1 files changed, 383 insertions, 440 deletions

diff --git a/drivers/char/agp/intel-gtt.c b/drivers/char/agp/intel-gtt.c
index 9272c38dd3c..fab3d3265ad 100644
--- a/drivers/char/agp/intel-gtt.c
+++ b/drivers/char/agp/intel-gtt.c
@@ -24,7 +24,6 @@
 #include <asm/smp.h>
 #include "agp.h"
 #include "intel-agp.h"
-#include <linux/intel-gtt.h>
 #include <drm/intel-gtt.h>
 
 /*
@@ -39,40 +38,12 @@
 #define USE_PCI_DMA_API 0
 #endif
 
-/* Max amount of stolen space, anything above will be returned to Linux */
-int intel_max_stolen = 32 * 1024 * 1024;
-
-static const struct aper_size_info_fixed intel_i810_sizes[] =
-{
-	{64, 16384, 4},
-	/* The 32M mode still requires a 64k gatt */
-	{32, 8192, 4}
-};
-
-#define AGP_DCACHE_MEMORY	1
-#define AGP_PHYS_MEMORY		2
-#define INTEL_AGP_CACHED_MEMORY 3
-
-static struct gatt_mask intel_i810_masks[] =
-{
-	{.mask = I810_PTE_VALID, .type = 0},
-	{.mask = (I810_PTE_VALID | I810_PTE_LOCAL), .type = AGP_DCACHE_MEMORY},
-	{.mask = I810_PTE_VALID, .type = 0},
-	{.mask = I810_PTE_VALID | I830_PTE_SYSTEM_CACHED,
-	 .type = INTEL_AGP_CACHED_MEMORY}
-};
-
-#define INTEL_AGP_UNCACHED_MEMORY              0
-#define INTEL_AGP_CACHED_MEMORY_LLC            1
-#define INTEL_AGP_CACHED_MEMORY_LLC_GFDT       2
-#define INTEL_AGP_CACHED_MEMORY_LLC_MLC        3
-#define INTEL_AGP_CACHED_MEMORY_LLC_MLC_GFDT   4
-
 struct intel_gtt_driver {
 	unsigned int gen : 8;
 	unsigned int is_g33 : 1;
 	unsigned int is_pineview : 1;
 	unsigned int is_ironlake : 1;
+	unsigned int has_pgtbl_enable : 1;
 	unsigned int dma_mask_size : 8;
 	/* Chipset specific GTT setup */
 	int (*setup)(void);
@@ -95,13 +66,15 @@ static struct _intel_private {
 	u8 __iomem *registers;
 	phys_addr_t gtt_bus_addr;
 	phys_addr_t gma_bus_addr;
-	phys_addr_t pte_bus_addr;
+	u32 PGETBL_save;
 	u32 __iomem *gtt;		/* I915G */
+	bool clear_fake_agp; /* on first access via agp, fill with scratch */
 	int num_dcache_entries;
 	union {
 		void __iomem *i9xx_flush_page;
 		void *i8xx_flush_page;
 	};
+	char *i81x_gtt_table;
 	struct page *i8xx_page;
 	struct resource ifp_resource;
 	int resource_valid;
@@ -113,42 +86,31 @@ static struct _intel_private {
 #define IS_G33		intel_private.driver->is_g33
 #define IS_PINEVIEW	intel_private.driver->is_pineview
 #define IS_IRONLAKE	intel_private.driver->is_ironlake
+#define HAS_PGTBL_EN	intel_private.driver->has_pgtbl_enable
 
-static void intel_agp_free_sglist(struct agp_memory *mem)
-{
-	struct sg_table st;
-
-	st.sgl = mem->sg_list;
-	st.orig_nents = st.nents = mem->page_count;
-
-	sg_free_table(&st);
-
-	mem->sg_list = NULL;
-	mem->num_sg = 0;
-}
-
-static int intel_agp_map_memory(struct agp_memory *mem)
+int intel_gtt_map_memory(struct page **pages, unsigned int num_entries,
+			 struct scatterlist **sg_list, int *num_sg)
 {
 	struct sg_table st;
 	struct scatterlist *sg;
 	int i;
 
-	if (mem->sg_list)
+	if (*sg_list)
 		return 0; /* already mapped (for e.g. resume */
 
-	DBG("try mapping %lu pages\n", (unsigned long)mem->page_count);
+	DBG("try mapping %lu pages\n", (unsigned long)num_entries);
 
-	if (sg_alloc_table(&st, mem->page_count, GFP_KERNEL))
+	if (sg_alloc_table(&st, num_entries, GFP_KERNEL))
 		goto err;
 
-	mem->sg_list = sg = st.sgl;
+	*sg_list = sg = st.sgl;
 
-	for (i = 0 ; i < mem->page_count; i++, sg = sg_next(sg))
-		sg_set_page(sg, mem->pages[i], PAGE_SIZE, 0);
+	for (i = 0 ; i < num_entries; i++, sg = sg_next(sg))
+		sg_set_page(sg, pages[i], PAGE_SIZE, 0);
 
-	mem->num_sg = pci_map_sg(intel_private.pcidev, mem->sg_list,
-				 mem->page_count, PCI_DMA_BIDIRECTIONAL);
-	if (unlikely(!mem->num_sg))
+	*num_sg = pci_map_sg(intel_private.pcidev, *sg_list,
+				 num_entries, PCI_DMA_BIDIRECTIONAL);
+	if (unlikely(!*num_sg))
 		goto err;
 
 	return 0;
@@ -157,90 +119,22 @@ err:
 	sg_free_table(&st);
 	return -ENOMEM;
 }
+EXPORT_SYMBOL(intel_gtt_map_memory);
 
-static void intel_agp_unmap_memory(struct agp_memory *mem)
+void intel_gtt_unmap_memory(struct scatterlist *sg_list, int num_sg)
 {
+	struct sg_table st;
 	DBG("try unmapping %lu pages\n", (unsigned long)mem->page_count);
 
-	pci_unmap_sg(intel_private.pcidev, mem->sg_list,
-		     mem->page_count, PCI_DMA_BIDIRECTIONAL);
-	intel_agp_free_sglist(mem);
-}
-
-static int intel_i810_fetch_size(void)
-{
-	u32 smram_miscc;
-	struct aper_size_info_fixed *values;
-
-	pci_read_config_dword(intel_private.bridge_dev,
-			      I810_SMRAM_MISCC, &smram_miscc);
-	values = A_SIZE_FIX(agp_bridge->driver->aperture_sizes);
-
-	if ((smram_miscc & I810_GMS) == I810_GMS_DISABLE) {
-		dev_warn(&intel_private.bridge_dev->dev, "i810 is disabled\n");
-		return 0;
-	}
-	if ((smram_miscc & I810_GFX_MEM_WIN_SIZE) == I810_GFX_MEM_WIN_32M) {
-		agp_bridge->current_size = (void *) (values + 1);
-		agp_bridge->aperture_size_idx = 1;
-		return values[1].size;
-	} else {
-		agp_bridge->current_size = (void *) (values);
-		agp_bridge->aperture_size_idx = 0;
-		return values[0].size;
-	}
-
-	return 0;
-}
-
-static int intel_i810_configure(void)
-{
-	struct aper_size_info_fixed *current_size;
-	u32 temp;
-	int i;
-
-	current_size = A_SIZE_FIX(agp_bridge->current_size);
-
-	if (!intel_private.registers) {
-		pci_read_config_dword(intel_private.pcidev, I810_MMADDR, &temp);
-		temp &= 0xfff80000;
-
-		intel_private.registers = ioremap(temp, 128 * 4096);
-		if (!intel_private.registers) {
-			dev_err(&intel_private.pcidev->dev,
-				"can't remap memory\n");
-			return -ENOMEM;
-		}
-	}
+	pci_unmap_sg(intel_private.pcidev, sg_list,
+		     num_sg, PCI_DMA_BIDIRECTIONAL);
 
-	if ((readl(intel_private.registers+I810_DRAM_CTL)
-		& I810_DRAM_ROW_0) == I810_DRAM_ROW_0_SDRAM) {
-		/* This will need to be dynamically assigned */
-		dev_info(&intel_private.pcidev->dev,
-			 "detected 4MB dedicated video ram\n");
-		intel_private.num_dcache_entries = 1024;
-	}
-	pci_read_config_dword(intel_private.pcidev, I810_GMADDR, &temp);
-	agp_bridge->gart_bus_addr = (temp & PCI_BASE_ADDRESS_MEM_MASK);
-	writel(agp_bridge->gatt_bus_addr | I810_PGETBL_ENABLED, intel_private.registers+I810_PGETBL_CTL);
-	readl(intel_private.registers+I810_PGETBL_CTL);	/* PCI Posting. */
-
-	if (agp_bridge->driver->needs_scratch_page) {
-		for (i = 0; i < current_size->num_entries; i++) {
-			writel(agp_bridge->scratch_page, intel_private.registers+I810_PTE_BASE+(i*4));
-		}
-		readl(intel_private.registers+I810_PTE_BASE+((i-1)*4));	/* PCI posting. */
-	}
-	global_cache_flush();
-	return 0;
-}
+	st.sgl = sg_list;
+	st.orig_nents = st.nents = num_sg;
 
-static void intel_i810_cleanup(void)
-{
-	writel(0, intel_private.registers+I810_PGETBL_CTL);
-	readl(intel_private.registers);	/* PCI Posting. */
-	iounmap(intel_private.registers);
+	sg_free_table(&st);
 }
+EXPORT_SYMBOL(intel_gtt_unmap_memory);
 
 static void intel_fake_agp_enable(struct agp_bridge_data *bridge, u32 mode)
 {
@@ -277,80 +171,64 @@ static void i8xx_destroy_pages(struct page *page)
 	atomic_dec(&agp_bridge->current_memory_agp);
 }
 
-static int intel_i810_insert_entries(struct agp_memory *mem, off_t pg_start,
-				int type)
+#define I810_GTT_ORDER 4
+static int i810_setup(void)
 {
-	int i, j, num_entries;
-	void *temp;
-	int ret = -EINVAL;
-	int mask_type;
-
-	if (mem->page_count == 0)
-		goto out;
-
-	temp = agp_bridge->current_size;
-	num_entries = A_SIZE_FIX(temp)->num_entries;
+	u32 reg_addr;
+	char *gtt_table;
 
-	if ((pg_start + mem->page_count) > num_entries)
-		goto out_err;
+	/* i81x does not preallocate the gtt. It's always 64kb in size. */
+	gtt_table = alloc_gatt_pages(I810_GTT_ORDER);
+	if (gtt_table == NULL)
+		return -ENOMEM;
+	intel_private.i81x_gtt_table = gtt_table;
 
+	pci_read_config_dword(intel_private.pcidev, I810_MMADDR, &reg_addr);
+	reg_addr &= 0xfff80000;
 
-	for (j = pg_start; j < (pg_start + mem->page_count); j++) {
-		if (!PGE_EMPTY(agp_bridge, readl(agp_bridge->gatt_table+j))) {
-			ret = -EBUSY;
-			goto out_err;
-		}
-	}
+	intel_private.registers = ioremap(reg_addr, KB(64));
+	if (!intel_private.registers)
+		return -ENOMEM;
 
-	if (type != mem->type)
-		goto out_err;
+	writel(virt_to_phys(gtt_table) | I810_PGETBL_ENABLED,
+	       intel_private.registers+I810_PGETBL_CTL);
 
-	mask_type = agp_bridge->driver->agp_type_to_mask_type(agp_bridge, type);
+	intel_private.gtt_bus_addr = reg_addr + I810_PTE_BASE;
 
-	switch (mask_type) {
-	case AGP_DCACHE_MEMORY:
-		if (!mem->is_flushed)
-			global_cache_flush();
-		for (i = pg_start; i < (pg_start + mem->page_count); i++) {
-			writel((i*4096)|I810_PTE_LOCAL|I810_PTE_VALID,
-			       intel_private.registers+I810_PTE_BASE+(i*4));
-		}
-		readl(intel_private.registers+I810_PTE_BASE+((i-1)*4));
-		break;
-	case AGP_PHYS_MEMORY:
-	case AGP_NORMAL_MEMORY:
-		if (!mem->is_flushed)
-			global_cache_flush();
-		for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
-			writel(agp_bridge->driver->mask_memory(agp_bridge,
-					page_to_phys(mem->pages[i]), mask_type),
-			       intel_private.registers+I810_PTE_BASE+(j*4));
-		}
-		readl(intel_private.registers+I810_PTE_BASE+((j-1)*4));
-		break;
-	default:
-		goto out_err;
+	if ((readl(intel_private.registers+I810_DRAM_CTL)
+		& I810_DRAM_ROW_0) == I810_DRAM_ROW_0_SDRAM) {
+		dev_info(&intel_private.pcidev->dev,
+			 "detected 4MB dedicated video ram\n");
+		intel_private.num_dcache_entries = 1024;
 	}
 
-out:
-	ret = 0;
-out_err:
-	mem->is_flushed = true;
-	return ret;
+	return 0;
+}
+
+static void i810_cleanup(void)
+{
+	writel(0, intel_private.registers+I810_PGETBL_CTL);
+	free_gatt_pages(intel_private.i81x_gtt_table, I810_GTT_ORDER);
 }
 
-static int intel_i810_remove_entries(struct agp_memory *mem, off_t pg_start,
-				int type)
+static int i810_insert_dcache_entries(struct agp_memory *mem, off_t pg_start,
+				      int type)
 {
 	int i;
 
-	if (mem->page_count == 0)
-		return 0;
+	if ((pg_start + mem->page_count)
+			> intel_private.num_dcache_entries)
+		return -EINVAL;
 
-	for (i = pg_start; i < (mem->page_count + pg_start); i++) {
-		writel(agp_bridge->scratch_page, intel_private.registers+I810_PTE_BASE+(i*4));
+	if (!mem->is_flushed)
+		global_cache_flush();
+
+	for (i = pg_start; i < (pg_start + mem->page_count); i++) {
+		dma_addr_t addr = i << PAGE_SHIFT;
+		intel_private.driver->write_entry(addr,
+						  i, type);
 	}
-	readl(intel_private.registers+I810_PTE_BASE+((i-1)*4));
+	readl(intel_private.gtt+i-1);
 
 	return 0;
 }
@@ -397,29 +275,6 @@ static struct agp_memory *alloc_agpphysmem_i8xx(size_t pg_count, int type)
 	return new;
 }
 
-static struct agp_memory *intel_i810_alloc_by_type(size_t pg_count, int type)
-{
-	struct agp_memory *new;
-
-	if (type == AGP_DCACHE_MEMORY) {
-		if (pg_count != intel_private.num_dcache_entries)
-			return NULL;
-
-		new = agp_create_memory(1);
-		if (new == NULL)
-			return NULL;
-
-		new->type = AGP_DCACHE_MEMORY;
-		new->page_count = pg_count;
-		new->num_scratch_pages = 0;
-		agp_free_page_array(new);
-		return new;
-	}
-	if (type == AGP_PHYS_MEMORY)
-		return alloc_agpphysmem_i8xx(pg_count, type);
-	return NULL;
-}
-
 static void intel_i810_free_by_type(struct agp_memory *curr)
 {
 	agp_free_key(curr->key);
@@ -437,13 +292,6 @@ static void intel_i810_free_by_type(struct agp_memory *curr)
 	kfree(curr);
 }
 
-static unsigned long intel_i810_mask_memory(struct agp_bridge_data *bridge,
-					    dma_addr_t addr, int type)
-{
-	/* Type checking must be done elsewhere */
-	return addr | bridge->driver->masks[type].mask;
-}
-
 static int intel_gtt_setup_scratch_page(void)
 {
 	struct page *page;
@@ -455,7 +303,7 @@ static int intel_gtt_setup_scratch_page(void)
 	get_page(page);
 	set_pages_uc(page, 1);
 
-	if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2) {
+	if (intel_private.base.needs_dmar) {
 		dma_addr = pci_map_page(intel_private.pcidev, page, 0,
 				    PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 		if (pci_dma_mapping_error(intel_private.pcidev, dma_addr))
@@ -470,34 +318,45 @@ static int intel_gtt_setup_scratch_page(void)
 	return 0;
 }
 
-static const struct aper_size_info_fixed const intel_fake_agp_sizes[] = {
+static void i810_write_entry(dma_addr_t addr, unsigned int entry,
+			     unsigned int flags)
+{
+	u32 pte_flags = I810_PTE_VALID;
+
+	switch (flags) {
+	case AGP_DCACHE_MEMORY:
+		pte_flags |= I810_PTE_LOCAL;
+		break;
+	case AGP_USER_CACHED_MEMORY:
+		pte_flags |= I830_PTE_SYSTEM_CACHED;
+		break;
+	}
+
+	writel(addr | pte_flags, intel_private.gtt + entry);
+}
+
+static const struct aper_size_info_fixed intel_fake_agp_sizes[] = {
+	{32, 8192, 3},
+	{64, 16384, 4},
 	{128, 32768, 5},
-	/* The 64M mode still requires a 128k gatt */
-	{64, 16384, 5},
 	{256, 65536, 6},
 	{512, 131072, 7},
 };
 
-static unsigned int intel_gtt_stolen_entries(void)
+static unsigned int intel_gtt_stolen_size(void)
 {
 	u16 gmch_ctrl;
 	u8 rdct;
 	int local = 0;
 	static const int ddt[4] = { 0, 16, 32, 64 };
-	unsigned int overhead_entries, stolen_entries;
 	unsigned int stolen_size = 0;
 
+	if (INTEL_GTT_GEN == 1)
+		return 0; /* no stolen mem on i81x */
+
 	pci_read_config_word(intel_private.bridge_dev,
 			     I830_GMCH_CTRL, &gmch_ctrl);
 
-	if (INTEL_GTT_GEN > 4 || IS_PINEVIEW)
-		overhead_entries = 0;
-	else
-		overhead_entries = intel_private.base.gtt_mappable_entries
-			/ 1024;
-
-	overhead_entries += 1; /* BIOS popup */
-
 	if (intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82830_HB ||
 	    intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82845G_HB) {
 		switch (gmch_ctrl & I830_GMCH_GMS_MASK) {
@@ -623,12 +482,7 @@ static unsigned int intel_gtt_stolen_entries(void)
 		}
 	}
 
-	if (!local && stolen_size > intel_max_stolen) {
-		dev_info(&intel_private.bridge_dev->dev,
-			 "detected %dK stolen memory, trimming to %dK\n",
-			 stolen_size / KB(1), intel_max_stolen / KB(1));
-		stolen_size = intel_max_stolen;
-	} else if (stolen_size > 0) {
+	if (stolen_size > 0) {
 		dev_info(&intel_private.bridge_dev->dev, "detected %dK %s memory\n",
 		       stolen_size / KB(1), local ? "local" : "stolen");
 	} else {
@@ -637,46 +491,88 @@ static unsigned int intel_gtt_stolen_entries(void)
 		stolen_size = 0;
 	}
 
-	stolen_entries = stolen_size/KB(4) - overhead_entries;
+	return stolen_size;
+}
 
-	return stolen_entries;
+static void i965_adjust_pgetbl_size(unsigned int size_flag)
+{
+	u32 pgetbl_ctl, pgetbl_ctl2;
+
+	/* ensure that ppgtt is disabled */
+	pgetbl_ctl2 = readl(intel_private.registers+I965_PGETBL_CTL2);
+	pgetbl_ctl2 &= ~I810_PGETBL_ENABLED;
+	writel(pgetbl_ctl2, intel_private.registers+I965_PGETBL_CTL2);
+
+	/* write the new ggtt size */
+	pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL);
+	pgetbl_ctl &= ~I965_PGETBL_SIZE_MASK;
+	pgetbl_ctl |= size_flag;
+	writel(pgetbl_ctl, intel_private.registers+I810_PGETBL_CTL);
 }
 
-static unsigned int intel_gtt_total_entries(void)
+static unsigned int i965_gtt_total_entries(void)
 {
 	int size;
+	u32 pgetbl_ctl;
+	u16 gmch_ctl;
 
-	if (IS_G33 || INTEL_GTT_GEN == 4 || INTEL_GTT_GEN == 5) {
-		u32 pgetbl_ctl;
-		pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL);
+	pci_read_config_word(intel_private.bridge_dev,
+			     I830_GMCH_CTRL, &gmch_ctl);
 
-		switch (pgetbl_ctl & I965_PGETBL_SIZE_MASK) {
-		case I965_PGETBL_SIZE_128KB:
-			size = KB(128);
-			break;
-		case I965_PGETBL_SIZE_256KB:
-			size = KB(256);
-			break;
-		case I965_PGETBL_SIZE_512KB:
-			size = KB(512);
-			break;
-		case I965_PGETBL_SIZE_1MB:
-			size = KB(1024);
+	if (INTEL_GTT_GEN == 5) {
+		switch (gmch_ctl & G4x_GMCH_SIZE_MASK) {
+		case G4x_GMCH_SIZE_1M:
+		case G4x_GMCH_SIZE_VT_1M:
+			i965_adjust_pgetbl_size(I965_PGETBL_SIZE_1MB);
 			break;
-		case I965_PGETBL_SIZE_2MB:
-			size = KB(2048);
+		case G4x_GMCH_SIZE_VT_1_5M:
+			i965_adjust_pgetbl_size(I965_PGETBL_SIZE_1_5MB);
 			break;
-		case I965_PGETBL_SIZE_1_5MB:
-			size = KB(1024 + 512);
+		case G4x_GMCH_SIZE_2M:
+		case G4x_GMCH_SIZE_VT_2M:
+			i965_adjust_pgetbl_size(I965_PGETBL_SIZE_2MB);
 			break;
-		default:
-			dev_info(&intel_private.pcidev->dev,
-				 "unknown page table size, assuming 512KB\n");
-			size = KB(512);
 		}
+	}
 
-		return size/4;
-	} else if (INTEL_GTT_GEN == 6) {
+	pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL);
+
+	switch (pgetbl_ctl & I965_PGETBL_SIZE_MASK) {
+	case I965_PGETBL_SIZE_128KB:
+		size = KB(128);
+		break;
+	case I965_PGETBL_SIZE_256KB:
+		size = KB(256);
+		break;
+	case I965_PGETBL_SIZE_512KB:
+		size = KB(512);
+		break;
+	/* GTT pagetable sizes bigger than 512KB are not possible on G33! */
+	case I965_PGETBL_SIZE_1MB:
+		size = KB(1024);
+		break;
+	case I965_PGETBL_SIZE_2MB:
+		size = KB(2048);
+		break;
+	case I965_PGETBL_SIZE_1_5MB:
+		size = KB(1024 + 512);
+		break;
+	default:
+		dev_info(&intel_private.pcidev->dev,
+			 "unknown page table size, assuming 512KB\n");
+		size = KB(512);
+	}
+
+	return size/4;
+}
+
+static unsigned int intel_gtt_total_entries(void)
+{
+	int size;
+
+	if (IS_G33 || INTEL_GTT_GEN == 4 || INTEL_GTT_GEN == 5)
+		return i965_gtt_total_entries();
+	else if (INTEL_GTT_GEN == 6) {
 		u16 snb_gmch_ctl;
 
 		pci_read_config_word(intel_private.pcidev, SNB_GMCH_CTRL, &snb_gmch_ctl);
@@ -706,7 +602,18 @@ static unsigned int intel_gtt_mappable_entries(void)
 {
 	unsigned int aperture_size;
 
-	if (INTEL_GTT_GEN == 2) {
+	if (INTEL_GTT_GEN == 1) {
+		u32 smram_miscc;
+
+		pci_read_config_dword(intel_private.bridge_dev,
+				      I810_SMRAM_MISCC, &smram_miscc);
+
+		if ((smram_miscc & I810_GFX_MEM_WIN_SIZE)
+				== I810_GFX_MEM_WIN_32M)
+			aperture_size = MB(32);
+		else
+			aperture_size = MB(64);
+	} else if (INTEL_GTT_GEN == 2) {
 		u16 gmch_ctrl;
 
 		pci_read_config_word(intel_private.bridge_dev,
@@ -739,7 +646,7 @@ static void intel_gtt_cleanup(void)
 
 	iounmap(intel_private.gtt);
 	iounmap(intel_private.registers);
-	
+
 	intel_gtt_teardown_scratch_page();
 }
 
@@ -755,6 +662,14 @@ static int intel_gtt_init(void)
 	intel_private.base.gtt_mappable_entries = intel_gtt_mappable_entries();
 	intel_private.base.gtt_total_entries = intel_gtt_total_entries();
 
+	/* save the PGETBL reg for resume */
+	intel_private.PGETBL_save =
+		readl(intel_private.registers+I810_PGETBL_CTL)
+			& ~I810_PGETBL_ENABLED;
+	/* we only ever restore the register when enabling the PGTBL... */
+	if (HAS_PGTBL_EN)
+		intel_private.PGETBL_save |= I810_PGETBL_ENABLED;
+
 	dev_info(&intel_private.bridge_dev->dev,
 			"detected gtt size: %dK total, %dK mappable\n",
 			intel_private.base.gtt_total_entries * 4,
@@ -772,14 +687,9 @@ static int intel_gtt_init(void)
 
 	global_cache_flush();   /* FIXME: ? */
 
-	/* we have to call this as early as possible after the MMIO base address is known */
-	intel_private.base.gtt_stolen_entries = intel_gtt_stolen_entries();
-	if (intel_private.base.gtt_stolen_entries == 0) {
-		intel_private.driver->cleanup();
-		iounmap(intel_private.registers);
-		iounmap(intel_private.gtt);
-		return -ENOMEM;
-	}
+	intel_private.base.stolen_size = intel_gtt_stolen_size();
+
+	intel_private.base.needs_dmar = USE_PCI_DMA_API && INTEL_GTT_GEN > 2;
 
 	ret = intel_gtt_setup_scratch_page();
 	if (ret != 0) {
@@ -812,8 +722,10 @@ static int intel_fake_agp_fetch_size(void)
 
 static void i830_cleanup(void)
 {
-	kunmap(intel_private.i8xx_page);
-	intel_private.i8xx_flush_page = NULL;
+	if (intel_private.i8xx_flush_page) {
+		kunmap(intel_private.i8xx_flush_page);
+		intel_private.i8xx_flush_page = NULL;
+	}
 
 	__free_page(intel_private.i8xx_page);
 	intel_private.i8xx_page = NULL;
@@ -860,25 +772,19 @@ static void i830_write_entry(dma_addr_t addr, unsigned int entry,
 			     unsigned int flags)
 {
 	u32 pte_flags = I810_PTE_VALID;
-	
-	switch (flags) {
-	case AGP_DCACHE_MEMORY:
-		pte_flags |= I810_PTE_LOCAL;
-		break;
-	case AGP_USER_CACHED_MEMORY:
+
+	if (flags ==  AGP_USER_CACHED_MEMORY)
 		pte_flags |= I830_PTE_SYSTEM_CACHED;
-		break;
-	}
 
 	writel(addr | pte_flags, intel_private.gtt + entry);
 }
 
-static void intel_enable_gtt(void)
+static bool intel_enable_gtt(void)
 {
 	u32 gma_addr;
-	u16 gmch_ctrl;
+	u8 __iomem *reg;
 
-	if (INTEL_GTT_GEN == 2)
+	if (INTEL_GTT_GEN <= 2)
 		pci_read_config_dword(intel_private.pcidev, I810_GMADDR,
 				      &gma_addr);
 	else
@@ -887,13 +793,47 @@ static void intel_enable_gtt(void)
 
 	intel_private.gma_bus_addr = (gma_addr & PCI_BASE_ADDRESS_MEM_MASK);
 
-	pci_read_config_word(intel_private.bridge_dev, I830_GMCH_CTRL, &gmch_ctrl);
-	gmch_ctrl |= I830_GMCH_ENABLED;
-	pci_write_config_word(intel_private.bridge_dev, I830_GMCH_CTRL, gmch_ctrl);
+	if (INTEL_GTT_GEN >= 6)
+	    return true;
 
-	writel(intel_private.pte_bus_addr|I810_PGETBL_ENABLED,
-	       intel_private.registers+I810_PGETBL_CTL);
-	readl(intel_private.registers+I810_PGETBL_CTL);	/* PCI Posting. */
+	if (INTEL_GTT_GEN == 2) {
+		u16 gmch_ctrl;
+
+		pci_read_config_word(intel_private.bridge_dev,
+				     I830_GMCH_CTRL, &gmch_ctrl);
+		gmch_ctrl |= I830_GMCH_ENABLED;
+		pci_write_config_word(intel_private.bridge_dev,
+				      I830_GMCH_CTRL, gmch_ctrl);
+
+		pci_read_config_word(intel_private.bridge_dev,
+				     I830_GMCH_CTRL, &gmch_ctrl);
+		if ((gmch_ctrl & I830_GMCH_ENABLED) == 0) {
+			dev_err(&intel_private.pcidev->dev,
+				"failed to enable the GTT: GMCH_CTRL=%x\n",
+				gmch_ctrl);
+			return false;
+		}
+	}
+
+	/* On the resume path we may be adjusting the PGTBL value, so
+	 * be paranoid and flush all chipset write buffers...
+	 */
+	if (INTEL_GTT_GEN >= 3)
+		writel(0, intel_private.registers+GFX_FLSH_CNTL);
+
+	reg = intel_private.registers+I810_PGETBL_CTL;
+	writel(intel_private.PGETBL_save, reg);
+	if (HAS_PGTBL_EN && (readl(reg) & I810_PGETBL_ENABLED) == 0) {
+		dev_err(&intel_private.pcidev->dev,
+			"failed to enable the GTT: PGETBL=%x [expected %x]\n",
+			readl(reg), intel_private.PGETBL_save);
+		return false;
+	}
+
+	if (INTEL_GTT_GEN >= 3)
+		writel(0, intel_private.registers+GFX_FLSH_CNTL);
+
+	return true;
 }
 
 static int i830_setup(void)
@@ -908,8 +848,6 @@ static int i830_setup(void)
 		return -ENOMEM;
 
 	intel_private.gtt_bus_addr = reg_addr + I810_PTE_BASE;
-	intel_private.pte_bus_addr =
-		readl(intel_private.registers+I810_PGETBL_CTL) & 0xfffff000;
 
 	intel_i830_setup_flush();
 
@@ -932,21 +870,12 @@ static int intel_fake_agp_free_gatt_table(struct agp_bridge_data *bridge)
 
 static int intel_fake_agp_configure(void)
 {
-	int i;
-
-	intel_enable_gtt();
+	if (!intel_enable_gtt())
+	    return -EIO;
 
+	intel_private.clear_fake_agp = true;
 	agp_bridge->gart_bus_addr = intel_private.gma_bus_addr;
 
-	for (i = intel_private.base.gtt_stolen_entries;
-			i < intel_private.base.gtt_total_entries; i++) {
-		intel_private.driver->write_entry(intel_private.scratch_page_dma,
-						  i, 0);
-	}
-	readl(intel_private.gtt+i-1);	/* PCI Posting. */
-
-	global_cache_flush();
-
 	return 0;
 }
 
@@ -963,10 +892,10 @@ static bool i830_check_flags(unsigned int flags)
 	return false;
 }
 
-static void intel_gtt_insert_sg_entries(struct scatterlist *sg_list,
-					unsigned int sg_len,
-					unsigned int pg_start,
-					unsigned int flags)
+void intel_gtt_insert_sg_entries(struct scatterlist *sg_list,
+				 unsigned int sg_len,
+				 unsigned int pg_start,
+				 unsigned int flags)
 {
 	struct scatterlist *sg;
 	unsigned int len, m;
@@ -987,27 +916,41 @@ static void intel_gtt_insert_sg_entries(struct scatterlist *sg_list,
 	}
 	readl(intel_private.gtt+j-1);
 }
+EXPORT_SYMBOL(intel_gtt_insert_sg_entries);
+
+void intel_gtt_insert_pages(unsigned int first_entry, unsigned int num_entries,
+			    struct page **pages, unsigned int flags)
+{
+	int i, j;
+
+	for (i = 0, j = first_entry; i < num_entries; i++, j++) {
+		dma_addr_t addr = page_to_phys(pages[i]);
+		intel_private.driver->write_entry(addr,
+						  j, flags);
+	}
+	readl(intel_private.gtt+j-1);
+}
+EXPORT_SYMBOL(intel_gtt_insert_pages);
 
 static int intel_fake_agp_insert_entries(struct agp_memory *mem,
 					 off_t pg_start, int type)
 {
-	int i, j;
 	int ret = -EINVAL;
 
-	if (mem->page_count == 0)
-		goto out;
+	if (intel_private.clear_fake_agp) {
+		int start = intel_private.base.stolen_size / PAGE_SIZE;
+		int end = intel_private.base.gtt_mappable_entries;
+		intel_gtt_clear_range(start, end - start);
+		intel_private.clear_fake_agp = false;
+	}
 
-	if (pg_start < intel_private.base.gtt_stolen_entries) {
-		dev_printk(KERN_DEBUG, &intel_private.pcidev->dev,
-			   "pg_start == 0x%.8lx, gtt_stolen_entries == 0x%.8x\n",
-			   pg_start, intel_private.base.gtt_stolen_entries);
+	if (INTEL_GTT_GEN == 1 && type == AGP_DCACHE_MEMORY)
+		return i810_insert_dcache_entries(mem, pg_start, type);
 
-		dev_info(&intel_private.pcidev->dev,
-			 "trying to insert into local/stolen memory\n");
-		goto out_err;
-	}
+	if (mem->page_count == 0)
+		goto out;
 
-	if ((pg_start + mem->page_count) > intel_private.base.gtt_total_entries)
+	if (pg_start + mem->page_count > intel_private.base.gtt_total_entries)
 		goto out_err;
 
 	if (type != mem->type)
@@ -1019,21 +962,17 @@ static int intel_fake_agp_insert_entries(struct agp_memory *mem,
 	if (!mem->is_flushed)
 		global_cache_flush();
 
-	if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2) {
-		ret = intel_agp_map_memory(mem);
+	if (intel_private.base.needs_dmar) {
+		ret = intel_gtt_map_memory(mem->pages, mem->page_count,
+					   &mem->sg_list, &mem->num_sg);
 		if (ret != 0)
 			return ret;
 
 		intel_gtt_insert_sg_entries(mem->sg_list, mem->num_sg,
 					    pg_start, type);
-	} else {
-		for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
-			dma_addr_t addr = page_to_phys(mem->pages[i]);
-			intel_private.driver->write_entry(addr,
-							  j, type);
-		}
-		readl(intel_private.gtt+j-1);
-	}
+	} else
+		intel_gtt_insert_pages(pg_start, mem->page_count, mem->pages,
+				       type);
 
 out:
 	ret = 0;
@@ -1042,40 +981,54 @@ out_err:
 	return ret;
 }
 
+void intel_gtt_clear_range(unsigned int first_entry, unsigned int num_entries)
+{
+	unsigned int i;
+
+	for (i = first_entry; i < (first_entry + num_entries); i++) {
+		intel_private.driver->write_entry(intel_private.scratch_page_dma,
+						  i, 0);
+	}
+	readl(intel_private.gtt+i-1);
+}
+EXPORT_SYMBOL(intel_gtt_clear_range);
+
 static int intel_fake_agp_remove_entries(struct agp_memory *mem,
 					 off_t pg_start, int type)
 {
-	int i;
-
 	if (mem->page_count == 0)
 		return 0;
 
-	if (pg_start < intel_private.base.gtt_stolen_entries) {
-		dev_info(&intel_private.pcidev->dev,
-			 "trying to disable local/stolen memory\n");
-		return -EINVAL;
-	}
+	intel_gtt_clear_range(pg_start, mem->page_count);
 
-	if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2)
-		intel_agp_unmap_memory(mem);
-
-	for (i = pg_start; i < (mem->page_count + pg_start); i++) {
-		intel_private.driver->write_entry(intel_private.scratch_page_dma,
-						  i, 0);
+	if (intel_private.base.needs_dmar) {
+		intel_gtt_unmap_memory(mem->sg_list, mem->num_sg);
+		mem->sg_list = NULL;
+		mem->num_sg = 0;
 	}
-	readl(intel_private.gtt+i-1);
 
 	return 0;
 }
 
-static void intel_fake_agp_chipset_flush(struct agp_bridge_data *bridge)
-{
-	intel_private.driver->chipset_flush();
-}
-
 static struct agp_memory *intel_fake_agp_alloc_by_type(size_t pg_count,
 						       int type)
 {
+	struct agp_memory *new;
+
+	if (type == AGP_DCACHE_MEMORY && INTEL_GTT_GEN == 1) {
+		if (pg_count != intel_private.num_dcache_entries)
+			return NULL;
+
+		new = agp_create_memory(1);
+		if (new == NULL)
+			return NULL;
+
+		new->type = AGP_DCACHE_MEMORY;
+		new->page_count = pg_count;
+		new->num_scratch_pages = 0;
+		agp_free_page_array(new);
+		return new;
+	}
 	if (type == AGP_PHYS_MEMORY)
 		return alloc_agpphysmem_i8xx(pg_count, type);
 	/* always return NULL for other allocation types for now */
@@ -1190,12 +1143,19 @@ static void i9xx_chipset_flush(void)
 		writel(1, intel_private.i9xx_flush_page);
 }
 
-static void i965_write_entry(dma_addr_t addr, unsigned int entry,
+static void i965_write_entry(dma_addr_t addr,
+			     unsigned int entry,
 			     unsigned int flags)
 {
+	u32 pte_flags;
+
+	pte_flags = I810_PTE_VALID;
+	if (flags == AGP_USER_CACHED_MEMORY)
+		pte_flags |= I830_PTE_SYSTEM_CACHED;
+
 	/* Shift high bits down */
 	addr |= (addr >> 28) & 0xf0;
-	writel(addr | I810_PTE_VALID, intel_private.gtt + entry);
+	writel(addr | pte_flags, intel_private.gtt + entry);
 }
 
 static bool gen6_check_flags(unsigned int flags)
@@ -1265,40 +1225,11 @@ static int i9xx_setup(void)
 		intel_private.gtt_bus_addr = reg_addr + gtt_offset;
 	}
 
-	intel_private.pte_bus_addr =
-		readl(intel_private.registers+I810_PGETBL_CTL) & 0xfffff000;
-
 	intel_i9xx_setup_flush();
 
 	return 0;
 }
 
-static const struct agp_bridge_driver intel_810_driver = {
-	.owner			= THIS_MODULE,
-	.aperture_sizes		= intel_i810_sizes,
-	.size_type		= FIXED_APER_SIZE,
-	.num_aperture_sizes	= 2,
-	.needs_scratch_page	= true,
-	.configure		= intel_i810_configure,
-	.fetch_size		= intel_i810_fetch_size,
-	.cleanup		= intel_i810_cleanup,
-	.mask_memory		= intel_i810_mask_memory,
-	.masks			= intel_i810_masks,
-	.agp_enable		= intel_fake_agp_enable,
-	.cache_flush		= global_cache_flush,
-	.create_gatt_table	= agp_generic_create_gatt_table,
-	.free_gatt_table	= agp_generic_free_gatt_table,
-	.insert_memory		= intel_i810_insert_entries,
-	.remove_memory		= intel_i810_remove_entries,
-	.alloc_by_type		= intel_i810_alloc_by_type,
-	.free_by_type		= intel_i810_free_by_type,
-	.agp_alloc_page		= agp_generic_alloc_page,
-	.agp_alloc_pages        = agp_generic_alloc_pages,
-	.agp_destroy_page	= agp_generic_destroy_page,
-	.agp_destroy_pages      = agp_generic_destroy_pages,
-	.agp_type_to_mask_type  = agp_generic_type_to_mask_type,
-};
-
 static const struct agp_bridge_driver intel_fake_agp_driver = {
 	.owner			= THIS_MODULE,
 	.size_type		= FIXED_APER_SIZE,
@@ -1319,15 +1250,20 @@ static const struct agp_bridge_driver intel_fake_agp_driver = {
 	.agp_alloc_pages        = agp_generic_alloc_pages,
 	.agp_destroy_page	= agp_generic_destroy_page,
 	.agp_destroy_pages      = agp_generic_destroy_pages,
-	.chipset_flush		= intel_fake_agp_chipset_flush,
 };
 
 static const struct intel_gtt_driver i81x_gtt_driver = {
 	.gen = 1,
+	.has_pgtbl_enable = 1,
 	.dma_mask_size = 32,
+	.setup = i810_setup,
+	.cleanup = i810_cleanup,
+	.check_flags = i830_check_flags,
+	.write_entry = i810_write_entry,
 };
 static const struct intel_gtt_driver i8xx_gtt_driver = {
 	.gen = 2,
+	.has_pgtbl_enable = 1,
 	.setup = i830_setup,
 	.cleanup = i830_cleanup,
 	.write_entry = i830_write_entry,
@@ -1337,10 +1273,11 @@ static const struct intel_gtt_driver i8xx_gtt_driver = {
 };
 static const struct intel_gtt_driver i915_gtt_driver = {
 	.gen = 3,
+	.has_pgtbl_enable = 1,
 	.setup = i9xx_setup,
 	.cleanup = i9xx_cleanup,
 	/* i945 is the last gpu to need phys mem (for overlay and cursors). */
-	.write_entry = i830_write_entry, 
+	.write_entry = i830_write_entry,
 	.dma_mask_size = 32,
 	.check_flags = i830_check_flags,
 	.chipset_flush = i9xx_chipset_flush,
@@ -1367,6 +1304,7 @@ static const struct intel_gtt_driver pineview_gtt_driver = {
 };
 static const struct intel_gtt_driver i965_gtt_driver = {
 	.gen = 4,
+	.has_pgtbl_enable = 1,
 	.setup = i9xx_setup,
 	.cleanup = i9xx_cleanup,
 	.write_entry = i965_write_entry,
@@ -1410,93 +1348,92 @@ static const struct intel_gtt_driver sandybridge_gtt_driver = {
 static const struct intel_gtt_driver_description {
 	unsigned int gmch_chip_id;
 	char *name;
-	const struct agp_bridge_driver *gmch_driver;
 	const struct intel_gtt_driver *gtt_driver;
 } intel_gtt_chipsets[] = {
-	{ PCI_DEVICE_ID_INTEL_82810_IG1, "i810", &intel_810_driver,
+	{ PCI_DEVICE_ID_INTEL_82810_IG1, "i810",
 		&i81x_gtt_driver},
-	{ PCI_DEVICE_ID_INTEL_82810_IG3, "i810", &intel_810_driver,
+	{ PCI_DEVICE_ID_INTEL_82810_IG3, "i810",
 		&i81x_gtt_driver},
-	{ PCI_DEVICE_ID_INTEL_82810E_IG, "i810", &intel_810_driver,
+	{ PCI_DEVICE_ID_INTEL_82810E_IG, "i810",
 		&i81x_gtt_driver},
-	{ PCI_DEVICE_ID_INTEL_82815_CGC, "i815", &intel_810_driver,
+	{ PCI_DEVICE_ID_INTEL_82815_CGC, "i815",
 		&i81x_gtt_driver},
 	{ PCI_DEVICE_ID_INTEL_82830_CGC, "830M",
-		&intel_fake_agp_driver, &i8xx_gtt_driver},
-	{ PCI_DEVICE_ID_INTEL_82845G_IG, "830M",
-		&intel_fake_agp_driver, &i8xx_gtt_driver},
+		&i8xx_gtt_driver},
+	{ PCI_DEVICE_ID_INTEL_82845G_IG, "845G",
+		&i8xx_gtt_driver},
 	{ PCI_DEVICE_ID_INTEL_82854_IG, "854",
-		&intel_fake_agp_driver, &i8xx_gtt_driver},
+		&i8xx_gtt_driver},
 	{ PCI_DEVICE_ID_INTEL_82855GM_IG, "855GM",
-		&intel_fake_agp_driver, &i8xx_gtt_driver},
+		&i8xx_gtt_driver},
 	{ PCI_DEVICE_ID_INTEL_82865_IG, "865",
-		&intel_fake_agp_driver, &i8xx_gtt_driver},
+		&i8xx_gtt_driver},
 	{ PCI_DEVICE_ID_INTEL_E7221_IG, "E7221 (i915)",
-		&intel_fake_agp_driver, &i915_gtt_driver },
+		&i915_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_82915G_IG, "915G",
-		&intel_fake_agp_driver, &i915_gtt_driver },
+		&i915_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_82915GM_IG, "915GM",
-		&intel_fake_agp_driver, &i915_gtt_driver },
+		&i915_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_82945G_IG, "945G",
-		&intel_fake_agp_driver, &i915_gtt_driver },
+		&i915_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_82945GM_IG, "945GM",
-		&intel_fake_agp_driver, &i915_gtt_driver },
+		&i915_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_82945GME_IG, "945GME",
-		&intel_fake_agp_driver, &i915_gtt_driver },
+		&i915_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_82946GZ_IG, "946GZ",
-		&intel_fake_agp_driver, &i965_gtt_driver },
+		&i965_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_82G35_IG, "G35",
-		&intel_fake_agp_driver, &i965_gtt_driver },
+		&i965_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_82965Q_IG, "965Q",
-		&intel_fake_agp_driver, &i965_gtt_driver },
+		&i965_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_82965G_IG, "965G",
-		&intel_fake_agp_driver, &i965_gtt_driver },
+		&i965_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_82965GM_IG, "965GM",
-		&intel_fake_agp_driver, &i965_gtt_driver },
+		&i965_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_82965GME_IG, "965GME/GLE",
-		&intel_fake_agp_driver, &i965_gtt_driver },
+		&i965_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_G33_IG, "G33",
-		&intel_fake_agp_driver, &g33_gtt_driver },
+		&g33_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_Q35_IG, "Q35",
-		&intel_fake_agp_driver, &g33_gtt_driver },
+		&g33_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_Q33_IG, "Q33",
-		&intel_fake_agp_driver, &g33_gtt_driver },
+		&g33_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_PINEVIEW_M_IG, "GMA3150",
-		&intel_fake_agp_driver, &pineview_gtt_driver },
+		&pineview_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_PINEVIEW_IG, "GMA3150",
-		&intel_fake_agp_driver, &pineview_gtt_driver },
+		&pineview_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_GM45_IG, "GM45",
-		&intel_fake_agp_driver, &g4x_gtt_driver },
+		&g4x_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_EAGLELAKE_IG, "Eaglelake",
-		&intel_fake_agp_driver, &g4x_gtt_driver },
+		&g4x_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_Q45_IG, "Q45/Q43",
-		&intel_fake_agp_driver, &g4x_gtt_driver },
+		&g4x_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_G45_IG, "G45/G43",
-		&intel_fake_agp_driver, &g4x_gtt_driver },
+		&g4x_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_B43_IG, "B43",
-		&intel_fake_agp_driver, &g4x_gtt_driver },
+		&g4x_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_B43_1_IG, "B43",
-		&intel_fake_agp_driver, &g4x_gtt_driver },
+		&g4x_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_G41_IG, "G41",
-		&intel_fake_agp_driver, &g4x_gtt_driver },
+		&g4x_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_IRONLAKE_D_IG,
-	    "HD Graphics", &intel_fake_agp_driver, &ironlake_gtt_driver },
+	    "HD Graphics", &ironlake_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_IRONLAKE_M_IG,
-	    "HD Graphics", &intel_fake_agp_driver, &ironlake_gtt_driver },
+	    "HD Graphics", &ironlake_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT1_IG,
-	    "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
+	    "Sandybridge", &sandybridge_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT2_IG,
-	    "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
+	    "Sandybridge", &sandybridge_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT2_PLUS_IG,
-	    "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
+	    "Sandybridge", &sandybridge_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT1_IG,
-	    "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
+	    "Sandybridge", &sandybridge_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT2_IG,
-	    "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
+	    "Sandybridge", &sandybridge_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT2_PLUS_IG,
-	    "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
+	    "Sandybridge", &sandybridge_gtt_driver },
 	{ PCI_DEVICE_ID_INTEL_SANDYBRIDGE_S_IG,
-	    "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
+	    "Sandybridge", &sandybridge_gtt_driver },
 	{ 0, NULL, NULL }
 };
 
@@ -1521,21 +1458,20 @@ int intel_gmch_probe(struct pci_dev *pdev,
 				      struct agp_bridge_data *bridge)
 {
 	int i, mask;
-	bridge->driver = NULL;
+	intel_private.driver = NULL;
 
 	for (i = 0; intel_gtt_chipsets[i].name != NULL; i++) {
 		if (find_gmch(intel_gtt_chipsets[i].gmch_chip_id)) {
-			bridge->driver =
-				intel_gtt_chipsets[i].gmch_driver;
-			intel_private.driver = 
+			intel_private.driver =
 				intel_gtt_chipsets[i].gtt_driver;
 			break;
 		}
 	}
 
-	if (!bridge->driver)
+	if (!intel_private.driver)
 		return 0;
 
+	bridge->driver = &intel_fake_agp_driver;
 	bridge->dev_private_data = &intel_private;
 	bridge->dev = pdev;
 
@@ -1551,8 +1487,8 @@ int intel_gmch_probe(struct pci_dev *pdev,
 		pci_set_consistent_dma_mask(intel_private.pcidev,
 					    DMA_BIT_MASK(mask));
 
-	if (bridge->driver == &intel_810_driver)
-		return 1;
+	/*if (bridge->driver == &intel_810_driver)
+		return 1;*/
 
 	if (intel_gtt_init() != 0)
 		return 0;
@@ -1561,12 +1497,19 @@ int intel_gmch_probe(struct pci_dev *pdev,
 }
 EXPORT_SYMBOL(intel_gmch_probe);
 
-struct intel_gtt *intel_gtt_get(void)
+const struct intel_gtt *intel_gtt_get(void)
 {
 	return &intel_private.base;
 }
 EXPORT_SYMBOL(intel_gtt_get);
 
+void intel_gtt_chipset_flush(void)
+{
+	if (intel_private.driver->chipset_flush)
+		intel_private.driver->chipset_flush();
+}
+EXPORT_SYMBOL(intel_gtt_chipset_flush);
+
 void intel_gmch_remove(struct pci_dev *pdev)
 {
 	if (intel_private.pcidev)