diff options
Diffstat (limited to 'drivers/char/agp/intel-gtt.c')
| -rw-r--r-- | drivers/char/agp/intel-gtt.c | 864 |
1 files changed, 392 insertions, 472 deletions
diff --git a/drivers/char/agp/intel-gtt.c b/drivers/char/agp/intel-gtt.c index 29ac6d499fa..0d09b537bb9 100644 --- a/drivers/char/agp/intel-gtt.c +++ b/drivers/char/agp/intel-gtt.c @@ -21,10 +21,10 @@ #include <linux/kernel.h> #include <linux/pagemap.h> #include <linux/agp_backend.h> +#include <linux/delay.h> #include <asm/smp.h> #include "agp.h" #include "intel-agp.h" -#include <linux/intel-gtt.h> #include <drm/intel-gtt.h> /* @@ -39,40 +39,12 @@ #define USE_PCI_DMA_API 0 #endif -/* Max amount of stolen space, anything above will be returned to Linux */ -int intel_max_stolen = 32 * 1024 * 1024; - -static const struct aper_size_info_fixed intel_i810_sizes[] = -{ - {64, 16384, 4}, - /* The 32M mode still requires a 64k gatt */ - {32, 8192, 4} -}; - -#define AGP_DCACHE_MEMORY 1 -#define AGP_PHYS_MEMORY 2 -#define INTEL_AGP_CACHED_MEMORY 3 - -static struct gatt_mask intel_i810_masks[] = -{ - {.mask = I810_PTE_VALID, .type = 0}, - {.mask = (I810_PTE_VALID | I810_PTE_LOCAL), .type = AGP_DCACHE_MEMORY}, - {.mask = I810_PTE_VALID, .type = 0}, - {.mask = I810_PTE_VALID | I830_PTE_SYSTEM_CACHED, - .type = INTEL_AGP_CACHED_MEMORY} -}; - -#define INTEL_AGP_UNCACHED_MEMORY 0 -#define INTEL_AGP_CACHED_MEMORY_LLC 1 -#define INTEL_AGP_CACHED_MEMORY_LLC_GFDT 2 -#define INTEL_AGP_CACHED_MEMORY_LLC_MLC 3 -#define INTEL_AGP_CACHED_MEMORY_LLC_MLC_GFDT 4 - struct intel_gtt_driver { unsigned int gen : 8; unsigned int is_g33 : 1; unsigned int is_pineview : 1; unsigned int is_ironlake : 1; + unsigned int has_pgtbl_enable : 1; unsigned int dma_mask_size : 8; /* Chipset specific GTT setup */ int (*setup)(void); @@ -95,14 +67,12 @@ static struct _intel_private { u8 __iomem *registers; phys_addr_t gtt_bus_addr; phys_addr_t gma_bus_addr; - phys_addr_t pte_bus_addr; + u32 PGETBL_save; u32 __iomem *gtt; /* I915G */ + bool clear_fake_agp; /* on first access via agp, fill with scratch */ int num_dcache_entries; - union { - void __iomem *i9xx_flush_page; - void *i8xx_flush_page; - }; - struct page *i8xx_page; + void __iomem *i9xx_flush_page; + char *i81x_gtt_table; struct resource ifp_resource; int resource_valid; struct page *scratch_page; @@ -113,42 +83,31 @@ static struct _intel_private { #define IS_G33 intel_private.driver->is_g33 #define IS_PINEVIEW intel_private.driver->is_pineview #define IS_IRONLAKE intel_private.driver->is_ironlake +#define HAS_PGTBL_EN intel_private.driver->has_pgtbl_enable -static void intel_agp_free_sglist(struct agp_memory *mem) -{ - struct sg_table st; - - st.sgl = mem->sg_list; - st.orig_nents = st.nents = mem->page_count; - - sg_free_table(&st); - - mem->sg_list = NULL; - mem->num_sg = 0; -} - -static int intel_agp_map_memory(struct agp_memory *mem) +int intel_gtt_map_memory(struct page **pages, unsigned int num_entries, + struct scatterlist **sg_list, int *num_sg) { struct sg_table st; struct scatterlist *sg; int i; - if (mem->sg_list) + if (*sg_list) return 0; /* already mapped (for e.g. resume */ - DBG("try mapping %lu pages\n", (unsigned long)mem->page_count); + DBG("try mapping %lu pages\n", (unsigned long)num_entries); - if (sg_alloc_table(&st, mem->page_count, GFP_KERNEL)) + if (sg_alloc_table(&st, num_entries, GFP_KERNEL)) goto err; - mem->sg_list = sg = st.sgl; + *sg_list = sg = st.sgl; - for (i = 0 ; i < mem->page_count; i++, sg = sg_next(sg)) - sg_set_page(sg, mem->pages[i], PAGE_SIZE, 0); + for (i = 0 ; i < num_entries; i++, sg = sg_next(sg)) + sg_set_page(sg, pages[i], PAGE_SIZE, 0); - mem->num_sg = pci_map_sg(intel_private.pcidev, mem->sg_list, - mem->page_count, PCI_DMA_BIDIRECTIONAL); - if (unlikely(!mem->num_sg)) + *num_sg = pci_map_sg(intel_private.pcidev, *sg_list, + num_entries, PCI_DMA_BIDIRECTIONAL); + if (unlikely(!*num_sg)) goto err; return 0; @@ -157,90 +116,22 @@ err: sg_free_table(&st); return -ENOMEM; } +EXPORT_SYMBOL(intel_gtt_map_memory); -static void intel_agp_unmap_memory(struct agp_memory *mem) +void intel_gtt_unmap_memory(struct scatterlist *sg_list, int num_sg) { + struct sg_table st; DBG("try unmapping %lu pages\n", (unsigned long)mem->page_count); - pci_unmap_sg(intel_private.pcidev, mem->sg_list, - mem->page_count, PCI_DMA_BIDIRECTIONAL); - intel_agp_free_sglist(mem); -} - -static int intel_i810_fetch_size(void) -{ - u32 smram_miscc; - struct aper_size_info_fixed *values; - - pci_read_config_dword(intel_private.bridge_dev, - I810_SMRAM_MISCC, &smram_miscc); - values = A_SIZE_FIX(agp_bridge->driver->aperture_sizes); - - if ((smram_miscc & I810_GMS) == I810_GMS_DISABLE) { - dev_warn(&intel_private.bridge_dev->dev, "i810 is disabled\n"); - return 0; - } - if ((smram_miscc & I810_GFX_MEM_WIN_SIZE) == I810_GFX_MEM_WIN_32M) { - agp_bridge->current_size = (void *) (values + 1); - agp_bridge->aperture_size_idx = 1; - return values[1].size; - } else { - agp_bridge->current_size = (void *) (values); - agp_bridge->aperture_size_idx = 0; - return values[0].size; - } - - return 0; -} - -static int intel_i810_configure(void) -{ - struct aper_size_info_fixed *current_size; - u32 temp; - int i; - - current_size = A_SIZE_FIX(agp_bridge->current_size); - - if (!intel_private.registers) { - pci_read_config_dword(intel_private.pcidev, I810_MMADDR, &temp); - temp &= 0xfff80000; - - intel_private.registers = ioremap(temp, 128 * 4096); - if (!intel_private.registers) { - dev_err(&intel_private.pcidev->dev, - "can't remap memory\n"); - return -ENOMEM; - } - } + pci_unmap_sg(intel_private.pcidev, sg_list, + num_sg, PCI_DMA_BIDIRECTIONAL); - if ((readl(intel_private.registers+I810_DRAM_CTL) - & I810_DRAM_ROW_0) == I810_DRAM_ROW_0_SDRAM) { - /* This will need to be dynamically assigned */ - dev_info(&intel_private.pcidev->dev, - "detected 4MB dedicated video ram\n"); - intel_private.num_dcache_entries = 1024; - } - pci_read_config_dword(intel_private.pcidev, I810_GMADDR, &temp); - agp_bridge->gart_bus_addr = (temp & PCI_BASE_ADDRESS_MEM_MASK); - writel(agp_bridge->gatt_bus_addr | I810_PGETBL_ENABLED, intel_private.registers+I810_PGETBL_CTL); - readl(intel_private.registers+I810_PGETBL_CTL); /* PCI Posting. */ - - if (agp_bridge->driver->needs_scratch_page) { - for (i = 0; i < current_size->num_entries; i++) { - writel(agp_bridge->scratch_page, intel_private.registers+I810_PTE_BASE+(i*4)); - } - readl(intel_private.registers+I810_PTE_BASE+((i-1)*4)); /* PCI posting. */ - } - global_cache_flush(); - return 0; -} + st.sgl = sg_list; + st.orig_nents = st.nents = num_sg; -static void intel_i810_cleanup(void) -{ - writel(0, intel_private.registers+I810_PGETBL_CTL); - readl(intel_private.registers); /* PCI Posting. */ - iounmap(intel_private.registers); + sg_free_table(&st); } +EXPORT_SYMBOL(intel_gtt_unmap_memory); static void intel_fake_agp_enable(struct agp_bridge_data *bridge, u32 mode) { @@ -277,80 +168,64 @@ static void i8xx_destroy_pages(struct page *page) atomic_dec(&agp_bridge->current_memory_agp); } -static int intel_i810_insert_entries(struct agp_memory *mem, off_t pg_start, - int type) +#define I810_GTT_ORDER 4 +static int i810_setup(void) { - int i, j, num_entries; - void *temp; - int ret = -EINVAL; - int mask_type; - - if (mem->page_count == 0) - goto out; - - temp = agp_bridge->current_size; - num_entries = A_SIZE_FIX(temp)->num_entries; + u32 reg_addr; + char *gtt_table; - if ((pg_start + mem->page_count) > num_entries) - goto out_err; + /* i81x does not preallocate the gtt. It's always 64kb in size. */ + gtt_table = alloc_gatt_pages(I810_GTT_ORDER); + if (gtt_table == NULL) + return -ENOMEM; + intel_private.i81x_gtt_table = gtt_table; + pci_read_config_dword(intel_private.pcidev, I810_MMADDR, ®_addr); + reg_addr &= 0xfff80000; - for (j = pg_start; j < (pg_start + mem->page_count); j++) { - if (!PGE_EMPTY(agp_bridge, readl(agp_bridge->gatt_table+j))) { - ret = -EBUSY; - goto out_err; - } - } + intel_private.registers = ioremap(reg_addr, KB(64)); + if (!intel_private.registers) + return -ENOMEM; - if (type != mem->type) - goto out_err; + writel(virt_to_phys(gtt_table) | I810_PGETBL_ENABLED, + intel_private.registers+I810_PGETBL_CTL); - mask_type = agp_bridge->driver->agp_type_to_mask_type(agp_bridge, type); + intel_private.gtt_bus_addr = reg_addr + I810_PTE_BASE; - switch (mask_type) { - case AGP_DCACHE_MEMORY: - if (!mem->is_flushed) - global_cache_flush(); - for (i = pg_start; i < (pg_start + mem->page_count); i++) { - writel((i*4096)|I810_PTE_LOCAL|I810_PTE_VALID, - intel_private.registers+I810_PTE_BASE+(i*4)); - } - readl(intel_private.registers+I810_PTE_BASE+((i-1)*4)); - break; - case AGP_PHYS_MEMORY: - case AGP_NORMAL_MEMORY: - if (!mem->is_flushed) - global_cache_flush(); - for (i = 0, j = pg_start; i < mem->page_count; i++, j++) { - writel(agp_bridge->driver->mask_memory(agp_bridge, - page_to_phys(mem->pages[i]), mask_type), - intel_private.registers+I810_PTE_BASE+(j*4)); - } - readl(intel_private.registers+I810_PTE_BASE+((j-1)*4)); - break; - default: - goto out_err; + if ((readl(intel_private.registers+I810_DRAM_CTL) + & I810_DRAM_ROW_0) == I810_DRAM_ROW_0_SDRAM) { + dev_info(&intel_private.pcidev->dev, + "detected 4MB dedicated video ram\n"); + intel_private.num_dcache_entries = 1024; } -out: - ret = 0; -out_err: - mem->is_flushed = true; - return ret; + return 0; } -static int intel_i810_remove_entries(struct agp_memory *mem, off_t pg_start, - int type) +static void i810_cleanup(void) +{ + writel(0, intel_private.registers+I810_PGETBL_CTL); + free_gatt_pages(intel_private.i81x_gtt_table, I810_GTT_ORDER); +} + +static int i810_insert_dcache_entries(struct agp_memory *mem, off_t pg_start, + int type) { int i; - if (mem->page_count == 0) - return 0; + if ((pg_start + mem->page_count) + > intel_private.num_dcache_entries) + return -EINVAL; + + if (!mem->is_flushed) + global_cache_flush(); - for (i = pg_start; i < (mem->page_count + pg_start); i++) { - writel(agp_bridge->scratch_page, intel_private.registers+I810_PTE_BASE+(i*4)); + for (i = pg_start; i < (pg_start + mem->page_count); i++) { + dma_addr_t addr = i << PAGE_SHIFT; + intel_private.driver->write_entry(addr, + i, type); } - readl(intel_private.registers+I810_PTE_BASE+((i-1)*4)); + readl(intel_private.gtt+i-1); return 0; } @@ -397,29 +272,6 @@ static struct agp_memory *alloc_agpphysmem_i8xx(size_t pg_count, int type) return new; } -static struct agp_memory *intel_i810_alloc_by_type(size_t pg_count, int type) -{ - struct agp_memory *new; - - if (type == AGP_DCACHE_MEMORY) { - if (pg_count != intel_private.num_dcache_entries) - return NULL; - - new = agp_create_memory(1); - if (new == NULL) - return NULL; - - new->type = AGP_DCACHE_MEMORY; - new->page_count = pg_count; - new->num_scratch_pages = 0; - agp_free_page_array(new); - return new; - } - if (type == AGP_PHYS_MEMORY) - return alloc_agpphysmem_i8xx(pg_count, type); - return NULL; -} - static void intel_i810_free_by_type(struct agp_memory *curr) { agp_free_key(curr->key); @@ -437,13 +289,6 @@ static void intel_i810_free_by_type(struct agp_memory *curr) kfree(curr); } -static unsigned long intel_i810_mask_memory(struct agp_bridge_data *bridge, - dma_addr_t addr, int type) -{ - /* Type checking must be done elsewhere */ - return addr | bridge->driver->masks[type].mask; -} - static int intel_gtt_setup_scratch_page(void) { struct page *page; @@ -455,7 +300,7 @@ static int intel_gtt_setup_scratch_page(void) get_page(page); set_pages_uc(page, 1); - if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2) { + if (intel_private.base.needs_dmar) { dma_addr = pci_map_page(intel_private.pcidev, page, 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); if (pci_dma_mapping_error(intel_private.pcidev, dma_addr)) @@ -470,34 +315,45 @@ static int intel_gtt_setup_scratch_page(void) return 0; } -static const struct aper_size_info_fixed const intel_fake_agp_sizes[] = { +static void i810_write_entry(dma_addr_t addr, unsigned int entry, + unsigned int flags) +{ + u32 pte_flags = I810_PTE_VALID; + + switch (flags) { + case AGP_DCACHE_MEMORY: + pte_flags |= I810_PTE_LOCAL; + break; + case AGP_USER_CACHED_MEMORY: + pte_flags |= I830_PTE_SYSTEM_CACHED; + break; + } + + writel(addr | pte_flags, intel_private.gtt + entry); +} + +static const struct aper_size_info_fixed intel_fake_agp_sizes[] = { + {32, 8192, 3}, + {64, 16384, 4}, {128, 32768, 5}, - /* The 64M mode still requires a 128k gatt */ - {64, 16384, 5}, {256, 65536, 6}, {512, 131072, 7}, }; -static unsigned int intel_gtt_stolen_entries(void) +static unsigned int intel_gtt_stolen_size(void) { u16 gmch_ctrl; u8 rdct; int local = 0; static const int ddt[4] = { 0, 16, 32, 64 }; - unsigned int overhead_entries, stolen_entries; unsigned int stolen_size = 0; + if (INTEL_GTT_GEN == 1) + return 0; /* no stolen mem on i81x */ + pci_read_config_word(intel_private.bridge_dev, I830_GMCH_CTRL, &gmch_ctrl); - if (INTEL_GTT_GEN > 4 || IS_PINEVIEW) - overhead_entries = 0; - else - overhead_entries = intel_private.base.gtt_mappable_entries - / 1024; - - overhead_entries += 1; /* BIOS popup */ - if (intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82830_HB || intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82845G_HB) { switch (gmch_ctrl & I830_GMCH_GMS_MASK) { @@ -623,12 +479,7 @@ static unsigned int intel_gtt_stolen_entries(void) } } - if (!local && stolen_size > intel_max_stolen) { - dev_info(&intel_private.bridge_dev->dev, - "detected %dK stolen memory, trimming to %dK\n", - stolen_size / KB(1), intel_max_stolen / KB(1)); - stolen_size = intel_max_stolen; - } else if (stolen_size > 0) { + if (stolen_size > 0) { dev_info(&intel_private.bridge_dev->dev, "detected %dK %s memory\n", stolen_size / KB(1), local ? "local" : "stolen"); } else { @@ -637,46 +488,88 @@ static unsigned int intel_gtt_stolen_entries(void) stolen_size = 0; } - stolen_entries = stolen_size/KB(4) - overhead_entries; + return stolen_size; +} - return stolen_entries; +static void i965_adjust_pgetbl_size(unsigned int size_flag) +{ + u32 pgetbl_ctl, pgetbl_ctl2; + + /* ensure that ppgtt is disabled */ + pgetbl_ctl2 = readl(intel_private.registers+I965_PGETBL_CTL2); + pgetbl_ctl2 &= ~I810_PGETBL_ENABLED; + writel(pgetbl_ctl2, intel_private.registers+I965_PGETBL_CTL2); + + /* write the new ggtt size */ + pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL); + pgetbl_ctl &= ~I965_PGETBL_SIZE_MASK; + pgetbl_ctl |= size_flag; + writel(pgetbl_ctl, intel_private.registers+I810_PGETBL_CTL); } -static unsigned int intel_gtt_total_entries(void) +static unsigned int i965_gtt_total_entries(void) { int size; + u32 pgetbl_ctl; + u16 gmch_ctl; - if (IS_G33 || INTEL_GTT_GEN == 4 || INTEL_GTT_GEN == 5) { - u32 pgetbl_ctl; - pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL); + pci_read_config_word(intel_private.bridge_dev, + I830_GMCH_CTRL, &gmch_ctl); - switch (pgetbl_ctl & I965_PGETBL_SIZE_MASK) { - case I965_PGETBL_SIZE_128KB: - size = KB(128); - break; - case I965_PGETBL_SIZE_256KB: - size = KB(256); - break; - case I965_PGETBL_SIZE_512KB: - size = KB(512); + if (INTEL_GTT_GEN == 5) { + switch (gmch_ctl & G4x_GMCH_SIZE_MASK) { + case G4x_GMCH_SIZE_1M: + case G4x_GMCH_SIZE_VT_1M: + i965_adjust_pgetbl_size(I965_PGETBL_SIZE_1MB); break; - case I965_PGETBL_SIZE_1MB: - size = KB(1024); + case G4x_GMCH_SIZE_VT_1_5M: + i965_adjust_pgetbl_size(I965_PGETBL_SIZE_1_5MB); break; - case I965_PGETBL_SIZE_2MB: - size = KB(2048); + case G4x_GMCH_SIZE_2M: + case G4x_GMCH_SIZE_VT_2M: + i965_adjust_pgetbl_size(I965_PGETBL_SIZE_2MB); break; - case I965_PGETBL_SIZE_1_5MB: - size = KB(1024 + 512); - break; - default: - dev_info(&intel_private.pcidev->dev, - "unknown page table size, assuming 512KB\n"); - size = KB(512); } + } - return size/4; - } else if (INTEL_GTT_GEN == 6) { + pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL); + + switch (pgetbl_ctl & I965_PGETBL_SIZE_MASK) { + case I965_PGETBL_SIZE_128KB: + size = KB(128); + break; + case I965_PGETBL_SIZE_256KB: + size = KB(256); + break; + case I965_PGETBL_SIZE_512KB: + size = KB(512); + break; + /* GTT pagetable sizes bigger than 512KB are not possible on G33! */ + case I965_PGETBL_SIZE_1MB: + size = KB(1024); + break; + case I965_PGETBL_SIZE_2MB: + size = KB(2048); + break; + case I965_PGETBL_SIZE_1_5MB: + size = KB(1024 + 512); + break; + default: + dev_info(&intel_private.pcidev->dev, + "unknown page table size, assuming 512KB\n"); + size = KB(512); + } + + return size/4; +} + +static unsigned int intel_gtt_total_entries(void) +{ + int size; + + if (IS_G33 || INTEL_GTT_GEN == 4 || INTEL_GTT_GEN == 5) + return i965_gtt_total_entries(); + else if (INTEL_GTT_GEN == 6) { u16 snb_gmch_ctl; pci_read_config_word(intel_private.pcidev, SNB_GMCH_CTRL, &snb_gmch_ctl); @@ -706,7 +599,18 @@ static unsigned int intel_gtt_mappable_entries(void) { unsigned int aperture_size; - if (INTEL_GTT_GEN == 2) { + if (INTEL_GTT_GEN == 1) { + u32 smram_miscc; + + pci_read_config_dword(intel_private.bridge_dev, + I810_SMRAM_MISCC, &smram_miscc); + + if ((smram_miscc & I810_GFX_MEM_WIN_SIZE) + == I810_GFX_MEM_WIN_32M) + aperture_size = MB(32); + else + aperture_size = MB(64); + } else if (INTEL_GTT_GEN == 2) { u16 gmch_ctrl; pci_read_config_word(intel_private.bridge_dev, @@ -739,7 +643,7 @@ static void intel_gtt_cleanup(void) iounmap(intel_private.gtt); iounmap(intel_private.registers); - + intel_gtt_teardown_scratch_page(); } @@ -755,6 +659,14 @@ static int intel_gtt_init(void) intel_private.base.gtt_mappable_entries = intel_gtt_mappable_entries(); intel_private.base.gtt_total_entries = intel_gtt_total_entries(); + /* save the PGETBL reg for resume */ + intel_private.PGETBL_save = + readl(intel_private.registers+I810_PGETBL_CTL) + & ~I810_PGETBL_ENABLED; + /* we only ever restore the register when enabling the PGTBL... */ + if (HAS_PGTBL_EN) + intel_private.PGETBL_save |= I810_PGETBL_ENABLED; + dev_info(&intel_private.bridge_dev->dev, "detected gtt size: %dK total, %dK mappable\n", intel_private.base.gtt_total_entries * 4, @@ -772,14 +684,9 @@ static int intel_gtt_init(void) global_cache_flush(); /* FIXME: ? */ - /* we have to call this as early as possible after the MMIO base address is known */ - intel_private.base.gtt_stolen_entries = intel_gtt_stolen_entries(); - if (intel_private.base.gtt_stolen_entries == 0) { - intel_private.driver->cleanup(); - iounmap(intel_private.registers); - iounmap(intel_private.gtt); - return -ENOMEM; - } + intel_private.base.stolen_size = intel_gtt_stolen_size(); + + intel_private.base.needs_dmar = USE_PCI_DMA_API && INTEL_GTT_GEN > 2; ret = intel_gtt_setup_scratch_page(); if (ret != 0) { @@ -812,28 +719,6 @@ static int intel_fake_agp_fetch_size(void) static void i830_cleanup(void) { - if (intel_private.i8xx_flush_page) { - kunmap(intel_private.i8xx_flush_page); - intel_private.i8xx_flush_page = NULL; - } - - __free_page(intel_private.i8xx_page); - intel_private.i8xx_page = NULL; -} - -static void intel_i830_setup_flush(void) -{ - /* return if we've already set the flush mechanism up */ - if (intel_private.i8xx_page) - return; - - intel_private.i8xx_page = alloc_page(GFP_KERNEL); - if (!intel_private.i8xx_page) - return; - - intel_private.i8xx_flush_page = kmap(intel_private.i8xx_page); - if (!intel_private.i8xx_flush_page) - i830_cleanup(); } /* The chipset_flush interface needs to get data that has already been @@ -848,39 +733,46 @@ static void intel_i830_setup_flush(void) */ static void i830_chipset_flush(void) { - unsigned int *pg = intel_private.i8xx_flush_page; - - memset(pg, 0, 1024); + unsigned long timeout = jiffies + msecs_to_jiffies(1000); + + /* Forcibly evict everything from the CPU write buffers. + * clflush appears to be insufficient. + */ + wbinvd_on_all_cpus(); + + /* Now we've only seen documents for this magic bit on 855GM, + * we hope it exists for the other gen2 chipsets... + * + * Also works as advertised on my 845G. + */ + writel(readl(intel_private.registers+I830_HIC) | (1<<31), + intel_private.registers+I830_HIC); + + while (readl(intel_private.registers+I830_HIC) & (1<<31)) { + if (time_after(jiffies, timeout)) + break; - if (cpu_has_clflush) - clflush_cache_range(pg, 1024); - else if (wbinvd_on_all_cpus() != 0) - printk(KERN_ERR "Timed out waiting for cache flush.\n"); + udelay(50); + } } static void i830_write_entry(dma_addr_t addr, unsigned int entry, unsigned int flags) { u32 pte_flags = I810_PTE_VALID; - - switch (flags) { - case AGP_DCACHE_MEMORY: - pte_flags |= I810_PTE_LOCAL; - break; - case AGP_USER_CACHED_MEMORY: + + if (flags == AGP_USER_CACHED_MEMORY) pte_flags |= I830_PTE_SYSTEM_CACHED; - break; - } writel(addr | pte_flags, intel_private.gtt + entry); } -static void intel_enable_gtt(void) +static bool intel_enable_gtt(void) { u32 gma_addr; - u16 gmch_ctrl; + u8 __iomem *reg; - if (INTEL_GTT_GEN == 2) + if (INTEL_GTT_GEN <= 2) pci_read_config_dword(intel_private.pcidev, I810_GMADDR, &gma_addr); else @@ -889,13 +781,47 @@ static void intel_enable_gtt(void) intel_private.gma_bus_addr = (gma_addr & PCI_BASE_ADDRESS_MEM_MASK); - pci_read_config_word(intel_private.bridge_dev, I830_GMCH_CTRL, &gmch_ctrl); - gmch_ctrl |= I830_GMCH_ENABLED; - pci_write_config_word(intel_private.bridge_dev, I830_GMCH_CTRL, gmch_ctrl); + if (INTEL_GTT_GEN >= 6) + return true; - writel(intel_private.pte_bus_addr|I810_PGETBL_ENABLED, - intel_private.registers+I810_PGETBL_CTL); - readl(intel_private.registers+I810_PGETBL_CTL); /* PCI Posting. */ + if (INTEL_GTT_GEN == 2) { + u16 gmch_ctrl; + + pci_read_config_word(intel_private.bridge_dev, + I830_GMCH_CTRL, &gmch_ctrl); + gmch_ctrl |= I830_GMCH_ENABLED; + pci_write_config_word(intel_private.bridge_dev, + I830_GMCH_CTRL, gmch_ctrl); + + pci_read_config_word(intel_private.bridge_dev, + I830_GMCH_CTRL, &gmch_ctrl); + if ((gmch_ctrl & I830_GMCH_ENABLED) == 0) { + dev_err(&intel_private.pcidev->dev, + "failed to enable the GTT: GMCH_CTRL=%x\n", + gmch_ctrl); + return false; + } + } + + /* On the resume path we may be adjusting the PGTBL value, so + * be paranoid and flush all chipset write buffers... + */ + if (INTEL_GTT_GEN >= 3) + writel(0, intel_private.registers+GFX_FLSH_CNTL); + + reg = intel_private.registers+I810_PGETBL_CTL; + writel(intel_private.PGETBL_save, reg); + if (HAS_PGTBL_EN && (readl(reg) & I810_PGETBL_ENABLED) == 0) { + dev_err(&intel_private.pcidev->dev, + "failed to enable the GTT: PGETBL=%x [expected %x]\n", + readl(reg), intel_private.PGETBL_save); + return false; + } + + if (INTEL_GTT_GEN >= 3) + writel(0, intel_private.registers+GFX_FLSH_CNTL); + + return true; } static int i830_setup(void) @@ -910,10 +836,6 @@ static int i830_setup(void) return -ENOMEM; intel_private.gtt_bus_addr = reg_addr + I810_PTE_BASE; - intel_private.pte_bus_addr = - readl(intel_private.registers+I810_PGETBL_CTL) & 0xfffff000; - - intel_i830_setup_flush(); return 0; } @@ -934,21 +856,12 @@ static int intel_fake_agp_free_gatt_table(struct agp_bridge_data *bridge) static int intel_fake_agp_configure(void) { - int i; - - intel_enable_gtt(); + if (!intel_enable_gtt()) + return -EIO; + intel_private.clear_fake_agp = true; agp_bridge->gart_bus_addr = intel_private.gma_bus_addr; - for (i = intel_private.base.gtt_stolen_entries; - i < intel_private.base.gtt_total_entries; i++) { - intel_private.driver->write_entry(intel_private.scratch_page_dma, - i, 0); - } - readl(intel_private.gtt+i-1); /* PCI Posting. */ - - global_cache_flush(); - return 0; } @@ -965,10 +878,10 @@ static bool i830_check_flags(unsigned int flags) return false; } -static void intel_gtt_insert_sg_entries(struct scatterlist *sg_list, - unsigned int sg_len, - unsigned int pg_start, - unsigned int flags) +void intel_gtt_insert_sg_entries(struct scatterlist *sg_list, + unsigned int sg_len, + unsigned int pg_start, + unsigned int flags) { struct scatterlist *sg; unsigned int len, m; @@ -989,27 +902,41 @@ static void intel_gtt_insert_sg_entries(struct scatterlist *sg_list, } readl(intel_private.gtt+j-1); } +EXPORT_SYMBOL(intel_gtt_insert_sg_entries); + +void intel_gtt_insert_pages(unsigned int first_entry, unsigned int num_entries, + struct page **pages, unsigned int flags) +{ + int i, j; + + for (i = 0, j = first_entry; i < num_entries; i++, j++) { + dma_addr_t addr = page_to_phys(pages[i]); + intel_private.driver->write_entry(addr, + j, flags); + } + readl(intel_private.gtt+j-1); +} +EXPORT_SYMBOL(intel_gtt_insert_pages); static int intel_fake_agp_insert_entries(struct agp_memory *mem, off_t pg_start, int type) { - int i, j; int ret = -EINVAL; - if (mem->page_count == 0) - goto out; + if (intel_private.clear_fake_agp) { + int start = intel_private.base.stolen_size / PAGE_SIZE; + int end = intel_private.base.gtt_mappable_entries; + intel_gtt_clear_range(start, end - start); + intel_private.clear_fake_agp = false; + } - if (pg_start < intel_private.base.gtt_stolen_entries) { - dev_printk(KERN_DEBUG, &intel_private.pcidev->dev, - "pg_start == 0x%.8lx, gtt_stolen_entries == 0x%.8x\n", - pg_start, intel_private.base.gtt_stolen_entries); + if (INTEL_GTT_GEN == 1 && type == AGP_DCACHE_MEMORY) + return i810_insert_dcache_entries(mem, pg_start, type); - dev_info(&intel_private.pcidev->dev, - "trying to insert into local/stolen memory\n"); - goto out_err; - } + if (mem->page_count == 0) + goto out; - if ((pg_start + mem->page_count) > intel_private.base.gtt_total_entries) + if (pg_start + mem->page_count > intel_private.base.gtt_total_entries) goto out_err; if (type != mem->type) @@ -1021,21 +948,17 @@ static int intel_fake_agp_insert_entries(struct agp_memory *mem, if (!mem->is_flushed) global_cache_flush(); - if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2) { - ret = intel_agp_map_memory(mem); + if (intel_private.base.needs_dmar) { + ret = intel_gtt_map_memory(mem->pages, mem->page_count, + &mem->sg_list, &mem->num_sg); if (ret != 0) return ret; intel_gtt_insert_sg_entries(mem->sg_list, mem->num_sg, pg_start, type); - } else { - for (i = 0, j = pg_start; i < mem->page_count; i++, j++) { - dma_addr_t addr = page_to_phys(mem->pages[i]); - intel_private.driver->write_entry(addr, - j, type); - } - readl(intel_private.gtt+j-1); - } + } else + intel_gtt_insert_pages(pg_start, mem->page_count, mem->pages, + type); out: ret = 0; @@ -1044,40 +967,54 @@ out_err: return ret; } +void intel_gtt_clear_range(unsigned int first_entry, unsigned int num_entries) +{ + unsigned int i; + + for (i = first_entry; i < (first_entry + num_entries); i++) { + intel_private.driver->write_entry(intel_private.scratch_page_dma, + i, 0); + } + readl(intel_private.gtt+i-1); +} +EXPORT_SYMBOL(intel_gtt_clear_range); + static int intel_fake_agp_remove_entries(struct agp_memory *mem, off_t pg_start, int type) { - int i; - if (mem->page_count == 0) return 0; - if (pg_start < intel_private.base.gtt_stolen_entries) { - dev_info(&intel_private.pcidev->dev, - "trying to disable local/stolen memory\n"); - return -EINVAL; - } - - if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2) - intel_agp_unmap_memory(mem); + intel_gtt_clear_range(pg_start, mem->page_count); - for (i = pg_start; i < (mem->page_count + pg_start); i++) { - intel_private.driver->write_entry(intel_private.scratch_page_dma, - i, 0); + if (intel_private.base.needs_dmar) { + intel_gtt_unmap_memory(mem->sg_list, mem->num_sg); + mem->sg_list = NULL; + mem->num_sg = 0; } - readl(intel_private.gtt+i-1); return 0; } -static void intel_fake_agp_chipset_flush(struct agp_bridge_data *bridge) -{ - intel_private.driver->chipset_flush(); -} - static struct agp_memory *intel_fake_agp_alloc_by_type(size_t pg_count, int type) { + struct agp_memory *new; + + if (type == AGP_DCACHE_MEMORY && INTEL_GTT_GEN == 1) { + if (pg_count != intel_private.num_dcache_entries) + return NULL; + + new = agp_create_memory(1); + if (new == NULL) + return NULL; + + new->type = AGP_DCACHE_MEMORY; + new->page_count = pg_count; + new->num_scratch_pages = 0; + agp_free_page_array(new); + return new; + } if (type == AGP_PHYS_MEMORY) return alloc_agpphysmem_i8xx(pg_count, type); /* always return NULL for other allocation types for now */ @@ -1274,40 +1211,11 @@ static int i9xx_setup(void) intel_private.gtt_bus_addr = reg_addr + gtt_offset; } - intel_private.pte_bus_addr = - readl(intel_private.registers+I810_PGETBL_CTL) & 0xfffff000; - intel_i9xx_setup_flush(); return 0; } -static const struct agp_bridge_driver intel_810_driver = { - .owner = THIS_MODULE, - .aperture_sizes = intel_i810_sizes, - .size_type = FIXED_APER_SIZE, - .num_aperture_sizes = 2, - .needs_scratch_page = true, - .configure = intel_i810_configure, - .fetch_size = intel_i810_fetch_size, - .cleanup = intel_i810_cleanup, - .mask_memory = intel_i810_mask_memory, - .masks = intel_i810_masks, - .agp_enable = intel_fake_agp_enable, - .cache_flush = global_cache_flush, - .create_gatt_table = agp_generic_create_gatt_table, - .free_gatt_table = agp_generic_free_gatt_table, - .insert_memory = intel_i810_insert_entries, - .remove_memory = intel_i810_remove_entries, - .alloc_by_type = intel_i810_alloc_by_type, - .free_by_type = intel_i810_free_by_type, - .agp_alloc_page = agp_generic_alloc_page, - .agp_alloc_pages = agp_generic_alloc_pages, - .agp_destroy_page = agp_generic_destroy_page, - .agp_destroy_pages = agp_generic_destroy_pages, - .agp_type_to_mask_type = agp_generic_type_to_mask_type, -}; - static const struct agp_bridge_driver intel_fake_agp_driver = { .owner = THIS_MODULE, .size_type = FIXED_APER_SIZE, @@ -1328,15 +1236,20 @@ static const struct agp_bridge_driver intel_fake_agp_driver = { .agp_alloc_pages = agp_generic_alloc_pages, .agp_destroy_page = agp_generic_destroy_page, .agp_destroy_pages = agp_generic_destroy_pages, - .chipset_flush = intel_fake_agp_chipset_flush, }; static const struct intel_gtt_driver i81x_gtt_driver = { .gen = 1, + .has_pgtbl_enable = 1, .dma_mask_size = 32, + .setup = i810_setup, + .cleanup = i810_cleanup, + .check_flags = i830_check_flags, + .write_entry = i810_write_entry, }; static const struct intel_gtt_driver i8xx_gtt_driver = { .gen = 2, + .has_pgtbl_enable = 1, .setup = i830_setup, .cleanup = i830_cleanup, .write_entry = i830_write_entry, @@ -1346,10 +1259,11 @@ static const struct intel_gtt_driver i8xx_gtt_driver = { }; static const struct intel_gtt_driver i915_gtt_driver = { .gen = 3, + .has_pgtbl_enable = 1, .setup = i9xx_setup, .cleanup = i9xx_cleanup, /* i945 is the last gpu to need phys mem (for overlay and cursors). */ - .write_entry = i830_write_entry, + .write_entry = i830_write_entry, .dma_mask_size = 32, .check_flags = i830_check_flags, .chipset_flush = i9xx_chipset_flush, @@ -1376,6 +1290,7 @@ static const struct intel_gtt_driver pineview_gtt_driver = { }; static const struct intel_gtt_driver i965_gtt_driver = { .gen = 4, + .has_pgtbl_enable = 1, .setup = i9xx_setup, .cleanup = i9xx_cleanup, .write_entry = i965_write_entry, @@ -1419,93 +1334,92 @@ static const struct intel_gtt_driver sandybridge_gtt_driver = { static const struct intel_gtt_driver_description { unsigned int gmch_chip_id; char *name; - const struct agp_bridge_driver *gmch_driver; const struct intel_gtt_driver *gtt_driver; } intel_gtt_chipsets[] = { - { PCI_DEVICE_ID_INTEL_82810_IG1, "i810", &intel_810_driver, + { PCI_DEVICE_ID_INTEL_82810_IG1, "i810", &i81x_gtt_driver}, - { PCI_DEVICE_ID_INTEL_82810_IG3, "i810", &intel_810_driver, + { PCI_DEVICE_ID_INTEL_82810_IG3, "i810", &i81x_gtt_driver}, - { PCI_DEVICE_ID_INTEL_82810E_IG, "i810", &intel_810_driver, + { PCI_DEVICE_ID_INTEL_82810E_IG, "i810", &i81x_gtt_driver}, - { PCI_DEVICE_ID_INTEL_82815_CGC, "i815", &intel_810_driver, + { PCI_DEVICE_ID_INTEL_82815_CGC, "i815", &i81x_gtt_driver}, { PCI_DEVICE_ID_INTEL_82830_CGC, "830M", - &intel_fake_agp_driver, &i8xx_gtt_driver}, - { PCI_DEVICE_ID_INTEL_82845G_IG, "830M", - &intel_fake_agp_driver, &i8xx_gtt_driver}, + &i8xx_gtt_driver}, + { PCI_DEVICE_ID_INTEL_82845G_IG, "845G", + &i8xx_gtt_driver}, { PCI_DEVICE_ID_INTEL_82854_IG, "854", - &intel_fake_agp_driver, &i8xx_gtt_driver}, + &i8xx_gtt_driver}, { PCI_DEVICE_ID_INTEL_82855GM_IG, "855GM", - &intel_fake_agp_driver, &i8xx_gtt_driver}, + &i8xx_gtt_driver}, { PCI_DEVICE_ID_INTEL_82865_IG, "865", - &intel_fake_agp_driver, &i8xx_gtt_driver}, + &i8xx_gtt_driver}, { PCI_DEVICE_ID_INTEL_E7221_IG, "E7221 (i915)", - &intel_fake_agp_driver, &i915_gtt_driver }, + &i915_gtt_driver }, { PCI_DEVICE_ID_INTEL_82915G_IG, "915G", - &intel_fake_agp_driver, &i915_gtt_driver }, + &i915_gtt_driver }, { PCI_DEVICE_ID_INTEL_82915GM_IG, "915GM", - &intel_fake_agp_driver, &i915_gtt_driver }, + &i915_gtt_driver }, { PCI_DEVICE_ID_INTEL_82945G_IG, "945G", - &intel_fake_agp_driver, &i915_gtt_driver }, + &i915_gtt_driver }, { PCI_DEVICE_ID_INTEL_82945GM_IG, "945GM", - &intel_fake_agp_driver, &i915_gtt_driver }, + &i915_gtt_driver }, { PCI_DEVICE_ID_INTEL_82945GME_IG, "945GME", - &intel_fake_agp_driver, &i915_gtt_driver }, + &i915_gtt_driver }, { PCI_DEVICE_ID_INTEL_82946GZ_IG, "946GZ", - &intel_fake_agp_driver, &i965_gtt_driver }, + &i965_gtt_driver }, { PCI_DEVICE_ID_INTEL_82G35_IG, "G35", - &intel_fake_agp_driver, &i965_gtt_driver }, + &i965_gtt_driver }, { PCI_DEVICE_ID_INTEL_82965Q_IG, "965Q", - &intel_fake_agp_driver, &i965_gtt_driver }, + &i965_gtt_driver }, { PCI_DEVICE_ID_INTEL_82965G_IG, "965G", - &intel_fake_agp_driver, &i965_gtt_driver }, + &i965_gtt_driver }, { PCI_DEVICE_ID_INTEL_82965GM_IG, "965GM", - &intel_fake_agp_driver, &i965_gtt_driver }, + &i965_gtt_driver }, { PCI_DEVICE_ID_INTEL_82965GME_IG, "965GME/GLE", - &intel_fake_agp_driver, &i965_gtt_driver }, + &i965_gtt_driver }, { PCI_DEVICE_ID_INTEL_G33_IG, "G33", - &intel_fake_agp_driver, &g33_gtt_driver }, + &g33_gtt_driver }, { PCI_DEVICE_ID_INTEL_Q35_IG, "Q35", - &intel_fake_agp_driver, &g33_gtt_driver }, + &g33_gtt_driver }, { PCI_DEVICE_ID_INTEL_Q33_IG, "Q33", - &intel_fake_agp_driver, &g33_gtt_driver }, + &g33_gtt_driver }, { PCI_DEVICE_ID_INTEL_PINEVIEW_M_IG, "GMA3150", - &intel_fake_agp_driver, &pineview_gtt_driver }, + &pineview_gtt_driver }, { PCI_DEVICE_ID_INTEL_PINEVIEW_IG, "GMA3150", - &intel_fake_agp_driver, &pineview_gtt_driver }, + &pineview_gtt_driver }, { PCI_DEVICE_ID_INTEL_GM45_IG, "GM45", - &intel_fake_agp_driver, &g4x_gtt_driver }, + &g4x_gtt_driver }, { PCI_DEVICE_ID_INTEL_EAGLELAKE_IG, "Eaglelake", - &intel_fake_agp_driver, &g4x_gtt_driver }, + &g4x_gtt_driver }, { PCI_DEVICE_ID_INTEL_Q45_IG, "Q45/Q43", - &intel_fake_agp_driver, &g4x_gtt_driver }, + &g4x_gtt_driver }, { PCI_DEVICE_ID_INTEL_G45_IG, "G45/G43", - &intel_fake_agp_driver, &g4x_gtt_driver }, + &g4x_gtt_driver }, { PCI_DEVICE_ID_INTEL_B43_IG, "B43", - &intel_fake_agp_driver, &g4x_gtt_driver }, + &g4x_gtt_driver }, { PCI_DEVICE_ID_INTEL_B43_1_IG, "B43", - &intel_fake_agp_driver, &g4x_gtt_driver }, + &g4x_gtt_driver }, { PCI_DEVICE_ID_INTEL_G41_IG, "G41", - &intel_fake_agp_driver, &g4x_gtt_driver }, + &g4x_gtt_driver }, { PCI_DEVICE_ID_INTEL_IRONLAKE_D_IG, - "HD Graphics", &intel_fake_agp_driver, &ironlake_gtt_driver }, + "HD Graphics", &ironlake_gtt_driver }, { PCI_DEVICE_ID_INTEL_IRONLAKE_M_IG, - "HD Graphics", &intel_fake_agp_driver, &ironlake_gtt_driver }, + "HD Graphics", &ironlake_gtt_driver }, { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT1_IG, - "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver }, + "Sandybridge", &sandybridge_gtt_driver }, { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT2_IG, - "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver }, + "Sandybridge", &sandybridge_gtt_driver }, { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT2_PLUS_IG, - "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver }, + "Sandybridge", &sandybridge_gtt_driver }, { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT1_IG, - "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver }, + "Sandybridge", &sandybridge_gtt_driver }, { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT2_IG, - "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver }, + "Sandybridge", &sandybridge_gtt_driver }, { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT2_PLUS_IG, - "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver }, + "Sandybridge", &sandybridge_gtt_driver }, { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_S_IG, - "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver }, + "Sandybridge", &sandybridge_gtt_driver }, { 0, NULL, NULL } }; @@ -1530,21 +1444,20 @@ int intel_gmch_probe(struct pci_dev *pdev, struct agp_bridge_data *bridge) { int i, mask; - bridge->driver = NULL; + intel_private.driver = NULL; for (i = 0; intel_gtt_chipsets[i].name != NULL; i++) { if (find_gmch(intel_gtt_chipsets[i].gmch_chip_id)) { - bridge->driver = - intel_gtt_chipsets[i].gmch_driver; - intel_private.driver = + intel_private.driver = intel_gtt_chipsets[i].gtt_driver; break; } } - if (!bridge->driver) + if (!intel_private.driver) return 0; + bridge->driver = &intel_fake_agp_driver; bridge->dev_private_data = &intel_private; bridge->dev = pdev; @@ -1560,8 +1473,8 @@ int intel_gmch_probe(struct pci_dev *pdev, pci_set_consistent_dma_mask(intel_private.pcidev, DMA_BIT_MASK(mask)); - if (bridge->driver == &intel_810_driver) - return 1; + /*if (bridge->driver == &intel_810_driver) + return 1;*/ if (intel_gtt_init() != 0) return 0; @@ -1570,12 +1483,19 @@ int intel_gmch_probe(struct pci_dev *pdev, } EXPORT_SYMBOL(intel_gmch_probe); -struct intel_gtt *intel_gtt_get(void) +const struct intel_gtt *intel_gtt_get(void) { return &intel_private.base; } EXPORT_SYMBOL(intel_gtt_get); +void intel_gtt_chipset_flush(void) +{ + if (intel_private.driver->chipset_flush) + intel_private.driver->chipset_flush(); +} +EXPORT_SYMBOL(intel_gtt_chipset_flush); + void intel_gmch_remove(struct pci_dev *pdev) { if (intel_private.pcidev) |