1f51b7662SDaniel Vetter /* 2f51b7662SDaniel Vetter * Intel GTT (Graphics Translation Table) routines 3f51b7662SDaniel Vetter * 4f51b7662SDaniel Vetter * Caveat: This driver implements the linux agp interface, but this is far from 5f51b7662SDaniel Vetter * a agp driver! GTT support ended up here for purely historical reasons: The 6f51b7662SDaniel Vetter * old userspace intel graphics drivers needed an interface to map memory into 7f51b7662SDaniel Vetter * the GTT. And the drm provides a default interface for graphic devices sitting 8f51b7662SDaniel Vetter * on an agp port. So it made sense to fake the GTT support as an agp port to 9f51b7662SDaniel Vetter * avoid having to create a new api. 10f51b7662SDaniel Vetter * 11f51b7662SDaniel Vetter * With gem this does not make much sense anymore, just needlessly complicates 12f51b7662SDaniel Vetter * the code. But as long as the old graphics stack is still support, it's stuck 13f51b7662SDaniel Vetter * here. 14f51b7662SDaniel Vetter * 15f51b7662SDaniel Vetter * /fairy-tale-mode off 16f51b7662SDaniel Vetter */ 17f51b7662SDaniel Vetter 18e2404e7cSDaniel Vetter #include <linux/module.h> 19e2404e7cSDaniel Vetter #include <linux/pci.h> 20e2404e7cSDaniel Vetter #include <linux/init.h> 21e2404e7cSDaniel Vetter #include <linux/kernel.h> 22e2404e7cSDaniel Vetter #include <linux/pagemap.h> 23e2404e7cSDaniel Vetter #include <linux/agp_backend.h> 24bdb8b975SChris Wilson #include <linux/delay.h> 25e2404e7cSDaniel Vetter #include <asm/smp.h> 26e2404e7cSDaniel Vetter #include "agp.h" 27e2404e7cSDaniel Vetter #include "intel-agp.h" 280ade6386SDaniel Vetter #include <drm/intel-gtt.h> 29e2404e7cSDaniel Vetter 30f51b7662SDaniel Vetter /* 31f51b7662SDaniel Vetter * If we have Intel graphics, we're not going to have anything other than 32f51b7662SDaniel Vetter * an Intel IOMMU. So make the correct use of the PCI DMA API contingent 33d3f13810SSuresh Siddha * on the Intel IOMMU support (CONFIG_INTEL_IOMMU). 34f51b7662SDaniel Vetter * Only newer chipsets need to bother with this, of course. 35f51b7662SDaniel Vetter */ 36d3f13810SSuresh Siddha #ifdef CONFIG_INTEL_IOMMU 37f51b7662SDaniel Vetter #define USE_PCI_DMA_API 1 380e87d2b0SDaniel Vetter #else 390e87d2b0SDaniel Vetter #define USE_PCI_DMA_API 0 40f51b7662SDaniel Vetter #endif 41f51b7662SDaniel Vetter 421a997ff2SDaniel Vetter struct intel_gtt_driver { 431a997ff2SDaniel Vetter unsigned int gen : 8; 441a997ff2SDaniel Vetter unsigned int is_g33 : 1; 451a997ff2SDaniel Vetter unsigned int is_pineview : 1; 461a997ff2SDaniel Vetter unsigned int is_ironlake : 1; 47100519e2SChris Wilson unsigned int has_pgtbl_enable : 1; 4822533b49SDaniel Vetter unsigned int dma_mask_size : 8; 4973800422SDaniel Vetter /* Chipset specific GTT setup */ 5073800422SDaniel Vetter int (*setup)(void); 51ae83dd5cSDaniel Vetter /* This should undo anything done in ->setup() save the unmapping 52ae83dd5cSDaniel Vetter * of the mmio register file, that's done in the generic code. */ 53ae83dd5cSDaniel Vetter void (*cleanup)(void); 54351bb278SDaniel Vetter void (*write_entry)(dma_addr_t addr, unsigned int entry, unsigned int flags); 55351bb278SDaniel Vetter /* Flags is a more or less chipset specific opaque value. 56351bb278SDaniel Vetter * For chipsets that need to support old ums (non-gem) code, this 57351bb278SDaniel Vetter * needs to be identical to the various supported agp memory types! */ 585cbecafcSDaniel Vetter bool (*check_flags)(unsigned int flags); 591b263f24SDaniel Vetter void (*chipset_flush)(void); 601a997ff2SDaniel Vetter }; 611a997ff2SDaniel Vetter 62f51b7662SDaniel Vetter static struct _intel_private { 630ade6386SDaniel Vetter struct intel_gtt base; 641a997ff2SDaniel Vetter const struct intel_gtt_driver *driver; 65f51b7662SDaniel Vetter struct pci_dev *pcidev; /* device one */ 66d7cca2f7SDaniel Vetter struct pci_dev *bridge_dev; 67f51b7662SDaniel Vetter u8 __iomem *registers; 68f67eab66SDaniel Vetter phys_addr_t gtt_bus_addr; 69b3eafc5aSDaniel Vetter u32 PGETBL_save; 70f51b7662SDaniel Vetter u32 __iomem *gtt; /* I915G */ 71bee4a186SChris Wilson bool clear_fake_agp; /* on first access via agp, fill with scratch */ 72f51b7662SDaniel Vetter int num_dcache_entries; 73f51b7662SDaniel Vetter void __iomem *i9xx_flush_page; 74820647b9SDaniel Vetter char *i81x_gtt_table; 75f51b7662SDaniel Vetter struct resource ifp_resource; 76f51b7662SDaniel Vetter int resource_valid; 770e87d2b0SDaniel Vetter struct page *scratch_page; 789c61a32dSBen Widawsky phys_addr_t scratch_page_dma; 7914be93ddSDaniel Vetter int refcount; 808d2e6308SBen Widawsky /* Whether i915 needs to use the dmar apis or not. */ 818d2e6308SBen Widawsky unsigned int needs_dmar : 1; 82f51b7662SDaniel Vetter } intel_private; 83f51b7662SDaniel Vetter 841a997ff2SDaniel Vetter #define INTEL_GTT_GEN intel_private.driver->gen 851a997ff2SDaniel Vetter #define IS_G33 intel_private.driver->is_g33 861a997ff2SDaniel Vetter #define IS_PINEVIEW intel_private.driver->is_pineview 871a997ff2SDaniel Vetter #define IS_IRONLAKE intel_private.driver->is_ironlake 88100519e2SChris Wilson #define HAS_PGTBL_EN intel_private.driver->has_pgtbl_enable 891a997ff2SDaniel Vetter 909da3da66SChris Wilson static int intel_gtt_map_memory(struct page **pages, 919da3da66SChris Wilson unsigned int num_entries, 929da3da66SChris Wilson struct sg_table *st) 93f51b7662SDaniel Vetter { 94f51b7662SDaniel Vetter struct scatterlist *sg; 95f51b7662SDaniel Vetter int i; 96f51b7662SDaniel Vetter 974080775bSDaniel Vetter DBG("try mapping %lu pages\n", (unsigned long)num_entries); 98f51b7662SDaniel Vetter 999da3da66SChris Wilson if (sg_alloc_table(st, num_entries, GFP_KERNEL)) 100831cd445SChris Wilson goto err; 101f51b7662SDaniel Vetter 1029da3da66SChris Wilson for_each_sg(st->sgl, sg, num_entries, i) 1034080775bSDaniel Vetter sg_set_page(sg, pages[i], PAGE_SIZE, 0); 104f51b7662SDaniel Vetter 1059da3da66SChris Wilson if (!pci_map_sg(intel_private.pcidev, 1069da3da66SChris Wilson st->sgl, st->nents, PCI_DMA_BIDIRECTIONAL)) 107831cd445SChris Wilson goto err; 108831cd445SChris Wilson 109f51b7662SDaniel Vetter return 0; 110831cd445SChris Wilson 111831cd445SChris Wilson err: 1129da3da66SChris Wilson sg_free_table(st); 113831cd445SChris Wilson return -ENOMEM; 114f51b7662SDaniel Vetter } 115f51b7662SDaniel Vetter 1169da3da66SChris Wilson static void intel_gtt_unmap_memory(struct scatterlist *sg_list, int num_sg) 117f51b7662SDaniel Vetter { 1184080775bSDaniel Vetter struct sg_table st; 119f51b7662SDaniel Vetter DBG("try unmapping %lu pages\n", (unsigned long)mem->page_count); 120f51b7662SDaniel Vetter 1214080775bSDaniel Vetter pci_unmap_sg(intel_private.pcidev, sg_list, 1224080775bSDaniel Vetter num_sg, PCI_DMA_BIDIRECTIONAL); 1234080775bSDaniel Vetter 1244080775bSDaniel Vetter st.sgl = sg_list; 1254080775bSDaniel Vetter st.orig_nents = st.nents = num_sg; 1264080775bSDaniel Vetter 1274080775bSDaniel Vetter sg_free_table(&st); 128f51b7662SDaniel Vetter } 129f51b7662SDaniel Vetter 130ffdd7510SDaniel Vetter static void intel_fake_agp_enable(struct agp_bridge_data *bridge, u32 mode) 131f51b7662SDaniel Vetter { 132f51b7662SDaniel Vetter return; 133f51b7662SDaniel Vetter } 134f51b7662SDaniel Vetter 135f51b7662SDaniel Vetter /* Exists to support ARGB cursors */ 136f51b7662SDaniel Vetter static struct page *i8xx_alloc_pages(void) 137f51b7662SDaniel Vetter { 138f51b7662SDaniel Vetter struct page *page; 139f51b7662SDaniel Vetter 140f51b7662SDaniel Vetter page = alloc_pages(GFP_KERNEL | GFP_DMA32, 2); 141f51b7662SDaniel Vetter if (page == NULL) 142f51b7662SDaniel Vetter return NULL; 143f51b7662SDaniel Vetter 144f51b7662SDaniel Vetter if (set_pages_uc(page, 4) < 0) { 145f51b7662SDaniel Vetter set_pages_wb(page, 4); 146f51b7662SDaniel Vetter __free_pages(page, 2); 147f51b7662SDaniel Vetter return NULL; 148f51b7662SDaniel Vetter } 149f51b7662SDaniel Vetter get_page(page); 150f51b7662SDaniel Vetter atomic_inc(&agp_bridge->current_memory_agp); 151f51b7662SDaniel Vetter return page; 152f51b7662SDaniel Vetter } 153f51b7662SDaniel Vetter 154f51b7662SDaniel Vetter static void i8xx_destroy_pages(struct page *page) 155f51b7662SDaniel Vetter { 156f51b7662SDaniel Vetter if (page == NULL) 157f51b7662SDaniel Vetter return; 158f51b7662SDaniel Vetter 159f51b7662SDaniel Vetter set_pages_wb(page, 4); 160f51b7662SDaniel Vetter put_page(page); 161f51b7662SDaniel Vetter __free_pages(page, 2); 162f51b7662SDaniel Vetter atomic_dec(&agp_bridge->current_memory_agp); 163f51b7662SDaniel Vetter } 164f51b7662SDaniel Vetter 165820647b9SDaniel Vetter #define I810_GTT_ORDER 4 166820647b9SDaniel Vetter static int i810_setup(void) 167820647b9SDaniel Vetter { 168820647b9SDaniel Vetter u32 reg_addr; 169820647b9SDaniel Vetter char *gtt_table; 170820647b9SDaniel Vetter 171820647b9SDaniel Vetter /* i81x does not preallocate the gtt. It's always 64kb in size. */ 172820647b9SDaniel Vetter gtt_table = alloc_gatt_pages(I810_GTT_ORDER); 173820647b9SDaniel Vetter if (gtt_table == NULL) 174820647b9SDaniel Vetter return -ENOMEM; 175820647b9SDaniel Vetter intel_private.i81x_gtt_table = gtt_table; 176820647b9SDaniel Vetter 177820647b9SDaniel Vetter pci_read_config_dword(intel_private.pcidev, I810_MMADDR, ®_addr); 178820647b9SDaniel Vetter reg_addr &= 0xfff80000; 179820647b9SDaniel Vetter 180820647b9SDaniel Vetter intel_private.registers = ioremap(reg_addr, KB(64)); 181820647b9SDaniel Vetter if (!intel_private.registers) 182820647b9SDaniel Vetter return -ENOMEM; 183820647b9SDaniel Vetter 184820647b9SDaniel Vetter writel(virt_to_phys(gtt_table) | I810_PGETBL_ENABLED, 185820647b9SDaniel Vetter intel_private.registers+I810_PGETBL_CTL); 186820647b9SDaniel Vetter 187820647b9SDaniel Vetter intel_private.gtt_bus_addr = reg_addr + I810_PTE_BASE; 188820647b9SDaniel Vetter 189820647b9SDaniel Vetter if ((readl(intel_private.registers+I810_DRAM_CTL) 190820647b9SDaniel Vetter & I810_DRAM_ROW_0) == I810_DRAM_ROW_0_SDRAM) { 191820647b9SDaniel Vetter dev_info(&intel_private.pcidev->dev, 192820647b9SDaniel Vetter "detected 4MB dedicated video ram\n"); 193820647b9SDaniel Vetter intel_private.num_dcache_entries = 1024; 194820647b9SDaniel Vetter } 195820647b9SDaniel Vetter 196820647b9SDaniel Vetter return 0; 197820647b9SDaniel Vetter } 198820647b9SDaniel Vetter 199820647b9SDaniel Vetter static void i810_cleanup(void) 200820647b9SDaniel Vetter { 201820647b9SDaniel Vetter writel(0, intel_private.registers+I810_PGETBL_CTL); 202820647b9SDaniel Vetter free_gatt_pages(intel_private.i81x_gtt_table, I810_GTT_ORDER); 203820647b9SDaniel Vetter } 204820647b9SDaniel Vetter 205ff26860fSDaniel Vetter static int i810_insert_dcache_entries(struct agp_memory *mem, off_t pg_start, 206f51b7662SDaniel Vetter int type) 207f51b7662SDaniel Vetter { 208f51b7662SDaniel Vetter int i; 209f51b7662SDaniel Vetter 210625dd9d3SDaniel Vetter if ((pg_start + mem->page_count) 211625dd9d3SDaniel Vetter > intel_private.num_dcache_entries) 212625dd9d3SDaniel Vetter return -EINVAL; 213f51b7662SDaniel Vetter 214625dd9d3SDaniel Vetter if (!mem->is_flushed) 215625dd9d3SDaniel Vetter global_cache_flush(); 216625dd9d3SDaniel Vetter 217625dd9d3SDaniel Vetter for (i = pg_start; i < (pg_start + mem->page_count); i++) { 218625dd9d3SDaniel Vetter dma_addr_t addr = i << PAGE_SHIFT; 219625dd9d3SDaniel Vetter intel_private.driver->write_entry(addr, 220625dd9d3SDaniel Vetter i, type); 221f51b7662SDaniel Vetter } 222625dd9d3SDaniel Vetter readl(intel_private.gtt+i-1); 223f51b7662SDaniel Vetter 224f51b7662SDaniel Vetter return 0; 225f51b7662SDaniel Vetter } 226f51b7662SDaniel Vetter 227f51b7662SDaniel Vetter /* 228f51b7662SDaniel Vetter * The i810/i830 requires a physical address to program its mouse 229f51b7662SDaniel Vetter * pointer into hardware. 230f51b7662SDaniel Vetter * However the Xserver still writes to it through the agp aperture. 231f51b7662SDaniel Vetter */ 232f51b7662SDaniel Vetter static struct agp_memory *alloc_agpphysmem_i8xx(size_t pg_count, int type) 233f51b7662SDaniel Vetter { 234f51b7662SDaniel Vetter struct agp_memory *new; 235f51b7662SDaniel Vetter struct page *page; 236f51b7662SDaniel Vetter 237f51b7662SDaniel Vetter switch (pg_count) { 238f51b7662SDaniel Vetter case 1: page = agp_bridge->driver->agp_alloc_page(agp_bridge); 239f51b7662SDaniel Vetter break; 240f51b7662SDaniel Vetter case 4: 241f51b7662SDaniel Vetter /* kludge to get 4 physical pages for ARGB cursor */ 242f51b7662SDaniel Vetter page = i8xx_alloc_pages(); 243f51b7662SDaniel Vetter break; 244f51b7662SDaniel Vetter default: 245f51b7662SDaniel Vetter return NULL; 246f51b7662SDaniel Vetter } 247f51b7662SDaniel Vetter 248f51b7662SDaniel Vetter if (page == NULL) 249f51b7662SDaniel Vetter return NULL; 250f51b7662SDaniel Vetter 251f51b7662SDaniel Vetter new = agp_create_memory(pg_count); 252f51b7662SDaniel Vetter if (new == NULL) 253f51b7662SDaniel Vetter return NULL; 254f51b7662SDaniel Vetter 255f51b7662SDaniel Vetter new->pages[0] = page; 256f51b7662SDaniel Vetter if (pg_count == 4) { 257f51b7662SDaniel Vetter /* kludge to get 4 physical pages for ARGB cursor */ 258f51b7662SDaniel Vetter new->pages[1] = new->pages[0] + 1; 259f51b7662SDaniel Vetter new->pages[2] = new->pages[1] + 1; 260f51b7662SDaniel Vetter new->pages[3] = new->pages[2] + 1; 261f51b7662SDaniel Vetter } 262f51b7662SDaniel Vetter new->page_count = pg_count; 263f51b7662SDaniel Vetter new->num_scratch_pages = pg_count; 264f51b7662SDaniel Vetter new->type = AGP_PHYS_MEMORY; 265f51b7662SDaniel Vetter new->physical = page_to_phys(new->pages[0]); 266f51b7662SDaniel Vetter return new; 267f51b7662SDaniel Vetter } 268f51b7662SDaniel Vetter 269f51b7662SDaniel Vetter static void intel_i810_free_by_type(struct agp_memory *curr) 270f51b7662SDaniel Vetter { 271f51b7662SDaniel Vetter agp_free_key(curr->key); 272f51b7662SDaniel Vetter if (curr->type == AGP_PHYS_MEMORY) { 273f51b7662SDaniel Vetter if (curr->page_count == 4) 274f51b7662SDaniel Vetter i8xx_destroy_pages(curr->pages[0]); 275f51b7662SDaniel Vetter else { 276f51b7662SDaniel Vetter agp_bridge->driver->agp_destroy_page(curr->pages[0], 277f51b7662SDaniel Vetter AGP_PAGE_DESTROY_UNMAP); 278f51b7662SDaniel Vetter agp_bridge->driver->agp_destroy_page(curr->pages[0], 279f51b7662SDaniel Vetter AGP_PAGE_DESTROY_FREE); 280f51b7662SDaniel Vetter } 281f51b7662SDaniel Vetter agp_free_page_array(curr); 282f51b7662SDaniel Vetter } 283f51b7662SDaniel Vetter kfree(curr); 284f51b7662SDaniel Vetter } 285f51b7662SDaniel Vetter 2860e87d2b0SDaniel Vetter static int intel_gtt_setup_scratch_page(void) 2870e87d2b0SDaniel Vetter { 2880e87d2b0SDaniel Vetter struct page *page; 2890e87d2b0SDaniel Vetter dma_addr_t dma_addr; 2900e87d2b0SDaniel Vetter 2910e87d2b0SDaniel Vetter page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO); 2920e87d2b0SDaniel Vetter if (page == NULL) 2930e87d2b0SDaniel Vetter return -ENOMEM; 2940e87d2b0SDaniel Vetter get_page(page); 2950e87d2b0SDaniel Vetter set_pages_uc(page, 1); 2960e87d2b0SDaniel Vetter 2978d2e6308SBen Widawsky if (intel_private.needs_dmar) { 2980e87d2b0SDaniel Vetter dma_addr = pci_map_page(intel_private.pcidev, page, 0, 2990e87d2b0SDaniel Vetter PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 3000e87d2b0SDaniel Vetter if (pci_dma_mapping_error(intel_private.pcidev, dma_addr)) 3010e87d2b0SDaniel Vetter return -EINVAL; 3020e87d2b0SDaniel Vetter 3039c61a32dSBen Widawsky intel_private.scratch_page_dma = dma_addr; 3040e87d2b0SDaniel Vetter } else 3059c61a32dSBen Widawsky intel_private.scratch_page_dma = page_to_phys(page); 3060e87d2b0SDaniel Vetter 3070e87d2b0SDaniel Vetter intel_private.scratch_page = page; 3080e87d2b0SDaniel Vetter 3090e87d2b0SDaniel Vetter return 0; 3100e87d2b0SDaniel Vetter } 3110e87d2b0SDaniel Vetter 312625dd9d3SDaniel Vetter static void i810_write_entry(dma_addr_t addr, unsigned int entry, 313625dd9d3SDaniel Vetter unsigned int flags) 314625dd9d3SDaniel Vetter { 315625dd9d3SDaniel Vetter u32 pte_flags = I810_PTE_VALID; 316625dd9d3SDaniel Vetter 317625dd9d3SDaniel Vetter switch (flags) { 318625dd9d3SDaniel Vetter case AGP_DCACHE_MEMORY: 319625dd9d3SDaniel Vetter pte_flags |= I810_PTE_LOCAL; 320625dd9d3SDaniel Vetter break; 321625dd9d3SDaniel Vetter case AGP_USER_CACHED_MEMORY: 322625dd9d3SDaniel Vetter pte_flags |= I830_PTE_SYSTEM_CACHED; 323625dd9d3SDaniel Vetter break; 324625dd9d3SDaniel Vetter } 325625dd9d3SDaniel Vetter 326625dd9d3SDaniel Vetter writel(addr | pte_flags, intel_private.gtt + entry); 327625dd9d3SDaniel Vetter } 328625dd9d3SDaniel Vetter 3297bdc9ab0SChris Wilson static const struct aper_size_info_fixed intel_fake_agp_sizes[] = { 330820647b9SDaniel Vetter {32, 8192, 3}, 331820647b9SDaniel Vetter {64, 16384, 4}, 332f51b7662SDaniel Vetter {128, 32768, 5}, 333f51b7662SDaniel Vetter {256, 65536, 6}, 334f51b7662SDaniel Vetter {512, 131072, 7}, 335f51b7662SDaniel Vetter }; 336f51b7662SDaniel Vetter 337c64f7ba5SChris Wilson static unsigned int intel_gtt_stolen_size(void) 338f51b7662SDaniel Vetter { 339f51b7662SDaniel Vetter u16 gmch_ctrl; 340f51b7662SDaniel Vetter u8 rdct; 341f51b7662SDaniel Vetter int local = 0; 342f51b7662SDaniel Vetter static const int ddt[4] = { 0, 16, 32, 64 }; 343d8d9abcdSDaniel Vetter unsigned int stolen_size = 0; 344f51b7662SDaniel Vetter 345820647b9SDaniel Vetter if (INTEL_GTT_GEN == 1) 346820647b9SDaniel Vetter return 0; /* no stolen mem on i81x */ 347820647b9SDaniel Vetter 348d7cca2f7SDaniel Vetter pci_read_config_word(intel_private.bridge_dev, 349d7cca2f7SDaniel Vetter I830_GMCH_CTRL, &gmch_ctrl); 350f51b7662SDaniel Vetter 351d7cca2f7SDaniel Vetter if (intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82830_HB || 352d7cca2f7SDaniel Vetter intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82845G_HB) { 353f51b7662SDaniel Vetter switch (gmch_ctrl & I830_GMCH_GMS_MASK) { 354f51b7662SDaniel Vetter case I830_GMCH_GMS_STOLEN_512: 355d8d9abcdSDaniel Vetter stolen_size = KB(512); 356f51b7662SDaniel Vetter break; 357f51b7662SDaniel Vetter case I830_GMCH_GMS_STOLEN_1024: 358d8d9abcdSDaniel Vetter stolen_size = MB(1); 359f51b7662SDaniel Vetter break; 360f51b7662SDaniel Vetter case I830_GMCH_GMS_STOLEN_8192: 361d8d9abcdSDaniel Vetter stolen_size = MB(8); 362f51b7662SDaniel Vetter break; 363f51b7662SDaniel Vetter case I830_GMCH_GMS_LOCAL: 364f51b7662SDaniel Vetter rdct = readb(intel_private.registers+I830_RDRAM_CHANNEL_TYPE); 365d8d9abcdSDaniel Vetter stolen_size = (I830_RDRAM_ND(rdct) + 1) * 366f51b7662SDaniel Vetter MB(ddt[I830_RDRAM_DDT(rdct)]); 367f51b7662SDaniel Vetter local = 1; 368f51b7662SDaniel Vetter break; 369f51b7662SDaniel Vetter default: 370d8d9abcdSDaniel Vetter stolen_size = 0; 371f51b7662SDaniel Vetter break; 372f51b7662SDaniel Vetter } 373f51b7662SDaniel Vetter } else { 374f51b7662SDaniel Vetter switch (gmch_ctrl & I855_GMCH_GMS_MASK) { 375f51b7662SDaniel Vetter case I855_GMCH_GMS_STOLEN_1M: 376d8d9abcdSDaniel Vetter stolen_size = MB(1); 377f51b7662SDaniel Vetter break; 378f51b7662SDaniel Vetter case I855_GMCH_GMS_STOLEN_4M: 379d8d9abcdSDaniel Vetter stolen_size = MB(4); 380f51b7662SDaniel Vetter break; 381f51b7662SDaniel Vetter case I855_GMCH_GMS_STOLEN_8M: 382d8d9abcdSDaniel Vetter stolen_size = MB(8); 383f51b7662SDaniel Vetter break; 384f51b7662SDaniel Vetter case I855_GMCH_GMS_STOLEN_16M: 385d8d9abcdSDaniel Vetter stolen_size = MB(16); 386f51b7662SDaniel Vetter break; 387f51b7662SDaniel Vetter case I855_GMCH_GMS_STOLEN_32M: 388d8d9abcdSDaniel Vetter stolen_size = MB(32); 389f51b7662SDaniel Vetter break; 390f51b7662SDaniel Vetter case I915_GMCH_GMS_STOLEN_48M: 391d8d9abcdSDaniel Vetter stolen_size = MB(48); 392f51b7662SDaniel Vetter break; 393f51b7662SDaniel Vetter case I915_GMCH_GMS_STOLEN_64M: 394d8d9abcdSDaniel Vetter stolen_size = MB(64); 395f51b7662SDaniel Vetter break; 396f51b7662SDaniel Vetter case G33_GMCH_GMS_STOLEN_128M: 397d8d9abcdSDaniel Vetter stolen_size = MB(128); 398f51b7662SDaniel Vetter break; 399f51b7662SDaniel Vetter case G33_GMCH_GMS_STOLEN_256M: 400d8d9abcdSDaniel Vetter stolen_size = MB(256); 401f51b7662SDaniel Vetter break; 402f51b7662SDaniel Vetter case INTEL_GMCH_GMS_STOLEN_96M: 403d8d9abcdSDaniel Vetter stolen_size = MB(96); 404f51b7662SDaniel Vetter break; 405f51b7662SDaniel Vetter case INTEL_GMCH_GMS_STOLEN_160M: 406d8d9abcdSDaniel Vetter stolen_size = MB(160); 407f51b7662SDaniel Vetter break; 408f51b7662SDaniel Vetter case INTEL_GMCH_GMS_STOLEN_224M: 409d8d9abcdSDaniel Vetter stolen_size = MB(224); 410f51b7662SDaniel Vetter break; 411f51b7662SDaniel Vetter case INTEL_GMCH_GMS_STOLEN_352M: 412d8d9abcdSDaniel Vetter stolen_size = MB(352); 413f51b7662SDaniel Vetter break; 414f51b7662SDaniel Vetter default: 415d8d9abcdSDaniel Vetter stolen_size = 0; 416f51b7662SDaniel Vetter break; 417f51b7662SDaniel Vetter } 418f51b7662SDaniel Vetter } 4191784a5fbSDaniel Vetter 4201b6064d7SChris Wilson if (stolen_size > 0) { 421d7cca2f7SDaniel Vetter dev_info(&intel_private.bridge_dev->dev, "detected %dK %s memory\n", 422d8d9abcdSDaniel Vetter stolen_size / KB(1), local ? "local" : "stolen"); 423f51b7662SDaniel Vetter } else { 424d7cca2f7SDaniel Vetter dev_info(&intel_private.bridge_dev->dev, 425f51b7662SDaniel Vetter "no pre-allocated video memory detected\n"); 426d8d9abcdSDaniel Vetter stolen_size = 0; 427f51b7662SDaniel Vetter } 428f51b7662SDaniel Vetter 429c64f7ba5SChris Wilson return stolen_size; 430f51b7662SDaniel Vetter } 431f51b7662SDaniel Vetter 43220172842SDaniel Vetter static void i965_adjust_pgetbl_size(unsigned int size_flag) 43320172842SDaniel Vetter { 43420172842SDaniel Vetter u32 pgetbl_ctl, pgetbl_ctl2; 43520172842SDaniel Vetter 43620172842SDaniel Vetter /* ensure that ppgtt is disabled */ 43720172842SDaniel Vetter pgetbl_ctl2 = readl(intel_private.registers+I965_PGETBL_CTL2); 43820172842SDaniel Vetter pgetbl_ctl2 &= ~I810_PGETBL_ENABLED; 43920172842SDaniel Vetter writel(pgetbl_ctl2, intel_private.registers+I965_PGETBL_CTL2); 44020172842SDaniel Vetter 44120172842SDaniel Vetter /* write the new ggtt size */ 44220172842SDaniel Vetter pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL); 44320172842SDaniel Vetter pgetbl_ctl &= ~I965_PGETBL_SIZE_MASK; 44420172842SDaniel Vetter pgetbl_ctl |= size_flag; 44520172842SDaniel Vetter writel(pgetbl_ctl, intel_private.registers+I810_PGETBL_CTL); 44620172842SDaniel Vetter } 44720172842SDaniel Vetter 44820172842SDaniel Vetter static unsigned int i965_gtt_total_entries(void) 449fbe40783SDaniel Vetter { 450fbe40783SDaniel Vetter int size; 451fbe40783SDaniel Vetter u32 pgetbl_ctl; 45220172842SDaniel Vetter u16 gmch_ctl; 45320172842SDaniel Vetter 45420172842SDaniel Vetter pci_read_config_word(intel_private.bridge_dev, 45520172842SDaniel Vetter I830_GMCH_CTRL, &gmch_ctl); 45620172842SDaniel Vetter 45720172842SDaniel Vetter if (INTEL_GTT_GEN == 5) { 45820172842SDaniel Vetter switch (gmch_ctl & G4x_GMCH_SIZE_MASK) { 45920172842SDaniel Vetter case G4x_GMCH_SIZE_1M: 46020172842SDaniel Vetter case G4x_GMCH_SIZE_VT_1M: 46120172842SDaniel Vetter i965_adjust_pgetbl_size(I965_PGETBL_SIZE_1MB); 46220172842SDaniel Vetter break; 46320172842SDaniel Vetter case G4x_GMCH_SIZE_VT_1_5M: 46420172842SDaniel Vetter i965_adjust_pgetbl_size(I965_PGETBL_SIZE_1_5MB); 46520172842SDaniel Vetter break; 46620172842SDaniel Vetter case G4x_GMCH_SIZE_2M: 46720172842SDaniel Vetter case G4x_GMCH_SIZE_VT_2M: 46820172842SDaniel Vetter i965_adjust_pgetbl_size(I965_PGETBL_SIZE_2MB); 46920172842SDaniel Vetter break; 47020172842SDaniel Vetter } 47120172842SDaniel Vetter } 47220172842SDaniel Vetter 473fbe40783SDaniel Vetter pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL); 474fbe40783SDaniel Vetter 475fbe40783SDaniel Vetter switch (pgetbl_ctl & I965_PGETBL_SIZE_MASK) { 476fbe40783SDaniel Vetter case I965_PGETBL_SIZE_128KB: 477e5e408fcSDaniel Vetter size = KB(128); 478fbe40783SDaniel Vetter break; 479fbe40783SDaniel Vetter case I965_PGETBL_SIZE_256KB: 480e5e408fcSDaniel Vetter size = KB(256); 481fbe40783SDaniel Vetter break; 482fbe40783SDaniel Vetter case I965_PGETBL_SIZE_512KB: 483e5e408fcSDaniel Vetter size = KB(512); 484fbe40783SDaniel Vetter break; 48520172842SDaniel Vetter /* GTT pagetable sizes bigger than 512KB are not possible on G33! */ 486fbe40783SDaniel Vetter case I965_PGETBL_SIZE_1MB: 487e5e408fcSDaniel Vetter size = KB(1024); 488fbe40783SDaniel Vetter break; 489fbe40783SDaniel Vetter case I965_PGETBL_SIZE_2MB: 490e5e408fcSDaniel Vetter size = KB(2048); 491fbe40783SDaniel Vetter break; 492fbe40783SDaniel Vetter case I965_PGETBL_SIZE_1_5MB: 493e5e408fcSDaniel Vetter size = KB(1024 + 512); 494fbe40783SDaniel Vetter break; 495fbe40783SDaniel Vetter default: 496fbe40783SDaniel Vetter dev_info(&intel_private.pcidev->dev, 497fbe40783SDaniel Vetter "unknown page table size, assuming 512KB\n"); 498e5e408fcSDaniel Vetter size = KB(512); 499fbe40783SDaniel Vetter } 500e5e408fcSDaniel Vetter 501e5e408fcSDaniel Vetter return size/4; 50220172842SDaniel Vetter } 50320172842SDaniel Vetter 50420172842SDaniel Vetter static unsigned int intel_gtt_total_entries(void) 50520172842SDaniel Vetter { 50620172842SDaniel Vetter if (IS_G33 || INTEL_GTT_GEN == 4 || INTEL_GTT_GEN == 5) 50720172842SDaniel Vetter return i965_gtt_total_entries(); 508009946f8SBen Widawsky else { 509fbe40783SDaniel Vetter /* On previous hardware, the GTT size was just what was 510fbe40783SDaniel Vetter * required to map the aperture. 511fbe40783SDaniel Vetter */ 512e5e408fcSDaniel Vetter return intel_private.base.gtt_mappable_entries; 513fbe40783SDaniel Vetter } 514fbe40783SDaniel Vetter } 515fbe40783SDaniel Vetter 5161784a5fbSDaniel Vetter static unsigned int intel_gtt_mappable_entries(void) 5171784a5fbSDaniel Vetter { 5181784a5fbSDaniel Vetter unsigned int aperture_size; 5191784a5fbSDaniel Vetter 520820647b9SDaniel Vetter if (INTEL_GTT_GEN == 1) { 521820647b9SDaniel Vetter u32 smram_miscc; 522820647b9SDaniel Vetter 523820647b9SDaniel Vetter pci_read_config_dword(intel_private.bridge_dev, 524820647b9SDaniel Vetter I810_SMRAM_MISCC, &smram_miscc); 525820647b9SDaniel Vetter 526820647b9SDaniel Vetter if ((smram_miscc & I810_GFX_MEM_WIN_SIZE) 527820647b9SDaniel Vetter == I810_GFX_MEM_WIN_32M) 528820647b9SDaniel Vetter aperture_size = MB(32); 529820647b9SDaniel Vetter else 530820647b9SDaniel Vetter aperture_size = MB(64); 531820647b9SDaniel Vetter } else if (INTEL_GTT_GEN == 2) { 532b1c5b0f8SChris Wilson u16 gmch_ctrl; 5331784a5fbSDaniel Vetter 5341784a5fbSDaniel Vetter pci_read_config_word(intel_private.bridge_dev, 5351784a5fbSDaniel Vetter I830_GMCH_CTRL, &gmch_ctrl); 5361784a5fbSDaniel Vetter 5371784a5fbSDaniel Vetter if ((gmch_ctrl & I830_GMCH_MEM_MASK) == I830_GMCH_MEM_64M) 538b1c5b0f8SChris Wilson aperture_size = MB(64); 5391784a5fbSDaniel Vetter else 540b1c5b0f8SChris Wilson aperture_size = MB(128); 541239918f7SDaniel Vetter } else { 5421784a5fbSDaniel Vetter /* 9xx supports large sizes, just look at the length */ 5431784a5fbSDaniel Vetter aperture_size = pci_resource_len(intel_private.pcidev, 2); 5441784a5fbSDaniel Vetter } 5451784a5fbSDaniel Vetter 5461784a5fbSDaniel Vetter return aperture_size >> PAGE_SHIFT; 5471784a5fbSDaniel Vetter } 5481784a5fbSDaniel Vetter 5490e87d2b0SDaniel Vetter static void intel_gtt_teardown_scratch_page(void) 5500e87d2b0SDaniel Vetter { 5510e87d2b0SDaniel Vetter set_pages_wb(intel_private.scratch_page, 1); 5529c61a32dSBen Widawsky pci_unmap_page(intel_private.pcidev, intel_private.scratch_page_dma, 5530e87d2b0SDaniel Vetter PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 5540e87d2b0SDaniel Vetter put_page(intel_private.scratch_page); 5550e87d2b0SDaniel Vetter __free_page(intel_private.scratch_page); 5560e87d2b0SDaniel Vetter } 5570e87d2b0SDaniel Vetter 5580e87d2b0SDaniel Vetter static void intel_gtt_cleanup(void) 5590e87d2b0SDaniel Vetter { 560ae83dd5cSDaniel Vetter intel_private.driver->cleanup(); 561ae83dd5cSDaniel Vetter 5620e87d2b0SDaniel Vetter iounmap(intel_private.gtt); 5630e87d2b0SDaniel Vetter iounmap(intel_private.registers); 5640e87d2b0SDaniel Vetter 5650e87d2b0SDaniel Vetter intel_gtt_teardown_scratch_page(); 5660e87d2b0SDaniel Vetter } 5670e87d2b0SDaniel Vetter 5681784a5fbSDaniel Vetter static int intel_gtt_init(void) 5691784a5fbSDaniel Vetter { 57032e3cd6eSDaniel Vetter u32 gma_addr; 571f67eab66SDaniel Vetter u32 gtt_map_size; 5723b15a9d7SDaniel Vetter int ret; 5733b15a9d7SDaniel Vetter 5743b15a9d7SDaniel Vetter ret = intel_private.driver->setup(); 5753b15a9d7SDaniel Vetter if (ret != 0) 5763b15a9d7SDaniel Vetter return ret; 577f67eab66SDaniel Vetter 578f67eab66SDaniel Vetter intel_private.base.gtt_mappable_entries = intel_gtt_mappable_entries(); 579f67eab66SDaniel Vetter intel_private.base.gtt_total_entries = intel_gtt_total_entries(); 580f67eab66SDaniel Vetter 581b3eafc5aSDaniel Vetter /* save the PGETBL reg for resume */ 582b3eafc5aSDaniel Vetter intel_private.PGETBL_save = 583b3eafc5aSDaniel Vetter readl(intel_private.registers+I810_PGETBL_CTL) 584b3eafc5aSDaniel Vetter & ~I810_PGETBL_ENABLED; 585100519e2SChris Wilson /* we only ever restore the register when enabling the PGTBL... */ 586100519e2SChris Wilson if (HAS_PGTBL_EN) 587100519e2SChris Wilson intel_private.PGETBL_save |= I810_PGETBL_ENABLED; 588b3eafc5aSDaniel Vetter 5890af9e92eSDaniel Vetter dev_info(&intel_private.bridge_dev->dev, 5900af9e92eSDaniel Vetter "detected gtt size: %dK total, %dK mappable\n", 5910af9e92eSDaniel Vetter intel_private.base.gtt_total_entries * 4, 5920af9e92eSDaniel Vetter intel_private.base.gtt_mappable_entries * 4); 5930af9e92eSDaniel Vetter 594f67eab66SDaniel Vetter gtt_map_size = intel_private.base.gtt_total_entries * 4; 595f67eab66SDaniel Vetter 596edef7e68SChris Wilson intel_private.gtt = NULL; 5979169d3a8SDaniel Vetter if (INTEL_GTT_GEN < 6 && INTEL_GTT_GEN > 2) 598edef7e68SChris Wilson intel_private.gtt = ioremap_wc(intel_private.gtt_bus_addr, 599edef7e68SChris Wilson gtt_map_size); 600edef7e68SChris Wilson if (intel_private.gtt == NULL) 601f67eab66SDaniel Vetter intel_private.gtt = ioremap(intel_private.gtt_bus_addr, 602f67eab66SDaniel Vetter gtt_map_size); 603edef7e68SChris Wilson if (intel_private.gtt == NULL) { 604ae83dd5cSDaniel Vetter intel_private.driver->cleanup(); 605f67eab66SDaniel Vetter iounmap(intel_private.registers); 606f67eab66SDaniel Vetter return -ENOMEM; 607f67eab66SDaniel Vetter } 608f67eab66SDaniel Vetter 609f67eab66SDaniel Vetter global_cache_flush(); /* FIXME: ? */ 610f67eab66SDaniel Vetter 611c64f7ba5SChris Wilson intel_private.base.stolen_size = intel_gtt_stolen_size(); 6121784a5fbSDaniel Vetter 6138d2e6308SBen Widawsky intel_private.needs_dmar = USE_PCI_DMA_API && INTEL_GTT_GEN > 2; 614a46f3108SDave Airlie 6150e87d2b0SDaniel Vetter ret = intel_gtt_setup_scratch_page(); 6160e87d2b0SDaniel Vetter if (ret != 0) { 6170e87d2b0SDaniel Vetter intel_gtt_cleanup(); 6180e87d2b0SDaniel Vetter return ret; 6190e87d2b0SDaniel Vetter } 6200e87d2b0SDaniel Vetter 62132e3cd6eSDaniel Vetter if (INTEL_GTT_GEN <= 2) 62232e3cd6eSDaniel Vetter pci_read_config_dword(intel_private.pcidev, I810_GMADDR, 62332e3cd6eSDaniel Vetter &gma_addr); 62432e3cd6eSDaniel Vetter else 62532e3cd6eSDaniel Vetter pci_read_config_dword(intel_private.pcidev, I915_GMADDR, 62632e3cd6eSDaniel Vetter &gma_addr); 62732e3cd6eSDaniel Vetter 62832e3cd6eSDaniel Vetter intel_private.base.gma_bus_addr = (gma_addr & PCI_BASE_ADDRESS_MEM_MASK); 62932e3cd6eSDaniel Vetter 6301784a5fbSDaniel Vetter return 0; 6311784a5fbSDaniel Vetter } 6321784a5fbSDaniel Vetter 6333e921f98SDaniel Vetter static int intel_fake_agp_fetch_size(void) 6343e921f98SDaniel Vetter { 6359e76e7b8SChris Wilson int num_sizes = ARRAY_SIZE(intel_fake_agp_sizes); 6363e921f98SDaniel Vetter unsigned int aper_size; 6373e921f98SDaniel Vetter int i; 6383e921f98SDaniel Vetter 6393e921f98SDaniel Vetter aper_size = (intel_private.base.gtt_mappable_entries << PAGE_SHIFT) 6403e921f98SDaniel Vetter / MB(1); 6413e921f98SDaniel Vetter 6423e921f98SDaniel Vetter for (i = 0; i < num_sizes; i++) { 643ffdd7510SDaniel Vetter if (aper_size == intel_fake_agp_sizes[i].size) { 6449e76e7b8SChris Wilson agp_bridge->current_size = 6459e76e7b8SChris Wilson (void *) (intel_fake_agp_sizes + i); 6463e921f98SDaniel Vetter return aper_size; 6473e921f98SDaniel Vetter } 6483e921f98SDaniel Vetter } 6493e921f98SDaniel Vetter 6503e921f98SDaniel Vetter return 0; 6513e921f98SDaniel Vetter } 6523e921f98SDaniel Vetter 653ae83dd5cSDaniel Vetter static void i830_cleanup(void) 654f51b7662SDaniel Vetter { 655f51b7662SDaniel Vetter } 656f51b7662SDaniel Vetter 657f51b7662SDaniel Vetter /* The chipset_flush interface needs to get data that has already been 658f51b7662SDaniel Vetter * flushed out of the CPU all the way out to main memory, because the GPU 659f51b7662SDaniel Vetter * doesn't snoop those buffers. 660f51b7662SDaniel Vetter * 661f51b7662SDaniel Vetter * The 8xx series doesn't have the same lovely interface for flushing the 662f51b7662SDaniel Vetter * chipset write buffers that the later chips do. According to the 865 663f51b7662SDaniel Vetter * specs, it's 64 octwords, or 1KB. So, to get those previous things in 664f51b7662SDaniel Vetter * that buffer out, we just fill 1KB and clflush it out, on the assumption 665f51b7662SDaniel Vetter * that it'll push whatever was in there out. It appears to work. 666f51b7662SDaniel Vetter */ 6671b263f24SDaniel Vetter static void i830_chipset_flush(void) 668f51b7662SDaniel Vetter { 669bdb8b975SChris Wilson unsigned long timeout = jiffies + msecs_to_jiffies(1000); 670f51b7662SDaniel Vetter 671bdb8b975SChris Wilson /* Forcibly evict everything from the CPU write buffers. 672bdb8b975SChris Wilson * clflush appears to be insufficient. 673bdb8b975SChris Wilson */ 674bdb8b975SChris Wilson wbinvd_on_all_cpus(); 675f51b7662SDaniel Vetter 676bdb8b975SChris Wilson /* Now we've only seen documents for this magic bit on 855GM, 677bdb8b975SChris Wilson * we hope it exists for the other gen2 chipsets... 678bdb8b975SChris Wilson * 679bdb8b975SChris Wilson * Also works as advertised on my 845G. 680bdb8b975SChris Wilson */ 681bdb8b975SChris Wilson writel(readl(intel_private.registers+I830_HIC) | (1<<31), 682bdb8b975SChris Wilson intel_private.registers+I830_HIC); 683bdb8b975SChris Wilson 684bdb8b975SChris Wilson while (readl(intel_private.registers+I830_HIC) & (1<<31)) { 685bdb8b975SChris Wilson if (time_after(jiffies, timeout)) 686bdb8b975SChris Wilson break; 687bdb8b975SChris Wilson 688bdb8b975SChris Wilson udelay(50); 689bdb8b975SChris Wilson } 690f51b7662SDaniel Vetter } 691f51b7662SDaniel Vetter 692351bb278SDaniel Vetter static void i830_write_entry(dma_addr_t addr, unsigned int entry, 693351bb278SDaniel Vetter unsigned int flags) 694351bb278SDaniel Vetter { 695351bb278SDaniel Vetter u32 pte_flags = I810_PTE_VALID; 696351bb278SDaniel Vetter 697b47cf66fSDaniel Vetter if (flags == AGP_USER_CACHED_MEMORY) 698351bb278SDaniel Vetter pte_flags |= I830_PTE_SYSTEM_CACHED; 699351bb278SDaniel Vetter 700351bb278SDaniel Vetter writel(addr | pte_flags, intel_private.gtt + entry); 701351bb278SDaniel Vetter } 702351bb278SDaniel Vetter 7038ecd1a66SDaniel Vetter bool intel_enable_gtt(void) 70473800422SDaniel Vetter { 705e380f60bSChris Wilson u8 __iomem *reg; 70673800422SDaniel Vetter 707100519e2SChris Wilson if (INTEL_GTT_GEN == 2) { 708100519e2SChris Wilson u16 gmch_ctrl; 709100519e2SChris Wilson 710e380f60bSChris Wilson pci_read_config_word(intel_private.bridge_dev, 711e380f60bSChris Wilson I830_GMCH_CTRL, &gmch_ctrl); 712e380f60bSChris Wilson gmch_ctrl |= I830_GMCH_ENABLED; 713e380f60bSChris Wilson pci_write_config_word(intel_private.bridge_dev, 714e380f60bSChris Wilson I830_GMCH_CTRL, gmch_ctrl); 715e380f60bSChris Wilson 716e380f60bSChris Wilson pci_read_config_word(intel_private.bridge_dev, 717e380f60bSChris Wilson I830_GMCH_CTRL, &gmch_ctrl); 718e380f60bSChris Wilson if ((gmch_ctrl & I830_GMCH_ENABLED) == 0) { 719e380f60bSChris Wilson dev_err(&intel_private.pcidev->dev, 720e380f60bSChris Wilson "failed to enable the GTT: GMCH_CTRL=%x\n", 721e380f60bSChris Wilson gmch_ctrl); 722e380f60bSChris Wilson return false; 723e380f60bSChris Wilson } 724100519e2SChris Wilson } 725e380f60bSChris Wilson 726c97689d8SChris Wilson /* On the resume path we may be adjusting the PGTBL value, so 727c97689d8SChris Wilson * be paranoid and flush all chipset write buffers... 728c97689d8SChris Wilson */ 729c97689d8SChris Wilson if (INTEL_GTT_GEN >= 3) 730c97689d8SChris Wilson writel(0, intel_private.registers+GFX_FLSH_CNTL); 731c97689d8SChris Wilson 732e380f60bSChris Wilson reg = intel_private.registers+I810_PGETBL_CTL; 733100519e2SChris Wilson writel(intel_private.PGETBL_save, reg); 734100519e2SChris Wilson if (HAS_PGTBL_EN && (readl(reg) & I810_PGETBL_ENABLED) == 0) { 735e380f60bSChris Wilson dev_err(&intel_private.pcidev->dev, 736100519e2SChris Wilson "failed to enable the GTT: PGETBL=%x [expected %x]\n", 737e380f60bSChris Wilson readl(reg), intel_private.PGETBL_save); 738e380f60bSChris Wilson return false; 739e380f60bSChris Wilson } 740e380f60bSChris Wilson 741c97689d8SChris Wilson if (INTEL_GTT_GEN >= 3) 742c97689d8SChris Wilson writel(0, intel_private.registers+GFX_FLSH_CNTL); 743c97689d8SChris Wilson 744e380f60bSChris Wilson return true; 74573800422SDaniel Vetter } 7468ecd1a66SDaniel Vetter EXPORT_SYMBOL(intel_enable_gtt); 74773800422SDaniel Vetter 74873800422SDaniel Vetter static int i830_setup(void) 74973800422SDaniel Vetter { 75073800422SDaniel Vetter u32 reg_addr; 75173800422SDaniel Vetter 75273800422SDaniel Vetter pci_read_config_dword(intel_private.pcidev, I810_MMADDR, ®_addr); 75373800422SDaniel Vetter reg_addr &= 0xfff80000; 75473800422SDaniel Vetter 75573800422SDaniel Vetter intel_private.registers = ioremap(reg_addr, KB(64)); 75673800422SDaniel Vetter if (!intel_private.registers) 75773800422SDaniel Vetter return -ENOMEM; 75873800422SDaniel Vetter 75973800422SDaniel Vetter intel_private.gtt_bus_addr = reg_addr + I810_PTE_BASE; 76073800422SDaniel Vetter 76173800422SDaniel Vetter return 0; 76273800422SDaniel Vetter } 76373800422SDaniel Vetter 7643b15a9d7SDaniel Vetter static int intel_fake_agp_create_gatt_table(struct agp_bridge_data *bridge) 765f51b7662SDaniel Vetter { 76673800422SDaniel Vetter agp_bridge->gatt_table_real = NULL; 767f51b7662SDaniel Vetter agp_bridge->gatt_table = NULL; 76873800422SDaniel Vetter agp_bridge->gatt_bus_addr = 0; 769f51b7662SDaniel Vetter 770f51b7662SDaniel Vetter return 0; 771f51b7662SDaniel Vetter } 772f51b7662SDaniel Vetter 773ffdd7510SDaniel Vetter static int intel_fake_agp_free_gatt_table(struct agp_bridge_data *bridge) 774f51b7662SDaniel Vetter { 775f51b7662SDaniel Vetter return 0; 776f51b7662SDaniel Vetter } 777f51b7662SDaniel Vetter 778351bb278SDaniel Vetter static int intel_fake_agp_configure(void) 779f51b7662SDaniel Vetter { 780e380f60bSChris Wilson if (!intel_enable_gtt()) 781e380f60bSChris Wilson return -EIO; 782f51b7662SDaniel Vetter 783bee4a186SChris Wilson intel_private.clear_fake_agp = true; 784dd2757f8SDaniel Vetter agp_bridge->gart_bus_addr = intel_private.base.gma_bus_addr; 785f51b7662SDaniel Vetter 786f51b7662SDaniel Vetter return 0; 787f51b7662SDaniel Vetter } 788f51b7662SDaniel Vetter 7895cbecafcSDaniel Vetter static bool i830_check_flags(unsigned int flags) 790f51b7662SDaniel Vetter { 7915cbecafcSDaniel Vetter switch (flags) { 7925cbecafcSDaniel Vetter case 0: 7935cbecafcSDaniel Vetter case AGP_PHYS_MEMORY: 7945cbecafcSDaniel Vetter case AGP_USER_CACHED_MEMORY: 7955cbecafcSDaniel Vetter case AGP_USER_MEMORY: 7965cbecafcSDaniel Vetter return true; 7975cbecafcSDaniel Vetter } 7985cbecafcSDaniel Vetter 7995cbecafcSDaniel Vetter return false; 8005cbecafcSDaniel Vetter } 8015cbecafcSDaniel Vetter 8029da3da66SChris Wilson void intel_gtt_insert_sg_entries(struct sg_table *st, 803fefaa70fSDaniel Vetter unsigned int pg_start, 804fefaa70fSDaniel Vetter unsigned int flags) 805fefaa70fSDaniel Vetter { 806fefaa70fSDaniel Vetter struct scatterlist *sg; 807fefaa70fSDaniel Vetter unsigned int len, m; 808fefaa70fSDaniel Vetter int i, j; 809fefaa70fSDaniel Vetter 810fefaa70fSDaniel Vetter j = pg_start; 811fefaa70fSDaniel Vetter 812fefaa70fSDaniel Vetter /* sg may merge pages, but we have to separate 813fefaa70fSDaniel Vetter * per-page addr for GTT */ 8149da3da66SChris Wilson for_each_sg(st->sgl, sg, st->nents, i) { 815fefaa70fSDaniel Vetter len = sg_dma_len(sg) >> PAGE_SHIFT; 816fefaa70fSDaniel Vetter for (m = 0; m < len; m++) { 817fefaa70fSDaniel Vetter dma_addr_t addr = sg_dma_address(sg) + (m << PAGE_SHIFT); 8189da3da66SChris Wilson intel_private.driver->write_entry(addr, j, flags); 819fefaa70fSDaniel Vetter j++; 820fefaa70fSDaniel Vetter } 821fefaa70fSDaniel Vetter } 822fefaa70fSDaniel Vetter readl(intel_private.gtt+j-1); 823fefaa70fSDaniel Vetter } 8244080775bSDaniel Vetter EXPORT_SYMBOL(intel_gtt_insert_sg_entries); 8254080775bSDaniel Vetter 8269da3da66SChris Wilson static void intel_gtt_insert_pages(unsigned int first_entry, 8279da3da66SChris Wilson unsigned int num_entries, 8289da3da66SChris Wilson struct page **pages, 8299da3da66SChris Wilson unsigned int flags) 8304080775bSDaniel Vetter { 8314080775bSDaniel Vetter int i, j; 8324080775bSDaniel Vetter 8334080775bSDaniel Vetter for (i = 0, j = first_entry; i < num_entries; i++, j++) { 8344080775bSDaniel Vetter dma_addr_t addr = page_to_phys(pages[i]); 8354080775bSDaniel Vetter intel_private.driver->write_entry(addr, 8364080775bSDaniel Vetter j, flags); 8374080775bSDaniel Vetter } 8384080775bSDaniel Vetter readl(intel_private.gtt+j-1); 8394080775bSDaniel Vetter } 840fefaa70fSDaniel Vetter 8415cbecafcSDaniel Vetter static int intel_fake_agp_insert_entries(struct agp_memory *mem, 8425cbecafcSDaniel Vetter off_t pg_start, int type) 8435cbecafcSDaniel Vetter { 844f51b7662SDaniel Vetter int ret = -EINVAL; 845f51b7662SDaniel Vetter 846bee4a186SChris Wilson if (intel_private.clear_fake_agp) { 847bee4a186SChris Wilson int start = intel_private.base.stolen_size / PAGE_SIZE; 848bee4a186SChris Wilson int end = intel_private.base.gtt_mappable_entries; 849bee4a186SChris Wilson intel_gtt_clear_range(start, end - start); 850bee4a186SChris Wilson intel_private.clear_fake_agp = false; 851bee4a186SChris Wilson } 852bee4a186SChris Wilson 853ff26860fSDaniel Vetter if (INTEL_GTT_GEN == 1 && type == AGP_DCACHE_MEMORY) 854ff26860fSDaniel Vetter return i810_insert_dcache_entries(mem, pg_start, type); 855ff26860fSDaniel Vetter 856f51b7662SDaniel Vetter if (mem->page_count == 0) 857f51b7662SDaniel Vetter goto out; 858f51b7662SDaniel Vetter 859c64f7ba5SChris Wilson if (pg_start + mem->page_count > intel_private.base.gtt_total_entries) 860f51b7662SDaniel Vetter goto out_err; 861f51b7662SDaniel Vetter 862f51b7662SDaniel Vetter if (type != mem->type) 863f51b7662SDaniel Vetter goto out_err; 864f51b7662SDaniel Vetter 8655cbecafcSDaniel Vetter if (!intel_private.driver->check_flags(type)) 866f51b7662SDaniel Vetter goto out_err; 867f51b7662SDaniel Vetter 868f51b7662SDaniel Vetter if (!mem->is_flushed) 869f51b7662SDaniel Vetter global_cache_flush(); 870f51b7662SDaniel Vetter 8718d2e6308SBen Widawsky if (intel_private.needs_dmar) { 8729da3da66SChris Wilson struct sg_table st; 8739da3da66SChris Wilson 8749da3da66SChris Wilson ret = intel_gtt_map_memory(mem->pages, mem->page_count, &st); 875fefaa70fSDaniel Vetter if (ret != 0) 876fefaa70fSDaniel Vetter return ret; 877fefaa70fSDaniel Vetter 8789da3da66SChris Wilson intel_gtt_insert_sg_entries(&st, pg_start, type); 8799da3da66SChris Wilson mem->sg_list = st.sgl; 8809da3da66SChris Wilson mem->num_sg = st.nents; 8814080775bSDaniel Vetter } else 8824080775bSDaniel Vetter intel_gtt_insert_pages(pg_start, mem->page_count, mem->pages, 8834080775bSDaniel Vetter type); 884f51b7662SDaniel Vetter 885f51b7662SDaniel Vetter out: 886f51b7662SDaniel Vetter ret = 0; 887f51b7662SDaniel Vetter out_err: 888f51b7662SDaniel Vetter mem->is_flushed = true; 889f51b7662SDaniel Vetter return ret; 890f51b7662SDaniel Vetter } 891f51b7662SDaniel Vetter 8924080775bSDaniel Vetter void intel_gtt_clear_range(unsigned int first_entry, unsigned int num_entries) 893f51b7662SDaniel Vetter { 8944080775bSDaniel Vetter unsigned int i; 895f51b7662SDaniel Vetter 8964080775bSDaniel Vetter for (i = first_entry; i < (first_entry + num_entries); i++) { 8979c61a32dSBen Widawsky intel_private.driver->write_entry(intel_private.scratch_page_dma, 8985cbecafcSDaniel Vetter i, 0); 899f51b7662SDaniel Vetter } 900fdfb58a9SDaniel Vetter readl(intel_private.gtt+i-1); 9014080775bSDaniel Vetter } 9024080775bSDaniel Vetter EXPORT_SYMBOL(intel_gtt_clear_range); 9034080775bSDaniel Vetter 9044080775bSDaniel Vetter static int intel_fake_agp_remove_entries(struct agp_memory *mem, 9054080775bSDaniel Vetter off_t pg_start, int type) 9064080775bSDaniel Vetter { 9074080775bSDaniel Vetter if (mem->page_count == 0) 9084080775bSDaniel Vetter return 0; 9094080775bSDaniel Vetter 910d15eda5cSDave Airlie intel_gtt_clear_range(pg_start, mem->page_count); 911d15eda5cSDave Airlie 9128d2e6308SBen Widawsky if (intel_private.needs_dmar) { 9134080775bSDaniel Vetter intel_gtt_unmap_memory(mem->sg_list, mem->num_sg); 9144080775bSDaniel Vetter mem->sg_list = NULL; 9154080775bSDaniel Vetter mem->num_sg = 0; 9164080775bSDaniel Vetter } 9174080775bSDaniel Vetter 918f51b7662SDaniel Vetter return 0; 919f51b7662SDaniel Vetter } 920f51b7662SDaniel Vetter 921ffdd7510SDaniel Vetter static struct agp_memory *intel_fake_agp_alloc_by_type(size_t pg_count, 922ffdd7510SDaniel Vetter int type) 923f51b7662SDaniel Vetter { 924625dd9d3SDaniel Vetter struct agp_memory *new; 925625dd9d3SDaniel Vetter 926625dd9d3SDaniel Vetter if (type == AGP_DCACHE_MEMORY && INTEL_GTT_GEN == 1) { 927625dd9d3SDaniel Vetter if (pg_count != intel_private.num_dcache_entries) 928625dd9d3SDaniel Vetter return NULL; 929625dd9d3SDaniel Vetter 930625dd9d3SDaniel Vetter new = agp_create_memory(1); 931625dd9d3SDaniel Vetter if (new == NULL) 932625dd9d3SDaniel Vetter return NULL; 933625dd9d3SDaniel Vetter 934625dd9d3SDaniel Vetter new->type = AGP_DCACHE_MEMORY; 935625dd9d3SDaniel Vetter new->page_count = pg_count; 936625dd9d3SDaniel Vetter new->num_scratch_pages = 0; 937625dd9d3SDaniel Vetter agp_free_page_array(new); 938625dd9d3SDaniel Vetter return new; 939625dd9d3SDaniel Vetter } 940f51b7662SDaniel Vetter if (type == AGP_PHYS_MEMORY) 941f51b7662SDaniel Vetter return alloc_agpphysmem_i8xx(pg_count, type); 942f51b7662SDaniel Vetter /* always return NULL for other allocation types for now */ 943f51b7662SDaniel Vetter return NULL; 944f51b7662SDaniel Vetter } 945f51b7662SDaniel Vetter 946f51b7662SDaniel Vetter static int intel_alloc_chipset_flush_resource(void) 947f51b7662SDaniel Vetter { 948f51b7662SDaniel Vetter int ret; 949d7cca2f7SDaniel Vetter ret = pci_bus_alloc_resource(intel_private.bridge_dev->bus, &intel_private.ifp_resource, PAGE_SIZE, 950f51b7662SDaniel Vetter PAGE_SIZE, PCIBIOS_MIN_MEM, 0, 951d7cca2f7SDaniel Vetter pcibios_align_resource, intel_private.bridge_dev); 952f51b7662SDaniel Vetter 953f51b7662SDaniel Vetter return ret; 954f51b7662SDaniel Vetter } 955f51b7662SDaniel Vetter 956f51b7662SDaniel Vetter static void intel_i915_setup_chipset_flush(void) 957f51b7662SDaniel Vetter { 958f51b7662SDaniel Vetter int ret; 959f51b7662SDaniel Vetter u32 temp; 960f51b7662SDaniel Vetter 961d7cca2f7SDaniel Vetter pci_read_config_dword(intel_private.bridge_dev, I915_IFPADDR, &temp); 962f51b7662SDaniel Vetter if (!(temp & 0x1)) { 963f51b7662SDaniel Vetter intel_alloc_chipset_flush_resource(); 964f51b7662SDaniel Vetter intel_private.resource_valid = 1; 965d7cca2f7SDaniel Vetter pci_write_config_dword(intel_private.bridge_dev, I915_IFPADDR, (intel_private.ifp_resource.start & 0xffffffff) | 0x1); 966f51b7662SDaniel Vetter } else { 967f51b7662SDaniel Vetter temp &= ~1; 968f51b7662SDaniel Vetter 969f51b7662SDaniel Vetter intel_private.resource_valid = 1; 970f51b7662SDaniel Vetter intel_private.ifp_resource.start = temp; 971f51b7662SDaniel Vetter intel_private.ifp_resource.end = temp + PAGE_SIZE; 972f51b7662SDaniel Vetter ret = request_resource(&iomem_resource, &intel_private.ifp_resource); 973f51b7662SDaniel Vetter /* some BIOSes reserve this area in a pnp some don't */ 974f51b7662SDaniel Vetter if (ret) 975f51b7662SDaniel Vetter intel_private.resource_valid = 0; 976f51b7662SDaniel Vetter } 977f51b7662SDaniel Vetter } 978f51b7662SDaniel Vetter 979f51b7662SDaniel Vetter static void intel_i965_g33_setup_chipset_flush(void) 980f51b7662SDaniel Vetter { 981f51b7662SDaniel Vetter u32 temp_hi, temp_lo; 982f51b7662SDaniel Vetter int ret; 983f51b7662SDaniel Vetter 984d7cca2f7SDaniel Vetter pci_read_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4, &temp_hi); 985d7cca2f7SDaniel Vetter pci_read_config_dword(intel_private.bridge_dev, I965_IFPADDR, &temp_lo); 986f51b7662SDaniel Vetter 987f51b7662SDaniel Vetter if (!(temp_lo & 0x1)) { 988f51b7662SDaniel Vetter 989f51b7662SDaniel Vetter intel_alloc_chipset_flush_resource(); 990f51b7662SDaniel Vetter 991f51b7662SDaniel Vetter intel_private.resource_valid = 1; 992d7cca2f7SDaniel Vetter pci_write_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4, 993f51b7662SDaniel Vetter upper_32_bits(intel_private.ifp_resource.start)); 994d7cca2f7SDaniel Vetter pci_write_config_dword(intel_private.bridge_dev, I965_IFPADDR, (intel_private.ifp_resource.start & 0xffffffff) | 0x1); 995f51b7662SDaniel Vetter } else { 996f51b7662SDaniel Vetter u64 l64; 997f51b7662SDaniel Vetter 998f51b7662SDaniel Vetter temp_lo &= ~0x1; 999f51b7662SDaniel Vetter l64 = ((u64)temp_hi << 32) | temp_lo; 1000f51b7662SDaniel Vetter 1001f51b7662SDaniel Vetter intel_private.resource_valid = 1; 1002f51b7662SDaniel Vetter intel_private.ifp_resource.start = l64; 1003f51b7662SDaniel Vetter intel_private.ifp_resource.end = l64 + PAGE_SIZE; 1004f51b7662SDaniel Vetter ret = request_resource(&iomem_resource, &intel_private.ifp_resource); 1005f51b7662SDaniel Vetter /* some BIOSes reserve this area in a pnp some don't */ 1006f51b7662SDaniel Vetter if (ret) 1007f51b7662SDaniel Vetter intel_private.resource_valid = 0; 1008f51b7662SDaniel Vetter } 1009f51b7662SDaniel Vetter } 1010f51b7662SDaniel Vetter 1011f51b7662SDaniel Vetter static void intel_i9xx_setup_flush(void) 1012f51b7662SDaniel Vetter { 1013f51b7662SDaniel Vetter /* return if already configured */ 1014f51b7662SDaniel Vetter if (intel_private.ifp_resource.start) 1015f51b7662SDaniel Vetter return; 1016f51b7662SDaniel Vetter 10171a997ff2SDaniel Vetter if (INTEL_GTT_GEN == 6) 1018f51b7662SDaniel Vetter return; 1019f51b7662SDaniel Vetter 1020f51b7662SDaniel Vetter /* setup a resource for this object */ 1021f51b7662SDaniel Vetter intel_private.ifp_resource.name = "Intel Flush Page"; 1022f51b7662SDaniel Vetter intel_private.ifp_resource.flags = IORESOURCE_MEM; 1023f51b7662SDaniel Vetter 1024f51b7662SDaniel Vetter /* Setup chipset flush for 915 */ 10251a997ff2SDaniel Vetter if (IS_G33 || INTEL_GTT_GEN >= 4) { 1026f51b7662SDaniel Vetter intel_i965_g33_setup_chipset_flush(); 1027f51b7662SDaniel Vetter } else { 1028f51b7662SDaniel Vetter intel_i915_setup_chipset_flush(); 1029f51b7662SDaniel Vetter } 1030f51b7662SDaniel Vetter 1031df51e7aaSChris Wilson if (intel_private.ifp_resource.start) 1032f51b7662SDaniel Vetter intel_private.i9xx_flush_page = ioremap_nocache(intel_private.ifp_resource.start, PAGE_SIZE); 1033f51b7662SDaniel Vetter if (!intel_private.i9xx_flush_page) 1034df51e7aaSChris Wilson dev_err(&intel_private.pcidev->dev, 1035df51e7aaSChris Wilson "can't ioremap flush page - no chipset flushing\n"); 1036f51b7662SDaniel Vetter } 1037f51b7662SDaniel Vetter 1038ae83dd5cSDaniel Vetter static void i9xx_cleanup(void) 1039ae83dd5cSDaniel Vetter { 1040ae83dd5cSDaniel Vetter if (intel_private.i9xx_flush_page) 1041ae83dd5cSDaniel Vetter iounmap(intel_private.i9xx_flush_page); 1042ae83dd5cSDaniel Vetter if (intel_private.resource_valid) 1043ae83dd5cSDaniel Vetter release_resource(&intel_private.ifp_resource); 1044ae83dd5cSDaniel Vetter intel_private.ifp_resource.start = 0; 1045ae83dd5cSDaniel Vetter intel_private.resource_valid = 0; 1046ae83dd5cSDaniel Vetter } 1047ae83dd5cSDaniel Vetter 10481b263f24SDaniel Vetter static void i9xx_chipset_flush(void) 1049f51b7662SDaniel Vetter { 1050f51b7662SDaniel Vetter if (intel_private.i9xx_flush_page) 1051f51b7662SDaniel Vetter writel(1, intel_private.i9xx_flush_page); 1052f51b7662SDaniel Vetter } 1053f51b7662SDaniel Vetter 105471f45660SChris Wilson static void i965_write_entry(dma_addr_t addr, 105571f45660SChris Wilson unsigned int entry, 1056a6963596SDaniel Vetter unsigned int flags) 1057a6963596SDaniel Vetter { 105871f45660SChris Wilson u32 pte_flags; 105971f45660SChris Wilson 106071f45660SChris Wilson pte_flags = I810_PTE_VALID; 106171f45660SChris Wilson if (flags == AGP_USER_CACHED_MEMORY) 106271f45660SChris Wilson pte_flags |= I830_PTE_SYSTEM_CACHED; 106371f45660SChris Wilson 1064a6963596SDaniel Vetter /* Shift high bits down */ 1065a6963596SDaniel Vetter addr |= (addr >> 28) & 0xf0; 106671f45660SChris Wilson writel(addr | pte_flags, intel_private.gtt + entry); 1067a6963596SDaniel Vetter } 1068a6963596SDaniel Vetter 10695c042287SBen Widawsky 10702d2430cfSDaniel Vetter static int i9xx_setup(void) 10712d2430cfSDaniel Vetter { 1072009946f8SBen Widawsky u32 reg_addr, gtt_addr; 10734b60d29eSJesse Barnes int size = KB(512); 10742d2430cfSDaniel Vetter 10752d2430cfSDaniel Vetter pci_read_config_dword(intel_private.pcidev, I915_MMADDR, ®_addr); 10762d2430cfSDaniel Vetter 10772d2430cfSDaniel Vetter reg_addr &= 0xfff80000; 10782d2430cfSDaniel Vetter 10794b60d29eSJesse Barnes intel_private.registers = ioremap(reg_addr, size); 10802d2430cfSDaniel Vetter if (!intel_private.registers) 10812d2430cfSDaniel Vetter return -ENOMEM; 10822d2430cfSDaniel Vetter 1083009946f8SBen Widawsky switch (INTEL_GTT_GEN) { 1084009946f8SBen Widawsky case 3: 10852d2430cfSDaniel Vetter pci_read_config_dword(intel_private.pcidev, 10862d2430cfSDaniel Vetter I915_PTEADDR, >t_addr); 10872d2430cfSDaniel Vetter intel_private.gtt_bus_addr = gtt_addr; 1088009946f8SBen Widawsky break; 10892d2430cfSDaniel Vetter case 5: 1090009946f8SBen Widawsky intel_private.gtt_bus_addr = reg_addr + MB(2); 10912d2430cfSDaniel Vetter break; 10922d2430cfSDaniel Vetter default: 1093009946f8SBen Widawsky intel_private.gtt_bus_addr = reg_addr + KB(512); 10942d2430cfSDaniel Vetter break; 10952d2430cfSDaniel Vetter } 10962d2430cfSDaniel Vetter 10972d2430cfSDaniel Vetter intel_i9xx_setup_flush(); 10982d2430cfSDaniel Vetter 10992d2430cfSDaniel Vetter return 0; 11002d2430cfSDaniel Vetter } 11012d2430cfSDaniel Vetter 1102e9b1cc81SDaniel Vetter static const struct agp_bridge_driver intel_fake_agp_driver = { 1103f51b7662SDaniel Vetter .owner = THIS_MODULE, 1104f51b7662SDaniel Vetter .size_type = FIXED_APER_SIZE, 11059e76e7b8SChris Wilson .aperture_sizes = intel_fake_agp_sizes, 11069e76e7b8SChris Wilson .num_aperture_sizes = ARRAY_SIZE(intel_fake_agp_sizes), 1107a6963596SDaniel Vetter .configure = intel_fake_agp_configure, 11083e921f98SDaniel Vetter .fetch_size = intel_fake_agp_fetch_size, 1109fdfb58a9SDaniel Vetter .cleanup = intel_gtt_cleanup, 1110ffdd7510SDaniel Vetter .agp_enable = intel_fake_agp_enable, 1111f51b7662SDaniel Vetter .cache_flush = global_cache_flush, 11123b15a9d7SDaniel Vetter .create_gatt_table = intel_fake_agp_create_gatt_table, 1113ffdd7510SDaniel Vetter .free_gatt_table = intel_fake_agp_free_gatt_table, 1114450f2b3dSDaniel Vetter .insert_memory = intel_fake_agp_insert_entries, 1115450f2b3dSDaniel Vetter .remove_memory = intel_fake_agp_remove_entries, 1116ffdd7510SDaniel Vetter .alloc_by_type = intel_fake_agp_alloc_by_type, 1117f51b7662SDaniel Vetter .free_by_type = intel_i810_free_by_type, 1118f51b7662SDaniel Vetter .agp_alloc_page = agp_generic_alloc_page, 1119f51b7662SDaniel Vetter .agp_alloc_pages = agp_generic_alloc_pages, 1120f51b7662SDaniel Vetter .agp_destroy_page = agp_generic_destroy_page, 1121f51b7662SDaniel Vetter .agp_destroy_pages = agp_generic_destroy_pages, 1122f51b7662SDaniel Vetter }; 112302c026ceSDaniel Vetter 1124bdd30729SDaniel Vetter static const struct intel_gtt_driver i81x_gtt_driver = { 1125bdd30729SDaniel Vetter .gen = 1, 1126820647b9SDaniel Vetter .has_pgtbl_enable = 1, 112722533b49SDaniel Vetter .dma_mask_size = 32, 1128820647b9SDaniel Vetter .setup = i810_setup, 1129820647b9SDaniel Vetter .cleanup = i810_cleanup, 1130625dd9d3SDaniel Vetter .check_flags = i830_check_flags, 1131625dd9d3SDaniel Vetter .write_entry = i810_write_entry, 1132bdd30729SDaniel Vetter }; 11331a997ff2SDaniel Vetter static const struct intel_gtt_driver i8xx_gtt_driver = { 11341a997ff2SDaniel Vetter .gen = 2, 1135100519e2SChris Wilson .has_pgtbl_enable = 1, 113673800422SDaniel Vetter .setup = i830_setup, 1137ae83dd5cSDaniel Vetter .cleanup = i830_cleanup, 1138351bb278SDaniel Vetter .write_entry = i830_write_entry, 113922533b49SDaniel Vetter .dma_mask_size = 32, 11405cbecafcSDaniel Vetter .check_flags = i830_check_flags, 11411b263f24SDaniel Vetter .chipset_flush = i830_chipset_flush, 11421a997ff2SDaniel Vetter }; 11431a997ff2SDaniel Vetter static const struct intel_gtt_driver i915_gtt_driver = { 11441a997ff2SDaniel Vetter .gen = 3, 1145100519e2SChris Wilson .has_pgtbl_enable = 1, 11462d2430cfSDaniel Vetter .setup = i9xx_setup, 1147ae83dd5cSDaniel Vetter .cleanup = i9xx_cleanup, 1148351bb278SDaniel Vetter /* i945 is the last gpu to need phys mem (for overlay and cursors). */ 1149351bb278SDaniel Vetter .write_entry = i830_write_entry, 115022533b49SDaniel Vetter .dma_mask_size = 32, 1151fefaa70fSDaniel Vetter .check_flags = i830_check_flags, 11521b263f24SDaniel Vetter .chipset_flush = i9xx_chipset_flush, 11531a997ff2SDaniel Vetter }; 11541a997ff2SDaniel Vetter static const struct intel_gtt_driver g33_gtt_driver = { 11551a997ff2SDaniel Vetter .gen = 3, 11561a997ff2SDaniel Vetter .is_g33 = 1, 11572d2430cfSDaniel Vetter .setup = i9xx_setup, 1158ae83dd5cSDaniel Vetter .cleanup = i9xx_cleanup, 1159a6963596SDaniel Vetter .write_entry = i965_write_entry, 116022533b49SDaniel Vetter .dma_mask_size = 36, 1161450f2b3dSDaniel Vetter .check_flags = i830_check_flags, 11621b263f24SDaniel Vetter .chipset_flush = i9xx_chipset_flush, 11631a997ff2SDaniel Vetter }; 11641a997ff2SDaniel Vetter static const struct intel_gtt_driver pineview_gtt_driver = { 11651a997ff2SDaniel Vetter .gen = 3, 11661a997ff2SDaniel Vetter .is_pineview = 1, .is_g33 = 1, 11672d2430cfSDaniel Vetter .setup = i9xx_setup, 1168ae83dd5cSDaniel Vetter .cleanup = i9xx_cleanup, 1169a6963596SDaniel Vetter .write_entry = i965_write_entry, 117022533b49SDaniel Vetter .dma_mask_size = 36, 1171450f2b3dSDaniel Vetter .check_flags = i830_check_flags, 11721b263f24SDaniel Vetter .chipset_flush = i9xx_chipset_flush, 11731a997ff2SDaniel Vetter }; 11741a997ff2SDaniel Vetter static const struct intel_gtt_driver i965_gtt_driver = { 11751a997ff2SDaniel Vetter .gen = 4, 1176100519e2SChris Wilson .has_pgtbl_enable = 1, 11772d2430cfSDaniel Vetter .setup = i9xx_setup, 1178ae83dd5cSDaniel Vetter .cleanup = i9xx_cleanup, 1179a6963596SDaniel Vetter .write_entry = i965_write_entry, 118022533b49SDaniel Vetter .dma_mask_size = 36, 1181450f2b3dSDaniel Vetter .check_flags = i830_check_flags, 11821b263f24SDaniel Vetter .chipset_flush = i9xx_chipset_flush, 11831a997ff2SDaniel Vetter }; 11841a997ff2SDaniel Vetter static const struct intel_gtt_driver g4x_gtt_driver = { 11851a997ff2SDaniel Vetter .gen = 5, 11862d2430cfSDaniel Vetter .setup = i9xx_setup, 1187ae83dd5cSDaniel Vetter .cleanup = i9xx_cleanup, 1188a6963596SDaniel Vetter .write_entry = i965_write_entry, 118922533b49SDaniel Vetter .dma_mask_size = 36, 1190450f2b3dSDaniel Vetter .check_flags = i830_check_flags, 11911b263f24SDaniel Vetter .chipset_flush = i9xx_chipset_flush, 11921a997ff2SDaniel Vetter }; 11931a997ff2SDaniel Vetter static const struct intel_gtt_driver ironlake_gtt_driver = { 11941a997ff2SDaniel Vetter .gen = 5, 11951a997ff2SDaniel Vetter .is_ironlake = 1, 11962d2430cfSDaniel Vetter .setup = i9xx_setup, 1197ae83dd5cSDaniel Vetter .cleanup = i9xx_cleanup, 1198a6963596SDaniel Vetter .write_entry = i965_write_entry, 119922533b49SDaniel Vetter .dma_mask_size = 36, 1200450f2b3dSDaniel Vetter .check_flags = i830_check_flags, 12011b263f24SDaniel Vetter .chipset_flush = i9xx_chipset_flush, 12021a997ff2SDaniel Vetter }; 12031a997ff2SDaniel Vetter 120402c026ceSDaniel Vetter /* Table to describe Intel GMCH and AGP/PCIE GART drivers. At least one of 120502c026ceSDaniel Vetter * driver and gmch_driver must be non-null, and find_gmch will determine 120602c026ceSDaniel Vetter * which one should be used if a gmch_chip_id is present. 120702c026ceSDaniel Vetter */ 120802c026ceSDaniel Vetter static const struct intel_gtt_driver_description { 120902c026ceSDaniel Vetter unsigned int gmch_chip_id; 121002c026ceSDaniel Vetter char *name; 12111a997ff2SDaniel Vetter const struct intel_gtt_driver *gtt_driver; 121202c026ceSDaniel Vetter } intel_gtt_chipsets[] = { 1213ff26860fSDaniel Vetter { PCI_DEVICE_ID_INTEL_82810_IG1, "i810", 1214bdd30729SDaniel Vetter &i81x_gtt_driver}, 1215ff26860fSDaniel Vetter { PCI_DEVICE_ID_INTEL_82810_IG3, "i810", 1216bdd30729SDaniel Vetter &i81x_gtt_driver}, 1217ff26860fSDaniel Vetter { PCI_DEVICE_ID_INTEL_82810E_IG, "i810", 1218bdd30729SDaniel Vetter &i81x_gtt_driver}, 1219ff26860fSDaniel Vetter { PCI_DEVICE_ID_INTEL_82815_CGC, "i815", 1220bdd30729SDaniel Vetter &i81x_gtt_driver}, 12211a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82830_CGC, "830M", 1222ff26860fSDaniel Vetter &i8xx_gtt_driver}, 122353371edaSOswald Buddenhagen { PCI_DEVICE_ID_INTEL_82845G_IG, "845G", 1224ff26860fSDaniel Vetter &i8xx_gtt_driver}, 12251a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82854_IG, "854", 1226ff26860fSDaniel Vetter &i8xx_gtt_driver}, 12271a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82855GM_IG, "855GM", 1228ff26860fSDaniel Vetter &i8xx_gtt_driver}, 12291a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82865_IG, "865", 1230ff26860fSDaniel Vetter &i8xx_gtt_driver}, 12311a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_E7221_IG, "E7221 (i915)", 1232ff26860fSDaniel Vetter &i915_gtt_driver }, 12331a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82915G_IG, "915G", 1234ff26860fSDaniel Vetter &i915_gtt_driver }, 12351a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82915GM_IG, "915GM", 1236ff26860fSDaniel Vetter &i915_gtt_driver }, 12371a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82945G_IG, "945G", 1238ff26860fSDaniel Vetter &i915_gtt_driver }, 12391a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82945GM_IG, "945GM", 1240ff26860fSDaniel Vetter &i915_gtt_driver }, 12411a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82945GME_IG, "945GME", 1242ff26860fSDaniel Vetter &i915_gtt_driver }, 12431a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82946GZ_IG, "946GZ", 1244ff26860fSDaniel Vetter &i965_gtt_driver }, 12451a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82G35_IG, "G35", 1246ff26860fSDaniel Vetter &i965_gtt_driver }, 12471a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82965Q_IG, "965Q", 1248ff26860fSDaniel Vetter &i965_gtt_driver }, 12491a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82965G_IG, "965G", 1250ff26860fSDaniel Vetter &i965_gtt_driver }, 12511a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82965GM_IG, "965GM", 1252ff26860fSDaniel Vetter &i965_gtt_driver }, 12531a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_82965GME_IG, "965GME/GLE", 1254ff26860fSDaniel Vetter &i965_gtt_driver }, 12551a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_G33_IG, "G33", 1256ff26860fSDaniel Vetter &g33_gtt_driver }, 12571a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_Q35_IG, "Q35", 1258ff26860fSDaniel Vetter &g33_gtt_driver }, 12591a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_Q33_IG, "Q33", 1260ff26860fSDaniel Vetter &g33_gtt_driver }, 12611a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_PINEVIEW_M_IG, "GMA3150", 1262ff26860fSDaniel Vetter &pineview_gtt_driver }, 12631a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_PINEVIEW_IG, "GMA3150", 1264ff26860fSDaniel Vetter &pineview_gtt_driver }, 12651a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_GM45_IG, "GM45", 1266ff26860fSDaniel Vetter &g4x_gtt_driver }, 12671a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_EAGLELAKE_IG, "Eaglelake", 1268ff26860fSDaniel Vetter &g4x_gtt_driver }, 12691a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_Q45_IG, "Q45/Q43", 1270ff26860fSDaniel Vetter &g4x_gtt_driver }, 12711a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_G45_IG, "G45/G43", 1272ff26860fSDaniel Vetter &g4x_gtt_driver }, 12731a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_B43_IG, "B43", 1274ff26860fSDaniel Vetter &g4x_gtt_driver }, 1275e9e5f8e8SChris Wilson { PCI_DEVICE_ID_INTEL_B43_1_IG, "B43", 1276ff26860fSDaniel Vetter &g4x_gtt_driver }, 12771a997ff2SDaniel Vetter { PCI_DEVICE_ID_INTEL_G41_IG, "G41", 1278ff26860fSDaniel Vetter &g4x_gtt_driver }, 127902c026ceSDaniel Vetter { PCI_DEVICE_ID_INTEL_IRONLAKE_D_IG, 1280ff26860fSDaniel Vetter "HD Graphics", &ironlake_gtt_driver }, 128102c026ceSDaniel Vetter { PCI_DEVICE_ID_INTEL_IRONLAKE_M_IG, 1282ff26860fSDaniel Vetter "HD Graphics", &ironlake_gtt_driver }, 128302c026ceSDaniel Vetter { 0, NULL, NULL } 128402c026ceSDaniel Vetter }; 128502c026ceSDaniel Vetter 128602c026ceSDaniel Vetter static int find_gmch(u16 device) 128702c026ceSDaniel Vetter { 128802c026ceSDaniel Vetter struct pci_dev *gmch_device; 128902c026ceSDaniel Vetter 129002c026ceSDaniel Vetter gmch_device = pci_get_device(PCI_VENDOR_ID_INTEL, device, NULL); 129102c026ceSDaniel Vetter if (gmch_device && PCI_FUNC(gmch_device->devfn) != 0) { 129202c026ceSDaniel Vetter gmch_device = pci_get_device(PCI_VENDOR_ID_INTEL, 129302c026ceSDaniel Vetter device, gmch_device); 129402c026ceSDaniel Vetter } 129502c026ceSDaniel Vetter 129602c026ceSDaniel Vetter if (!gmch_device) 129702c026ceSDaniel Vetter return 0; 129802c026ceSDaniel Vetter 129902c026ceSDaniel Vetter intel_private.pcidev = gmch_device; 130002c026ceSDaniel Vetter return 1; 130102c026ceSDaniel Vetter } 130202c026ceSDaniel Vetter 130314be93ddSDaniel Vetter int intel_gmch_probe(struct pci_dev *bridge_pdev, struct pci_dev *gpu_pdev, 130402c026ceSDaniel Vetter struct agp_bridge_data *bridge) 130502c026ceSDaniel Vetter { 130602c026ceSDaniel Vetter int i, mask; 130714be93ddSDaniel Vetter 130814be93ddSDaniel Vetter /* 130914be93ddSDaniel Vetter * Can be called from the fake agp driver but also directly from 131014be93ddSDaniel Vetter * drm/i915.ko. Hence we need to check whether everything is set up 131114be93ddSDaniel Vetter * already. 131214be93ddSDaniel Vetter */ 131314be93ddSDaniel Vetter if (intel_private.driver) { 131414be93ddSDaniel Vetter intel_private.refcount++; 131514be93ddSDaniel Vetter return 1; 131614be93ddSDaniel Vetter } 131702c026ceSDaniel Vetter 131802c026ceSDaniel Vetter for (i = 0; intel_gtt_chipsets[i].name != NULL; i++) { 131914be93ddSDaniel Vetter if (gpu_pdev) { 132014be93ddSDaniel Vetter if (gpu_pdev->device == 132114be93ddSDaniel Vetter intel_gtt_chipsets[i].gmch_chip_id) { 132214be93ddSDaniel Vetter intel_private.pcidev = pci_dev_get(gpu_pdev); 132314be93ddSDaniel Vetter intel_private.driver = 132414be93ddSDaniel Vetter intel_gtt_chipsets[i].gtt_driver; 132514be93ddSDaniel Vetter 132614be93ddSDaniel Vetter break; 132714be93ddSDaniel Vetter } 132814be93ddSDaniel Vetter } else if (find_gmch(intel_gtt_chipsets[i].gmch_chip_id)) { 13291a997ff2SDaniel Vetter intel_private.driver = 13301a997ff2SDaniel Vetter intel_gtt_chipsets[i].gtt_driver; 133102c026ceSDaniel Vetter break; 133202c026ceSDaniel Vetter } 133302c026ceSDaniel Vetter } 133402c026ceSDaniel Vetter 1335ff26860fSDaniel Vetter if (!intel_private.driver) 133602c026ceSDaniel Vetter return 0; 133702c026ceSDaniel Vetter 133814be93ddSDaniel Vetter intel_private.refcount++; 133914be93ddSDaniel Vetter 13407e8f6306SDaniel Vetter if (bridge) { 1341ff26860fSDaniel Vetter bridge->driver = &intel_fake_agp_driver; 134202c026ceSDaniel Vetter bridge->dev_private_data = &intel_private; 134314be93ddSDaniel Vetter bridge->dev = bridge_pdev; 13447e8f6306SDaniel Vetter } 134502c026ceSDaniel Vetter 134614be93ddSDaniel Vetter intel_private.bridge_dev = pci_dev_get(bridge_pdev); 1347d7cca2f7SDaniel Vetter 134814be93ddSDaniel Vetter dev_info(&bridge_pdev->dev, "Intel %s Chipset\n", intel_gtt_chipsets[i].name); 134902c026ceSDaniel Vetter 135022533b49SDaniel Vetter mask = intel_private.driver->dma_mask_size; 135102c026ceSDaniel Vetter if (pci_set_dma_mask(intel_private.pcidev, DMA_BIT_MASK(mask))) 135202c026ceSDaniel Vetter dev_err(&intel_private.pcidev->dev, 135302c026ceSDaniel Vetter "set gfx device dma mask %d-bit failed!\n", mask); 135402c026ceSDaniel Vetter else 135502c026ceSDaniel Vetter pci_set_consistent_dma_mask(intel_private.pcidev, 135602c026ceSDaniel Vetter DMA_BIT_MASK(mask)); 135702c026ceSDaniel Vetter 135814be93ddSDaniel Vetter if (intel_gtt_init() != 0) { 135914be93ddSDaniel Vetter intel_gmch_remove(); 136014be93ddSDaniel Vetter 13613b15a9d7SDaniel Vetter return 0; 136214be93ddSDaniel Vetter } 13631784a5fbSDaniel Vetter 136402c026ceSDaniel Vetter return 1; 136502c026ceSDaniel Vetter } 1366e2404e7cSDaniel Vetter EXPORT_SYMBOL(intel_gmch_probe); 136702c026ceSDaniel Vetter 1368e76e9aebSBen Widawsky struct intel_gtt *intel_gtt_get(void) 136919966754SDaniel Vetter { 137019966754SDaniel Vetter return &intel_private.base; 137119966754SDaniel Vetter } 137219966754SDaniel Vetter EXPORT_SYMBOL(intel_gtt_get); 137319966754SDaniel Vetter 137440ce6575SDaniel Vetter void intel_gtt_chipset_flush(void) 137540ce6575SDaniel Vetter { 137640ce6575SDaniel Vetter if (intel_private.driver->chipset_flush) 137740ce6575SDaniel Vetter intel_private.driver->chipset_flush(); 137840ce6575SDaniel Vetter } 137940ce6575SDaniel Vetter EXPORT_SYMBOL(intel_gtt_chipset_flush); 138040ce6575SDaniel Vetter 138114be93ddSDaniel Vetter void intel_gmch_remove(void) 138202c026ceSDaniel Vetter { 138314be93ddSDaniel Vetter if (--intel_private.refcount) 138414be93ddSDaniel Vetter return; 138514be93ddSDaniel Vetter 138602c026ceSDaniel Vetter if (intel_private.pcidev) 138702c026ceSDaniel Vetter pci_dev_put(intel_private.pcidev); 1388d7cca2f7SDaniel Vetter if (intel_private.bridge_dev) 1389d7cca2f7SDaniel Vetter pci_dev_put(intel_private.bridge_dev); 139014be93ddSDaniel Vetter intel_private.driver = NULL; 139102c026ceSDaniel Vetter } 1392e2404e7cSDaniel Vetter EXPORT_SYMBOL(intel_gmch_remove); 1393e2404e7cSDaniel Vetter 1394e2404e7cSDaniel Vetter MODULE_AUTHOR("Dave Jones <davej@redhat.com>"); 1395e2404e7cSDaniel Vetter MODULE_LICENSE("GPL and additional rights"); 1396