drivers/char/agp/intel-gtt.c

   1 /*
   2  * Intel GTT (Graphics Translation Table) routines
   3  *
   4  * Caveat: This driver implements the linux agp interface, but this is far from
   5  * a agp driver! GTT support ended up here for purely historical reasons: The
   6  * old userspace intel graphics drivers needed an interface to map memory into
   7  * the GTT. And the drm provides a default interface for graphic devices sitting
   8  * on an agp port. So it made sense to fake the GTT support as an agp port to
   9  * avoid having to create a new api.
  10  *
  11  * With gem this does not make much sense anymore, just needlessly complicates
  12  * the code. But as long as the old graphics stack is still support, it's stuck
  13  * here.
  14  *
  15  * /fairy-tale-mode off
  16  */
  17
  18 #include <linux/module.h>
  19 #include <linux/pci.h>
  20 #include <linux/init.h>
  21 #include <linux/kernel.h>
  22 #include <linux/pagemap.h>
  23 #include <linux/agp_backend.h>
  24 #include <linux/delay.h>
  25 #include <asm/smp.h>
  26 #include "agp.h"
  27 #include "intel-agp.h"
  28 #include <drm/intel-gtt.h>
  29
  30 /*
  31  * If we have Intel graphics, we're not going to have anything other than
  32  * an Intel IOMMU. So make the correct use of the PCI DMA API contingent
  33  * on the Intel IOMMU support (CONFIG_INTEL_IOMMU).
  34  * Only newer chipsets need to bother with this, of course.
  35  */
  36 #ifdef CONFIG_INTEL_IOMMU
  37 #define USE_PCI_DMA_API 1
  38 #else
  39 #define USE_PCI_DMA_API 0
  40 #endif
  41
  42 struct intel_gtt_driver {
  43         unsigned int gen : 8;
  44         unsigned int is_g33 : 1;
  45         unsigned int is_pineview : 1;
  46         unsigned int is_ironlake : 1;
  47         unsigned int has_pgtbl_enable : 1;
  48         unsigned int dma_mask_size : 8;
  49         /* Chipset specific GTT setup */
  50         int (*setup)(void);
  51         /* This should undo anything done in ->setup() save the unmapping
  52          * of the mmio register file, that's done in the generic code. */
  53         void (*cleanup)(void);
  54         void (*write_entry)(dma_addr_t addr, unsigned int entry, unsigned int flags);
  55         /* Flags is a more or less chipset specific opaque value.
  56          * For chipsets that need to support old ums (non-gem) code, this
  57          * needs to be identical to the various supported agp memory types! */
  58         bool (*check_flags)(unsigned int flags);
  59         void (*chipset_flush)(void);
  60 };
  61
  62 static struct _intel_private {
  63         struct intel_gtt base;
  64         const struct intel_gtt_driver *driver;
  65         struct pci_dev *pcidev; /* device one */
  66         struct pci_dev *bridge_dev;
  67         u8 __iomem *registers;
  68         phys_addr_t gtt_bus_addr;
  69         u32 PGETBL_save;
  70         u32 __iomem *gtt;               /* I915G */
  71         bool clear_fake_agp; /* on first access via agp, fill with scratch */
  72         int num_dcache_entries;
  73         void __iomem *i9xx_flush_page;
  74         char *i81x_gtt_table;
  75         struct resource ifp_resource;
  76         int resource_valid;
  77         struct page *scratch_page;
  78         phys_addr_t scratch_page_dma;
  79         int refcount;
  80         /* Whether i915 needs to use the dmar apis or not. */
  81         unsigned int needs_dmar : 1;
  82 } intel_private;
  83
  84 #define INTEL_GTT_GEN   intel_private.driver->gen
  85 #define IS_G33          intel_private.driver->is_g33
  86 #define IS_PINEVIEW     intel_private.driver->is_pineview
  87 #define IS_IRONLAKE     intel_private.driver->is_ironlake
  88 #define HAS_PGTBL_EN    intel_private.driver->has_pgtbl_enable
  89
  90 static int intel_gtt_map_memory(struct page **pages,
  91                                 unsigned int num_entries,
  92                                 struct sg_table *st)
  93 {
  94         struct scatterlist *sg;
  95         int i;
  96
  97         DBG("try mapping %lu pages\n", (unsigned long)num_entries);
  98
  99         if (sg_alloc_table(st, num_entries, GFP_KERNEL))
 100                 goto err;
 101
 102         for_each_sg(st->sgl, sg, num_entries, i)
 103                 sg_set_page(sg, pages[i], PAGE_SIZE, 0);
 104
 105         if (!pci_map_sg(intel_private.pcidev,
 106                         st->sgl, st->nents, PCI_DMA_BIDIRECTIONAL))
 107                 goto err;
 108
 109         return 0;
 110
 111 err:
 112         sg_free_table(st);
 113         return -ENOMEM;
 114 }
 115
 116 static void intel_gtt_unmap_memory(struct scatterlist *sg_list, int num_sg)
 117 {
 118         struct sg_table st;
 119         DBG("try unmapping %lu pages\n", (unsigned long)mem->page_count);
 120
 121         pci_unmap_sg(intel_private.pcidev, sg_list,
 122                      num_sg, PCI_DMA_BIDIRECTIONAL);
 123
 124         st.sgl = sg_list;
 125         st.orig_nents = st.nents = num_sg;
 126
 127         sg_free_table(&st);
 128 }
 129
 130 static void intel_fake_agp_enable(struct agp_bridge_data *bridge, u32 mode)
 131 {
 132         return;
 133 }
 134
 135 /* Exists to support ARGB cursors */
 136 static struct page *i8xx_alloc_pages(void)
 137 {
 138         struct page *page;
 139
 140         page = alloc_pages(GFP_KERNEL | GFP_DMA32, 2);
 141         if (page == NULL)
 142                 return NULL;
 143
 144         if (set_pages_uc(page, 4) < 0) {
 145                 set_pages_wb(page, 4);
 146                 __free_pages(page, 2);
 147                 return NULL;
 148         }
 149         get_page(page);
 150         atomic_inc(&agp_bridge->current_memory_agp);
 151         return page;
 152 }
 153
 154 static void i8xx_destroy_pages(struct page *page)
 155 {
 156         if (page == NULL)
 157                 return;
 158
 159         set_pages_wb(page, 4);
 160         put_page(page);
 161         __free_pages(page, 2);
 162         atomic_dec(&agp_bridge->current_memory_agp);
 163 }
 164
 165 #define I810_GTT_ORDER 4
 166 static int i810_setup(void)
 167 {
 168         u32 reg_addr;
 169         char *gtt_table;
 170
 171         /* i81x does not preallocate the gtt. It's always 64kb in size. */
 172         gtt_table = alloc_gatt_pages(I810_GTT_ORDER);
 173         if (gtt_table == NULL)
 174                 return -ENOMEM;
 175         intel_private.i81x_gtt_table = gtt_table;
 176
 177         pci_read_config_dword(intel_private.pcidev, I810_MMADDR, &reg_addr);
 178         reg_addr &= 0xfff80000;
 179
 180         intel_private.registers = ioremap(reg_addr, KB(64));
 181         if (!intel_private.registers)
 182                 return -ENOMEM;
 183
 184         writel(virt_to_phys(gtt_table) | I810_PGETBL_ENABLED,
 185                intel_private.registers+I810_PGETBL_CTL);
 186
 187         intel_private.gtt_bus_addr = reg_addr + I810_PTE_BASE;
 188
 189         if ((readl(intel_private.registers+I810_DRAM_CTL)
 190                 & I810_DRAM_ROW_0) == I810_DRAM_ROW_0_SDRAM) {
 191                 dev_info(&intel_private.pcidev->dev,
 192                          "detected 4MB dedicated video ram\n");
 193                 intel_private.num_dcache_entries = 1024;
 194         }
 195
 196         return 0;
 197 }
 198
 199 static void i810_cleanup(void)
 200 {
 201         writel(0, intel_private.registers+I810_PGETBL_CTL);
 202         free_gatt_pages(intel_private.i81x_gtt_table, I810_GTT_ORDER);
 203 }
 204
 205 static int i810_insert_dcache_entries(struct agp_memory *mem, off_t pg_start,
 206                                       int type)
 207 {
 208         int i;
 209
 210         if ((pg_start + mem->page_count)
 211                         > intel_private.num_dcache_entries)
 212                 return -EINVAL;
 213
 214         if (!mem->is_flushed)
 215                 global_cache_flush();
 216
 217         for (i = pg_start; i < (pg_start + mem->page_count); i++) {
 218                 dma_addr_t addr = i << PAGE_SHIFT;
 219                 intel_private.driver->write_entry(addr,
 220                                                   i, type);
 221         }
 222         readl(intel_private.gtt+i-1);
 223
 224         return 0;
 225 }
 226
 227 /*
 228  * The i810/i830 requires a physical address to program its mouse
 229  * pointer into hardware.
 230  * However the Xserver still writes to it through the agp aperture.
 231  */
 232 static struct agp_memory *alloc_agpphysmem_i8xx(size_t pg_count, int type)
 233 {
 234         struct agp_memory *new;
 235         struct page *page;
 236
 237         switch (pg_count) {
 238         case 1: page = agp_bridge->driver->agp_alloc_page(agp_bridge);
 239                 break;
 240         case 4:
 241                 /* kludge to get 4 physical pages for ARGB cursor */
 242                 page = i8xx_alloc_pages();
 243                 break;
 244         default:
 245                 return NULL;
 246         }
 247
 248         if (page == NULL)
 249                 return NULL;
 250
 251         new = agp_create_memory(pg_count);
 252         if (new == NULL)
 253                 return NULL;
 254
 255         new->pages[0] = page;
 256         if (pg_count == 4) {
 257                 /* kludge to get 4 physical pages for ARGB cursor */
 258                 new->pages[1] = new->pages[0] + 1;
 259                 new->pages[2] = new->pages[1] + 1;
 260                 new->pages[3] = new->pages[2] + 1;
 261         }
 262         new->page_count = pg_count;
 263         new->num_scratch_pages = pg_count;
 264         new->type = AGP_PHYS_MEMORY;
 265         new->physical = page_to_phys(new->pages[0]);
 266         return new;
 267 }
 268
 269 static void intel_i810_free_by_type(struct agp_memory *curr)
 270 {
 271         agp_free_key(curr->key);
 272         if (curr->type == AGP_PHYS_MEMORY) {
 273                 if (curr->page_count == 4)
 274                         i8xx_destroy_pages(curr->pages[0]);
 275                 else {
 276                         agp_bridge->driver->agp_destroy_page(curr->pages[0],
 277                                                              AGP_PAGE_DESTROY_UNMAP);
 278                         agp_bridge->driver->agp_destroy_page(curr->pages[0],
 279                                                              AGP_PAGE_DESTROY_FREE);
 280                 }
 281                 agp_free_page_array(curr);
 282         }
 283         kfree(curr);
 284 }
 285
 286 static int intel_gtt_setup_scratch_page(void)
 287 {
 288         struct page *page;
 289         dma_addr_t dma_addr;
 290
 291         page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
 292         if (page == NULL)
 293                 return -ENOMEM;
 294         get_page(page);
 295         set_pages_uc(page, 1);
 296
 297         if (intel_private.needs_dmar) {
 298                 dma_addr = pci_map_page(intel_private.pcidev, page, 0,
 299                                     PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 300                 if (pci_dma_mapping_error(intel_private.pcidev, dma_addr))
 301                         return -EINVAL;
 302
 303                 intel_private.scratch_page_dma = dma_addr;
 304         } else
 305                 intel_private.scratch_page_dma = page_to_phys(page);
 306
 307         intel_private.scratch_page = page;
 308
 309         return 0;
 310 }
 311
 312 static void i810_write_entry(dma_addr_t addr, unsigned int entry,
 313                              unsigned int flags)
 314 {
 315         u32 pte_flags = I810_PTE_VALID;
 316
 317         switch (flags) {
 318         case AGP_DCACHE_MEMORY:
 319                 pte_flags |= I810_PTE_LOCAL;
 320                 break;
 321         case AGP_USER_CACHED_MEMORY:
 322                 pte_flags |= I830_PTE_SYSTEM_CACHED;
 323                 break;
 324         }
 325
 326         writel(addr | pte_flags, intel_private.gtt + entry);
 327 }
 328
 329 static const struct aper_size_info_fixed intel_fake_agp_sizes[] = {
 330         {32, 8192, 3},
 331         {64, 16384, 4},
 332         {128, 32768, 5},
 333         {256, 65536, 6},
 334         {512, 131072, 7},
 335 };
 336
 337 static unsigned int intel_gtt_stolen_size(void)
 338 {
 339         u16 gmch_ctrl;
 340         u8 rdct;
 341         int local = 0;
 342         static const int ddt[4] = { 0, 16, 32, 64 };
 343         unsigned int stolen_size = 0;
 344
 345         if (INTEL_GTT_GEN == 1)
 346                 return 0; /* no stolen mem on i81x */
 347
 348         pci_read_config_word(intel_private.bridge_dev,
 349                              I830_GMCH_CTRL, &gmch_ctrl);
 350
 351         if (intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82830_HB ||
 352             intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82845G_HB) {
 353                 switch (gmch_ctrl & I830_GMCH_GMS_MASK) {
 354                 case I830_GMCH_GMS_STOLEN_512:
 355                         stolen_size = KB(512);
 356                         break;
 357                 case I830_GMCH_GMS_STOLEN_1024:
 358                         stolen_size = MB(1);
 359                         break;
 360                 case I830_GMCH_GMS_STOLEN_8192:
 361                         stolen_size = MB(8);
 362                         break;
 363                 case I830_GMCH_GMS_LOCAL:
 364                         rdct = readb(intel_private.registers+I830_RDRAM_CHANNEL_TYPE);
 365                         stolen_size = (I830_RDRAM_ND(rdct) + 1) *
 366                                         MB(ddt[I830_RDRAM_DDT(rdct)]);
 367                         local = 1;
 368                         break;
 369                 default:
 370                         stolen_size = 0;
 371                         break;
 372                 }
 373         } else {
 374                 switch (gmch_ctrl & I855_GMCH_GMS_MASK) {
 375                 case I855_GMCH_GMS_STOLEN_1M:
 376                         stolen_size = MB(1);
 377                         break;
 378                 case I855_GMCH_GMS_STOLEN_4M:
 379                         stolen_size = MB(4);
 380                         break;
 381                 case I855_GMCH_GMS_STOLEN_8M:
 382                         stolen_size = MB(8);
 383                         break;
 384                 case I855_GMCH_GMS_STOLEN_16M:
 385                         stolen_size = MB(16);
 386                         break;
 387                 case I855_GMCH_GMS_STOLEN_32M:
 388                         stolen_size = MB(32);
 389                         break;
 390                 case I915_GMCH_GMS_STOLEN_48M:
 391                         stolen_size = MB(48);
 392                         break;
 393                 case I915_GMCH_GMS_STOLEN_64M:
 394                         stolen_size = MB(64);
 395                         break;
 396                 case G33_GMCH_GMS_STOLEN_128M:
 397                         stolen_size = MB(128);
 398                         break;
 399                 case G33_GMCH_GMS_STOLEN_256M:
 400                         stolen_size = MB(256);
 401                         break;
 402                 case INTEL_GMCH_GMS_STOLEN_96M:
 403                         stolen_size = MB(96);
 404                         break;
 405                 case INTEL_GMCH_GMS_STOLEN_160M:
 406                         stolen_size = MB(160);
 407                         break;
 408                 case INTEL_GMCH_GMS_STOLEN_224M:
 409                         stolen_size = MB(224);
 410                         break;
 411                 case INTEL_GMCH_GMS_STOLEN_352M:
 412                         stolen_size = MB(352);
 413                         break;
 414                 default:
 415                         stolen_size = 0;
 416                         break;
 417                 }
 418         }
 419
 420         if (stolen_size > 0) {
 421                 dev_info(&intel_private.bridge_dev->dev, "detected %dK %s memory\n",
 422                        stolen_size / KB(1), local ? "local" : "stolen");
 423         } else {
 424                 dev_info(&intel_private.bridge_dev->dev,
 425                        "no pre-allocated video memory detected\n");
 426                 stolen_size = 0;
 427         }
 428
 429         return stolen_size;
 430 }
 431
 432 static void i965_adjust_pgetbl_size(unsigned int size_flag)
 433 {
 434         u32 pgetbl_ctl, pgetbl_ctl2;
 435
 436         /* ensure that ppgtt is disabled */
 437         pgetbl_ctl2 = readl(intel_private.registers+I965_PGETBL_CTL2);
 438         pgetbl_ctl2 &= ~I810_PGETBL_ENABLED;
 439         writel(pgetbl_ctl2, intel_private.registers+I965_PGETBL_CTL2);
 440
 441         /* write the new ggtt size */
 442         pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL);
 443         pgetbl_ctl &= ~I965_PGETBL_SIZE_MASK;
 444         pgetbl_ctl |= size_flag;
 445         writel(pgetbl_ctl, intel_private.registers+I810_PGETBL_CTL);
 446 }
 447
 448 static unsigned int i965_gtt_total_entries(void)
 449 {
 450         int size;
 451         u32 pgetbl_ctl;
 452         u16 gmch_ctl;
 453
 454         pci_read_config_word(intel_private.bridge_dev,
 455                              I830_GMCH_CTRL, &gmch_ctl);
 456
 457         if (INTEL_GTT_GEN == 5) {
 458                 switch (gmch_ctl & G4x_GMCH_SIZE_MASK) {
 459                 case G4x_GMCH_SIZE_1M:
 460                 case G4x_GMCH_SIZE_VT_1M:
 461                         i965_adjust_pgetbl_size(I965_PGETBL_SIZE_1MB);
 462                         break;
 463                 case G4x_GMCH_SIZE_VT_1_5M:
 464                         i965_adjust_pgetbl_size(I965_PGETBL_SIZE_1_5MB);
 465                         break;
 466                 case G4x_GMCH_SIZE_2M:
 467                 case G4x_GMCH_SIZE_VT_2M:
 468                         i965_adjust_pgetbl_size(I965_PGETBL_SIZE_2MB);
 469                         break;
 470                 }
 471         }
 472
 473         pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL);
 474
 475         switch (pgetbl_ctl & I965_PGETBL_SIZE_MASK) {
 476         case I965_PGETBL_SIZE_128KB:
 477                 size = KB(128);
 478                 break;
 479         case I965_PGETBL_SIZE_256KB:
 480                 size = KB(256);
 481                 break;
 482         case I965_PGETBL_SIZE_512KB:
 483                 size = KB(512);
 484                 break;
 485         /* GTT pagetable sizes bigger than 512KB are not possible on G33! */
 486         case I965_PGETBL_SIZE_1MB:
 487                 size = KB(1024);
 488                 break;
 489         case I965_PGETBL_SIZE_2MB:
 490                 size = KB(2048);
 491                 break;
 492         case I965_PGETBL_SIZE_1_5MB:
 493                 size = KB(1024 + 512);
 494                 break;
 495         default:
 496                 dev_info(&intel_private.pcidev->dev,
 497                          "unknown page table size, assuming 512KB\n");
 498                 size = KB(512);
 499         }
 500
 501         return size/4;
 502 }
 503
 504 static unsigned int intel_gtt_total_entries(void)
 505 {
 506         if (IS_G33 || INTEL_GTT_GEN == 4 || INTEL_GTT_GEN == 5)
 507                 return i965_gtt_total_entries();
 508         else {
 509                 /* On previous hardware, the GTT size was just what was
 510                  * required to map the aperture.
 511                  */
 512                 return intel_private.base.gtt_mappable_entries;
 513         }
 514 }
 515
 516 static unsigned int intel_gtt_mappable_entries(void)
 517 {
 518         unsigned int aperture_size;
 519
 520         if (INTEL_GTT_GEN == 1) {
 521                 u32 smram_miscc;
 522
 523                 pci_read_config_dword(intel_private.bridge_dev,
 524                                       I810_SMRAM_MISCC, &smram_miscc);
 525
 526                 if ((smram_miscc & I810_GFX_MEM_WIN_SIZE)
 527                                 == I810_GFX_MEM_WIN_32M)
 528                         aperture_size = MB(32);
 529                 else
 530                         aperture_size = MB(64);
 531         } else if (INTEL_GTT_GEN == 2) {
 532                 u16 gmch_ctrl;
 533
 534                 pci_read_config_word(intel_private.bridge_dev,
 535                                      I830_GMCH_CTRL, &gmch_ctrl);
 536
 537                 if ((gmch_ctrl & I830_GMCH_MEM_MASK) == I830_GMCH_MEM_64M)
 538                         aperture_size = MB(64);
 539                 else
 540                         aperture_size = MB(128);
 541         } else {
 542                 /* 9xx supports large sizes, just look at the length */
 543                 aperture_size = pci_resource_len(intel_private.pcidev, 2);
 544         }
 545
 546         return aperture_size >> PAGE_SHIFT;
 547 }
 548
 549 static void intel_gtt_teardown_scratch_page(void)
 550 {
 551         set_pages_wb(intel_private.scratch_page, 1);
 552         pci_unmap_page(intel_private.pcidev, intel_private.scratch_page_dma,
 553                        PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 554         put_page(intel_private.scratch_page);
 555         __free_page(intel_private.scratch_page);
 556 }
 557
 558 static void intel_gtt_cleanup(void)
 559 {
 560         intel_private.driver->cleanup();
 561
 562         iounmap(intel_private.gtt);
 563         iounmap(intel_private.registers);
 564
 565         intel_gtt_teardown_scratch_page();
 566 }
 567
 568 static int intel_gtt_init(void)
 569 {
 570         u32 gma_addr;
 571         u32 gtt_map_size;
 572         int ret;
 573
 574         ret = intel_private.driver->setup();
 575         if (ret != 0)
 576                 return ret;
 577
 578         intel_private.base.gtt_mappable_entries = intel_gtt_mappable_entries();
 579         intel_private.base.gtt_total_entries = intel_gtt_total_entries();
 580
 581         /* save the PGETBL reg for resume */
 582         intel_private.PGETBL_save =
 583                 readl(intel_private.registers+I810_PGETBL_CTL)
 584                         & ~I810_PGETBL_ENABLED;
 585         /* we only ever restore the register when enabling the PGTBL... */
 586         if (HAS_PGTBL_EN)
 587                 intel_private.PGETBL_save |= I810_PGETBL_ENABLED;
 588
 589         dev_info(&intel_private.bridge_dev->dev,
 590                         "detected gtt size: %dK total, %dK mappable\n",
 591                         intel_private.base.gtt_total_entries * 4,
 592                         intel_private.base.gtt_mappable_entries * 4);
 593
 594         gtt_map_size = intel_private.base.gtt_total_entries * 4;
 595
 596         intel_private.gtt = NULL;
 597         if (INTEL_GTT_GEN < 6 && INTEL_GTT_GEN > 2)
 598                 intel_private.gtt = ioremap_wc(intel_private.gtt_bus_addr,
 599                                                gtt_map_size);
 600         if (intel_private.gtt == NULL)
 601                 intel_private.gtt = ioremap(intel_private.gtt_bus_addr,
 602                                             gtt_map_size);
 603         if (intel_private.gtt == NULL) {
 604                 intel_private.driver->cleanup();
 605                 iounmap(intel_private.registers);
 606                 return -ENOMEM;
 607         }
 608
 609         global_cache_flush();   /* FIXME: ? */
 610
 611         intel_private.base.stolen_size = intel_gtt_stolen_size();
 612
 613         intel_private.needs_dmar = USE_PCI_DMA_API && INTEL_GTT_GEN > 2;
 614
 615         ret = intel_gtt_setup_scratch_page();
 616         if (ret != 0) {
 617                 intel_gtt_cleanup();
 618                 return ret;
 619         }
 620
 621         if (INTEL_GTT_GEN <= 2)
 622                 pci_read_config_dword(intel_private.pcidev, I810_GMADDR,
 623                                       &gma_addr);
 624         else
 625                 pci_read_config_dword(intel_private.pcidev, I915_GMADDR,
 626                                       &gma_addr);
 627
 628         intel_private.base.gma_bus_addr = (gma_addr & PCI_BASE_ADDRESS_MEM_MASK);
 629
 630         return 0;
 631 }
 632
 633 static int intel_fake_agp_fetch_size(void)
 634 {
 635         int num_sizes = ARRAY_SIZE(intel_fake_agp_sizes);
 636         unsigned int aper_size;
 637         int i;
 638
 639         aper_size = (intel_private.base.gtt_mappable_entries << PAGE_SHIFT)
 640                     / MB(1);
 641
 642         for (i = 0; i < num_sizes; i++) {
 643                 if (aper_size == intel_fake_agp_sizes[i].size) {
 644                         agp_bridge->current_size =
 645                                 (void *) (intel_fake_agp_sizes + i);
 646                         return aper_size;
 647                 }
 648         }
 649
 650         return 0;
 651 }
 652
 653 static void i830_cleanup(void)
 654 {
 655 }
 656
 657 /* The chipset_flush interface needs to get data that has already been
 658  * flushed out of the CPU all the way out to main memory, because the GPU
 659  * doesn't snoop those buffers.
 660  *
 661  * The 8xx series doesn't have the same lovely interface for flushing the
 662  * chipset write buffers that the later chips do. According to the 865
 663  * specs, it's 64 octwords, or 1KB.  So, to get those previous things in
 664  * that buffer out, we just fill 1KB and clflush it out, on the assumption
 665  * that it'll push whatever was in there out.  It appears to work.
 666  */
 667 static void i830_chipset_flush(void)
 668 {
 669         unsigned long timeout = jiffies + msecs_to_jiffies(1000);
 670
 671         /* Forcibly evict everything from the CPU write buffers.
 672          * clflush appears to be insufficient.
 673          */
 674         wbinvd_on_all_cpus();
 675
 676         /* Now we've only seen documents for this magic bit on 855GM,
 677          * we hope it exists for the other gen2 chipsets...
 678          *
 679          * Also works as advertised on my 845G.
 680          */
 681         writel(readl(intel_private.registers+I830_HIC) | (1<<31),
 682                intel_private.registers+I830_HIC);
 683
 684         while (readl(intel_private.registers+I830_HIC) & (1<<31)) {
 685                 if (time_after(jiffies, timeout))
 686                         break;
 687
 688                 udelay(50);
 689         }
 690 }
 691
 692 static void i830_write_entry(dma_addr_t addr, unsigned int entry,
 693                              unsigned int flags)
 694 {
 695         u32 pte_flags = I810_PTE_VALID;
 696
 697         if (flags ==  AGP_USER_CACHED_MEMORY)
 698                 pte_flags |= I830_PTE_SYSTEM_CACHED;
 699
 700         writel(addr | pte_flags, intel_private.gtt + entry);
 701 }
 702
 703 bool intel_enable_gtt(void)
 704 {
 705         u8 __iomem *reg;
 706
 707         if (INTEL_GTT_GEN == 2) {
 708                 u16 gmch_ctrl;
 709
 710                 pci_read_config_word(intel_private.bridge_dev,
 711                                      I830_GMCH_CTRL, &gmch_ctrl);
 712                 gmch_ctrl |= I830_GMCH_ENABLED;
 713                 pci_write_config_word(intel_private.bridge_dev,
 714                                       I830_GMCH_CTRL, gmch_ctrl);
 715
 716                 pci_read_config_word(intel_private.bridge_dev,
 717                                      I830_GMCH_CTRL, &gmch_ctrl);
 718                 if ((gmch_ctrl & I830_GMCH_ENABLED) == 0) {
 719                         dev_err(&intel_private.pcidev->dev,
 720                                 "failed to enable the GTT: GMCH_CTRL=%x\n",
 721                                 gmch_ctrl);
 722                         return false;
 723                 }
 724         }
 725
 726         /* On the resume path we may be adjusting the PGTBL value, so
 727          * be paranoid and flush all chipset write buffers...
 728          */
 729         if (INTEL_GTT_GEN >= 3)
 730                 writel(0, intel_private.registers+GFX_FLSH_CNTL);
 731
 732         reg = intel_private.registers+I810_PGETBL_CTL;
 733         writel(intel_private.PGETBL_save, reg);
 734         if (HAS_PGTBL_EN && (readl(reg) & I810_PGETBL_ENABLED) == 0) {
 735                 dev_err(&intel_private.pcidev->dev,
 736                         "failed to enable the GTT: PGETBL=%x [expected %x]\n",
 737                         readl(reg), intel_private.PGETBL_save);
 738                 return false;
 739         }
 740
 741         if (INTEL_GTT_GEN >= 3)
 742                 writel(0, intel_private.registers+GFX_FLSH_CNTL);
 743
 744         return true;
 745 }
 746 EXPORT_SYMBOL(intel_enable_gtt);
 747
 748 static int i830_setup(void)
 749 {
 750         u32 reg_addr;
 751
 752         pci_read_config_dword(intel_private.pcidev, I810_MMADDR, &reg_addr);
 753         reg_addr &= 0xfff80000;
 754
 755         intel_private.registers = ioremap(reg_addr, KB(64));
 756         if (!intel_private.registers)
 757                 return -ENOMEM;
 758
 759         intel_private.gtt_bus_addr = reg_addr + I810_PTE_BASE;
 760
 761         return 0;
 762 }
 763
 764 static int intel_fake_agp_create_gatt_table(struct agp_bridge_data *bridge)
 765 {
 766         agp_bridge->gatt_table_real = NULL;
 767         agp_bridge->gatt_table = NULL;
 768         agp_bridge->gatt_bus_addr = 0;
 769
 770         return 0;
 771 }
 772
 773 static int intel_fake_agp_free_gatt_table(struct agp_bridge_data *bridge)
 774 {
 775         return 0;
 776 }
 777
 778 static int intel_fake_agp_configure(void)
 779 {
 780         if (!intel_enable_gtt())
 781             return -EIO;
 782
 783         intel_private.clear_fake_agp = true;
 784         agp_bridge->gart_bus_addr = intel_private.base.gma_bus_addr;
 785
 786         return 0;
 787 }
 788
 789 static bool i830_check_flags(unsigned int flags)
 790 {
 791         switch (flags) {
 792         case 0:
 793         case AGP_PHYS_MEMORY:
 794         case AGP_USER_CACHED_MEMORY:
 795         case AGP_USER_MEMORY:
 796                 return true;
 797         }
 798
 799         return false;
 800 }
 801
 802 void intel_gtt_insert_sg_entries(struct sg_table *st,
 803                                  unsigned int pg_start,
 804                                  unsigned int flags)
 805 {
 806         struct scatterlist *sg;
 807         unsigned int len, m;
 808         int i, j;
 809
 810         j = pg_start;
 811
 812         /* sg may merge pages, but we have to separate
 813          * per-page addr for GTT */
 814         for_each_sg(st->sgl, sg, st->nents, i) {
 815                 len = sg_dma_len(sg) >> PAGE_SHIFT;
 816                 for (m = 0; m < len; m++) {
 817                         dma_addr_t addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
 818                         intel_private.driver->write_entry(addr, j, flags);
 819                         j++;
 820                 }
 821         }
 822         readl(intel_private.gtt+j-1);
 823 }
 824 EXPORT_SYMBOL(intel_gtt_insert_sg_entries);
 825
 826 static void intel_gtt_insert_pages(unsigned int first_entry,
 827                                    unsigned int num_entries,
 828                                    struct page **pages,
 829                                    unsigned int flags)
 830 {
 831         int i, j;
 832
 833         for (i = 0, j = first_entry; i < num_entries; i++, j++) {
 834                 dma_addr_t addr = page_to_phys(pages[i]);
 835                 intel_private.driver->write_entry(addr,
 836                                                   j, flags);
 837         }
 838         readl(intel_private.gtt+j-1);
 839 }
 840
 841 static int intel_fake_agp_insert_entries(struct agp_memory *mem,
 842                                          off_t pg_start, int type)
 843 {
 844         int ret = -EINVAL;
 845
 846         if (intel_private.clear_fake_agp) {
 847                 int start = intel_private.base.stolen_size / PAGE_SIZE;
 848                 int end = intel_private.base.gtt_mappable_entries;
 849                 intel_gtt_clear_range(start, end - start);
 850                 intel_private.clear_fake_agp = false;
 851         }
 852
 853         if (INTEL_GTT_GEN == 1 && type == AGP_DCACHE_MEMORY)
 854                 return i810_insert_dcache_entries(mem, pg_start, type);
 855
 856         if (mem->page_count == 0)
 857                 goto out;
 858
 859         if (pg_start + mem->page_count > intel_private.base.gtt_total_entries)
 860                 goto out_err;
 861
 862         if (type != mem->type)
 863                 goto out_err;
 864
 865         if (!intel_private.driver->check_flags(type))
 866                 goto out_err;
 867
 868         if (!mem->is_flushed)
 869                 global_cache_flush();
 870
 871         if (intel_private.needs_dmar) {
 872                 struct sg_table st;
 873
 874                 ret = intel_gtt_map_memory(mem->pages, mem->page_count, &st);
 875                 if (ret != 0)
 876                         return ret;
 877
 878                 intel_gtt_insert_sg_entries(&st, pg_start, type);
 879                 mem->sg_list = st.sgl;
 880                 mem->num_sg = st.nents;
 881         } else
 882                 intel_gtt_insert_pages(pg_start, mem->page_count, mem->pages,
 883                                        type);
 884
 885 out:
 886         ret = 0;
 887 out_err:
 888         mem->is_flushed = true;
 889         return ret;
 890 }
 891
 892 void intel_gtt_clear_range(unsigned int first_entry, unsigned int num_entries)
 893 {
 894         unsigned int i;
 895
 896         for (i = first_entry; i < (first_entry + num_entries); i++) {
 897                 intel_private.driver->write_entry(intel_private.scratch_page_dma,
 898                                                   i, 0);
 899         }
 900         readl(intel_private.gtt+i-1);
 901 }
 902 EXPORT_SYMBOL(intel_gtt_clear_range);
 903
 904 static int intel_fake_agp_remove_entries(struct agp_memory *mem,
 905                                          off_t pg_start, int type)
 906 {
 907         if (mem->page_count == 0)
 908                 return 0;
 909
 910         intel_gtt_clear_range(pg_start, mem->page_count);
 911
 912         if (intel_private.needs_dmar) {
 913                 intel_gtt_unmap_memory(mem->sg_list, mem->num_sg);
 914                 mem->sg_list = NULL;
 915                 mem->num_sg = 0;
 916         }
 917
 918         return 0;
 919 }
 920
 921 static struct agp_memory *intel_fake_agp_alloc_by_type(size_t pg_count,
 922                                                        int type)
 923 {
 924         struct agp_memory *new;
 925
 926         if (type == AGP_DCACHE_MEMORY && INTEL_GTT_GEN == 1) {
 927                 if (pg_count != intel_private.num_dcache_entries)
 928                         return NULL;
 929
 930                 new = agp_create_memory(1);
 931                 if (new == NULL)
 932                         return NULL;
 933
 934                 new->type = AGP_DCACHE_MEMORY;
 935                 new->page_count = pg_count;
 936                 new->num_scratch_pages = 0;
 937                 agp_free_page_array(new);
 938                 return new;
 939         }
 940         if (type == AGP_PHYS_MEMORY)
 941                 return alloc_agpphysmem_i8xx(pg_count, type);
 942         /* always return NULL for other allocation types for now */
 943         return NULL;
 944 }
 945
 946 static int intel_alloc_chipset_flush_resource(void)
 947 {
 948         int ret;
 949         ret = pci_bus_alloc_resource(intel_private.bridge_dev->bus, &intel_private.ifp_resource, PAGE_SIZE,
 950                                      PAGE_SIZE, PCIBIOS_MIN_MEM, 0,
 951                                      pcibios_align_resource, intel_private.bridge_dev);
 952
 953         return ret;
 954 }
 955
 956 static void intel_i915_setup_chipset_flush(void)
 957 {
 958         int ret;
 959         u32 temp;
 960
 961         pci_read_config_dword(intel_private.bridge_dev, I915_IFPADDR, &temp);
 962         if (!(temp & 0x1)) {
 963                 intel_alloc_chipset_flush_resource();
 964                 intel_private.resource_valid = 1;
 965                 pci_write_config_dword(intel_private.bridge_dev, I915_IFPADDR, (intel_private.ifp_resource.start & 0xffffffff) | 0x1);
 966         } else {
 967                 temp &= ~1;
 968
 969                 intel_private.resource_valid = 1;
 970                 intel_private.ifp_resource.start = temp;
 971                 intel_private.ifp_resource.end = temp + PAGE_SIZE;
 972                 ret = request_resource(&iomem_resource, &intel_private.ifp_resource);
 973                 /* some BIOSes reserve this area in a pnp some don't */
 974                 if (ret)
 975                         intel_private.resource_valid = 0;
 976         }
 977 }
 978
 979 static void intel_i965_g33_setup_chipset_flush(void)
 980 {
 981         u32 temp_hi, temp_lo;
 982         int ret;
 983
 984         pci_read_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4, &temp_hi);
 985         pci_read_config_dword(intel_private.bridge_dev, I965_IFPADDR, &temp_lo);
 986
 987         if (!(temp_lo & 0x1)) {
 988
 989                 intel_alloc_chipset_flush_resource();
 990
 991                 intel_private.resource_valid = 1;
 992                 pci_write_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4,
 993                         upper_32_bits(intel_private.ifp_resource.start));
 994                 pci_write_config_dword(intel_private.bridge_dev, I965_IFPADDR, (intel_private.ifp_resource.start & 0xffffffff) | 0x1);
 995         } else {
 996                 u64 l64;
 997
 998                 temp_lo &= ~0x1;
 999                 l64 = ((u64)temp_hi << 32) | temp_lo;
1000
1001                 intel_private.resource_valid = 1;
1002                 intel_private.ifp_resource.start = l64;
1003                 intel_private.ifp_resource.end = l64 + PAGE_SIZE;
1004                 ret = request_resource(&iomem_resource, &intel_private.ifp_resource);
1005                 /* some BIOSes reserve this area in a pnp some don't */
1006                 if (ret)
1007                         intel_private.resource_valid = 0;
1008         }
1009 }
1010
1011 static void intel_i9xx_setup_flush(void)
1012 {
1013         /* return if already configured */
1014         if (intel_private.ifp_resource.start)
1015                 return;
1016
1017         if (INTEL_GTT_GEN == 6)
1018                 return;
1019
1020         /* setup a resource for this object */
1021         intel_private.ifp_resource.name = "Intel Flush Page";
1022         intel_private.ifp_resource.flags = IORESOURCE_MEM;
1023
1024         /* Setup chipset flush for 915 */
1025         if (IS_G33 || INTEL_GTT_GEN >= 4) {
1026                 intel_i965_g33_setup_chipset_flush();
1027         } else {
1028                 intel_i915_setup_chipset_flush();
1029         }
1030
1031         if (intel_private.ifp_resource.start)
1032                 intel_private.i9xx_flush_page = ioremap_nocache(intel_private.ifp_resource.start, PAGE_SIZE);
1033         if (!intel_private.i9xx_flush_page)
1034                 dev_err(&intel_private.pcidev->dev,
1035                         "can't ioremap flush page - no chipset flushing\n");
1036 }
1037
1038 static void i9xx_cleanup(void)
1039 {
1040         if (intel_private.i9xx_flush_page)
1041                 iounmap(intel_private.i9xx_flush_page);
1042         if (intel_private.resource_valid)
1043                 release_resource(&intel_private.ifp_resource);
1044         intel_private.ifp_resource.start = 0;
1045         intel_private.resource_valid = 0;
1046 }
1047
1048 static void i9xx_chipset_flush(void)
1049 {
1050         if (intel_private.i9xx_flush_page)
1051                 writel(1, intel_private.i9xx_flush_page);
1052 }
1053
1054 static void i965_write_entry(dma_addr_t addr,
1055                              unsigned int entry,
1056                              unsigned int flags)
1057 {
1058         u32 pte_flags;
1059
1060         pte_flags = I810_PTE_VALID;
1061         if (flags == AGP_USER_CACHED_MEMORY)
1062                 pte_flags |= I830_PTE_SYSTEM_CACHED;
1063
1064         /* Shift high bits down */
1065         addr |= (addr >> 28) & 0xf0;
1066         writel(addr | pte_flags, intel_private.gtt + entry);
1067 }
1068
1069
1070 static int i9xx_setup(void)
1071 {
1072         u32 reg_addr, gtt_addr;
1073         int size = KB(512);
1074
1075         pci_read_config_dword(intel_private.pcidev, I915_MMADDR, &reg_addr);
1076
1077         reg_addr &= 0xfff80000;
1078
1079         intel_private.registers = ioremap(reg_addr, size);
1080         if (!intel_private.registers)
1081                 return -ENOMEM;
1082
1083         switch (INTEL_GTT_GEN) {
1084         case 3:
1085                 pci_read_config_dword(intel_private.pcidev,
1086                                       I915_PTEADDR, &gtt_addr);
1087                 intel_private.gtt_bus_addr = gtt_addr;
1088                 break;
1089         case 5:
1090                 intel_private.gtt_bus_addr = reg_addr + MB(2);
1091                 break;
1092         default:
1093                 intel_private.gtt_bus_addr = reg_addr + KB(512);
1094                 break;
1095         }
1096
1097         intel_i9xx_setup_flush();
1098
1099         return 0;
1100 }
1101
1102 static const struct agp_bridge_driver intel_fake_agp_driver = {
1103         .owner                  = THIS_MODULE,
1104         .size_type              = FIXED_APER_SIZE,
1105         .aperture_sizes         = intel_fake_agp_sizes,
1106         .num_aperture_sizes     = ARRAY_SIZE(intel_fake_agp_sizes),
1107         .configure              = intel_fake_agp_configure,
1108         .fetch_size             = intel_fake_agp_fetch_size,
1109         .cleanup                = intel_gtt_cleanup,
1110         .agp_enable             = intel_fake_agp_enable,
1111         .cache_flush            = global_cache_flush,
1112         .create_gatt_table      = intel_fake_agp_create_gatt_table,
1113         .free_gatt_table        = intel_fake_agp_free_gatt_table,
1114         .insert_memory          = intel_fake_agp_insert_entries,
1115         .remove_memory          = intel_fake_agp_remove_entries,
1116         .alloc_by_type          = intel_fake_agp_alloc_by_type,
1117         .free_by_type           = intel_i810_free_by_type,
1118         .agp_alloc_page         = agp_generic_alloc_page,
1119         .agp_alloc_pages        = agp_generic_alloc_pages,
1120         .agp_destroy_page       = agp_generic_destroy_page,
1121         .agp_destroy_pages      = agp_generic_destroy_pages,
1122 };
1123
1124 static const struct intel_gtt_driver i81x_gtt_driver = {
1125         .gen = 1,
1126         .has_pgtbl_enable = 1,
1127         .dma_mask_size = 32,
1128         .setup = i810_setup,
1129         .cleanup = i810_cleanup,
1130         .check_flags = i830_check_flags,
1131         .write_entry = i810_write_entry,
1132 };
1133 static const struct intel_gtt_driver i8xx_gtt_driver = {
1134         .gen = 2,
1135         .has_pgtbl_enable = 1,
1136         .setup = i830_setup,
1137         .cleanup = i830_cleanup,
1138         .write_entry = i830_write_entry,
1139         .dma_mask_size = 32,
1140         .check_flags = i830_check_flags,
1141         .chipset_flush = i830_chipset_flush,
1142 };
1143 static const struct intel_gtt_driver i915_gtt_driver = {
1144         .gen = 3,
1145         .has_pgtbl_enable = 1,
1146         .setup = i9xx_setup,
1147         .cleanup = i9xx_cleanup,
1148         /* i945 is the last gpu to need phys mem (for overlay and cursors). */
1149         .write_entry = i830_write_entry,
1150         .dma_mask_size = 32,
1151         .check_flags = i830_check_flags,
1152         .chipset_flush = i9xx_chipset_flush,
1153 };
1154 static const struct intel_gtt_driver g33_gtt_driver = {
1155         .gen = 3,
1156         .is_g33 = 1,
1157         .setup = i9xx_setup,
1158         .cleanup = i9xx_cleanup,
1159         .write_entry = i965_write_entry,
1160         .dma_mask_size = 36,
1161         .check_flags = i830_check_flags,
1162         .chipset_flush = i9xx_chipset_flush,
1163 };
1164 static const struct intel_gtt_driver pineview_gtt_driver = {
1165         .gen = 3,
1166         .is_pineview = 1, .is_g33 = 1,
1167         .setup = i9xx_setup,
1168         .cleanup = i9xx_cleanup,
1169         .write_entry = i965_write_entry,
1170         .dma_mask_size = 36,
1171         .check_flags = i830_check_flags,
1172         .chipset_flush = i9xx_chipset_flush,
1173 };
1174 static const struct intel_gtt_driver i965_gtt_driver = {
1175         .gen = 4,
1176         .has_pgtbl_enable = 1,
1177         .setup = i9xx_setup,
1178         .cleanup = i9xx_cleanup,
1179         .write_entry = i965_write_entry,
1180         .dma_mask_size = 36,
1181         .check_flags = i830_check_flags,
1182         .chipset_flush = i9xx_chipset_flush,
1183 };
1184 static const struct intel_gtt_driver g4x_gtt_driver = {
1185         .gen = 5,
1186         .setup = i9xx_setup,
1187         .cleanup = i9xx_cleanup,
1188         .write_entry = i965_write_entry,
1189         .dma_mask_size = 36,
1190         .check_flags = i830_check_flags,
1191         .chipset_flush = i9xx_chipset_flush,
1192 };
1193 static const struct intel_gtt_driver ironlake_gtt_driver = {
1194         .gen = 5,
1195         .is_ironlake = 1,
1196         .setup = i9xx_setup,
1197         .cleanup = i9xx_cleanup,
1198         .write_entry = i965_write_entry,
1199         .dma_mask_size = 36,
1200         .check_flags = i830_check_flags,
1201         .chipset_flush = i9xx_chipset_flush,
1202 };
1203
1204 /* Table to describe Intel GMCH and AGP/PCIE GART drivers.  At least one of
1205  * driver and gmch_driver must be non-null, and find_gmch will determine
1206  * which one should be used if a gmch_chip_id is present.
1207  */
1208 static const struct intel_gtt_driver_description {
1209         unsigned int gmch_chip_id;
1210         char *name;
1211         const struct intel_gtt_driver *gtt_driver;
1212 } intel_gtt_chipsets[] = {
1213         { PCI_DEVICE_ID_INTEL_82810_IG1, "i810",
1214                 &i81x_gtt_driver},
1215         { PCI_DEVICE_ID_INTEL_82810_IG3, "i810",
1216                 &i81x_gtt_driver},
1217         { PCI_DEVICE_ID_INTEL_82810E_IG, "i810",
1218                 &i81x_gtt_driver},
1219         { PCI_DEVICE_ID_INTEL_82815_CGC, "i815",
1220                 &i81x_gtt_driver},
1221         { PCI_DEVICE_ID_INTEL_82830_CGC, "830M",
1222                 &i8xx_gtt_driver},
1223         { PCI_DEVICE_ID_INTEL_82845G_IG, "845G",
1224                 &i8xx_gtt_driver},
1225         { PCI_DEVICE_ID_INTEL_82854_IG, "854",
1226                 &i8xx_gtt_driver},
1227         { PCI_DEVICE_ID_INTEL_82855GM_IG, "855GM",
1228                 &i8xx_gtt_driver},
1229         { PCI_DEVICE_ID_INTEL_82865_IG, "865",
1230                 &i8xx_gtt_driver},
1231         { PCI_DEVICE_ID_INTEL_E7221_IG, "E7221 (i915)",
1232                 &i915_gtt_driver },
1233         { PCI_DEVICE_ID_INTEL_82915G_IG, "915G",
1234                 &i915_gtt_driver },
1235         { PCI_DEVICE_ID_INTEL_82915GM_IG, "915GM",
1236                 &i915_gtt_driver },
1237         { PCI_DEVICE_ID_INTEL_82945G_IG, "945G",
1238                 &i915_gtt_driver },
1239         { PCI_DEVICE_ID_INTEL_82945GM_IG, "945GM",
1240                 &i915_gtt_driver },
1241         { PCI_DEVICE_ID_INTEL_82945GME_IG, "945GME",
1242                 &i915_gtt_driver },
1243         { PCI_DEVICE_ID_INTEL_82946GZ_IG, "946GZ",
1244                 &i965_gtt_driver },
1245         { PCI_DEVICE_ID_INTEL_82G35_IG, "G35",
1246                 &i965_gtt_driver },
1247         { PCI_DEVICE_ID_INTEL_82965Q_IG, "965Q",
1248                 &i965_gtt_driver },
1249         { PCI_DEVICE_ID_INTEL_82965G_IG, "965G",
1250                 &i965_gtt_driver },
1251         { PCI_DEVICE_ID_INTEL_82965GM_IG, "965GM",
1252                 &i965_gtt_driver },
1253         { PCI_DEVICE_ID_INTEL_82965GME_IG, "965GME/GLE",
1254                 &i965_gtt_driver },
1255         { PCI_DEVICE_ID_INTEL_G33_IG, "G33",
1256                 &g33_gtt_driver },
1257         { PCI_DEVICE_ID_INTEL_Q35_IG, "Q35",
1258                 &g33_gtt_driver },
1259         { PCI_DEVICE_ID_INTEL_Q33_IG, "Q33",
1260                 &g33_gtt_driver },
1261         { PCI_DEVICE_ID_INTEL_PINEVIEW_M_IG, "GMA3150",
1262                 &pineview_gtt_driver },
1263         { PCI_DEVICE_ID_INTEL_PINEVIEW_IG, "GMA3150",
1264                 &pineview_gtt_driver },
1265         { PCI_DEVICE_ID_INTEL_GM45_IG, "GM45",
1266                 &g4x_gtt_driver },
1267         { PCI_DEVICE_ID_INTEL_EAGLELAKE_IG, "Eaglelake",
1268                 &g4x_gtt_driver },
1269         { PCI_DEVICE_ID_INTEL_Q45_IG, "Q45/Q43",
1270                 &g4x_gtt_driver },
1271         { PCI_DEVICE_ID_INTEL_G45_IG, "G45/G43",
1272                 &g4x_gtt_driver },
1273         { PCI_DEVICE_ID_INTEL_B43_IG, "B43",
1274                 &g4x_gtt_driver },
1275         { PCI_DEVICE_ID_INTEL_B43_1_IG, "B43",
1276                 &g4x_gtt_driver },
1277         { PCI_DEVICE_ID_INTEL_G41_IG, "G41",
1278                 &g4x_gtt_driver },
1279         { PCI_DEVICE_ID_INTEL_IRONLAKE_D_IG,
1280             "HD Graphics", &ironlake_gtt_driver },
1281         { PCI_DEVICE_ID_INTEL_IRONLAKE_M_IG,
1282             "HD Graphics", &ironlake_gtt_driver },
1283         { 0, NULL, NULL }
1284 };
1285
1286 static int find_gmch(u16 device)
1287 {
1288         struct pci_dev *gmch_device;
1289
1290         gmch_device = pci_get_device(PCI_VENDOR_ID_INTEL, device, NULL);
1291         if (gmch_device && PCI_FUNC(gmch_device->devfn) != 0) {
1292                 gmch_device = pci_get_device(PCI_VENDOR_ID_INTEL,
1293                                              device, gmch_device);
1294         }
1295
1296         if (!gmch_device)
1297                 return 0;
1298
1299         intel_private.pcidev = gmch_device;
1300         return 1;
1301 }
1302
1303 int intel_gmch_probe(struct pci_dev *bridge_pdev, struct pci_dev *gpu_pdev,
1304                      struct agp_bridge_data *bridge)
1305 {
1306         int i, mask;
1307
1308         /*
1309          * Can be called from the fake agp driver but also directly from
1310          * drm/i915.ko. Hence we need to check whether everything is set up
1311          * already.
1312          */
1313         if (intel_private.driver) {
1314                 intel_private.refcount++;
1315                 return 1;
1316         }
1317
1318         for (i = 0; intel_gtt_chipsets[i].name != NULL; i++) {
1319                 if (gpu_pdev) {
1320                         if (gpu_pdev->device ==
1321                             intel_gtt_chipsets[i].gmch_chip_id) {
1322                                 intel_private.pcidev = pci_dev_get(gpu_pdev);
1323                                 intel_private.driver =
1324                                         intel_gtt_chipsets[i].gtt_driver;
1325
1326                                 break;
1327                         }
1328                 } else if (find_gmch(intel_gtt_chipsets[i].gmch_chip_id)) {
1329                         intel_private.driver =
1330                                 intel_gtt_chipsets[i].gtt_driver;
1331                         break;
1332                 }
1333         }
1334
1335         if (!intel_private.driver)
1336                 return 0;
1337
1338         intel_private.refcount++;
1339
1340         if (bridge) {
1341                 bridge->driver = &intel_fake_agp_driver;
1342                 bridge->dev_private_data = &intel_private;
1343                 bridge->dev = bridge_pdev;
1344         }
1345
1346         intel_private.bridge_dev = pci_dev_get(bridge_pdev);
1347
1348         dev_info(&bridge_pdev->dev, "Intel %s Chipset\n", intel_gtt_chipsets[i].name);
1349
1350         mask = intel_private.driver->dma_mask_size;
1351         if (pci_set_dma_mask(intel_private.pcidev, DMA_BIT_MASK(mask)))
1352                 dev_err(&intel_private.pcidev->dev,
1353                         "set gfx device dma mask %d-bit failed!\n", mask);
1354         else
1355                 pci_set_consistent_dma_mask(intel_private.pcidev,
1356                                             DMA_BIT_MASK(mask));
1357
1358         if (intel_gtt_init() != 0) {
1359                 intel_gmch_remove();
1360
1361                 return 0;
1362         }
1363
1364         return 1;
1365 }
1366 EXPORT_SYMBOL(intel_gmch_probe);
1367
1368 struct intel_gtt *intel_gtt_get(void)
1369 {
1370         return &intel_private.base;
1371 }
1372 EXPORT_SYMBOL(intel_gtt_get);
1373
1374 void intel_gtt_chipset_flush(void)
1375 {
1376         if (intel_private.driver->chipset_flush)
1377                 intel_private.driver->chipset_flush();
1378 }
1379 EXPORT_SYMBOL(intel_gtt_chipset_flush);
1380
1381 void intel_gmch_remove(void)
1382 {
1383         if (--intel_private.refcount)
1384                 return;
1385
1386         if (intel_private.pcidev)
1387                 pci_dev_put(intel_private.pcidev);
1388         if (intel_private.bridge_dev)
1389                 pci_dev_put(intel_private.bridge_dev);
1390         intel_private.driver = NULL;
1391 }
1392 EXPORT_SYMBOL(intel_gmch_remove);
1393
1394 MODULE_AUTHOR("Dave Jones <davej@redhat.com>");
1395 MODULE_LICENSE("GPL and additional rights");