1 /* 2 * Firmware replacement code. 3 * 4 * Work around broken BIOSes that don't set an aperture, only set the 5 * aperture in the AGP bridge, or set too small aperture. 6 * 7 * If all fails map the aperture over some low memory. This is cheaper than 8 * doing bounce buffering. The memory is lost. This is done at early boot 9 * because only the bootmem allocator can allocate 32+MB. 10 * 11 * Copyright 2002 Andi Kleen, SuSE Labs. 12 */ 13 #include <linux/kernel.h> 14 #include <linux/types.h> 15 #include <linux/init.h> 16 #include <linux/memblock.h> 17 #include <linux/mmzone.h> 18 #include <linux/pci_ids.h> 19 #include <linux/pci.h> 20 #include <linux/bitops.h> 21 #include <linux/suspend.h> 22 #include <asm/e820.h> 23 #include <asm/io.h> 24 #include <asm/iommu.h> 25 #include <asm/gart.h> 26 #include <asm/pci-direct.h> 27 #include <asm/dma.h> 28 #include <asm/amd_nb.h> 29 #include <asm/x86_init.h> 30 31 /* 32 * Using 512M as goal, in case kexec will load kernel_big 33 * that will do the on-position decompress, and could overlap with 34 * with the gart aperture that is used. 35 * Sequence: 36 * kernel_small 37 * ==> kexec (with kdump trigger path or gart still enabled) 38 * ==> kernel_small (gart area become e820_reserved) 39 * ==> kexec (with kdump trigger path or gart still enabled) 40 * ==> kerne_big (uncompressed size will be big than 64M or 128M) 41 * So don't use 512M below as gart iommu, leave the space for kernel 42 * code for safe. 43 */ 44 #define GART_MIN_ADDR (512ULL << 20) 45 #define GART_MAX_ADDR (1ULL << 32) 46 47 int gart_iommu_aperture; 48 int gart_iommu_aperture_disabled __initdata; 49 int gart_iommu_aperture_allowed __initdata; 50 51 int fallback_aper_order __initdata = 1; /* 64MB */ 52 int fallback_aper_force __initdata; 53 54 int fix_aperture __initdata = 1; 55 56 /* This code runs before the PCI subsystem is initialized, so just 57 access the northbridge directly. */ 58 59 static u32 __init allocate_aperture(void) 60 { 61 u32 aper_size; 62 unsigned long addr; 63 64 /* aper_size should <= 1G */ 65 if (fallback_aper_order > 5) 66 fallback_aper_order = 5; 67 aper_size = (32 * 1024 * 1024) << fallback_aper_order; 68 69 /* 70 * Aperture has to be naturally aligned. This means a 2GB aperture 71 * won't have much chance of finding a place in the lower 4GB of 72 * memory. Unfortunately we cannot move it up because that would 73 * make the IOMMU useless. 74 */ 75 addr = memblock_find_in_range(GART_MIN_ADDR, GART_MAX_ADDR, 76 aper_size, aper_size); 77 if (!addr) { 78 printk(KERN_ERR 79 "Cannot allocate aperture memory hole (%lx,%uK)\n", 80 addr, aper_size>>10); 81 return 0; 82 } 83 memblock_reserve(addr, aper_size); 84 printk(KERN_INFO "Mapping aperture over %d KB of RAM @ %lx\n", 85 aper_size >> 10, addr); 86 register_nosave_region(addr >> PAGE_SHIFT, 87 (addr+aper_size) >> PAGE_SHIFT); 88 89 return (u32)addr; 90 } 91 92 93 /* Find a PCI capability */ 94 static u32 __init find_cap(int bus, int slot, int func, int cap) 95 { 96 int bytes; 97 u8 pos; 98 99 if (!(read_pci_config_16(bus, slot, func, PCI_STATUS) & 100 PCI_STATUS_CAP_LIST)) 101 return 0; 102 103 pos = read_pci_config_byte(bus, slot, func, PCI_CAPABILITY_LIST); 104 for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) { 105 u8 id; 106 107 pos &= ~3; 108 id = read_pci_config_byte(bus, slot, func, pos+PCI_CAP_LIST_ID); 109 if (id == 0xff) 110 break; 111 if (id == cap) 112 return pos; 113 pos = read_pci_config_byte(bus, slot, func, 114 pos+PCI_CAP_LIST_NEXT); 115 } 116 return 0; 117 } 118 119 /* Read a standard AGPv3 bridge header */ 120 static u32 __init read_agp(int bus, int slot, int func, int cap, u32 *order) 121 { 122 u32 apsize; 123 u32 apsizereg; 124 int nbits; 125 u32 aper_low, aper_hi; 126 u64 aper; 127 u32 old_order; 128 129 printk(KERN_INFO "AGP bridge at %02x:%02x:%02x\n", bus, slot, func); 130 apsizereg = read_pci_config_16(bus, slot, func, cap + 0x14); 131 if (apsizereg == 0xffffffff) { 132 printk(KERN_ERR "APSIZE in AGP bridge unreadable\n"); 133 return 0; 134 } 135 136 /* old_order could be the value from NB gart setting */ 137 old_order = *order; 138 139 apsize = apsizereg & 0xfff; 140 /* Some BIOS use weird encodings not in the AGPv3 table. */ 141 if (apsize & 0xff) 142 apsize |= 0xf00; 143 nbits = hweight16(apsize); 144 *order = 7 - nbits; 145 if ((int)*order < 0) /* < 32MB */ 146 *order = 0; 147 148 aper_low = read_pci_config(bus, slot, func, 0x10); 149 aper_hi = read_pci_config(bus, slot, func, 0x14); 150 aper = (aper_low & ~((1<<22)-1)) | ((u64)aper_hi << 32); 151 152 /* 153 * On some sick chips, APSIZE is 0. It means it wants 4G 154 * so let double check that order, and lets trust AMD NB settings: 155 */ 156 printk(KERN_INFO "Aperture from AGP @ %Lx old size %u MB\n", 157 aper, 32 << old_order); 158 if (aper + (32ULL<<(20 + *order)) > 0x100000000ULL) { 159 printk(KERN_INFO "Aperture size %u MB (APSIZE %x) is not right, using settings from NB\n", 160 32 << *order, apsizereg); 161 *order = old_order; 162 } 163 164 printk(KERN_INFO "Aperture from AGP @ %Lx size %u MB (APSIZE %x)\n", 165 aper, 32 << *order, apsizereg); 166 167 if (!aperture_valid(aper, (32*1024*1024) << *order, 32<<20)) 168 return 0; 169 return (u32)aper; 170 } 171 172 /* 173 * Look for an AGP bridge. Windows only expects the aperture in the 174 * AGP bridge and some BIOS forget to initialize the Northbridge too. 175 * Work around this here. 176 * 177 * Do an PCI bus scan by hand because we're running before the PCI 178 * subsystem. 179 * 180 * All AMD AGP bridges are AGPv3 compliant, so we can do this scan 181 * generically. It's probably overkill to always scan all slots because 182 * the AGP bridges should be always an own bus on the HT hierarchy, 183 * but do it here for future safety. 184 */ 185 static u32 __init search_agp_bridge(u32 *order, int *valid_agp) 186 { 187 int bus, slot, func; 188 189 /* Poor man's PCI discovery */ 190 for (bus = 0; bus < 256; bus++) { 191 for (slot = 0; slot < 32; slot++) { 192 for (func = 0; func < 8; func++) { 193 u32 class, cap; 194 u8 type; 195 class = read_pci_config(bus, slot, func, 196 PCI_CLASS_REVISION); 197 if (class == 0xffffffff) 198 break; 199 200 switch (class >> 16) { 201 case PCI_CLASS_BRIDGE_HOST: 202 case PCI_CLASS_BRIDGE_OTHER: /* needed? */ 203 /* AGP bridge? */ 204 cap = find_cap(bus, slot, func, 205 PCI_CAP_ID_AGP); 206 if (!cap) 207 break; 208 *valid_agp = 1; 209 return read_agp(bus, slot, func, cap, 210 order); 211 } 212 213 /* No multi-function device? */ 214 type = read_pci_config_byte(bus, slot, func, 215 PCI_HEADER_TYPE); 216 if (!(type & 0x80)) 217 break; 218 } 219 } 220 } 221 printk(KERN_INFO "No AGP bridge found\n"); 222 223 return 0; 224 } 225 226 static int gart_fix_e820 __initdata = 1; 227 228 static int __init parse_gart_mem(char *p) 229 { 230 if (!p) 231 return -EINVAL; 232 233 if (!strncmp(p, "off", 3)) 234 gart_fix_e820 = 0; 235 else if (!strncmp(p, "on", 2)) 236 gart_fix_e820 = 1; 237 238 return 0; 239 } 240 early_param("gart_fix_e820", parse_gart_mem); 241 242 void __init early_gart_iommu_check(void) 243 { 244 /* 245 * in case it is enabled before, esp for kexec/kdump, 246 * previous kernel already enable that. memset called 247 * by allocate_aperture/__alloc_bootmem_nopanic cause restart. 248 * or second kernel have different position for GART hole. and new 249 * kernel could use hole as RAM that is still used by GART set by 250 * first kernel 251 * or BIOS forget to put that in reserved. 252 * try to update e820 to make that region as reserved. 253 */ 254 u32 agp_aper_order = 0; 255 int i, fix, slot, valid_agp = 0; 256 u32 ctl; 257 u32 aper_size = 0, aper_order = 0, last_aper_order = 0; 258 u64 aper_base = 0, last_aper_base = 0; 259 int aper_enabled = 0, last_aper_enabled = 0, last_valid = 0; 260 261 if (!early_pci_allowed()) 262 return; 263 264 /* This is mostly duplicate of iommu_hole_init */ 265 search_agp_bridge(&agp_aper_order, &valid_agp); 266 267 fix = 0; 268 for (i = 0; amd_nb_bus_dev_ranges[i].dev_limit; i++) { 269 int bus; 270 int dev_base, dev_limit; 271 272 bus = amd_nb_bus_dev_ranges[i].bus; 273 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 274 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 275 276 for (slot = dev_base; slot < dev_limit; slot++) { 277 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 278 continue; 279 280 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 281 aper_enabled = ctl & GARTEN; 282 aper_order = (ctl >> 1) & 7; 283 aper_size = (32 * 1024 * 1024) << aper_order; 284 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 285 aper_base <<= 25; 286 287 if (last_valid) { 288 if ((aper_order != last_aper_order) || 289 (aper_base != last_aper_base) || 290 (aper_enabled != last_aper_enabled)) { 291 fix = 1; 292 break; 293 } 294 } 295 296 last_aper_order = aper_order; 297 last_aper_base = aper_base; 298 last_aper_enabled = aper_enabled; 299 last_valid = 1; 300 } 301 } 302 303 if (!fix && !aper_enabled) 304 return; 305 306 if (!aper_base || !aper_size || aper_base + aper_size > 0x100000000UL) 307 fix = 1; 308 309 if (gart_fix_e820 && !fix && aper_enabled) { 310 if (e820_any_mapped(aper_base, aper_base + aper_size, 311 E820_RAM)) { 312 /* reserve it, so we can reuse it in second kernel */ 313 printk(KERN_INFO "update e820 for GART\n"); 314 e820_add_region(aper_base, aper_size, E820_RESERVED); 315 update_e820(); 316 } 317 } 318 319 if (valid_agp) 320 return; 321 322 /* disable them all at first */ 323 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 324 int bus; 325 int dev_base, dev_limit; 326 327 bus = amd_nb_bus_dev_ranges[i].bus; 328 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 329 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 330 331 for (slot = dev_base; slot < dev_limit; slot++) { 332 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 333 continue; 334 335 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 336 ctl &= ~GARTEN; 337 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 338 } 339 } 340 341 } 342 343 static int __initdata printed_gart_size_msg; 344 345 int __init gart_iommu_hole_init(void) 346 { 347 u32 agp_aper_base = 0, agp_aper_order = 0; 348 u32 aper_size, aper_alloc = 0, aper_order = 0, last_aper_order = 0; 349 u64 aper_base, last_aper_base = 0; 350 int fix, slot, valid_agp = 0; 351 int i, node; 352 353 if (gart_iommu_aperture_disabled || !fix_aperture || 354 !early_pci_allowed()) 355 return -ENODEV; 356 357 printk(KERN_INFO "Checking aperture...\n"); 358 359 if (!fallback_aper_force) 360 agp_aper_base = search_agp_bridge(&agp_aper_order, &valid_agp); 361 362 fix = 0; 363 node = 0; 364 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 365 int bus; 366 int dev_base, dev_limit; 367 u32 ctl; 368 369 bus = amd_nb_bus_dev_ranges[i].bus; 370 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 371 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 372 373 for (slot = dev_base; slot < dev_limit; slot++) { 374 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 375 continue; 376 377 iommu_detected = 1; 378 gart_iommu_aperture = 1; 379 x86_init.iommu.iommu_init = gart_iommu_init; 380 381 ctl = read_pci_config(bus, slot, 3, 382 AMD64_GARTAPERTURECTL); 383 384 /* 385 * Before we do anything else disable the GART. It may 386 * still be enabled if we boot into a crash-kernel here. 387 * Reconfiguring the GART while it is enabled could have 388 * unknown side-effects. 389 */ 390 ctl &= ~GARTEN; 391 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 392 393 aper_order = (ctl >> 1) & 7; 394 aper_size = (32 * 1024 * 1024) << aper_order; 395 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 396 aper_base <<= 25; 397 398 printk(KERN_INFO "Node %d: aperture @ %Lx size %u MB\n", 399 node, aper_base, aper_size >> 20); 400 node++; 401 402 if (!aperture_valid(aper_base, aper_size, 64<<20)) { 403 if (valid_agp && agp_aper_base && 404 agp_aper_base == aper_base && 405 agp_aper_order == aper_order) { 406 /* the same between two setting from NB and agp */ 407 if (!no_iommu && 408 max_pfn > MAX_DMA32_PFN && 409 !printed_gart_size_msg) { 410 printk(KERN_ERR "you are using iommu with agp, but GART size is less than 64M\n"); 411 printk(KERN_ERR "please increase GART size in your BIOS setup\n"); 412 printk(KERN_ERR "if BIOS doesn't have that option, contact your HW vendor!\n"); 413 printed_gart_size_msg = 1; 414 } 415 } else { 416 fix = 1; 417 goto out; 418 } 419 } 420 421 if ((last_aper_order && aper_order != last_aper_order) || 422 (last_aper_base && aper_base != last_aper_base)) { 423 fix = 1; 424 goto out; 425 } 426 last_aper_order = aper_order; 427 last_aper_base = aper_base; 428 } 429 } 430 431 out: 432 if (!fix && !fallback_aper_force) { 433 if (last_aper_base) 434 return 1; 435 return 0; 436 } 437 438 if (!fallback_aper_force) { 439 aper_alloc = agp_aper_base; 440 aper_order = agp_aper_order; 441 } 442 443 if (aper_alloc) { 444 /* Got the aperture from the AGP bridge */ 445 } else if ((!no_iommu && max_pfn > MAX_DMA32_PFN) || 446 force_iommu || 447 valid_agp || 448 fallback_aper_force) { 449 printk(KERN_INFO 450 "Your BIOS doesn't leave a aperture memory hole\n"); 451 printk(KERN_INFO 452 "Please enable the IOMMU option in the BIOS setup\n"); 453 printk(KERN_INFO 454 "This costs you %d MB of RAM\n", 455 32 << fallback_aper_order); 456 457 aper_order = fallback_aper_order; 458 aper_alloc = allocate_aperture(); 459 if (!aper_alloc) { 460 /* 461 * Could disable AGP and IOMMU here, but it's 462 * probably not worth it. But the later users 463 * cannot deal with bad apertures and turning 464 * on the aperture over memory causes very 465 * strange problems, so it's better to panic 466 * early. 467 */ 468 panic("Not enough memory for aperture"); 469 } 470 } else { 471 return 0; 472 } 473 474 /* Fix up the north bridges */ 475 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 476 int bus, dev_base, dev_limit; 477 478 /* 479 * Don't enable translation yet but enable GART IO and CPU 480 * accesses and set DISTLBWALKPRB since GART table memory is UC. 481 */ 482 u32 ctl = aper_order << 1; 483 484 bus = amd_nb_bus_dev_ranges[i].bus; 485 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 486 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 487 for (slot = dev_base; slot < dev_limit; slot++) { 488 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 489 continue; 490 491 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 492 write_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE, aper_alloc >> 25); 493 } 494 } 495 496 set_up_gart_resume(aper_order, aper_alloc); 497 498 return 1; 499 } 500