1 /* 2 * Firmware replacement code. 3 * 4 * Work around broken BIOSes that don't set an aperture, only set the 5 * aperture in the AGP bridge, or set too small aperture. 6 * 7 * If all fails map the aperture over some low memory. This is cheaper than 8 * doing bounce buffering. The memory is lost. This is done at early boot 9 * because only the bootmem allocator can allocate 32+MB. 10 * 11 * Copyright 2002 Andi Kleen, SuSE Labs. 12 */ 13 #include <linux/kernel.h> 14 #include <linux/types.h> 15 #include <linux/init.h> 16 #include <linux/bootmem.h> 17 #include <linux/mmzone.h> 18 #include <linux/pci_ids.h> 19 #include <linux/pci.h> 20 #include <linux/bitops.h> 21 #include <linux/ioport.h> 22 #include <linux/suspend.h> 23 #include <asm/e820.h> 24 #include <asm/io.h> 25 #include <asm/iommu.h> 26 #include <asm/gart.h> 27 #include <asm/pci-direct.h> 28 #include <asm/dma.h> 29 #include <asm/k8.h> 30 31 int gart_iommu_aperture; 32 int gart_iommu_aperture_disabled __initdata; 33 int gart_iommu_aperture_allowed __initdata; 34 35 int fallback_aper_order __initdata = 1; /* 64MB */ 36 int fallback_aper_force __initdata; 37 38 int fix_aperture __initdata = 1; 39 40 struct bus_dev_range { 41 int bus; 42 int dev_base; 43 int dev_limit; 44 }; 45 46 static struct bus_dev_range bus_dev_ranges[] __initdata = { 47 { 0x00, 0x18, 0x20}, 48 { 0xff, 0x00, 0x20}, 49 { 0xfe, 0x00, 0x20} 50 }; 51 52 static struct resource gart_resource = { 53 .name = "GART", 54 .flags = IORESOURCE_MEM, 55 }; 56 57 static void __init insert_aperture_resource(u32 aper_base, u32 aper_size) 58 { 59 gart_resource.start = aper_base; 60 gart_resource.end = aper_base + aper_size - 1; 61 insert_resource(&iomem_resource, &gart_resource); 62 } 63 64 /* This code runs before the PCI subsystem is initialized, so just 65 access the northbridge directly. */ 66 67 static u32 __init allocate_aperture(void) 68 { 69 u32 aper_size; 70 void *p; 71 72 /* aper_size should <= 1G */ 73 if (fallback_aper_order > 5) 74 fallback_aper_order = 5; 75 aper_size = (32 * 1024 * 1024) << fallback_aper_order; 76 77 /* 78 * Aperture has to be naturally aligned. This means a 2GB aperture 79 * won't have much chance of finding a place in the lower 4GB of 80 * memory. Unfortunately we cannot move it up because that would 81 * make the IOMMU useless. 82 */ 83 /* 84 * using 512M as goal, in case kexec will load kernel_big 85 * that will do the on position decompress, and could overlap with 86 * that positon with gart that is used. 87 * sequende: 88 * kernel_small 89 * ==> kexec (with kdump trigger path or previous doesn't shutdown gart) 90 * ==> kernel_small(gart area become e820_reserved) 91 * ==> kexec (with kdump trigger path or previous doesn't shutdown gart) 92 * ==> kerne_big (uncompressed size will be big than 64M or 128M) 93 * so don't use 512M below as gart iommu, leave the space for kernel 94 * code for safe 95 */ 96 p = __alloc_bootmem_nopanic(aper_size, aper_size, 512ULL<<20); 97 if (!p || __pa(p)+aper_size > 0xffffffff) { 98 printk(KERN_ERR 99 "Cannot allocate aperture memory hole (%p,%uK)\n", 100 p, aper_size>>10); 101 if (p) 102 free_bootmem(__pa(p), aper_size); 103 return 0; 104 } 105 printk(KERN_INFO "Mapping aperture over %d KB of RAM @ %lx\n", 106 aper_size >> 10, __pa(p)); 107 insert_aperture_resource((u32)__pa(p), aper_size); 108 register_nosave_region((u32)__pa(p) >> PAGE_SHIFT, 109 (u32)__pa(p+aper_size) >> PAGE_SHIFT); 110 111 return (u32)__pa(p); 112 } 113 114 115 /* Find a PCI capability */ 116 static u32 __init find_cap(int bus, int slot, int func, int cap) 117 { 118 int bytes; 119 u8 pos; 120 121 if (!(read_pci_config_16(bus, slot, func, PCI_STATUS) & 122 PCI_STATUS_CAP_LIST)) 123 return 0; 124 125 pos = read_pci_config_byte(bus, slot, func, PCI_CAPABILITY_LIST); 126 for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) { 127 u8 id; 128 129 pos &= ~3; 130 id = read_pci_config_byte(bus, slot, func, pos+PCI_CAP_LIST_ID); 131 if (id == 0xff) 132 break; 133 if (id == cap) 134 return pos; 135 pos = read_pci_config_byte(bus, slot, func, 136 pos+PCI_CAP_LIST_NEXT); 137 } 138 return 0; 139 } 140 141 /* Read a standard AGPv3 bridge header */ 142 static u32 __init read_agp(int bus, int slot, int func, int cap, u32 *order) 143 { 144 u32 apsize; 145 u32 apsizereg; 146 int nbits; 147 u32 aper_low, aper_hi; 148 u64 aper; 149 u32 old_order; 150 151 printk(KERN_INFO "AGP bridge at %02x:%02x:%02x\n", bus, slot, func); 152 apsizereg = read_pci_config_16(bus, slot, func, cap + 0x14); 153 if (apsizereg == 0xffffffff) { 154 printk(KERN_ERR "APSIZE in AGP bridge unreadable\n"); 155 return 0; 156 } 157 158 /* old_order could be the value from NB gart setting */ 159 old_order = *order; 160 161 apsize = apsizereg & 0xfff; 162 /* Some BIOS use weird encodings not in the AGPv3 table. */ 163 if (apsize & 0xff) 164 apsize |= 0xf00; 165 nbits = hweight16(apsize); 166 *order = 7 - nbits; 167 if ((int)*order < 0) /* < 32MB */ 168 *order = 0; 169 170 aper_low = read_pci_config(bus, slot, func, 0x10); 171 aper_hi = read_pci_config(bus, slot, func, 0x14); 172 aper = (aper_low & ~((1<<22)-1)) | ((u64)aper_hi << 32); 173 174 /* 175 * On some sick chips, APSIZE is 0. It means it wants 4G 176 * so let double check that order, and lets trust AMD NB settings: 177 */ 178 printk(KERN_INFO "Aperture from AGP @ %Lx old size %u MB\n", 179 aper, 32 << old_order); 180 if (aper + (32ULL<<(20 + *order)) > 0x100000000ULL) { 181 printk(KERN_INFO "Aperture size %u MB (APSIZE %x) is not right, using settings from NB\n", 182 32 << *order, apsizereg); 183 *order = old_order; 184 } 185 186 printk(KERN_INFO "Aperture from AGP @ %Lx size %u MB (APSIZE %x)\n", 187 aper, 32 << *order, apsizereg); 188 189 if (!aperture_valid(aper, (32*1024*1024) << *order, 32<<20)) 190 return 0; 191 return (u32)aper; 192 } 193 194 /* 195 * Look for an AGP bridge. Windows only expects the aperture in the 196 * AGP bridge and some BIOS forget to initialize the Northbridge too. 197 * Work around this here. 198 * 199 * Do an PCI bus scan by hand because we're running before the PCI 200 * subsystem. 201 * 202 * All K8 AGP bridges are AGPv3 compliant, so we can do this scan 203 * generically. It's probably overkill to always scan all slots because 204 * the AGP bridges should be always an own bus on the HT hierarchy, 205 * but do it here for future safety. 206 */ 207 static u32 __init search_agp_bridge(u32 *order, int *valid_agp) 208 { 209 int bus, slot, func; 210 211 /* Poor man's PCI discovery */ 212 for (bus = 0; bus < 256; bus++) { 213 for (slot = 0; slot < 32; slot++) { 214 for (func = 0; func < 8; func++) { 215 u32 class, cap; 216 u8 type; 217 class = read_pci_config(bus, slot, func, 218 PCI_CLASS_REVISION); 219 if (class == 0xffffffff) 220 break; 221 222 switch (class >> 16) { 223 case PCI_CLASS_BRIDGE_HOST: 224 case PCI_CLASS_BRIDGE_OTHER: /* needed? */ 225 /* AGP bridge? */ 226 cap = find_cap(bus, slot, func, 227 PCI_CAP_ID_AGP); 228 if (!cap) 229 break; 230 *valid_agp = 1; 231 return read_agp(bus, slot, func, cap, 232 order); 233 } 234 235 /* No multi-function device? */ 236 type = read_pci_config_byte(bus, slot, func, 237 PCI_HEADER_TYPE); 238 if (!(type & 0x80)) 239 break; 240 } 241 } 242 } 243 printk(KERN_INFO "No AGP bridge found\n"); 244 245 return 0; 246 } 247 248 static int gart_fix_e820 __initdata = 1; 249 250 static int __init parse_gart_mem(char *p) 251 { 252 if (!p) 253 return -EINVAL; 254 255 if (!strncmp(p, "off", 3)) 256 gart_fix_e820 = 0; 257 else if (!strncmp(p, "on", 2)) 258 gart_fix_e820 = 1; 259 260 return 0; 261 } 262 early_param("gart_fix_e820", parse_gart_mem); 263 264 void __init early_gart_iommu_check(void) 265 { 266 /* 267 * in case it is enabled before, esp for kexec/kdump, 268 * previous kernel already enable that. memset called 269 * by allocate_aperture/__alloc_bootmem_nopanic cause restart. 270 * or second kernel have different position for GART hole. and new 271 * kernel could use hole as RAM that is still used by GART set by 272 * first kernel 273 * or BIOS forget to put that in reserved. 274 * try to update e820 to make that region as reserved. 275 */ 276 int i, fix, slot; 277 u32 ctl; 278 u32 aper_size = 0, aper_order = 0, last_aper_order = 0; 279 u64 aper_base = 0, last_aper_base = 0; 280 int aper_enabled = 0, last_aper_enabled = 0, last_valid = 0; 281 282 if (!early_pci_allowed()) 283 return; 284 285 /* This is mostly duplicate of iommu_hole_init */ 286 fix = 0; 287 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) { 288 int bus; 289 int dev_base, dev_limit; 290 291 bus = bus_dev_ranges[i].bus; 292 dev_base = bus_dev_ranges[i].dev_base; 293 dev_limit = bus_dev_ranges[i].dev_limit; 294 295 for (slot = dev_base; slot < dev_limit; slot++) { 296 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00))) 297 continue; 298 299 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 300 aper_enabled = ctl & AMD64_GARTEN; 301 aper_order = (ctl >> 1) & 7; 302 aper_size = (32 * 1024 * 1024) << aper_order; 303 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 304 aper_base <<= 25; 305 306 if (last_valid) { 307 if ((aper_order != last_aper_order) || 308 (aper_base != last_aper_base) || 309 (aper_enabled != last_aper_enabled)) { 310 fix = 1; 311 break; 312 } 313 } 314 315 last_aper_order = aper_order; 316 last_aper_base = aper_base; 317 last_aper_enabled = aper_enabled; 318 last_valid = 1; 319 } 320 } 321 322 if (!fix && !aper_enabled) 323 return; 324 325 if (!aper_base || !aper_size || aper_base + aper_size > 0x100000000UL) 326 fix = 1; 327 328 if (gart_fix_e820 && !fix && aper_enabled) { 329 if (e820_any_mapped(aper_base, aper_base + aper_size, 330 E820_RAM)) { 331 /* reserve it, so we can reuse it in second kernel */ 332 printk(KERN_INFO "update e820 for GART\n"); 333 e820_add_region(aper_base, aper_size, E820_RESERVED); 334 update_e820(); 335 } 336 } 337 338 if (!fix) 339 return; 340 341 /* different nodes have different setting, disable them all at first*/ 342 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) { 343 int bus; 344 int dev_base, dev_limit; 345 346 bus = bus_dev_ranges[i].bus; 347 dev_base = bus_dev_ranges[i].dev_base; 348 dev_limit = bus_dev_ranges[i].dev_limit; 349 350 for (slot = dev_base; slot < dev_limit; slot++) { 351 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00))) 352 continue; 353 354 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 355 ctl &= ~AMD64_GARTEN; 356 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 357 } 358 } 359 360 } 361 362 static int __initdata printed_gart_size_msg; 363 364 void __init gart_iommu_hole_init(void) 365 { 366 u32 agp_aper_base = 0, agp_aper_order = 0; 367 u32 aper_size, aper_alloc = 0, aper_order = 0, last_aper_order = 0; 368 u64 aper_base, last_aper_base = 0; 369 int fix, slot, valid_agp = 0; 370 int i, node; 371 372 if (gart_iommu_aperture_disabled || !fix_aperture || 373 !early_pci_allowed()) 374 return; 375 376 printk(KERN_INFO "Checking aperture...\n"); 377 378 if (!fallback_aper_force) 379 agp_aper_base = search_agp_bridge(&agp_aper_order, &valid_agp); 380 381 fix = 0; 382 node = 0; 383 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) { 384 int bus; 385 int dev_base, dev_limit; 386 387 bus = bus_dev_ranges[i].bus; 388 dev_base = bus_dev_ranges[i].dev_base; 389 dev_limit = bus_dev_ranges[i].dev_limit; 390 391 for (slot = dev_base; slot < dev_limit; slot++) { 392 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00))) 393 continue; 394 395 iommu_detected = 1; 396 gart_iommu_aperture = 1; 397 398 aper_order = (read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL) >> 1) & 7; 399 aper_size = (32 * 1024 * 1024) << aper_order; 400 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 401 aper_base <<= 25; 402 403 printk(KERN_INFO "Node %d: aperture @ %Lx size %u MB\n", 404 node, aper_base, aper_size >> 20); 405 node++; 406 407 if (!aperture_valid(aper_base, aper_size, 64<<20)) { 408 if (valid_agp && agp_aper_base && 409 agp_aper_base == aper_base && 410 agp_aper_order == aper_order) { 411 /* the same between two setting from NB and agp */ 412 if (!no_iommu && 413 max_pfn > MAX_DMA32_PFN && 414 !printed_gart_size_msg) { 415 printk(KERN_ERR "you are using iommu with agp, but GART size is less than 64M\n"); 416 printk(KERN_ERR "please increase GART size in your BIOS setup\n"); 417 printk(KERN_ERR "if BIOS doesn't have that option, contact your HW vendor!\n"); 418 printed_gart_size_msg = 1; 419 } 420 } else { 421 fix = 1; 422 goto out; 423 } 424 } 425 426 if ((last_aper_order && aper_order != last_aper_order) || 427 (last_aper_base && aper_base != last_aper_base)) { 428 fix = 1; 429 goto out; 430 } 431 last_aper_order = aper_order; 432 last_aper_base = aper_base; 433 } 434 } 435 436 out: 437 if (!fix && !fallback_aper_force) { 438 if (last_aper_base) { 439 unsigned long n = (32 * 1024 * 1024) << last_aper_order; 440 441 insert_aperture_resource((u32)last_aper_base, n); 442 } 443 return; 444 } 445 446 if (!fallback_aper_force) { 447 aper_alloc = agp_aper_base; 448 aper_order = agp_aper_order; 449 } 450 451 if (aper_alloc) { 452 /* Got the aperture from the AGP bridge */ 453 } else if (swiotlb && !valid_agp) { 454 /* Do nothing */ 455 } else if ((!no_iommu && max_pfn > MAX_DMA32_PFN) || 456 force_iommu || 457 valid_agp || 458 fallback_aper_force) { 459 printk(KERN_INFO 460 "Your BIOS doesn't leave a aperture memory hole\n"); 461 printk(KERN_INFO 462 "Please enable the IOMMU option in the BIOS setup\n"); 463 printk(KERN_INFO 464 "This costs you %d MB of RAM\n", 465 32 << fallback_aper_order); 466 467 aper_order = fallback_aper_order; 468 aper_alloc = allocate_aperture(); 469 if (!aper_alloc) { 470 /* 471 * Could disable AGP and IOMMU here, but it's 472 * probably not worth it. But the later users 473 * cannot deal with bad apertures and turning 474 * on the aperture over memory causes very 475 * strange problems, so it's better to panic 476 * early. 477 */ 478 panic("Not enough memory for aperture"); 479 } 480 } else { 481 return; 482 } 483 484 /* Fix up the north bridges */ 485 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) { 486 int bus; 487 int dev_base, dev_limit; 488 489 bus = bus_dev_ranges[i].bus; 490 dev_base = bus_dev_ranges[i].dev_base; 491 dev_limit = bus_dev_ranges[i].dev_limit; 492 for (slot = dev_base; slot < dev_limit; slot++) { 493 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00))) 494 continue; 495 496 /* Don't enable translation yet. That is done later. 497 Assume this BIOS didn't initialise the GART so 498 just overwrite all previous bits */ 499 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, aper_order << 1); 500 write_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE, aper_alloc >> 25); 501 } 502 } 503 504 set_up_gart_resume(aper_order, aper_alloc); 505 } 506