1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Firmware replacement code. 4 * 5 * Work around broken BIOSes that don't set an aperture, only set the 6 * aperture in the AGP bridge, or set too small aperture. 7 * 8 * If all fails map the aperture over some low memory. This is cheaper than 9 * doing bounce buffering. The memory is lost. This is done at early boot 10 * because only the bootmem allocator can allocate 32+MB. 11 * 12 * Copyright 2002 Andi Kleen, SuSE Labs. 13 */ 14 #define pr_fmt(fmt) "AGP: " fmt 15 16 #include <linux/kernel.h> 17 #include <linux/kcore.h> 18 #include <linux/types.h> 19 #include <linux/init.h> 20 #include <linux/memblock.h> 21 #include <linux/mmzone.h> 22 #include <linux/pci_ids.h> 23 #include <linux/pci.h> 24 #include <linux/bitops.h> 25 #include <linux/suspend.h> 26 #include <asm/e820/api.h> 27 #include <asm/io.h> 28 #include <asm/iommu.h> 29 #include <asm/gart.h> 30 #include <asm/pci-direct.h> 31 #include <asm/dma.h> 32 #include <asm/amd_nb.h> 33 #include <asm/x86_init.h> 34 #include <linux/crash_dump.h> 35 36 /* 37 * Using 512M as goal, in case kexec will load kernel_big 38 * that will do the on-position decompress, and could overlap with 39 * with the gart aperture that is used. 40 * Sequence: 41 * kernel_small 42 * ==> kexec (with kdump trigger path or gart still enabled) 43 * ==> kernel_small (gart area become e820_reserved) 44 * ==> kexec (with kdump trigger path or gart still enabled) 45 * ==> kerne_big (uncompressed size will be big than 64M or 128M) 46 * So don't use 512M below as gart iommu, leave the space for kernel 47 * code for safe. 48 */ 49 #define GART_MIN_ADDR (512ULL << 20) 50 #define GART_MAX_ADDR (1ULL << 32) 51 52 int gart_iommu_aperture; 53 int gart_iommu_aperture_disabled __initdata; 54 int gart_iommu_aperture_allowed __initdata; 55 56 int fallback_aper_order __initdata = 1; /* 64MB */ 57 int fallback_aper_force __initdata; 58 59 int fix_aperture __initdata = 1; 60 61 #if defined(CONFIG_PROC_VMCORE) || defined(CONFIG_PROC_KCORE) 62 /* 63 * If the first kernel maps the aperture over e820 RAM, the kdump kernel will 64 * use the same range because it will remain configured in the northbridge. 65 * Trying to dump this area via /proc/vmcore may crash the machine, so exclude 66 * it from vmcore. 67 */ 68 static unsigned long aperture_pfn_start, aperture_page_count; 69 70 static int gart_mem_pfn_is_ram(unsigned long pfn) 71 { 72 return likely((pfn < aperture_pfn_start) || 73 (pfn >= aperture_pfn_start + aperture_page_count)); 74 } 75 76 static void __init exclude_from_core(u64 aper_base, u32 aper_order) 77 { 78 aperture_pfn_start = aper_base >> PAGE_SHIFT; 79 aperture_page_count = (32 * 1024 * 1024) << aper_order >> PAGE_SHIFT; 80 #ifdef CONFIG_PROC_VMCORE 81 WARN_ON(register_oldmem_pfn_is_ram(&gart_mem_pfn_is_ram)); 82 #endif 83 #ifdef CONFIG_PROC_KCORE 84 WARN_ON(register_mem_pfn_is_ram(&gart_mem_pfn_is_ram)); 85 #endif 86 } 87 #else 88 static void exclude_from_core(u64 aper_base, u32 aper_order) 89 { 90 } 91 #endif 92 93 /* This code runs before the PCI subsystem is initialized, so just 94 access the northbridge directly. */ 95 96 static u32 __init allocate_aperture(void) 97 { 98 u32 aper_size; 99 unsigned long addr; 100 101 /* aper_size should <= 1G */ 102 if (fallback_aper_order > 5) 103 fallback_aper_order = 5; 104 aper_size = (32 * 1024 * 1024) << fallback_aper_order; 105 106 /* 107 * Aperture has to be naturally aligned. This means a 2GB aperture 108 * won't have much chance of finding a place in the lower 4GB of 109 * memory. Unfortunately we cannot move it up because that would 110 * make the IOMMU useless. 111 */ 112 addr = memblock_phys_alloc_range(aper_size, aper_size, 113 GART_MIN_ADDR, GART_MAX_ADDR); 114 if (!addr) { 115 pr_err("Cannot allocate aperture memory hole [mem %#010lx-%#010lx] (%uKB)\n", 116 addr, addr + aper_size - 1, aper_size >> 10); 117 return 0; 118 } 119 pr_info("Mapping aperture over RAM [mem %#010lx-%#010lx] (%uKB)\n", 120 addr, addr + aper_size - 1, aper_size >> 10); 121 register_nosave_region(addr >> PAGE_SHIFT, 122 (addr+aper_size) >> PAGE_SHIFT); 123 124 return (u32)addr; 125 } 126 127 128 /* Find a PCI capability */ 129 static u32 __init find_cap(int bus, int slot, int func, int cap) 130 { 131 int bytes; 132 u8 pos; 133 134 if (!(read_pci_config_16(bus, slot, func, PCI_STATUS) & 135 PCI_STATUS_CAP_LIST)) 136 return 0; 137 138 pos = read_pci_config_byte(bus, slot, func, PCI_CAPABILITY_LIST); 139 for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) { 140 u8 id; 141 142 pos &= ~3; 143 id = read_pci_config_byte(bus, slot, func, pos+PCI_CAP_LIST_ID); 144 if (id == 0xff) 145 break; 146 if (id == cap) 147 return pos; 148 pos = read_pci_config_byte(bus, slot, func, 149 pos+PCI_CAP_LIST_NEXT); 150 } 151 return 0; 152 } 153 154 /* Read a standard AGPv3 bridge header */ 155 static u32 __init read_agp(int bus, int slot, int func, int cap, u32 *order) 156 { 157 u32 apsize; 158 u32 apsizereg; 159 int nbits; 160 u32 aper_low, aper_hi; 161 u64 aper; 162 u32 old_order; 163 164 pr_info("pci 0000:%02x:%02x:%02x: AGP bridge\n", bus, slot, func); 165 apsizereg = read_pci_config_16(bus, slot, func, cap + 0x14); 166 if (apsizereg == 0xffffffff) { 167 pr_err("pci 0000:%02x:%02x.%d: APSIZE unreadable\n", 168 bus, slot, func); 169 return 0; 170 } 171 172 /* old_order could be the value from NB gart setting */ 173 old_order = *order; 174 175 apsize = apsizereg & 0xfff; 176 /* Some BIOS use weird encodings not in the AGPv3 table. */ 177 if (apsize & 0xff) 178 apsize |= 0xf00; 179 nbits = hweight16(apsize); 180 *order = 7 - nbits; 181 if ((int)*order < 0) /* < 32MB */ 182 *order = 0; 183 184 aper_low = read_pci_config(bus, slot, func, 0x10); 185 aper_hi = read_pci_config(bus, slot, func, 0x14); 186 aper = (aper_low & ~((1<<22)-1)) | ((u64)aper_hi << 32); 187 188 /* 189 * On some sick chips, APSIZE is 0. It means it wants 4G 190 * so let double check that order, and lets trust AMD NB settings: 191 */ 192 pr_info("pci 0000:%02x:%02x.%d: AGP aperture [bus addr %#010Lx-%#010Lx] (old size %uMB)\n", 193 bus, slot, func, aper, aper + (32ULL << (old_order + 20)) - 1, 194 32 << old_order); 195 if (aper + (32ULL<<(20 + *order)) > 0x100000000ULL) { 196 pr_info("pci 0000:%02x:%02x.%d: AGP aperture size %uMB (APSIZE %#x) is not right, using settings from NB\n", 197 bus, slot, func, 32 << *order, apsizereg); 198 *order = old_order; 199 } 200 201 pr_info("pci 0000:%02x:%02x.%d: AGP aperture [bus addr %#010Lx-%#010Lx] (%uMB, APSIZE %#x)\n", 202 bus, slot, func, aper, aper + (32ULL << (*order + 20)) - 1, 203 32 << *order, apsizereg); 204 205 if (!aperture_valid(aper, (32*1024*1024) << *order, 32<<20)) 206 return 0; 207 return (u32)aper; 208 } 209 210 /* 211 * Look for an AGP bridge. Windows only expects the aperture in the 212 * AGP bridge and some BIOS forget to initialize the Northbridge too. 213 * Work around this here. 214 * 215 * Do an PCI bus scan by hand because we're running before the PCI 216 * subsystem. 217 * 218 * All AMD AGP bridges are AGPv3 compliant, so we can do this scan 219 * generically. It's probably overkill to always scan all slots because 220 * the AGP bridges should be always an own bus on the HT hierarchy, 221 * but do it here for future safety. 222 */ 223 static u32 __init search_agp_bridge(u32 *order, int *valid_agp) 224 { 225 int bus, slot, func; 226 227 /* Poor man's PCI discovery */ 228 for (bus = 0; bus < 256; bus++) { 229 for (slot = 0; slot < 32; slot++) { 230 for (func = 0; func < 8; func++) { 231 u32 class, cap; 232 u8 type; 233 class = read_pci_config(bus, slot, func, 234 PCI_CLASS_REVISION); 235 if (class == 0xffffffff) 236 break; 237 238 switch (class >> 16) { 239 case PCI_CLASS_BRIDGE_HOST: 240 case PCI_CLASS_BRIDGE_OTHER: /* needed? */ 241 /* AGP bridge? */ 242 cap = find_cap(bus, slot, func, 243 PCI_CAP_ID_AGP); 244 if (!cap) 245 break; 246 *valid_agp = 1; 247 return read_agp(bus, slot, func, cap, 248 order); 249 } 250 251 /* No multi-function device? */ 252 type = read_pci_config_byte(bus, slot, func, 253 PCI_HEADER_TYPE); 254 if (!(type & 0x80)) 255 break; 256 } 257 } 258 } 259 pr_info("No AGP bridge found\n"); 260 261 return 0; 262 } 263 264 static bool gart_fix_e820 __initdata = true; 265 266 static int __init parse_gart_mem(char *p) 267 { 268 return kstrtobool(p, &gart_fix_e820); 269 } 270 early_param("gart_fix_e820", parse_gart_mem); 271 272 /* 273 * With kexec/kdump, if the first kernel doesn't shut down the GART and the 274 * second kernel allocates a different GART region, there might be two 275 * overlapping GART regions present: 276 * 277 * - the first still used by the GART initialized in the first kernel. 278 * - (sub-)set of it used as normal RAM by the second kernel. 279 * 280 * which leads to memory corruptions and a kernel panic eventually. 281 * 282 * This can also happen if the BIOS has forgotten to mark the GART region 283 * as reserved. 284 * 285 * Try to update the e820 map to mark that new region as reserved. 286 */ 287 void __init early_gart_iommu_check(void) 288 { 289 u32 agp_aper_order = 0; 290 int i, fix, slot, valid_agp = 0; 291 u32 ctl; 292 u32 aper_size = 0, aper_order = 0, last_aper_order = 0; 293 u64 aper_base = 0, last_aper_base = 0; 294 int aper_enabled = 0, last_aper_enabled = 0, last_valid = 0; 295 296 if (!amd_gart_present()) 297 return; 298 299 if (!early_pci_allowed()) 300 return; 301 302 /* This is mostly duplicate of iommu_hole_init */ 303 search_agp_bridge(&agp_aper_order, &valid_agp); 304 305 fix = 0; 306 for (i = 0; amd_nb_bus_dev_ranges[i].dev_limit; i++) { 307 int bus; 308 int dev_base, dev_limit; 309 310 bus = amd_nb_bus_dev_ranges[i].bus; 311 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 312 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 313 314 for (slot = dev_base; slot < dev_limit; slot++) { 315 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 316 continue; 317 318 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 319 aper_enabled = ctl & GARTEN; 320 aper_order = (ctl >> 1) & 7; 321 aper_size = (32 * 1024 * 1024) << aper_order; 322 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 323 aper_base <<= 25; 324 325 if (last_valid) { 326 if ((aper_order != last_aper_order) || 327 (aper_base != last_aper_base) || 328 (aper_enabled != last_aper_enabled)) { 329 fix = 1; 330 break; 331 } 332 } 333 334 last_aper_order = aper_order; 335 last_aper_base = aper_base; 336 last_aper_enabled = aper_enabled; 337 last_valid = 1; 338 } 339 } 340 341 if (!fix && !aper_enabled) 342 return; 343 344 if (!aper_base || !aper_size || aper_base + aper_size > 0x100000000UL) 345 fix = 1; 346 347 if (gart_fix_e820 && !fix && aper_enabled) { 348 if (e820__mapped_any(aper_base, aper_base + aper_size, 349 E820_TYPE_RAM)) { 350 /* reserve it, so we can reuse it in second kernel */ 351 pr_info("e820: reserve [mem %#010Lx-%#010Lx] for GART\n", 352 aper_base, aper_base + aper_size - 1); 353 e820__range_add(aper_base, aper_size, E820_TYPE_RESERVED); 354 e820__update_table_print(); 355 } 356 } 357 358 if (valid_agp) 359 return; 360 361 /* disable them all at first */ 362 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 363 int bus; 364 int dev_base, dev_limit; 365 366 bus = amd_nb_bus_dev_ranges[i].bus; 367 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 368 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 369 370 for (slot = dev_base; slot < dev_limit; slot++) { 371 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 372 continue; 373 374 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 375 ctl &= ~GARTEN; 376 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 377 } 378 } 379 380 } 381 382 static int __initdata printed_gart_size_msg; 383 384 int __init gart_iommu_hole_init(void) 385 { 386 u32 agp_aper_base = 0, agp_aper_order = 0; 387 u32 aper_size, aper_alloc = 0, aper_order = 0, last_aper_order = 0; 388 u64 aper_base, last_aper_base = 0; 389 int fix, slot, valid_agp = 0; 390 int i, node; 391 392 if (!amd_gart_present()) 393 return -ENODEV; 394 395 if (gart_iommu_aperture_disabled || !fix_aperture || 396 !early_pci_allowed()) 397 return -ENODEV; 398 399 pr_info("Checking aperture...\n"); 400 401 if (!fallback_aper_force) 402 agp_aper_base = search_agp_bridge(&agp_aper_order, &valid_agp); 403 404 fix = 0; 405 node = 0; 406 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 407 int bus; 408 int dev_base, dev_limit; 409 u32 ctl; 410 411 bus = amd_nb_bus_dev_ranges[i].bus; 412 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 413 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 414 415 for (slot = dev_base; slot < dev_limit; slot++) { 416 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 417 continue; 418 419 iommu_detected = 1; 420 gart_iommu_aperture = 1; 421 x86_init.iommu.iommu_init = gart_iommu_init; 422 423 ctl = read_pci_config(bus, slot, 3, 424 AMD64_GARTAPERTURECTL); 425 426 /* 427 * Before we do anything else disable the GART. It may 428 * still be enabled if we boot into a crash-kernel here. 429 * Reconfiguring the GART while it is enabled could have 430 * unknown side-effects. 431 */ 432 ctl &= ~GARTEN; 433 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 434 435 aper_order = (ctl >> 1) & 7; 436 aper_size = (32 * 1024 * 1024) << aper_order; 437 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 438 aper_base <<= 25; 439 440 pr_info("Node %d: aperture [bus addr %#010Lx-%#010Lx] (%uMB)\n", 441 node, aper_base, aper_base + aper_size - 1, 442 aper_size >> 20); 443 node++; 444 445 if (!aperture_valid(aper_base, aper_size, 64<<20)) { 446 if (valid_agp && agp_aper_base && 447 agp_aper_base == aper_base && 448 agp_aper_order == aper_order) { 449 /* the same between two setting from NB and agp */ 450 if (!no_iommu && 451 max_pfn > MAX_DMA32_PFN && 452 !printed_gart_size_msg) { 453 pr_err("you are using iommu with agp, but GART size is less than 64MB\n"); 454 pr_err("please increase GART size in your BIOS setup\n"); 455 pr_err("if BIOS doesn't have that option, contact your HW vendor!\n"); 456 printed_gart_size_msg = 1; 457 } 458 } else { 459 fix = 1; 460 goto out; 461 } 462 } 463 464 if ((last_aper_order && aper_order != last_aper_order) || 465 (last_aper_base && aper_base != last_aper_base)) { 466 fix = 1; 467 goto out; 468 } 469 last_aper_order = aper_order; 470 last_aper_base = aper_base; 471 } 472 } 473 474 out: 475 if (!fix && !fallback_aper_force) { 476 if (last_aper_base) { 477 /* 478 * If this is the kdump kernel, the first kernel 479 * may have allocated the range over its e820 RAM 480 * and fixed up the northbridge 481 */ 482 exclude_from_core(last_aper_base, last_aper_order); 483 484 return 1; 485 } 486 return 0; 487 } 488 489 if (!fallback_aper_force) { 490 aper_alloc = agp_aper_base; 491 aper_order = agp_aper_order; 492 } 493 494 if (aper_alloc) { 495 /* Got the aperture from the AGP bridge */ 496 } else if ((!no_iommu && max_pfn > MAX_DMA32_PFN) || 497 force_iommu || 498 valid_agp || 499 fallback_aper_force) { 500 pr_info("Your BIOS doesn't leave an aperture memory hole\n"); 501 pr_info("Please enable the IOMMU option in the BIOS setup\n"); 502 pr_info("This costs you %dMB of RAM\n", 503 32 << fallback_aper_order); 504 505 aper_order = fallback_aper_order; 506 aper_alloc = allocate_aperture(); 507 if (!aper_alloc) { 508 /* 509 * Could disable AGP and IOMMU here, but it's 510 * probably not worth it. But the later users 511 * cannot deal with bad apertures and turning 512 * on the aperture over memory causes very 513 * strange problems, so it's better to panic 514 * early. 515 */ 516 panic("Not enough memory for aperture"); 517 } 518 } else { 519 return 0; 520 } 521 522 /* 523 * If this is the kdump kernel _and_ the first kernel did not 524 * configure the aperture in the northbridge, this range may 525 * overlap with the first kernel's memory. We can't access the 526 * range through vmcore even though it should be part of the dump. 527 */ 528 exclude_from_core(aper_alloc, aper_order); 529 530 /* Fix up the north bridges */ 531 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 532 int bus, dev_base, dev_limit; 533 534 /* 535 * Don't enable translation yet but enable GART IO and CPU 536 * accesses and set DISTLBWALKPRB since GART table memory is UC. 537 */ 538 u32 ctl = aper_order << 1; 539 540 bus = amd_nb_bus_dev_ranges[i].bus; 541 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 542 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 543 for (slot = dev_base; slot < dev_limit; slot++) { 544 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 545 continue; 546 547 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 548 write_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE, aper_alloc >> 25); 549 } 550 } 551 552 set_up_gart_resume(aper_order, aper_alloc); 553 554 return 1; 555 } 556