1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Firmware replacement code. 4 * 5 * Work around broken BIOSes that don't set an aperture, only set the 6 * aperture in the AGP bridge, or set too small aperture. 7 * 8 * If all fails map the aperture over some low memory. This is cheaper than 9 * doing bounce buffering. The memory is lost. This is done at early boot 10 * because only the bootmem allocator can allocate 32+MB. 11 * 12 * Copyright 2002 Andi Kleen, SuSE Labs. 13 */ 14 #define pr_fmt(fmt) "AGP: " fmt 15 16 #include <linux/kernel.h> 17 #include <linux/types.h> 18 #include <linux/init.h> 19 #include <linux/memblock.h> 20 #include <linux/mmzone.h> 21 #include <linux/pci_ids.h> 22 #include <linux/pci.h> 23 #include <linux/bitops.h> 24 #include <linux/suspend.h> 25 #include <asm/e820/api.h> 26 #include <asm/io.h> 27 #include <asm/iommu.h> 28 #include <asm/gart.h> 29 #include <asm/pci-direct.h> 30 #include <asm/dma.h> 31 #include <asm/amd_nb.h> 32 #include <asm/x86_init.h> 33 34 /* 35 * Using 512M as goal, in case kexec will load kernel_big 36 * that will do the on-position decompress, and could overlap with 37 * with the gart aperture that is used. 38 * Sequence: 39 * kernel_small 40 * ==> kexec (with kdump trigger path or gart still enabled) 41 * ==> kernel_small (gart area become e820_reserved) 42 * ==> kexec (with kdump trigger path or gart still enabled) 43 * ==> kerne_big (uncompressed size will be big than 64M or 128M) 44 * So don't use 512M below as gart iommu, leave the space for kernel 45 * code for safe. 46 */ 47 #define GART_MIN_ADDR (512ULL << 20) 48 #define GART_MAX_ADDR (1ULL << 32) 49 50 int gart_iommu_aperture; 51 int gart_iommu_aperture_disabled __initdata; 52 int gart_iommu_aperture_allowed __initdata; 53 54 int fallback_aper_order __initdata = 1; /* 64MB */ 55 int fallback_aper_force __initdata; 56 57 int fix_aperture __initdata = 1; 58 59 /* This code runs before the PCI subsystem is initialized, so just 60 access the northbridge directly. */ 61 62 static u32 __init allocate_aperture(void) 63 { 64 u32 aper_size; 65 unsigned long addr; 66 67 /* aper_size should <= 1G */ 68 if (fallback_aper_order > 5) 69 fallback_aper_order = 5; 70 aper_size = (32 * 1024 * 1024) << fallback_aper_order; 71 72 /* 73 * Aperture has to be naturally aligned. This means a 2GB aperture 74 * won't have much chance of finding a place in the lower 4GB of 75 * memory. Unfortunately we cannot move it up because that would 76 * make the IOMMU useless. 77 */ 78 addr = memblock_find_in_range(GART_MIN_ADDR, GART_MAX_ADDR, 79 aper_size, aper_size); 80 if (!addr) { 81 pr_err("Cannot allocate aperture memory hole [mem %#010lx-%#010lx] (%uKB)\n", 82 addr, addr + aper_size - 1, aper_size >> 10); 83 return 0; 84 } 85 memblock_reserve(addr, aper_size); 86 pr_info("Mapping aperture over RAM [mem %#010lx-%#010lx] (%uKB)\n", 87 addr, addr + aper_size - 1, aper_size >> 10); 88 register_nosave_region(addr >> PAGE_SHIFT, 89 (addr+aper_size) >> PAGE_SHIFT); 90 91 return (u32)addr; 92 } 93 94 95 /* Find a PCI capability */ 96 static u32 __init find_cap(int bus, int slot, int func, int cap) 97 { 98 int bytes; 99 u8 pos; 100 101 if (!(read_pci_config_16(bus, slot, func, PCI_STATUS) & 102 PCI_STATUS_CAP_LIST)) 103 return 0; 104 105 pos = read_pci_config_byte(bus, slot, func, PCI_CAPABILITY_LIST); 106 for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) { 107 u8 id; 108 109 pos &= ~3; 110 id = read_pci_config_byte(bus, slot, func, pos+PCI_CAP_LIST_ID); 111 if (id == 0xff) 112 break; 113 if (id == cap) 114 return pos; 115 pos = read_pci_config_byte(bus, slot, func, 116 pos+PCI_CAP_LIST_NEXT); 117 } 118 return 0; 119 } 120 121 /* Read a standard AGPv3 bridge header */ 122 static u32 __init read_agp(int bus, int slot, int func, int cap, u32 *order) 123 { 124 u32 apsize; 125 u32 apsizereg; 126 int nbits; 127 u32 aper_low, aper_hi; 128 u64 aper; 129 u32 old_order; 130 131 pr_info("pci 0000:%02x:%02x:%02x: AGP bridge\n", bus, slot, func); 132 apsizereg = read_pci_config_16(bus, slot, func, cap + 0x14); 133 if (apsizereg == 0xffffffff) { 134 pr_err("pci 0000:%02x:%02x.%d: APSIZE unreadable\n", 135 bus, slot, func); 136 return 0; 137 } 138 139 /* old_order could be the value from NB gart setting */ 140 old_order = *order; 141 142 apsize = apsizereg & 0xfff; 143 /* Some BIOS use weird encodings not in the AGPv3 table. */ 144 if (apsize & 0xff) 145 apsize |= 0xf00; 146 nbits = hweight16(apsize); 147 *order = 7 - nbits; 148 if ((int)*order < 0) /* < 32MB */ 149 *order = 0; 150 151 aper_low = read_pci_config(bus, slot, func, 0x10); 152 aper_hi = read_pci_config(bus, slot, func, 0x14); 153 aper = (aper_low & ~((1<<22)-1)) | ((u64)aper_hi << 32); 154 155 /* 156 * On some sick chips, APSIZE is 0. It means it wants 4G 157 * so let double check that order, and lets trust AMD NB settings: 158 */ 159 pr_info("pci 0000:%02x:%02x.%d: AGP aperture [bus addr %#010Lx-%#010Lx] (old size %uMB)\n", 160 bus, slot, func, aper, aper + (32ULL << (old_order + 20)) - 1, 161 32 << old_order); 162 if (aper + (32ULL<<(20 + *order)) > 0x100000000ULL) { 163 pr_info("pci 0000:%02x:%02x.%d: AGP aperture size %uMB (APSIZE %#x) is not right, using settings from NB\n", 164 bus, slot, func, 32 << *order, apsizereg); 165 *order = old_order; 166 } 167 168 pr_info("pci 0000:%02x:%02x.%d: AGP aperture [bus addr %#010Lx-%#010Lx] (%uMB, APSIZE %#x)\n", 169 bus, slot, func, aper, aper + (32ULL << (*order + 20)) - 1, 170 32 << *order, apsizereg); 171 172 if (!aperture_valid(aper, (32*1024*1024) << *order, 32<<20)) 173 return 0; 174 return (u32)aper; 175 } 176 177 /* 178 * Look for an AGP bridge. Windows only expects the aperture in the 179 * AGP bridge and some BIOS forget to initialize the Northbridge too. 180 * Work around this here. 181 * 182 * Do an PCI bus scan by hand because we're running before the PCI 183 * subsystem. 184 * 185 * All AMD AGP bridges are AGPv3 compliant, so we can do this scan 186 * generically. It's probably overkill to always scan all slots because 187 * the AGP bridges should be always an own bus on the HT hierarchy, 188 * but do it here for future safety. 189 */ 190 static u32 __init search_agp_bridge(u32 *order, int *valid_agp) 191 { 192 int bus, slot, func; 193 194 /* Poor man's PCI discovery */ 195 for (bus = 0; bus < 256; bus++) { 196 for (slot = 0; slot < 32; slot++) { 197 for (func = 0; func < 8; func++) { 198 u32 class, cap; 199 u8 type; 200 class = read_pci_config(bus, slot, func, 201 PCI_CLASS_REVISION); 202 if (class == 0xffffffff) 203 break; 204 205 switch (class >> 16) { 206 case PCI_CLASS_BRIDGE_HOST: 207 case PCI_CLASS_BRIDGE_OTHER: /* needed? */ 208 /* AGP bridge? */ 209 cap = find_cap(bus, slot, func, 210 PCI_CAP_ID_AGP); 211 if (!cap) 212 break; 213 *valid_agp = 1; 214 return read_agp(bus, slot, func, cap, 215 order); 216 } 217 218 /* No multi-function device? */ 219 type = read_pci_config_byte(bus, slot, func, 220 PCI_HEADER_TYPE); 221 if (!(type & 0x80)) 222 break; 223 } 224 } 225 } 226 pr_info("No AGP bridge found\n"); 227 228 return 0; 229 } 230 231 static bool gart_fix_e820 __initdata = true; 232 233 static int __init parse_gart_mem(char *p) 234 { 235 return kstrtobool(p, &gart_fix_e820); 236 } 237 early_param("gart_fix_e820", parse_gart_mem); 238 239 void __init early_gart_iommu_check(void) 240 { 241 /* 242 * in case it is enabled before, esp for kexec/kdump, 243 * previous kernel already enable that. memset called 244 * by allocate_aperture/__alloc_bootmem_nopanic cause restart. 245 * or second kernel have different position for GART hole. and new 246 * kernel could use hole as RAM that is still used by GART set by 247 * first kernel 248 * or BIOS forget to put that in reserved. 249 * try to update e820 to make that region as reserved. 250 */ 251 u32 agp_aper_order = 0; 252 int i, fix, slot, valid_agp = 0; 253 u32 ctl; 254 u32 aper_size = 0, aper_order = 0, last_aper_order = 0; 255 u64 aper_base = 0, last_aper_base = 0; 256 int aper_enabled = 0, last_aper_enabled = 0, last_valid = 0; 257 258 if (!amd_gart_present()) 259 return; 260 261 if (!early_pci_allowed()) 262 return; 263 264 /* This is mostly duplicate of iommu_hole_init */ 265 search_agp_bridge(&agp_aper_order, &valid_agp); 266 267 fix = 0; 268 for (i = 0; amd_nb_bus_dev_ranges[i].dev_limit; i++) { 269 int bus; 270 int dev_base, dev_limit; 271 272 bus = amd_nb_bus_dev_ranges[i].bus; 273 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 274 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 275 276 for (slot = dev_base; slot < dev_limit; slot++) { 277 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 278 continue; 279 280 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 281 aper_enabled = ctl & GARTEN; 282 aper_order = (ctl >> 1) & 7; 283 aper_size = (32 * 1024 * 1024) << aper_order; 284 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 285 aper_base <<= 25; 286 287 if (last_valid) { 288 if ((aper_order != last_aper_order) || 289 (aper_base != last_aper_base) || 290 (aper_enabled != last_aper_enabled)) { 291 fix = 1; 292 break; 293 } 294 } 295 296 last_aper_order = aper_order; 297 last_aper_base = aper_base; 298 last_aper_enabled = aper_enabled; 299 last_valid = 1; 300 } 301 } 302 303 if (!fix && !aper_enabled) 304 return; 305 306 if (!aper_base || !aper_size || aper_base + aper_size > 0x100000000UL) 307 fix = 1; 308 309 if (gart_fix_e820 && !fix && aper_enabled) { 310 if (e820__mapped_any(aper_base, aper_base + aper_size, 311 E820_TYPE_RAM)) { 312 /* reserve it, so we can reuse it in second kernel */ 313 pr_info("e820: reserve [mem %#010Lx-%#010Lx] for GART\n", 314 aper_base, aper_base + aper_size - 1); 315 e820__range_add(aper_base, aper_size, E820_TYPE_RESERVED); 316 e820__update_table_print(); 317 } 318 } 319 320 if (valid_agp) 321 return; 322 323 /* disable them all at first */ 324 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 325 int bus; 326 int dev_base, dev_limit; 327 328 bus = amd_nb_bus_dev_ranges[i].bus; 329 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 330 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 331 332 for (slot = dev_base; slot < dev_limit; slot++) { 333 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 334 continue; 335 336 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 337 ctl &= ~GARTEN; 338 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 339 } 340 } 341 342 } 343 344 static int __initdata printed_gart_size_msg; 345 346 int __init gart_iommu_hole_init(void) 347 { 348 u32 agp_aper_base = 0, agp_aper_order = 0; 349 u32 aper_size, aper_alloc = 0, aper_order = 0, last_aper_order = 0; 350 u64 aper_base, last_aper_base = 0; 351 int fix, slot, valid_agp = 0; 352 int i, node; 353 354 if (!amd_gart_present()) 355 return -ENODEV; 356 357 if (gart_iommu_aperture_disabled || !fix_aperture || 358 !early_pci_allowed()) 359 return -ENODEV; 360 361 pr_info("Checking aperture...\n"); 362 363 if (!fallback_aper_force) 364 agp_aper_base = search_agp_bridge(&agp_aper_order, &valid_agp); 365 366 fix = 0; 367 node = 0; 368 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 369 int bus; 370 int dev_base, dev_limit; 371 u32 ctl; 372 373 bus = amd_nb_bus_dev_ranges[i].bus; 374 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 375 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 376 377 for (slot = dev_base; slot < dev_limit; slot++) { 378 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 379 continue; 380 381 iommu_detected = 1; 382 gart_iommu_aperture = 1; 383 x86_init.iommu.iommu_init = gart_iommu_init; 384 385 ctl = read_pci_config(bus, slot, 3, 386 AMD64_GARTAPERTURECTL); 387 388 /* 389 * Before we do anything else disable the GART. It may 390 * still be enabled if we boot into a crash-kernel here. 391 * Reconfiguring the GART while it is enabled could have 392 * unknown side-effects. 393 */ 394 ctl &= ~GARTEN; 395 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 396 397 aper_order = (ctl >> 1) & 7; 398 aper_size = (32 * 1024 * 1024) << aper_order; 399 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 400 aper_base <<= 25; 401 402 pr_info("Node %d: aperture [bus addr %#010Lx-%#010Lx] (%uMB)\n", 403 node, aper_base, aper_base + aper_size - 1, 404 aper_size >> 20); 405 node++; 406 407 if (!aperture_valid(aper_base, aper_size, 64<<20)) { 408 if (valid_agp && agp_aper_base && 409 agp_aper_base == aper_base && 410 agp_aper_order == aper_order) { 411 /* the same between two setting from NB and agp */ 412 if (!no_iommu && 413 max_pfn > MAX_DMA32_PFN && 414 !printed_gart_size_msg) { 415 pr_err("you are using iommu with agp, but GART size is less than 64MB\n"); 416 pr_err("please increase GART size in your BIOS setup\n"); 417 pr_err("if BIOS doesn't have that option, contact your HW vendor!\n"); 418 printed_gart_size_msg = 1; 419 } 420 } else { 421 fix = 1; 422 goto out; 423 } 424 } 425 426 if ((last_aper_order && aper_order != last_aper_order) || 427 (last_aper_base && aper_base != last_aper_base)) { 428 fix = 1; 429 goto out; 430 } 431 last_aper_order = aper_order; 432 last_aper_base = aper_base; 433 } 434 } 435 436 out: 437 if (!fix && !fallback_aper_force) { 438 if (last_aper_base) 439 return 1; 440 return 0; 441 } 442 443 if (!fallback_aper_force) { 444 aper_alloc = agp_aper_base; 445 aper_order = agp_aper_order; 446 } 447 448 if (aper_alloc) { 449 /* Got the aperture from the AGP bridge */ 450 } else if ((!no_iommu && max_pfn > MAX_DMA32_PFN) || 451 force_iommu || 452 valid_agp || 453 fallback_aper_force) { 454 pr_info("Your BIOS doesn't leave an aperture memory hole\n"); 455 pr_info("Please enable the IOMMU option in the BIOS setup\n"); 456 pr_info("This costs you %dMB of RAM\n", 457 32 << fallback_aper_order); 458 459 aper_order = fallback_aper_order; 460 aper_alloc = allocate_aperture(); 461 if (!aper_alloc) { 462 /* 463 * Could disable AGP and IOMMU here, but it's 464 * probably not worth it. But the later users 465 * cannot deal with bad apertures and turning 466 * on the aperture over memory causes very 467 * strange problems, so it's better to panic 468 * early. 469 */ 470 panic("Not enough memory for aperture"); 471 } 472 } else { 473 return 0; 474 } 475 476 /* Fix up the north bridges */ 477 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 478 int bus, dev_base, dev_limit; 479 480 /* 481 * Don't enable translation yet but enable GART IO and CPU 482 * accesses and set DISTLBWALKPRB since GART table memory is UC. 483 */ 484 u32 ctl = aper_order << 1; 485 486 bus = amd_nb_bus_dev_ranges[i].bus; 487 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 488 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 489 for (slot = dev_base; slot < dev_limit; slot++) { 490 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 491 continue; 492 493 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 494 write_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE, aper_alloc >> 25); 495 } 496 } 497 498 set_up_gart_resume(aper_order, aper_alloc); 499 500 return 1; 501 } 502