1 /* SPDX-License-Identifier: MIT */ 2 /* 3 * Copyright © 2020 Intel Corporation 4 * 5 * Please try to maintain the following order within this file unless it makes 6 * sense to do otherwise. From top to bottom: 7 * 1. typedefs 8 * 2. #defines, and macros 9 * 3. structure definitions 10 * 4. function prototypes 11 * 12 * Within each section, please try to order by generation in ascending order, 13 * from top to bottom (ie. gen6 on the top, gen8 on the bottom). 14 */ 15 16 #ifndef __INTEL_GTT_H__ 17 #define __INTEL_GTT_H__ 18 19 #include <linux/io-mapping.h> 20 #include <linux/kref.h> 21 #include <linux/mm.h> 22 #include <linux/pagevec.h> 23 #include <linux/scatterlist.h> 24 #include <linux/workqueue.h> 25 26 #include <drm/drm_mm.h> 27 28 #include "gt/intel_reset.h" 29 #include "i915_selftest.h" 30 #include "i915_vma_types.h" 31 32 #define I915_GFP_ALLOW_FAIL (GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN) 33 34 #if IS_ENABLED(CONFIG_DRM_I915_TRACE_GTT) 35 #define DBG(...) trace_printk(__VA_ARGS__) 36 #else 37 #define DBG(...) 38 #endif 39 40 #define NALLOC 3 /* 1 normal, 1 for concurrent threads, 1 for preallocation */ 41 42 #define I915_GTT_PAGE_SIZE_4K BIT_ULL(12) 43 #define I915_GTT_PAGE_SIZE_64K BIT_ULL(16) 44 #define I915_GTT_PAGE_SIZE_2M BIT_ULL(21) 45 46 #define I915_GTT_PAGE_SIZE I915_GTT_PAGE_SIZE_4K 47 #define I915_GTT_MAX_PAGE_SIZE I915_GTT_PAGE_SIZE_2M 48 49 #define I915_GTT_PAGE_MASK -I915_GTT_PAGE_SIZE 50 51 #define I915_GTT_MIN_ALIGNMENT I915_GTT_PAGE_SIZE 52 53 #define I915_FENCE_REG_NONE -1 54 #define I915_MAX_NUM_FENCES 32 55 /* 32 fences + sign bit for FENCE_REG_NONE */ 56 #define I915_MAX_NUM_FENCE_BITS 6 57 58 typedef u32 gen6_pte_t; 59 typedef u64 gen8_pte_t; 60 61 #define ggtt_total_entries(ggtt) ((ggtt)->vm.total >> PAGE_SHIFT) 62 63 #define I915_PTES(pte_len) ((unsigned int)(PAGE_SIZE / (pte_len))) 64 #define I915_PTE_MASK(pte_len) (I915_PTES(pte_len) - 1) 65 #define I915_PDES 512 66 #define I915_PDE_MASK (I915_PDES - 1) 67 68 /* gen6-hsw has bit 11-4 for physical addr bit 39-32 */ 69 #define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0)) 70 #define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) 71 #define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) 72 #define GEN6_PTE_CACHE_LLC (2 << 1) 73 #define GEN6_PTE_UNCACHED (1 << 1) 74 #define GEN6_PTE_VALID REG_BIT(0) 75 76 #define GEN6_PTES I915_PTES(sizeof(gen6_pte_t)) 77 #define GEN6_PD_SIZE (I915_PDES * PAGE_SIZE) 78 #define GEN6_PD_ALIGN (PAGE_SIZE * 16) 79 #define GEN6_PDE_SHIFT 22 80 #define GEN6_PDE_VALID REG_BIT(0) 81 #define NUM_PTE(pde_shift) (1 << (pde_shift - PAGE_SHIFT)) 82 83 #define GEN7_PTE_CACHE_L3_LLC (3 << 1) 84 85 #define BYT_PTE_SNOOPED_BY_CPU_CACHES REG_BIT(2) 86 #define BYT_PTE_WRITEABLE REG_BIT(1) 87 88 /* 89 * Cacheability Control is a 4-bit value. The low three bits are stored in bits 90 * 3:1 of the PTE, while the fourth bit is stored in bit 11 of the PTE. 91 */ 92 #define HSW_CACHEABILITY_CONTROL(bits) ((((bits) & 0x7) << 1) | \ 93 (((bits) & 0x8) << (11 - 3))) 94 #define HSW_WB_LLC_AGE3 HSW_CACHEABILITY_CONTROL(0x2) 95 #define HSW_WB_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0x3) 96 #define HSW_WB_ELLC_LLC_AGE3 HSW_CACHEABILITY_CONTROL(0x8) 97 #define HSW_WB_ELLC_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0xb) 98 #define HSW_WT_ELLC_LLC_AGE3 HSW_CACHEABILITY_CONTROL(0x7) 99 #define HSW_WT_ELLC_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0x6) 100 #define HSW_PTE_UNCACHED (0) 101 #define HSW_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0x7f0)) 102 #define HSW_PTE_ADDR_ENCODE(addr) HSW_GTT_ADDR_ENCODE(addr) 103 104 /* 105 * GEN8 32b style address is defined as a 3 level page table: 106 * 31:30 | 29:21 | 20:12 | 11:0 107 * PDPE | PDE | PTE | offset 108 * The difference as compared to normal x86 3 level page table is the PDPEs are 109 * programmed via register. 110 * 111 * GEN8 48b style address is defined as a 4 level page table: 112 * 47:39 | 38:30 | 29:21 | 20:12 | 11:0 113 * PML4E | PDPE | PDE | PTE | offset 114 */ 115 #define GEN8_3LVL_PDPES 4 116 117 #define PPAT_UNCACHED (_PAGE_PWT | _PAGE_PCD) 118 #define PPAT_CACHED_PDE 0 /* WB LLC */ 119 #define PPAT_CACHED _PAGE_PAT /* WB LLCeLLC */ 120 #define PPAT_DISPLAY_ELLC _PAGE_PCD /* WT eLLC */ 121 122 #define CHV_PPAT_SNOOP REG_BIT(6) 123 #define GEN8_PPAT_AGE(x) ((x)<<4) 124 #define GEN8_PPAT_LLCeLLC (3<<2) 125 #define GEN8_PPAT_LLCELLC (2<<2) 126 #define GEN8_PPAT_LLC (1<<2) 127 #define GEN8_PPAT_WB (3<<0) 128 #define GEN8_PPAT_WT (2<<0) 129 #define GEN8_PPAT_WC (1<<0) 130 #define GEN8_PPAT_UC (0<<0) 131 #define GEN8_PPAT_ELLC_OVERRIDE (0<<2) 132 #define GEN8_PPAT(i, x) ((u64)(x) << ((i) * 8)) 133 134 #define GEN8_PDE_IPS_64K BIT(11) 135 #define GEN8_PDE_PS_2M BIT(7) 136 137 enum i915_cache_level; 138 139 struct drm_i915_file_private; 140 struct drm_i915_gem_object; 141 struct i915_fence_reg; 142 struct i915_vma; 143 struct intel_gt; 144 145 #define for_each_sgt_daddr(__dp, __iter, __sgt) \ 146 __for_each_sgt_daddr(__dp, __iter, __sgt, I915_GTT_PAGE_SIZE) 147 148 struct i915_page_table { 149 struct drm_i915_gem_object *base; 150 union { 151 atomic_t used; 152 struct i915_page_table *stash; 153 }; 154 }; 155 156 struct i915_page_directory { 157 struct i915_page_table pt; 158 spinlock_t lock; 159 void **entry; 160 }; 161 162 #define __px_choose_expr(x, type, expr, other) \ 163 __builtin_choose_expr( \ 164 __builtin_types_compatible_p(typeof(x), type) || \ 165 __builtin_types_compatible_p(typeof(x), const type), \ 166 ({ type __x = (type)(x); expr; }), \ 167 other) 168 169 #define px_base(px) \ 170 __px_choose_expr(px, struct drm_i915_gem_object *, __x, \ 171 __px_choose_expr(px, struct i915_page_table *, __x->base, \ 172 __px_choose_expr(px, struct i915_page_directory *, __x->pt.base, \ 173 (void)0))) 174 175 struct page *__px_page(struct drm_i915_gem_object *p); 176 dma_addr_t __px_dma(struct drm_i915_gem_object *p); 177 #define px_dma(px) (__px_dma(px_base(px))) 178 179 #define px_pt(px) \ 180 __px_choose_expr(px, struct i915_page_table *, __x, \ 181 __px_choose_expr(px, struct i915_page_directory *, &__x->pt, \ 182 (void)0)) 183 #define px_used(px) (&px_pt(px)->used) 184 185 struct i915_vm_pt_stash { 186 /* preallocated chains of page tables/directories */ 187 struct i915_page_table *pt[2]; 188 }; 189 190 struct i915_vma_ops { 191 /* Map an object into an address space with the given cache flags. */ 192 void (*bind_vma)(struct i915_address_space *vm, 193 struct i915_vm_pt_stash *stash, 194 struct i915_vma *vma, 195 enum i915_cache_level cache_level, 196 u32 flags); 197 /* 198 * Unmap an object from an address space. This usually consists of 199 * setting the valid PTE entries to a reserved scratch page. 200 */ 201 void (*unbind_vma)(struct i915_address_space *vm, 202 struct i915_vma *vma); 203 204 int (*set_pages)(struct i915_vma *vma); 205 void (*clear_pages)(struct i915_vma *vma); 206 }; 207 208 struct i915_address_space { 209 struct kref ref; 210 struct rcu_work rcu; 211 212 struct drm_mm mm; 213 struct intel_gt *gt; 214 struct drm_i915_private *i915; 215 struct device *dma; 216 /* 217 * Every address space belongs to a struct file - except for the global 218 * GTT that is owned by the driver (and so @file is set to NULL). In 219 * principle, no information should leak from one context to another 220 * (or between files/processes etc) unless explicitly shared by the 221 * owner. Tracking the owner is important in order to free up per-file 222 * objects along with the file, to aide resource tracking, and to 223 * assign blame. 224 */ 225 struct drm_i915_file_private *file; 226 u64 total; /* size addr space maps (ex. 2GB for ggtt) */ 227 u64 reserved; /* size addr space reserved */ 228 229 unsigned int bind_async_flags; 230 231 /* 232 * Each active user context has its own address space (in full-ppgtt). 233 * Since the vm may be shared between multiple contexts, we count how 234 * many contexts keep us "open". Once open hits zero, we are closed 235 * and do not allow any new attachments, and proceed to shutdown our 236 * vma and page directories. 237 */ 238 atomic_t open; 239 240 struct mutex mutex; /* protects vma and our lists */ 241 #define VM_CLASS_GGTT 0 242 #define VM_CLASS_PPGTT 1 243 244 struct drm_i915_gem_object *scratch[4]; 245 /** 246 * List of vma currently bound. 247 */ 248 struct list_head bound_list; 249 250 /* Global GTT */ 251 bool is_ggtt:1; 252 253 /* Some systems support read-only mappings for GGTT and/or PPGTT */ 254 bool has_read_only:1; 255 256 u8 top; 257 u8 pd_shift; 258 u8 scratch_order; 259 260 struct drm_i915_gem_object * 261 (*alloc_pt_dma)(struct i915_address_space *vm, int sz); 262 263 u64 (*pte_encode)(dma_addr_t addr, 264 enum i915_cache_level level, 265 u32 flags); /* Create a valid PTE */ 266 #define PTE_READ_ONLY BIT(0) 267 268 void (*allocate_va_range)(struct i915_address_space *vm, 269 struct i915_vm_pt_stash *stash, 270 u64 start, u64 length); 271 void (*clear_range)(struct i915_address_space *vm, 272 u64 start, u64 length); 273 void (*insert_page)(struct i915_address_space *vm, 274 dma_addr_t addr, 275 u64 offset, 276 enum i915_cache_level cache_level, 277 u32 flags); 278 void (*insert_entries)(struct i915_address_space *vm, 279 struct i915_vma *vma, 280 enum i915_cache_level cache_level, 281 u32 flags); 282 void (*cleanup)(struct i915_address_space *vm); 283 284 struct i915_vma_ops vma_ops; 285 286 I915_SELFTEST_DECLARE(struct fault_attr fault_attr); 287 I915_SELFTEST_DECLARE(bool scrub_64K); 288 }; 289 290 /* 291 * The Graphics Translation Table is the way in which GEN hardware translates a 292 * Graphics Virtual Address into a Physical Address. In addition to the normal 293 * collateral associated with any va->pa translations GEN hardware also has a 294 * portion of the GTT which can be mapped by the CPU and remain both coherent 295 * and correct (in cases like swizzling). That region is referred to as GMADR in 296 * the spec. 297 */ 298 struct i915_ggtt { 299 struct i915_address_space vm; 300 301 struct io_mapping iomap; /* Mapping to our CPU mappable region */ 302 struct resource gmadr; /* GMADR resource */ 303 resource_size_t mappable_end; /* End offset that we can CPU map */ 304 305 /** "Graphics Stolen Memory" holds the global PTEs */ 306 void __iomem *gsm; 307 void (*invalidate)(struct i915_ggtt *ggtt); 308 309 /** PPGTT used for aliasing the PPGTT with the GTT */ 310 struct i915_ppgtt *alias; 311 312 bool do_idle_maps; 313 314 int mtrr; 315 316 /** Bit 6 swizzling required for X tiling */ 317 u32 bit_6_swizzle_x; 318 /** Bit 6 swizzling required for Y tiling */ 319 u32 bit_6_swizzle_y; 320 321 u32 pin_bias; 322 323 unsigned int num_fences; 324 struct i915_fence_reg *fence_regs; 325 struct list_head fence_list; 326 327 /** 328 * List of all objects in gtt_space, currently mmaped by userspace. 329 * All objects within this list must also be on bound_list. 330 */ 331 struct list_head userfault_list; 332 333 /* Manual runtime pm autosuspend delay for user GGTT mmaps */ 334 struct intel_wakeref_auto userfault_wakeref; 335 336 struct mutex error_mutex; 337 struct drm_mm_node error_capture; 338 struct drm_mm_node uc_fw; 339 }; 340 341 struct i915_ppgtt { 342 struct i915_address_space vm; 343 344 struct i915_page_directory *pd; 345 }; 346 347 #define i915_is_ggtt(vm) ((vm)->is_ggtt) 348 349 static inline bool 350 i915_vm_is_4lvl(const struct i915_address_space *vm) 351 { 352 return (vm->total - 1) >> 32; 353 } 354 355 static inline bool 356 i915_vm_has_scratch_64K(struct i915_address_space *vm) 357 { 358 return vm->scratch_order == get_order(I915_GTT_PAGE_SIZE_64K); 359 } 360 361 static inline bool 362 i915_vm_has_cache_coloring(struct i915_address_space *vm) 363 { 364 return i915_is_ggtt(vm) && vm->mm.color_adjust; 365 } 366 367 static inline struct i915_ggtt * 368 i915_vm_to_ggtt(struct i915_address_space *vm) 369 { 370 BUILD_BUG_ON(offsetof(struct i915_ggtt, vm)); 371 GEM_BUG_ON(!i915_is_ggtt(vm)); 372 return container_of(vm, struct i915_ggtt, vm); 373 } 374 375 static inline struct i915_ppgtt * 376 i915_vm_to_ppgtt(struct i915_address_space *vm) 377 { 378 BUILD_BUG_ON(offsetof(struct i915_ppgtt, vm)); 379 GEM_BUG_ON(i915_is_ggtt(vm)); 380 return container_of(vm, struct i915_ppgtt, vm); 381 } 382 383 static inline struct i915_address_space * 384 i915_vm_get(struct i915_address_space *vm) 385 { 386 kref_get(&vm->ref); 387 return vm; 388 } 389 390 void i915_vm_release(struct kref *kref); 391 392 static inline void i915_vm_put(struct i915_address_space *vm) 393 { 394 kref_put(&vm->ref, i915_vm_release); 395 } 396 397 static inline struct i915_address_space * 398 i915_vm_open(struct i915_address_space *vm) 399 { 400 GEM_BUG_ON(!atomic_read(&vm->open)); 401 atomic_inc(&vm->open); 402 return i915_vm_get(vm); 403 } 404 405 static inline bool 406 i915_vm_tryopen(struct i915_address_space *vm) 407 { 408 if (atomic_add_unless(&vm->open, 1, 0)) 409 return i915_vm_get(vm); 410 411 return false; 412 } 413 414 void __i915_vm_close(struct i915_address_space *vm); 415 416 static inline void 417 i915_vm_close(struct i915_address_space *vm) 418 { 419 GEM_BUG_ON(!atomic_read(&vm->open)); 420 __i915_vm_close(vm); 421 422 i915_vm_put(vm); 423 } 424 425 void i915_address_space_init(struct i915_address_space *vm, int subclass); 426 void i915_address_space_fini(struct i915_address_space *vm); 427 428 static inline u32 i915_pte_index(u64 address, unsigned int pde_shift) 429 { 430 const u32 mask = NUM_PTE(pde_shift) - 1; 431 432 return (address >> PAGE_SHIFT) & mask; 433 } 434 435 /* 436 * Helper to counts the number of PTEs within the given length. This count 437 * does not cross a page table boundary, so the max value would be 438 * GEN6_PTES for GEN6, and GEN8_PTES for GEN8. 439 */ 440 static inline u32 i915_pte_count(u64 addr, u64 length, unsigned int pde_shift) 441 { 442 const u64 mask = ~((1ULL << pde_shift) - 1); 443 u64 end; 444 445 GEM_BUG_ON(length == 0); 446 GEM_BUG_ON(offset_in_page(addr | length)); 447 448 end = addr + length; 449 450 if ((addr & mask) != (end & mask)) 451 return NUM_PTE(pde_shift) - i915_pte_index(addr, pde_shift); 452 453 return i915_pte_index(end, pde_shift) - i915_pte_index(addr, pde_shift); 454 } 455 456 static inline u32 i915_pde_index(u64 addr, u32 shift) 457 { 458 return (addr >> shift) & I915_PDE_MASK; 459 } 460 461 static inline struct i915_page_table * 462 i915_pt_entry(const struct i915_page_directory * const pd, 463 const unsigned short n) 464 { 465 return pd->entry[n]; 466 } 467 468 static inline struct i915_page_directory * 469 i915_pd_entry(const struct i915_page_directory * const pdp, 470 const unsigned short n) 471 { 472 return pdp->entry[n]; 473 } 474 475 static inline dma_addr_t 476 i915_page_dir_dma_addr(const struct i915_ppgtt *ppgtt, const unsigned int n) 477 { 478 struct i915_page_table *pt = ppgtt->pd->entry[n]; 479 480 return __px_dma(pt ? px_base(pt) : ppgtt->vm.scratch[ppgtt->vm.top]); 481 } 482 483 void ppgtt_init(struct i915_ppgtt *ppgtt, struct intel_gt *gt); 484 485 int i915_ggtt_probe_hw(struct drm_i915_private *i915); 486 int i915_ggtt_init_hw(struct drm_i915_private *i915); 487 int i915_ggtt_enable_hw(struct drm_i915_private *i915); 488 void i915_ggtt_enable_guc(struct i915_ggtt *ggtt); 489 void i915_ggtt_disable_guc(struct i915_ggtt *ggtt); 490 int i915_init_ggtt(struct drm_i915_private *i915); 491 void i915_ggtt_driver_release(struct drm_i915_private *i915); 492 493 static inline bool i915_ggtt_has_aperture(const struct i915_ggtt *ggtt) 494 { 495 return ggtt->mappable_end > 0; 496 } 497 498 int i915_ppgtt_init_hw(struct intel_gt *gt); 499 500 struct i915_ppgtt *i915_ppgtt_create(struct intel_gt *gt); 501 502 void i915_ggtt_suspend(struct i915_ggtt *gtt); 503 void i915_ggtt_resume(struct i915_ggtt *ggtt); 504 505 #define kmap_atomic_px(px) kmap_atomic(__px_page(px_base(px))) 506 507 void 508 fill_page_dma(struct drm_i915_gem_object *p, const u64 val, unsigned int count); 509 510 #define fill_px(px, v) fill_page_dma(px_base(px), (v), PAGE_SIZE / sizeof(u64)) 511 #define fill32_px(px, v) do { \ 512 u64 v__ = lower_32_bits(v); \ 513 fill_px((px), v__ << 32 | v__); \ 514 } while (0) 515 516 int setup_scratch_page(struct i915_address_space *vm); 517 void free_scratch(struct i915_address_space *vm); 518 519 struct drm_i915_gem_object *alloc_pt_dma(struct i915_address_space *vm, int sz); 520 struct i915_page_table *alloc_pt(struct i915_address_space *vm); 521 struct i915_page_directory *alloc_pd(struct i915_address_space *vm); 522 struct i915_page_directory *__alloc_pd(int npde); 523 524 int pin_pt_dma(struct i915_address_space *vm, struct drm_i915_gem_object *obj); 525 526 void free_px(struct i915_address_space *vm, 527 struct i915_page_table *pt, int lvl); 528 #define free_pt(vm, px) free_px(vm, px, 0) 529 #define free_pd(vm, px) free_px(vm, px_pt(px), 1) 530 531 void 532 __set_pd_entry(struct i915_page_directory * const pd, 533 const unsigned short idx, 534 struct i915_page_table *pt, 535 u64 (*encode)(const dma_addr_t, const enum i915_cache_level)); 536 537 #define set_pd_entry(pd, idx, to) \ 538 __set_pd_entry((pd), (idx), px_pt(to), gen8_pde_encode) 539 540 void 541 clear_pd_entry(struct i915_page_directory * const pd, 542 const unsigned short idx, 543 const struct drm_i915_gem_object * const scratch); 544 545 bool 546 release_pd_entry(struct i915_page_directory * const pd, 547 const unsigned short idx, 548 struct i915_page_table * const pt, 549 const struct drm_i915_gem_object * const scratch); 550 void gen6_ggtt_invalidate(struct i915_ggtt *ggtt); 551 552 int ggtt_set_pages(struct i915_vma *vma); 553 int ppgtt_set_pages(struct i915_vma *vma); 554 void clear_pages(struct i915_vma *vma); 555 556 void ppgtt_bind_vma(struct i915_address_space *vm, 557 struct i915_vm_pt_stash *stash, 558 struct i915_vma *vma, 559 enum i915_cache_level cache_level, 560 u32 flags); 561 void ppgtt_unbind_vma(struct i915_address_space *vm, 562 struct i915_vma *vma); 563 564 void gtt_write_workarounds(struct intel_gt *gt); 565 566 void setup_private_pat(struct intel_uncore *uncore); 567 568 int i915_vm_alloc_pt_stash(struct i915_address_space *vm, 569 struct i915_vm_pt_stash *stash, 570 u64 size); 571 int i915_vm_pin_pt_stash(struct i915_address_space *vm, 572 struct i915_vm_pt_stash *stash); 573 void i915_vm_free_pt_stash(struct i915_address_space *vm, 574 struct i915_vm_pt_stash *stash); 575 576 struct i915_vma * 577 __vm_create_scratch_for_read(struct i915_address_space *vm, unsigned long size); 578 579 static inline struct sgt_dma { 580 struct scatterlist *sg; 581 dma_addr_t dma, max; 582 } sgt_dma(struct i915_vma *vma) { 583 struct scatterlist *sg = vma->pages->sgl; 584 dma_addr_t addr = sg_dma_address(sg); 585 586 return (struct sgt_dma){ sg, addr, addr + sg_dma_len(sg) }; 587 } 588 589 #endif 590