1b414fcd5SChris Wilson /* 2b414fcd5SChris Wilson * SPDX-License-Identifier: MIT 3b414fcd5SChris Wilson * 4b414fcd5SChris Wilson * Copyright © 2016 Intel Corporation 5b414fcd5SChris Wilson */ 6b414fcd5SChris Wilson 7b414fcd5SChris Wilson #include <linux/prime_numbers.h> 8b414fcd5SChris Wilson 9a1c8a09eSTvrtko Ursulin #include "gt/intel_gt.h" 10b414fcd5SChris Wilson #include "gt/intel_gt_pm.h" 1110be98a7SChris Wilson #include "huge_gem_object.h" 12b414fcd5SChris Wilson #include "i915_selftest.h" 1307e98eb0SChris Wilson #include "selftests/i915_random.h" 14b414fcd5SChris Wilson #include "selftests/igt_flush_test.h" 15b414fcd5SChris Wilson 16b414fcd5SChris Wilson struct tile { 17b414fcd5SChris Wilson unsigned int width; 18b414fcd5SChris Wilson unsigned int height; 19b414fcd5SChris Wilson unsigned int stride; 20b414fcd5SChris Wilson unsigned int size; 21b414fcd5SChris Wilson unsigned int tiling; 22b414fcd5SChris Wilson unsigned int swizzle; 23b414fcd5SChris Wilson }; 24b414fcd5SChris Wilson 25b414fcd5SChris Wilson static u64 swizzle_bit(unsigned int bit, u64 offset) 26b414fcd5SChris Wilson { 27b414fcd5SChris Wilson return (offset & BIT_ULL(bit)) >> (bit - 6); 28b414fcd5SChris Wilson } 29b414fcd5SChris Wilson 30b414fcd5SChris Wilson static u64 tiled_offset(const struct tile *tile, u64 v) 31b414fcd5SChris Wilson { 32b414fcd5SChris Wilson u64 x, y; 33b414fcd5SChris Wilson 34b414fcd5SChris Wilson if (tile->tiling == I915_TILING_NONE) 35b414fcd5SChris Wilson return v; 36b414fcd5SChris Wilson 37b414fcd5SChris Wilson y = div64_u64_rem(v, tile->stride, &x); 38b414fcd5SChris Wilson v = div64_u64_rem(y, tile->height, &y) * tile->stride * tile->height; 39b414fcd5SChris Wilson 40b414fcd5SChris Wilson if (tile->tiling == I915_TILING_X) { 41b414fcd5SChris Wilson v += y * tile->width; 42b414fcd5SChris Wilson v += div64_u64_rem(x, tile->width, &x) << tile->size; 43b414fcd5SChris Wilson v += x; 44b414fcd5SChris Wilson } else if (tile->width == 128) { 45b414fcd5SChris Wilson const unsigned int ytile_span = 16; 46b414fcd5SChris Wilson const unsigned int ytile_height = 512; 47b414fcd5SChris Wilson 48b414fcd5SChris Wilson v += y * ytile_span; 49b414fcd5SChris Wilson v += div64_u64_rem(x, ytile_span, &x) * ytile_height; 50b414fcd5SChris Wilson v += x; 51b414fcd5SChris Wilson } else { 52b414fcd5SChris Wilson const unsigned int ytile_span = 32; 53b414fcd5SChris Wilson const unsigned int ytile_height = 256; 54b414fcd5SChris Wilson 55b414fcd5SChris Wilson v += y * ytile_span; 56b414fcd5SChris Wilson v += div64_u64_rem(x, ytile_span, &x) * ytile_height; 57b414fcd5SChris Wilson v += x; 58b414fcd5SChris Wilson } 59b414fcd5SChris Wilson 60b414fcd5SChris Wilson switch (tile->swizzle) { 61b414fcd5SChris Wilson case I915_BIT_6_SWIZZLE_9: 62b414fcd5SChris Wilson v ^= swizzle_bit(9, v); 63b414fcd5SChris Wilson break; 64b414fcd5SChris Wilson case I915_BIT_6_SWIZZLE_9_10: 65b414fcd5SChris Wilson v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v); 66b414fcd5SChris Wilson break; 67b414fcd5SChris Wilson case I915_BIT_6_SWIZZLE_9_11: 68b414fcd5SChris Wilson v ^= swizzle_bit(9, v) ^ swizzle_bit(11, v); 69b414fcd5SChris Wilson break; 70b414fcd5SChris Wilson case I915_BIT_6_SWIZZLE_9_10_11: 71b414fcd5SChris Wilson v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v) ^ swizzle_bit(11, v); 72b414fcd5SChris Wilson break; 73b414fcd5SChris Wilson } 74b414fcd5SChris Wilson 75b414fcd5SChris Wilson return v; 76b414fcd5SChris Wilson } 77b414fcd5SChris Wilson 78b414fcd5SChris Wilson static int check_partial_mapping(struct drm_i915_gem_object *obj, 79b414fcd5SChris Wilson const struct tile *tile, 8007e98eb0SChris Wilson struct rnd_state *prng) 8107e98eb0SChris Wilson { 8207e98eb0SChris Wilson const unsigned long npages = obj->base.size / PAGE_SIZE; 8307e98eb0SChris Wilson struct i915_ggtt_view view; 8407e98eb0SChris Wilson struct i915_vma *vma; 8507e98eb0SChris Wilson unsigned long page; 8607e98eb0SChris Wilson u32 __iomem *io; 8707e98eb0SChris Wilson struct page *p; 8807e98eb0SChris Wilson unsigned int n; 8907e98eb0SChris Wilson u64 offset; 9007e98eb0SChris Wilson u32 *cpu; 9107e98eb0SChris Wilson int err; 9207e98eb0SChris Wilson 9307e98eb0SChris Wilson err = i915_gem_object_set_tiling(obj, tile->tiling, tile->stride); 9407e98eb0SChris Wilson if (err) { 9507e98eb0SChris Wilson pr_err("Failed to set tiling mode=%u, stride=%u, err=%d\n", 9607e98eb0SChris Wilson tile->tiling, tile->stride, err); 9707e98eb0SChris Wilson return err; 9807e98eb0SChris Wilson } 9907e98eb0SChris Wilson 10007e98eb0SChris Wilson GEM_BUG_ON(i915_gem_object_get_tiling(obj) != tile->tiling); 10107e98eb0SChris Wilson GEM_BUG_ON(i915_gem_object_get_stride(obj) != tile->stride); 10207e98eb0SChris Wilson 10307e98eb0SChris Wilson i915_gem_object_lock(obj); 10407e98eb0SChris Wilson err = i915_gem_object_set_to_gtt_domain(obj, true); 10507e98eb0SChris Wilson i915_gem_object_unlock(obj); 10607e98eb0SChris Wilson if (err) { 10707e98eb0SChris Wilson pr_err("Failed to flush to GTT write domain; err=%d\n", err); 10807e98eb0SChris Wilson return err; 10907e98eb0SChris Wilson } 11007e98eb0SChris Wilson 11107e98eb0SChris Wilson page = i915_prandom_u32_max_state(npages, prng); 11207e98eb0SChris Wilson view = compute_partial_view(obj, page, MIN_CHUNK_PAGES); 11307e98eb0SChris Wilson 11407e98eb0SChris Wilson vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE); 11507e98eb0SChris Wilson if (IS_ERR(vma)) { 11607e98eb0SChris Wilson pr_err("Failed to pin partial view: offset=%lu; err=%d\n", 11707e98eb0SChris Wilson page, (int)PTR_ERR(vma)); 11807e98eb0SChris Wilson return PTR_ERR(vma); 11907e98eb0SChris Wilson } 12007e98eb0SChris Wilson 12107e98eb0SChris Wilson n = page - view.partial.offset; 12207e98eb0SChris Wilson GEM_BUG_ON(n >= view.partial.size); 12307e98eb0SChris Wilson 12407e98eb0SChris Wilson io = i915_vma_pin_iomap(vma); 12507e98eb0SChris Wilson i915_vma_unpin(vma); 12607e98eb0SChris Wilson if (IS_ERR(io)) { 12707e98eb0SChris Wilson pr_err("Failed to iomap partial view: offset=%lu; err=%d\n", 12807e98eb0SChris Wilson page, (int)PTR_ERR(io)); 12907e98eb0SChris Wilson err = PTR_ERR(io); 13007e98eb0SChris Wilson goto out; 13107e98eb0SChris Wilson } 13207e98eb0SChris Wilson 13307e98eb0SChris Wilson iowrite32(page, io + n * PAGE_SIZE / sizeof(*io)); 13407e98eb0SChris Wilson i915_vma_unpin_iomap(vma); 13507e98eb0SChris Wilson 13607e98eb0SChris Wilson offset = tiled_offset(tile, page << PAGE_SHIFT); 13707e98eb0SChris Wilson if (offset >= obj->base.size) 13807e98eb0SChris Wilson goto out; 13907e98eb0SChris Wilson 14007e98eb0SChris Wilson intel_gt_flush_ggtt_writes(&to_i915(obj->base.dev)->gt); 14107e98eb0SChris Wilson 14207e98eb0SChris Wilson p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT); 14307e98eb0SChris Wilson cpu = kmap(p) + offset_in_page(offset); 14407e98eb0SChris Wilson drm_clflush_virt_range(cpu, sizeof(*cpu)); 14507e98eb0SChris Wilson if (*cpu != (u32)page) { 14607e98eb0SChris Wilson pr_err("Partial view for %lu [%u] (offset=%llu, size=%u [%llu, row size %u], fence=%d, tiling=%d, stride=%d) misalignment, expected write to page (%llu + %u [0x%llx]) of 0x%x, found 0x%x\n", 14707e98eb0SChris Wilson page, n, 14807e98eb0SChris Wilson view.partial.offset, 14907e98eb0SChris Wilson view.partial.size, 15007e98eb0SChris Wilson vma->size >> PAGE_SHIFT, 15107e98eb0SChris Wilson tile->tiling ? tile_row_pages(obj) : 0, 15207e98eb0SChris Wilson vma->fence ? vma->fence->id : -1, tile->tiling, tile->stride, 15307e98eb0SChris Wilson offset >> PAGE_SHIFT, 15407e98eb0SChris Wilson (unsigned int)offset_in_page(offset), 15507e98eb0SChris Wilson offset, 15607e98eb0SChris Wilson (u32)page, *cpu); 15707e98eb0SChris Wilson err = -EINVAL; 15807e98eb0SChris Wilson } 15907e98eb0SChris Wilson *cpu = 0; 16007e98eb0SChris Wilson drm_clflush_virt_range(cpu, sizeof(*cpu)); 16107e98eb0SChris Wilson kunmap(p); 16207e98eb0SChris Wilson 16307e98eb0SChris Wilson out: 16407e98eb0SChris Wilson i915_vma_destroy(vma); 16507e98eb0SChris Wilson return err; 16607e98eb0SChris Wilson } 16707e98eb0SChris Wilson 16807e98eb0SChris Wilson static int check_partial_mappings(struct drm_i915_gem_object *obj, 16907e98eb0SChris Wilson const struct tile *tile, 170b414fcd5SChris Wilson unsigned long end_time) 171b414fcd5SChris Wilson { 172b414fcd5SChris Wilson const unsigned int nreal = obj->scratch / PAGE_SIZE; 173b414fcd5SChris Wilson const unsigned long npages = obj->base.size / PAGE_SIZE; 174b414fcd5SChris Wilson struct i915_vma *vma; 175b414fcd5SChris Wilson unsigned long page; 176b414fcd5SChris Wilson int err; 177b414fcd5SChris Wilson 178b414fcd5SChris Wilson err = i915_gem_object_set_tiling(obj, tile->tiling, tile->stride); 179b414fcd5SChris Wilson if (err) { 180b414fcd5SChris Wilson pr_err("Failed to set tiling mode=%u, stride=%u, err=%d\n", 181b414fcd5SChris Wilson tile->tiling, tile->stride, err); 182b414fcd5SChris Wilson return err; 183b414fcd5SChris Wilson } 184b414fcd5SChris Wilson 185b414fcd5SChris Wilson GEM_BUG_ON(i915_gem_object_get_tiling(obj) != tile->tiling); 186b414fcd5SChris Wilson GEM_BUG_ON(i915_gem_object_get_stride(obj) != tile->stride); 187b414fcd5SChris Wilson 18887d1372dSChris Wilson i915_gem_object_lock(obj); 18987d1372dSChris Wilson err = i915_gem_object_set_to_gtt_domain(obj, true); 19087d1372dSChris Wilson i915_gem_object_unlock(obj); 19187d1372dSChris Wilson if (err) { 19287d1372dSChris Wilson pr_err("Failed to flush to GTT write domain; err=%d\n", err); 19387d1372dSChris Wilson return err; 19487d1372dSChris Wilson } 19587d1372dSChris Wilson 196b414fcd5SChris Wilson for_each_prime_number_from(page, 1, npages) { 197b414fcd5SChris Wilson struct i915_ggtt_view view = 198b414fcd5SChris Wilson compute_partial_view(obj, page, MIN_CHUNK_PAGES); 199b414fcd5SChris Wilson u32 __iomem *io; 200b414fcd5SChris Wilson struct page *p; 201b414fcd5SChris Wilson unsigned int n; 202b414fcd5SChris Wilson u64 offset; 203b414fcd5SChris Wilson u32 *cpu; 204b414fcd5SChris Wilson 205b414fcd5SChris Wilson GEM_BUG_ON(view.partial.size > nreal); 206b414fcd5SChris Wilson cond_resched(); 207b414fcd5SChris Wilson 208b414fcd5SChris Wilson vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE); 209b414fcd5SChris Wilson if (IS_ERR(vma)) { 210b414fcd5SChris Wilson pr_err("Failed to pin partial view: offset=%lu; err=%d\n", 211b414fcd5SChris Wilson page, (int)PTR_ERR(vma)); 212b414fcd5SChris Wilson return PTR_ERR(vma); 213b414fcd5SChris Wilson } 214b414fcd5SChris Wilson 215b414fcd5SChris Wilson n = page - view.partial.offset; 216b414fcd5SChris Wilson GEM_BUG_ON(n >= view.partial.size); 217b414fcd5SChris Wilson 218b414fcd5SChris Wilson io = i915_vma_pin_iomap(vma); 219b414fcd5SChris Wilson i915_vma_unpin(vma); 220b414fcd5SChris Wilson if (IS_ERR(io)) { 221b414fcd5SChris Wilson pr_err("Failed to iomap partial view: offset=%lu; err=%d\n", 222b414fcd5SChris Wilson page, (int)PTR_ERR(io)); 223b414fcd5SChris Wilson return PTR_ERR(io); 224b414fcd5SChris Wilson } 225b414fcd5SChris Wilson 226b414fcd5SChris Wilson iowrite32(page, io + n * PAGE_SIZE / sizeof(*io)); 227b414fcd5SChris Wilson i915_vma_unpin_iomap(vma); 228b414fcd5SChris Wilson 229b414fcd5SChris Wilson offset = tiled_offset(tile, page << PAGE_SHIFT); 230b414fcd5SChris Wilson if (offset >= obj->base.size) 231b414fcd5SChris Wilson continue; 232b414fcd5SChris Wilson 233a1c8a09eSTvrtko Ursulin intel_gt_flush_ggtt_writes(&to_i915(obj->base.dev)->gt); 234b414fcd5SChris Wilson 235b414fcd5SChris Wilson p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT); 236b414fcd5SChris Wilson cpu = kmap(p) + offset_in_page(offset); 237b414fcd5SChris Wilson drm_clflush_virt_range(cpu, sizeof(*cpu)); 238b414fcd5SChris Wilson if (*cpu != (u32)page) { 239b414fcd5SChris Wilson pr_err("Partial view for %lu [%u] (offset=%llu, size=%u [%llu, row size %u], fence=%d, tiling=%d, stride=%d) misalignment, expected write to page (%llu + %u [0x%llx]) of 0x%x, found 0x%x\n", 240b414fcd5SChris Wilson page, n, 241b414fcd5SChris Wilson view.partial.offset, 242b414fcd5SChris Wilson view.partial.size, 243b414fcd5SChris Wilson vma->size >> PAGE_SHIFT, 244b414fcd5SChris Wilson tile->tiling ? tile_row_pages(obj) : 0, 245b414fcd5SChris Wilson vma->fence ? vma->fence->id : -1, tile->tiling, tile->stride, 246b414fcd5SChris Wilson offset >> PAGE_SHIFT, 247b414fcd5SChris Wilson (unsigned int)offset_in_page(offset), 248b414fcd5SChris Wilson offset, 249b414fcd5SChris Wilson (u32)page, *cpu); 250b414fcd5SChris Wilson err = -EINVAL; 251b414fcd5SChris Wilson } 252b414fcd5SChris Wilson *cpu = 0; 253b414fcd5SChris Wilson drm_clflush_virt_range(cpu, sizeof(*cpu)); 254b414fcd5SChris Wilson kunmap(p); 255b414fcd5SChris Wilson if (err) 256b414fcd5SChris Wilson return err; 257b414fcd5SChris Wilson 258b414fcd5SChris Wilson i915_vma_destroy(vma); 25907e98eb0SChris Wilson 26007e98eb0SChris Wilson if (igt_timeout(end_time, 26107e98eb0SChris Wilson "%s: timed out after tiling=%d stride=%d\n", 26207e98eb0SChris Wilson __func__, tile->tiling, tile->stride)) 26307e98eb0SChris Wilson return -EINTR; 264b414fcd5SChris Wilson } 265b414fcd5SChris Wilson 266b414fcd5SChris Wilson return 0; 267b414fcd5SChris Wilson } 268b414fcd5SChris Wilson 26907e98eb0SChris Wilson static unsigned int 27007e98eb0SChris Wilson setup_tile_size(struct tile *tile, struct drm_i915_private *i915) 27107e98eb0SChris Wilson { 27207e98eb0SChris Wilson if (INTEL_GEN(i915) <= 2) { 27307e98eb0SChris Wilson tile->height = 16; 27407e98eb0SChris Wilson tile->width = 128; 27507e98eb0SChris Wilson tile->size = 11; 27607e98eb0SChris Wilson } else if (tile->tiling == I915_TILING_Y && 27707e98eb0SChris Wilson HAS_128_BYTE_Y_TILING(i915)) { 27807e98eb0SChris Wilson tile->height = 32; 27907e98eb0SChris Wilson tile->width = 128; 28007e98eb0SChris Wilson tile->size = 12; 28107e98eb0SChris Wilson } else { 28207e98eb0SChris Wilson tile->height = 8; 28307e98eb0SChris Wilson tile->width = 512; 28407e98eb0SChris Wilson tile->size = 12; 28507e98eb0SChris Wilson } 28607e98eb0SChris Wilson 28707e98eb0SChris Wilson if (INTEL_GEN(i915) < 4) 28807e98eb0SChris Wilson return 8192 / tile->width; 28907e98eb0SChris Wilson else if (INTEL_GEN(i915) < 7) 29007e98eb0SChris Wilson return 128 * I965_FENCE_MAX_PITCH_VAL / tile->width; 29107e98eb0SChris Wilson else 29207e98eb0SChris Wilson return 128 * GEN7_FENCE_MAX_PITCH_VAL / tile->width; 29307e98eb0SChris Wilson } 29407e98eb0SChris Wilson 295b414fcd5SChris Wilson static int igt_partial_tiling(void *arg) 296b414fcd5SChris Wilson { 297b414fcd5SChris Wilson const unsigned int nreal = 1 << 12; /* largest tile row x2 */ 298b414fcd5SChris Wilson struct drm_i915_private *i915 = arg; 299b414fcd5SChris Wilson struct drm_i915_gem_object *obj; 300b414fcd5SChris Wilson intel_wakeref_t wakeref; 301b414fcd5SChris Wilson int tiling; 302b414fcd5SChris Wilson int err; 303b414fcd5SChris Wilson 304e60f7bb7SMatthew Auld if (!i915_ggtt_has_aperture(&i915->ggtt)) 305e60f7bb7SMatthew Auld return 0; 306e60f7bb7SMatthew Auld 307b414fcd5SChris Wilson /* We want to check the page mapping and fencing of a large object 308b414fcd5SChris Wilson * mmapped through the GTT. The object we create is larger than can 309b414fcd5SChris Wilson * possibly be mmaped as a whole, and so we must use partial GGTT vma. 310b414fcd5SChris Wilson * We then check that a write through each partial GGTT vma ends up 311b414fcd5SChris Wilson * in the right set of pages within the object, and with the expected 312b414fcd5SChris Wilson * tiling, which we verify by manual swizzling. 313b414fcd5SChris Wilson */ 314b414fcd5SChris Wilson 315b414fcd5SChris Wilson obj = huge_gem_object(i915, 316b414fcd5SChris Wilson nreal << PAGE_SHIFT, 317b414fcd5SChris Wilson (1 + next_prime_number(i915->ggtt.vm.total >> PAGE_SHIFT)) << PAGE_SHIFT); 318b414fcd5SChris Wilson if (IS_ERR(obj)) 319b414fcd5SChris Wilson return PTR_ERR(obj); 320b414fcd5SChris Wilson 321b414fcd5SChris Wilson err = i915_gem_object_pin_pages(obj); 322b414fcd5SChris Wilson if (err) { 323b414fcd5SChris Wilson pr_err("Failed to allocate %u pages (%lu total), err=%d\n", 324b414fcd5SChris Wilson nreal, obj->base.size / PAGE_SIZE, err); 325b414fcd5SChris Wilson goto out; 326b414fcd5SChris Wilson } 327b414fcd5SChris Wilson 328d858d569SDaniele Ceraolo Spurio wakeref = intel_runtime_pm_get(&i915->runtime_pm); 329b414fcd5SChris Wilson 330b414fcd5SChris Wilson if (1) { 331b414fcd5SChris Wilson IGT_TIMEOUT(end); 332b414fcd5SChris Wilson struct tile tile; 333b414fcd5SChris Wilson 334b414fcd5SChris Wilson tile.height = 1; 335b414fcd5SChris Wilson tile.width = 1; 336b414fcd5SChris Wilson tile.size = 0; 337b414fcd5SChris Wilson tile.stride = 0; 338b414fcd5SChris Wilson tile.swizzle = I915_BIT_6_SWIZZLE_NONE; 339b414fcd5SChris Wilson tile.tiling = I915_TILING_NONE; 340b414fcd5SChris Wilson 34107e98eb0SChris Wilson err = check_partial_mappings(obj, &tile, end); 342b414fcd5SChris Wilson if (err && err != -EINTR) 343b414fcd5SChris Wilson goto out_unlock; 344b414fcd5SChris Wilson } 345b414fcd5SChris Wilson 346b414fcd5SChris Wilson for (tiling = I915_TILING_X; tiling <= I915_TILING_Y; tiling++) { 347b414fcd5SChris Wilson IGT_TIMEOUT(end); 348b414fcd5SChris Wilson unsigned int max_pitch; 349b414fcd5SChris Wilson unsigned int pitch; 350b414fcd5SChris Wilson struct tile tile; 351b414fcd5SChris Wilson 352b414fcd5SChris Wilson if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) 353b414fcd5SChris Wilson /* 354b414fcd5SChris Wilson * The swizzling pattern is actually unknown as it 355b414fcd5SChris Wilson * varies based on physical address of each page. 356b414fcd5SChris Wilson * See i915_gem_detect_bit_6_swizzle(). 357b414fcd5SChris Wilson */ 358b414fcd5SChris Wilson break; 359b414fcd5SChris Wilson 360b414fcd5SChris Wilson tile.tiling = tiling; 361b414fcd5SChris Wilson switch (tiling) { 362b414fcd5SChris Wilson case I915_TILING_X: 363972c646fSChris Wilson tile.swizzle = i915->ggtt.bit_6_swizzle_x; 364b414fcd5SChris Wilson break; 365b414fcd5SChris Wilson case I915_TILING_Y: 366972c646fSChris Wilson tile.swizzle = i915->ggtt.bit_6_swizzle_y; 367b414fcd5SChris Wilson break; 368b414fcd5SChris Wilson } 369b414fcd5SChris Wilson 370b414fcd5SChris Wilson GEM_BUG_ON(tile.swizzle == I915_BIT_6_SWIZZLE_UNKNOWN); 371b414fcd5SChris Wilson if (tile.swizzle == I915_BIT_6_SWIZZLE_9_17 || 372b414fcd5SChris Wilson tile.swizzle == I915_BIT_6_SWIZZLE_9_10_17) 373b414fcd5SChris Wilson continue; 374b414fcd5SChris Wilson 37507e98eb0SChris Wilson max_pitch = setup_tile_size(&tile, i915); 376b414fcd5SChris Wilson 377b414fcd5SChris Wilson for (pitch = max_pitch; pitch; pitch >>= 1) { 378b414fcd5SChris Wilson tile.stride = tile.width * pitch; 37907e98eb0SChris Wilson err = check_partial_mappings(obj, &tile, end); 380b414fcd5SChris Wilson if (err == -EINTR) 381b414fcd5SChris Wilson goto next_tiling; 382b414fcd5SChris Wilson if (err) 383b414fcd5SChris Wilson goto out_unlock; 384b414fcd5SChris Wilson 385b414fcd5SChris Wilson if (pitch > 2 && INTEL_GEN(i915) >= 4) { 386b414fcd5SChris Wilson tile.stride = tile.width * (pitch - 1); 38707e98eb0SChris Wilson err = check_partial_mappings(obj, &tile, end); 388b414fcd5SChris Wilson if (err == -EINTR) 389b414fcd5SChris Wilson goto next_tiling; 390b414fcd5SChris Wilson if (err) 391b414fcd5SChris Wilson goto out_unlock; 392b414fcd5SChris Wilson } 393b414fcd5SChris Wilson 394b414fcd5SChris Wilson if (pitch < max_pitch && INTEL_GEN(i915) >= 4) { 395b414fcd5SChris Wilson tile.stride = tile.width * (pitch + 1); 39607e98eb0SChris Wilson err = check_partial_mappings(obj, &tile, end); 397b414fcd5SChris Wilson if (err == -EINTR) 398b414fcd5SChris Wilson goto next_tiling; 399b414fcd5SChris Wilson if (err) 400b414fcd5SChris Wilson goto out_unlock; 401b414fcd5SChris Wilson } 402b414fcd5SChris Wilson } 403b414fcd5SChris Wilson 404b414fcd5SChris Wilson if (INTEL_GEN(i915) >= 4) { 405b414fcd5SChris Wilson for_each_prime_number(pitch, max_pitch) { 406b414fcd5SChris Wilson tile.stride = tile.width * pitch; 40707e98eb0SChris Wilson err = check_partial_mappings(obj, &tile, end); 408b414fcd5SChris Wilson if (err == -EINTR) 409b414fcd5SChris Wilson goto next_tiling; 410b414fcd5SChris Wilson if (err) 411b414fcd5SChris Wilson goto out_unlock; 412b414fcd5SChris Wilson } 413b414fcd5SChris Wilson } 414b414fcd5SChris Wilson 415b414fcd5SChris Wilson next_tiling: ; 416b414fcd5SChris Wilson } 417b414fcd5SChris Wilson 418b414fcd5SChris Wilson out_unlock: 419d858d569SDaniele Ceraolo Spurio intel_runtime_pm_put(&i915->runtime_pm, wakeref); 420b414fcd5SChris Wilson i915_gem_object_unpin_pages(obj); 421b414fcd5SChris Wilson out: 422b414fcd5SChris Wilson i915_gem_object_put(obj); 423b414fcd5SChris Wilson return err; 424b414fcd5SChris Wilson } 425b414fcd5SChris Wilson 42607e98eb0SChris Wilson static int igt_smoke_tiling(void *arg) 42707e98eb0SChris Wilson { 42807e98eb0SChris Wilson const unsigned int nreal = 1 << 12; /* largest tile row x2 */ 42907e98eb0SChris Wilson struct drm_i915_private *i915 = arg; 43007e98eb0SChris Wilson struct drm_i915_gem_object *obj; 43107e98eb0SChris Wilson intel_wakeref_t wakeref; 43207e98eb0SChris Wilson I915_RND_STATE(prng); 43307e98eb0SChris Wilson unsigned long count; 43407e98eb0SChris Wilson IGT_TIMEOUT(end); 43507e98eb0SChris Wilson int err; 43607e98eb0SChris Wilson 437e60f7bb7SMatthew Auld if (!i915_ggtt_has_aperture(&i915->ggtt)) 438e60f7bb7SMatthew Auld return 0; 439e60f7bb7SMatthew Auld 44007e98eb0SChris Wilson /* 44107e98eb0SChris Wilson * igt_partial_tiling() does an exhastive check of partial tiling 44207e98eb0SChris Wilson * chunking, but will undoubtably run out of time. Here, we do a 44307e98eb0SChris Wilson * randomised search and hope over many runs of 1s with different 44407e98eb0SChris Wilson * seeds we will do a thorough check. 44507e98eb0SChris Wilson * 44607e98eb0SChris Wilson * Remember to look at the st_seed if we see a flip-flop in BAT! 44707e98eb0SChris Wilson */ 44807e98eb0SChris Wilson 44907e98eb0SChris Wilson if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) 45007e98eb0SChris Wilson return 0; 45107e98eb0SChris Wilson 45207e98eb0SChris Wilson obj = huge_gem_object(i915, 45307e98eb0SChris Wilson nreal << PAGE_SHIFT, 45407e98eb0SChris Wilson (1 + next_prime_number(i915->ggtt.vm.total >> PAGE_SHIFT)) << PAGE_SHIFT); 45507e98eb0SChris Wilson if (IS_ERR(obj)) 45607e98eb0SChris Wilson return PTR_ERR(obj); 45707e98eb0SChris Wilson 45807e98eb0SChris Wilson err = i915_gem_object_pin_pages(obj); 45907e98eb0SChris Wilson if (err) { 46007e98eb0SChris Wilson pr_err("Failed to allocate %u pages (%lu total), err=%d\n", 46107e98eb0SChris Wilson nreal, obj->base.size / PAGE_SIZE, err); 46207e98eb0SChris Wilson goto out; 46307e98eb0SChris Wilson } 46407e98eb0SChris Wilson 46507e98eb0SChris Wilson wakeref = intel_runtime_pm_get(&i915->runtime_pm); 46607e98eb0SChris Wilson 46707e98eb0SChris Wilson count = 0; 46807e98eb0SChris Wilson do { 46907e98eb0SChris Wilson struct tile tile; 47007e98eb0SChris Wilson 47107e98eb0SChris Wilson tile.tiling = 47207e98eb0SChris Wilson i915_prandom_u32_max_state(I915_TILING_Y + 1, &prng); 47307e98eb0SChris Wilson switch (tile.tiling) { 47407e98eb0SChris Wilson case I915_TILING_NONE: 47507e98eb0SChris Wilson tile.height = 1; 47607e98eb0SChris Wilson tile.width = 1; 47707e98eb0SChris Wilson tile.size = 0; 47807e98eb0SChris Wilson tile.stride = 0; 47907e98eb0SChris Wilson tile.swizzle = I915_BIT_6_SWIZZLE_NONE; 48007e98eb0SChris Wilson break; 48107e98eb0SChris Wilson 48207e98eb0SChris Wilson case I915_TILING_X: 483972c646fSChris Wilson tile.swizzle = i915->ggtt.bit_6_swizzle_x; 48407e98eb0SChris Wilson break; 48507e98eb0SChris Wilson case I915_TILING_Y: 486972c646fSChris Wilson tile.swizzle = i915->ggtt.bit_6_swizzle_y; 48707e98eb0SChris Wilson break; 48807e98eb0SChris Wilson } 48907e98eb0SChris Wilson 49007e98eb0SChris Wilson if (tile.swizzle == I915_BIT_6_SWIZZLE_9_17 || 49107e98eb0SChris Wilson tile.swizzle == I915_BIT_6_SWIZZLE_9_10_17) 49207e98eb0SChris Wilson continue; 49307e98eb0SChris Wilson 49407e98eb0SChris Wilson if (tile.tiling != I915_TILING_NONE) { 49507e98eb0SChris Wilson unsigned int max_pitch = setup_tile_size(&tile, i915); 49607e98eb0SChris Wilson 49707e98eb0SChris Wilson tile.stride = 49807e98eb0SChris Wilson i915_prandom_u32_max_state(max_pitch, &prng); 49907e98eb0SChris Wilson tile.stride = (1 + tile.stride) * tile.width; 50007e98eb0SChris Wilson if (INTEL_GEN(i915) < 4) 50107e98eb0SChris Wilson tile.stride = rounddown_pow_of_two(tile.stride); 50207e98eb0SChris Wilson } 50307e98eb0SChris Wilson 50407e98eb0SChris Wilson err = check_partial_mapping(obj, &tile, &prng); 50507e98eb0SChris Wilson if (err) 50607e98eb0SChris Wilson break; 50707e98eb0SChris Wilson 50807e98eb0SChris Wilson count++; 50907e98eb0SChris Wilson } while (!__igt_timeout(end, NULL)); 51007e98eb0SChris Wilson 51107e98eb0SChris Wilson pr_info("%s: Completed %lu trials\n", __func__, count); 51207e98eb0SChris Wilson 51307e98eb0SChris Wilson intel_runtime_pm_put(&i915->runtime_pm, wakeref); 51407e98eb0SChris Wilson i915_gem_object_unpin_pages(obj); 51507e98eb0SChris Wilson out: 51607e98eb0SChris Wilson i915_gem_object_put(obj); 51707e98eb0SChris Wilson return err; 51807e98eb0SChris Wilson } 51907e98eb0SChris Wilson 520b414fcd5SChris Wilson static int make_obj_busy(struct drm_i915_gem_object *obj) 521b414fcd5SChris Wilson { 522b414fcd5SChris Wilson struct drm_i915_private *i915 = to_i915(obj->base.dev); 5238f856c74SChris Wilson struct intel_engine_cs *engine; 524e948761fSChris Wilson 525e948761fSChris Wilson for_each_uabi_engine(engine, i915) { 526e948761fSChris Wilson struct i915_request *rq; 527b414fcd5SChris Wilson struct i915_vma *vma; 528b414fcd5SChris Wilson int err; 529b414fcd5SChris Wilson 530e948761fSChris Wilson vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); 531b414fcd5SChris Wilson if (IS_ERR(vma)) 532b414fcd5SChris Wilson return PTR_ERR(vma); 533b414fcd5SChris Wilson 534b414fcd5SChris Wilson err = i915_vma_pin(vma, 0, 0, PIN_USER); 535b414fcd5SChris Wilson if (err) 536b414fcd5SChris Wilson return err; 537b414fcd5SChris Wilson 5388f856c74SChris Wilson rq = i915_request_create(engine->kernel_context); 539b414fcd5SChris Wilson if (IS_ERR(rq)) { 540b414fcd5SChris Wilson i915_vma_unpin(vma); 541b414fcd5SChris Wilson return PTR_ERR(rq); 542b414fcd5SChris Wilson } 543b414fcd5SChris Wilson 5446951e589SChris Wilson i915_vma_lock(vma); 54570d6894dSChris Wilson err = i915_request_await_object(rq, vma->obj, true); 54670d6894dSChris Wilson if (err == 0) 54770d6894dSChris Wilson err = i915_vma_move_to_active(vma, rq, 54870d6894dSChris Wilson EXEC_OBJECT_WRITE); 5496951e589SChris Wilson i915_vma_unlock(vma); 550b414fcd5SChris Wilson 551b414fcd5SChris Wilson i915_request_add(rq); 552e948761fSChris Wilson i915_vma_unpin(vma); 553e948761fSChris Wilson if (err) 554e948761fSChris Wilson return err; 5558f856c74SChris Wilson } 556b414fcd5SChris Wilson 557c017cf6bSChris Wilson i915_gem_object_put(obj); /* leave it only alive via its active ref */ 558e948761fSChris Wilson return 0; 559b414fcd5SChris Wilson } 560b414fcd5SChris Wilson 561b414fcd5SChris Wilson static bool assert_mmap_offset(struct drm_i915_private *i915, 562b414fcd5SChris Wilson unsigned long size, 563b414fcd5SChris Wilson int expected) 564b414fcd5SChris Wilson { 565b414fcd5SChris Wilson struct drm_i915_gem_object *obj; 566b414fcd5SChris Wilson int err; 567b414fcd5SChris Wilson 568b414fcd5SChris Wilson obj = i915_gem_object_create_internal(i915, size); 569b414fcd5SChris Wilson if (IS_ERR(obj)) 570b414fcd5SChris Wilson return PTR_ERR(obj); 571b414fcd5SChris Wilson 572b414fcd5SChris Wilson err = create_mmap_offset(obj); 573b414fcd5SChris Wilson i915_gem_object_put(obj); 574b414fcd5SChris Wilson 575b414fcd5SChris Wilson return err == expected; 576b414fcd5SChris Wilson } 577b414fcd5SChris Wilson 578b414fcd5SChris Wilson static void disable_retire_worker(struct drm_i915_private *i915) 579b414fcd5SChris Wilson { 580c29579d2SChris Wilson i915_gem_driver_unregister__shrinker(i915); 5810c91621cSChris Wilson intel_gt_pm_get(&i915->gt); 58266101975SChris Wilson cancel_delayed_work_sync(&i915->gt.requests.retire_work); 583b414fcd5SChris Wilson } 584b414fcd5SChris Wilson 585b414fcd5SChris Wilson static void restore_retire_worker(struct drm_i915_private *i915) 586b414fcd5SChris Wilson { 5877e805762SChris Wilson igt_flush_test(i915); 5880c91621cSChris Wilson intel_gt_pm_put(&i915->gt); 589c29579d2SChris Wilson i915_gem_driver_register__shrinker(i915); 590b414fcd5SChris Wilson } 591b414fcd5SChris Wilson 592f63dfc14SChris Wilson static void mmap_offset_lock(struct drm_i915_private *i915) 593f63dfc14SChris Wilson __acquires(&i915->drm.vma_offset_manager->vm_lock) 594f63dfc14SChris Wilson { 595f63dfc14SChris Wilson write_lock(&i915->drm.vma_offset_manager->vm_lock); 596f63dfc14SChris Wilson } 597f63dfc14SChris Wilson 598f63dfc14SChris Wilson static void mmap_offset_unlock(struct drm_i915_private *i915) 599f63dfc14SChris Wilson __releases(&i915->drm.vma_offset_manager->vm_lock) 600f63dfc14SChris Wilson { 601f63dfc14SChris Wilson write_unlock(&i915->drm.vma_offset_manager->vm_lock); 602f63dfc14SChris Wilson } 603f63dfc14SChris Wilson 604b414fcd5SChris Wilson static int igt_mmap_offset_exhaustion(void *arg) 605b414fcd5SChris Wilson { 606b414fcd5SChris Wilson struct drm_i915_private *i915 = arg; 607b414fcd5SChris Wilson struct drm_mm *mm = &i915->drm.vma_offset_manager->vm_addr_space_mm; 608b414fcd5SChris Wilson struct drm_i915_gem_object *obj; 609b414fcd5SChris Wilson struct drm_mm_node resv, *hole; 610b414fcd5SChris Wilson u64 hole_start, hole_end; 611b414fcd5SChris Wilson int loop, err; 612b414fcd5SChris Wilson 613b414fcd5SChris Wilson /* Disable background reaper */ 614b414fcd5SChris Wilson disable_retire_worker(i915); 615b414fcd5SChris Wilson GEM_BUG_ON(!i915->gt.awake); 616b414fcd5SChris Wilson 617b414fcd5SChris Wilson /* Trim the device mmap space to only a page */ 618b414fcd5SChris Wilson memset(&resv, 0, sizeof(resv)); 619b414fcd5SChris Wilson drm_mm_for_each_hole(hole, mm, hole_start, hole_end) { 620b414fcd5SChris Wilson resv.start = hole_start; 621b414fcd5SChris Wilson resv.size = hole_end - hole_start - 1; /* PAGE_SIZE units */ 622f63dfc14SChris Wilson mmap_offset_lock(i915); 623b414fcd5SChris Wilson err = drm_mm_reserve_node(mm, &resv); 624f63dfc14SChris Wilson mmap_offset_unlock(i915); 625b414fcd5SChris Wilson if (err) { 626b414fcd5SChris Wilson pr_err("Failed to trim VMA manager, err=%d\n", err); 627b414fcd5SChris Wilson goto out_park; 628b414fcd5SChris Wilson } 629b414fcd5SChris Wilson break; 630b414fcd5SChris Wilson } 631b414fcd5SChris Wilson 632b414fcd5SChris Wilson /* Just fits! */ 633b414fcd5SChris Wilson if (!assert_mmap_offset(i915, PAGE_SIZE, 0)) { 634b414fcd5SChris Wilson pr_err("Unable to insert object into single page hole\n"); 635b414fcd5SChris Wilson err = -EINVAL; 636b414fcd5SChris Wilson goto out; 637b414fcd5SChris Wilson } 638b414fcd5SChris Wilson 639b414fcd5SChris Wilson /* Too large */ 640b414fcd5SChris Wilson if (!assert_mmap_offset(i915, 2 * PAGE_SIZE, -ENOSPC)) { 641b414fcd5SChris Wilson pr_err("Unexpectedly succeeded in inserting too large object into single page hole\n"); 642b414fcd5SChris Wilson err = -EINVAL; 643b414fcd5SChris Wilson goto out; 644b414fcd5SChris Wilson } 645b414fcd5SChris Wilson 646b414fcd5SChris Wilson /* Fill the hole, further allocation attempts should then fail */ 647b414fcd5SChris Wilson obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 648b414fcd5SChris Wilson if (IS_ERR(obj)) { 649b414fcd5SChris Wilson err = PTR_ERR(obj); 650b414fcd5SChris Wilson goto out; 651b414fcd5SChris Wilson } 652b414fcd5SChris Wilson 653b414fcd5SChris Wilson err = create_mmap_offset(obj); 654b414fcd5SChris Wilson if (err) { 655b414fcd5SChris Wilson pr_err("Unable to insert object into reclaimed hole\n"); 656b414fcd5SChris Wilson goto err_obj; 657b414fcd5SChris Wilson } 658b414fcd5SChris Wilson 659b414fcd5SChris Wilson if (!assert_mmap_offset(i915, PAGE_SIZE, -ENOSPC)) { 660b414fcd5SChris Wilson pr_err("Unexpectedly succeeded in inserting object into no holes!\n"); 661b414fcd5SChris Wilson err = -EINVAL; 662b414fcd5SChris Wilson goto err_obj; 663b414fcd5SChris Wilson } 664b414fcd5SChris Wilson 665b414fcd5SChris Wilson i915_gem_object_put(obj); 666b414fcd5SChris Wilson 667b414fcd5SChris Wilson /* Now fill with busy dead objects that we expect to reap */ 668b414fcd5SChris Wilson for (loop = 0; loop < 3; loop++) { 669cb823ed9SChris Wilson if (intel_gt_is_wedged(&i915->gt)) 670b414fcd5SChris Wilson break; 671b414fcd5SChris Wilson 672b414fcd5SChris Wilson obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 673b414fcd5SChris Wilson if (IS_ERR(obj)) { 674b414fcd5SChris Wilson err = PTR_ERR(obj); 675b414fcd5SChris Wilson goto out; 676b414fcd5SChris Wilson } 677b414fcd5SChris Wilson 678b414fcd5SChris Wilson err = make_obj_busy(obj); 679b414fcd5SChris Wilson if (err) { 680b414fcd5SChris Wilson pr_err("[loop %d] Failed to busy the object\n", loop); 681b414fcd5SChris Wilson goto err_obj; 682b414fcd5SChris Wilson } 683b414fcd5SChris Wilson } 684b414fcd5SChris Wilson 685b414fcd5SChris Wilson out: 686f63dfc14SChris Wilson mmap_offset_lock(i915); 687b414fcd5SChris Wilson drm_mm_remove_node(&resv); 688f63dfc14SChris Wilson mmap_offset_unlock(i915); 689b414fcd5SChris Wilson out_park: 690b414fcd5SChris Wilson restore_retire_worker(i915); 691b414fcd5SChris Wilson return err; 692b414fcd5SChris Wilson err_obj: 693b414fcd5SChris Wilson i915_gem_object_put(obj); 694b414fcd5SChris Wilson goto out; 695b414fcd5SChris Wilson } 696b414fcd5SChris Wilson 697b414fcd5SChris Wilson int i915_gem_mman_live_selftests(struct drm_i915_private *i915) 698b414fcd5SChris Wilson { 699b414fcd5SChris Wilson static const struct i915_subtest tests[] = { 700b414fcd5SChris Wilson SUBTEST(igt_partial_tiling), 70107e98eb0SChris Wilson SUBTEST(igt_smoke_tiling), 702b414fcd5SChris Wilson SUBTEST(igt_mmap_offset_exhaustion), 703b414fcd5SChris Wilson }; 704b414fcd5SChris Wilson 705b414fcd5SChris Wilson return i915_subtests(tests, i915); 706b414fcd5SChris Wilson } 707