1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2014-2016 Intel Corporation 5 */ 6 7 #include <linux/pagevec.h> 8 #include <linux/shmem_fs.h> 9 #include <linux/swap.h> 10 11 #include <drm/drm_cache.h> 12 13 #include "gem/i915_gem_region.h" 14 #include "i915_drv.h" 15 #include "i915_gem_object.h" 16 #include "i915_gem_tiling.h" 17 #include "i915_gemfs.h" 18 #include "i915_scatterlist.h" 19 #include "i915_trace.h" 20 21 /* 22 * Move folios to appropriate lru and release the batch, decrementing the 23 * ref count of those folios. 24 */ 25 static void check_release_folio_batch(struct folio_batch *fbatch) 26 { 27 check_move_unevictable_folios(fbatch); 28 __folio_batch_release(fbatch); 29 cond_resched(); 30 } 31 32 void shmem_sg_free_table(struct sg_table *st, struct address_space *mapping, 33 bool dirty, bool backup) 34 { 35 struct sgt_iter sgt_iter; 36 struct folio_batch fbatch; 37 struct folio *last = NULL; 38 struct page *page; 39 40 mapping_clear_unevictable(mapping); 41 42 folio_batch_init(&fbatch); 43 for_each_sgt_page(page, sgt_iter, st) { 44 struct folio *folio = page_folio(page); 45 46 if (folio == last) 47 continue; 48 last = folio; 49 if (dirty) 50 folio_mark_dirty(folio); 51 if (backup) 52 folio_mark_accessed(folio); 53 54 if (!folio_batch_add(&fbatch, folio)) 55 check_release_folio_batch(&fbatch); 56 } 57 if (fbatch.nr) 58 check_release_folio_batch(&fbatch); 59 60 sg_free_table(st); 61 } 62 63 int shmem_sg_alloc_table(struct drm_i915_private *i915, struct sg_table *st, 64 size_t size, struct intel_memory_region *mr, 65 struct address_space *mapping, 66 unsigned int max_segment) 67 { 68 unsigned int page_count; /* restricted by sg_alloc_table */ 69 unsigned long i; 70 struct scatterlist *sg; 71 unsigned long next_pfn = 0; /* suppress gcc warning */ 72 gfp_t noreclaim; 73 int ret; 74 75 if (overflows_type(size / PAGE_SIZE, page_count)) 76 return -E2BIG; 77 78 page_count = size / PAGE_SIZE; 79 /* 80 * If there's no chance of allocating enough pages for the whole 81 * object, bail early. 82 */ 83 if (size > resource_size(&mr->region)) 84 return -ENOMEM; 85 86 if (sg_alloc_table(st, page_count, GFP_KERNEL | __GFP_NOWARN)) 87 return -ENOMEM; 88 89 /* 90 * Get the list of pages out of our struct file. They'll be pinned 91 * at this point until we release them. 92 * 93 * Fail silently without starting the shrinker 94 */ 95 mapping_set_unevictable(mapping); 96 noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM); 97 noreclaim |= __GFP_NORETRY | __GFP_NOWARN; 98 99 sg = st->sgl; 100 st->nents = 0; 101 for (i = 0; i < page_count; i++) { 102 struct folio *folio; 103 unsigned long nr_pages; 104 const unsigned int shrink[] = { 105 I915_SHRINK_BOUND | I915_SHRINK_UNBOUND, 106 0, 107 }, *s = shrink; 108 gfp_t gfp = noreclaim; 109 110 do { 111 cond_resched(); 112 folio = shmem_read_folio_gfp(mapping, i, gfp); 113 if (!IS_ERR(folio)) 114 break; 115 116 if (!*s) { 117 ret = PTR_ERR(folio); 118 goto err_sg; 119 } 120 121 i915_gem_shrink(NULL, i915, 2 * page_count, NULL, *s++); 122 123 /* 124 * We've tried hard to allocate the memory by reaping 125 * our own buffer, now let the real VM do its job and 126 * go down in flames if truly OOM. 127 * 128 * However, since graphics tend to be disposable, 129 * defer the oom here by reporting the ENOMEM back 130 * to userspace. 131 */ 132 if (!*s) { 133 /* reclaim and warn, but no oom */ 134 gfp = mapping_gfp_mask(mapping); 135 136 /* 137 * Our bo are always dirty and so we require 138 * kswapd to reclaim our pages (direct reclaim 139 * does not effectively begin pageout of our 140 * buffers on its own). However, direct reclaim 141 * only waits for kswapd when under allocation 142 * congestion. So as a result __GFP_RECLAIM is 143 * unreliable and fails to actually reclaim our 144 * dirty pages -- unless you try over and over 145 * again with !__GFP_NORETRY. However, we still 146 * want to fail this allocation rather than 147 * trigger the out-of-memory killer and for 148 * this we want __GFP_RETRY_MAYFAIL. 149 */ 150 gfp |= __GFP_RETRY_MAYFAIL | __GFP_NOWARN; 151 } 152 } while (1); 153 154 nr_pages = min_t(unsigned long, 155 folio_nr_pages(folio), page_count - i); 156 if (!i || 157 sg->length >= max_segment || 158 folio_pfn(folio) != next_pfn) { 159 if (i) 160 sg = sg_next(sg); 161 162 st->nents++; 163 sg_set_folio(sg, folio, nr_pages * PAGE_SIZE, 0); 164 } else { 165 /* XXX: could overflow? */ 166 sg->length += nr_pages * PAGE_SIZE; 167 } 168 next_pfn = folio_pfn(folio) + nr_pages; 169 i += nr_pages - 1; 170 171 /* Check that the i965g/gm workaround works. */ 172 GEM_BUG_ON(gfp & __GFP_DMA32 && next_pfn >= 0x00100000UL); 173 } 174 if (sg) /* loop terminated early; short sg table */ 175 sg_mark_end(sg); 176 177 /* Trim unused sg entries to avoid wasting memory. */ 178 i915_sg_trim(st); 179 180 return 0; 181 err_sg: 182 sg_mark_end(sg); 183 if (sg != st->sgl) { 184 shmem_sg_free_table(st, mapping, false, false); 185 } else { 186 mapping_clear_unevictable(mapping); 187 sg_free_table(st); 188 } 189 190 /* 191 * shmemfs first checks if there is enough memory to allocate the page 192 * and reports ENOSPC should there be insufficient, along with the usual 193 * ENOMEM for a genuine allocation failure. 194 * 195 * We use ENOSPC in our driver to mean that we have run out of aperture 196 * space and so want to translate the error from shmemfs back to our 197 * usual understanding of ENOMEM. 198 */ 199 if (ret == -ENOSPC) 200 ret = -ENOMEM; 201 202 return ret; 203 } 204 205 static int shmem_get_pages(struct drm_i915_gem_object *obj) 206 { 207 struct drm_i915_private *i915 = to_i915(obj->base.dev); 208 struct intel_memory_region *mem = obj->mm.region; 209 struct address_space *mapping = obj->base.filp->f_mapping; 210 unsigned int max_segment = i915_sg_segment_size(i915->drm.dev); 211 struct sg_table *st; 212 struct sgt_iter sgt_iter; 213 struct page *page; 214 int ret; 215 216 /* 217 * Assert that the object is not currently in any GPU domain. As it 218 * wasn't in the GTT, there shouldn't be any way it could have been in 219 * a GPU cache 220 */ 221 GEM_BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS); 222 GEM_BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS); 223 224 rebuild_st: 225 st = kmalloc(sizeof(*st), GFP_KERNEL | __GFP_NOWARN); 226 if (!st) 227 return -ENOMEM; 228 229 ret = shmem_sg_alloc_table(i915, st, obj->base.size, mem, mapping, 230 max_segment); 231 if (ret) 232 goto err_st; 233 234 ret = i915_gem_gtt_prepare_pages(obj, st); 235 if (ret) { 236 /* 237 * DMA remapping failed? One possible cause is that 238 * it could not reserve enough large entries, asking 239 * for PAGE_SIZE chunks instead may be helpful. 240 */ 241 if (max_segment > PAGE_SIZE) { 242 for_each_sgt_page(page, sgt_iter, st) 243 put_page(page); 244 sg_free_table(st); 245 kfree(st); 246 247 max_segment = PAGE_SIZE; 248 goto rebuild_st; 249 } else { 250 dev_warn(i915->drm.dev, 251 "Failed to DMA remap %zu pages\n", 252 obj->base.size >> PAGE_SHIFT); 253 goto err_pages; 254 } 255 } 256 257 if (i915_gem_object_needs_bit17_swizzle(obj)) 258 i915_gem_object_do_bit_17_swizzle(obj, st); 259 260 if (i915_gem_object_can_bypass_llc(obj)) 261 obj->cache_dirty = true; 262 263 __i915_gem_object_set_pages(obj, st); 264 265 return 0; 266 267 err_pages: 268 shmem_sg_free_table(st, mapping, false, false); 269 /* 270 * shmemfs first checks if there is enough memory to allocate the page 271 * and reports ENOSPC should there be insufficient, along with the usual 272 * ENOMEM for a genuine allocation failure. 273 * 274 * We use ENOSPC in our driver to mean that we have run out of aperture 275 * space and so want to translate the error from shmemfs back to our 276 * usual understanding of ENOMEM. 277 */ 278 err_st: 279 if (ret == -ENOSPC) 280 ret = -ENOMEM; 281 282 kfree(st); 283 284 return ret; 285 } 286 287 static int 288 shmem_truncate(struct drm_i915_gem_object *obj) 289 { 290 /* 291 * Our goal here is to return as much of the memory as 292 * is possible back to the system as we are called from OOM. 293 * To do this we must instruct the shmfs to drop all of its 294 * backing pages, *now*. 295 */ 296 shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1); 297 obj->mm.madv = __I915_MADV_PURGED; 298 obj->mm.pages = ERR_PTR(-EFAULT); 299 300 return 0; 301 } 302 303 void __shmem_writeback(size_t size, struct address_space *mapping) 304 { 305 struct writeback_control wbc = { 306 .sync_mode = WB_SYNC_NONE, 307 .nr_to_write = SWAP_CLUSTER_MAX, 308 .range_start = 0, 309 .range_end = LLONG_MAX, 310 .for_reclaim = 1, 311 }; 312 unsigned long i; 313 314 /* 315 * Leave mmapings intact (GTT will have been revoked on unbinding, 316 * leaving only CPU mmapings around) and add those pages to the LRU 317 * instead of invoking writeback so they are aged and paged out 318 * as normal. 319 */ 320 321 /* Begin writeback on each dirty page */ 322 for (i = 0; i < size >> PAGE_SHIFT; i++) { 323 struct page *page; 324 325 page = find_lock_page(mapping, i); 326 if (!page) 327 continue; 328 329 if (!page_mapped(page) && clear_page_dirty_for_io(page)) { 330 int ret; 331 332 SetPageReclaim(page); 333 ret = mapping->a_ops->writepage(page, &wbc); 334 if (!PageWriteback(page)) 335 ClearPageReclaim(page); 336 if (!ret) 337 goto put; 338 } 339 unlock_page(page); 340 put: 341 put_page(page); 342 } 343 } 344 345 static void 346 shmem_writeback(struct drm_i915_gem_object *obj) 347 { 348 __shmem_writeback(obj->base.size, obj->base.filp->f_mapping); 349 } 350 351 static int shmem_shrink(struct drm_i915_gem_object *obj, unsigned int flags) 352 { 353 switch (obj->mm.madv) { 354 case I915_MADV_DONTNEED: 355 return i915_gem_object_truncate(obj); 356 case __I915_MADV_PURGED: 357 return 0; 358 } 359 360 if (flags & I915_GEM_OBJECT_SHRINK_WRITEBACK) 361 shmem_writeback(obj); 362 363 return 0; 364 } 365 366 void 367 __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj, 368 struct sg_table *pages, 369 bool needs_clflush) 370 { 371 struct drm_i915_private *i915 = to_i915(obj->base.dev); 372 373 GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED); 374 375 if (obj->mm.madv == I915_MADV_DONTNEED) 376 obj->mm.dirty = false; 377 378 if (needs_clflush && 379 (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0 && 380 !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)) 381 drm_clflush_sg(pages); 382 383 __start_cpu_write(obj); 384 /* 385 * On non-LLC igfx platforms, force the flush-on-acquire if this is ever 386 * swapped-in. Our async flush path is not trust worthy enough yet(and 387 * happens in the wrong order), and with some tricks it's conceivable 388 * for userspace to change the cache-level to I915_CACHE_NONE after the 389 * pages are swapped-in, and since execbuf binds the object before doing 390 * the async flush, we have a race window. 391 */ 392 if (!HAS_LLC(i915) && !IS_DGFX(i915)) 393 obj->cache_dirty = true; 394 } 395 396 void i915_gem_object_put_pages_shmem(struct drm_i915_gem_object *obj, struct sg_table *pages) 397 { 398 __i915_gem_object_release_shmem(obj, pages, true); 399 400 i915_gem_gtt_finish_pages(obj, pages); 401 402 if (i915_gem_object_needs_bit17_swizzle(obj)) 403 i915_gem_object_save_bit_17_swizzle(obj, pages); 404 405 shmem_sg_free_table(pages, file_inode(obj->base.filp)->i_mapping, 406 obj->mm.dirty, obj->mm.madv == I915_MADV_WILLNEED); 407 kfree(pages); 408 obj->mm.dirty = false; 409 } 410 411 static void 412 shmem_put_pages(struct drm_i915_gem_object *obj, struct sg_table *pages) 413 { 414 if (likely(i915_gem_object_has_struct_page(obj))) 415 i915_gem_object_put_pages_shmem(obj, pages); 416 else 417 i915_gem_object_put_pages_phys(obj, pages); 418 } 419 420 static int 421 shmem_pwrite(struct drm_i915_gem_object *obj, 422 const struct drm_i915_gem_pwrite *arg) 423 { 424 struct address_space *mapping = obj->base.filp->f_mapping; 425 const struct address_space_operations *aops = mapping->a_ops; 426 char __user *user_data = u64_to_user_ptr(arg->data_ptr); 427 u64 remain, offset; 428 unsigned int pg; 429 430 /* Caller already validated user args */ 431 GEM_BUG_ON(!access_ok(user_data, arg->size)); 432 433 if (!i915_gem_object_has_struct_page(obj)) 434 return i915_gem_object_pwrite_phys(obj, arg); 435 436 /* 437 * Before we instantiate/pin the backing store for our use, we 438 * can prepopulate the shmemfs filp efficiently using a write into 439 * the pagecache. We avoid the penalty of instantiating all the 440 * pages, important if the user is just writing to a few and never 441 * uses the object on the GPU, and using a direct write into shmemfs 442 * allows it to avoid the cost of retrieving a page (either swapin 443 * or clearing-before-use) before it is overwritten. 444 */ 445 if (i915_gem_object_has_pages(obj)) 446 return -ENODEV; 447 448 if (obj->mm.madv != I915_MADV_WILLNEED) 449 return -EFAULT; 450 451 /* 452 * Before the pages are instantiated the object is treated as being 453 * in the CPU domain. The pages will be clflushed as required before 454 * use, and we can freely write into the pages directly. If userspace 455 * races pwrite with any other operation; corruption will ensue - 456 * that is userspace's prerogative! 457 */ 458 459 remain = arg->size; 460 offset = arg->offset; 461 pg = offset_in_page(offset); 462 463 do { 464 unsigned int len, unwritten; 465 struct page *page; 466 void *data, *vaddr; 467 int err; 468 char __maybe_unused c; 469 470 len = PAGE_SIZE - pg; 471 if (len > remain) 472 len = remain; 473 474 /* Prefault the user page to reduce potential recursion */ 475 err = __get_user(c, user_data); 476 if (err) 477 return err; 478 479 err = __get_user(c, user_data + len - 1); 480 if (err) 481 return err; 482 483 err = aops->write_begin(obj->base.filp, mapping, offset, len, 484 &page, &data); 485 if (err < 0) 486 return err; 487 488 vaddr = kmap_atomic(page); 489 unwritten = __copy_from_user_inatomic(vaddr + pg, 490 user_data, 491 len); 492 kunmap_atomic(vaddr); 493 494 err = aops->write_end(obj->base.filp, mapping, offset, len, 495 len - unwritten, page, data); 496 if (err < 0) 497 return err; 498 499 /* We don't handle -EFAULT, leave it to the caller to check */ 500 if (unwritten) 501 return -ENODEV; 502 503 remain -= len; 504 user_data += len; 505 offset += len; 506 pg = 0; 507 } while (remain); 508 509 return 0; 510 } 511 512 static int 513 shmem_pread(struct drm_i915_gem_object *obj, 514 const struct drm_i915_gem_pread *arg) 515 { 516 if (!i915_gem_object_has_struct_page(obj)) 517 return i915_gem_object_pread_phys(obj, arg); 518 519 return -ENODEV; 520 } 521 522 static void shmem_release(struct drm_i915_gem_object *obj) 523 { 524 if (i915_gem_object_has_struct_page(obj)) 525 i915_gem_object_release_memory_region(obj); 526 527 fput(obj->base.filp); 528 } 529 530 const struct drm_i915_gem_object_ops i915_gem_shmem_ops = { 531 .name = "i915_gem_object_shmem", 532 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 533 534 .get_pages = shmem_get_pages, 535 .put_pages = shmem_put_pages, 536 .truncate = shmem_truncate, 537 .shrink = shmem_shrink, 538 539 .pwrite = shmem_pwrite, 540 .pread = shmem_pread, 541 542 .release = shmem_release, 543 }; 544 545 static int __create_shmem(struct drm_i915_private *i915, 546 struct drm_gem_object *obj, 547 resource_size_t size) 548 { 549 unsigned long flags = VM_NORESERVE; 550 struct file *filp; 551 552 drm_gem_private_object_init(&i915->drm, obj, size); 553 554 /* XXX: The __shmem_file_setup() function returns -EINVAL if size is 555 * greater than MAX_LFS_FILESIZE. 556 * To handle the same error as other code that returns -E2BIG when 557 * the size is too large, we add a code that returns -E2BIG when the 558 * size is larger than the size that can be handled. 559 * If BITS_PER_LONG is 32, size > MAX_LFS_FILESIZE is always false, 560 * so we only needs to check when BITS_PER_LONG is 64. 561 * If BITS_PER_LONG is 32, E2BIG checks are processed when 562 * i915_gem_object_size_2big() is called before init_object() callback 563 * is called. 564 */ 565 if (BITS_PER_LONG == 64 && size > MAX_LFS_FILESIZE) 566 return -E2BIG; 567 568 if (i915->mm.gemfs) 569 filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size, 570 flags); 571 else 572 filp = shmem_file_setup("i915", size, flags); 573 if (IS_ERR(filp)) 574 return PTR_ERR(filp); 575 576 obj->filp = filp; 577 return 0; 578 } 579 580 static int shmem_object_init(struct intel_memory_region *mem, 581 struct drm_i915_gem_object *obj, 582 resource_size_t offset, 583 resource_size_t size, 584 resource_size_t page_size, 585 unsigned int flags) 586 { 587 static struct lock_class_key lock_class; 588 struct drm_i915_private *i915 = mem->i915; 589 struct address_space *mapping; 590 unsigned int cache_level; 591 gfp_t mask; 592 int ret; 593 594 ret = __create_shmem(i915, &obj->base, size); 595 if (ret) 596 return ret; 597 598 mask = GFP_HIGHUSER | __GFP_RECLAIMABLE; 599 if (IS_I965GM(i915) || IS_I965G(i915)) { 600 /* 965gm cannot relocate objects above 4GiB. */ 601 mask &= ~__GFP_HIGHMEM; 602 mask |= __GFP_DMA32; 603 } 604 605 mapping = obj->base.filp->f_mapping; 606 mapping_set_gfp_mask(mapping, mask); 607 GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM)); 608 609 i915_gem_object_init(obj, &i915_gem_shmem_ops, &lock_class, flags); 610 obj->mem_flags |= I915_BO_FLAG_STRUCT_PAGE; 611 obj->write_domain = I915_GEM_DOMAIN_CPU; 612 obj->read_domains = I915_GEM_DOMAIN_CPU; 613 614 /* 615 * MTL doesn't snoop CPU cache by default for GPU access (namely 616 * 1-way coherency). However some UMD's are currently depending on 617 * that. Make 1-way coherent the default setting for MTL. A follow 618 * up patch will extend the GEM_CREATE uAPI to allow UMD's specify 619 * caching mode at BO creation time 620 */ 621 if (HAS_LLC(i915) || (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))) 622 /* On some devices, we can have the GPU use the LLC (the CPU 623 * cache) for about a 10% performance improvement 624 * compared to uncached. Graphics requests other than 625 * display scanout are coherent with the CPU in 626 * accessing this cache. This means in this mode we 627 * don't need to clflush on the CPU side, and on the 628 * GPU side we only need to flush internal caches to 629 * get data visible to the CPU. 630 * 631 * However, we maintain the display planes as UC, and so 632 * need to rebind when first used as such. 633 */ 634 cache_level = I915_CACHE_LLC; 635 else 636 cache_level = I915_CACHE_NONE; 637 638 i915_gem_object_set_cache_coherency(obj, cache_level); 639 640 i915_gem_object_init_memory_region(obj, mem); 641 642 return 0; 643 } 644 645 struct drm_i915_gem_object * 646 i915_gem_object_create_shmem(struct drm_i915_private *i915, 647 resource_size_t size) 648 { 649 return i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_SMEM], 650 size, 0, 0); 651 } 652 653 /* Allocate a new GEM object and fill it with the supplied data */ 654 struct drm_i915_gem_object * 655 i915_gem_object_create_shmem_from_data(struct drm_i915_private *dev_priv, 656 const void *data, resource_size_t size) 657 { 658 struct drm_i915_gem_object *obj; 659 struct file *file; 660 const struct address_space_operations *aops; 661 resource_size_t offset; 662 int err; 663 664 GEM_WARN_ON(IS_DGFX(dev_priv)); 665 obj = i915_gem_object_create_shmem(dev_priv, round_up(size, PAGE_SIZE)); 666 if (IS_ERR(obj)) 667 return obj; 668 669 GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU); 670 671 file = obj->base.filp; 672 aops = file->f_mapping->a_ops; 673 offset = 0; 674 do { 675 unsigned int len = min_t(typeof(size), size, PAGE_SIZE); 676 struct page *page; 677 void *pgdata, *vaddr; 678 679 err = aops->write_begin(file, file->f_mapping, offset, len, 680 &page, &pgdata); 681 if (err < 0) 682 goto fail; 683 684 vaddr = kmap(page); 685 memcpy(vaddr, data, len); 686 kunmap(page); 687 688 err = aops->write_end(file, file->f_mapping, offset, len, len, 689 page, pgdata); 690 if (err < 0) 691 goto fail; 692 693 size -= len; 694 data += len; 695 offset += len; 696 } while (size); 697 698 return obj; 699 700 fail: 701 i915_gem_object_put(obj); 702 return ERR_PTR(err); 703 } 704 705 static int init_shmem(struct intel_memory_region *mem) 706 { 707 i915_gemfs_init(mem->i915); 708 intel_memory_region_set_name(mem, "system"); 709 710 return 0; /* We have fallback to the kernel mnt if gemfs init failed. */ 711 } 712 713 static int release_shmem(struct intel_memory_region *mem) 714 { 715 i915_gemfs_fini(mem->i915); 716 return 0; 717 } 718 719 static const struct intel_memory_region_ops shmem_region_ops = { 720 .init = init_shmem, 721 .release = release_shmem, 722 .init_object = shmem_object_init, 723 }; 724 725 struct intel_memory_region *i915_gem_shmem_setup(struct drm_i915_private *i915, 726 u16 type, u16 instance) 727 { 728 return intel_memory_region_create(i915, 0, 729 totalram_pages() << PAGE_SHIFT, 730 PAGE_SIZE, 0, 0, 731 type, instance, 732 &shmem_region_ops); 733 } 734 735 bool i915_gem_object_is_shmem(const struct drm_i915_gem_object *obj) 736 { 737 return obj->ops == &i915_gem_shmem_ops; 738 } 739