1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2014-2016 Intel Corporation 5 */ 6 7 #include "i915_drv.h" 8 #include "i915_gem_object.h" 9 #include "i915_scatterlist.h" 10 11 void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, 12 struct sg_table *pages, 13 unsigned int sg_page_sizes) 14 { 15 struct drm_i915_private *i915 = to_i915(obj->base.dev); 16 unsigned long supported = INTEL_INFO(i915)->page_sizes; 17 int i; 18 19 lockdep_assert_held(&obj->mm.lock); 20 21 /* Make the pages coherent with the GPU (flushing any swapin). */ 22 if (obj->cache_dirty) { 23 obj->write_domain = 0; 24 if (i915_gem_object_has_struct_page(obj)) 25 drm_clflush_sg(pages); 26 obj->cache_dirty = false; 27 } 28 29 obj->mm.get_page.sg_pos = pages->sgl; 30 obj->mm.get_page.sg_idx = 0; 31 32 obj->mm.pages = pages; 33 34 if (i915_gem_object_is_tiled(obj) && 35 i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) { 36 GEM_BUG_ON(obj->mm.quirked); 37 __i915_gem_object_pin_pages(obj); 38 obj->mm.quirked = true; 39 } 40 41 GEM_BUG_ON(!sg_page_sizes); 42 obj->mm.page_sizes.phys = sg_page_sizes; 43 44 /* 45 * Calculate the supported page-sizes which fit into the given 46 * sg_page_sizes. This will give us the page-sizes which we may be able 47 * to use opportunistically when later inserting into the GTT. For 48 * example if phys=2G, then in theory we should be able to use 1G, 2M, 49 * 64K or 4K pages, although in practice this will depend on a number of 50 * other factors. 51 */ 52 obj->mm.page_sizes.sg = 0; 53 for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) { 54 if (obj->mm.page_sizes.phys & ~0u << i) 55 obj->mm.page_sizes.sg |= BIT(i); 56 } 57 GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg)); 58 59 if (i915_gem_object_is_shrinkable(obj)) { 60 struct list_head *list; 61 unsigned long flags; 62 63 spin_lock_irqsave(&i915->mm.obj_lock, flags); 64 65 i915->mm.shrink_count++; 66 i915->mm.shrink_memory += obj->base.size; 67 68 if (obj->mm.madv != I915_MADV_WILLNEED) 69 list = &i915->mm.purge_list; 70 else 71 list = &i915->mm.shrink_list; 72 list_add_tail(&obj->mm.link, list); 73 74 spin_unlock_irqrestore(&i915->mm.obj_lock, flags); 75 } 76 } 77 78 int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj) 79 { 80 int err; 81 82 if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) { 83 DRM_DEBUG("Attempting to obtain a purgeable object\n"); 84 return -EFAULT; 85 } 86 87 err = obj->ops->get_pages(obj); 88 GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj)); 89 90 return err; 91 } 92 93 /* Ensure that the associated pages are gathered from the backing storage 94 * and pinned into our object. i915_gem_object_pin_pages() may be called 95 * multiple times before they are released by a single call to 96 * i915_gem_object_unpin_pages() - once the pages are no longer referenced 97 * either as a result of memory pressure (reaping pages under the shrinker) 98 * or as the object is itself released. 99 */ 100 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj) 101 { 102 int err; 103 104 err = mutex_lock_interruptible(&obj->mm.lock); 105 if (err) 106 return err; 107 108 if (unlikely(!i915_gem_object_has_pages(obj))) { 109 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj)); 110 111 err = ____i915_gem_object_get_pages(obj); 112 if (err) 113 goto unlock; 114 115 smp_mb__before_atomic(); 116 } 117 atomic_inc(&obj->mm.pages_pin_count); 118 119 unlock: 120 mutex_unlock(&obj->mm.lock); 121 return err; 122 } 123 124 /* Immediately discard the backing storage */ 125 void i915_gem_object_truncate(struct drm_i915_gem_object *obj) 126 { 127 drm_gem_free_mmap_offset(&obj->base); 128 if (obj->ops->truncate) 129 obj->ops->truncate(obj); 130 } 131 132 /* Try to discard unwanted pages */ 133 void i915_gem_object_writeback(struct drm_i915_gem_object *obj) 134 { 135 lockdep_assert_held(&obj->mm.lock); 136 GEM_BUG_ON(i915_gem_object_has_pages(obj)); 137 138 if (obj->ops->writeback) 139 obj->ops->writeback(obj); 140 } 141 142 static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj) 143 { 144 struct radix_tree_iter iter; 145 void __rcu **slot; 146 147 rcu_read_lock(); 148 radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0) 149 radix_tree_delete(&obj->mm.get_page.radix, iter.index); 150 rcu_read_unlock(); 151 } 152 153 struct sg_table * 154 __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj) 155 { 156 struct drm_i915_private *i915 = to_i915(obj->base.dev); 157 struct sg_table *pages; 158 159 pages = fetch_and_zero(&obj->mm.pages); 160 if (IS_ERR_OR_NULL(pages)) 161 return pages; 162 163 if (i915_gem_object_is_shrinkable(obj)) { 164 unsigned long flags; 165 166 spin_lock_irqsave(&i915->mm.obj_lock, flags); 167 168 list_del(&obj->mm.link); 169 i915->mm.shrink_count--; 170 i915->mm.shrink_memory -= obj->base.size; 171 172 spin_unlock_irqrestore(&i915->mm.obj_lock, flags); 173 } 174 175 if (obj->mm.mapping) { 176 void *ptr; 177 178 ptr = page_mask_bits(obj->mm.mapping); 179 if (is_vmalloc_addr(ptr)) 180 vunmap(ptr); 181 else 182 kunmap(kmap_to_page(ptr)); 183 184 obj->mm.mapping = NULL; 185 } 186 187 __i915_gem_object_reset_page_iter(obj); 188 obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0; 189 190 return pages; 191 } 192 193 int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj, 194 enum i915_mm_subclass subclass) 195 { 196 struct sg_table *pages; 197 int err; 198 199 if (i915_gem_object_has_pinned_pages(obj)) 200 return -EBUSY; 201 202 GEM_BUG_ON(atomic_read(&obj->bind_count)); 203 204 /* May be called by shrinker from within get_pages() (on another bo) */ 205 mutex_lock_nested(&obj->mm.lock, subclass); 206 if (unlikely(atomic_read(&obj->mm.pages_pin_count))) { 207 err = -EBUSY; 208 goto unlock; 209 } 210 211 /* 212 * ->put_pages might need to allocate memory for the bit17 swizzle 213 * array, hence protect them from being reaped by removing them from gtt 214 * lists early. 215 */ 216 pages = __i915_gem_object_unset_pages(obj); 217 218 /* 219 * XXX Temporary hijinx to avoid updating all backends to handle 220 * NULL pages. In the future, when we have more asynchronous 221 * get_pages backends we should be better able to handle the 222 * cancellation of the async task in a more uniform manner. 223 */ 224 if (!pages && !i915_gem_object_needs_async_cancel(obj)) 225 pages = ERR_PTR(-EINVAL); 226 227 if (!IS_ERR(pages)) 228 obj->ops->put_pages(obj, pages); 229 230 err = 0; 231 unlock: 232 mutex_unlock(&obj->mm.lock); 233 234 return err; 235 } 236 237 /* The 'mapping' part of i915_gem_object_pin_map() below */ 238 static void *i915_gem_object_map(const struct drm_i915_gem_object *obj, 239 enum i915_map_type type) 240 { 241 unsigned long n_pages = obj->base.size >> PAGE_SHIFT; 242 struct sg_table *sgt = obj->mm.pages; 243 struct sgt_iter sgt_iter; 244 struct page *page; 245 struct page *stack_pages[32]; 246 struct page **pages = stack_pages; 247 unsigned long i = 0; 248 pgprot_t pgprot; 249 void *addr; 250 251 /* A single page can always be kmapped */ 252 if (n_pages == 1 && type == I915_MAP_WB) 253 return kmap(sg_page(sgt->sgl)); 254 255 if (n_pages > ARRAY_SIZE(stack_pages)) { 256 /* Too big for stack -- allocate temporary array instead */ 257 pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL); 258 if (!pages) 259 return NULL; 260 } 261 262 for_each_sgt_page(page, sgt_iter, sgt) 263 pages[i++] = page; 264 265 /* Check that we have the expected number of pages */ 266 GEM_BUG_ON(i != n_pages); 267 268 switch (type) { 269 default: 270 MISSING_CASE(type); 271 /* fallthrough to use PAGE_KERNEL anyway */ 272 case I915_MAP_WB: 273 pgprot = PAGE_KERNEL; 274 break; 275 case I915_MAP_WC: 276 pgprot = pgprot_writecombine(PAGE_KERNEL_IO); 277 break; 278 } 279 addr = vmap(pages, n_pages, 0, pgprot); 280 281 if (pages != stack_pages) 282 kvfree(pages); 283 284 return addr; 285 } 286 287 /* get, pin, and map the pages of the object into kernel space */ 288 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, 289 enum i915_map_type type) 290 { 291 enum i915_map_type has_type; 292 bool pinned; 293 void *ptr; 294 int err; 295 296 if (unlikely(!i915_gem_object_has_struct_page(obj))) 297 return ERR_PTR(-ENXIO); 298 299 err = mutex_lock_interruptible(&obj->mm.lock); 300 if (err) 301 return ERR_PTR(err); 302 303 pinned = !(type & I915_MAP_OVERRIDE); 304 type &= ~I915_MAP_OVERRIDE; 305 306 if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) { 307 if (unlikely(!i915_gem_object_has_pages(obj))) { 308 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj)); 309 310 err = ____i915_gem_object_get_pages(obj); 311 if (err) 312 goto err_unlock; 313 314 smp_mb__before_atomic(); 315 } 316 atomic_inc(&obj->mm.pages_pin_count); 317 pinned = false; 318 } 319 GEM_BUG_ON(!i915_gem_object_has_pages(obj)); 320 321 ptr = page_unpack_bits(obj->mm.mapping, &has_type); 322 if (ptr && has_type != type) { 323 if (pinned) { 324 err = -EBUSY; 325 goto err_unpin; 326 } 327 328 if (is_vmalloc_addr(ptr)) 329 vunmap(ptr); 330 else 331 kunmap(kmap_to_page(ptr)); 332 333 ptr = obj->mm.mapping = NULL; 334 } 335 336 if (!ptr) { 337 ptr = i915_gem_object_map(obj, type); 338 if (!ptr) { 339 err = -ENOMEM; 340 goto err_unpin; 341 } 342 343 obj->mm.mapping = page_pack_bits(ptr, type); 344 } 345 346 out_unlock: 347 mutex_unlock(&obj->mm.lock); 348 return ptr; 349 350 err_unpin: 351 atomic_dec(&obj->mm.pages_pin_count); 352 err_unlock: 353 ptr = ERR_PTR(err); 354 goto out_unlock; 355 } 356 357 void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj, 358 unsigned long offset, 359 unsigned long size) 360 { 361 enum i915_map_type has_type; 362 void *ptr; 363 364 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 365 GEM_BUG_ON(range_overflows_t(typeof(obj->base.size), 366 offset, size, obj->base.size)); 367 368 obj->mm.dirty = true; 369 370 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE) 371 return; 372 373 ptr = page_unpack_bits(obj->mm.mapping, &has_type); 374 if (has_type == I915_MAP_WC) 375 return; 376 377 drm_clflush_virt_range(ptr + offset, size); 378 if (size == obj->base.size) { 379 obj->write_domain &= ~I915_GEM_DOMAIN_CPU; 380 obj->cache_dirty = false; 381 } 382 } 383 384 struct scatterlist * 385 i915_gem_object_get_sg(struct drm_i915_gem_object *obj, 386 unsigned int n, 387 unsigned int *offset) 388 { 389 struct i915_gem_object_page_iter *iter = &obj->mm.get_page; 390 struct scatterlist *sg; 391 unsigned int idx, count; 392 393 might_sleep(); 394 GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT); 395 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 396 397 /* As we iterate forward through the sg, we record each entry in a 398 * radixtree for quick repeated (backwards) lookups. If we have seen 399 * this index previously, we will have an entry for it. 400 * 401 * Initial lookup is O(N), but this is amortized to O(1) for 402 * sequential page access (where each new request is consecutive 403 * to the previous one). Repeated lookups are O(lg(obj->base.size)), 404 * i.e. O(1) with a large constant! 405 */ 406 if (n < READ_ONCE(iter->sg_idx)) 407 goto lookup; 408 409 mutex_lock(&iter->lock); 410 411 /* We prefer to reuse the last sg so that repeated lookup of this 412 * (or the subsequent) sg are fast - comparing against the last 413 * sg is faster than going through the radixtree. 414 */ 415 416 sg = iter->sg_pos; 417 idx = iter->sg_idx; 418 count = __sg_page_count(sg); 419 420 while (idx + count <= n) { 421 void *entry; 422 unsigned long i; 423 int ret; 424 425 /* If we cannot allocate and insert this entry, or the 426 * individual pages from this range, cancel updating the 427 * sg_idx so that on this lookup we are forced to linearly 428 * scan onwards, but on future lookups we will try the 429 * insertion again (in which case we need to be careful of 430 * the error return reporting that we have already inserted 431 * this index). 432 */ 433 ret = radix_tree_insert(&iter->radix, idx, sg); 434 if (ret && ret != -EEXIST) 435 goto scan; 436 437 entry = xa_mk_value(idx); 438 for (i = 1; i < count; i++) { 439 ret = radix_tree_insert(&iter->radix, idx + i, entry); 440 if (ret && ret != -EEXIST) 441 goto scan; 442 } 443 444 idx += count; 445 sg = ____sg_next(sg); 446 count = __sg_page_count(sg); 447 } 448 449 scan: 450 iter->sg_pos = sg; 451 iter->sg_idx = idx; 452 453 mutex_unlock(&iter->lock); 454 455 if (unlikely(n < idx)) /* insertion completed by another thread */ 456 goto lookup; 457 458 /* In case we failed to insert the entry into the radixtree, we need 459 * to look beyond the current sg. 460 */ 461 while (idx + count <= n) { 462 idx += count; 463 sg = ____sg_next(sg); 464 count = __sg_page_count(sg); 465 } 466 467 *offset = n - idx; 468 return sg; 469 470 lookup: 471 rcu_read_lock(); 472 473 sg = radix_tree_lookup(&iter->radix, n); 474 GEM_BUG_ON(!sg); 475 476 /* If this index is in the middle of multi-page sg entry, 477 * the radix tree will contain a value entry that points 478 * to the start of that range. We will return the pointer to 479 * the base page and the offset of this page within the 480 * sg entry's range. 481 */ 482 *offset = 0; 483 if (unlikely(xa_is_value(sg))) { 484 unsigned long base = xa_to_value(sg); 485 486 sg = radix_tree_lookup(&iter->radix, base); 487 GEM_BUG_ON(!sg); 488 489 *offset = n - base; 490 } 491 492 rcu_read_unlock(); 493 494 return sg; 495 } 496 497 struct page * 498 i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n) 499 { 500 struct scatterlist *sg; 501 unsigned int offset; 502 503 GEM_BUG_ON(!i915_gem_object_has_struct_page(obj)); 504 505 sg = i915_gem_object_get_sg(obj, n, &offset); 506 return nth_page(sg_page(sg), offset); 507 } 508 509 /* Like i915_gem_object_get_page(), but mark the returned page dirty */ 510 struct page * 511 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, 512 unsigned int n) 513 { 514 struct page *page; 515 516 page = i915_gem_object_get_page(obj, n); 517 if (!obj->mm.dirty) 518 set_page_dirty(page); 519 520 return page; 521 } 522 523 dma_addr_t 524 i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj, 525 unsigned long n, 526 unsigned int *len) 527 { 528 struct scatterlist *sg; 529 unsigned int offset; 530 531 sg = i915_gem_object_get_sg(obj, n, &offset); 532 533 if (len) 534 *len = sg_dma_len(sg) - (offset << PAGE_SHIFT); 535 536 return sg_dma_address(sg) + (offset << PAGE_SHIFT); 537 } 538 539 dma_addr_t 540 i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, 541 unsigned long n) 542 { 543 return i915_gem_object_get_dma_address_len(obj, n, NULL); 544 } 545