1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2014-2016 Intel Corporation
5  */
6 
7 #include "i915_drv.h"
8 #include "i915_gem_object.h"
9 #include "i915_scatterlist.h"
10 #include "i915_gem_lmem.h"
11 #include "i915_gem_mman.h"
12 
13 void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
14 				 struct sg_table *pages,
15 				 unsigned int sg_page_sizes)
16 {
17 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
18 	unsigned long supported = INTEL_INFO(i915)->page_sizes;
19 	bool shrinkable;
20 	int i;
21 
22 	assert_object_held_shared(obj);
23 
24 	if (i915_gem_object_is_volatile(obj))
25 		obj->mm.madv = I915_MADV_DONTNEED;
26 
27 	/* Make the pages coherent with the GPU (flushing any swapin). */
28 	if (obj->cache_dirty) {
29 		obj->write_domain = 0;
30 		if (i915_gem_object_has_struct_page(obj))
31 			drm_clflush_sg(pages);
32 		obj->cache_dirty = false;
33 	}
34 
35 	obj->mm.get_page.sg_pos = pages->sgl;
36 	obj->mm.get_page.sg_idx = 0;
37 	obj->mm.get_dma_page.sg_pos = pages->sgl;
38 	obj->mm.get_dma_page.sg_idx = 0;
39 
40 	obj->mm.pages = pages;
41 
42 	GEM_BUG_ON(!sg_page_sizes);
43 	obj->mm.page_sizes.phys = sg_page_sizes;
44 
45 	/*
46 	 * Calculate the supported page-sizes which fit into the given
47 	 * sg_page_sizes. This will give us the page-sizes which we may be able
48 	 * to use opportunistically when later inserting into the GTT. For
49 	 * example if phys=2G, then in theory we should be able to use 1G, 2M,
50 	 * 64K or 4K pages, although in practice this will depend on a number of
51 	 * other factors.
52 	 */
53 	obj->mm.page_sizes.sg = 0;
54 	for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
55 		if (obj->mm.page_sizes.phys & ~0u << i)
56 			obj->mm.page_sizes.sg |= BIT(i);
57 	}
58 	GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg));
59 
60 	shrinkable = i915_gem_object_is_shrinkable(obj);
61 
62 	if (i915_gem_object_is_tiled(obj) &&
63 	    i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
64 		GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj));
65 		i915_gem_object_set_tiling_quirk(obj);
66 		GEM_BUG_ON(!list_empty(&obj->mm.link));
67 		atomic_inc(&obj->mm.shrink_pin);
68 		shrinkable = false;
69 	}
70 
71 	if (shrinkable) {
72 		struct list_head *list;
73 		unsigned long flags;
74 
75 		assert_object_held(obj);
76 		spin_lock_irqsave(&i915->mm.obj_lock, flags);
77 
78 		i915->mm.shrink_count++;
79 		i915->mm.shrink_memory += obj->base.size;
80 
81 		if (obj->mm.madv != I915_MADV_WILLNEED)
82 			list = &i915->mm.purge_list;
83 		else
84 			list = &i915->mm.shrink_list;
85 		list_add_tail(&obj->mm.link, list);
86 
87 		atomic_set(&obj->mm.shrink_pin, 0);
88 		spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
89 	}
90 }
91 
92 int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
93 {
94 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
95 	int err;
96 
97 	assert_object_held_shared(obj);
98 
99 	if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
100 		drm_dbg(&i915->drm,
101 			"Attempting to obtain a purgeable object\n");
102 		return -EFAULT;
103 	}
104 
105 	err = obj->ops->get_pages(obj);
106 	GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj));
107 
108 	return err;
109 }
110 
111 /* Ensure that the associated pages are gathered from the backing storage
112  * and pinned into our object. i915_gem_object_pin_pages() may be called
113  * multiple times before they are released by a single call to
114  * i915_gem_object_unpin_pages() - once the pages are no longer referenced
115  * either as a result of memory pressure (reaping pages under the shrinker)
116  * or as the object is itself released.
117  */
118 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
119 {
120 	int err;
121 
122 	assert_object_held(obj);
123 
124 	assert_object_held_shared(obj);
125 
126 	if (unlikely(!i915_gem_object_has_pages(obj))) {
127 		GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
128 
129 		err = ____i915_gem_object_get_pages(obj);
130 		if (err)
131 			return err;
132 
133 		smp_mb__before_atomic();
134 	}
135 	atomic_inc(&obj->mm.pages_pin_count);
136 
137 	return 0;
138 }
139 
140 int i915_gem_object_pin_pages_unlocked(struct drm_i915_gem_object *obj)
141 {
142 	struct i915_gem_ww_ctx ww;
143 	int err;
144 
145 	i915_gem_ww_ctx_init(&ww, true);
146 retry:
147 	err = i915_gem_object_lock(obj, &ww);
148 	if (!err)
149 		err = i915_gem_object_pin_pages(obj);
150 
151 	if (err == -EDEADLK) {
152 		err = i915_gem_ww_ctx_backoff(&ww);
153 		if (!err)
154 			goto retry;
155 	}
156 	i915_gem_ww_ctx_fini(&ww);
157 	return err;
158 }
159 
160 /* Immediately discard the backing storage */
161 void i915_gem_object_truncate(struct drm_i915_gem_object *obj)
162 {
163 	drm_gem_free_mmap_offset(&obj->base);
164 	if (obj->ops->truncate)
165 		obj->ops->truncate(obj);
166 }
167 
168 /* Try to discard unwanted pages */
169 void i915_gem_object_writeback(struct drm_i915_gem_object *obj)
170 {
171 	assert_object_held_shared(obj);
172 	GEM_BUG_ON(i915_gem_object_has_pages(obj));
173 
174 	if (obj->ops->writeback)
175 		obj->ops->writeback(obj);
176 }
177 
178 static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
179 {
180 	struct radix_tree_iter iter;
181 	void __rcu **slot;
182 
183 	rcu_read_lock();
184 	radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)
185 		radix_tree_delete(&obj->mm.get_page.radix, iter.index);
186 	radix_tree_for_each_slot(slot, &obj->mm.get_dma_page.radix, &iter, 0)
187 		radix_tree_delete(&obj->mm.get_dma_page.radix, iter.index);
188 	rcu_read_unlock();
189 }
190 
191 static void unmap_object(struct drm_i915_gem_object *obj, void *ptr)
192 {
193 	if (is_vmalloc_addr(ptr))
194 		vunmap(ptr);
195 }
196 
197 struct sg_table *
198 __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj)
199 {
200 	struct sg_table *pages;
201 
202 	assert_object_held_shared(obj);
203 
204 	pages = fetch_and_zero(&obj->mm.pages);
205 	if (IS_ERR_OR_NULL(pages))
206 		return pages;
207 
208 	if (i915_gem_object_is_volatile(obj))
209 		obj->mm.madv = I915_MADV_WILLNEED;
210 
211 	i915_gem_object_make_unshrinkable(obj);
212 
213 	if (obj->mm.mapping) {
214 		unmap_object(obj, page_mask_bits(obj->mm.mapping));
215 		obj->mm.mapping = NULL;
216 	}
217 
218 	__i915_gem_object_reset_page_iter(obj);
219 	obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0;
220 
221 	return pages;
222 }
223 
224 int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
225 {
226 	struct sg_table *pages;
227 
228 	if (i915_gem_object_has_pinned_pages(obj))
229 		return -EBUSY;
230 
231 	/* May be called by shrinker from within get_pages() (on another bo) */
232 	assert_object_held_shared(obj);
233 
234 	i915_gem_object_release_mmap_offset(obj);
235 
236 	/*
237 	 * ->put_pages might need to allocate memory for the bit17 swizzle
238 	 * array, hence protect them from being reaped by removing them from gtt
239 	 * lists early.
240 	 */
241 	pages = __i915_gem_object_unset_pages(obj);
242 
243 	/*
244 	 * XXX Temporary hijinx to avoid updating all backends to handle
245 	 * NULL pages. In the future, when we have more asynchronous
246 	 * get_pages backends we should be better able to handle the
247 	 * cancellation of the async task in a more uniform manner.
248 	 */
249 	if (!IS_ERR_OR_NULL(pages))
250 		obj->ops->put_pages(obj, pages);
251 
252 	return 0;
253 }
254 
255 /* The 'mapping' part of i915_gem_object_pin_map() below */
256 static void *i915_gem_object_map_page(struct drm_i915_gem_object *obj,
257 				      enum i915_map_type type)
258 {
259 	unsigned long n_pages = obj->base.size >> PAGE_SHIFT, i;
260 	struct page *stack[32], **pages = stack, *page;
261 	struct sgt_iter iter;
262 	pgprot_t pgprot;
263 	void *vaddr;
264 
265 	switch (type) {
266 	default:
267 		MISSING_CASE(type);
268 		fallthrough;	/* to use PAGE_KERNEL anyway */
269 	case I915_MAP_WB:
270 		/*
271 		 * On 32b, highmem using a finite set of indirect PTE (i.e.
272 		 * vmap) to provide virtual mappings of the high pages.
273 		 * As these are finite, map_new_virtual() must wait for some
274 		 * other kmap() to finish when it runs out. If we map a large
275 		 * number of objects, there is no method for it to tell us
276 		 * to release the mappings, and we deadlock.
277 		 *
278 		 * However, if we make an explicit vmap of the page, that
279 		 * uses a larger vmalloc arena, and also has the ability
280 		 * to tell us to release unwanted mappings. Most importantly,
281 		 * it will fail and propagate an error instead of waiting
282 		 * forever.
283 		 *
284 		 * So if the page is beyond the 32b boundary, make an explicit
285 		 * vmap.
286 		 */
287 		if (n_pages == 1 && !PageHighMem(sg_page(obj->mm.pages->sgl)))
288 			return page_address(sg_page(obj->mm.pages->sgl));
289 		pgprot = PAGE_KERNEL;
290 		break;
291 	case I915_MAP_WC:
292 		pgprot = pgprot_writecombine(PAGE_KERNEL_IO);
293 		break;
294 	}
295 
296 	if (n_pages > ARRAY_SIZE(stack)) {
297 		/* Too big for stack -- allocate temporary array instead */
298 		pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL);
299 		if (!pages)
300 			return ERR_PTR(-ENOMEM);
301 	}
302 
303 	i = 0;
304 	for_each_sgt_page(page, iter, obj->mm.pages)
305 		pages[i++] = page;
306 	vaddr = vmap(pages, n_pages, 0, pgprot);
307 	if (pages != stack)
308 		kvfree(pages);
309 
310 	return vaddr ?: ERR_PTR(-ENOMEM);
311 }
312 
313 static void *i915_gem_object_map_pfn(struct drm_i915_gem_object *obj,
314 				     enum i915_map_type type)
315 {
316 	resource_size_t iomap = obj->mm.region->iomap.base -
317 		obj->mm.region->region.start;
318 	unsigned long n_pfn = obj->base.size >> PAGE_SHIFT;
319 	unsigned long stack[32], *pfns = stack, i;
320 	struct sgt_iter iter;
321 	dma_addr_t addr;
322 	void *vaddr;
323 
324 	if (type != I915_MAP_WC)
325 		return ERR_PTR(-ENODEV);
326 
327 	if (n_pfn > ARRAY_SIZE(stack)) {
328 		/* Too big for stack -- allocate temporary array instead */
329 		pfns = kvmalloc_array(n_pfn, sizeof(*pfns), GFP_KERNEL);
330 		if (!pfns)
331 			return ERR_PTR(-ENOMEM);
332 	}
333 
334 	i = 0;
335 	for_each_sgt_daddr(addr, iter, obj->mm.pages)
336 		pfns[i++] = (iomap + addr) >> PAGE_SHIFT;
337 	vaddr = vmap_pfn(pfns, n_pfn, pgprot_writecombine(PAGE_KERNEL_IO));
338 	if (pfns != stack)
339 		kvfree(pfns);
340 
341 	return vaddr ?: ERR_PTR(-ENOMEM);
342 }
343 
344 /* get, pin, and map the pages of the object into kernel space */
345 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
346 			      enum i915_map_type type)
347 {
348 	enum i915_map_type has_type;
349 	bool pinned;
350 	void *ptr;
351 	int err;
352 
353 	if (!i915_gem_object_has_struct_page(obj) &&
354 	    !i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_IOMEM))
355 		return ERR_PTR(-ENXIO);
356 
357 	assert_object_held(obj);
358 
359 	pinned = !(type & I915_MAP_OVERRIDE);
360 	type &= ~I915_MAP_OVERRIDE;
361 
362 	if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) {
363 		if (unlikely(!i915_gem_object_has_pages(obj))) {
364 			GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
365 
366 			err = ____i915_gem_object_get_pages(obj);
367 			if (err)
368 				return ERR_PTR(err);
369 
370 			smp_mb__before_atomic();
371 		}
372 		atomic_inc(&obj->mm.pages_pin_count);
373 		pinned = false;
374 	}
375 	GEM_BUG_ON(!i915_gem_object_has_pages(obj));
376 
377 	ptr = page_unpack_bits(obj->mm.mapping, &has_type);
378 	if (ptr && has_type != type) {
379 		if (pinned) {
380 			ptr = ERR_PTR(-EBUSY);
381 			goto err_unpin;
382 		}
383 
384 		unmap_object(obj, ptr);
385 
386 		ptr = obj->mm.mapping = NULL;
387 	}
388 
389 	if (!ptr) {
390 		if (GEM_WARN_ON(type == I915_MAP_WC &&
391 				!static_cpu_has(X86_FEATURE_PAT)))
392 			ptr = ERR_PTR(-ENODEV);
393 		else if (i915_gem_object_has_struct_page(obj))
394 			ptr = i915_gem_object_map_page(obj, type);
395 		else
396 			ptr = i915_gem_object_map_pfn(obj, type);
397 		if (IS_ERR(ptr))
398 			goto err_unpin;
399 
400 		obj->mm.mapping = page_pack_bits(ptr, type);
401 	}
402 
403 	return ptr;
404 
405 err_unpin:
406 	atomic_dec(&obj->mm.pages_pin_count);
407 	return ptr;
408 }
409 
410 void *i915_gem_object_pin_map_unlocked(struct drm_i915_gem_object *obj,
411 				       enum i915_map_type type)
412 {
413 	void *ret;
414 
415 	i915_gem_object_lock(obj, NULL);
416 	ret = i915_gem_object_pin_map(obj, type);
417 	i915_gem_object_unlock(obj);
418 
419 	return ret;
420 }
421 
422 void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,
423 				 unsigned long offset,
424 				 unsigned long size)
425 {
426 	enum i915_map_type has_type;
427 	void *ptr;
428 
429 	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
430 	GEM_BUG_ON(range_overflows_t(typeof(obj->base.size),
431 				     offset, size, obj->base.size));
432 
433 	wmb(); /* let all previous writes be visible to coherent partners */
434 	obj->mm.dirty = true;
435 
436 	if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE)
437 		return;
438 
439 	ptr = page_unpack_bits(obj->mm.mapping, &has_type);
440 	if (has_type == I915_MAP_WC)
441 		return;
442 
443 	drm_clflush_virt_range(ptr + offset, size);
444 	if (size == obj->base.size) {
445 		obj->write_domain &= ~I915_GEM_DOMAIN_CPU;
446 		obj->cache_dirty = false;
447 	}
448 }
449 
450 void __i915_gem_object_release_map(struct drm_i915_gem_object *obj)
451 {
452 	GEM_BUG_ON(!obj->mm.mapping);
453 
454 	/*
455 	 * We allow removing the mapping from underneath pinned pages!
456 	 *
457 	 * Furthermore, since this is an unsafe operation reserved only
458 	 * for construction time manipulation, we ignore locking prudence.
459 	 */
460 	unmap_object(obj, page_mask_bits(fetch_and_zero(&obj->mm.mapping)));
461 
462 	i915_gem_object_unpin_map(obj);
463 }
464 
465 struct scatterlist *
466 __i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
467 			 struct i915_gem_object_page_iter *iter,
468 			 unsigned int n,
469 			 unsigned int *offset,
470 			 bool allow_alloc)
471 {
472 	const bool dma = iter == &obj->mm.get_dma_page;
473 	struct scatterlist *sg;
474 	unsigned int idx, count;
475 
476 	might_sleep();
477 	GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT);
478 	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
479 
480 	/* As we iterate forward through the sg, we record each entry in a
481 	 * radixtree for quick repeated (backwards) lookups. If we have seen
482 	 * this index previously, we will have an entry for it.
483 	 *
484 	 * Initial lookup is O(N), but this is amortized to O(1) for
485 	 * sequential page access (where each new request is consecutive
486 	 * to the previous one). Repeated lookups are O(lg(obj->base.size)),
487 	 * i.e. O(1) with a large constant!
488 	 */
489 	if (n < READ_ONCE(iter->sg_idx))
490 		goto lookup;
491 
492 	if (!allow_alloc)
493 		goto manual_lookup;
494 
495 	mutex_lock(&iter->lock);
496 
497 	/* We prefer to reuse the last sg so that repeated lookup of this
498 	 * (or the subsequent) sg are fast - comparing against the last
499 	 * sg is faster than going through the radixtree.
500 	 */
501 
502 	sg = iter->sg_pos;
503 	idx = iter->sg_idx;
504 	count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);
505 
506 	while (idx + count <= n) {
507 		void *entry;
508 		unsigned long i;
509 		int ret;
510 
511 		/* If we cannot allocate and insert this entry, or the
512 		 * individual pages from this range, cancel updating the
513 		 * sg_idx so that on this lookup we are forced to linearly
514 		 * scan onwards, but on future lookups we will try the
515 		 * insertion again (in which case we need to be careful of
516 		 * the error return reporting that we have already inserted
517 		 * this index).
518 		 */
519 		ret = radix_tree_insert(&iter->radix, idx, sg);
520 		if (ret && ret != -EEXIST)
521 			goto scan;
522 
523 		entry = xa_mk_value(idx);
524 		for (i = 1; i < count; i++) {
525 			ret = radix_tree_insert(&iter->radix, idx + i, entry);
526 			if (ret && ret != -EEXIST)
527 				goto scan;
528 		}
529 
530 		idx += count;
531 		sg = ____sg_next(sg);
532 		count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);
533 	}
534 
535 scan:
536 	iter->sg_pos = sg;
537 	iter->sg_idx = idx;
538 
539 	mutex_unlock(&iter->lock);
540 
541 	if (unlikely(n < idx)) /* insertion completed by another thread */
542 		goto lookup;
543 
544 	goto manual_walk;
545 
546 manual_lookup:
547 	idx = 0;
548 	sg = obj->mm.pages->sgl;
549 	count = __sg_page_count(sg);
550 
551 manual_walk:
552 	/*
553 	 * In case we failed to insert the entry into the radixtree, we need
554 	 * to look beyond the current sg.
555 	 */
556 	while (idx + count <= n) {
557 		idx += count;
558 		sg = ____sg_next(sg);
559 		count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);
560 	}
561 
562 	*offset = n - idx;
563 	return sg;
564 
565 lookup:
566 	rcu_read_lock();
567 
568 	sg = radix_tree_lookup(&iter->radix, n);
569 	GEM_BUG_ON(!sg);
570 
571 	/* If this index is in the middle of multi-page sg entry,
572 	 * the radix tree will contain a value entry that points
573 	 * to the start of that range. We will return the pointer to
574 	 * the base page and the offset of this page within the
575 	 * sg entry's range.
576 	 */
577 	*offset = 0;
578 	if (unlikely(xa_is_value(sg))) {
579 		unsigned long base = xa_to_value(sg);
580 
581 		sg = radix_tree_lookup(&iter->radix, base);
582 		GEM_BUG_ON(!sg);
583 
584 		*offset = n - base;
585 	}
586 
587 	rcu_read_unlock();
588 
589 	return sg;
590 }
591 
592 struct page *
593 i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n)
594 {
595 	struct scatterlist *sg;
596 	unsigned int offset;
597 
598 	GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
599 
600 	sg = i915_gem_object_get_sg(obj, n, &offset, true);
601 	return nth_page(sg_page(sg), offset);
602 }
603 
604 /* Like i915_gem_object_get_page(), but mark the returned page dirty */
605 struct page *
606 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
607 			       unsigned int n)
608 {
609 	struct page *page;
610 
611 	page = i915_gem_object_get_page(obj, n);
612 	if (!obj->mm.dirty)
613 		set_page_dirty(page);
614 
615 	return page;
616 }
617 
618 dma_addr_t
619 i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj,
620 				    unsigned long n,
621 				    unsigned int *len)
622 {
623 	struct scatterlist *sg;
624 	unsigned int offset;
625 
626 	sg = i915_gem_object_get_sg_dma(obj, n, &offset, true);
627 
628 	if (len)
629 		*len = sg_dma_len(sg) - (offset << PAGE_SHIFT);
630 
631 	return sg_dma_address(sg) + (offset << PAGE_SHIFT);
632 }
633 
634 dma_addr_t
635 i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
636 				unsigned long n)
637 {
638 	return i915_gem_object_get_dma_address_len(obj, n, NULL);
639 }
640