1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2014-2016 Intel Corporation
5  */
6 
7 #include <linux/pagevec.h>
8 #include <linux/swap.h>
9 
10 #include "gem/i915_gem_region.h"
11 #include "i915_drv.h"
12 #include "i915_gemfs.h"
13 #include "i915_gem_object.h"
14 #include "i915_scatterlist.h"
15 #include "i915_trace.h"
16 
17 /*
18  * Move pages to appropriate lru and release the pagevec, decrementing the
19  * ref count of those pages.
20  */
21 static void check_release_pagevec(struct pagevec *pvec)
22 {
23 	check_move_unevictable_pages(pvec);
24 	__pagevec_release(pvec);
25 	cond_resched();
26 }
27 
28 static int shmem_get_pages(struct drm_i915_gem_object *obj)
29 {
30 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
31 	struct intel_memory_region *mem = obj->mm.region;
32 	const unsigned long page_count = obj->base.size / PAGE_SIZE;
33 	unsigned long i;
34 	struct address_space *mapping;
35 	struct sg_table *st;
36 	struct scatterlist *sg;
37 	struct sgt_iter sgt_iter;
38 	struct page *page;
39 	unsigned long last_pfn = 0;	/* suppress gcc warning */
40 	unsigned int max_segment = i915_sg_segment_size();
41 	unsigned int sg_page_sizes;
42 	struct pagevec pvec;
43 	gfp_t noreclaim;
44 	int ret;
45 
46 	/*
47 	 * Assert that the object is not currently in any GPU domain. As it
48 	 * wasn't in the GTT, there shouldn't be any way it could have been in
49 	 * a GPU cache
50 	 */
51 	GEM_BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
52 	GEM_BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
53 
54 	/*
55 	 * If there's no chance of allocating enough pages for the whole
56 	 * object, bail early.
57 	 */
58 	if (obj->base.size > resource_size(&mem->region))
59 		return -ENOMEM;
60 
61 	st = kmalloc(sizeof(*st), GFP_KERNEL);
62 	if (!st)
63 		return -ENOMEM;
64 
65 rebuild_st:
66 	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
67 		kfree(st);
68 		return -ENOMEM;
69 	}
70 
71 	/*
72 	 * Get the list of pages out of our struct file.  They'll be pinned
73 	 * at this point until we release them.
74 	 *
75 	 * Fail silently without starting the shrinker
76 	 */
77 	mapping = obj->base.filp->f_mapping;
78 	mapping_set_unevictable(mapping);
79 	noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM);
80 	noreclaim |= __GFP_NORETRY | __GFP_NOWARN;
81 
82 	sg = st->sgl;
83 	st->nents = 0;
84 	sg_page_sizes = 0;
85 	for (i = 0; i < page_count; i++) {
86 		const unsigned int shrink[] = {
87 			I915_SHRINK_BOUND | I915_SHRINK_UNBOUND,
88 			0,
89 		}, *s = shrink;
90 		gfp_t gfp = noreclaim;
91 
92 		do {
93 			cond_resched();
94 			page = shmem_read_mapping_page_gfp(mapping, i, gfp);
95 			if (!IS_ERR(page))
96 				break;
97 
98 			if (!*s) {
99 				ret = PTR_ERR(page);
100 				goto err_sg;
101 			}
102 
103 			i915_gem_shrink(i915, 2 * page_count, NULL, *s++);
104 
105 			/*
106 			 * We've tried hard to allocate the memory by reaping
107 			 * our own buffer, now let the real VM do its job and
108 			 * go down in flames if truly OOM.
109 			 *
110 			 * However, since graphics tend to be disposable,
111 			 * defer the oom here by reporting the ENOMEM back
112 			 * to userspace.
113 			 */
114 			if (!*s) {
115 				/* reclaim and warn, but no oom */
116 				gfp = mapping_gfp_mask(mapping);
117 
118 				/*
119 				 * Our bo are always dirty and so we require
120 				 * kswapd to reclaim our pages (direct reclaim
121 				 * does not effectively begin pageout of our
122 				 * buffers on its own). However, direct reclaim
123 				 * only waits for kswapd when under allocation
124 				 * congestion. So as a result __GFP_RECLAIM is
125 				 * unreliable and fails to actually reclaim our
126 				 * dirty pages -- unless you try over and over
127 				 * again with !__GFP_NORETRY. However, we still
128 				 * want to fail this allocation rather than
129 				 * trigger the out-of-memory killer and for
130 				 * this we want __GFP_RETRY_MAYFAIL.
131 				 */
132 				gfp |= __GFP_RETRY_MAYFAIL;
133 			}
134 		} while (1);
135 
136 		if (!i ||
137 		    sg->length >= max_segment ||
138 		    page_to_pfn(page) != last_pfn + 1) {
139 			if (i) {
140 				sg_page_sizes |= sg->length;
141 				sg = sg_next(sg);
142 			}
143 			st->nents++;
144 			sg_set_page(sg, page, PAGE_SIZE, 0);
145 		} else {
146 			sg->length += PAGE_SIZE;
147 		}
148 		last_pfn = page_to_pfn(page);
149 
150 		/* Check that the i965g/gm workaround works. */
151 		drm_WARN_ON(&i915->drm,
152 			    (gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
153 	}
154 	if (sg) { /* loop terminated early; short sg table */
155 		sg_page_sizes |= sg->length;
156 		sg_mark_end(sg);
157 	}
158 
159 	/* Trim unused sg entries to avoid wasting memory. */
160 	i915_sg_trim(st);
161 
162 	ret = i915_gem_gtt_prepare_pages(obj, st);
163 	if (ret) {
164 		/*
165 		 * DMA remapping failed? One possible cause is that
166 		 * it could not reserve enough large entries, asking
167 		 * for PAGE_SIZE chunks instead may be helpful.
168 		 */
169 		if (max_segment > PAGE_SIZE) {
170 			for_each_sgt_page(page, sgt_iter, st)
171 				put_page(page);
172 			sg_free_table(st);
173 
174 			max_segment = PAGE_SIZE;
175 			goto rebuild_st;
176 		} else {
177 			dev_warn(&i915->drm.pdev->dev,
178 				 "Failed to DMA remap %lu pages\n",
179 				 page_count);
180 			goto err_pages;
181 		}
182 	}
183 
184 	if (i915_gem_object_needs_bit17_swizzle(obj))
185 		i915_gem_object_do_bit_17_swizzle(obj, st);
186 
187 	__i915_gem_object_set_pages(obj, st, sg_page_sizes);
188 
189 	return 0;
190 
191 err_sg:
192 	sg_mark_end(sg);
193 err_pages:
194 	mapping_clear_unevictable(mapping);
195 	pagevec_init(&pvec);
196 	for_each_sgt_page(page, sgt_iter, st) {
197 		if (!pagevec_add(&pvec, page))
198 			check_release_pagevec(&pvec);
199 	}
200 	if (pagevec_count(&pvec))
201 		check_release_pagevec(&pvec);
202 	sg_free_table(st);
203 	kfree(st);
204 
205 	/*
206 	 * shmemfs first checks if there is enough memory to allocate the page
207 	 * and reports ENOSPC should there be insufficient, along with the usual
208 	 * ENOMEM for a genuine allocation failure.
209 	 *
210 	 * We use ENOSPC in our driver to mean that we have run out of aperture
211 	 * space and so want to translate the error from shmemfs back to our
212 	 * usual understanding of ENOMEM.
213 	 */
214 	if (ret == -ENOSPC)
215 		ret = -ENOMEM;
216 
217 	return ret;
218 }
219 
220 static void
221 shmem_truncate(struct drm_i915_gem_object *obj)
222 {
223 	/*
224 	 * Our goal here is to return as much of the memory as
225 	 * is possible back to the system as we are called from OOM.
226 	 * To do this we must instruct the shmfs to drop all of its
227 	 * backing pages, *now*.
228 	 */
229 	shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
230 	obj->mm.madv = __I915_MADV_PURGED;
231 	obj->mm.pages = ERR_PTR(-EFAULT);
232 }
233 
234 static void
235 shmem_writeback(struct drm_i915_gem_object *obj)
236 {
237 	struct address_space *mapping;
238 	struct writeback_control wbc = {
239 		.sync_mode = WB_SYNC_NONE,
240 		.nr_to_write = SWAP_CLUSTER_MAX,
241 		.range_start = 0,
242 		.range_end = LLONG_MAX,
243 		.for_reclaim = 1,
244 	};
245 	unsigned long i;
246 
247 	/*
248 	 * Leave mmapings intact (GTT will have been revoked on unbinding,
249 	 * leaving only CPU mmapings around) and add those pages to the LRU
250 	 * instead of invoking writeback so they are aged and paged out
251 	 * as normal.
252 	 */
253 	mapping = obj->base.filp->f_mapping;
254 
255 	/* Begin writeback on each dirty page */
256 	for (i = 0; i < obj->base.size >> PAGE_SHIFT; i++) {
257 		struct page *page;
258 
259 		page = find_lock_entry(mapping, i);
260 		if (!page || xa_is_value(page))
261 			continue;
262 
263 		if (!page_mapped(page) && clear_page_dirty_for_io(page)) {
264 			int ret;
265 
266 			SetPageReclaim(page);
267 			ret = mapping->a_ops->writepage(page, &wbc);
268 			if (!PageWriteback(page))
269 				ClearPageReclaim(page);
270 			if (!ret)
271 				goto put;
272 		}
273 		unlock_page(page);
274 put:
275 		put_page(page);
276 	}
277 }
278 
279 void
280 __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
281 				struct sg_table *pages,
282 				bool needs_clflush)
283 {
284 	GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);
285 
286 	if (obj->mm.madv == I915_MADV_DONTNEED)
287 		obj->mm.dirty = false;
288 
289 	if (needs_clflush &&
290 	    (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0 &&
291 	    !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
292 		drm_clflush_sg(pages);
293 
294 	__start_cpu_write(obj);
295 }
296 
297 static void
298 shmem_put_pages(struct drm_i915_gem_object *obj, struct sg_table *pages)
299 {
300 	struct sgt_iter sgt_iter;
301 	struct pagevec pvec;
302 	struct page *page;
303 
304 	__i915_gem_object_release_shmem(obj, pages, true);
305 
306 	i915_gem_gtt_finish_pages(obj, pages);
307 
308 	if (i915_gem_object_needs_bit17_swizzle(obj))
309 		i915_gem_object_save_bit_17_swizzle(obj, pages);
310 
311 	mapping_clear_unevictable(file_inode(obj->base.filp)->i_mapping);
312 
313 	pagevec_init(&pvec);
314 	for_each_sgt_page(page, sgt_iter, pages) {
315 		if (obj->mm.dirty)
316 			set_page_dirty(page);
317 
318 		if (obj->mm.madv == I915_MADV_WILLNEED)
319 			mark_page_accessed(page);
320 
321 		if (!pagevec_add(&pvec, page))
322 			check_release_pagevec(&pvec);
323 	}
324 	if (pagevec_count(&pvec))
325 		check_release_pagevec(&pvec);
326 	obj->mm.dirty = false;
327 
328 	sg_free_table(pages);
329 	kfree(pages);
330 }
331 
332 static int
333 shmem_pwrite(struct drm_i915_gem_object *obj,
334 	     const struct drm_i915_gem_pwrite *arg)
335 {
336 	struct address_space *mapping = obj->base.filp->f_mapping;
337 	char __user *user_data = u64_to_user_ptr(arg->data_ptr);
338 	u64 remain, offset;
339 	unsigned int pg;
340 
341 	/* Caller already validated user args */
342 	GEM_BUG_ON(!access_ok(user_data, arg->size));
343 
344 	/*
345 	 * Before we instantiate/pin the backing store for our use, we
346 	 * can prepopulate the shmemfs filp efficiently using a write into
347 	 * the pagecache. We avoid the penalty of instantiating all the
348 	 * pages, important if the user is just writing to a few and never
349 	 * uses the object on the GPU, and using a direct write into shmemfs
350 	 * allows it to avoid the cost of retrieving a page (either swapin
351 	 * or clearing-before-use) before it is overwritten.
352 	 */
353 	if (i915_gem_object_has_pages(obj))
354 		return -ENODEV;
355 
356 	if (obj->mm.madv != I915_MADV_WILLNEED)
357 		return -EFAULT;
358 
359 	/*
360 	 * Before the pages are instantiated the object is treated as being
361 	 * in the CPU domain. The pages will be clflushed as required before
362 	 * use, and we can freely write into the pages directly. If userspace
363 	 * races pwrite with any other operation; corruption will ensue -
364 	 * that is userspace's prerogative!
365 	 */
366 
367 	remain = arg->size;
368 	offset = arg->offset;
369 	pg = offset_in_page(offset);
370 
371 	do {
372 		unsigned int len, unwritten;
373 		struct page *page;
374 		void *data, *vaddr;
375 		int err;
376 		char c;
377 
378 		len = PAGE_SIZE - pg;
379 		if (len > remain)
380 			len = remain;
381 
382 		/* Prefault the user page to reduce potential recursion */
383 		err = __get_user(c, user_data);
384 		if (err)
385 			return err;
386 
387 		err = __get_user(c, user_data + len - 1);
388 		if (err)
389 			return err;
390 
391 		err = pagecache_write_begin(obj->base.filp, mapping,
392 					    offset, len, 0,
393 					    &page, &data);
394 		if (err < 0)
395 			return err;
396 
397 		vaddr = kmap_atomic(page);
398 		unwritten = __copy_from_user_inatomic(vaddr + pg,
399 						      user_data,
400 						      len);
401 		kunmap_atomic(vaddr);
402 
403 		err = pagecache_write_end(obj->base.filp, mapping,
404 					  offset, len, len - unwritten,
405 					  page, data);
406 		if (err < 0)
407 			return err;
408 
409 		/* We don't handle -EFAULT, leave it to the caller to check */
410 		if (unwritten)
411 			return -ENODEV;
412 
413 		remain -= len;
414 		user_data += len;
415 		offset += len;
416 		pg = 0;
417 	} while (remain);
418 
419 	return 0;
420 }
421 
422 static void shmem_release(struct drm_i915_gem_object *obj)
423 {
424 	i915_gem_object_release_memory_region(obj);
425 
426 	fput(obj->base.filp);
427 }
428 
429 const struct drm_i915_gem_object_ops i915_gem_shmem_ops = {
430 	.flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
431 		 I915_GEM_OBJECT_IS_SHRINKABLE,
432 
433 	.get_pages = shmem_get_pages,
434 	.put_pages = shmem_put_pages,
435 	.truncate = shmem_truncate,
436 	.writeback = shmem_writeback,
437 
438 	.pwrite = shmem_pwrite,
439 
440 	.release = shmem_release,
441 };
442 
443 static int __create_shmem(struct drm_i915_private *i915,
444 			  struct drm_gem_object *obj,
445 			  resource_size_t size)
446 {
447 	unsigned long flags = VM_NORESERVE;
448 	struct file *filp;
449 
450 	drm_gem_private_object_init(&i915->drm, obj, size);
451 
452 	if (i915->mm.gemfs)
453 		filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size,
454 						 flags);
455 	else
456 		filp = shmem_file_setup("i915", size, flags);
457 	if (IS_ERR(filp))
458 		return PTR_ERR(filp);
459 
460 	obj->filp = filp;
461 	return 0;
462 }
463 
464 static struct drm_i915_gem_object *
465 create_shmem(struct intel_memory_region *mem,
466 	     resource_size_t size,
467 	     unsigned int flags)
468 {
469 	static struct lock_class_key lock_class;
470 	struct drm_i915_private *i915 = mem->i915;
471 	struct drm_i915_gem_object *obj;
472 	struct address_space *mapping;
473 	unsigned int cache_level;
474 	gfp_t mask;
475 	int ret;
476 
477 	obj = i915_gem_object_alloc();
478 	if (!obj)
479 		return ERR_PTR(-ENOMEM);
480 
481 	ret = __create_shmem(i915, &obj->base, size);
482 	if (ret)
483 		goto fail;
484 
485 	mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
486 	if (IS_I965GM(i915) || IS_I965G(i915)) {
487 		/* 965gm cannot relocate objects above 4GiB. */
488 		mask &= ~__GFP_HIGHMEM;
489 		mask |= __GFP_DMA32;
490 	}
491 
492 	mapping = obj->base.filp->f_mapping;
493 	mapping_set_gfp_mask(mapping, mask);
494 	GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM));
495 
496 	i915_gem_object_init(obj, &i915_gem_shmem_ops, &lock_class);
497 
498 	obj->write_domain = I915_GEM_DOMAIN_CPU;
499 	obj->read_domains = I915_GEM_DOMAIN_CPU;
500 
501 	if (HAS_LLC(i915))
502 		/* On some devices, we can have the GPU use the LLC (the CPU
503 		 * cache) for about a 10% performance improvement
504 		 * compared to uncached.  Graphics requests other than
505 		 * display scanout are coherent with the CPU in
506 		 * accessing this cache.  This means in this mode we
507 		 * don't need to clflush on the CPU side, and on the
508 		 * GPU side we only need to flush internal caches to
509 		 * get data visible to the CPU.
510 		 *
511 		 * However, we maintain the display planes as UC, and so
512 		 * need to rebind when first used as such.
513 		 */
514 		cache_level = I915_CACHE_LLC;
515 	else
516 		cache_level = I915_CACHE_NONE;
517 
518 	i915_gem_object_set_cache_coherency(obj, cache_level);
519 
520 	i915_gem_object_init_memory_region(obj, mem, 0);
521 
522 	return obj;
523 
524 fail:
525 	i915_gem_object_free(obj);
526 	return ERR_PTR(ret);
527 }
528 
529 struct drm_i915_gem_object *
530 i915_gem_object_create_shmem(struct drm_i915_private *i915,
531 			     resource_size_t size)
532 {
533 	return i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_SMEM],
534 					     size, 0);
535 }
536 
537 /* Allocate a new GEM object and fill it with the supplied data */
538 struct drm_i915_gem_object *
539 i915_gem_object_create_shmem_from_data(struct drm_i915_private *dev_priv,
540 				       const void *data, resource_size_t size)
541 {
542 	struct drm_i915_gem_object *obj;
543 	struct file *file;
544 	resource_size_t offset;
545 	int err;
546 
547 	obj = i915_gem_object_create_shmem(dev_priv, round_up(size, PAGE_SIZE));
548 	if (IS_ERR(obj))
549 		return obj;
550 
551 	GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU);
552 
553 	file = obj->base.filp;
554 	offset = 0;
555 	do {
556 		unsigned int len = min_t(typeof(size), size, PAGE_SIZE);
557 		struct page *page;
558 		void *pgdata, *vaddr;
559 
560 		err = pagecache_write_begin(file, file->f_mapping,
561 					    offset, len, 0,
562 					    &page, &pgdata);
563 		if (err < 0)
564 			goto fail;
565 
566 		vaddr = kmap(page);
567 		memcpy(vaddr, data, len);
568 		kunmap(page);
569 
570 		err = pagecache_write_end(file, file->f_mapping,
571 					  offset, len, len,
572 					  page, pgdata);
573 		if (err < 0)
574 			goto fail;
575 
576 		size -= len;
577 		data += len;
578 		offset += len;
579 	} while (size);
580 
581 	return obj;
582 
583 fail:
584 	i915_gem_object_put(obj);
585 	return ERR_PTR(err);
586 }
587 
588 static int init_shmem(struct intel_memory_region *mem)
589 {
590 	int err;
591 
592 	err = i915_gemfs_init(mem->i915);
593 	if (err) {
594 		DRM_NOTE("Unable to create a private tmpfs mount, hugepage support will be disabled(%d).\n",
595 			 err);
596 	}
597 
598 	intel_memory_region_set_name(mem, "system");
599 
600 	return 0; /* Don't error, we can simply fallback to the kernel mnt */
601 }
602 
603 static void release_shmem(struct intel_memory_region *mem)
604 {
605 	i915_gemfs_fini(mem->i915);
606 }
607 
608 static const struct intel_memory_region_ops shmem_region_ops = {
609 	.init = init_shmem,
610 	.release = release_shmem,
611 	.create_object = create_shmem,
612 };
613 
614 struct intel_memory_region *i915_gem_shmem_setup(struct drm_i915_private *i915)
615 {
616 	return intel_memory_region_create(i915, 0,
617 					  totalram_pages() << PAGE_SHIFT,
618 					  PAGE_SIZE, 0,
619 					  &shmem_region_ops);
620 }
621