1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2014-2016 Intel Corporation
5  */
6 
7 #include <linux/pagevec.h>
8 #include <linux/swap.h>
9 
10 #include "gem/i915_gem_region.h"
11 #include "i915_drv.h"
12 #include "i915_gemfs.h"
13 #include "i915_gem_object.h"
14 #include "i915_scatterlist.h"
15 #include "i915_trace.h"
16 
17 /*
18  * Move pages to appropriate lru and release the pagevec, decrementing the
19  * ref count of those pages.
20  */
21 static void check_release_pagevec(struct pagevec *pvec)
22 {
23 	check_move_unevictable_pages(pvec);
24 	__pagevec_release(pvec);
25 	cond_resched();
26 }
27 
28 static int shmem_get_pages(struct drm_i915_gem_object *obj)
29 {
30 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
31 	struct intel_memory_region *mem = obj->mm.region;
32 	const unsigned long page_count = obj->base.size / PAGE_SIZE;
33 	unsigned long i;
34 	struct address_space *mapping;
35 	struct sg_table *st;
36 	struct scatterlist *sg;
37 	struct sgt_iter sgt_iter;
38 	struct page *page;
39 	unsigned long last_pfn = 0;	/* suppress gcc warning */
40 	unsigned int max_segment = i915_sg_segment_size();
41 	unsigned int sg_page_sizes;
42 	gfp_t noreclaim;
43 	int ret;
44 
45 	/*
46 	 * Assert that the object is not currently in any GPU domain. As it
47 	 * wasn't in the GTT, there shouldn't be any way it could have been in
48 	 * a GPU cache
49 	 */
50 	GEM_BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
51 	GEM_BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
52 
53 	/*
54 	 * If there's no chance of allocating enough pages for the whole
55 	 * object, bail early.
56 	 */
57 	if (obj->base.size > resource_size(&mem->region))
58 		return -ENOMEM;
59 
60 	st = kmalloc(sizeof(*st), GFP_KERNEL);
61 	if (!st)
62 		return -ENOMEM;
63 
64 rebuild_st:
65 	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
66 		kfree(st);
67 		return -ENOMEM;
68 	}
69 
70 	/*
71 	 * Get the list of pages out of our struct file.  They'll be pinned
72 	 * at this point until we release them.
73 	 *
74 	 * Fail silently without starting the shrinker
75 	 */
76 	mapping = obj->base.filp->f_mapping;
77 	mapping_set_unevictable(mapping);
78 	noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM);
79 	noreclaim |= __GFP_NORETRY | __GFP_NOWARN;
80 
81 	sg = st->sgl;
82 	st->nents = 0;
83 	sg_page_sizes = 0;
84 	for (i = 0; i < page_count; i++) {
85 		const unsigned int shrink[] = {
86 			I915_SHRINK_BOUND | I915_SHRINK_UNBOUND,
87 			0,
88 		}, *s = shrink;
89 		gfp_t gfp = noreclaim;
90 
91 		do {
92 			cond_resched();
93 			page = shmem_read_mapping_page_gfp(mapping, i, gfp);
94 			if (!IS_ERR(page))
95 				break;
96 
97 			if (!*s) {
98 				ret = PTR_ERR(page);
99 				goto err_sg;
100 			}
101 
102 			i915_gem_shrink(i915, 2 * page_count, NULL, *s++);
103 
104 			/*
105 			 * We've tried hard to allocate the memory by reaping
106 			 * our own buffer, now let the real VM do its job and
107 			 * go down in flames if truly OOM.
108 			 *
109 			 * However, since graphics tend to be disposable,
110 			 * defer the oom here by reporting the ENOMEM back
111 			 * to userspace.
112 			 */
113 			if (!*s) {
114 				/* reclaim and warn, but no oom */
115 				gfp = mapping_gfp_mask(mapping);
116 
117 				/*
118 				 * Our bo are always dirty and so we require
119 				 * kswapd to reclaim our pages (direct reclaim
120 				 * does not effectively begin pageout of our
121 				 * buffers on its own). However, direct reclaim
122 				 * only waits for kswapd when under allocation
123 				 * congestion. So as a result __GFP_RECLAIM is
124 				 * unreliable and fails to actually reclaim our
125 				 * dirty pages -- unless you try over and over
126 				 * again with !__GFP_NORETRY. However, we still
127 				 * want to fail this allocation rather than
128 				 * trigger the out-of-memory killer and for
129 				 * this we want __GFP_RETRY_MAYFAIL.
130 				 */
131 				gfp |= __GFP_RETRY_MAYFAIL;
132 			}
133 		} while (1);
134 
135 		if (!i ||
136 		    sg->length >= max_segment ||
137 		    page_to_pfn(page) != last_pfn + 1) {
138 			if (i) {
139 				sg_page_sizes |= sg->length;
140 				sg = sg_next(sg);
141 			}
142 			st->nents++;
143 			sg_set_page(sg, page, PAGE_SIZE, 0);
144 		} else {
145 			sg->length += PAGE_SIZE;
146 		}
147 		last_pfn = page_to_pfn(page);
148 
149 		/* Check that the i965g/gm workaround works. */
150 		GEM_BUG_ON(gfp & __GFP_DMA32 && last_pfn >= 0x00100000UL);
151 	}
152 	if (sg) { /* loop terminated early; short sg table */
153 		sg_page_sizes |= sg->length;
154 		sg_mark_end(sg);
155 	}
156 
157 	/* Trim unused sg entries to avoid wasting memory. */
158 	i915_sg_trim(st);
159 
160 	ret = i915_gem_gtt_prepare_pages(obj, st);
161 	if (ret) {
162 		/*
163 		 * DMA remapping failed? One possible cause is that
164 		 * it could not reserve enough large entries, asking
165 		 * for PAGE_SIZE chunks instead may be helpful.
166 		 */
167 		if (max_segment > PAGE_SIZE) {
168 			for_each_sgt_page(page, sgt_iter, st)
169 				put_page(page);
170 			sg_free_table(st);
171 
172 			max_segment = PAGE_SIZE;
173 			goto rebuild_st;
174 		} else {
175 			dev_warn(&i915->drm.pdev->dev,
176 				 "Failed to DMA remap %lu pages\n",
177 				 page_count);
178 			goto err_pages;
179 		}
180 	}
181 
182 	if (i915_gem_object_needs_bit17_swizzle(obj))
183 		i915_gem_object_do_bit_17_swizzle(obj, st);
184 
185 	__i915_gem_object_set_pages(obj, st, sg_page_sizes);
186 
187 	return 0;
188 
189 err_sg:
190 	sg_mark_end(sg);
191 err_pages:
192 	mapping_clear_unevictable(mapping);
193 	if (sg != st->sgl) {
194 		struct pagevec pvec;
195 
196 		pagevec_init(&pvec);
197 		for_each_sgt_page(page, sgt_iter, st) {
198 			if (!pagevec_add(&pvec, page))
199 				check_release_pagevec(&pvec);
200 		}
201 		if (pagevec_count(&pvec))
202 			check_release_pagevec(&pvec);
203 	}
204 	sg_free_table(st);
205 	kfree(st);
206 
207 	/*
208 	 * shmemfs first checks if there is enough memory to allocate the page
209 	 * and reports ENOSPC should there be insufficient, along with the usual
210 	 * ENOMEM for a genuine allocation failure.
211 	 *
212 	 * We use ENOSPC in our driver to mean that we have run out of aperture
213 	 * space and so want to translate the error from shmemfs back to our
214 	 * usual understanding of ENOMEM.
215 	 */
216 	if (ret == -ENOSPC)
217 		ret = -ENOMEM;
218 
219 	return ret;
220 }
221 
222 static void
223 shmem_truncate(struct drm_i915_gem_object *obj)
224 {
225 	/*
226 	 * Our goal here is to return as much of the memory as
227 	 * is possible back to the system as we are called from OOM.
228 	 * To do this we must instruct the shmfs to drop all of its
229 	 * backing pages, *now*.
230 	 */
231 	shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
232 	obj->mm.madv = __I915_MADV_PURGED;
233 	obj->mm.pages = ERR_PTR(-EFAULT);
234 }
235 
236 static void
237 shmem_writeback(struct drm_i915_gem_object *obj)
238 {
239 	struct address_space *mapping;
240 	struct writeback_control wbc = {
241 		.sync_mode = WB_SYNC_NONE,
242 		.nr_to_write = SWAP_CLUSTER_MAX,
243 		.range_start = 0,
244 		.range_end = LLONG_MAX,
245 		.for_reclaim = 1,
246 	};
247 	unsigned long i;
248 
249 	/*
250 	 * Leave mmapings intact (GTT will have been revoked on unbinding,
251 	 * leaving only CPU mmapings around) and add those pages to the LRU
252 	 * instead of invoking writeback so they are aged and paged out
253 	 * as normal.
254 	 */
255 	mapping = obj->base.filp->f_mapping;
256 
257 	/* Begin writeback on each dirty page */
258 	for (i = 0; i < obj->base.size >> PAGE_SHIFT; i++) {
259 		struct page *page;
260 
261 		page = find_lock_page(mapping, i);
262 		if (!page)
263 			continue;
264 
265 		if (!page_mapped(page) && clear_page_dirty_for_io(page)) {
266 			int ret;
267 
268 			SetPageReclaim(page);
269 			ret = mapping->a_ops->writepage(page, &wbc);
270 			if (!PageWriteback(page))
271 				ClearPageReclaim(page);
272 			if (!ret)
273 				goto put;
274 		}
275 		unlock_page(page);
276 put:
277 		put_page(page);
278 	}
279 }
280 
281 void
282 __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
283 				struct sg_table *pages,
284 				bool needs_clflush)
285 {
286 	GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);
287 
288 	if (obj->mm.madv == I915_MADV_DONTNEED)
289 		obj->mm.dirty = false;
290 
291 	if (needs_clflush &&
292 	    (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0 &&
293 	    !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
294 		drm_clflush_sg(pages);
295 
296 	__start_cpu_write(obj);
297 }
298 
299 static void
300 shmem_put_pages(struct drm_i915_gem_object *obj, struct sg_table *pages)
301 {
302 	struct sgt_iter sgt_iter;
303 	struct pagevec pvec;
304 	struct page *page;
305 
306 	__i915_gem_object_release_shmem(obj, pages, true);
307 
308 	i915_gem_gtt_finish_pages(obj, pages);
309 
310 	if (i915_gem_object_needs_bit17_swizzle(obj))
311 		i915_gem_object_save_bit_17_swizzle(obj, pages);
312 
313 	mapping_clear_unevictable(file_inode(obj->base.filp)->i_mapping);
314 
315 	pagevec_init(&pvec);
316 	for_each_sgt_page(page, sgt_iter, pages) {
317 		if (obj->mm.dirty)
318 			set_page_dirty(page);
319 
320 		if (obj->mm.madv == I915_MADV_WILLNEED)
321 			mark_page_accessed(page);
322 
323 		if (!pagevec_add(&pvec, page))
324 			check_release_pagevec(&pvec);
325 	}
326 	if (pagevec_count(&pvec))
327 		check_release_pagevec(&pvec);
328 	obj->mm.dirty = false;
329 
330 	sg_free_table(pages);
331 	kfree(pages);
332 }
333 
334 static int
335 shmem_pwrite(struct drm_i915_gem_object *obj,
336 	     const struct drm_i915_gem_pwrite *arg)
337 {
338 	struct address_space *mapping = obj->base.filp->f_mapping;
339 	char __user *user_data = u64_to_user_ptr(arg->data_ptr);
340 	u64 remain, offset;
341 	unsigned int pg;
342 
343 	/* Caller already validated user args */
344 	GEM_BUG_ON(!access_ok(user_data, arg->size));
345 
346 	/*
347 	 * Before we instantiate/pin the backing store for our use, we
348 	 * can prepopulate the shmemfs filp efficiently using a write into
349 	 * the pagecache. We avoid the penalty of instantiating all the
350 	 * pages, important if the user is just writing to a few and never
351 	 * uses the object on the GPU, and using a direct write into shmemfs
352 	 * allows it to avoid the cost of retrieving a page (either swapin
353 	 * or clearing-before-use) before it is overwritten.
354 	 */
355 	if (i915_gem_object_has_pages(obj))
356 		return -ENODEV;
357 
358 	if (obj->mm.madv != I915_MADV_WILLNEED)
359 		return -EFAULT;
360 
361 	/*
362 	 * Before the pages are instantiated the object is treated as being
363 	 * in the CPU domain. The pages will be clflushed as required before
364 	 * use, and we can freely write into the pages directly. If userspace
365 	 * races pwrite with any other operation; corruption will ensue -
366 	 * that is userspace's prerogative!
367 	 */
368 
369 	remain = arg->size;
370 	offset = arg->offset;
371 	pg = offset_in_page(offset);
372 
373 	do {
374 		unsigned int len, unwritten;
375 		struct page *page;
376 		void *data, *vaddr;
377 		int err;
378 		char c;
379 
380 		len = PAGE_SIZE - pg;
381 		if (len > remain)
382 			len = remain;
383 
384 		/* Prefault the user page to reduce potential recursion */
385 		err = __get_user(c, user_data);
386 		if (err)
387 			return err;
388 
389 		err = __get_user(c, user_data + len - 1);
390 		if (err)
391 			return err;
392 
393 		err = pagecache_write_begin(obj->base.filp, mapping,
394 					    offset, len, 0,
395 					    &page, &data);
396 		if (err < 0)
397 			return err;
398 
399 		vaddr = kmap_atomic(page);
400 		unwritten = __copy_from_user_inatomic(vaddr + pg,
401 						      user_data,
402 						      len);
403 		kunmap_atomic(vaddr);
404 
405 		err = pagecache_write_end(obj->base.filp, mapping,
406 					  offset, len, len - unwritten,
407 					  page, data);
408 		if (err < 0)
409 			return err;
410 
411 		/* We don't handle -EFAULT, leave it to the caller to check */
412 		if (unwritten)
413 			return -ENODEV;
414 
415 		remain -= len;
416 		user_data += len;
417 		offset += len;
418 		pg = 0;
419 	} while (remain);
420 
421 	return 0;
422 }
423 
424 static void shmem_release(struct drm_i915_gem_object *obj)
425 {
426 	i915_gem_object_release_memory_region(obj);
427 
428 	fput(obj->base.filp);
429 }
430 
431 const struct drm_i915_gem_object_ops i915_gem_shmem_ops = {
432 	.name = "i915_gem_object_shmem",
433 	.flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
434 		 I915_GEM_OBJECT_IS_SHRINKABLE,
435 
436 	.get_pages = shmem_get_pages,
437 	.put_pages = shmem_put_pages,
438 	.truncate = shmem_truncate,
439 	.writeback = shmem_writeback,
440 
441 	.pwrite = shmem_pwrite,
442 
443 	.release = shmem_release,
444 };
445 
446 static int __create_shmem(struct drm_i915_private *i915,
447 			  struct drm_gem_object *obj,
448 			  resource_size_t size)
449 {
450 	unsigned long flags = VM_NORESERVE;
451 	struct file *filp;
452 
453 	drm_gem_private_object_init(&i915->drm, obj, size);
454 
455 	if (i915->mm.gemfs)
456 		filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size,
457 						 flags);
458 	else
459 		filp = shmem_file_setup("i915", size, flags);
460 	if (IS_ERR(filp))
461 		return PTR_ERR(filp);
462 
463 	obj->filp = filp;
464 	return 0;
465 }
466 
467 static int shmem_object_init(struct intel_memory_region *mem,
468 			     struct drm_i915_gem_object *obj,
469 			     resource_size_t size,
470 			     unsigned int flags)
471 {
472 	static struct lock_class_key lock_class;
473 	struct drm_i915_private *i915 = mem->i915;
474 	struct address_space *mapping;
475 	unsigned int cache_level;
476 	gfp_t mask;
477 	int ret;
478 
479 	ret = __create_shmem(i915, &obj->base, size);
480 	if (ret)
481 		return ret;
482 
483 	mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
484 	if (IS_I965GM(i915) || IS_I965G(i915)) {
485 		/* 965gm cannot relocate objects above 4GiB. */
486 		mask &= ~__GFP_HIGHMEM;
487 		mask |= __GFP_DMA32;
488 	}
489 
490 	mapping = obj->base.filp->f_mapping;
491 	mapping_set_gfp_mask(mapping, mask);
492 	GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM));
493 
494 	i915_gem_object_init(obj, &i915_gem_shmem_ops, &lock_class);
495 
496 	obj->write_domain = I915_GEM_DOMAIN_CPU;
497 	obj->read_domains = I915_GEM_DOMAIN_CPU;
498 
499 	if (HAS_LLC(i915))
500 		/* On some devices, we can have the GPU use the LLC (the CPU
501 		 * cache) for about a 10% performance improvement
502 		 * compared to uncached.  Graphics requests other than
503 		 * display scanout are coherent with the CPU in
504 		 * accessing this cache.  This means in this mode we
505 		 * don't need to clflush on the CPU side, and on the
506 		 * GPU side we only need to flush internal caches to
507 		 * get data visible to the CPU.
508 		 *
509 		 * However, we maintain the display planes as UC, and so
510 		 * need to rebind when first used as such.
511 		 */
512 		cache_level = I915_CACHE_LLC;
513 	else
514 		cache_level = I915_CACHE_NONE;
515 
516 	i915_gem_object_set_cache_coherency(obj, cache_level);
517 
518 	i915_gem_object_init_memory_region(obj, mem, 0);
519 
520 	return 0;
521 }
522 
523 struct drm_i915_gem_object *
524 i915_gem_object_create_shmem(struct drm_i915_private *i915,
525 			     resource_size_t size)
526 {
527 	return i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_SMEM],
528 					     size, 0);
529 }
530 
531 /* Allocate a new GEM object and fill it with the supplied data */
532 struct drm_i915_gem_object *
533 i915_gem_object_create_shmem_from_data(struct drm_i915_private *dev_priv,
534 				       const void *data, resource_size_t size)
535 {
536 	struct drm_i915_gem_object *obj;
537 	struct file *file;
538 	resource_size_t offset;
539 	int err;
540 
541 	obj = i915_gem_object_create_shmem(dev_priv, round_up(size, PAGE_SIZE));
542 	if (IS_ERR(obj))
543 		return obj;
544 
545 	GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU);
546 
547 	file = obj->base.filp;
548 	offset = 0;
549 	do {
550 		unsigned int len = min_t(typeof(size), size, PAGE_SIZE);
551 		struct page *page;
552 		void *pgdata, *vaddr;
553 
554 		err = pagecache_write_begin(file, file->f_mapping,
555 					    offset, len, 0,
556 					    &page, &pgdata);
557 		if (err < 0)
558 			goto fail;
559 
560 		vaddr = kmap(page);
561 		memcpy(vaddr, data, len);
562 		kunmap(page);
563 
564 		err = pagecache_write_end(file, file->f_mapping,
565 					  offset, len, len,
566 					  page, pgdata);
567 		if (err < 0)
568 			goto fail;
569 
570 		size -= len;
571 		data += len;
572 		offset += len;
573 	} while (size);
574 
575 	return obj;
576 
577 fail:
578 	i915_gem_object_put(obj);
579 	return ERR_PTR(err);
580 }
581 
582 static int init_shmem(struct intel_memory_region *mem)
583 {
584 	int err;
585 
586 	err = i915_gemfs_init(mem->i915);
587 	if (err) {
588 		DRM_NOTE("Unable to create a private tmpfs mount, hugepage support will be disabled(%d).\n",
589 			 err);
590 	}
591 
592 	intel_memory_region_set_name(mem, "system");
593 
594 	return 0; /* Don't error, we can simply fallback to the kernel mnt */
595 }
596 
597 static void release_shmem(struct intel_memory_region *mem)
598 {
599 	i915_gemfs_fini(mem->i915);
600 }
601 
602 static const struct intel_memory_region_ops shmem_region_ops = {
603 	.init = init_shmem,
604 	.release = release_shmem,
605 	.init_object = shmem_object_init,
606 };
607 
608 struct intel_memory_region *i915_gem_shmem_setup(struct drm_i915_private *i915)
609 {
610 	return intel_memory_region_create(i915, 0,
611 					  totalram_pages() << PAGE_SHIFT,
612 					  PAGE_SIZE, 0,
613 					  &shmem_region_ops);
614 }
615 
616 bool i915_gem_object_is_shmem(const struct drm_i915_gem_object *obj)
617 {
618 	return obj->ops == &i915_gem_shmem_ops;
619 }
620