xref: /openbmc/linux/drivers/gpu/drm/i915/i915_gem.c (revision b34e08d5)
1 /*
2  * Copyright © 2008 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  * Authors:
24  *    Eric Anholt <eric@anholt.net>
25  *
26  */
27 
28 #include <drm/drmP.h>
29 #include <drm/drm_vma_manager.h>
30 #include <drm/i915_drm.h>
31 #include "i915_drv.h"
32 #include "i915_trace.h"
33 #include "intel_drv.h"
34 #include <linux/shmem_fs.h>
35 #include <linux/slab.h>
36 #include <linux/swap.h>
37 #include <linux/pci.h>
38 #include <linux/dma-buf.h>
39 
40 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
41 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
42 						   bool force);
43 static __must_check int
44 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
45 			       bool readonly);
46 static int i915_gem_phys_pwrite(struct drm_device *dev,
47 				struct drm_i915_gem_object *obj,
48 				struct drm_i915_gem_pwrite *args,
49 				struct drm_file *file);
50 
51 static void i915_gem_write_fence(struct drm_device *dev, int reg,
52 				 struct drm_i915_gem_object *obj);
53 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
54 					 struct drm_i915_fence_reg *fence,
55 					 bool enable);
56 
57 static unsigned long i915_gem_inactive_count(struct shrinker *shrinker,
58 					     struct shrink_control *sc);
59 static unsigned long i915_gem_inactive_scan(struct shrinker *shrinker,
60 					    struct shrink_control *sc);
61 static unsigned long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
62 static unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
63 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
64 static void i915_gem_retire_requests_ring(struct intel_ring_buffer *ring);
65 
66 static bool cpu_cache_is_coherent(struct drm_device *dev,
67 				  enum i915_cache_level level)
68 {
69 	return HAS_LLC(dev) || level != I915_CACHE_NONE;
70 }
71 
72 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
73 {
74 	if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
75 		return true;
76 
77 	return obj->pin_display;
78 }
79 
80 static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
81 {
82 	if (obj->tiling_mode)
83 		i915_gem_release_mmap(obj);
84 
85 	/* As we do not have an associated fence register, we will force
86 	 * a tiling change if we ever need to acquire one.
87 	 */
88 	obj->fence_dirty = false;
89 	obj->fence_reg = I915_FENCE_REG_NONE;
90 }
91 
92 /* some bookkeeping */
93 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
94 				  size_t size)
95 {
96 	spin_lock(&dev_priv->mm.object_stat_lock);
97 	dev_priv->mm.object_count++;
98 	dev_priv->mm.object_memory += size;
99 	spin_unlock(&dev_priv->mm.object_stat_lock);
100 }
101 
102 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
103 				     size_t size)
104 {
105 	spin_lock(&dev_priv->mm.object_stat_lock);
106 	dev_priv->mm.object_count--;
107 	dev_priv->mm.object_memory -= size;
108 	spin_unlock(&dev_priv->mm.object_stat_lock);
109 }
110 
111 static int
112 i915_gem_wait_for_error(struct i915_gpu_error *error)
113 {
114 	int ret;
115 
116 #define EXIT_COND (!i915_reset_in_progress(error) || \
117 		   i915_terminally_wedged(error))
118 	if (EXIT_COND)
119 		return 0;
120 
121 	/*
122 	 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
123 	 * userspace. If it takes that long something really bad is going on and
124 	 * we should simply try to bail out and fail as gracefully as possible.
125 	 */
126 	ret = wait_event_interruptible_timeout(error->reset_queue,
127 					       EXIT_COND,
128 					       10*HZ);
129 	if (ret == 0) {
130 		DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
131 		return -EIO;
132 	} else if (ret < 0) {
133 		return ret;
134 	}
135 #undef EXIT_COND
136 
137 	return 0;
138 }
139 
140 int i915_mutex_lock_interruptible(struct drm_device *dev)
141 {
142 	struct drm_i915_private *dev_priv = dev->dev_private;
143 	int ret;
144 
145 	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
146 	if (ret)
147 		return ret;
148 
149 	ret = mutex_lock_interruptible(&dev->struct_mutex);
150 	if (ret)
151 		return ret;
152 
153 	WARN_ON(i915_verify_lists(dev));
154 	return 0;
155 }
156 
157 static inline bool
158 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
159 {
160 	return i915_gem_obj_bound_any(obj) && !obj->active;
161 }
162 
163 int
164 i915_gem_init_ioctl(struct drm_device *dev, void *data,
165 		    struct drm_file *file)
166 {
167 	struct drm_i915_private *dev_priv = dev->dev_private;
168 	struct drm_i915_gem_init *args = data;
169 
170 	if (drm_core_check_feature(dev, DRIVER_MODESET))
171 		return -ENODEV;
172 
173 	if (args->gtt_start >= args->gtt_end ||
174 	    (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
175 		return -EINVAL;
176 
177 	/* GEM with user mode setting was never supported on ilk and later. */
178 	if (INTEL_INFO(dev)->gen >= 5)
179 		return -ENODEV;
180 
181 	mutex_lock(&dev->struct_mutex);
182 	i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
183 				  args->gtt_end);
184 	dev_priv->gtt.mappable_end = args->gtt_end;
185 	mutex_unlock(&dev->struct_mutex);
186 
187 	return 0;
188 }
189 
190 int
191 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
192 			    struct drm_file *file)
193 {
194 	struct drm_i915_private *dev_priv = dev->dev_private;
195 	struct drm_i915_gem_get_aperture *args = data;
196 	struct drm_i915_gem_object *obj;
197 	size_t pinned;
198 
199 	pinned = 0;
200 	mutex_lock(&dev->struct_mutex);
201 	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
202 		if (i915_gem_obj_is_pinned(obj))
203 			pinned += i915_gem_obj_ggtt_size(obj);
204 	mutex_unlock(&dev->struct_mutex);
205 
206 	args->aper_size = dev_priv->gtt.base.total;
207 	args->aper_available_size = args->aper_size - pinned;
208 
209 	return 0;
210 }
211 
212 void *i915_gem_object_alloc(struct drm_device *dev)
213 {
214 	struct drm_i915_private *dev_priv = dev->dev_private;
215 	return kmem_cache_zalloc(dev_priv->slab, GFP_KERNEL);
216 }
217 
218 void i915_gem_object_free(struct drm_i915_gem_object *obj)
219 {
220 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
221 	kmem_cache_free(dev_priv->slab, obj);
222 }
223 
224 static int
225 i915_gem_create(struct drm_file *file,
226 		struct drm_device *dev,
227 		uint64_t size,
228 		uint32_t *handle_p)
229 {
230 	struct drm_i915_gem_object *obj;
231 	int ret;
232 	u32 handle;
233 
234 	size = roundup(size, PAGE_SIZE);
235 	if (size == 0)
236 		return -EINVAL;
237 
238 	/* Allocate the new object */
239 	obj = i915_gem_alloc_object(dev, size);
240 	if (obj == NULL)
241 		return -ENOMEM;
242 
243 	ret = drm_gem_handle_create(file, &obj->base, &handle);
244 	/* drop reference from allocate - handle holds it now */
245 	drm_gem_object_unreference_unlocked(&obj->base);
246 	if (ret)
247 		return ret;
248 
249 	*handle_p = handle;
250 	return 0;
251 }
252 
253 int
254 i915_gem_dumb_create(struct drm_file *file,
255 		     struct drm_device *dev,
256 		     struct drm_mode_create_dumb *args)
257 {
258 	/* have to work out size/pitch and return them */
259 	args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
260 	args->size = args->pitch * args->height;
261 	return i915_gem_create(file, dev,
262 			       args->size, &args->handle);
263 }
264 
265 /**
266  * Creates a new mm object and returns a handle to it.
267  */
268 int
269 i915_gem_create_ioctl(struct drm_device *dev, void *data,
270 		      struct drm_file *file)
271 {
272 	struct drm_i915_gem_create *args = data;
273 
274 	return i915_gem_create(file, dev,
275 			       args->size, &args->handle);
276 }
277 
278 static inline int
279 __copy_to_user_swizzled(char __user *cpu_vaddr,
280 			const char *gpu_vaddr, int gpu_offset,
281 			int length)
282 {
283 	int ret, cpu_offset = 0;
284 
285 	while (length > 0) {
286 		int cacheline_end = ALIGN(gpu_offset + 1, 64);
287 		int this_length = min(cacheline_end - gpu_offset, length);
288 		int swizzled_gpu_offset = gpu_offset ^ 64;
289 
290 		ret = __copy_to_user(cpu_vaddr + cpu_offset,
291 				     gpu_vaddr + swizzled_gpu_offset,
292 				     this_length);
293 		if (ret)
294 			return ret + length;
295 
296 		cpu_offset += this_length;
297 		gpu_offset += this_length;
298 		length -= this_length;
299 	}
300 
301 	return 0;
302 }
303 
304 static inline int
305 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
306 			  const char __user *cpu_vaddr,
307 			  int length)
308 {
309 	int ret, cpu_offset = 0;
310 
311 	while (length > 0) {
312 		int cacheline_end = ALIGN(gpu_offset + 1, 64);
313 		int this_length = min(cacheline_end - gpu_offset, length);
314 		int swizzled_gpu_offset = gpu_offset ^ 64;
315 
316 		ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
317 				       cpu_vaddr + cpu_offset,
318 				       this_length);
319 		if (ret)
320 			return ret + length;
321 
322 		cpu_offset += this_length;
323 		gpu_offset += this_length;
324 		length -= this_length;
325 	}
326 
327 	return 0;
328 }
329 
330 /*
331  * Pins the specified object's pages and synchronizes the object with
332  * GPU accesses. Sets needs_clflush to non-zero if the caller should
333  * flush the object from the CPU cache.
334  */
335 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
336 				    int *needs_clflush)
337 {
338 	int ret;
339 
340 	*needs_clflush = 0;
341 
342 	if (!obj->base.filp)
343 		return -EINVAL;
344 
345 	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
346 		/* If we're not in the cpu read domain, set ourself into the gtt
347 		 * read domain and manually flush cachelines (if required). This
348 		 * optimizes for the case when the gpu will dirty the data
349 		 * anyway again before the next pread happens. */
350 		*needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
351 							obj->cache_level);
352 		ret = i915_gem_object_wait_rendering(obj, true);
353 		if (ret)
354 			return ret;
355 	}
356 
357 	ret = i915_gem_object_get_pages(obj);
358 	if (ret)
359 		return ret;
360 
361 	i915_gem_object_pin_pages(obj);
362 
363 	return ret;
364 }
365 
366 /* Per-page copy function for the shmem pread fastpath.
367  * Flushes invalid cachelines before reading the target if
368  * needs_clflush is set. */
369 static int
370 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
371 		 char __user *user_data,
372 		 bool page_do_bit17_swizzling, bool needs_clflush)
373 {
374 	char *vaddr;
375 	int ret;
376 
377 	if (unlikely(page_do_bit17_swizzling))
378 		return -EINVAL;
379 
380 	vaddr = kmap_atomic(page);
381 	if (needs_clflush)
382 		drm_clflush_virt_range(vaddr + shmem_page_offset,
383 				       page_length);
384 	ret = __copy_to_user_inatomic(user_data,
385 				      vaddr + shmem_page_offset,
386 				      page_length);
387 	kunmap_atomic(vaddr);
388 
389 	return ret ? -EFAULT : 0;
390 }
391 
392 static void
393 shmem_clflush_swizzled_range(char *addr, unsigned long length,
394 			     bool swizzled)
395 {
396 	if (unlikely(swizzled)) {
397 		unsigned long start = (unsigned long) addr;
398 		unsigned long end = (unsigned long) addr + length;
399 
400 		/* For swizzling simply ensure that we always flush both
401 		 * channels. Lame, but simple and it works. Swizzled
402 		 * pwrite/pread is far from a hotpath - current userspace
403 		 * doesn't use it at all. */
404 		start = round_down(start, 128);
405 		end = round_up(end, 128);
406 
407 		drm_clflush_virt_range((void *)start, end - start);
408 	} else {
409 		drm_clflush_virt_range(addr, length);
410 	}
411 
412 }
413 
414 /* Only difference to the fast-path function is that this can handle bit17
415  * and uses non-atomic copy and kmap functions. */
416 static int
417 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
418 		 char __user *user_data,
419 		 bool page_do_bit17_swizzling, bool needs_clflush)
420 {
421 	char *vaddr;
422 	int ret;
423 
424 	vaddr = kmap(page);
425 	if (needs_clflush)
426 		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
427 					     page_length,
428 					     page_do_bit17_swizzling);
429 
430 	if (page_do_bit17_swizzling)
431 		ret = __copy_to_user_swizzled(user_data,
432 					      vaddr, shmem_page_offset,
433 					      page_length);
434 	else
435 		ret = __copy_to_user(user_data,
436 				     vaddr + shmem_page_offset,
437 				     page_length);
438 	kunmap(page);
439 
440 	return ret ? - EFAULT : 0;
441 }
442 
443 static int
444 i915_gem_shmem_pread(struct drm_device *dev,
445 		     struct drm_i915_gem_object *obj,
446 		     struct drm_i915_gem_pread *args,
447 		     struct drm_file *file)
448 {
449 	char __user *user_data;
450 	ssize_t remain;
451 	loff_t offset;
452 	int shmem_page_offset, page_length, ret = 0;
453 	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
454 	int prefaulted = 0;
455 	int needs_clflush = 0;
456 	struct sg_page_iter sg_iter;
457 
458 	user_data = to_user_ptr(args->data_ptr);
459 	remain = args->size;
460 
461 	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
462 
463 	ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
464 	if (ret)
465 		return ret;
466 
467 	offset = args->offset;
468 
469 	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
470 			 offset >> PAGE_SHIFT) {
471 		struct page *page = sg_page_iter_page(&sg_iter);
472 
473 		if (remain <= 0)
474 			break;
475 
476 		/* Operation in this page
477 		 *
478 		 * shmem_page_offset = offset within page in shmem file
479 		 * page_length = bytes to copy for this page
480 		 */
481 		shmem_page_offset = offset_in_page(offset);
482 		page_length = remain;
483 		if ((shmem_page_offset + page_length) > PAGE_SIZE)
484 			page_length = PAGE_SIZE - shmem_page_offset;
485 
486 		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
487 			(page_to_phys(page) & (1 << 17)) != 0;
488 
489 		ret = shmem_pread_fast(page, shmem_page_offset, page_length,
490 				       user_data, page_do_bit17_swizzling,
491 				       needs_clflush);
492 		if (ret == 0)
493 			goto next_page;
494 
495 		mutex_unlock(&dev->struct_mutex);
496 
497 		if (likely(!i915.prefault_disable) && !prefaulted) {
498 			ret = fault_in_multipages_writeable(user_data, remain);
499 			/* Userspace is tricking us, but we've already clobbered
500 			 * its pages with the prefault and promised to write the
501 			 * data up to the first fault. Hence ignore any errors
502 			 * and just continue. */
503 			(void)ret;
504 			prefaulted = 1;
505 		}
506 
507 		ret = shmem_pread_slow(page, shmem_page_offset, page_length,
508 				       user_data, page_do_bit17_swizzling,
509 				       needs_clflush);
510 
511 		mutex_lock(&dev->struct_mutex);
512 
513 		if (ret)
514 			goto out;
515 
516 next_page:
517 		remain -= page_length;
518 		user_data += page_length;
519 		offset += page_length;
520 	}
521 
522 out:
523 	i915_gem_object_unpin_pages(obj);
524 
525 	return ret;
526 }
527 
528 /**
529  * Reads data from the object referenced by handle.
530  *
531  * On error, the contents of *data are undefined.
532  */
533 int
534 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
535 		     struct drm_file *file)
536 {
537 	struct drm_i915_gem_pread *args = data;
538 	struct drm_i915_gem_object *obj;
539 	int ret = 0;
540 
541 	if (args->size == 0)
542 		return 0;
543 
544 	if (!access_ok(VERIFY_WRITE,
545 		       to_user_ptr(args->data_ptr),
546 		       args->size))
547 		return -EFAULT;
548 
549 	ret = i915_mutex_lock_interruptible(dev);
550 	if (ret)
551 		return ret;
552 
553 	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
554 	if (&obj->base == NULL) {
555 		ret = -ENOENT;
556 		goto unlock;
557 	}
558 
559 	/* Bounds check source.  */
560 	if (args->offset > obj->base.size ||
561 	    args->size > obj->base.size - args->offset) {
562 		ret = -EINVAL;
563 		goto out;
564 	}
565 
566 	/* prime objects have no backing filp to GEM pread/pwrite
567 	 * pages from.
568 	 */
569 	if (!obj->base.filp) {
570 		ret = -EINVAL;
571 		goto out;
572 	}
573 
574 	trace_i915_gem_object_pread(obj, args->offset, args->size);
575 
576 	ret = i915_gem_shmem_pread(dev, obj, args, file);
577 
578 out:
579 	drm_gem_object_unreference(&obj->base);
580 unlock:
581 	mutex_unlock(&dev->struct_mutex);
582 	return ret;
583 }
584 
585 /* This is the fast write path which cannot handle
586  * page faults in the source data
587  */
588 
589 static inline int
590 fast_user_write(struct io_mapping *mapping,
591 		loff_t page_base, int page_offset,
592 		char __user *user_data,
593 		int length)
594 {
595 	void __iomem *vaddr_atomic;
596 	void *vaddr;
597 	unsigned long unwritten;
598 
599 	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
600 	/* We can use the cpu mem copy function because this is X86. */
601 	vaddr = (void __force*)vaddr_atomic + page_offset;
602 	unwritten = __copy_from_user_inatomic_nocache(vaddr,
603 						      user_data, length);
604 	io_mapping_unmap_atomic(vaddr_atomic);
605 	return unwritten;
606 }
607 
608 /**
609  * This is the fast pwrite path, where we copy the data directly from the
610  * user into the GTT, uncached.
611  */
612 static int
613 i915_gem_gtt_pwrite_fast(struct drm_device *dev,
614 			 struct drm_i915_gem_object *obj,
615 			 struct drm_i915_gem_pwrite *args,
616 			 struct drm_file *file)
617 {
618 	struct drm_i915_private *dev_priv = dev->dev_private;
619 	ssize_t remain;
620 	loff_t offset, page_base;
621 	char __user *user_data;
622 	int page_offset, page_length, ret;
623 
624 	ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
625 	if (ret)
626 		goto out;
627 
628 	ret = i915_gem_object_set_to_gtt_domain(obj, true);
629 	if (ret)
630 		goto out_unpin;
631 
632 	ret = i915_gem_object_put_fence(obj);
633 	if (ret)
634 		goto out_unpin;
635 
636 	user_data = to_user_ptr(args->data_ptr);
637 	remain = args->size;
638 
639 	offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
640 
641 	while (remain > 0) {
642 		/* Operation in this page
643 		 *
644 		 * page_base = page offset within aperture
645 		 * page_offset = offset within page
646 		 * page_length = bytes to copy for this page
647 		 */
648 		page_base = offset & PAGE_MASK;
649 		page_offset = offset_in_page(offset);
650 		page_length = remain;
651 		if ((page_offset + remain) > PAGE_SIZE)
652 			page_length = PAGE_SIZE - page_offset;
653 
654 		/* If we get a fault while copying data, then (presumably) our
655 		 * source page isn't available.  Return the error and we'll
656 		 * retry in the slow path.
657 		 */
658 		if (fast_user_write(dev_priv->gtt.mappable, page_base,
659 				    page_offset, user_data, page_length)) {
660 			ret = -EFAULT;
661 			goto out_unpin;
662 		}
663 
664 		remain -= page_length;
665 		user_data += page_length;
666 		offset += page_length;
667 	}
668 
669 out_unpin:
670 	i915_gem_object_ggtt_unpin(obj);
671 out:
672 	return ret;
673 }
674 
675 /* Per-page copy function for the shmem pwrite fastpath.
676  * Flushes invalid cachelines before writing to the target if
677  * needs_clflush_before is set and flushes out any written cachelines after
678  * writing if needs_clflush is set. */
679 static int
680 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
681 		  char __user *user_data,
682 		  bool page_do_bit17_swizzling,
683 		  bool needs_clflush_before,
684 		  bool needs_clflush_after)
685 {
686 	char *vaddr;
687 	int ret;
688 
689 	if (unlikely(page_do_bit17_swizzling))
690 		return -EINVAL;
691 
692 	vaddr = kmap_atomic(page);
693 	if (needs_clflush_before)
694 		drm_clflush_virt_range(vaddr + shmem_page_offset,
695 				       page_length);
696 	ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
697 					user_data, page_length);
698 	if (needs_clflush_after)
699 		drm_clflush_virt_range(vaddr + shmem_page_offset,
700 				       page_length);
701 	kunmap_atomic(vaddr);
702 
703 	return ret ? -EFAULT : 0;
704 }
705 
706 /* Only difference to the fast-path function is that this can handle bit17
707  * and uses non-atomic copy and kmap functions. */
708 static int
709 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
710 		  char __user *user_data,
711 		  bool page_do_bit17_swizzling,
712 		  bool needs_clflush_before,
713 		  bool needs_clflush_after)
714 {
715 	char *vaddr;
716 	int ret;
717 
718 	vaddr = kmap(page);
719 	if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
720 		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
721 					     page_length,
722 					     page_do_bit17_swizzling);
723 	if (page_do_bit17_swizzling)
724 		ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
725 						user_data,
726 						page_length);
727 	else
728 		ret = __copy_from_user(vaddr + shmem_page_offset,
729 				       user_data,
730 				       page_length);
731 	if (needs_clflush_after)
732 		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
733 					     page_length,
734 					     page_do_bit17_swizzling);
735 	kunmap(page);
736 
737 	return ret ? -EFAULT : 0;
738 }
739 
740 static int
741 i915_gem_shmem_pwrite(struct drm_device *dev,
742 		      struct drm_i915_gem_object *obj,
743 		      struct drm_i915_gem_pwrite *args,
744 		      struct drm_file *file)
745 {
746 	ssize_t remain;
747 	loff_t offset;
748 	char __user *user_data;
749 	int shmem_page_offset, page_length, ret = 0;
750 	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
751 	int hit_slowpath = 0;
752 	int needs_clflush_after = 0;
753 	int needs_clflush_before = 0;
754 	struct sg_page_iter sg_iter;
755 
756 	user_data = to_user_ptr(args->data_ptr);
757 	remain = args->size;
758 
759 	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
760 
761 	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
762 		/* If we're not in the cpu write domain, set ourself into the gtt
763 		 * write domain and manually flush cachelines (if required). This
764 		 * optimizes for the case when the gpu will use the data
765 		 * right away and we therefore have to clflush anyway. */
766 		needs_clflush_after = cpu_write_needs_clflush(obj);
767 		ret = i915_gem_object_wait_rendering(obj, false);
768 		if (ret)
769 			return ret;
770 	}
771 	/* Same trick applies to invalidate partially written cachelines read
772 	 * before writing. */
773 	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
774 		needs_clflush_before =
775 			!cpu_cache_is_coherent(dev, obj->cache_level);
776 
777 	ret = i915_gem_object_get_pages(obj);
778 	if (ret)
779 		return ret;
780 
781 	i915_gem_object_pin_pages(obj);
782 
783 	offset = args->offset;
784 	obj->dirty = 1;
785 
786 	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
787 			 offset >> PAGE_SHIFT) {
788 		struct page *page = sg_page_iter_page(&sg_iter);
789 		int partial_cacheline_write;
790 
791 		if (remain <= 0)
792 			break;
793 
794 		/* Operation in this page
795 		 *
796 		 * shmem_page_offset = offset within page in shmem file
797 		 * page_length = bytes to copy for this page
798 		 */
799 		shmem_page_offset = offset_in_page(offset);
800 
801 		page_length = remain;
802 		if ((shmem_page_offset + page_length) > PAGE_SIZE)
803 			page_length = PAGE_SIZE - shmem_page_offset;
804 
805 		/* If we don't overwrite a cacheline completely we need to be
806 		 * careful to have up-to-date data by first clflushing. Don't
807 		 * overcomplicate things and flush the entire patch. */
808 		partial_cacheline_write = needs_clflush_before &&
809 			((shmem_page_offset | page_length)
810 				& (boot_cpu_data.x86_clflush_size - 1));
811 
812 		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
813 			(page_to_phys(page) & (1 << 17)) != 0;
814 
815 		ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
816 					user_data, page_do_bit17_swizzling,
817 					partial_cacheline_write,
818 					needs_clflush_after);
819 		if (ret == 0)
820 			goto next_page;
821 
822 		hit_slowpath = 1;
823 		mutex_unlock(&dev->struct_mutex);
824 		ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
825 					user_data, page_do_bit17_swizzling,
826 					partial_cacheline_write,
827 					needs_clflush_after);
828 
829 		mutex_lock(&dev->struct_mutex);
830 
831 		if (ret)
832 			goto out;
833 
834 next_page:
835 		remain -= page_length;
836 		user_data += page_length;
837 		offset += page_length;
838 	}
839 
840 out:
841 	i915_gem_object_unpin_pages(obj);
842 
843 	if (hit_slowpath) {
844 		/*
845 		 * Fixup: Flush cpu caches in case we didn't flush the dirty
846 		 * cachelines in-line while writing and the object moved
847 		 * out of the cpu write domain while we've dropped the lock.
848 		 */
849 		if (!needs_clflush_after &&
850 		    obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
851 			if (i915_gem_clflush_object(obj, obj->pin_display))
852 				i915_gem_chipset_flush(dev);
853 		}
854 	}
855 
856 	if (needs_clflush_after)
857 		i915_gem_chipset_flush(dev);
858 
859 	return ret;
860 }
861 
862 /**
863  * Writes data to the object referenced by handle.
864  *
865  * On error, the contents of the buffer that were to be modified are undefined.
866  */
867 int
868 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
869 		      struct drm_file *file)
870 {
871 	struct drm_i915_gem_pwrite *args = data;
872 	struct drm_i915_gem_object *obj;
873 	int ret;
874 
875 	if (args->size == 0)
876 		return 0;
877 
878 	if (!access_ok(VERIFY_READ,
879 		       to_user_ptr(args->data_ptr),
880 		       args->size))
881 		return -EFAULT;
882 
883 	if (likely(!i915.prefault_disable)) {
884 		ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
885 						   args->size);
886 		if (ret)
887 			return -EFAULT;
888 	}
889 
890 	ret = i915_mutex_lock_interruptible(dev);
891 	if (ret)
892 		return ret;
893 
894 	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
895 	if (&obj->base == NULL) {
896 		ret = -ENOENT;
897 		goto unlock;
898 	}
899 
900 	/* Bounds check destination. */
901 	if (args->offset > obj->base.size ||
902 	    args->size > obj->base.size - args->offset) {
903 		ret = -EINVAL;
904 		goto out;
905 	}
906 
907 	/* prime objects have no backing filp to GEM pread/pwrite
908 	 * pages from.
909 	 */
910 	if (!obj->base.filp) {
911 		ret = -EINVAL;
912 		goto out;
913 	}
914 
915 	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
916 
917 	ret = -EFAULT;
918 	/* We can only do the GTT pwrite on untiled buffers, as otherwise
919 	 * it would end up going through the fenced access, and we'll get
920 	 * different detiling behavior between reading and writing.
921 	 * pread/pwrite currently are reading and writing from the CPU
922 	 * perspective, requiring manual detiling by the client.
923 	 */
924 	if (obj->phys_obj) {
925 		ret = i915_gem_phys_pwrite(dev, obj, args, file);
926 		goto out;
927 	}
928 
929 	if (obj->tiling_mode == I915_TILING_NONE &&
930 	    obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
931 	    cpu_write_needs_clflush(obj)) {
932 		ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
933 		/* Note that the gtt paths might fail with non-page-backed user
934 		 * pointers (e.g. gtt mappings when moving data between
935 		 * textures). Fallback to the shmem path in that case. */
936 	}
937 
938 	if (ret == -EFAULT || ret == -ENOSPC)
939 		ret = i915_gem_shmem_pwrite(dev, obj, args, file);
940 
941 out:
942 	drm_gem_object_unreference(&obj->base);
943 unlock:
944 	mutex_unlock(&dev->struct_mutex);
945 	return ret;
946 }
947 
948 int
949 i915_gem_check_wedge(struct i915_gpu_error *error,
950 		     bool interruptible)
951 {
952 	if (i915_reset_in_progress(error)) {
953 		/* Non-interruptible callers can't handle -EAGAIN, hence return
954 		 * -EIO unconditionally for these. */
955 		if (!interruptible)
956 			return -EIO;
957 
958 		/* Recovery complete, but the reset failed ... */
959 		if (i915_terminally_wedged(error))
960 			return -EIO;
961 
962 		return -EAGAIN;
963 	}
964 
965 	return 0;
966 }
967 
968 /*
969  * Compare seqno against outstanding lazy request. Emit a request if they are
970  * equal.
971  */
972 static int
973 i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
974 {
975 	int ret;
976 
977 	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
978 
979 	ret = 0;
980 	if (seqno == ring->outstanding_lazy_seqno)
981 		ret = i915_add_request(ring, NULL);
982 
983 	return ret;
984 }
985 
986 static void fake_irq(unsigned long data)
987 {
988 	wake_up_process((struct task_struct *)data);
989 }
990 
991 static bool missed_irq(struct drm_i915_private *dev_priv,
992 		       struct intel_ring_buffer *ring)
993 {
994 	return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
995 }
996 
997 static bool can_wait_boost(struct drm_i915_file_private *file_priv)
998 {
999 	if (file_priv == NULL)
1000 		return true;
1001 
1002 	return !atomic_xchg(&file_priv->rps_wait_boost, true);
1003 }
1004 
1005 /**
1006  * __wait_seqno - wait until execution of seqno has finished
1007  * @ring: the ring expected to report seqno
1008  * @seqno: duh!
1009  * @reset_counter: reset sequence associated with the given seqno
1010  * @interruptible: do an interruptible wait (normally yes)
1011  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
1012  *
1013  * Note: It is of utmost importance that the passed in seqno and reset_counter
1014  * values have been read by the caller in an smp safe manner. Where read-side
1015  * locks are involved, it is sufficient to read the reset_counter before
1016  * unlocking the lock that protects the seqno. For lockless tricks, the
1017  * reset_counter _must_ be read before, and an appropriate smp_rmb must be
1018  * inserted.
1019  *
1020  * Returns 0 if the seqno was found within the alloted time. Else returns the
1021  * errno with remaining time filled in timeout argument.
1022  */
1023 static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
1024 			unsigned reset_counter,
1025 			bool interruptible,
1026 			struct timespec *timeout,
1027 			struct drm_i915_file_private *file_priv)
1028 {
1029 	struct drm_device *dev = ring->dev;
1030 	struct drm_i915_private *dev_priv = dev->dev_private;
1031 	const bool irq_test_in_progress =
1032 		ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring);
1033 	struct timespec before, now;
1034 	DEFINE_WAIT(wait);
1035 	unsigned long timeout_expire;
1036 	int ret;
1037 
1038 	WARN(dev_priv->pm.irqs_disabled, "IRQs disabled\n");
1039 
1040 	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
1041 		return 0;
1042 
1043 	timeout_expire = timeout ? jiffies + timespec_to_jiffies_timeout(timeout) : 0;
1044 
1045 	if (INTEL_INFO(dev)->gen >= 6 && can_wait_boost(file_priv)) {
1046 		gen6_rps_boost(dev_priv);
1047 		if (file_priv)
1048 			mod_delayed_work(dev_priv->wq,
1049 					 &file_priv->mm.idle_work,
1050 					 msecs_to_jiffies(100));
1051 	}
1052 
1053 	if (!irq_test_in_progress && WARN_ON(!ring->irq_get(ring)))
1054 		return -ENODEV;
1055 
1056 	/* Record current time in case interrupted by signal, or wedged */
1057 	trace_i915_gem_request_wait_begin(ring, seqno);
1058 	getrawmonotonic(&before);
1059 	for (;;) {
1060 		struct timer_list timer;
1061 
1062 		prepare_to_wait(&ring->irq_queue, &wait,
1063 				interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE);
1064 
1065 		/* We need to check whether any gpu reset happened in between
1066 		 * the caller grabbing the seqno and now ... */
1067 		if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
1068 			/* ... but upgrade the -EAGAIN to an -EIO if the gpu
1069 			 * is truely gone. */
1070 			ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1071 			if (ret == 0)
1072 				ret = -EAGAIN;
1073 			break;
1074 		}
1075 
1076 		if (i915_seqno_passed(ring->get_seqno(ring, false), seqno)) {
1077 			ret = 0;
1078 			break;
1079 		}
1080 
1081 		if (interruptible && signal_pending(current)) {
1082 			ret = -ERESTARTSYS;
1083 			break;
1084 		}
1085 
1086 		if (timeout && time_after_eq(jiffies, timeout_expire)) {
1087 			ret = -ETIME;
1088 			break;
1089 		}
1090 
1091 		timer.function = NULL;
1092 		if (timeout || missed_irq(dev_priv, ring)) {
1093 			unsigned long expire;
1094 
1095 			setup_timer_on_stack(&timer, fake_irq, (unsigned long)current);
1096 			expire = missed_irq(dev_priv, ring) ? jiffies + 1 : timeout_expire;
1097 			mod_timer(&timer, expire);
1098 		}
1099 
1100 		io_schedule();
1101 
1102 		if (timer.function) {
1103 			del_singleshot_timer_sync(&timer);
1104 			destroy_timer_on_stack(&timer);
1105 		}
1106 	}
1107 	getrawmonotonic(&now);
1108 	trace_i915_gem_request_wait_end(ring, seqno);
1109 
1110 	if (!irq_test_in_progress)
1111 		ring->irq_put(ring);
1112 
1113 	finish_wait(&ring->irq_queue, &wait);
1114 
1115 	if (timeout) {
1116 		struct timespec sleep_time = timespec_sub(now, before);
1117 		*timeout = timespec_sub(*timeout, sleep_time);
1118 		if (!timespec_valid(timeout)) /* i.e. negative time remains */
1119 			set_normalized_timespec(timeout, 0, 0);
1120 	}
1121 
1122 	return ret;
1123 }
1124 
1125 /**
1126  * Waits for a sequence number to be signaled, and cleans up the
1127  * request and object lists appropriately for that event.
1128  */
1129 int
1130 i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
1131 {
1132 	struct drm_device *dev = ring->dev;
1133 	struct drm_i915_private *dev_priv = dev->dev_private;
1134 	bool interruptible = dev_priv->mm.interruptible;
1135 	int ret;
1136 
1137 	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1138 	BUG_ON(seqno == 0);
1139 
1140 	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1141 	if (ret)
1142 		return ret;
1143 
1144 	ret = i915_gem_check_olr(ring, seqno);
1145 	if (ret)
1146 		return ret;
1147 
1148 	return __wait_seqno(ring, seqno,
1149 			    atomic_read(&dev_priv->gpu_error.reset_counter),
1150 			    interruptible, NULL, NULL);
1151 }
1152 
1153 static int
1154 i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
1155 				     struct intel_ring_buffer *ring)
1156 {
1157 	i915_gem_retire_requests_ring(ring);
1158 
1159 	/* Manually manage the write flush as we may have not yet
1160 	 * retired the buffer.
1161 	 *
1162 	 * Note that the last_write_seqno is always the earlier of
1163 	 * the two (read/write) seqno, so if we haved successfully waited,
1164 	 * we know we have passed the last write.
1165 	 */
1166 	obj->last_write_seqno = 0;
1167 	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
1168 
1169 	return 0;
1170 }
1171 
1172 /**
1173  * Ensures that all rendering to the object has completed and the object is
1174  * safe to unbind from the GTT or access from the CPU.
1175  */
1176 static __must_check int
1177 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1178 			       bool readonly)
1179 {
1180 	struct intel_ring_buffer *ring = obj->ring;
1181 	u32 seqno;
1182 	int ret;
1183 
1184 	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
1185 	if (seqno == 0)
1186 		return 0;
1187 
1188 	ret = i915_wait_seqno(ring, seqno);
1189 	if (ret)
1190 		return ret;
1191 
1192 	return i915_gem_object_wait_rendering__tail(obj, ring);
1193 }
1194 
1195 /* A nonblocking variant of the above wait. This is a highly dangerous routine
1196  * as the object state may change during this call.
1197  */
1198 static __must_check int
1199 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
1200 					    struct drm_i915_file_private *file_priv,
1201 					    bool readonly)
1202 {
1203 	struct drm_device *dev = obj->base.dev;
1204 	struct drm_i915_private *dev_priv = dev->dev_private;
1205 	struct intel_ring_buffer *ring = obj->ring;
1206 	unsigned reset_counter;
1207 	u32 seqno;
1208 	int ret;
1209 
1210 	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1211 	BUG_ON(!dev_priv->mm.interruptible);
1212 
1213 	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
1214 	if (seqno == 0)
1215 		return 0;
1216 
1217 	ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
1218 	if (ret)
1219 		return ret;
1220 
1221 	ret = i915_gem_check_olr(ring, seqno);
1222 	if (ret)
1223 		return ret;
1224 
1225 	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
1226 	mutex_unlock(&dev->struct_mutex);
1227 	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, file_priv);
1228 	mutex_lock(&dev->struct_mutex);
1229 	if (ret)
1230 		return ret;
1231 
1232 	return i915_gem_object_wait_rendering__tail(obj, ring);
1233 }
1234 
1235 /**
1236  * Called when user space prepares to use an object with the CPU, either
1237  * through the mmap ioctl's mapping or a GTT mapping.
1238  */
1239 int
1240 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1241 			  struct drm_file *file)
1242 {
1243 	struct drm_i915_gem_set_domain *args = data;
1244 	struct drm_i915_gem_object *obj;
1245 	uint32_t read_domains = args->read_domains;
1246 	uint32_t write_domain = args->write_domain;
1247 	int ret;
1248 
1249 	/* Only handle setting domains to types used by the CPU. */
1250 	if (write_domain & I915_GEM_GPU_DOMAINS)
1251 		return -EINVAL;
1252 
1253 	if (read_domains & I915_GEM_GPU_DOMAINS)
1254 		return -EINVAL;
1255 
1256 	/* Having something in the write domain implies it's in the read
1257 	 * domain, and only that read domain.  Enforce that in the request.
1258 	 */
1259 	if (write_domain != 0 && read_domains != write_domain)
1260 		return -EINVAL;
1261 
1262 	ret = i915_mutex_lock_interruptible(dev);
1263 	if (ret)
1264 		return ret;
1265 
1266 	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1267 	if (&obj->base == NULL) {
1268 		ret = -ENOENT;
1269 		goto unlock;
1270 	}
1271 
1272 	/* Try to flush the object off the GPU without holding the lock.
1273 	 * We will repeat the flush holding the lock in the normal manner
1274 	 * to catch cases where we are gazumped.
1275 	 */
1276 	ret = i915_gem_object_wait_rendering__nonblocking(obj,
1277 							  file->driver_priv,
1278 							  !write_domain);
1279 	if (ret)
1280 		goto unref;
1281 
1282 	if (read_domains & I915_GEM_DOMAIN_GTT) {
1283 		ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1284 
1285 		/* Silently promote "you're not bound, there was nothing to do"
1286 		 * to success, since the client was just asking us to
1287 		 * make sure everything was done.
1288 		 */
1289 		if (ret == -EINVAL)
1290 			ret = 0;
1291 	} else {
1292 		ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1293 	}
1294 
1295 unref:
1296 	drm_gem_object_unreference(&obj->base);
1297 unlock:
1298 	mutex_unlock(&dev->struct_mutex);
1299 	return ret;
1300 }
1301 
1302 /**
1303  * Called when user space has done writes to this buffer
1304  */
1305 int
1306 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1307 			 struct drm_file *file)
1308 {
1309 	struct drm_i915_gem_sw_finish *args = data;
1310 	struct drm_i915_gem_object *obj;
1311 	int ret = 0;
1312 
1313 	ret = i915_mutex_lock_interruptible(dev);
1314 	if (ret)
1315 		return ret;
1316 
1317 	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1318 	if (&obj->base == NULL) {
1319 		ret = -ENOENT;
1320 		goto unlock;
1321 	}
1322 
1323 	/* Pinned buffers may be scanout, so flush the cache */
1324 	if (obj->pin_display)
1325 		i915_gem_object_flush_cpu_write_domain(obj, true);
1326 
1327 	drm_gem_object_unreference(&obj->base);
1328 unlock:
1329 	mutex_unlock(&dev->struct_mutex);
1330 	return ret;
1331 }
1332 
1333 /**
1334  * Maps the contents of an object, returning the address it is mapped
1335  * into.
1336  *
1337  * While the mapping holds a reference on the contents of the object, it doesn't
1338  * imply a ref on the object itself.
1339  */
1340 int
1341 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1342 		    struct drm_file *file)
1343 {
1344 	struct drm_i915_gem_mmap *args = data;
1345 	struct drm_gem_object *obj;
1346 	unsigned long addr;
1347 
1348 	obj = drm_gem_object_lookup(dev, file, args->handle);
1349 	if (obj == NULL)
1350 		return -ENOENT;
1351 
1352 	/* prime objects have no backing filp to GEM mmap
1353 	 * pages from.
1354 	 */
1355 	if (!obj->filp) {
1356 		drm_gem_object_unreference_unlocked(obj);
1357 		return -EINVAL;
1358 	}
1359 
1360 	addr = vm_mmap(obj->filp, 0, args->size,
1361 		       PROT_READ | PROT_WRITE, MAP_SHARED,
1362 		       args->offset);
1363 	drm_gem_object_unreference_unlocked(obj);
1364 	if (IS_ERR((void *)addr))
1365 		return addr;
1366 
1367 	args->addr_ptr = (uint64_t) addr;
1368 
1369 	return 0;
1370 }
1371 
1372 /**
1373  * i915_gem_fault - fault a page into the GTT
1374  * vma: VMA in question
1375  * vmf: fault info
1376  *
1377  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1378  * from userspace.  The fault handler takes care of binding the object to
1379  * the GTT (if needed), allocating and programming a fence register (again,
1380  * only if needed based on whether the old reg is still valid or the object
1381  * is tiled) and inserting a new PTE into the faulting process.
1382  *
1383  * Note that the faulting process may involve evicting existing objects
1384  * from the GTT and/or fence registers to make room.  So performance may
1385  * suffer if the GTT working set is large or there are few fence registers
1386  * left.
1387  */
1388 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1389 {
1390 	struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1391 	struct drm_device *dev = obj->base.dev;
1392 	struct drm_i915_private *dev_priv = dev->dev_private;
1393 	pgoff_t page_offset;
1394 	unsigned long pfn;
1395 	int ret = 0;
1396 	bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1397 
1398 	intel_runtime_pm_get(dev_priv);
1399 
1400 	/* We don't use vmf->pgoff since that has the fake offset */
1401 	page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1402 		PAGE_SHIFT;
1403 
1404 	ret = i915_mutex_lock_interruptible(dev);
1405 	if (ret)
1406 		goto out;
1407 
1408 	trace_i915_gem_object_fault(obj, page_offset, true, write);
1409 
1410 	/* Try to flush the object off the GPU first without holding the lock.
1411 	 * Upon reacquiring the lock, we will perform our sanity checks and then
1412 	 * repeat the flush holding the lock in the normal manner to catch cases
1413 	 * where we are gazumped.
1414 	 */
1415 	ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
1416 	if (ret)
1417 		goto unlock;
1418 
1419 	/* Access to snoopable pages through the GTT is incoherent. */
1420 	if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
1421 		ret = -EINVAL;
1422 		goto unlock;
1423 	}
1424 
1425 	/* Now bind it into the GTT if needed */
1426 	ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE);
1427 	if (ret)
1428 		goto unlock;
1429 
1430 	ret = i915_gem_object_set_to_gtt_domain(obj, write);
1431 	if (ret)
1432 		goto unpin;
1433 
1434 	ret = i915_gem_object_get_fence(obj);
1435 	if (ret)
1436 		goto unpin;
1437 
1438 	obj->fault_mappable = true;
1439 
1440 	pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
1441 	pfn >>= PAGE_SHIFT;
1442 	pfn += page_offset;
1443 
1444 	/* Finally, remap it using the new GTT offset */
1445 	ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1446 unpin:
1447 	i915_gem_object_ggtt_unpin(obj);
1448 unlock:
1449 	mutex_unlock(&dev->struct_mutex);
1450 out:
1451 	switch (ret) {
1452 	case -EIO:
1453 		/* If this -EIO is due to a gpu hang, give the reset code a
1454 		 * chance to clean up the mess. Otherwise return the proper
1455 		 * SIGBUS. */
1456 		if (i915_terminally_wedged(&dev_priv->gpu_error)) {
1457 			ret = VM_FAULT_SIGBUS;
1458 			break;
1459 		}
1460 	case -EAGAIN:
1461 		/*
1462 		 * EAGAIN means the gpu is hung and we'll wait for the error
1463 		 * handler to reset everything when re-faulting in
1464 		 * i915_mutex_lock_interruptible.
1465 		 */
1466 	case 0:
1467 	case -ERESTARTSYS:
1468 	case -EINTR:
1469 	case -EBUSY:
1470 		/*
1471 		 * EBUSY is ok: this just means that another thread
1472 		 * already did the job.
1473 		 */
1474 		ret = VM_FAULT_NOPAGE;
1475 		break;
1476 	case -ENOMEM:
1477 		ret = VM_FAULT_OOM;
1478 		break;
1479 	case -ENOSPC:
1480 	case -EFAULT:
1481 		ret = VM_FAULT_SIGBUS;
1482 		break;
1483 	default:
1484 		WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1485 		ret = VM_FAULT_SIGBUS;
1486 		break;
1487 	}
1488 
1489 	intel_runtime_pm_put(dev_priv);
1490 	return ret;
1491 }
1492 
1493 void i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
1494 {
1495 	struct i915_vma *vma;
1496 
1497 	/*
1498 	 * Only the global gtt is relevant for gtt memory mappings, so restrict
1499 	 * list traversal to objects bound into the global address space. Note
1500 	 * that the active list should be empty, but better safe than sorry.
1501 	 */
1502 	WARN_ON(!list_empty(&dev_priv->gtt.base.active_list));
1503 	list_for_each_entry(vma, &dev_priv->gtt.base.active_list, mm_list)
1504 		i915_gem_release_mmap(vma->obj);
1505 	list_for_each_entry(vma, &dev_priv->gtt.base.inactive_list, mm_list)
1506 		i915_gem_release_mmap(vma->obj);
1507 }
1508 
1509 /**
1510  * i915_gem_release_mmap - remove physical page mappings
1511  * @obj: obj in question
1512  *
1513  * Preserve the reservation of the mmapping with the DRM core code, but
1514  * relinquish ownership of the pages back to the system.
1515  *
1516  * It is vital that we remove the page mapping if we have mapped a tiled
1517  * object through the GTT and then lose the fence register due to
1518  * resource pressure. Similarly if the object has been moved out of the
1519  * aperture, than pages mapped into userspace must be revoked. Removing the
1520  * mapping will then trigger a page fault on the next user access, allowing
1521  * fixup by i915_gem_fault().
1522  */
1523 void
1524 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1525 {
1526 	if (!obj->fault_mappable)
1527 		return;
1528 
1529 	drm_vma_node_unmap(&obj->base.vma_node,
1530 			   obj->base.dev->anon_inode->i_mapping);
1531 	obj->fault_mappable = false;
1532 }
1533 
1534 uint32_t
1535 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1536 {
1537 	uint32_t gtt_size;
1538 
1539 	if (INTEL_INFO(dev)->gen >= 4 ||
1540 	    tiling_mode == I915_TILING_NONE)
1541 		return size;
1542 
1543 	/* Previous chips need a power-of-two fence region when tiling */
1544 	if (INTEL_INFO(dev)->gen == 3)
1545 		gtt_size = 1024*1024;
1546 	else
1547 		gtt_size = 512*1024;
1548 
1549 	while (gtt_size < size)
1550 		gtt_size <<= 1;
1551 
1552 	return gtt_size;
1553 }
1554 
1555 /**
1556  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1557  * @obj: object to check
1558  *
1559  * Return the required GTT alignment for an object, taking into account
1560  * potential fence register mapping.
1561  */
1562 uint32_t
1563 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
1564 			   int tiling_mode, bool fenced)
1565 {
1566 	/*
1567 	 * Minimum alignment is 4k (GTT page size), but might be greater
1568 	 * if a fence register is needed for the object.
1569 	 */
1570 	if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
1571 	    tiling_mode == I915_TILING_NONE)
1572 		return 4096;
1573 
1574 	/*
1575 	 * Previous chips need to be aligned to the size of the smallest
1576 	 * fence register that can contain the object.
1577 	 */
1578 	return i915_gem_get_gtt_size(dev, size, tiling_mode);
1579 }
1580 
1581 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
1582 {
1583 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1584 	int ret;
1585 
1586 	if (drm_vma_node_has_offset(&obj->base.vma_node))
1587 		return 0;
1588 
1589 	dev_priv->mm.shrinker_no_lock_stealing = true;
1590 
1591 	ret = drm_gem_create_mmap_offset(&obj->base);
1592 	if (ret != -ENOSPC)
1593 		goto out;
1594 
1595 	/* Badly fragmented mmap space? The only way we can recover
1596 	 * space is by destroying unwanted objects. We can't randomly release
1597 	 * mmap_offsets as userspace expects them to be persistent for the
1598 	 * lifetime of the objects. The closest we can is to release the
1599 	 * offsets on purgeable objects by truncating it and marking it purged,
1600 	 * which prevents userspace from ever using that object again.
1601 	 */
1602 	i915_gem_purge(dev_priv, obj->base.size >> PAGE_SHIFT);
1603 	ret = drm_gem_create_mmap_offset(&obj->base);
1604 	if (ret != -ENOSPC)
1605 		goto out;
1606 
1607 	i915_gem_shrink_all(dev_priv);
1608 	ret = drm_gem_create_mmap_offset(&obj->base);
1609 out:
1610 	dev_priv->mm.shrinker_no_lock_stealing = false;
1611 
1612 	return ret;
1613 }
1614 
1615 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
1616 {
1617 	drm_gem_free_mmap_offset(&obj->base);
1618 }
1619 
1620 int
1621 i915_gem_mmap_gtt(struct drm_file *file,
1622 		  struct drm_device *dev,
1623 		  uint32_t handle,
1624 		  uint64_t *offset)
1625 {
1626 	struct drm_i915_private *dev_priv = dev->dev_private;
1627 	struct drm_i915_gem_object *obj;
1628 	int ret;
1629 
1630 	ret = i915_mutex_lock_interruptible(dev);
1631 	if (ret)
1632 		return ret;
1633 
1634 	obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1635 	if (&obj->base == NULL) {
1636 		ret = -ENOENT;
1637 		goto unlock;
1638 	}
1639 
1640 	if (obj->base.size > dev_priv->gtt.mappable_end) {
1641 		ret = -E2BIG;
1642 		goto out;
1643 	}
1644 
1645 	if (obj->madv != I915_MADV_WILLNEED) {
1646 		DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
1647 		ret = -EFAULT;
1648 		goto out;
1649 	}
1650 
1651 	ret = i915_gem_object_create_mmap_offset(obj);
1652 	if (ret)
1653 		goto out;
1654 
1655 	*offset = drm_vma_node_offset_addr(&obj->base.vma_node);
1656 
1657 out:
1658 	drm_gem_object_unreference(&obj->base);
1659 unlock:
1660 	mutex_unlock(&dev->struct_mutex);
1661 	return ret;
1662 }
1663 
1664 /**
1665  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1666  * @dev: DRM device
1667  * @data: GTT mapping ioctl data
1668  * @file: GEM object info
1669  *
1670  * Simply returns the fake offset to userspace so it can mmap it.
1671  * The mmap call will end up in drm_gem_mmap(), which will set things
1672  * up so we can get faults in the handler above.
1673  *
1674  * The fault handler will take care of binding the object into the GTT
1675  * (since it may have been evicted to make room for something), allocating
1676  * a fence register, and mapping the appropriate aperture address into
1677  * userspace.
1678  */
1679 int
1680 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1681 			struct drm_file *file)
1682 {
1683 	struct drm_i915_gem_mmap_gtt *args = data;
1684 
1685 	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1686 }
1687 
1688 /* Immediately discard the backing storage */
1689 static void
1690 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1691 {
1692 	struct inode *inode;
1693 
1694 	i915_gem_object_free_mmap_offset(obj);
1695 
1696 	if (obj->base.filp == NULL)
1697 		return;
1698 
1699 	/* Our goal here is to return as much of the memory as
1700 	 * is possible back to the system as we are called from OOM.
1701 	 * To do this we must instruct the shmfs to drop all of its
1702 	 * backing pages, *now*.
1703 	 */
1704 	inode = file_inode(obj->base.filp);
1705 	shmem_truncate_range(inode, 0, (loff_t)-1);
1706 
1707 	obj->madv = __I915_MADV_PURGED;
1708 }
1709 
1710 static inline int
1711 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1712 {
1713 	return obj->madv == I915_MADV_DONTNEED;
1714 }
1715 
1716 static void
1717 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1718 {
1719 	struct sg_page_iter sg_iter;
1720 	int ret;
1721 
1722 	BUG_ON(obj->madv == __I915_MADV_PURGED);
1723 
1724 	ret = i915_gem_object_set_to_cpu_domain(obj, true);
1725 	if (ret) {
1726 		/* In the event of a disaster, abandon all caches and
1727 		 * hope for the best.
1728 		 */
1729 		WARN_ON(ret != -EIO);
1730 		i915_gem_clflush_object(obj, true);
1731 		obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
1732 	}
1733 
1734 	if (i915_gem_object_needs_bit17_swizzle(obj))
1735 		i915_gem_object_save_bit_17_swizzle(obj);
1736 
1737 	if (obj->madv == I915_MADV_DONTNEED)
1738 		obj->dirty = 0;
1739 
1740 	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
1741 		struct page *page = sg_page_iter_page(&sg_iter);
1742 
1743 		if (obj->dirty)
1744 			set_page_dirty(page);
1745 
1746 		if (obj->madv == I915_MADV_WILLNEED)
1747 			mark_page_accessed(page);
1748 
1749 		page_cache_release(page);
1750 	}
1751 	obj->dirty = 0;
1752 
1753 	sg_free_table(obj->pages);
1754 	kfree(obj->pages);
1755 }
1756 
1757 int
1758 i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
1759 {
1760 	const struct drm_i915_gem_object_ops *ops = obj->ops;
1761 
1762 	if (obj->pages == NULL)
1763 		return 0;
1764 
1765 	if (obj->pages_pin_count)
1766 		return -EBUSY;
1767 
1768 	BUG_ON(i915_gem_obj_bound_any(obj));
1769 
1770 	/* ->put_pages might need to allocate memory for the bit17 swizzle
1771 	 * array, hence protect them from being reaped by removing them from gtt
1772 	 * lists early. */
1773 	list_del(&obj->global_list);
1774 
1775 	ops->put_pages(obj);
1776 	obj->pages = NULL;
1777 
1778 	if (i915_gem_object_is_purgeable(obj))
1779 		i915_gem_object_truncate(obj);
1780 
1781 	return 0;
1782 }
1783 
1784 static unsigned long
1785 __i915_gem_shrink(struct drm_i915_private *dev_priv, long target,
1786 		  bool purgeable_only)
1787 {
1788 	struct list_head still_bound_list;
1789 	struct drm_i915_gem_object *obj, *next;
1790 	unsigned long count = 0;
1791 
1792 	list_for_each_entry_safe(obj, next,
1793 				 &dev_priv->mm.unbound_list,
1794 				 global_list) {
1795 		if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) &&
1796 		    i915_gem_object_put_pages(obj) == 0) {
1797 			count += obj->base.size >> PAGE_SHIFT;
1798 			if (count >= target)
1799 				return count;
1800 		}
1801 	}
1802 
1803 	/*
1804 	 * As we may completely rewrite the bound list whilst unbinding
1805 	 * (due to retiring requests) we have to strictly process only
1806 	 * one element of the list at the time, and recheck the list
1807 	 * on every iteration.
1808 	 */
1809 	INIT_LIST_HEAD(&still_bound_list);
1810 	while (count < target && !list_empty(&dev_priv->mm.bound_list)) {
1811 		struct i915_vma *vma, *v;
1812 
1813 		obj = list_first_entry(&dev_priv->mm.bound_list,
1814 				       typeof(*obj), global_list);
1815 		list_move_tail(&obj->global_list, &still_bound_list);
1816 
1817 		if (!i915_gem_object_is_purgeable(obj) && purgeable_only)
1818 			continue;
1819 
1820 		/*
1821 		 * Hold a reference whilst we unbind this object, as we may
1822 		 * end up waiting for and retiring requests. This might
1823 		 * release the final reference (held by the active list)
1824 		 * and result in the object being freed from under us.
1825 		 * in this object being freed.
1826 		 *
1827 		 * Note 1: Shrinking the bound list is special since only active
1828 		 * (and hence bound objects) can contain such limbo objects, so
1829 		 * we don't need special tricks for shrinking the unbound list.
1830 		 * The only other place where we have to be careful with active
1831 		 * objects suddenly disappearing due to retiring requests is the
1832 		 * eviction code.
1833 		 *
1834 		 * Note 2: Even though the bound list doesn't hold a reference
1835 		 * to the object we can safely grab one here: The final object
1836 		 * unreferencing and the bound_list are both protected by the
1837 		 * dev->struct_mutex and so we won't ever be able to observe an
1838 		 * object on the bound_list with a reference count equals 0.
1839 		 */
1840 		drm_gem_object_reference(&obj->base);
1841 
1842 		list_for_each_entry_safe(vma, v, &obj->vma_list, vma_link)
1843 			if (i915_vma_unbind(vma))
1844 				break;
1845 
1846 		if (i915_gem_object_put_pages(obj) == 0)
1847 			count += obj->base.size >> PAGE_SHIFT;
1848 
1849 		drm_gem_object_unreference(&obj->base);
1850 	}
1851 	list_splice(&still_bound_list, &dev_priv->mm.bound_list);
1852 
1853 	return count;
1854 }
1855 
1856 static unsigned long
1857 i915_gem_purge(struct drm_i915_private *dev_priv, long target)
1858 {
1859 	return __i915_gem_shrink(dev_priv, target, true);
1860 }
1861 
1862 static unsigned long
1863 i915_gem_shrink_all(struct drm_i915_private *dev_priv)
1864 {
1865 	struct drm_i915_gem_object *obj, *next;
1866 	long freed = 0;
1867 
1868 	i915_gem_evict_everything(dev_priv->dev);
1869 
1870 	list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list,
1871 				 global_list) {
1872 		if (i915_gem_object_put_pages(obj) == 0)
1873 			freed += obj->base.size >> PAGE_SHIFT;
1874 	}
1875 	return freed;
1876 }
1877 
1878 static int
1879 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
1880 {
1881 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1882 	int page_count, i;
1883 	struct address_space *mapping;
1884 	struct sg_table *st;
1885 	struct scatterlist *sg;
1886 	struct sg_page_iter sg_iter;
1887 	struct page *page;
1888 	unsigned long last_pfn = 0;	/* suppress gcc warning */
1889 	gfp_t gfp;
1890 
1891 	/* Assert that the object is not currently in any GPU domain. As it
1892 	 * wasn't in the GTT, there shouldn't be any way it could have been in
1893 	 * a GPU cache
1894 	 */
1895 	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
1896 	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
1897 
1898 	st = kmalloc(sizeof(*st), GFP_KERNEL);
1899 	if (st == NULL)
1900 		return -ENOMEM;
1901 
1902 	page_count = obj->base.size / PAGE_SIZE;
1903 	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
1904 		kfree(st);
1905 		return -ENOMEM;
1906 	}
1907 
1908 	/* Get the list of pages out of our struct file.  They'll be pinned
1909 	 * at this point until we release them.
1910 	 *
1911 	 * Fail silently without starting the shrinker
1912 	 */
1913 	mapping = file_inode(obj->base.filp)->i_mapping;
1914 	gfp = mapping_gfp_mask(mapping);
1915 	gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
1916 	gfp &= ~(__GFP_IO | __GFP_WAIT);
1917 	sg = st->sgl;
1918 	st->nents = 0;
1919 	for (i = 0; i < page_count; i++) {
1920 		page = shmem_read_mapping_page_gfp(mapping, i, gfp);
1921 		if (IS_ERR(page)) {
1922 			i915_gem_purge(dev_priv, page_count);
1923 			page = shmem_read_mapping_page_gfp(mapping, i, gfp);
1924 		}
1925 		if (IS_ERR(page)) {
1926 			/* We've tried hard to allocate the memory by reaping
1927 			 * our own buffer, now let the real VM do its job and
1928 			 * go down in flames if truly OOM.
1929 			 */
1930 			gfp &= ~(__GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD);
1931 			gfp |= __GFP_IO | __GFP_WAIT;
1932 
1933 			i915_gem_shrink_all(dev_priv);
1934 			page = shmem_read_mapping_page_gfp(mapping, i, gfp);
1935 			if (IS_ERR(page))
1936 				goto err_pages;
1937 
1938 			gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
1939 			gfp &= ~(__GFP_IO | __GFP_WAIT);
1940 		}
1941 #ifdef CONFIG_SWIOTLB
1942 		if (swiotlb_nr_tbl()) {
1943 			st->nents++;
1944 			sg_set_page(sg, page, PAGE_SIZE, 0);
1945 			sg = sg_next(sg);
1946 			continue;
1947 		}
1948 #endif
1949 		if (!i || page_to_pfn(page) != last_pfn + 1) {
1950 			if (i)
1951 				sg = sg_next(sg);
1952 			st->nents++;
1953 			sg_set_page(sg, page, PAGE_SIZE, 0);
1954 		} else {
1955 			sg->length += PAGE_SIZE;
1956 		}
1957 		last_pfn = page_to_pfn(page);
1958 
1959 		/* Check that the i965g/gm workaround works. */
1960 		WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
1961 	}
1962 #ifdef CONFIG_SWIOTLB
1963 	if (!swiotlb_nr_tbl())
1964 #endif
1965 		sg_mark_end(sg);
1966 	obj->pages = st;
1967 
1968 	if (i915_gem_object_needs_bit17_swizzle(obj))
1969 		i915_gem_object_do_bit_17_swizzle(obj);
1970 
1971 	return 0;
1972 
1973 err_pages:
1974 	sg_mark_end(sg);
1975 	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
1976 		page_cache_release(sg_page_iter_page(&sg_iter));
1977 	sg_free_table(st);
1978 	kfree(st);
1979 	return PTR_ERR(page);
1980 }
1981 
1982 /* Ensure that the associated pages are gathered from the backing storage
1983  * and pinned into our object. i915_gem_object_get_pages() may be called
1984  * multiple times before they are released by a single call to
1985  * i915_gem_object_put_pages() - once the pages are no longer referenced
1986  * either as a result of memory pressure (reaping pages under the shrinker)
1987  * or as the object is itself released.
1988  */
1989 int
1990 i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
1991 {
1992 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1993 	const struct drm_i915_gem_object_ops *ops = obj->ops;
1994 	int ret;
1995 
1996 	if (obj->pages)
1997 		return 0;
1998 
1999 	if (obj->madv != I915_MADV_WILLNEED) {
2000 		DRM_DEBUG("Attempting to obtain a purgeable object\n");
2001 		return -EFAULT;
2002 	}
2003 
2004 	BUG_ON(obj->pages_pin_count);
2005 
2006 	ret = ops->get_pages(obj);
2007 	if (ret)
2008 		return ret;
2009 
2010 	list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2011 	return 0;
2012 }
2013 
2014 static void
2015 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
2016 			       struct intel_ring_buffer *ring)
2017 {
2018 	struct drm_device *dev = obj->base.dev;
2019 	struct drm_i915_private *dev_priv = dev->dev_private;
2020 	u32 seqno = intel_ring_get_seqno(ring);
2021 
2022 	BUG_ON(ring == NULL);
2023 	if (obj->ring != ring && obj->last_write_seqno) {
2024 		/* Keep the seqno relative to the current ring */
2025 		obj->last_write_seqno = seqno;
2026 	}
2027 	obj->ring = ring;
2028 
2029 	/* Add a reference if we're newly entering the active list. */
2030 	if (!obj->active) {
2031 		drm_gem_object_reference(&obj->base);
2032 		obj->active = 1;
2033 	}
2034 
2035 	list_move_tail(&obj->ring_list, &ring->active_list);
2036 
2037 	obj->last_read_seqno = seqno;
2038 
2039 	if (obj->fenced_gpu_access) {
2040 		obj->last_fenced_seqno = seqno;
2041 
2042 		/* Bump MRU to take account of the delayed flush */
2043 		if (obj->fence_reg != I915_FENCE_REG_NONE) {
2044 			struct drm_i915_fence_reg *reg;
2045 
2046 			reg = &dev_priv->fence_regs[obj->fence_reg];
2047 			list_move_tail(&reg->lru_list,
2048 				       &dev_priv->mm.fence_list);
2049 		}
2050 	}
2051 }
2052 
2053 void i915_vma_move_to_active(struct i915_vma *vma,
2054 			     struct intel_ring_buffer *ring)
2055 {
2056 	list_move_tail(&vma->mm_list, &vma->vm->active_list);
2057 	return i915_gem_object_move_to_active(vma->obj, ring);
2058 }
2059 
2060 static void
2061 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2062 {
2063 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2064 	struct i915_address_space *vm;
2065 	struct i915_vma *vma;
2066 
2067 	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2068 	BUG_ON(!obj->active);
2069 
2070 	list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
2071 		vma = i915_gem_obj_to_vma(obj, vm);
2072 		if (vma && !list_empty(&vma->mm_list))
2073 			list_move_tail(&vma->mm_list, &vm->inactive_list);
2074 	}
2075 
2076 	list_del_init(&obj->ring_list);
2077 	obj->ring = NULL;
2078 
2079 	obj->last_read_seqno = 0;
2080 	obj->last_write_seqno = 0;
2081 	obj->base.write_domain = 0;
2082 
2083 	obj->last_fenced_seqno = 0;
2084 	obj->fenced_gpu_access = false;
2085 
2086 	obj->active = 0;
2087 	drm_gem_object_unreference(&obj->base);
2088 
2089 	WARN_ON(i915_verify_lists(dev));
2090 }
2091 
2092 static int
2093 i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
2094 {
2095 	struct drm_i915_private *dev_priv = dev->dev_private;
2096 	struct intel_ring_buffer *ring;
2097 	int ret, i, j;
2098 
2099 	/* Carefully retire all requests without writing to the rings */
2100 	for_each_ring(ring, dev_priv, i) {
2101 		ret = intel_ring_idle(ring);
2102 		if (ret)
2103 			return ret;
2104 	}
2105 	i915_gem_retire_requests(dev);
2106 
2107 	/* Finally reset hw state */
2108 	for_each_ring(ring, dev_priv, i) {
2109 		intel_ring_init_seqno(ring, seqno);
2110 
2111 		for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
2112 			ring->sync_seqno[j] = 0;
2113 	}
2114 
2115 	return 0;
2116 }
2117 
2118 int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
2119 {
2120 	struct drm_i915_private *dev_priv = dev->dev_private;
2121 	int ret;
2122 
2123 	if (seqno == 0)
2124 		return -EINVAL;
2125 
2126 	/* HWS page needs to be set less than what we
2127 	 * will inject to ring
2128 	 */
2129 	ret = i915_gem_init_seqno(dev, seqno - 1);
2130 	if (ret)
2131 		return ret;
2132 
2133 	/* Carefully set the last_seqno value so that wrap
2134 	 * detection still works
2135 	 */
2136 	dev_priv->next_seqno = seqno;
2137 	dev_priv->last_seqno = seqno - 1;
2138 	if (dev_priv->last_seqno == 0)
2139 		dev_priv->last_seqno--;
2140 
2141 	return 0;
2142 }
2143 
2144 int
2145 i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
2146 {
2147 	struct drm_i915_private *dev_priv = dev->dev_private;
2148 
2149 	/* reserve 0 for non-seqno */
2150 	if (dev_priv->next_seqno == 0) {
2151 		int ret = i915_gem_init_seqno(dev, 0);
2152 		if (ret)
2153 			return ret;
2154 
2155 		dev_priv->next_seqno = 1;
2156 	}
2157 
2158 	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
2159 	return 0;
2160 }
2161 
2162 int __i915_add_request(struct intel_ring_buffer *ring,
2163 		       struct drm_file *file,
2164 		       struct drm_i915_gem_object *obj,
2165 		       u32 *out_seqno)
2166 {
2167 	struct drm_i915_private *dev_priv = ring->dev->dev_private;
2168 	struct drm_i915_gem_request *request;
2169 	u32 request_ring_position, request_start;
2170 	int ret;
2171 
2172 	request_start = intel_ring_get_tail(ring);
2173 	/*
2174 	 * Emit any outstanding flushes - execbuf can fail to emit the flush
2175 	 * after having emitted the batchbuffer command. Hence we need to fix
2176 	 * things up similar to emitting the lazy request. The difference here
2177 	 * is that the flush _must_ happen before the next request, no matter
2178 	 * what.
2179 	 */
2180 	ret = intel_ring_flush_all_caches(ring);
2181 	if (ret)
2182 		return ret;
2183 
2184 	request = ring->preallocated_lazy_request;
2185 	if (WARN_ON(request == NULL))
2186 		return -ENOMEM;
2187 
2188 	/* Record the position of the start of the request so that
2189 	 * should we detect the updated seqno part-way through the
2190 	 * GPU processing the request, we never over-estimate the
2191 	 * position of the head.
2192 	 */
2193 	request_ring_position = intel_ring_get_tail(ring);
2194 
2195 	ret = ring->add_request(ring);
2196 	if (ret)
2197 		return ret;
2198 
2199 	request->seqno = intel_ring_get_seqno(ring);
2200 	request->ring = ring;
2201 	request->head = request_start;
2202 	request->tail = request_ring_position;
2203 
2204 	/* Whilst this request exists, batch_obj will be on the
2205 	 * active_list, and so will hold the active reference. Only when this
2206 	 * request is retired will the the batch_obj be moved onto the
2207 	 * inactive_list and lose its active reference. Hence we do not need
2208 	 * to explicitly hold another reference here.
2209 	 */
2210 	request->batch_obj = obj;
2211 
2212 	/* Hold a reference to the current context so that we can inspect
2213 	 * it later in case a hangcheck error event fires.
2214 	 */
2215 	request->ctx = ring->last_context;
2216 	if (request->ctx)
2217 		i915_gem_context_reference(request->ctx);
2218 
2219 	request->emitted_jiffies = jiffies;
2220 	list_add_tail(&request->list, &ring->request_list);
2221 	request->file_priv = NULL;
2222 
2223 	if (file) {
2224 		struct drm_i915_file_private *file_priv = file->driver_priv;
2225 
2226 		spin_lock(&file_priv->mm.lock);
2227 		request->file_priv = file_priv;
2228 		list_add_tail(&request->client_list,
2229 			      &file_priv->mm.request_list);
2230 		spin_unlock(&file_priv->mm.lock);
2231 	}
2232 
2233 	trace_i915_gem_request_add(ring, request->seqno);
2234 	ring->outstanding_lazy_seqno = 0;
2235 	ring->preallocated_lazy_request = NULL;
2236 
2237 	if (!dev_priv->ums.mm_suspended) {
2238 		i915_queue_hangcheck(ring->dev);
2239 
2240 		cancel_delayed_work_sync(&dev_priv->mm.idle_work);
2241 		queue_delayed_work(dev_priv->wq,
2242 				   &dev_priv->mm.retire_work,
2243 				   round_jiffies_up_relative(HZ));
2244 		intel_mark_busy(dev_priv->dev);
2245 	}
2246 
2247 	if (out_seqno)
2248 		*out_seqno = request->seqno;
2249 	return 0;
2250 }
2251 
2252 static inline void
2253 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
2254 {
2255 	struct drm_i915_file_private *file_priv = request->file_priv;
2256 
2257 	if (!file_priv)
2258 		return;
2259 
2260 	spin_lock(&file_priv->mm.lock);
2261 	list_del(&request->client_list);
2262 	request->file_priv = NULL;
2263 	spin_unlock(&file_priv->mm.lock);
2264 }
2265 
2266 static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
2267 				   const struct i915_hw_context *ctx)
2268 {
2269 	unsigned long elapsed;
2270 
2271 	elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
2272 
2273 	if (ctx->hang_stats.banned)
2274 		return true;
2275 
2276 	if (elapsed <= DRM_I915_CTX_BAN_PERIOD) {
2277 		if (!i915_gem_context_is_default(ctx)) {
2278 			DRM_DEBUG("context hanging too fast, banning!\n");
2279 			return true;
2280 		} else if (dev_priv->gpu_error.stop_rings == 0) {
2281 			DRM_ERROR("gpu hanging too fast, banning!\n");
2282 			return true;
2283 		}
2284 	}
2285 
2286 	return false;
2287 }
2288 
2289 static void i915_set_reset_status(struct drm_i915_private *dev_priv,
2290 				  struct i915_hw_context *ctx,
2291 				  const bool guilty)
2292 {
2293 	struct i915_ctx_hang_stats *hs;
2294 
2295 	if (WARN_ON(!ctx))
2296 		return;
2297 
2298 	hs = &ctx->hang_stats;
2299 
2300 	if (guilty) {
2301 		hs->banned = i915_context_is_banned(dev_priv, ctx);
2302 		hs->batch_active++;
2303 		hs->guilty_ts = get_seconds();
2304 	} else {
2305 		hs->batch_pending++;
2306 	}
2307 }
2308 
2309 static void i915_gem_free_request(struct drm_i915_gem_request *request)
2310 {
2311 	list_del(&request->list);
2312 	i915_gem_request_remove_from_client(request);
2313 
2314 	if (request->ctx)
2315 		i915_gem_context_unreference(request->ctx);
2316 
2317 	kfree(request);
2318 }
2319 
2320 struct drm_i915_gem_request *
2321 i915_gem_find_active_request(struct intel_ring_buffer *ring)
2322 {
2323 	struct drm_i915_gem_request *request;
2324 	u32 completed_seqno;
2325 
2326 	completed_seqno = ring->get_seqno(ring, false);
2327 
2328 	list_for_each_entry(request, &ring->request_list, list) {
2329 		if (i915_seqno_passed(completed_seqno, request->seqno))
2330 			continue;
2331 
2332 		return request;
2333 	}
2334 
2335 	return NULL;
2336 }
2337 
2338 static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
2339 				       struct intel_ring_buffer *ring)
2340 {
2341 	struct drm_i915_gem_request *request;
2342 	bool ring_hung;
2343 
2344 	request = i915_gem_find_active_request(ring);
2345 
2346 	if (request == NULL)
2347 		return;
2348 
2349 	ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
2350 
2351 	i915_set_reset_status(dev_priv, request->ctx, ring_hung);
2352 
2353 	list_for_each_entry_continue(request, &ring->request_list, list)
2354 		i915_set_reset_status(dev_priv, request->ctx, false);
2355 }
2356 
2357 static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
2358 					struct intel_ring_buffer *ring)
2359 {
2360 	while (!list_empty(&ring->active_list)) {
2361 		struct drm_i915_gem_object *obj;
2362 
2363 		obj = list_first_entry(&ring->active_list,
2364 				       struct drm_i915_gem_object,
2365 				       ring_list);
2366 
2367 		i915_gem_object_move_to_inactive(obj);
2368 	}
2369 
2370 	/*
2371 	 * We must free the requests after all the corresponding objects have
2372 	 * been moved off active lists. Which is the same order as the normal
2373 	 * retire_requests function does. This is important if object hold
2374 	 * implicit references on things like e.g. ppgtt address spaces through
2375 	 * the request.
2376 	 */
2377 	while (!list_empty(&ring->request_list)) {
2378 		struct drm_i915_gem_request *request;
2379 
2380 		request = list_first_entry(&ring->request_list,
2381 					   struct drm_i915_gem_request,
2382 					   list);
2383 
2384 		i915_gem_free_request(request);
2385 	}
2386 }
2387 
2388 void i915_gem_restore_fences(struct drm_device *dev)
2389 {
2390 	struct drm_i915_private *dev_priv = dev->dev_private;
2391 	int i;
2392 
2393 	for (i = 0; i < dev_priv->num_fence_regs; i++) {
2394 		struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2395 
2396 		/*
2397 		 * Commit delayed tiling changes if we have an object still
2398 		 * attached to the fence, otherwise just clear the fence.
2399 		 */
2400 		if (reg->obj) {
2401 			i915_gem_object_update_fence(reg->obj, reg,
2402 						     reg->obj->tiling_mode);
2403 		} else {
2404 			i915_gem_write_fence(dev, i, NULL);
2405 		}
2406 	}
2407 }
2408 
2409 void i915_gem_reset(struct drm_device *dev)
2410 {
2411 	struct drm_i915_private *dev_priv = dev->dev_private;
2412 	struct intel_ring_buffer *ring;
2413 	int i;
2414 
2415 	/*
2416 	 * Before we free the objects from the requests, we need to inspect
2417 	 * them for finding the guilty party. As the requests only borrow
2418 	 * their reference to the objects, the inspection must be done first.
2419 	 */
2420 	for_each_ring(ring, dev_priv, i)
2421 		i915_gem_reset_ring_status(dev_priv, ring);
2422 
2423 	for_each_ring(ring, dev_priv, i)
2424 		i915_gem_reset_ring_cleanup(dev_priv, ring);
2425 
2426 	i915_gem_cleanup_ringbuffer(dev);
2427 
2428 	i915_gem_context_reset(dev);
2429 
2430 	i915_gem_restore_fences(dev);
2431 }
2432 
2433 /**
2434  * This function clears the request list as sequence numbers are passed.
2435  */
2436 static void
2437 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
2438 {
2439 	uint32_t seqno;
2440 
2441 	if (list_empty(&ring->request_list))
2442 		return;
2443 
2444 	WARN_ON(i915_verify_lists(ring->dev));
2445 
2446 	seqno = ring->get_seqno(ring, true);
2447 
2448 	/* Move any buffers on the active list that are no longer referenced
2449 	 * by the ringbuffer to the flushing/inactive lists as appropriate,
2450 	 * before we free the context associated with the requests.
2451 	 */
2452 	while (!list_empty(&ring->active_list)) {
2453 		struct drm_i915_gem_object *obj;
2454 
2455 		obj = list_first_entry(&ring->active_list,
2456 				      struct drm_i915_gem_object,
2457 				      ring_list);
2458 
2459 		if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2460 			break;
2461 
2462 		i915_gem_object_move_to_inactive(obj);
2463 	}
2464 
2465 
2466 	while (!list_empty(&ring->request_list)) {
2467 		struct drm_i915_gem_request *request;
2468 
2469 		request = list_first_entry(&ring->request_list,
2470 					   struct drm_i915_gem_request,
2471 					   list);
2472 
2473 		if (!i915_seqno_passed(seqno, request->seqno))
2474 			break;
2475 
2476 		trace_i915_gem_request_retire(ring, request->seqno);
2477 		/* We know the GPU must have read the request to have
2478 		 * sent us the seqno + interrupt, so use the position
2479 		 * of tail of the request to update the last known position
2480 		 * of the GPU head.
2481 		 */
2482 		ring->last_retired_head = request->tail;
2483 
2484 		i915_gem_free_request(request);
2485 	}
2486 
2487 	if (unlikely(ring->trace_irq_seqno &&
2488 		     i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
2489 		ring->irq_put(ring);
2490 		ring->trace_irq_seqno = 0;
2491 	}
2492 
2493 	WARN_ON(i915_verify_lists(ring->dev));
2494 }
2495 
2496 bool
2497 i915_gem_retire_requests(struct drm_device *dev)
2498 {
2499 	struct drm_i915_private *dev_priv = dev->dev_private;
2500 	struct intel_ring_buffer *ring;
2501 	bool idle = true;
2502 	int i;
2503 
2504 	for_each_ring(ring, dev_priv, i) {
2505 		i915_gem_retire_requests_ring(ring);
2506 		idle &= list_empty(&ring->request_list);
2507 	}
2508 
2509 	if (idle)
2510 		mod_delayed_work(dev_priv->wq,
2511 				   &dev_priv->mm.idle_work,
2512 				   msecs_to_jiffies(100));
2513 
2514 	return idle;
2515 }
2516 
2517 static void
2518 i915_gem_retire_work_handler(struct work_struct *work)
2519 {
2520 	struct drm_i915_private *dev_priv =
2521 		container_of(work, typeof(*dev_priv), mm.retire_work.work);
2522 	struct drm_device *dev = dev_priv->dev;
2523 	bool idle;
2524 
2525 	/* Come back later if the device is busy... */
2526 	idle = false;
2527 	if (mutex_trylock(&dev->struct_mutex)) {
2528 		idle = i915_gem_retire_requests(dev);
2529 		mutex_unlock(&dev->struct_mutex);
2530 	}
2531 	if (!idle)
2532 		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
2533 				   round_jiffies_up_relative(HZ));
2534 }
2535 
2536 static void
2537 i915_gem_idle_work_handler(struct work_struct *work)
2538 {
2539 	struct drm_i915_private *dev_priv =
2540 		container_of(work, typeof(*dev_priv), mm.idle_work.work);
2541 
2542 	intel_mark_idle(dev_priv->dev);
2543 }
2544 
2545 /**
2546  * Ensures that an object will eventually get non-busy by flushing any required
2547  * write domains, emitting any outstanding lazy request and retiring and
2548  * completed requests.
2549  */
2550 static int
2551 i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2552 {
2553 	int ret;
2554 
2555 	if (obj->active) {
2556 		ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
2557 		if (ret)
2558 			return ret;
2559 
2560 		i915_gem_retire_requests_ring(obj->ring);
2561 	}
2562 
2563 	return 0;
2564 }
2565 
2566 /**
2567  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
2568  * @DRM_IOCTL_ARGS: standard ioctl arguments
2569  *
2570  * Returns 0 if successful, else an error is returned with the remaining time in
2571  * the timeout parameter.
2572  *  -ETIME: object is still busy after timeout
2573  *  -ERESTARTSYS: signal interrupted the wait
2574  *  -ENONENT: object doesn't exist
2575  * Also possible, but rare:
2576  *  -EAGAIN: GPU wedged
2577  *  -ENOMEM: damn
2578  *  -ENODEV: Internal IRQ fail
2579  *  -E?: The add request failed
2580  *
2581  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
2582  * non-zero timeout parameter the wait ioctl will wait for the given number of
2583  * nanoseconds on an object becoming unbusy. Since the wait itself does so
2584  * without holding struct_mutex the object may become re-busied before this
2585  * function completes. A similar but shorter * race condition exists in the busy
2586  * ioctl
2587  */
2588 int
2589 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2590 {
2591 	struct drm_i915_private *dev_priv = dev->dev_private;
2592 	struct drm_i915_gem_wait *args = data;
2593 	struct drm_i915_gem_object *obj;
2594 	struct intel_ring_buffer *ring = NULL;
2595 	struct timespec timeout_stack, *timeout = NULL;
2596 	unsigned reset_counter;
2597 	u32 seqno = 0;
2598 	int ret = 0;
2599 
2600 	if (args->timeout_ns >= 0) {
2601 		timeout_stack = ns_to_timespec(args->timeout_ns);
2602 		timeout = &timeout_stack;
2603 	}
2604 
2605 	ret = i915_mutex_lock_interruptible(dev);
2606 	if (ret)
2607 		return ret;
2608 
2609 	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
2610 	if (&obj->base == NULL) {
2611 		mutex_unlock(&dev->struct_mutex);
2612 		return -ENOENT;
2613 	}
2614 
2615 	/* Need to make sure the object gets inactive eventually. */
2616 	ret = i915_gem_object_flush_active(obj);
2617 	if (ret)
2618 		goto out;
2619 
2620 	if (obj->active) {
2621 		seqno = obj->last_read_seqno;
2622 		ring = obj->ring;
2623 	}
2624 
2625 	if (seqno == 0)
2626 		 goto out;
2627 
2628 	/* Do this after OLR check to make sure we make forward progress polling
2629 	 * on this IOCTL with a 0 timeout (like busy ioctl)
2630 	 */
2631 	if (!args->timeout_ns) {
2632 		ret = -ETIME;
2633 		goto out;
2634 	}
2635 
2636 	drm_gem_object_unreference(&obj->base);
2637 	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
2638 	mutex_unlock(&dev->struct_mutex);
2639 
2640 	ret = __wait_seqno(ring, seqno, reset_counter, true, timeout, file->driver_priv);
2641 	if (timeout)
2642 		args->timeout_ns = timespec_to_ns(timeout);
2643 	return ret;
2644 
2645 out:
2646 	drm_gem_object_unreference(&obj->base);
2647 	mutex_unlock(&dev->struct_mutex);
2648 	return ret;
2649 }
2650 
2651 /**
2652  * i915_gem_object_sync - sync an object to a ring.
2653  *
2654  * @obj: object which may be in use on another ring.
2655  * @to: ring we wish to use the object on. May be NULL.
2656  *
2657  * This code is meant to abstract object synchronization with the GPU.
2658  * Calling with NULL implies synchronizing the object with the CPU
2659  * rather than a particular GPU ring.
2660  *
2661  * Returns 0 if successful, else propagates up the lower layer error.
2662  */
2663 int
2664 i915_gem_object_sync(struct drm_i915_gem_object *obj,
2665 		     struct intel_ring_buffer *to)
2666 {
2667 	struct intel_ring_buffer *from = obj->ring;
2668 	u32 seqno;
2669 	int ret, idx;
2670 
2671 	if (from == NULL || to == from)
2672 		return 0;
2673 
2674 	if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
2675 		return i915_gem_object_wait_rendering(obj, false);
2676 
2677 	idx = intel_ring_sync_index(from, to);
2678 
2679 	seqno = obj->last_read_seqno;
2680 	if (seqno <= from->sync_seqno[idx])
2681 		return 0;
2682 
2683 	ret = i915_gem_check_olr(obj->ring, seqno);
2684 	if (ret)
2685 		return ret;
2686 
2687 	trace_i915_gem_ring_sync_to(from, to, seqno);
2688 	ret = to->sync_to(to, from, seqno);
2689 	if (!ret)
2690 		/* We use last_read_seqno because sync_to()
2691 		 * might have just caused seqno wrap under
2692 		 * the radar.
2693 		 */
2694 		from->sync_seqno[idx] = obj->last_read_seqno;
2695 
2696 	return ret;
2697 }
2698 
2699 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
2700 {
2701 	u32 old_write_domain, old_read_domains;
2702 
2703 	/* Force a pagefault for domain tracking on next user access */
2704 	i915_gem_release_mmap(obj);
2705 
2706 	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
2707 		return;
2708 
2709 	/* Wait for any direct GTT access to complete */
2710 	mb();
2711 
2712 	old_read_domains = obj->base.read_domains;
2713 	old_write_domain = obj->base.write_domain;
2714 
2715 	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
2716 	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2717 
2718 	trace_i915_gem_object_change_domain(obj,
2719 					    old_read_domains,
2720 					    old_write_domain);
2721 }
2722 
2723 int i915_vma_unbind(struct i915_vma *vma)
2724 {
2725 	struct drm_i915_gem_object *obj = vma->obj;
2726 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2727 	int ret;
2728 
2729 	if (list_empty(&vma->vma_link))
2730 		return 0;
2731 
2732 	if (!drm_mm_node_allocated(&vma->node)) {
2733 		i915_gem_vma_destroy(vma);
2734 		return 0;
2735 	}
2736 
2737 	if (vma->pin_count)
2738 		return -EBUSY;
2739 
2740 	BUG_ON(obj->pages == NULL);
2741 
2742 	ret = i915_gem_object_finish_gpu(obj);
2743 	if (ret)
2744 		return ret;
2745 	/* Continue on if we fail due to EIO, the GPU is hung so we
2746 	 * should be safe and we need to cleanup or else we might
2747 	 * cause memory corruption through use-after-free.
2748 	 */
2749 
2750 	i915_gem_object_finish_gtt(obj);
2751 
2752 	/* release the fence reg _after_ flushing */
2753 	ret = i915_gem_object_put_fence(obj);
2754 	if (ret)
2755 		return ret;
2756 
2757 	trace_i915_vma_unbind(vma);
2758 
2759 	vma->unbind_vma(vma);
2760 
2761 	i915_gem_gtt_finish_object(obj);
2762 
2763 	list_del_init(&vma->mm_list);
2764 	/* Avoid an unnecessary call to unbind on rebind. */
2765 	if (i915_is_ggtt(vma->vm))
2766 		obj->map_and_fenceable = true;
2767 
2768 	drm_mm_remove_node(&vma->node);
2769 	i915_gem_vma_destroy(vma);
2770 
2771 	/* Since the unbound list is global, only move to that list if
2772 	 * no more VMAs exist. */
2773 	if (list_empty(&obj->vma_list))
2774 		list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2775 
2776 	/* And finally now the object is completely decoupled from this vma,
2777 	 * we can drop its hold on the backing storage and allow it to be
2778 	 * reaped by the shrinker.
2779 	 */
2780 	i915_gem_object_unpin_pages(obj);
2781 
2782 	return 0;
2783 }
2784 
2785 int i915_gpu_idle(struct drm_device *dev)
2786 {
2787 	struct drm_i915_private *dev_priv = dev->dev_private;
2788 	struct intel_ring_buffer *ring;
2789 	int ret, i;
2790 
2791 	/* Flush everything onto the inactive list. */
2792 	for_each_ring(ring, dev_priv, i) {
2793 		ret = i915_switch_context(ring, NULL, ring->default_context);
2794 		if (ret)
2795 			return ret;
2796 
2797 		ret = intel_ring_idle(ring);
2798 		if (ret)
2799 			return ret;
2800 	}
2801 
2802 	return 0;
2803 }
2804 
2805 static void i965_write_fence_reg(struct drm_device *dev, int reg,
2806 				 struct drm_i915_gem_object *obj)
2807 {
2808 	struct drm_i915_private *dev_priv = dev->dev_private;
2809 	int fence_reg;
2810 	int fence_pitch_shift;
2811 
2812 	if (INTEL_INFO(dev)->gen >= 6) {
2813 		fence_reg = FENCE_REG_SANDYBRIDGE_0;
2814 		fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
2815 	} else {
2816 		fence_reg = FENCE_REG_965_0;
2817 		fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
2818 	}
2819 
2820 	fence_reg += reg * 8;
2821 
2822 	/* To w/a incoherency with non-atomic 64-bit register updates,
2823 	 * we split the 64-bit update into two 32-bit writes. In order
2824 	 * for a partial fence not to be evaluated between writes, we
2825 	 * precede the update with write to turn off the fence register,
2826 	 * and only enable the fence as the last step.
2827 	 *
2828 	 * For extra levels of paranoia, we make sure each step lands
2829 	 * before applying the next step.
2830 	 */
2831 	I915_WRITE(fence_reg, 0);
2832 	POSTING_READ(fence_reg);
2833 
2834 	if (obj) {
2835 		u32 size = i915_gem_obj_ggtt_size(obj);
2836 		uint64_t val;
2837 
2838 		val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
2839 				 0xfffff000) << 32;
2840 		val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
2841 		val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
2842 		if (obj->tiling_mode == I915_TILING_Y)
2843 			val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2844 		val |= I965_FENCE_REG_VALID;
2845 
2846 		I915_WRITE(fence_reg + 4, val >> 32);
2847 		POSTING_READ(fence_reg + 4);
2848 
2849 		I915_WRITE(fence_reg + 0, val);
2850 		POSTING_READ(fence_reg);
2851 	} else {
2852 		I915_WRITE(fence_reg + 4, 0);
2853 		POSTING_READ(fence_reg + 4);
2854 	}
2855 }
2856 
2857 static void i915_write_fence_reg(struct drm_device *dev, int reg,
2858 				 struct drm_i915_gem_object *obj)
2859 {
2860 	struct drm_i915_private *dev_priv = dev->dev_private;
2861 	u32 val;
2862 
2863 	if (obj) {
2864 		u32 size = i915_gem_obj_ggtt_size(obj);
2865 		int pitch_val;
2866 		int tile_width;
2867 
2868 		WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) ||
2869 		     (size & -size) != size ||
2870 		     (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
2871 		     "object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2872 		     i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
2873 
2874 		if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2875 			tile_width = 128;
2876 		else
2877 			tile_width = 512;
2878 
2879 		/* Note: pitch better be a power of two tile widths */
2880 		pitch_val = obj->stride / tile_width;
2881 		pitch_val = ffs(pitch_val) - 1;
2882 
2883 		val = i915_gem_obj_ggtt_offset(obj);
2884 		if (obj->tiling_mode == I915_TILING_Y)
2885 			val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2886 		val |= I915_FENCE_SIZE_BITS(size);
2887 		val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2888 		val |= I830_FENCE_REG_VALID;
2889 	} else
2890 		val = 0;
2891 
2892 	if (reg < 8)
2893 		reg = FENCE_REG_830_0 + reg * 4;
2894 	else
2895 		reg = FENCE_REG_945_8 + (reg - 8) * 4;
2896 
2897 	I915_WRITE(reg, val);
2898 	POSTING_READ(reg);
2899 }
2900 
2901 static void i830_write_fence_reg(struct drm_device *dev, int reg,
2902 				struct drm_i915_gem_object *obj)
2903 {
2904 	struct drm_i915_private *dev_priv = dev->dev_private;
2905 	uint32_t val;
2906 
2907 	if (obj) {
2908 		u32 size = i915_gem_obj_ggtt_size(obj);
2909 		uint32_t pitch_val;
2910 
2911 		WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) ||
2912 		     (size & -size) != size ||
2913 		     (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
2914 		     "object 0x%08lx not 512K or pot-size 0x%08x aligned\n",
2915 		     i915_gem_obj_ggtt_offset(obj), size);
2916 
2917 		pitch_val = obj->stride / 128;
2918 		pitch_val = ffs(pitch_val) - 1;
2919 
2920 		val = i915_gem_obj_ggtt_offset(obj);
2921 		if (obj->tiling_mode == I915_TILING_Y)
2922 			val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2923 		val |= I830_FENCE_SIZE_BITS(size);
2924 		val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2925 		val |= I830_FENCE_REG_VALID;
2926 	} else
2927 		val = 0;
2928 
2929 	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
2930 	POSTING_READ(FENCE_REG_830_0 + reg * 4);
2931 }
2932 
2933 inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
2934 {
2935 	return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
2936 }
2937 
2938 static void i915_gem_write_fence(struct drm_device *dev, int reg,
2939 				 struct drm_i915_gem_object *obj)
2940 {
2941 	struct drm_i915_private *dev_priv = dev->dev_private;
2942 
2943 	/* Ensure that all CPU reads are completed before installing a fence
2944 	 * and all writes before removing the fence.
2945 	 */
2946 	if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
2947 		mb();
2948 
2949 	WARN(obj && (!obj->stride || !obj->tiling_mode),
2950 	     "bogus fence setup with stride: 0x%x, tiling mode: %i\n",
2951 	     obj->stride, obj->tiling_mode);
2952 
2953 	switch (INTEL_INFO(dev)->gen) {
2954 	case 8:
2955 	case 7:
2956 	case 6:
2957 	case 5:
2958 	case 4: i965_write_fence_reg(dev, reg, obj); break;
2959 	case 3: i915_write_fence_reg(dev, reg, obj); break;
2960 	case 2: i830_write_fence_reg(dev, reg, obj); break;
2961 	default: BUG();
2962 	}
2963 
2964 	/* And similarly be paranoid that no direct access to this region
2965 	 * is reordered to before the fence is installed.
2966 	 */
2967 	if (i915_gem_object_needs_mb(obj))
2968 		mb();
2969 }
2970 
2971 static inline int fence_number(struct drm_i915_private *dev_priv,
2972 			       struct drm_i915_fence_reg *fence)
2973 {
2974 	return fence - dev_priv->fence_regs;
2975 }
2976 
2977 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
2978 					 struct drm_i915_fence_reg *fence,
2979 					 bool enable)
2980 {
2981 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2982 	int reg = fence_number(dev_priv, fence);
2983 
2984 	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
2985 
2986 	if (enable) {
2987 		obj->fence_reg = reg;
2988 		fence->obj = obj;
2989 		list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
2990 	} else {
2991 		obj->fence_reg = I915_FENCE_REG_NONE;
2992 		fence->obj = NULL;
2993 		list_del_init(&fence->lru_list);
2994 	}
2995 	obj->fence_dirty = false;
2996 }
2997 
2998 static int
2999 i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
3000 {
3001 	if (obj->last_fenced_seqno) {
3002 		int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
3003 		if (ret)
3004 			return ret;
3005 
3006 		obj->last_fenced_seqno = 0;
3007 	}
3008 
3009 	obj->fenced_gpu_access = false;
3010 	return 0;
3011 }
3012 
3013 int
3014 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
3015 {
3016 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3017 	struct drm_i915_fence_reg *fence;
3018 	int ret;
3019 
3020 	ret = i915_gem_object_wait_fence(obj);
3021 	if (ret)
3022 		return ret;
3023 
3024 	if (obj->fence_reg == I915_FENCE_REG_NONE)
3025 		return 0;
3026 
3027 	fence = &dev_priv->fence_regs[obj->fence_reg];
3028 
3029 	i915_gem_object_fence_lost(obj);
3030 	i915_gem_object_update_fence(obj, fence, false);
3031 
3032 	return 0;
3033 }
3034 
3035 static struct drm_i915_fence_reg *
3036 i915_find_fence_reg(struct drm_device *dev)
3037 {
3038 	struct drm_i915_private *dev_priv = dev->dev_private;
3039 	struct drm_i915_fence_reg *reg, *avail;
3040 	int i;
3041 
3042 	/* First try to find a free reg */
3043 	avail = NULL;
3044 	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
3045 		reg = &dev_priv->fence_regs[i];
3046 		if (!reg->obj)
3047 			return reg;
3048 
3049 		if (!reg->pin_count)
3050 			avail = reg;
3051 	}
3052 
3053 	if (avail == NULL)
3054 		goto deadlock;
3055 
3056 	/* None available, try to steal one or wait for a user to finish */
3057 	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
3058 		if (reg->pin_count)
3059 			continue;
3060 
3061 		return reg;
3062 	}
3063 
3064 deadlock:
3065 	/* Wait for completion of pending flips which consume fences */
3066 	if (intel_has_pending_fb_unpin(dev))
3067 		return ERR_PTR(-EAGAIN);
3068 
3069 	return ERR_PTR(-EDEADLK);
3070 }
3071 
3072 /**
3073  * i915_gem_object_get_fence - set up fencing for an object
3074  * @obj: object to map through a fence reg
3075  *
3076  * When mapping objects through the GTT, userspace wants to be able to write
3077  * to them without having to worry about swizzling if the object is tiled.
3078  * This function walks the fence regs looking for a free one for @obj,
3079  * stealing one if it can't find any.
3080  *
3081  * It then sets up the reg based on the object's properties: address, pitch
3082  * and tiling format.
3083  *
3084  * For an untiled surface, this removes any existing fence.
3085  */
3086 int
3087 i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
3088 {
3089 	struct drm_device *dev = obj->base.dev;
3090 	struct drm_i915_private *dev_priv = dev->dev_private;
3091 	bool enable = obj->tiling_mode != I915_TILING_NONE;
3092 	struct drm_i915_fence_reg *reg;
3093 	int ret;
3094 
3095 	/* Have we updated the tiling parameters upon the object and so
3096 	 * will need to serialise the write to the associated fence register?
3097 	 */
3098 	if (obj->fence_dirty) {
3099 		ret = i915_gem_object_wait_fence(obj);
3100 		if (ret)
3101 			return ret;
3102 	}
3103 
3104 	/* Just update our place in the LRU if our fence is getting reused. */
3105 	if (obj->fence_reg != I915_FENCE_REG_NONE) {
3106 		reg = &dev_priv->fence_regs[obj->fence_reg];
3107 		if (!obj->fence_dirty) {
3108 			list_move_tail(&reg->lru_list,
3109 				       &dev_priv->mm.fence_list);
3110 			return 0;
3111 		}
3112 	} else if (enable) {
3113 		reg = i915_find_fence_reg(dev);
3114 		if (IS_ERR(reg))
3115 			return PTR_ERR(reg);
3116 
3117 		if (reg->obj) {
3118 			struct drm_i915_gem_object *old = reg->obj;
3119 
3120 			ret = i915_gem_object_wait_fence(old);
3121 			if (ret)
3122 				return ret;
3123 
3124 			i915_gem_object_fence_lost(old);
3125 		}
3126 	} else
3127 		return 0;
3128 
3129 	i915_gem_object_update_fence(obj, reg, enable);
3130 
3131 	return 0;
3132 }
3133 
3134 static bool i915_gem_valid_gtt_space(struct drm_device *dev,
3135 				     struct drm_mm_node *gtt_space,
3136 				     unsigned long cache_level)
3137 {
3138 	struct drm_mm_node *other;
3139 
3140 	/* On non-LLC machines we have to be careful when putting differing
3141 	 * types of snoopable memory together to avoid the prefetcher
3142 	 * crossing memory domains and dying.
3143 	 */
3144 	if (HAS_LLC(dev))
3145 		return true;
3146 
3147 	if (!drm_mm_node_allocated(gtt_space))
3148 		return true;
3149 
3150 	if (list_empty(&gtt_space->node_list))
3151 		return true;
3152 
3153 	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
3154 	if (other->allocated && !other->hole_follows && other->color != cache_level)
3155 		return false;
3156 
3157 	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
3158 	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
3159 		return false;
3160 
3161 	return true;
3162 }
3163 
3164 static void i915_gem_verify_gtt(struct drm_device *dev)
3165 {
3166 #if WATCH_GTT
3167 	struct drm_i915_private *dev_priv = dev->dev_private;
3168 	struct drm_i915_gem_object *obj;
3169 	int err = 0;
3170 
3171 	list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) {
3172 		if (obj->gtt_space == NULL) {
3173 			printk(KERN_ERR "object found on GTT list with no space reserved\n");
3174 			err++;
3175 			continue;
3176 		}
3177 
3178 		if (obj->cache_level != obj->gtt_space->color) {
3179 			printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
3180 			       i915_gem_obj_ggtt_offset(obj),
3181 			       i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3182 			       obj->cache_level,
3183 			       obj->gtt_space->color);
3184 			err++;
3185 			continue;
3186 		}
3187 
3188 		if (!i915_gem_valid_gtt_space(dev,
3189 					      obj->gtt_space,
3190 					      obj->cache_level)) {
3191 			printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
3192 			       i915_gem_obj_ggtt_offset(obj),
3193 			       i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3194 			       obj->cache_level);
3195 			err++;
3196 			continue;
3197 		}
3198 	}
3199 
3200 	WARN_ON(err);
3201 #endif
3202 }
3203 
3204 /**
3205  * Finds free space in the GTT aperture and binds the object there.
3206  */
3207 static struct i915_vma *
3208 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
3209 			   struct i915_address_space *vm,
3210 			   unsigned alignment,
3211 			   unsigned flags)
3212 {
3213 	struct drm_device *dev = obj->base.dev;
3214 	struct drm_i915_private *dev_priv = dev->dev_private;
3215 	u32 size, fence_size, fence_alignment, unfenced_alignment;
3216 	size_t gtt_max =
3217 		flags & PIN_MAPPABLE ? dev_priv->gtt.mappable_end : vm->total;
3218 	struct i915_vma *vma;
3219 	int ret;
3220 
3221 	fence_size = i915_gem_get_gtt_size(dev,
3222 					   obj->base.size,
3223 					   obj->tiling_mode);
3224 	fence_alignment = i915_gem_get_gtt_alignment(dev,
3225 						     obj->base.size,
3226 						     obj->tiling_mode, true);
3227 	unfenced_alignment =
3228 		i915_gem_get_gtt_alignment(dev,
3229 					   obj->base.size,
3230 					   obj->tiling_mode, false);
3231 
3232 	if (alignment == 0)
3233 		alignment = flags & PIN_MAPPABLE ? fence_alignment :
3234 						unfenced_alignment;
3235 	if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
3236 		DRM_DEBUG("Invalid object alignment requested %u\n", alignment);
3237 		return ERR_PTR(-EINVAL);
3238 	}
3239 
3240 	size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
3241 
3242 	/* If the object is bigger than the entire aperture, reject it early
3243 	 * before evicting everything in a vain attempt to find space.
3244 	 */
3245 	if (obj->base.size > gtt_max) {
3246 		DRM_DEBUG("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%zu\n",
3247 			  obj->base.size,
3248 			  flags & PIN_MAPPABLE ? "mappable" : "total",
3249 			  gtt_max);
3250 		return ERR_PTR(-E2BIG);
3251 	}
3252 
3253 	ret = i915_gem_object_get_pages(obj);
3254 	if (ret)
3255 		return ERR_PTR(ret);
3256 
3257 	i915_gem_object_pin_pages(obj);
3258 
3259 	vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
3260 	if (IS_ERR(vma))
3261 		goto err_unpin;
3262 
3263 search_free:
3264 	ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
3265 						  size, alignment,
3266 						  obj->cache_level, 0, gtt_max,
3267 						  DRM_MM_SEARCH_DEFAULT,
3268 						  DRM_MM_CREATE_DEFAULT);
3269 	if (ret) {
3270 		ret = i915_gem_evict_something(dev, vm, size, alignment,
3271 					       obj->cache_level, flags);
3272 		if (ret == 0)
3273 			goto search_free;
3274 
3275 		goto err_free_vma;
3276 	}
3277 	if (WARN_ON(!i915_gem_valid_gtt_space(dev, &vma->node,
3278 					      obj->cache_level))) {
3279 		ret = -EINVAL;
3280 		goto err_remove_node;
3281 	}
3282 
3283 	ret = i915_gem_gtt_prepare_object(obj);
3284 	if (ret)
3285 		goto err_remove_node;
3286 
3287 	list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
3288 	list_add_tail(&vma->mm_list, &vm->inactive_list);
3289 
3290 	if (i915_is_ggtt(vm)) {
3291 		bool mappable, fenceable;
3292 
3293 		fenceable = (vma->node.size == fence_size &&
3294 			     (vma->node.start & (fence_alignment - 1)) == 0);
3295 
3296 		mappable = (vma->node.start + obj->base.size <=
3297 			    dev_priv->gtt.mappable_end);
3298 
3299 		obj->map_and_fenceable = mappable && fenceable;
3300 	}
3301 
3302 	WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
3303 
3304 	trace_i915_vma_bind(vma, flags);
3305 	vma->bind_vma(vma, obj->cache_level,
3306 		      flags & (PIN_MAPPABLE | PIN_GLOBAL) ? GLOBAL_BIND : 0);
3307 
3308 	i915_gem_verify_gtt(dev);
3309 	return vma;
3310 
3311 err_remove_node:
3312 	drm_mm_remove_node(&vma->node);
3313 err_free_vma:
3314 	i915_gem_vma_destroy(vma);
3315 	vma = ERR_PTR(ret);
3316 err_unpin:
3317 	i915_gem_object_unpin_pages(obj);
3318 	return vma;
3319 }
3320 
3321 bool
3322 i915_gem_clflush_object(struct drm_i915_gem_object *obj,
3323 			bool force)
3324 {
3325 	/* If we don't have a page list set up, then we're not pinned
3326 	 * to GPU, and we can ignore the cache flush because it'll happen
3327 	 * again at bind time.
3328 	 */
3329 	if (obj->pages == NULL)
3330 		return false;
3331 
3332 	/*
3333 	 * Stolen memory is always coherent with the GPU as it is explicitly
3334 	 * marked as wc by the system, or the system is cache-coherent.
3335 	 */
3336 	if (obj->stolen)
3337 		return false;
3338 
3339 	/* If the GPU is snooping the contents of the CPU cache,
3340 	 * we do not need to manually clear the CPU cache lines.  However,
3341 	 * the caches are only snooped when the render cache is
3342 	 * flushed/invalidated.  As we always have to emit invalidations
3343 	 * and flushes when moving into and out of the RENDER domain, correct
3344 	 * snooping behaviour occurs naturally as the result of our domain
3345 	 * tracking.
3346 	 */
3347 	if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
3348 		return false;
3349 
3350 	trace_i915_gem_object_clflush(obj);
3351 	drm_clflush_sg(obj->pages);
3352 
3353 	return true;
3354 }
3355 
3356 /** Flushes the GTT write domain for the object if it's dirty. */
3357 static void
3358 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3359 {
3360 	uint32_t old_write_domain;
3361 
3362 	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3363 		return;
3364 
3365 	/* No actual flushing is required for the GTT write domain.  Writes
3366 	 * to it immediately go to main memory as far as we know, so there's
3367 	 * no chipset flush.  It also doesn't land in render cache.
3368 	 *
3369 	 * However, we do have to enforce the order so that all writes through
3370 	 * the GTT land before any writes to the device, such as updates to
3371 	 * the GATT itself.
3372 	 */
3373 	wmb();
3374 
3375 	old_write_domain = obj->base.write_domain;
3376 	obj->base.write_domain = 0;
3377 
3378 	trace_i915_gem_object_change_domain(obj,
3379 					    obj->base.read_domains,
3380 					    old_write_domain);
3381 }
3382 
3383 /** Flushes the CPU write domain for the object if it's dirty. */
3384 static void
3385 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
3386 				       bool force)
3387 {
3388 	uint32_t old_write_domain;
3389 
3390 	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3391 		return;
3392 
3393 	if (i915_gem_clflush_object(obj, force))
3394 		i915_gem_chipset_flush(obj->base.dev);
3395 
3396 	old_write_domain = obj->base.write_domain;
3397 	obj->base.write_domain = 0;
3398 
3399 	trace_i915_gem_object_change_domain(obj,
3400 					    obj->base.read_domains,
3401 					    old_write_domain);
3402 }
3403 
3404 /**
3405  * Moves a single object to the GTT read, and possibly write domain.
3406  *
3407  * This function returns when the move is complete, including waiting on
3408  * flushes to occur.
3409  */
3410 int
3411 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3412 {
3413 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3414 	uint32_t old_write_domain, old_read_domains;
3415 	int ret;
3416 
3417 	/* Not valid to be called on unbound objects. */
3418 	if (!i915_gem_obj_bound_any(obj))
3419 		return -EINVAL;
3420 
3421 	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3422 		return 0;
3423 
3424 	ret = i915_gem_object_wait_rendering(obj, !write);
3425 	if (ret)
3426 		return ret;
3427 
3428 	i915_gem_object_flush_cpu_write_domain(obj, false);
3429 
3430 	/* Serialise direct access to this object with the barriers for
3431 	 * coherent writes from the GPU, by effectively invalidating the
3432 	 * GTT domain upon first access.
3433 	 */
3434 	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3435 		mb();
3436 
3437 	old_write_domain = obj->base.write_domain;
3438 	old_read_domains = obj->base.read_domains;
3439 
3440 	/* It should now be out of any other write domains, and we can update
3441 	 * the domain values for our changes.
3442 	 */
3443 	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3444 	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3445 	if (write) {
3446 		obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3447 		obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3448 		obj->dirty = 1;
3449 	}
3450 
3451 	trace_i915_gem_object_change_domain(obj,
3452 					    old_read_domains,
3453 					    old_write_domain);
3454 
3455 	/* And bump the LRU for this access */
3456 	if (i915_gem_object_is_inactive(obj)) {
3457 		struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
3458 		if (vma)
3459 			list_move_tail(&vma->mm_list,
3460 				       &dev_priv->gtt.base.inactive_list);
3461 
3462 	}
3463 
3464 	return 0;
3465 }
3466 
3467 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3468 				    enum i915_cache_level cache_level)
3469 {
3470 	struct drm_device *dev = obj->base.dev;
3471 	struct i915_vma *vma, *next;
3472 	int ret;
3473 
3474 	if (obj->cache_level == cache_level)
3475 		return 0;
3476 
3477 	if (i915_gem_obj_is_pinned(obj)) {
3478 		DRM_DEBUG("can not change the cache level of pinned objects\n");
3479 		return -EBUSY;
3480 	}
3481 
3482 	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
3483 		if (!i915_gem_valid_gtt_space(dev, &vma->node, cache_level)) {
3484 			ret = i915_vma_unbind(vma);
3485 			if (ret)
3486 				return ret;
3487 		}
3488 	}
3489 
3490 	if (i915_gem_obj_bound_any(obj)) {
3491 		ret = i915_gem_object_finish_gpu(obj);
3492 		if (ret)
3493 			return ret;
3494 
3495 		i915_gem_object_finish_gtt(obj);
3496 
3497 		/* Before SandyBridge, you could not use tiling or fence
3498 		 * registers with snooped memory, so relinquish any fences
3499 		 * currently pointing to our region in the aperture.
3500 		 */
3501 		if (INTEL_INFO(dev)->gen < 6) {
3502 			ret = i915_gem_object_put_fence(obj);
3503 			if (ret)
3504 				return ret;
3505 		}
3506 
3507 		list_for_each_entry(vma, &obj->vma_list, vma_link)
3508 			if (drm_mm_node_allocated(&vma->node))
3509 				vma->bind_vma(vma, cache_level,
3510 					      obj->has_global_gtt_mapping ? GLOBAL_BIND : 0);
3511 	}
3512 
3513 	list_for_each_entry(vma, &obj->vma_list, vma_link)
3514 		vma->node.color = cache_level;
3515 	obj->cache_level = cache_level;
3516 
3517 	if (cpu_write_needs_clflush(obj)) {
3518 		u32 old_read_domains, old_write_domain;
3519 
3520 		/* If we're coming from LLC cached, then we haven't
3521 		 * actually been tracking whether the data is in the
3522 		 * CPU cache or not, since we only allow one bit set
3523 		 * in obj->write_domain and have been skipping the clflushes.
3524 		 * Just set it to the CPU cache for now.
3525 		 */
3526 		WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
3527 
3528 		old_read_domains = obj->base.read_domains;
3529 		old_write_domain = obj->base.write_domain;
3530 
3531 		obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3532 		obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3533 
3534 		trace_i915_gem_object_change_domain(obj,
3535 						    old_read_domains,
3536 						    old_write_domain);
3537 	}
3538 
3539 	i915_gem_verify_gtt(dev);
3540 	return 0;
3541 }
3542 
3543 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
3544 			       struct drm_file *file)
3545 {
3546 	struct drm_i915_gem_caching *args = data;
3547 	struct drm_i915_gem_object *obj;
3548 	int ret;
3549 
3550 	ret = i915_mutex_lock_interruptible(dev);
3551 	if (ret)
3552 		return ret;
3553 
3554 	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3555 	if (&obj->base == NULL) {
3556 		ret = -ENOENT;
3557 		goto unlock;
3558 	}
3559 
3560 	switch (obj->cache_level) {
3561 	case I915_CACHE_LLC:
3562 	case I915_CACHE_L3_LLC:
3563 		args->caching = I915_CACHING_CACHED;
3564 		break;
3565 
3566 	case I915_CACHE_WT:
3567 		args->caching = I915_CACHING_DISPLAY;
3568 		break;
3569 
3570 	default:
3571 		args->caching = I915_CACHING_NONE;
3572 		break;
3573 	}
3574 
3575 	drm_gem_object_unreference(&obj->base);
3576 unlock:
3577 	mutex_unlock(&dev->struct_mutex);
3578 	return ret;
3579 }
3580 
3581 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
3582 			       struct drm_file *file)
3583 {
3584 	struct drm_i915_gem_caching *args = data;
3585 	struct drm_i915_gem_object *obj;
3586 	enum i915_cache_level level;
3587 	int ret;
3588 
3589 	switch (args->caching) {
3590 	case I915_CACHING_NONE:
3591 		level = I915_CACHE_NONE;
3592 		break;
3593 	case I915_CACHING_CACHED:
3594 		level = I915_CACHE_LLC;
3595 		break;
3596 	case I915_CACHING_DISPLAY:
3597 		level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
3598 		break;
3599 	default:
3600 		return -EINVAL;
3601 	}
3602 
3603 	ret = i915_mutex_lock_interruptible(dev);
3604 	if (ret)
3605 		return ret;
3606 
3607 	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3608 	if (&obj->base == NULL) {
3609 		ret = -ENOENT;
3610 		goto unlock;
3611 	}
3612 
3613 	ret = i915_gem_object_set_cache_level(obj, level);
3614 
3615 	drm_gem_object_unreference(&obj->base);
3616 unlock:
3617 	mutex_unlock(&dev->struct_mutex);
3618 	return ret;
3619 }
3620 
3621 static bool is_pin_display(struct drm_i915_gem_object *obj)
3622 {
3623 	/* There are 3 sources that pin objects:
3624 	 *   1. The display engine (scanouts, sprites, cursors);
3625 	 *   2. Reservations for execbuffer;
3626 	 *   3. The user.
3627 	 *
3628 	 * We can ignore reservations as we hold the struct_mutex and
3629 	 * are only called outside of the reservation path.  The user
3630 	 * can only increment pin_count once, and so if after
3631 	 * subtracting the potential reference by the user, any pin_count
3632 	 * remains, it must be due to another use by the display engine.
3633 	 */
3634 	return i915_gem_obj_to_ggtt(obj)->pin_count - !!obj->user_pin_count;
3635 }
3636 
3637 /*
3638  * Prepare buffer for display plane (scanout, cursors, etc).
3639  * Can be called from an uninterruptible phase (modesetting) and allows
3640  * any flushes to be pipelined (for pageflips).
3641  */
3642 int
3643 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3644 				     u32 alignment,
3645 				     struct intel_ring_buffer *pipelined)
3646 {
3647 	u32 old_read_domains, old_write_domain;
3648 	int ret;
3649 
3650 	if (pipelined != obj->ring) {
3651 		ret = i915_gem_object_sync(obj, pipelined);
3652 		if (ret)
3653 			return ret;
3654 	}
3655 
3656 	/* Mark the pin_display early so that we account for the
3657 	 * display coherency whilst setting up the cache domains.
3658 	 */
3659 	obj->pin_display = true;
3660 
3661 	/* The display engine is not coherent with the LLC cache on gen6.  As
3662 	 * a result, we make sure that the pinning that is about to occur is
3663 	 * done with uncached PTEs. This is lowest common denominator for all
3664 	 * chipsets.
3665 	 *
3666 	 * However for gen6+, we could do better by using the GFDT bit instead
3667 	 * of uncaching, which would allow us to flush all the LLC-cached data
3668 	 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
3669 	 */
3670 	ret = i915_gem_object_set_cache_level(obj,
3671 					      HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
3672 	if (ret)
3673 		goto err_unpin_display;
3674 
3675 	/* As the user may map the buffer once pinned in the display plane
3676 	 * (e.g. libkms for the bootup splash), we have to ensure that we
3677 	 * always use map_and_fenceable for all scanout buffers.
3678 	 */
3679 	ret = i915_gem_obj_ggtt_pin(obj, alignment, PIN_MAPPABLE);
3680 	if (ret)
3681 		goto err_unpin_display;
3682 
3683 	i915_gem_object_flush_cpu_write_domain(obj, true);
3684 
3685 	old_write_domain = obj->base.write_domain;
3686 	old_read_domains = obj->base.read_domains;
3687 
3688 	/* It should now be out of any other write domains, and we can update
3689 	 * the domain values for our changes.
3690 	 */
3691 	obj->base.write_domain = 0;
3692 	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3693 
3694 	trace_i915_gem_object_change_domain(obj,
3695 					    old_read_domains,
3696 					    old_write_domain);
3697 
3698 	return 0;
3699 
3700 err_unpin_display:
3701 	obj->pin_display = is_pin_display(obj);
3702 	return ret;
3703 }
3704 
3705 void
3706 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj)
3707 {
3708 	i915_gem_object_ggtt_unpin(obj);
3709 	obj->pin_display = is_pin_display(obj);
3710 }
3711 
3712 int
3713 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
3714 {
3715 	int ret;
3716 
3717 	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
3718 		return 0;
3719 
3720 	ret = i915_gem_object_wait_rendering(obj, false);
3721 	if (ret)
3722 		return ret;
3723 
3724 	/* Ensure that we invalidate the GPU's caches and TLBs. */
3725 	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3726 	return 0;
3727 }
3728 
3729 /**
3730  * Moves a single object to the CPU read, and possibly write domain.
3731  *
3732  * This function returns when the move is complete, including waiting on
3733  * flushes to occur.
3734  */
3735 int
3736 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3737 {
3738 	uint32_t old_write_domain, old_read_domains;
3739 	int ret;
3740 
3741 	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
3742 		return 0;
3743 
3744 	ret = i915_gem_object_wait_rendering(obj, !write);
3745 	if (ret)
3746 		return ret;
3747 
3748 	i915_gem_object_flush_gtt_write_domain(obj);
3749 
3750 	old_write_domain = obj->base.write_domain;
3751 	old_read_domains = obj->base.read_domains;
3752 
3753 	/* Flush the CPU cache if it's still invalid. */
3754 	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3755 		i915_gem_clflush_object(obj, false);
3756 
3757 		obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3758 	}
3759 
3760 	/* It should now be out of any other write domains, and we can update
3761 	 * the domain values for our changes.
3762 	 */
3763 	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3764 
3765 	/* If we're writing through the CPU, then the GPU read domains will
3766 	 * need to be invalidated at next use.
3767 	 */
3768 	if (write) {
3769 		obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3770 		obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3771 	}
3772 
3773 	trace_i915_gem_object_change_domain(obj,
3774 					    old_read_domains,
3775 					    old_write_domain);
3776 
3777 	return 0;
3778 }
3779 
3780 /* Throttle our rendering by waiting until the ring has completed our requests
3781  * emitted over 20 msec ago.
3782  *
3783  * Note that if we were to use the current jiffies each time around the loop,
3784  * we wouldn't escape the function with any frames outstanding if the time to
3785  * render a frame was over 20ms.
3786  *
3787  * This should get us reasonable parallelism between CPU and GPU but also
3788  * relatively low latency when blocking on a particular request to finish.
3789  */
3790 static int
3791 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3792 {
3793 	struct drm_i915_private *dev_priv = dev->dev_private;
3794 	struct drm_i915_file_private *file_priv = file->driver_priv;
3795 	unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3796 	struct drm_i915_gem_request *request;
3797 	struct intel_ring_buffer *ring = NULL;
3798 	unsigned reset_counter;
3799 	u32 seqno = 0;
3800 	int ret;
3801 
3802 	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
3803 	if (ret)
3804 		return ret;
3805 
3806 	ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
3807 	if (ret)
3808 		return ret;
3809 
3810 	spin_lock(&file_priv->mm.lock);
3811 	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3812 		if (time_after_eq(request->emitted_jiffies, recent_enough))
3813 			break;
3814 
3815 		ring = request->ring;
3816 		seqno = request->seqno;
3817 	}
3818 	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3819 	spin_unlock(&file_priv->mm.lock);
3820 
3821 	if (seqno == 0)
3822 		return 0;
3823 
3824 	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, NULL);
3825 	if (ret == 0)
3826 		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3827 
3828 	return ret;
3829 }
3830 
3831 int
3832 i915_gem_object_pin(struct drm_i915_gem_object *obj,
3833 		    struct i915_address_space *vm,
3834 		    uint32_t alignment,
3835 		    unsigned flags)
3836 {
3837 	struct i915_vma *vma;
3838 	int ret;
3839 
3840 	if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
3841 		return -EINVAL;
3842 
3843 	vma = i915_gem_obj_to_vma(obj, vm);
3844 	if (vma) {
3845 		if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
3846 			return -EBUSY;
3847 
3848 		if ((alignment &&
3849 		     vma->node.start & (alignment - 1)) ||
3850 		    (flags & PIN_MAPPABLE && !obj->map_and_fenceable)) {
3851 			WARN(vma->pin_count,
3852 			     "bo is already pinned with incorrect alignment:"
3853 			     " offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
3854 			     " obj->map_and_fenceable=%d\n",
3855 			     i915_gem_obj_offset(obj, vm), alignment,
3856 			     flags & PIN_MAPPABLE,
3857 			     obj->map_and_fenceable);
3858 			ret = i915_vma_unbind(vma);
3859 			if (ret)
3860 				return ret;
3861 
3862 			vma = NULL;
3863 		}
3864 	}
3865 
3866 	if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
3867 		vma = i915_gem_object_bind_to_vm(obj, vm, alignment, flags);
3868 		if (IS_ERR(vma))
3869 			return PTR_ERR(vma);
3870 	}
3871 
3872 	if (flags & PIN_GLOBAL && !obj->has_global_gtt_mapping)
3873 		vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND);
3874 
3875 	vma->pin_count++;
3876 	if (flags & PIN_MAPPABLE)
3877 		obj->pin_mappable |= true;
3878 
3879 	return 0;
3880 }
3881 
3882 void
3883 i915_gem_object_ggtt_unpin(struct drm_i915_gem_object *obj)
3884 {
3885 	struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
3886 
3887 	BUG_ON(!vma);
3888 	BUG_ON(vma->pin_count == 0);
3889 	BUG_ON(!i915_gem_obj_ggtt_bound(obj));
3890 
3891 	if (--vma->pin_count == 0)
3892 		obj->pin_mappable = false;
3893 }
3894 
3895 int
3896 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
3897 		   struct drm_file *file)
3898 {
3899 	struct drm_i915_gem_pin *args = data;
3900 	struct drm_i915_gem_object *obj;
3901 	int ret;
3902 
3903 	if (INTEL_INFO(dev)->gen >= 6)
3904 		return -ENODEV;
3905 
3906 	ret = i915_mutex_lock_interruptible(dev);
3907 	if (ret)
3908 		return ret;
3909 
3910 	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3911 	if (&obj->base == NULL) {
3912 		ret = -ENOENT;
3913 		goto unlock;
3914 	}
3915 
3916 	if (obj->madv != I915_MADV_WILLNEED) {
3917 		DRM_DEBUG("Attempting to pin a purgeable buffer\n");
3918 		ret = -EFAULT;
3919 		goto out;
3920 	}
3921 
3922 	if (obj->pin_filp != NULL && obj->pin_filp != file) {
3923 		DRM_DEBUG("Already pinned in i915_gem_pin_ioctl(): %d\n",
3924 			  args->handle);
3925 		ret = -EINVAL;
3926 		goto out;
3927 	}
3928 
3929 	if (obj->user_pin_count == ULONG_MAX) {
3930 		ret = -EBUSY;
3931 		goto out;
3932 	}
3933 
3934 	if (obj->user_pin_count == 0) {
3935 		ret = i915_gem_obj_ggtt_pin(obj, args->alignment, PIN_MAPPABLE);
3936 		if (ret)
3937 			goto out;
3938 	}
3939 
3940 	obj->user_pin_count++;
3941 	obj->pin_filp = file;
3942 
3943 	args->offset = i915_gem_obj_ggtt_offset(obj);
3944 out:
3945 	drm_gem_object_unreference(&obj->base);
3946 unlock:
3947 	mutex_unlock(&dev->struct_mutex);
3948 	return ret;
3949 }
3950 
3951 int
3952 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
3953 		     struct drm_file *file)
3954 {
3955 	struct drm_i915_gem_pin *args = data;
3956 	struct drm_i915_gem_object *obj;
3957 	int ret;
3958 
3959 	ret = i915_mutex_lock_interruptible(dev);
3960 	if (ret)
3961 		return ret;
3962 
3963 	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3964 	if (&obj->base == NULL) {
3965 		ret = -ENOENT;
3966 		goto unlock;
3967 	}
3968 
3969 	if (obj->pin_filp != file) {
3970 		DRM_DEBUG("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3971 			  args->handle);
3972 		ret = -EINVAL;
3973 		goto out;
3974 	}
3975 	obj->user_pin_count--;
3976 	if (obj->user_pin_count == 0) {
3977 		obj->pin_filp = NULL;
3978 		i915_gem_object_ggtt_unpin(obj);
3979 	}
3980 
3981 out:
3982 	drm_gem_object_unreference(&obj->base);
3983 unlock:
3984 	mutex_unlock(&dev->struct_mutex);
3985 	return ret;
3986 }
3987 
3988 int
3989 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3990 		    struct drm_file *file)
3991 {
3992 	struct drm_i915_gem_busy *args = data;
3993 	struct drm_i915_gem_object *obj;
3994 	int ret;
3995 
3996 	ret = i915_mutex_lock_interruptible(dev);
3997 	if (ret)
3998 		return ret;
3999 
4000 	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4001 	if (&obj->base == NULL) {
4002 		ret = -ENOENT;
4003 		goto unlock;
4004 	}
4005 
4006 	/* Count all active objects as busy, even if they are currently not used
4007 	 * by the gpu. Users of this interface expect objects to eventually
4008 	 * become non-busy without any further actions, therefore emit any
4009 	 * necessary flushes here.
4010 	 */
4011 	ret = i915_gem_object_flush_active(obj);
4012 
4013 	args->busy = obj->active;
4014 	if (obj->ring) {
4015 		BUILD_BUG_ON(I915_NUM_RINGS > 16);
4016 		args->busy |= intel_ring_flag(obj->ring) << 16;
4017 	}
4018 
4019 	drm_gem_object_unreference(&obj->base);
4020 unlock:
4021 	mutex_unlock(&dev->struct_mutex);
4022 	return ret;
4023 }
4024 
4025 int
4026 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4027 			struct drm_file *file_priv)
4028 {
4029 	return i915_gem_ring_throttle(dev, file_priv);
4030 }
4031 
4032 int
4033 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4034 		       struct drm_file *file_priv)
4035 {
4036 	struct drm_i915_gem_madvise *args = data;
4037 	struct drm_i915_gem_object *obj;
4038 	int ret;
4039 
4040 	switch (args->madv) {
4041 	case I915_MADV_DONTNEED:
4042 	case I915_MADV_WILLNEED:
4043 	    break;
4044 	default:
4045 	    return -EINVAL;
4046 	}
4047 
4048 	ret = i915_mutex_lock_interruptible(dev);
4049 	if (ret)
4050 		return ret;
4051 
4052 	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
4053 	if (&obj->base == NULL) {
4054 		ret = -ENOENT;
4055 		goto unlock;
4056 	}
4057 
4058 	if (i915_gem_obj_is_pinned(obj)) {
4059 		ret = -EINVAL;
4060 		goto out;
4061 	}
4062 
4063 	if (obj->madv != __I915_MADV_PURGED)
4064 		obj->madv = args->madv;
4065 
4066 	/* if the object is no longer attached, discard its backing storage */
4067 	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
4068 		i915_gem_object_truncate(obj);
4069 
4070 	args->retained = obj->madv != __I915_MADV_PURGED;
4071 
4072 out:
4073 	drm_gem_object_unreference(&obj->base);
4074 unlock:
4075 	mutex_unlock(&dev->struct_mutex);
4076 	return ret;
4077 }
4078 
4079 void i915_gem_object_init(struct drm_i915_gem_object *obj,
4080 			  const struct drm_i915_gem_object_ops *ops)
4081 {
4082 	INIT_LIST_HEAD(&obj->global_list);
4083 	INIT_LIST_HEAD(&obj->ring_list);
4084 	INIT_LIST_HEAD(&obj->obj_exec_link);
4085 	INIT_LIST_HEAD(&obj->vma_list);
4086 
4087 	obj->ops = ops;
4088 
4089 	obj->fence_reg = I915_FENCE_REG_NONE;
4090 	obj->madv = I915_MADV_WILLNEED;
4091 	/* Avoid an unnecessary call to unbind on the first bind. */
4092 	obj->map_and_fenceable = true;
4093 
4094 	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
4095 }
4096 
4097 static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
4098 	.get_pages = i915_gem_object_get_pages_gtt,
4099 	.put_pages = i915_gem_object_put_pages_gtt,
4100 };
4101 
4102 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
4103 						  size_t size)
4104 {
4105 	struct drm_i915_gem_object *obj;
4106 	struct address_space *mapping;
4107 	gfp_t mask;
4108 
4109 	obj = i915_gem_object_alloc(dev);
4110 	if (obj == NULL)
4111 		return NULL;
4112 
4113 	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4114 		i915_gem_object_free(obj);
4115 		return NULL;
4116 	}
4117 
4118 	mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
4119 	if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
4120 		/* 965gm cannot relocate objects above 4GiB. */
4121 		mask &= ~__GFP_HIGHMEM;
4122 		mask |= __GFP_DMA32;
4123 	}
4124 
4125 	mapping = file_inode(obj->base.filp)->i_mapping;
4126 	mapping_set_gfp_mask(mapping, mask);
4127 
4128 	i915_gem_object_init(obj, &i915_gem_object_ops);
4129 
4130 	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4131 	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4132 
4133 	if (HAS_LLC(dev)) {
4134 		/* On some devices, we can have the GPU use the LLC (the CPU
4135 		 * cache) for about a 10% performance improvement
4136 		 * compared to uncached.  Graphics requests other than
4137 		 * display scanout are coherent with the CPU in
4138 		 * accessing this cache.  This means in this mode we
4139 		 * don't need to clflush on the CPU side, and on the
4140 		 * GPU side we only need to flush internal caches to
4141 		 * get data visible to the CPU.
4142 		 *
4143 		 * However, we maintain the display planes as UC, and so
4144 		 * need to rebind when first used as such.
4145 		 */
4146 		obj->cache_level = I915_CACHE_LLC;
4147 	} else
4148 		obj->cache_level = I915_CACHE_NONE;
4149 
4150 	trace_i915_gem_object_create(obj);
4151 
4152 	return obj;
4153 }
4154 
4155 void i915_gem_free_object(struct drm_gem_object *gem_obj)
4156 {
4157 	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4158 	struct drm_device *dev = obj->base.dev;
4159 	struct drm_i915_private *dev_priv = dev->dev_private;
4160 	struct i915_vma *vma, *next;
4161 
4162 	intel_runtime_pm_get(dev_priv);
4163 
4164 	trace_i915_gem_object_destroy(obj);
4165 
4166 	if (obj->phys_obj)
4167 		i915_gem_detach_phys_object(dev, obj);
4168 
4169 	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
4170 		int ret;
4171 
4172 		vma->pin_count = 0;
4173 		ret = i915_vma_unbind(vma);
4174 		if (WARN_ON(ret == -ERESTARTSYS)) {
4175 			bool was_interruptible;
4176 
4177 			was_interruptible = dev_priv->mm.interruptible;
4178 			dev_priv->mm.interruptible = false;
4179 
4180 			WARN_ON(i915_vma_unbind(vma));
4181 
4182 			dev_priv->mm.interruptible = was_interruptible;
4183 		}
4184 	}
4185 
4186 	/* Stolen objects don't hold a ref, but do hold pin count. Fix that up
4187 	 * before progressing. */
4188 	if (obj->stolen)
4189 		i915_gem_object_unpin_pages(obj);
4190 
4191 	if (WARN_ON(obj->pages_pin_count))
4192 		obj->pages_pin_count = 0;
4193 	i915_gem_object_put_pages(obj);
4194 	i915_gem_object_free_mmap_offset(obj);
4195 	i915_gem_object_release_stolen(obj);
4196 
4197 	BUG_ON(obj->pages);
4198 
4199 	if (obj->base.import_attach)
4200 		drm_prime_gem_destroy(&obj->base, NULL);
4201 
4202 	drm_gem_object_release(&obj->base);
4203 	i915_gem_info_remove_obj(dev_priv, obj->base.size);
4204 
4205 	kfree(obj->bit_17);
4206 	i915_gem_object_free(obj);
4207 
4208 	intel_runtime_pm_put(dev_priv);
4209 }
4210 
4211 struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
4212 				     struct i915_address_space *vm)
4213 {
4214 	struct i915_vma *vma;
4215 	list_for_each_entry(vma, &obj->vma_list, vma_link)
4216 		if (vma->vm == vm)
4217 			return vma;
4218 
4219 	return NULL;
4220 }
4221 
4222 void i915_gem_vma_destroy(struct i915_vma *vma)
4223 {
4224 	WARN_ON(vma->node.allocated);
4225 
4226 	/* Keep the vma as a placeholder in the execbuffer reservation lists */
4227 	if (!list_empty(&vma->exec_list))
4228 		return;
4229 
4230 	list_del(&vma->vma_link);
4231 
4232 	kfree(vma);
4233 }
4234 
4235 int
4236 i915_gem_suspend(struct drm_device *dev)
4237 {
4238 	struct drm_i915_private *dev_priv = dev->dev_private;
4239 	int ret = 0;
4240 
4241 	mutex_lock(&dev->struct_mutex);
4242 	if (dev_priv->ums.mm_suspended)
4243 		goto err;
4244 
4245 	ret = i915_gpu_idle(dev);
4246 	if (ret)
4247 		goto err;
4248 
4249 	i915_gem_retire_requests(dev);
4250 
4251 	/* Under UMS, be paranoid and evict. */
4252 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
4253 		i915_gem_evict_everything(dev);
4254 
4255 	i915_kernel_lost_context(dev);
4256 	i915_gem_cleanup_ringbuffer(dev);
4257 
4258 	/* Hack!  Don't let anybody do execbuf while we don't control the chip.
4259 	 * We need to replace this with a semaphore, or something.
4260 	 * And not confound ums.mm_suspended!
4261 	 */
4262 	dev_priv->ums.mm_suspended = !drm_core_check_feature(dev,
4263 							     DRIVER_MODESET);
4264 	mutex_unlock(&dev->struct_mutex);
4265 
4266 	del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
4267 	cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4268 	cancel_delayed_work_sync(&dev_priv->mm.idle_work);
4269 
4270 	return 0;
4271 
4272 err:
4273 	mutex_unlock(&dev->struct_mutex);
4274 	return ret;
4275 }
4276 
4277 int i915_gem_l3_remap(struct intel_ring_buffer *ring, int slice)
4278 {
4279 	struct drm_device *dev = ring->dev;
4280 	struct drm_i915_private *dev_priv = dev->dev_private;
4281 	u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200);
4282 	u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
4283 	int i, ret;
4284 
4285 	if (!HAS_L3_DPF(dev) || !remap_info)
4286 		return 0;
4287 
4288 	ret = intel_ring_begin(ring, GEN7_L3LOG_SIZE / 4 * 3);
4289 	if (ret)
4290 		return ret;
4291 
4292 	/*
4293 	 * Note: We do not worry about the concurrent register cacheline hang
4294 	 * here because no other code should access these registers other than
4295 	 * at initialization time.
4296 	 */
4297 	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
4298 		intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
4299 		intel_ring_emit(ring, reg_base + i);
4300 		intel_ring_emit(ring, remap_info[i/4]);
4301 	}
4302 
4303 	intel_ring_advance(ring);
4304 
4305 	return ret;
4306 }
4307 
4308 void i915_gem_init_swizzling(struct drm_device *dev)
4309 {
4310 	struct drm_i915_private *dev_priv = dev->dev_private;
4311 
4312 	if (INTEL_INFO(dev)->gen < 5 ||
4313 	    dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
4314 		return;
4315 
4316 	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
4317 				 DISP_TILE_SURFACE_SWIZZLING);
4318 
4319 	if (IS_GEN5(dev))
4320 		return;
4321 
4322 	I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
4323 	if (IS_GEN6(dev))
4324 		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4325 	else if (IS_GEN7(dev))
4326 		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4327 	else if (IS_GEN8(dev))
4328 		I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
4329 	else
4330 		BUG();
4331 }
4332 
4333 static bool
4334 intel_enable_blt(struct drm_device *dev)
4335 {
4336 	if (!HAS_BLT(dev))
4337 		return false;
4338 
4339 	/* The blitter was dysfunctional on early prototypes */
4340 	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
4341 		DRM_INFO("BLT not supported on this pre-production hardware;"
4342 			 " graphics performance will be degraded.\n");
4343 		return false;
4344 	}
4345 
4346 	return true;
4347 }
4348 
4349 static int i915_gem_init_rings(struct drm_device *dev)
4350 {
4351 	struct drm_i915_private *dev_priv = dev->dev_private;
4352 	int ret;
4353 
4354 	ret = intel_init_render_ring_buffer(dev);
4355 	if (ret)
4356 		return ret;
4357 
4358 	if (HAS_BSD(dev)) {
4359 		ret = intel_init_bsd_ring_buffer(dev);
4360 		if (ret)
4361 			goto cleanup_render_ring;
4362 	}
4363 
4364 	if (intel_enable_blt(dev)) {
4365 		ret = intel_init_blt_ring_buffer(dev);
4366 		if (ret)
4367 			goto cleanup_bsd_ring;
4368 	}
4369 
4370 	if (HAS_VEBOX(dev)) {
4371 		ret = intel_init_vebox_ring_buffer(dev);
4372 		if (ret)
4373 			goto cleanup_blt_ring;
4374 	}
4375 
4376 
4377 	ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
4378 	if (ret)
4379 		goto cleanup_vebox_ring;
4380 
4381 	return 0;
4382 
4383 cleanup_vebox_ring:
4384 	intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
4385 cleanup_blt_ring:
4386 	intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
4387 cleanup_bsd_ring:
4388 	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
4389 cleanup_render_ring:
4390 	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
4391 
4392 	return ret;
4393 }
4394 
4395 int
4396 i915_gem_init_hw(struct drm_device *dev)
4397 {
4398 	struct drm_i915_private *dev_priv = dev->dev_private;
4399 	int ret, i;
4400 
4401 	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
4402 		return -EIO;
4403 
4404 	if (dev_priv->ellc_size)
4405 		I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
4406 
4407 	if (IS_HASWELL(dev))
4408 		I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
4409 			   LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
4410 
4411 	if (HAS_PCH_NOP(dev)) {
4412 		if (IS_IVYBRIDGE(dev)) {
4413 			u32 temp = I915_READ(GEN7_MSG_CTL);
4414 			temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
4415 			I915_WRITE(GEN7_MSG_CTL, temp);
4416 		} else if (INTEL_INFO(dev)->gen >= 7) {
4417 			u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
4418 			temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
4419 			I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
4420 		}
4421 	}
4422 
4423 	i915_gem_init_swizzling(dev);
4424 
4425 	ret = i915_gem_init_rings(dev);
4426 	if (ret)
4427 		return ret;
4428 
4429 	for (i = 0; i < NUM_L3_SLICES(dev); i++)
4430 		i915_gem_l3_remap(&dev_priv->ring[RCS], i);
4431 
4432 	/*
4433 	 * XXX: Contexts should only be initialized once. Doing a switch to the
4434 	 * default context switch however is something we'd like to do after
4435 	 * reset or thaw (the latter may not actually be necessary for HW, but
4436 	 * goes with our code better). Context switching requires rings (for
4437 	 * the do_switch), but before enabling PPGTT. So don't move this.
4438 	 */
4439 	ret = i915_gem_context_enable(dev_priv);
4440 	if (ret) {
4441 		DRM_ERROR("Context enable failed %d\n", ret);
4442 		goto err_out;
4443 	}
4444 
4445 	return 0;
4446 
4447 err_out:
4448 	i915_gem_cleanup_ringbuffer(dev);
4449 	return ret;
4450 }
4451 
4452 int i915_gem_init(struct drm_device *dev)
4453 {
4454 	struct drm_i915_private *dev_priv = dev->dev_private;
4455 	int ret;
4456 
4457 	mutex_lock(&dev->struct_mutex);
4458 
4459 	if (IS_VALLEYVIEW(dev)) {
4460 		/* VLVA0 (potential hack), BIOS isn't actually waking us */
4461 		I915_WRITE(VLV_GTLC_WAKE_CTRL, 1);
4462 		if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) & 1) == 1, 10))
4463 			DRM_DEBUG_DRIVER("allow wake ack timed out\n");
4464 	}
4465 
4466 	i915_gem_init_global_gtt(dev);
4467 
4468 	ret = i915_gem_context_init(dev);
4469 	if (ret) {
4470 		mutex_unlock(&dev->struct_mutex);
4471 		return ret;
4472 	}
4473 
4474 	ret = i915_gem_init_hw(dev);
4475 	mutex_unlock(&dev->struct_mutex);
4476 	if (ret) {
4477 		WARN_ON(dev_priv->mm.aliasing_ppgtt);
4478 		i915_gem_context_fini(dev);
4479 		drm_mm_takedown(&dev_priv->gtt.base.mm);
4480 		return ret;
4481 	}
4482 
4483 	/* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
4484 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
4485 		dev_priv->dri1.allow_batchbuffer = 1;
4486 	return 0;
4487 }
4488 
4489 void
4490 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4491 {
4492 	struct drm_i915_private *dev_priv = dev->dev_private;
4493 	struct intel_ring_buffer *ring;
4494 	int i;
4495 
4496 	for_each_ring(ring, dev_priv, i)
4497 		intel_cleanup_ring_buffer(ring);
4498 }
4499 
4500 int
4501 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4502 		       struct drm_file *file_priv)
4503 {
4504 	struct drm_i915_private *dev_priv = dev->dev_private;
4505 	int ret;
4506 
4507 	if (drm_core_check_feature(dev, DRIVER_MODESET))
4508 		return 0;
4509 
4510 	if (i915_reset_in_progress(&dev_priv->gpu_error)) {
4511 		DRM_ERROR("Reenabling wedged hardware, good luck\n");
4512 		atomic_set(&dev_priv->gpu_error.reset_counter, 0);
4513 	}
4514 
4515 	mutex_lock(&dev->struct_mutex);
4516 	dev_priv->ums.mm_suspended = 0;
4517 
4518 	ret = i915_gem_init_hw(dev);
4519 	if (ret != 0) {
4520 		mutex_unlock(&dev->struct_mutex);
4521 		return ret;
4522 	}
4523 
4524 	BUG_ON(!list_empty(&dev_priv->gtt.base.active_list));
4525 	mutex_unlock(&dev->struct_mutex);
4526 
4527 	ret = drm_irq_install(dev);
4528 	if (ret)
4529 		goto cleanup_ringbuffer;
4530 
4531 	return 0;
4532 
4533 cleanup_ringbuffer:
4534 	mutex_lock(&dev->struct_mutex);
4535 	i915_gem_cleanup_ringbuffer(dev);
4536 	dev_priv->ums.mm_suspended = 1;
4537 	mutex_unlock(&dev->struct_mutex);
4538 
4539 	return ret;
4540 }
4541 
4542 int
4543 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4544 		       struct drm_file *file_priv)
4545 {
4546 	if (drm_core_check_feature(dev, DRIVER_MODESET))
4547 		return 0;
4548 
4549 	drm_irq_uninstall(dev);
4550 
4551 	return i915_gem_suspend(dev);
4552 }
4553 
4554 void
4555 i915_gem_lastclose(struct drm_device *dev)
4556 {
4557 	int ret;
4558 
4559 	if (drm_core_check_feature(dev, DRIVER_MODESET))
4560 		return;
4561 
4562 	ret = i915_gem_suspend(dev);
4563 	if (ret)
4564 		DRM_ERROR("failed to idle hardware: %d\n", ret);
4565 }
4566 
4567 static void
4568 init_ring_lists(struct intel_ring_buffer *ring)
4569 {
4570 	INIT_LIST_HEAD(&ring->active_list);
4571 	INIT_LIST_HEAD(&ring->request_list);
4572 }
4573 
4574 void i915_init_vm(struct drm_i915_private *dev_priv,
4575 		  struct i915_address_space *vm)
4576 {
4577 	if (!i915_is_ggtt(vm))
4578 		drm_mm_init(&vm->mm, vm->start, vm->total);
4579 	vm->dev = dev_priv->dev;
4580 	INIT_LIST_HEAD(&vm->active_list);
4581 	INIT_LIST_HEAD(&vm->inactive_list);
4582 	INIT_LIST_HEAD(&vm->global_link);
4583 	list_add_tail(&vm->global_link, &dev_priv->vm_list);
4584 }
4585 
4586 void
4587 i915_gem_load(struct drm_device *dev)
4588 {
4589 	struct drm_i915_private *dev_priv = dev->dev_private;
4590 	int i;
4591 
4592 	dev_priv->slab =
4593 		kmem_cache_create("i915_gem_object",
4594 				  sizeof(struct drm_i915_gem_object), 0,
4595 				  SLAB_HWCACHE_ALIGN,
4596 				  NULL);
4597 
4598 	INIT_LIST_HEAD(&dev_priv->vm_list);
4599 	i915_init_vm(dev_priv, &dev_priv->gtt.base);
4600 
4601 	INIT_LIST_HEAD(&dev_priv->context_list);
4602 	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
4603 	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
4604 	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4605 	for (i = 0; i < I915_NUM_RINGS; i++)
4606 		init_ring_lists(&dev_priv->ring[i]);
4607 	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
4608 		INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4609 	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4610 			  i915_gem_retire_work_handler);
4611 	INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
4612 			  i915_gem_idle_work_handler);
4613 	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
4614 
4615 	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4616 	if (IS_GEN3(dev)) {
4617 		I915_WRITE(MI_ARB_STATE,
4618 			   _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
4619 	}
4620 
4621 	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
4622 
4623 	/* Old X drivers will take 0-2 for front, back, depth buffers */
4624 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
4625 		dev_priv->fence_reg_start = 3;
4626 
4627 	if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
4628 		dev_priv->num_fence_regs = 32;
4629 	else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4630 		dev_priv->num_fence_regs = 16;
4631 	else
4632 		dev_priv->num_fence_regs = 8;
4633 
4634 	/* Initialize fence registers to zero */
4635 	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4636 	i915_gem_restore_fences(dev);
4637 
4638 	i915_gem_detect_bit_6_swizzle(dev);
4639 	init_waitqueue_head(&dev_priv->pending_flip_queue);
4640 
4641 	dev_priv->mm.interruptible = true;
4642 
4643 	dev_priv->mm.inactive_shrinker.scan_objects = i915_gem_inactive_scan;
4644 	dev_priv->mm.inactive_shrinker.count_objects = i915_gem_inactive_count;
4645 	dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
4646 	register_shrinker(&dev_priv->mm.inactive_shrinker);
4647 }
4648 
4649 /*
4650  * Create a physically contiguous memory object for this object
4651  * e.g. for cursor + overlay regs
4652  */
4653 static int i915_gem_init_phys_object(struct drm_device *dev,
4654 				     int id, int size, int align)
4655 {
4656 	struct drm_i915_private *dev_priv = dev->dev_private;
4657 	struct drm_i915_gem_phys_object *phys_obj;
4658 	int ret;
4659 
4660 	if (dev_priv->mm.phys_objs[id - 1] || !size)
4661 		return 0;
4662 
4663 	phys_obj = kzalloc(sizeof(*phys_obj), GFP_KERNEL);
4664 	if (!phys_obj)
4665 		return -ENOMEM;
4666 
4667 	phys_obj->id = id;
4668 
4669 	phys_obj->handle = drm_pci_alloc(dev, size, align);
4670 	if (!phys_obj->handle) {
4671 		ret = -ENOMEM;
4672 		goto kfree_obj;
4673 	}
4674 #ifdef CONFIG_X86
4675 	set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4676 #endif
4677 
4678 	dev_priv->mm.phys_objs[id - 1] = phys_obj;
4679 
4680 	return 0;
4681 kfree_obj:
4682 	kfree(phys_obj);
4683 	return ret;
4684 }
4685 
4686 static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4687 {
4688 	struct drm_i915_private *dev_priv = dev->dev_private;
4689 	struct drm_i915_gem_phys_object *phys_obj;
4690 
4691 	if (!dev_priv->mm.phys_objs[id - 1])
4692 		return;
4693 
4694 	phys_obj = dev_priv->mm.phys_objs[id - 1];
4695 	if (phys_obj->cur_obj) {
4696 		i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4697 	}
4698 
4699 #ifdef CONFIG_X86
4700 	set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4701 #endif
4702 	drm_pci_free(dev, phys_obj->handle);
4703 	kfree(phys_obj);
4704 	dev_priv->mm.phys_objs[id - 1] = NULL;
4705 }
4706 
4707 void i915_gem_free_all_phys_object(struct drm_device *dev)
4708 {
4709 	int i;
4710 
4711 	for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4712 		i915_gem_free_phys_object(dev, i);
4713 }
4714 
4715 void i915_gem_detach_phys_object(struct drm_device *dev,
4716 				 struct drm_i915_gem_object *obj)
4717 {
4718 	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
4719 	char *vaddr;
4720 	int i;
4721 	int page_count;
4722 
4723 	if (!obj->phys_obj)
4724 		return;
4725 	vaddr = obj->phys_obj->handle->vaddr;
4726 
4727 	page_count = obj->base.size / PAGE_SIZE;
4728 	for (i = 0; i < page_count; i++) {
4729 		struct page *page = shmem_read_mapping_page(mapping, i);
4730 		if (!IS_ERR(page)) {
4731 			char *dst = kmap_atomic(page);
4732 			memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
4733 			kunmap_atomic(dst);
4734 
4735 			drm_clflush_pages(&page, 1);
4736 
4737 			set_page_dirty(page);
4738 			mark_page_accessed(page);
4739 			page_cache_release(page);
4740 		}
4741 	}
4742 	i915_gem_chipset_flush(dev);
4743 
4744 	obj->phys_obj->cur_obj = NULL;
4745 	obj->phys_obj = NULL;
4746 }
4747 
4748 int
4749 i915_gem_attach_phys_object(struct drm_device *dev,
4750 			    struct drm_i915_gem_object *obj,
4751 			    int id,
4752 			    int align)
4753 {
4754 	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
4755 	struct drm_i915_private *dev_priv = dev->dev_private;
4756 	int ret = 0;
4757 	int page_count;
4758 	int i;
4759 
4760 	if (id > I915_MAX_PHYS_OBJECT)
4761 		return -EINVAL;
4762 
4763 	if (obj->phys_obj) {
4764 		if (obj->phys_obj->id == id)
4765 			return 0;
4766 		i915_gem_detach_phys_object(dev, obj);
4767 	}
4768 
4769 	/* create a new object */
4770 	if (!dev_priv->mm.phys_objs[id - 1]) {
4771 		ret = i915_gem_init_phys_object(dev, id,
4772 						obj->base.size, align);
4773 		if (ret) {
4774 			DRM_ERROR("failed to init phys object %d size: %zu\n",
4775 				  id, obj->base.size);
4776 			return ret;
4777 		}
4778 	}
4779 
4780 	/* bind to the object */
4781 	obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
4782 	obj->phys_obj->cur_obj = obj;
4783 
4784 	page_count = obj->base.size / PAGE_SIZE;
4785 
4786 	for (i = 0; i < page_count; i++) {
4787 		struct page *page;
4788 		char *dst, *src;
4789 
4790 		page = shmem_read_mapping_page(mapping, i);
4791 		if (IS_ERR(page))
4792 			return PTR_ERR(page);
4793 
4794 		src = kmap_atomic(page);
4795 		dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4796 		memcpy(dst, src, PAGE_SIZE);
4797 		kunmap_atomic(src);
4798 
4799 		mark_page_accessed(page);
4800 		page_cache_release(page);
4801 	}
4802 
4803 	return 0;
4804 }
4805 
4806 static int
4807 i915_gem_phys_pwrite(struct drm_device *dev,
4808 		     struct drm_i915_gem_object *obj,
4809 		     struct drm_i915_gem_pwrite *args,
4810 		     struct drm_file *file_priv)
4811 {
4812 	void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
4813 	char __user *user_data = to_user_ptr(args->data_ptr);
4814 
4815 	if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
4816 		unsigned long unwritten;
4817 
4818 		/* The physical object once assigned is fixed for the lifetime
4819 		 * of the obj, so we can safely drop the lock and continue
4820 		 * to access vaddr.
4821 		 */
4822 		mutex_unlock(&dev->struct_mutex);
4823 		unwritten = copy_from_user(vaddr, user_data, args->size);
4824 		mutex_lock(&dev->struct_mutex);
4825 		if (unwritten)
4826 			return -EFAULT;
4827 	}
4828 
4829 	i915_gem_chipset_flush(dev);
4830 	return 0;
4831 }
4832 
4833 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4834 {
4835 	struct drm_i915_file_private *file_priv = file->driver_priv;
4836 
4837 	cancel_delayed_work_sync(&file_priv->mm.idle_work);
4838 
4839 	/* Clean up our request list when the client is going away, so that
4840 	 * later retire_requests won't dereference our soon-to-be-gone
4841 	 * file_priv.
4842 	 */
4843 	spin_lock(&file_priv->mm.lock);
4844 	while (!list_empty(&file_priv->mm.request_list)) {
4845 		struct drm_i915_gem_request *request;
4846 
4847 		request = list_first_entry(&file_priv->mm.request_list,
4848 					   struct drm_i915_gem_request,
4849 					   client_list);
4850 		list_del(&request->client_list);
4851 		request->file_priv = NULL;
4852 	}
4853 	spin_unlock(&file_priv->mm.lock);
4854 }
4855 
4856 static void
4857 i915_gem_file_idle_work_handler(struct work_struct *work)
4858 {
4859 	struct drm_i915_file_private *file_priv =
4860 		container_of(work, typeof(*file_priv), mm.idle_work.work);
4861 
4862 	atomic_set(&file_priv->rps_wait_boost, false);
4863 }
4864 
4865 int i915_gem_open(struct drm_device *dev, struct drm_file *file)
4866 {
4867 	struct drm_i915_file_private *file_priv;
4868 	int ret;
4869 
4870 	DRM_DEBUG_DRIVER("\n");
4871 
4872 	file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
4873 	if (!file_priv)
4874 		return -ENOMEM;
4875 
4876 	file->driver_priv = file_priv;
4877 	file_priv->dev_priv = dev->dev_private;
4878 	file_priv->file = file;
4879 
4880 	spin_lock_init(&file_priv->mm.lock);
4881 	INIT_LIST_HEAD(&file_priv->mm.request_list);
4882 	INIT_DELAYED_WORK(&file_priv->mm.idle_work,
4883 			  i915_gem_file_idle_work_handler);
4884 
4885 	ret = i915_gem_context_open(dev, file);
4886 	if (ret)
4887 		kfree(file_priv);
4888 
4889 	return ret;
4890 }
4891 
4892 static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
4893 {
4894 	if (!mutex_is_locked(mutex))
4895 		return false;
4896 
4897 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES)
4898 	return mutex->owner == task;
4899 #else
4900 	/* Since UP may be pre-empted, we cannot assume that we own the lock */
4901 	return false;
4902 #endif
4903 }
4904 
4905 static unsigned long
4906 i915_gem_inactive_count(struct shrinker *shrinker, struct shrink_control *sc)
4907 {
4908 	struct drm_i915_private *dev_priv =
4909 		container_of(shrinker,
4910 			     struct drm_i915_private,
4911 			     mm.inactive_shrinker);
4912 	struct drm_device *dev = dev_priv->dev;
4913 	struct drm_i915_gem_object *obj;
4914 	bool unlock = true;
4915 	unsigned long count;
4916 
4917 	if (!mutex_trylock(&dev->struct_mutex)) {
4918 		if (!mutex_is_locked_by(&dev->struct_mutex, current))
4919 			return 0;
4920 
4921 		if (dev_priv->mm.shrinker_no_lock_stealing)
4922 			return 0;
4923 
4924 		unlock = false;
4925 	}
4926 
4927 	count = 0;
4928 	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list)
4929 		if (obj->pages_pin_count == 0)
4930 			count += obj->base.size >> PAGE_SHIFT;
4931 
4932 	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
4933 		if (obj->active)
4934 			continue;
4935 
4936 		if (!i915_gem_obj_is_pinned(obj) && obj->pages_pin_count == 0)
4937 			count += obj->base.size >> PAGE_SHIFT;
4938 	}
4939 
4940 	if (unlock)
4941 		mutex_unlock(&dev->struct_mutex);
4942 
4943 	return count;
4944 }
4945 
4946 /* All the new VM stuff */
4947 unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o,
4948 				  struct i915_address_space *vm)
4949 {
4950 	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
4951 	struct i915_vma *vma;
4952 
4953 	if (!dev_priv->mm.aliasing_ppgtt ||
4954 	    vm == &dev_priv->mm.aliasing_ppgtt->base)
4955 		vm = &dev_priv->gtt.base;
4956 
4957 	BUG_ON(list_empty(&o->vma_list));
4958 	list_for_each_entry(vma, &o->vma_list, vma_link) {
4959 		if (vma->vm == vm)
4960 			return vma->node.start;
4961 
4962 	}
4963 	return -1;
4964 }
4965 
4966 bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
4967 			struct i915_address_space *vm)
4968 {
4969 	struct i915_vma *vma;
4970 
4971 	list_for_each_entry(vma, &o->vma_list, vma_link)
4972 		if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
4973 			return true;
4974 
4975 	return false;
4976 }
4977 
4978 bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
4979 {
4980 	struct i915_vma *vma;
4981 
4982 	list_for_each_entry(vma, &o->vma_list, vma_link)
4983 		if (drm_mm_node_allocated(&vma->node))
4984 			return true;
4985 
4986 	return false;
4987 }
4988 
4989 unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
4990 				struct i915_address_space *vm)
4991 {
4992 	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
4993 	struct i915_vma *vma;
4994 
4995 	if (!dev_priv->mm.aliasing_ppgtt ||
4996 	    vm == &dev_priv->mm.aliasing_ppgtt->base)
4997 		vm = &dev_priv->gtt.base;
4998 
4999 	BUG_ON(list_empty(&o->vma_list));
5000 
5001 	list_for_each_entry(vma, &o->vma_list, vma_link)
5002 		if (vma->vm == vm)
5003 			return vma->node.size;
5004 
5005 	return 0;
5006 }
5007 
5008 static unsigned long
5009 i915_gem_inactive_scan(struct shrinker *shrinker, struct shrink_control *sc)
5010 {
5011 	struct drm_i915_private *dev_priv =
5012 		container_of(shrinker,
5013 			     struct drm_i915_private,
5014 			     mm.inactive_shrinker);
5015 	struct drm_device *dev = dev_priv->dev;
5016 	unsigned long freed;
5017 	bool unlock = true;
5018 
5019 	if (!mutex_trylock(&dev->struct_mutex)) {
5020 		if (!mutex_is_locked_by(&dev->struct_mutex, current))
5021 			return SHRINK_STOP;
5022 
5023 		if (dev_priv->mm.shrinker_no_lock_stealing)
5024 			return SHRINK_STOP;
5025 
5026 		unlock = false;
5027 	}
5028 
5029 	freed = i915_gem_purge(dev_priv, sc->nr_to_scan);
5030 	if (freed < sc->nr_to_scan)
5031 		freed += __i915_gem_shrink(dev_priv,
5032 					   sc->nr_to_scan - freed,
5033 					   false);
5034 	if (freed < sc->nr_to_scan)
5035 		freed += i915_gem_shrink_all(dev_priv);
5036 
5037 	if (unlock)
5038 		mutex_unlock(&dev->struct_mutex);
5039 
5040 	return freed;
5041 }
5042 
5043 struct i915_vma *i915_gem_obj_to_ggtt(struct drm_i915_gem_object *obj)
5044 {
5045 	struct i915_vma *vma;
5046 
5047 	if (WARN_ON(list_empty(&obj->vma_list)))
5048 		return NULL;
5049 
5050 	vma = list_first_entry(&obj->vma_list, typeof(*vma), vma_link);
5051 	if (vma->vm != obj_to_ggtt(obj))
5052 		return NULL;
5053 
5054 	return vma;
5055 }
5056