1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2014-2016 Intel Corporation
5  */
6 
7 #include <linux/highmem.h>
8 #include <linux/shmem_fs.h>
9 #include <linux/swap.h>
10 
11 #include <drm/drm_cache.h>
12 
13 #include "gt/intel_gt.h"
14 #include "i915_drv.h"
15 #include "i915_gem_object.h"
16 #include "i915_gem_region.h"
17 #include "i915_gem_tiling.h"
18 #include "i915_scatterlist.h"
19 
20 static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
21 {
22 	struct address_space *mapping = obj->base.filp->f_mapping;
23 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
24 	struct scatterlist *sg;
25 	struct sg_table *st;
26 	dma_addr_t dma;
27 	void *vaddr;
28 	void *dst;
29 	int i;
30 
31 	if (GEM_WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
32 		return -EINVAL;
33 
34 	/*
35 	 * Always aligning to the object size, allows a single allocation
36 	 * to handle all possible callers, and given typical object sizes,
37 	 * the alignment of the buddy allocation will naturally match.
38 	 */
39 	vaddr = dma_alloc_coherent(obj->base.dev->dev,
40 				   roundup_pow_of_two(obj->base.size),
41 				   &dma, GFP_KERNEL);
42 	if (!vaddr)
43 		return -ENOMEM;
44 
45 	st = kmalloc(sizeof(*st), GFP_KERNEL);
46 	if (!st)
47 		goto err_pci;
48 
49 	if (sg_alloc_table(st, 1, GFP_KERNEL))
50 		goto err_st;
51 
52 	sg = st->sgl;
53 	sg->offset = 0;
54 	sg->length = obj->base.size;
55 
56 	sg_assign_page(sg, (struct page *)vaddr);
57 	sg_dma_address(sg) = dma;
58 	sg_dma_len(sg) = obj->base.size;
59 
60 	dst = vaddr;
61 	for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
62 		struct page *page;
63 		void *src;
64 
65 		page = shmem_read_mapping_page(mapping, i);
66 		if (IS_ERR(page))
67 			goto err_st;
68 
69 		src = kmap_atomic(page);
70 		memcpy(dst, src, PAGE_SIZE);
71 		drm_clflush_virt_range(dst, PAGE_SIZE);
72 		kunmap_atomic(src);
73 
74 		put_page(page);
75 		dst += PAGE_SIZE;
76 	}
77 
78 	intel_gt_chipset_flush(to_gt(i915));
79 
80 	/* We're no longer struct page backed */
81 	obj->mem_flags &= ~I915_BO_FLAG_STRUCT_PAGE;
82 	__i915_gem_object_set_pages(obj, st);
83 
84 	return 0;
85 
86 err_st:
87 	kfree(st);
88 err_pci:
89 	dma_free_coherent(obj->base.dev->dev,
90 			  roundup_pow_of_two(obj->base.size),
91 			  vaddr, dma);
92 	return -ENOMEM;
93 }
94 
95 void
96 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj,
97 			       struct sg_table *pages)
98 {
99 	dma_addr_t dma = sg_dma_address(pages->sgl);
100 	void *vaddr = sg_page(pages->sgl);
101 
102 	__i915_gem_object_release_shmem(obj, pages, false);
103 
104 	if (obj->mm.dirty) {
105 		struct address_space *mapping = obj->base.filp->f_mapping;
106 		void *src = vaddr;
107 		int i;
108 
109 		for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
110 			struct page *page;
111 			char *dst;
112 
113 			page = shmem_read_mapping_page(mapping, i);
114 			if (IS_ERR(page))
115 				continue;
116 
117 			dst = kmap_atomic(page);
118 			drm_clflush_virt_range(src, PAGE_SIZE);
119 			memcpy(dst, src, PAGE_SIZE);
120 			kunmap_atomic(dst);
121 
122 			set_page_dirty(page);
123 			if (obj->mm.madv == I915_MADV_WILLNEED)
124 				mark_page_accessed(page);
125 			put_page(page);
126 
127 			src += PAGE_SIZE;
128 		}
129 		obj->mm.dirty = false;
130 	}
131 
132 	sg_free_table(pages);
133 	kfree(pages);
134 
135 	dma_free_coherent(obj->base.dev->dev,
136 			  roundup_pow_of_two(obj->base.size),
137 			  vaddr, dma);
138 }
139 
140 int i915_gem_object_pwrite_phys(struct drm_i915_gem_object *obj,
141 				const struct drm_i915_gem_pwrite *args)
142 {
143 	void *vaddr = sg_page(obj->mm.pages->sgl) + args->offset;
144 	char __user *user_data = u64_to_user_ptr(args->data_ptr);
145 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
146 	int err;
147 
148 	err = i915_gem_object_wait(obj,
149 				   I915_WAIT_INTERRUPTIBLE |
150 				   I915_WAIT_ALL,
151 				   MAX_SCHEDULE_TIMEOUT);
152 	if (err)
153 		return err;
154 
155 	/*
156 	 * We manually control the domain here and pretend that it
157 	 * remains coherent i.e. in the GTT domain, like shmem_pwrite.
158 	 */
159 	i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
160 
161 	if (copy_from_user(vaddr, user_data, args->size))
162 		return -EFAULT;
163 
164 	drm_clflush_virt_range(vaddr, args->size);
165 	intel_gt_chipset_flush(to_gt(i915));
166 
167 	i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
168 	return 0;
169 }
170 
171 int i915_gem_object_pread_phys(struct drm_i915_gem_object *obj,
172 			       const struct drm_i915_gem_pread *args)
173 {
174 	void *vaddr = sg_page(obj->mm.pages->sgl) + args->offset;
175 	char __user *user_data = u64_to_user_ptr(args->data_ptr);
176 	int err;
177 
178 	err = i915_gem_object_wait(obj,
179 				   I915_WAIT_INTERRUPTIBLE,
180 				   MAX_SCHEDULE_TIMEOUT);
181 	if (err)
182 		return err;
183 
184 	drm_clflush_virt_range(vaddr, args->size);
185 	if (copy_to_user(user_data, vaddr, args->size))
186 		return -EFAULT;
187 
188 	return 0;
189 }
190 
191 static int i915_gem_object_shmem_to_phys(struct drm_i915_gem_object *obj)
192 {
193 	struct sg_table *pages;
194 	int err;
195 
196 	pages = __i915_gem_object_unset_pages(obj);
197 
198 	err = i915_gem_object_get_pages_phys(obj);
199 	if (err)
200 		goto err_xfer;
201 
202 	/* Perma-pin (until release) the physical set of pages */
203 	__i915_gem_object_pin_pages(obj);
204 
205 	if (!IS_ERR_OR_NULL(pages))
206 		i915_gem_object_put_pages_shmem(obj, pages);
207 
208 	i915_gem_object_release_memory_region(obj);
209 	return 0;
210 
211 err_xfer:
212 	if (!IS_ERR_OR_NULL(pages))
213 		__i915_gem_object_set_pages(obj, pages);
214 	return err;
215 }
216 
217 int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align)
218 {
219 	int err;
220 
221 	assert_object_held(obj);
222 
223 	if (align > obj->base.size)
224 		return -EINVAL;
225 
226 	if (!i915_gem_object_is_shmem(obj))
227 		return -EINVAL;
228 
229 	if (!i915_gem_object_has_struct_page(obj))
230 		return 0;
231 
232 	err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE);
233 	if (err)
234 		return err;
235 
236 	if (obj->mm.madv != I915_MADV_WILLNEED)
237 		return -EFAULT;
238 
239 	if (i915_gem_object_has_tiling_quirk(obj))
240 		return -EFAULT;
241 
242 	if (obj->mm.mapping || i915_gem_object_has_pinned_pages(obj))
243 		return -EBUSY;
244 
245 	if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
246 		drm_dbg(obj->base.dev,
247 			"Attempting to obtain a purgeable object\n");
248 		return -EFAULT;
249 	}
250 
251 	return i915_gem_object_shmem_to_phys(obj);
252 }
253 
254 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
255 #include "selftests/i915_gem_phys.c"
256 #endif
257