1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2014-2016 Intel Corporation
5  */
6 
7 #include <linux/highmem.h>
8 #include <linux/shmem_fs.h>
9 #include <linux/swap.h>
10 
11 #include <drm/drm_cache.h>
12 
13 #include "gt/intel_gt.h"
14 #include "i915_drv.h"
15 #include "i915_gem_object.h"
16 #include "i915_gem_region.h"
17 #include "i915_scatterlist.h"
18 
19 static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
20 {
21 	struct address_space *mapping = obj->base.filp->f_mapping;
22 	struct scatterlist *sg;
23 	struct sg_table *st;
24 	dma_addr_t dma;
25 	void *vaddr;
26 	void *dst;
27 	int i;
28 
29 	if (GEM_WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
30 		return -EINVAL;
31 
32 	/*
33 	 * Always aligning to the object size, allows a single allocation
34 	 * to handle all possible callers, and given typical object sizes,
35 	 * the alignment of the buddy allocation will naturally match.
36 	 */
37 	vaddr = dma_alloc_coherent(obj->base.dev->dev,
38 				   roundup_pow_of_two(obj->base.size),
39 				   &dma, GFP_KERNEL);
40 	if (!vaddr)
41 		return -ENOMEM;
42 
43 	st = kmalloc(sizeof(*st), GFP_KERNEL);
44 	if (!st)
45 		goto err_pci;
46 
47 	if (sg_alloc_table(st, 1, GFP_KERNEL))
48 		goto err_st;
49 
50 	sg = st->sgl;
51 	sg->offset = 0;
52 	sg->length = obj->base.size;
53 
54 	sg_assign_page(sg, (struct page *)vaddr);
55 	sg_dma_address(sg) = dma;
56 	sg_dma_len(sg) = obj->base.size;
57 
58 	dst = vaddr;
59 	for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
60 		struct page *page;
61 		void *src;
62 
63 		page = shmem_read_mapping_page(mapping, i);
64 		if (IS_ERR(page))
65 			goto err_st;
66 
67 		src = kmap_atomic(page);
68 		memcpy(dst, src, PAGE_SIZE);
69 		drm_clflush_virt_range(dst, PAGE_SIZE);
70 		kunmap_atomic(src);
71 
72 		put_page(page);
73 		dst += PAGE_SIZE;
74 	}
75 
76 	intel_gt_chipset_flush(&to_i915(obj->base.dev)->gt);
77 
78 	/* We're no longer struct page backed */
79 	obj->mem_flags &= ~I915_BO_FLAG_STRUCT_PAGE;
80 	__i915_gem_object_set_pages(obj, st, sg->length);
81 
82 	return 0;
83 
84 err_st:
85 	kfree(st);
86 err_pci:
87 	dma_free_coherent(obj->base.dev->dev,
88 			  roundup_pow_of_two(obj->base.size),
89 			  vaddr, dma);
90 	return -ENOMEM;
91 }
92 
93 void
94 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj,
95 			       struct sg_table *pages)
96 {
97 	dma_addr_t dma = sg_dma_address(pages->sgl);
98 	void *vaddr = sg_page(pages->sgl);
99 
100 	__i915_gem_object_release_shmem(obj, pages, false);
101 
102 	if (obj->mm.dirty) {
103 		struct address_space *mapping = obj->base.filp->f_mapping;
104 		void *src = vaddr;
105 		int i;
106 
107 		for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
108 			struct page *page;
109 			char *dst;
110 
111 			page = shmem_read_mapping_page(mapping, i);
112 			if (IS_ERR(page))
113 				continue;
114 
115 			dst = kmap_atomic(page);
116 			drm_clflush_virt_range(src, PAGE_SIZE);
117 			memcpy(dst, src, PAGE_SIZE);
118 			kunmap_atomic(dst);
119 
120 			set_page_dirty(page);
121 			if (obj->mm.madv == I915_MADV_WILLNEED)
122 				mark_page_accessed(page);
123 			put_page(page);
124 
125 			src += PAGE_SIZE;
126 		}
127 		obj->mm.dirty = false;
128 	}
129 
130 	sg_free_table(pages);
131 	kfree(pages);
132 
133 	dma_free_coherent(obj->base.dev->dev,
134 			  roundup_pow_of_two(obj->base.size),
135 			  vaddr, dma);
136 }
137 
138 int i915_gem_object_pwrite_phys(struct drm_i915_gem_object *obj,
139 				const struct drm_i915_gem_pwrite *args)
140 {
141 	void *vaddr = sg_page(obj->mm.pages->sgl) + args->offset;
142 	char __user *user_data = u64_to_user_ptr(args->data_ptr);
143 	int err;
144 
145 	err = i915_gem_object_wait(obj,
146 				   I915_WAIT_INTERRUPTIBLE |
147 				   I915_WAIT_ALL,
148 				   MAX_SCHEDULE_TIMEOUT);
149 	if (err)
150 		return err;
151 
152 	/*
153 	 * We manually control the domain here and pretend that it
154 	 * remains coherent i.e. in the GTT domain, like shmem_pwrite.
155 	 */
156 	i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
157 
158 	if (copy_from_user(vaddr, user_data, args->size))
159 		return -EFAULT;
160 
161 	drm_clflush_virt_range(vaddr, args->size);
162 	intel_gt_chipset_flush(&to_i915(obj->base.dev)->gt);
163 
164 	i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
165 	return 0;
166 }
167 
168 int i915_gem_object_pread_phys(struct drm_i915_gem_object *obj,
169 			       const struct drm_i915_gem_pread *args)
170 {
171 	void *vaddr = sg_page(obj->mm.pages->sgl) + args->offset;
172 	char __user *user_data = u64_to_user_ptr(args->data_ptr);
173 	int err;
174 
175 	err = i915_gem_object_wait(obj,
176 				   I915_WAIT_INTERRUPTIBLE,
177 				   MAX_SCHEDULE_TIMEOUT);
178 	if (err)
179 		return err;
180 
181 	drm_clflush_virt_range(vaddr, args->size);
182 	if (copy_to_user(user_data, vaddr, args->size))
183 		return -EFAULT;
184 
185 	return 0;
186 }
187 
188 static int i915_gem_object_shmem_to_phys(struct drm_i915_gem_object *obj)
189 {
190 	struct sg_table *pages;
191 	int err;
192 
193 	pages = __i915_gem_object_unset_pages(obj);
194 
195 	err = i915_gem_object_get_pages_phys(obj);
196 	if (err)
197 		goto err_xfer;
198 
199 	/* Perma-pin (until release) the physical set of pages */
200 	__i915_gem_object_pin_pages(obj);
201 
202 	if (!IS_ERR_OR_NULL(pages))
203 		i915_gem_object_put_pages_shmem(obj, pages);
204 
205 	i915_gem_object_release_memory_region(obj);
206 	return 0;
207 
208 err_xfer:
209 	if (!IS_ERR_OR_NULL(pages)) {
210 		unsigned int sg_page_sizes = i915_sg_dma_sizes(pages->sgl);
211 
212 		__i915_gem_object_set_pages(obj, pages, sg_page_sizes);
213 	}
214 	return err;
215 }
216 
217 int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align)
218 {
219 	int err;
220 
221 	assert_object_held(obj);
222 
223 	if (align > obj->base.size)
224 		return -EINVAL;
225 
226 	if (!i915_gem_object_is_shmem(obj))
227 		return -EINVAL;
228 
229 	if (!i915_gem_object_has_struct_page(obj))
230 		return 0;
231 
232 	err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE);
233 	if (err)
234 		return err;
235 
236 	if (obj->mm.madv != I915_MADV_WILLNEED)
237 		return -EFAULT;
238 
239 	if (i915_gem_object_has_tiling_quirk(obj))
240 		return -EFAULT;
241 
242 	if (obj->mm.mapping || i915_gem_object_has_pinned_pages(obj))
243 		return -EBUSY;
244 
245 	if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
246 		drm_dbg(obj->base.dev,
247 			"Attempting to obtain a purgeable object\n");
248 		return -EFAULT;
249 	}
250 
251 	return i915_gem_object_shmem_to_phys(obj);
252 }
253 
254 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
255 #include "selftests/i915_gem_phys.c"
256 #endif
257