xref: /openbmc/linux/drivers/gpu/drm/msm/msm_gem.c (revision 68209390)
1c8afe684SRob Clark /*
2c8afe684SRob Clark  * Copyright (C) 2013 Red Hat
3c8afe684SRob Clark  * Author: Rob Clark <robdclark@gmail.com>
4c8afe684SRob Clark  *
5c8afe684SRob Clark  * This program is free software; you can redistribute it and/or modify it
6c8afe684SRob Clark  * under the terms of the GNU General Public License version 2 as published by
7c8afe684SRob Clark  * the Free Software Foundation.
8c8afe684SRob Clark  *
9c8afe684SRob Clark  * This program is distributed in the hope that it will be useful, but WITHOUT
10c8afe684SRob Clark  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11c8afe684SRob Clark  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
12c8afe684SRob Clark  * more details.
13c8afe684SRob Clark  *
14c8afe684SRob Clark  * You should have received a copy of the GNU General Public License along with
15c8afe684SRob Clark  * this program.  If not, see <http://www.gnu.org/licenses/>.
16c8afe684SRob Clark  */
17c8afe684SRob Clark 
18c8afe684SRob Clark #include <linux/spinlock.h>
19c8afe684SRob Clark #include <linux/shmem_fs.h>
2005b84911SRob Clark #include <linux/dma-buf.h>
2101c8f1c4SDan Williams #include <linux/pfn_t.h>
22c8afe684SRob Clark 
23c8afe684SRob Clark #include "msm_drv.h"
24fde5de6cSRob Clark #include "msm_fence.h"
25c8afe684SRob Clark #include "msm_gem.h"
267198e6b0SRob Clark #include "msm_gpu.h"
27871d812aSRob Clark #include "msm_mmu.h"
28c8afe684SRob Clark 
29871d812aSRob Clark static dma_addr_t physaddr(struct drm_gem_object *obj)
30871d812aSRob Clark {
31871d812aSRob Clark 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
32871d812aSRob Clark 	struct msm_drm_private *priv = obj->dev->dev_private;
33871d812aSRob Clark 	return (((dma_addr_t)msm_obj->vram_node->start) << PAGE_SHIFT) +
34871d812aSRob Clark 			priv->vram.paddr;
35871d812aSRob Clark }
36871d812aSRob Clark 
37072f1f91SRob Clark static bool use_pages(struct drm_gem_object *obj)
38072f1f91SRob Clark {
39072f1f91SRob Clark 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
40072f1f91SRob Clark 	return !msm_obj->vram_node;
41072f1f91SRob Clark }
42072f1f91SRob Clark 
43871d812aSRob Clark /* allocate pages from VRAM carveout, used when no IOMMU: */
44871d812aSRob Clark static struct page **get_pages_vram(struct drm_gem_object *obj,
45871d812aSRob Clark 		int npages)
46871d812aSRob Clark {
47871d812aSRob Clark 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
48871d812aSRob Clark 	struct msm_drm_private *priv = obj->dev->dev_private;
49871d812aSRob Clark 	dma_addr_t paddr;
50871d812aSRob Clark 	struct page **p;
51871d812aSRob Clark 	int ret, i;
52871d812aSRob Clark 
53871d812aSRob Clark 	p = drm_malloc_ab(npages, sizeof(struct page *));
54871d812aSRob Clark 	if (!p)
55871d812aSRob Clark 		return ERR_PTR(-ENOMEM);
56871d812aSRob Clark 
57871d812aSRob Clark 	ret = drm_mm_insert_node(&priv->vram.mm, msm_obj->vram_node,
58871d812aSRob Clark 			npages, 0, DRM_MM_SEARCH_DEFAULT);
59871d812aSRob Clark 	if (ret) {
60871d812aSRob Clark 		drm_free_large(p);
61871d812aSRob Clark 		return ERR_PTR(ret);
62871d812aSRob Clark 	}
63871d812aSRob Clark 
64871d812aSRob Clark 	paddr = physaddr(obj);
65871d812aSRob Clark 	for (i = 0; i < npages; i++) {
66871d812aSRob Clark 		p[i] = phys_to_page(paddr);
67871d812aSRob Clark 		paddr += PAGE_SIZE;
68871d812aSRob Clark 	}
69871d812aSRob Clark 
70871d812aSRob Clark 	return p;
71871d812aSRob Clark }
72c8afe684SRob Clark 
73c8afe684SRob Clark /* called with dev->struct_mutex held */
74c8afe684SRob Clark static struct page **get_pages(struct drm_gem_object *obj)
75c8afe684SRob Clark {
76c8afe684SRob Clark 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
77c8afe684SRob Clark 
78c8afe684SRob Clark 	if (!msm_obj->pages) {
79c8afe684SRob Clark 		struct drm_device *dev = obj->dev;
80871d812aSRob Clark 		struct page **p;
81c8afe684SRob Clark 		int npages = obj->size >> PAGE_SHIFT;
82c8afe684SRob Clark 
83072f1f91SRob Clark 		if (use_pages(obj))
840cdbe8acSDavid Herrmann 			p = drm_gem_get_pages(obj);
85871d812aSRob Clark 		else
86871d812aSRob Clark 			p = get_pages_vram(obj, npages);
87871d812aSRob Clark 
88c8afe684SRob Clark 		if (IS_ERR(p)) {
89c8afe684SRob Clark 			dev_err(dev->dev, "could not get pages: %ld\n",
90c8afe684SRob Clark 					PTR_ERR(p));
91c8afe684SRob Clark 			return p;
92c8afe684SRob Clark 		}
93c8afe684SRob Clark 
94c8afe684SRob Clark 		msm_obj->sgt = drm_prime_pages_to_sg(p, npages);
951f70e079SWei Yongjun 		if (IS_ERR(msm_obj->sgt)) {
96c8afe684SRob Clark 			dev_err(dev->dev, "failed to allocate sgt\n");
971f70e079SWei Yongjun 			return ERR_CAST(msm_obj->sgt);
98c8afe684SRob Clark 		}
99c8afe684SRob Clark 
100c8afe684SRob Clark 		msm_obj->pages = p;
101c8afe684SRob Clark 
102c8afe684SRob Clark 		/* For non-cached buffers, ensure the new pages are clean
103c8afe684SRob Clark 		 * because display controller, GPU, etc. are not coherent:
104c8afe684SRob Clark 		 */
105c8afe684SRob Clark 		if (msm_obj->flags & (MSM_BO_WC|MSM_BO_UNCACHED))
106c8afe684SRob Clark 			dma_map_sg(dev->dev, msm_obj->sgt->sgl,
107c8afe684SRob Clark 					msm_obj->sgt->nents, DMA_BIDIRECTIONAL);
108c8afe684SRob Clark 	}
109c8afe684SRob Clark 
110c8afe684SRob Clark 	return msm_obj->pages;
111c8afe684SRob Clark }
112c8afe684SRob Clark 
113c8afe684SRob Clark static void put_pages(struct drm_gem_object *obj)
114c8afe684SRob Clark {
115c8afe684SRob Clark 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
116c8afe684SRob Clark 
117c8afe684SRob Clark 	if (msm_obj->pages) {
118c8afe684SRob Clark 		/* For non-cached buffers, ensure the new pages are clean
119c8afe684SRob Clark 		 * because display controller, GPU, etc. are not coherent:
120c8afe684SRob Clark 		 */
121c8afe684SRob Clark 		if (msm_obj->flags & (MSM_BO_WC|MSM_BO_UNCACHED))
122c8afe684SRob Clark 			dma_unmap_sg(obj->dev->dev, msm_obj->sgt->sgl,
123c8afe684SRob Clark 					msm_obj->sgt->nents, DMA_BIDIRECTIONAL);
124c8afe684SRob Clark 		sg_free_table(msm_obj->sgt);
125c8afe684SRob Clark 		kfree(msm_obj->sgt);
126c8afe684SRob Clark 
127072f1f91SRob Clark 		if (use_pages(obj))
128c8afe684SRob Clark 			drm_gem_put_pages(obj, msm_obj->pages, true, false);
1291ffa2425SMicah Richert 		else {
130871d812aSRob Clark 			drm_mm_remove_node(msm_obj->vram_node);
1311ffa2425SMicah Richert 			drm_free_large(msm_obj->pages);
1321ffa2425SMicah Richert 		}
133871d812aSRob Clark 
134c8afe684SRob Clark 		msm_obj->pages = NULL;
135c8afe684SRob Clark 	}
136c8afe684SRob Clark }
137c8afe684SRob Clark 
13805b84911SRob Clark struct page **msm_gem_get_pages(struct drm_gem_object *obj)
13905b84911SRob Clark {
14005b84911SRob Clark 	struct drm_device *dev = obj->dev;
14105b84911SRob Clark 	struct page **p;
14205b84911SRob Clark 	mutex_lock(&dev->struct_mutex);
14305b84911SRob Clark 	p = get_pages(obj);
14405b84911SRob Clark 	mutex_unlock(&dev->struct_mutex);
14505b84911SRob Clark 	return p;
14605b84911SRob Clark }
14705b84911SRob Clark 
14805b84911SRob Clark void msm_gem_put_pages(struct drm_gem_object *obj)
14905b84911SRob Clark {
15005b84911SRob Clark 	/* when we start tracking the pin count, then do something here */
15105b84911SRob Clark }
15205b84911SRob Clark 
153c8afe684SRob Clark int msm_gem_mmap_obj(struct drm_gem_object *obj,
154c8afe684SRob Clark 		struct vm_area_struct *vma)
155c8afe684SRob Clark {
156c8afe684SRob Clark 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
157c8afe684SRob Clark 
158c8afe684SRob Clark 	vma->vm_flags &= ~VM_PFNMAP;
159c8afe684SRob Clark 	vma->vm_flags |= VM_MIXEDMAP;
160c8afe684SRob Clark 
161c8afe684SRob Clark 	if (msm_obj->flags & MSM_BO_WC) {
162c8afe684SRob Clark 		vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
163c8afe684SRob Clark 	} else if (msm_obj->flags & MSM_BO_UNCACHED) {
164c8afe684SRob Clark 		vma->vm_page_prot = pgprot_noncached(vm_get_page_prot(vma->vm_flags));
165c8afe684SRob Clark 	} else {
166c8afe684SRob Clark 		/*
167c8afe684SRob Clark 		 * Shunt off cached objs to shmem file so they have their own
168c8afe684SRob Clark 		 * address_space (so unmap_mapping_range does what we want,
169c8afe684SRob Clark 		 * in particular in the case of mmap'd dmabufs)
170c8afe684SRob Clark 		 */
171c8afe684SRob Clark 		fput(vma->vm_file);
172c8afe684SRob Clark 		get_file(obj->filp);
173c8afe684SRob Clark 		vma->vm_pgoff = 0;
174c8afe684SRob Clark 		vma->vm_file  = obj->filp;
175c8afe684SRob Clark 
176c8afe684SRob Clark 		vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
177c8afe684SRob Clark 	}
178c8afe684SRob Clark 
179c8afe684SRob Clark 	return 0;
180c8afe684SRob Clark }
181c8afe684SRob Clark 
182c8afe684SRob Clark int msm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
183c8afe684SRob Clark {
184c8afe684SRob Clark 	int ret;
185c8afe684SRob Clark 
186c8afe684SRob Clark 	ret = drm_gem_mmap(filp, vma);
187c8afe684SRob Clark 	if (ret) {
188c8afe684SRob Clark 		DBG("mmap failed: %d", ret);
189c8afe684SRob Clark 		return ret;
190c8afe684SRob Clark 	}
191c8afe684SRob Clark 
192c8afe684SRob Clark 	return msm_gem_mmap_obj(vma->vm_private_data, vma);
193c8afe684SRob Clark }
194c8afe684SRob Clark 
195c8afe684SRob Clark int msm_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
196c8afe684SRob Clark {
197c8afe684SRob Clark 	struct drm_gem_object *obj = vma->vm_private_data;
198c8afe684SRob Clark 	struct drm_device *dev = obj->dev;
199c8afe684SRob Clark 	struct page **pages;
200c8afe684SRob Clark 	unsigned long pfn;
201c8afe684SRob Clark 	pgoff_t pgoff;
202c8afe684SRob Clark 	int ret;
203c8afe684SRob Clark 
204c8afe684SRob Clark 	/* Make sure we don't parallel update on a fault, nor move or remove
205c8afe684SRob Clark 	 * something from beneath our feet
206c8afe684SRob Clark 	 */
207c8afe684SRob Clark 	ret = mutex_lock_interruptible(&dev->struct_mutex);
208c8afe684SRob Clark 	if (ret)
209c8afe684SRob Clark 		goto out;
210c8afe684SRob Clark 
211c8afe684SRob Clark 	/* make sure we have pages attached now */
212c8afe684SRob Clark 	pages = get_pages(obj);
213c8afe684SRob Clark 	if (IS_ERR(pages)) {
214c8afe684SRob Clark 		ret = PTR_ERR(pages);
215c8afe684SRob Clark 		goto out_unlock;
216c8afe684SRob Clark 	}
217c8afe684SRob Clark 
218c8afe684SRob Clark 	/* We don't use vmf->pgoff since that has the fake offset: */
219c8afe684SRob Clark 	pgoff = ((unsigned long)vmf->virtual_address -
220c8afe684SRob Clark 			vma->vm_start) >> PAGE_SHIFT;
221c8afe684SRob Clark 
222871d812aSRob Clark 	pfn = page_to_pfn(pages[pgoff]);
223c8afe684SRob Clark 
224c8afe684SRob Clark 	VERB("Inserting %p pfn %lx, pa %lx", vmf->virtual_address,
225c8afe684SRob Clark 			pfn, pfn << PAGE_SHIFT);
226c8afe684SRob Clark 
22701c8f1c4SDan Williams 	ret = vm_insert_mixed(vma, (unsigned long)vmf->virtual_address,
22801c8f1c4SDan Williams 			__pfn_to_pfn_t(pfn, PFN_DEV));
229c8afe684SRob Clark 
230c8afe684SRob Clark out_unlock:
231c8afe684SRob Clark 	mutex_unlock(&dev->struct_mutex);
232c8afe684SRob Clark out:
233c8afe684SRob Clark 	switch (ret) {
234c8afe684SRob Clark 	case -EAGAIN:
235c8afe684SRob Clark 	case 0:
236c8afe684SRob Clark 	case -ERESTARTSYS:
237c8afe684SRob Clark 	case -EINTR:
238505886d5SRob Clark 	case -EBUSY:
239505886d5SRob Clark 		/*
240505886d5SRob Clark 		 * EBUSY is ok: this just means that another thread
241505886d5SRob Clark 		 * already did the job.
242505886d5SRob Clark 		 */
243c8afe684SRob Clark 		return VM_FAULT_NOPAGE;
244c8afe684SRob Clark 	case -ENOMEM:
245c8afe684SRob Clark 		return VM_FAULT_OOM;
246c8afe684SRob Clark 	default:
247c8afe684SRob Clark 		return VM_FAULT_SIGBUS;
248c8afe684SRob Clark 	}
249c8afe684SRob Clark }
250c8afe684SRob Clark 
251c8afe684SRob Clark /** get mmap offset */
252c8afe684SRob Clark static uint64_t mmap_offset(struct drm_gem_object *obj)
253c8afe684SRob Clark {
254c8afe684SRob Clark 	struct drm_device *dev = obj->dev;
255c8afe684SRob Clark 	int ret;
256c8afe684SRob Clark 
257c8afe684SRob Clark 	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
258c8afe684SRob Clark 
259c8afe684SRob Clark 	/* Make it mmapable */
260c8afe684SRob Clark 	ret = drm_gem_create_mmap_offset(obj);
261c8afe684SRob Clark 
262c8afe684SRob Clark 	if (ret) {
263c8afe684SRob Clark 		dev_err(dev->dev, "could not allocate mmap offset\n");
264c8afe684SRob Clark 		return 0;
265c8afe684SRob Clark 	}
266c8afe684SRob Clark 
267c8afe684SRob Clark 	return drm_vma_node_offset_addr(&obj->vma_node);
268c8afe684SRob Clark }
269c8afe684SRob Clark 
270c8afe684SRob Clark uint64_t msm_gem_mmap_offset(struct drm_gem_object *obj)
271c8afe684SRob Clark {
272c8afe684SRob Clark 	uint64_t offset;
273c8afe684SRob Clark 	mutex_lock(&obj->dev->struct_mutex);
274c8afe684SRob Clark 	offset = mmap_offset(obj);
275c8afe684SRob Clark 	mutex_unlock(&obj->dev->struct_mutex);
276c8afe684SRob Clark 	return offset;
277c8afe684SRob Clark }
278c8afe684SRob Clark 
2794fe5f65eSRob Clark static void
2804fe5f65eSRob Clark put_iova(struct drm_gem_object *obj)
2814fe5f65eSRob Clark {
2824fe5f65eSRob Clark 	struct drm_device *dev = obj->dev;
2834fe5f65eSRob Clark 	struct msm_drm_private *priv = obj->dev->dev_private;
2844fe5f65eSRob Clark 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
2854fe5f65eSRob Clark 	int id;
2864fe5f65eSRob Clark 
2874fe5f65eSRob Clark 	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
2884fe5f65eSRob Clark 
2894fe5f65eSRob Clark 	for (id = 0; id < ARRAY_SIZE(msm_obj->domain); id++) {
2904fe5f65eSRob Clark 		struct msm_mmu *mmu = priv->mmus[id];
2914fe5f65eSRob Clark 		if (mmu && msm_obj->domain[id].iova) {
2924fe5f65eSRob Clark 			uint32_t offset = msm_obj->domain[id].iova;
2934fe5f65eSRob Clark 			mmu->funcs->unmap(mmu, offset, msm_obj->sgt, obj->size);
2944fe5f65eSRob Clark 			msm_obj->domain[id].iova = 0;
2954fe5f65eSRob Clark 		}
2964fe5f65eSRob Clark 	}
2974fe5f65eSRob Clark }
2984fe5f65eSRob Clark 
299c8afe684SRob Clark /* should be called under struct_mutex.. although it can be called
300c8afe684SRob Clark  * from atomic context without struct_mutex to acquire an extra
301c8afe684SRob Clark  * iova ref if you know one is already held.
302c8afe684SRob Clark  *
303c8afe684SRob Clark  * That means when I do eventually need to add support for unpinning
304c8afe684SRob Clark  * the refcnt counter needs to be atomic_t.
305c8afe684SRob Clark  */
306c8afe684SRob Clark int msm_gem_get_iova_locked(struct drm_gem_object *obj, int id,
307c8afe684SRob Clark 		uint32_t *iova)
308c8afe684SRob Clark {
309c8afe684SRob Clark 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
310c8afe684SRob Clark 	int ret = 0;
311c8afe684SRob Clark 
312c8afe684SRob Clark 	if (!msm_obj->domain[id].iova) {
313c8afe684SRob Clark 		struct msm_drm_private *priv = obj->dev->dev_private;
314871d812aSRob Clark 		struct page **pages = get_pages(obj);
315871d812aSRob Clark 
316c8afe684SRob Clark 		if (IS_ERR(pages))
317c8afe684SRob Clark 			return PTR_ERR(pages);
318871d812aSRob Clark 
319871d812aSRob Clark 		if (iommu_present(&platform_bus_type)) {
3201c4997feSRob Clark 			struct msm_mmu *mmu = priv->mmus[id];
3211c4997feSRob Clark 			uint32_t offset;
3221c4997feSRob Clark 
3231c4997feSRob Clark 			if (WARN_ON(!mmu))
3241c4997feSRob Clark 				return -EINVAL;
3251c4997feSRob Clark 
3261c4997feSRob Clark 			offset = (uint32_t)mmap_offset(obj);
327871d812aSRob Clark 			ret = mmu->funcs->map(mmu, offset, msm_obj->sgt,
328c8afe684SRob Clark 					obj->size, IOMMU_READ | IOMMU_WRITE);
329c8afe684SRob Clark 			msm_obj->domain[id].iova = offset;
330871d812aSRob Clark 		} else {
331871d812aSRob Clark 			msm_obj->domain[id].iova = physaddr(obj);
332871d812aSRob Clark 		}
333c8afe684SRob Clark 	}
334c8afe684SRob Clark 
335c8afe684SRob Clark 	if (!ret)
336c8afe684SRob Clark 		*iova = msm_obj->domain[id].iova;
337c8afe684SRob Clark 
338c8afe684SRob Clark 	return ret;
339c8afe684SRob Clark }
340c8afe684SRob Clark 
3412638d90aSRob Clark /* get iova, taking a reference.  Should have a matching put */
342c8afe684SRob Clark int msm_gem_get_iova(struct drm_gem_object *obj, int id, uint32_t *iova)
343c8afe684SRob Clark {
344edd4fc63SRob Clark 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
345c8afe684SRob Clark 	int ret;
346edd4fc63SRob Clark 
347edd4fc63SRob Clark 	/* this is safe right now because we don't unmap until the
348edd4fc63SRob Clark 	 * bo is deleted:
349edd4fc63SRob Clark 	 */
350edd4fc63SRob Clark 	if (msm_obj->domain[id].iova) {
351edd4fc63SRob Clark 		*iova = msm_obj->domain[id].iova;
352edd4fc63SRob Clark 		return 0;
353edd4fc63SRob Clark 	}
354edd4fc63SRob Clark 
355c8afe684SRob Clark 	mutex_lock(&obj->dev->struct_mutex);
356c8afe684SRob Clark 	ret = msm_gem_get_iova_locked(obj, id, iova);
357c8afe684SRob Clark 	mutex_unlock(&obj->dev->struct_mutex);
358c8afe684SRob Clark 	return ret;
359c8afe684SRob Clark }
360c8afe684SRob Clark 
3612638d90aSRob Clark /* get iova without taking a reference, used in places where you have
3622638d90aSRob Clark  * already done a 'msm_gem_get_iova()'.
3632638d90aSRob Clark  */
3642638d90aSRob Clark uint32_t msm_gem_iova(struct drm_gem_object *obj, int id)
3652638d90aSRob Clark {
3662638d90aSRob Clark 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
3672638d90aSRob Clark 	WARN_ON(!msm_obj->domain[id].iova);
3682638d90aSRob Clark 	return msm_obj->domain[id].iova;
3692638d90aSRob Clark }
3702638d90aSRob Clark 
371c8afe684SRob Clark void msm_gem_put_iova(struct drm_gem_object *obj, int id)
372c8afe684SRob Clark {
373c8afe684SRob Clark 	// XXX TODO ..
374c8afe684SRob Clark 	// NOTE: probably don't need a _locked() version.. we wouldn't
375c8afe684SRob Clark 	// normally unmap here, but instead just mark that it could be
376c8afe684SRob Clark 	// unmapped (if the iova refcnt drops to zero), but then later
377c8afe684SRob Clark 	// if another _get_iova_locked() fails we can start unmapping
378c8afe684SRob Clark 	// things that are no longer needed..
379c8afe684SRob Clark }
380c8afe684SRob Clark 
381c8afe684SRob Clark int msm_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
382c8afe684SRob Clark 		struct drm_mode_create_dumb *args)
383c8afe684SRob Clark {
384c8afe684SRob Clark 	args->pitch = align_pitch(args->width, args->bpp);
385c8afe684SRob Clark 	args->size  = PAGE_ALIGN(args->pitch * args->height);
386c8afe684SRob Clark 	return msm_gem_new_handle(dev, file, args->size,
387c8afe684SRob Clark 			MSM_BO_SCANOUT | MSM_BO_WC, &args->handle);
388c8afe684SRob Clark }
389c8afe684SRob Clark 
390c8afe684SRob Clark int msm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
391c8afe684SRob Clark 		uint32_t handle, uint64_t *offset)
392c8afe684SRob Clark {
393c8afe684SRob Clark 	struct drm_gem_object *obj;
394c8afe684SRob Clark 	int ret = 0;
395c8afe684SRob Clark 
396c8afe684SRob Clark 	/* GEM does all our handle to object mapping */
397a8ad0bd8SChris Wilson 	obj = drm_gem_object_lookup(file, handle);
398c8afe684SRob Clark 	if (obj == NULL) {
399c8afe684SRob Clark 		ret = -ENOENT;
400c8afe684SRob Clark 		goto fail;
401c8afe684SRob Clark 	}
402c8afe684SRob Clark 
403c8afe684SRob Clark 	*offset = msm_gem_mmap_offset(obj);
404c8afe684SRob Clark 
405c8afe684SRob Clark 	drm_gem_object_unreference_unlocked(obj);
406c8afe684SRob Clark 
407c8afe684SRob Clark fail:
408c8afe684SRob Clark 	return ret;
409c8afe684SRob Clark }
410c8afe684SRob Clark 
411c8afe684SRob Clark void *msm_gem_vaddr_locked(struct drm_gem_object *obj)
412c8afe684SRob Clark {
413c8afe684SRob Clark 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
414c8afe684SRob Clark 	WARN_ON(!mutex_is_locked(&obj->dev->struct_mutex));
415c8afe684SRob Clark 	if (!msm_obj->vaddr) {
416c8afe684SRob Clark 		struct page **pages = get_pages(obj);
417c8afe684SRob Clark 		if (IS_ERR(pages))
418c8afe684SRob Clark 			return ERR_CAST(pages);
419c8afe684SRob Clark 		msm_obj->vaddr = vmap(pages, obj->size >> PAGE_SHIFT,
420c8afe684SRob Clark 				VM_MAP, pgprot_writecombine(PAGE_KERNEL));
42169a834c2SRob Clark 		if (msm_obj->vaddr == NULL)
42269a834c2SRob Clark 			return ERR_PTR(-ENOMEM);
423c8afe684SRob Clark 	}
424c8afe684SRob Clark 	return msm_obj->vaddr;
425c8afe684SRob Clark }
426c8afe684SRob Clark 
427c8afe684SRob Clark void *msm_gem_vaddr(struct drm_gem_object *obj)
428c8afe684SRob Clark {
429c8afe684SRob Clark 	void *ret;
430c8afe684SRob Clark 	mutex_lock(&obj->dev->struct_mutex);
431c8afe684SRob Clark 	ret = msm_gem_vaddr_locked(obj);
432c8afe684SRob Clark 	mutex_unlock(&obj->dev->struct_mutex);
433c8afe684SRob Clark 	return ret;
434c8afe684SRob Clark }
435c8afe684SRob Clark 
4364cd33c48SRob Clark /* Update madvise status, returns true if not purged, else
4374cd33c48SRob Clark  * false or -errno.
4384cd33c48SRob Clark  */
4394cd33c48SRob Clark int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv)
4404cd33c48SRob Clark {
4414cd33c48SRob Clark 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
4424cd33c48SRob Clark 
4434cd33c48SRob Clark 	WARN_ON(!mutex_is_locked(&obj->dev->struct_mutex));
4444cd33c48SRob Clark 
4454cd33c48SRob Clark 	if (msm_obj->madv != __MSM_MADV_PURGED)
4464cd33c48SRob Clark 		msm_obj->madv = madv;
4474cd33c48SRob Clark 
4484cd33c48SRob Clark 	return (msm_obj->madv != __MSM_MADV_PURGED);
4494cd33c48SRob Clark }
4504cd33c48SRob Clark 
45168209390SRob Clark void msm_gem_purge(struct drm_gem_object *obj)
45268209390SRob Clark {
45368209390SRob Clark 	struct drm_device *dev = obj->dev;
45468209390SRob Clark 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
45568209390SRob Clark 
45668209390SRob Clark 	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
45768209390SRob Clark 	WARN_ON(!is_purgeable(msm_obj));
45868209390SRob Clark 	WARN_ON(obj->import_attach);
45968209390SRob Clark 
46068209390SRob Clark 	put_iova(obj);
46168209390SRob Clark 
46268209390SRob Clark 	vunmap(msm_obj->vaddr);
46368209390SRob Clark 	msm_obj->vaddr = NULL;
46468209390SRob Clark 
46568209390SRob Clark 	put_pages(obj);
46668209390SRob Clark 
46768209390SRob Clark 	msm_obj->madv = __MSM_MADV_PURGED;
46868209390SRob Clark 
46968209390SRob Clark 	drm_vma_node_unmap(&obj->vma_node, dev->anon_inode->i_mapping);
47068209390SRob Clark 	drm_gem_free_mmap_offset(obj);
47168209390SRob Clark 
47268209390SRob Clark 	/* Our goal here is to return as much of the memory as
47368209390SRob Clark 	 * is possible back to the system as we are called from OOM.
47468209390SRob Clark 	 * To do this we must instruct the shmfs to drop all of its
47568209390SRob Clark 	 * backing pages, *now*.
47668209390SRob Clark 	 */
47768209390SRob Clark 	shmem_truncate_range(file_inode(obj->filp), 0, (loff_t)-1);
47868209390SRob Clark 
47968209390SRob Clark 	invalidate_mapping_pages(file_inode(obj->filp)->i_mapping,
48068209390SRob Clark 			0, (loff_t)-1);
48168209390SRob Clark }
48268209390SRob Clark 
483b6295f9aSRob Clark /* must be called before _move_to_active().. */
484b6295f9aSRob Clark int msm_gem_sync_object(struct drm_gem_object *obj,
485b6295f9aSRob Clark 		struct msm_fence_context *fctx, bool exclusive)
486b6295f9aSRob Clark {
487b6295f9aSRob Clark 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
488b6295f9aSRob Clark 	struct reservation_object_list *fobj;
489b6295f9aSRob Clark 	struct fence *fence;
490b6295f9aSRob Clark 	int i, ret;
491b6295f9aSRob Clark 
492b6295f9aSRob Clark 	if (!exclusive) {
493b6295f9aSRob Clark 		/* NOTE: _reserve_shared() must happen before _add_shared_fence(),
494b6295f9aSRob Clark 		 * which makes this a slightly strange place to call it.  OTOH this
495b6295f9aSRob Clark 		 * is a convenient can-fail point to hook it in.  (And similar to
496b6295f9aSRob Clark 		 * how etnaviv and nouveau handle this.)
497b6295f9aSRob Clark 		 */
498b6295f9aSRob Clark 		ret = reservation_object_reserve_shared(msm_obj->resv);
499b6295f9aSRob Clark 		if (ret)
500b6295f9aSRob Clark 			return ret;
501b6295f9aSRob Clark 	}
502b6295f9aSRob Clark 
503b6295f9aSRob Clark 	fobj = reservation_object_get_list(msm_obj->resv);
504b6295f9aSRob Clark 	if (!fobj || (fobj->shared_count == 0)) {
505b6295f9aSRob Clark 		fence = reservation_object_get_excl(msm_obj->resv);
506b6295f9aSRob Clark 		/* don't need to wait on our own fences, since ring is fifo */
507b6295f9aSRob Clark 		if (fence && (fence->context != fctx->context)) {
508b6295f9aSRob Clark 			ret = fence_wait(fence, true);
509b6295f9aSRob Clark 			if (ret)
510b6295f9aSRob Clark 				return ret;
511b6295f9aSRob Clark 		}
512b6295f9aSRob Clark 	}
513b6295f9aSRob Clark 
514b6295f9aSRob Clark 	if (!exclusive || !fobj)
515b6295f9aSRob Clark 		return 0;
516b6295f9aSRob Clark 
517b6295f9aSRob Clark 	for (i = 0; i < fobj->shared_count; i++) {
518b6295f9aSRob Clark 		fence = rcu_dereference_protected(fobj->shared[i],
519b6295f9aSRob Clark 						reservation_object_held(msm_obj->resv));
520b6295f9aSRob Clark 		if (fence->context != fctx->context) {
521b6295f9aSRob Clark 			ret = fence_wait(fence, true);
522b6295f9aSRob Clark 			if (ret)
523b6295f9aSRob Clark 				return ret;
524b6295f9aSRob Clark 		}
525b6295f9aSRob Clark 	}
526b6295f9aSRob Clark 
527b6295f9aSRob Clark 	return 0;
528b6295f9aSRob Clark }
529b6295f9aSRob Clark 
5307198e6b0SRob Clark void msm_gem_move_to_active(struct drm_gem_object *obj,
531b6295f9aSRob Clark 		struct msm_gpu *gpu, bool exclusive, struct fence *fence)
5327198e6b0SRob Clark {
5337198e6b0SRob Clark 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
5344cd33c48SRob Clark 	WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED);
5357198e6b0SRob Clark 	msm_obj->gpu = gpu;
536b6295f9aSRob Clark 	if (exclusive)
537b6295f9aSRob Clark 		reservation_object_add_excl_fence(msm_obj->resv, fence);
538bf6811f3SRob Clark 	else
539b6295f9aSRob Clark 		reservation_object_add_shared_fence(msm_obj->resv, fence);
5407198e6b0SRob Clark 	list_del_init(&msm_obj->mm_list);
5417198e6b0SRob Clark 	list_add_tail(&msm_obj->mm_list, &gpu->active_list);
5427198e6b0SRob Clark }
5437198e6b0SRob Clark 
5447198e6b0SRob Clark void msm_gem_move_to_inactive(struct drm_gem_object *obj)
5457198e6b0SRob Clark {
5467198e6b0SRob Clark 	struct drm_device *dev = obj->dev;
5477198e6b0SRob Clark 	struct msm_drm_private *priv = dev->dev_private;
5487198e6b0SRob Clark 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
5497198e6b0SRob Clark 
5507198e6b0SRob Clark 	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
5517198e6b0SRob Clark 
5527198e6b0SRob Clark 	msm_obj->gpu = NULL;
5537198e6b0SRob Clark 	list_del_init(&msm_obj->mm_list);
5547198e6b0SRob Clark 	list_add_tail(&msm_obj->mm_list, &priv->inactive_list);
5557198e6b0SRob Clark }
5567198e6b0SRob Clark 
557ba00c3f2SRob Clark int msm_gem_cpu_prep(struct drm_gem_object *obj, uint32_t op, ktime_t *timeout)
558ba00c3f2SRob Clark {
559b6295f9aSRob Clark 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
560b6295f9aSRob Clark 	bool write = !!(op & MSM_PREP_WRITE);
561b6295f9aSRob Clark 
562b6295f9aSRob Clark 	if (op & MSM_PREP_NOSYNC) {
563b6295f9aSRob Clark 		if (!reservation_object_test_signaled_rcu(msm_obj->resv, write))
564b6295f9aSRob Clark 			return -EBUSY;
565b6295f9aSRob Clark 	} else {
566b6295f9aSRob Clark 		int ret;
567b6295f9aSRob Clark 
568b6295f9aSRob Clark 		ret = reservation_object_wait_timeout_rcu(msm_obj->resv, write,
569b6295f9aSRob Clark 				true, timeout_to_jiffies(timeout));
570b6295f9aSRob Clark 		if (ret <= 0)
571b6295f9aSRob Clark 			return ret == 0 ? -ETIMEDOUT : ret;
572b6295f9aSRob Clark 	}
573ba00c3f2SRob Clark 
5747198e6b0SRob Clark 	/* TODO cache maintenance */
5757198e6b0SRob Clark 
576b6295f9aSRob Clark 	return 0;
5777198e6b0SRob Clark }
5787198e6b0SRob Clark 
5797198e6b0SRob Clark int msm_gem_cpu_fini(struct drm_gem_object *obj)
5807198e6b0SRob Clark {
5817198e6b0SRob Clark 	/* TODO cache maintenance */
582c8afe684SRob Clark 	return 0;
583c8afe684SRob Clark }
584c8afe684SRob Clark 
585c8afe684SRob Clark #ifdef CONFIG_DEBUG_FS
586b6295f9aSRob Clark static void describe_fence(struct fence *fence, const char *type,
587b6295f9aSRob Clark 		struct seq_file *m)
588b6295f9aSRob Clark {
589b6295f9aSRob Clark 	if (!fence_is_signaled(fence))
590b6295f9aSRob Clark 		seq_printf(m, "\t%9s: %s %s seq %u\n", type,
591b6295f9aSRob Clark 				fence->ops->get_driver_name(fence),
592b6295f9aSRob Clark 				fence->ops->get_timeline_name(fence),
593b6295f9aSRob Clark 				fence->seqno);
594b6295f9aSRob Clark }
595b6295f9aSRob Clark 
596c8afe684SRob Clark void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m)
597c8afe684SRob Clark {
598c8afe684SRob Clark 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
599b6295f9aSRob Clark 	struct reservation_object *robj = msm_obj->resv;
600b6295f9aSRob Clark 	struct reservation_object_list *fobj;
601b6295f9aSRob Clark 	struct fence *fence;
602c8afe684SRob Clark 	uint64_t off = drm_vma_node_start(&obj->vma_node);
6034cd33c48SRob Clark 	const char *madv;
604c8afe684SRob Clark 
605b6295f9aSRob Clark 	WARN_ON(!mutex_is_locked(&obj->dev->struct_mutex));
606b6295f9aSRob Clark 
6074cd33c48SRob Clark 	switch (msm_obj->madv) {
6084cd33c48SRob Clark 	case __MSM_MADV_PURGED:
6094cd33c48SRob Clark 		madv = " purged";
6104cd33c48SRob Clark 		break;
6114cd33c48SRob Clark 	case MSM_MADV_DONTNEED:
6124cd33c48SRob Clark 		madv = " purgeable";
6134cd33c48SRob Clark 		break;
6144cd33c48SRob Clark 	case MSM_MADV_WILLNEED:
6154cd33c48SRob Clark 	default:
6164cd33c48SRob Clark 		madv = "";
6174cd33c48SRob Clark 		break;
6184cd33c48SRob Clark 	}
6194cd33c48SRob Clark 
6204cd33c48SRob Clark 	seq_printf(m, "%08x: %c %2d (%2d) %08llx %p %zu%s\n",
6217198e6b0SRob Clark 			msm_obj->flags, is_active(msm_obj) ? 'A' : 'I',
622bf6811f3SRob Clark 			obj->name, obj->refcount.refcount.counter,
6234cd33c48SRob Clark 			off, msm_obj->vaddr, obj->size, madv);
624b6295f9aSRob Clark 
625b6295f9aSRob Clark 	rcu_read_lock();
626b6295f9aSRob Clark 	fobj = rcu_dereference(robj->fence);
627b6295f9aSRob Clark 	if (fobj) {
628b6295f9aSRob Clark 		unsigned int i, shared_count = fobj->shared_count;
629b6295f9aSRob Clark 
630b6295f9aSRob Clark 		for (i = 0; i < shared_count; i++) {
631b6295f9aSRob Clark 			fence = rcu_dereference(fobj->shared[i]);
632b6295f9aSRob Clark 			describe_fence(fence, "Shared", m);
633b6295f9aSRob Clark 		}
634b6295f9aSRob Clark 	}
635b6295f9aSRob Clark 
636b6295f9aSRob Clark 	fence = rcu_dereference(robj->fence_excl);
637b6295f9aSRob Clark 	if (fence)
638b6295f9aSRob Clark 		describe_fence(fence, "Exclusive", m);
639b6295f9aSRob Clark 	rcu_read_unlock();
640c8afe684SRob Clark }
641c8afe684SRob Clark 
642c8afe684SRob Clark void msm_gem_describe_objects(struct list_head *list, struct seq_file *m)
643c8afe684SRob Clark {
644c8afe684SRob Clark 	struct msm_gem_object *msm_obj;
645c8afe684SRob Clark 	int count = 0;
646c8afe684SRob Clark 	size_t size = 0;
647c8afe684SRob Clark 
648c8afe684SRob Clark 	list_for_each_entry(msm_obj, list, mm_list) {
649c8afe684SRob Clark 		struct drm_gem_object *obj = &msm_obj->base;
650c8afe684SRob Clark 		seq_printf(m, "   ");
651c8afe684SRob Clark 		msm_gem_describe(obj, m);
652c8afe684SRob Clark 		count++;
653c8afe684SRob Clark 		size += obj->size;
654c8afe684SRob Clark 	}
655c8afe684SRob Clark 
656c8afe684SRob Clark 	seq_printf(m, "Total %d objects, %zu bytes\n", count, size);
657c8afe684SRob Clark }
658c8afe684SRob Clark #endif
659c8afe684SRob Clark 
660c8afe684SRob Clark void msm_gem_free_object(struct drm_gem_object *obj)
661c8afe684SRob Clark {
662c8afe684SRob Clark 	struct drm_device *dev = obj->dev;
663c8afe684SRob Clark 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
664c8afe684SRob Clark 
665c8afe684SRob Clark 	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
666c8afe684SRob Clark 
6677198e6b0SRob Clark 	/* object should not be on active list: */
6687198e6b0SRob Clark 	WARN_ON(is_active(msm_obj));
6697198e6b0SRob Clark 
670c8afe684SRob Clark 	list_del(&msm_obj->mm_list);
671c8afe684SRob Clark 
6724fe5f65eSRob Clark 	put_iova(obj);
673c8afe684SRob Clark 
67405b84911SRob Clark 	if (obj->import_attach) {
67505b84911SRob Clark 		if (msm_obj->vaddr)
67605b84911SRob Clark 			dma_buf_vunmap(obj->import_attach->dmabuf, msm_obj->vaddr);
67705b84911SRob Clark 
67805b84911SRob Clark 		/* Don't drop the pages for imported dmabuf, as they are not
67905b84911SRob Clark 		 * ours, just free the array we allocated:
68005b84911SRob Clark 		 */
68105b84911SRob Clark 		if (msm_obj->pages)
68205b84911SRob Clark 			drm_free_large(msm_obj->pages);
68305b84911SRob Clark 
684f28730c8Sjilai wang 		drm_prime_gem_destroy(obj, msm_obj->sgt);
68505b84911SRob Clark 	} else {
686c8afe684SRob Clark 		vunmap(msm_obj->vaddr);
687c8afe684SRob Clark 		put_pages(obj);
68805b84911SRob Clark 	}
689c8afe684SRob Clark 
6907198e6b0SRob Clark 	if (msm_obj->resv == &msm_obj->_resv)
6917198e6b0SRob Clark 		reservation_object_fini(msm_obj->resv);
6927198e6b0SRob Clark 
693c8afe684SRob Clark 	drm_gem_object_release(obj);
694c8afe684SRob Clark 
695c8afe684SRob Clark 	kfree(msm_obj);
696c8afe684SRob Clark }
697c8afe684SRob Clark 
698c8afe684SRob Clark /* convenience method to construct a GEM buffer object, and userspace handle */
699c8afe684SRob Clark int msm_gem_new_handle(struct drm_device *dev, struct drm_file *file,
700c8afe684SRob Clark 		uint32_t size, uint32_t flags, uint32_t *handle)
701c8afe684SRob Clark {
702c8afe684SRob Clark 	struct drm_gem_object *obj;
703c8afe684SRob Clark 	int ret;
704c8afe684SRob Clark 
705c8afe684SRob Clark 	ret = mutex_lock_interruptible(&dev->struct_mutex);
706c8afe684SRob Clark 	if (ret)
707c8afe684SRob Clark 		return ret;
708c8afe684SRob Clark 
709c8afe684SRob Clark 	obj = msm_gem_new(dev, size, flags);
710c8afe684SRob Clark 
711c8afe684SRob Clark 	mutex_unlock(&dev->struct_mutex);
712c8afe684SRob Clark 
713c8afe684SRob Clark 	if (IS_ERR(obj))
714c8afe684SRob Clark 		return PTR_ERR(obj);
715c8afe684SRob Clark 
716c8afe684SRob Clark 	ret = drm_gem_handle_create(file, obj, handle);
717c8afe684SRob Clark 
718c8afe684SRob Clark 	/* drop reference from allocate - handle holds it now */
719c8afe684SRob Clark 	drm_gem_object_unreference_unlocked(obj);
720c8afe684SRob Clark 
721c8afe684SRob Clark 	return ret;
722c8afe684SRob Clark }
723c8afe684SRob Clark 
72405b84911SRob Clark static int msm_gem_new_impl(struct drm_device *dev,
72505b84911SRob Clark 		uint32_t size, uint32_t flags,
72679f0e202SRob Clark 		struct reservation_object *resv,
72705b84911SRob Clark 		struct drm_gem_object **obj)
728c8afe684SRob Clark {
729c8afe684SRob Clark 	struct msm_drm_private *priv = dev->dev_private;
730c8afe684SRob Clark 	struct msm_gem_object *msm_obj;
731871d812aSRob Clark 	unsigned sz;
732072f1f91SRob Clark 	bool use_vram = false;
733c8afe684SRob Clark 
734c8afe684SRob Clark 	switch (flags & MSM_BO_CACHE_MASK) {
735c8afe684SRob Clark 	case MSM_BO_UNCACHED:
736c8afe684SRob Clark 	case MSM_BO_CACHED:
737c8afe684SRob Clark 	case MSM_BO_WC:
738c8afe684SRob Clark 		break;
739c8afe684SRob Clark 	default:
740c8afe684SRob Clark 		dev_err(dev->dev, "invalid cache flag: %x\n",
741c8afe684SRob Clark 				(flags & MSM_BO_CACHE_MASK));
74205b84911SRob Clark 		return -EINVAL;
743c8afe684SRob Clark 	}
744c8afe684SRob Clark 
745871d812aSRob Clark 	if (!iommu_present(&platform_bus_type))
746072f1f91SRob Clark 		use_vram = true;
747072f1f91SRob Clark 	else if ((flags & MSM_BO_STOLEN) && priv->vram.size)
748072f1f91SRob Clark 		use_vram = true;
749072f1f91SRob Clark 
750072f1f91SRob Clark 	if (WARN_ON(use_vram && !priv->vram.size))
751072f1f91SRob Clark 		return -EINVAL;
752072f1f91SRob Clark 
753072f1f91SRob Clark 	sz = sizeof(*msm_obj);
754072f1f91SRob Clark 	if (use_vram)
755871d812aSRob Clark 		sz += sizeof(struct drm_mm_node);
756871d812aSRob Clark 
757871d812aSRob Clark 	msm_obj = kzalloc(sz, GFP_KERNEL);
75805b84911SRob Clark 	if (!msm_obj)
75905b84911SRob Clark 		return -ENOMEM;
760c8afe684SRob Clark 
761072f1f91SRob Clark 	if (use_vram)
762871d812aSRob Clark 		msm_obj->vram_node = (void *)&msm_obj[1];
763871d812aSRob Clark 
764c8afe684SRob Clark 	msm_obj->flags = flags;
7654cd33c48SRob Clark 	msm_obj->madv = MSM_MADV_WILLNEED;
766c8afe684SRob Clark 
76779f0e202SRob Clark 	if (resv) {
76879f0e202SRob Clark 		msm_obj->resv = resv;
76979f0e202SRob Clark 	} else {
7707198e6b0SRob Clark 		msm_obj->resv = &msm_obj->_resv;
7717198e6b0SRob Clark 		reservation_object_init(msm_obj->resv);
77279f0e202SRob Clark 	}
773c8afe684SRob Clark 
7747198e6b0SRob Clark 	INIT_LIST_HEAD(&msm_obj->submit_entry);
775c8afe684SRob Clark 	list_add_tail(&msm_obj->mm_list, &priv->inactive_list);
776c8afe684SRob Clark 
77705b84911SRob Clark 	*obj = &msm_obj->base;
77805b84911SRob Clark 
77905b84911SRob Clark 	return 0;
78005b84911SRob Clark }
78105b84911SRob Clark 
78205b84911SRob Clark struct drm_gem_object *msm_gem_new(struct drm_device *dev,
78305b84911SRob Clark 		uint32_t size, uint32_t flags)
78405b84911SRob Clark {
785871d812aSRob Clark 	struct drm_gem_object *obj = NULL;
78605b84911SRob Clark 	int ret;
78705b84911SRob Clark 
78805b84911SRob Clark 	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
78905b84911SRob Clark 
79005b84911SRob Clark 	size = PAGE_ALIGN(size);
79105b84911SRob Clark 
79279f0e202SRob Clark 	ret = msm_gem_new_impl(dev, size, flags, NULL, &obj);
79305b84911SRob Clark 	if (ret)
79405b84911SRob Clark 		goto fail;
79505b84911SRob Clark 
796072f1f91SRob Clark 	if (use_pages(obj)) {
79705b84911SRob Clark 		ret = drm_gem_object_init(dev, obj, size);
79805b84911SRob Clark 		if (ret)
79905b84911SRob Clark 			goto fail;
800871d812aSRob Clark 	} else {
801871d812aSRob Clark 		drm_gem_private_object_init(dev, obj, size);
802871d812aSRob Clark 	}
80305b84911SRob Clark 
80405b84911SRob Clark 	return obj;
80505b84911SRob Clark 
80605b84911SRob Clark fail:
80705b84911SRob Clark 	if (obj)
8089999f105SRob Clark 		drm_gem_object_unreference(obj);
80905b84911SRob Clark 
81005b84911SRob Clark 	return ERR_PTR(ret);
81105b84911SRob Clark }
81205b84911SRob Clark 
81305b84911SRob Clark struct drm_gem_object *msm_gem_import(struct drm_device *dev,
81479f0e202SRob Clark 		struct dma_buf *dmabuf, struct sg_table *sgt)
81505b84911SRob Clark {
81605b84911SRob Clark 	struct msm_gem_object *msm_obj;
81705b84911SRob Clark 	struct drm_gem_object *obj;
81879f0e202SRob Clark 	uint32_t size;
81905b84911SRob Clark 	int ret, npages;
82005b84911SRob Clark 
821871d812aSRob Clark 	/* if we don't have IOMMU, don't bother pretending we can import: */
822871d812aSRob Clark 	if (!iommu_present(&platform_bus_type)) {
823871d812aSRob Clark 		dev_err(dev->dev, "cannot import without IOMMU\n");
824871d812aSRob Clark 		return ERR_PTR(-EINVAL);
825871d812aSRob Clark 	}
826871d812aSRob Clark 
82779f0e202SRob Clark 	size = PAGE_ALIGN(dmabuf->size);
82805b84911SRob Clark 
82979f0e202SRob Clark 	ret = msm_gem_new_impl(dev, size, MSM_BO_WC, dmabuf->resv, &obj);
83005b84911SRob Clark 	if (ret)
83105b84911SRob Clark 		goto fail;
83205b84911SRob Clark 
83305b84911SRob Clark 	drm_gem_private_object_init(dev, obj, size);
83405b84911SRob Clark 
83505b84911SRob Clark 	npages = size / PAGE_SIZE;
83605b84911SRob Clark 
83705b84911SRob Clark 	msm_obj = to_msm_bo(obj);
83805b84911SRob Clark 	msm_obj->sgt = sgt;
83905b84911SRob Clark 	msm_obj->pages = drm_malloc_ab(npages, sizeof(struct page *));
84005b84911SRob Clark 	if (!msm_obj->pages) {
84105b84911SRob Clark 		ret = -ENOMEM;
84205b84911SRob Clark 		goto fail;
84305b84911SRob Clark 	}
84405b84911SRob Clark 
84505b84911SRob Clark 	ret = drm_prime_sg_to_page_addr_arrays(sgt, msm_obj->pages, NULL, npages);
84605b84911SRob Clark 	if (ret)
84705b84911SRob Clark 		goto fail;
84805b84911SRob Clark 
849c8afe684SRob Clark 	return obj;
850c8afe684SRob Clark 
851c8afe684SRob Clark fail:
852c8afe684SRob Clark 	if (obj)
853c8afe684SRob Clark 		drm_gem_object_unreference_unlocked(obj);
854c8afe684SRob Clark 
855c8afe684SRob Clark 	return ERR_PTR(ret);
856c8afe684SRob Clark }
857