1e4812ab8SDmitry Osipenko // SPDX-License-Identifier: MIT
2e4812ab8SDmitry Osipenko /*
3e4812ab8SDmitry Osipenko * Copyright (C) 2015 Red Hat, Inc.
4e4812ab8SDmitry Osipenko * All Rights Reserved.
5e4812ab8SDmitry Osipenko *
6e4812ab8SDmitry Osipenko * Authors:
7e4812ab8SDmitry Osipenko * Dave Airlie
8e4812ab8SDmitry Osipenko * Alon Levy
9e4812ab8SDmitry Osipenko */
10e4812ab8SDmitry Osipenko
11e4812ab8SDmitry Osipenko #include <linux/dma-fence-unwrap.h>
12e4812ab8SDmitry Osipenko #include <linux/file.h>
13e4812ab8SDmitry Osipenko #include <linux/sync_file.h>
14e4812ab8SDmitry Osipenko #include <linux/uaccess.h>
15e4812ab8SDmitry Osipenko
16e4812ab8SDmitry Osipenko #include <drm/drm_file.h>
17*7cb8d1abSDmitry Osipenko #include <drm/drm_syncobj.h>
18e4812ab8SDmitry Osipenko #include <drm/virtgpu_drm.h>
19e4812ab8SDmitry Osipenko
20e4812ab8SDmitry Osipenko #include "virtgpu_drv.h"
21e4812ab8SDmitry Osipenko
22*7cb8d1abSDmitry Osipenko struct virtio_gpu_submit_post_dep {
23*7cb8d1abSDmitry Osipenko struct drm_syncobj *syncobj;
24*7cb8d1abSDmitry Osipenko struct dma_fence_chain *chain;
25*7cb8d1abSDmitry Osipenko u64 point;
26*7cb8d1abSDmitry Osipenko };
27*7cb8d1abSDmitry Osipenko
28e4812ab8SDmitry Osipenko struct virtio_gpu_submit {
29*7cb8d1abSDmitry Osipenko struct virtio_gpu_submit_post_dep *post_deps;
30*7cb8d1abSDmitry Osipenko unsigned int num_out_syncobjs;
31*7cb8d1abSDmitry Osipenko
32*7cb8d1abSDmitry Osipenko struct drm_syncobj **in_syncobjs;
33*7cb8d1abSDmitry Osipenko unsigned int num_in_syncobjs;
34*7cb8d1abSDmitry Osipenko
35e4812ab8SDmitry Osipenko struct virtio_gpu_object_array *buflist;
36e4812ab8SDmitry Osipenko struct drm_virtgpu_execbuffer *exbuf;
37e4812ab8SDmitry Osipenko struct virtio_gpu_fence *out_fence;
38e4812ab8SDmitry Osipenko struct virtio_gpu_fpriv *vfpriv;
39e4812ab8SDmitry Osipenko struct virtio_gpu_device *vgdev;
40e4812ab8SDmitry Osipenko struct sync_file *sync_file;
41e4812ab8SDmitry Osipenko struct drm_file *file;
42e4812ab8SDmitry Osipenko int out_fence_fd;
43e4812ab8SDmitry Osipenko u64 fence_ctx;
44e4812ab8SDmitry Osipenko u32 ring_idx;
45e4812ab8SDmitry Osipenko void *buf;
46e4812ab8SDmitry Osipenko };
47e4812ab8SDmitry Osipenko
virtio_gpu_do_fence_wait(struct virtio_gpu_submit * submit,struct dma_fence * in_fence)48eba57fb5SDmitry Osipenko static int virtio_gpu_do_fence_wait(struct virtio_gpu_submit *submit,
49e4812ab8SDmitry Osipenko struct dma_fence *in_fence)
50e4812ab8SDmitry Osipenko {
51e4812ab8SDmitry Osipenko u32 context = submit->fence_ctx + submit->ring_idx;
52e4812ab8SDmitry Osipenko
53e4812ab8SDmitry Osipenko if (dma_fence_match_context(in_fence, context))
54e4812ab8SDmitry Osipenko return 0;
55e4812ab8SDmitry Osipenko
56e4812ab8SDmitry Osipenko return dma_fence_wait(in_fence, true);
57e4812ab8SDmitry Osipenko }
58e4812ab8SDmitry Osipenko
virtio_gpu_dma_fence_wait(struct virtio_gpu_submit * submit,struct dma_fence * fence)59eba57fb5SDmitry Osipenko static int virtio_gpu_dma_fence_wait(struct virtio_gpu_submit *submit,
60eba57fb5SDmitry Osipenko struct dma_fence *fence)
61eba57fb5SDmitry Osipenko {
62eba57fb5SDmitry Osipenko struct dma_fence_unwrap itr;
63eba57fb5SDmitry Osipenko struct dma_fence *f;
64eba57fb5SDmitry Osipenko int err;
65eba57fb5SDmitry Osipenko
66eba57fb5SDmitry Osipenko dma_fence_unwrap_for_each(f, &itr, fence) {
67eba57fb5SDmitry Osipenko err = virtio_gpu_do_fence_wait(submit, f);
68eba57fb5SDmitry Osipenko if (err)
69eba57fb5SDmitry Osipenko return err;
70eba57fb5SDmitry Osipenko }
71eba57fb5SDmitry Osipenko
72eba57fb5SDmitry Osipenko return 0;
73eba57fb5SDmitry Osipenko }
74eba57fb5SDmitry Osipenko
virtio_gpu_free_syncobjs(struct drm_syncobj ** syncobjs,u32 nr_syncobjs)75*7cb8d1abSDmitry Osipenko static void virtio_gpu_free_syncobjs(struct drm_syncobj **syncobjs,
76*7cb8d1abSDmitry Osipenko u32 nr_syncobjs)
77*7cb8d1abSDmitry Osipenko {
78*7cb8d1abSDmitry Osipenko u32 i = nr_syncobjs;
79*7cb8d1abSDmitry Osipenko
80*7cb8d1abSDmitry Osipenko while (i--) {
81*7cb8d1abSDmitry Osipenko if (syncobjs[i])
82*7cb8d1abSDmitry Osipenko drm_syncobj_put(syncobjs[i]);
83*7cb8d1abSDmitry Osipenko }
84*7cb8d1abSDmitry Osipenko
85*7cb8d1abSDmitry Osipenko kvfree(syncobjs);
86*7cb8d1abSDmitry Osipenko }
87*7cb8d1abSDmitry Osipenko
88*7cb8d1abSDmitry Osipenko static int
virtio_gpu_parse_deps(struct virtio_gpu_submit * submit)89*7cb8d1abSDmitry Osipenko virtio_gpu_parse_deps(struct virtio_gpu_submit *submit)
90*7cb8d1abSDmitry Osipenko {
91*7cb8d1abSDmitry Osipenko struct drm_virtgpu_execbuffer *exbuf = submit->exbuf;
92*7cb8d1abSDmitry Osipenko struct drm_virtgpu_execbuffer_syncobj syncobj_desc;
93*7cb8d1abSDmitry Osipenko size_t syncobj_stride = exbuf->syncobj_stride;
94*7cb8d1abSDmitry Osipenko u32 num_in_syncobjs = exbuf->num_in_syncobjs;
95*7cb8d1abSDmitry Osipenko struct drm_syncobj **syncobjs;
96*7cb8d1abSDmitry Osipenko int ret = 0, i;
97*7cb8d1abSDmitry Osipenko
98*7cb8d1abSDmitry Osipenko if (!num_in_syncobjs)
99*7cb8d1abSDmitry Osipenko return 0;
100*7cb8d1abSDmitry Osipenko
101*7cb8d1abSDmitry Osipenko /*
102*7cb8d1abSDmitry Osipenko * kvalloc at first tries to allocate memory using kmalloc and
103*7cb8d1abSDmitry Osipenko * falls back to vmalloc only on failure. It also uses __GFP_NOWARN
104*7cb8d1abSDmitry Osipenko * internally for allocations larger than a page size, preventing
105*7cb8d1abSDmitry Osipenko * storm of KMSG warnings.
106*7cb8d1abSDmitry Osipenko */
107*7cb8d1abSDmitry Osipenko syncobjs = kvcalloc(num_in_syncobjs, sizeof(*syncobjs), GFP_KERNEL);
108*7cb8d1abSDmitry Osipenko if (!syncobjs)
109*7cb8d1abSDmitry Osipenko return -ENOMEM;
110*7cb8d1abSDmitry Osipenko
111*7cb8d1abSDmitry Osipenko for (i = 0; i < num_in_syncobjs; i++) {
112*7cb8d1abSDmitry Osipenko u64 address = exbuf->in_syncobjs + i * syncobj_stride;
113*7cb8d1abSDmitry Osipenko struct dma_fence *fence;
114*7cb8d1abSDmitry Osipenko
115*7cb8d1abSDmitry Osipenko memset(&syncobj_desc, 0, sizeof(syncobj_desc));
116*7cb8d1abSDmitry Osipenko
117*7cb8d1abSDmitry Osipenko if (copy_from_user(&syncobj_desc,
118*7cb8d1abSDmitry Osipenko u64_to_user_ptr(address),
119*7cb8d1abSDmitry Osipenko min(syncobj_stride, sizeof(syncobj_desc)))) {
120*7cb8d1abSDmitry Osipenko ret = -EFAULT;
121*7cb8d1abSDmitry Osipenko break;
122*7cb8d1abSDmitry Osipenko }
123*7cb8d1abSDmitry Osipenko
124*7cb8d1abSDmitry Osipenko if (syncobj_desc.flags & ~VIRTGPU_EXECBUF_SYNCOBJ_FLAGS) {
125*7cb8d1abSDmitry Osipenko ret = -EINVAL;
126*7cb8d1abSDmitry Osipenko break;
127*7cb8d1abSDmitry Osipenko }
128*7cb8d1abSDmitry Osipenko
129*7cb8d1abSDmitry Osipenko ret = drm_syncobj_find_fence(submit->file, syncobj_desc.handle,
130*7cb8d1abSDmitry Osipenko syncobj_desc.point, 0, &fence);
131*7cb8d1abSDmitry Osipenko if (ret)
132*7cb8d1abSDmitry Osipenko break;
133*7cb8d1abSDmitry Osipenko
134*7cb8d1abSDmitry Osipenko ret = virtio_gpu_dma_fence_wait(submit, fence);
135*7cb8d1abSDmitry Osipenko
136*7cb8d1abSDmitry Osipenko dma_fence_put(fence);
137*7cb8d1abSDmitry Osipenko if (ret)
138*7cb8d1abSDmitry Osipenko break;
139*7cb8d1abSDmitry Osipenko
140*7cb8d1abSDmitry Osipenko if (syncobj_desc.flags & VIRTGPU_EXECBUF_SYNCOBJ_RESET) {
141*7cb8d1abSDmitry Osipenko syncobjs[i] = drm_syncobj_find(submit->file,
142*7cb8d1abSDmitry Osipenko syncobj_desc.handle);
143*7cb8d1abSDmitry Osipenko if (!syncobjs[i]) {
144*7cb8d1abSDmitry Osipenko ret = -EINVAL;
145*7cb8d1abSDmitry Osipenko break;
146*7cb8d1abSDmitry Osipenko }
147*7cb8d1abSDmitry Osipenko }
148*7cb8d1abSDmitry Osipenko }
149*7cb8d1abSDmitry Osipenko
150*7cb8d1abSDmitry Osipenko if (ret) {
151*7cb8d1abSDmitry Osipenko virtio_gpu_free_syncobjs(syncobjs, i);
152*7cb8d1abSDmitry Osipenko return ret;
153*7cb8d1abSDmitry Osipenko }
154*7cb8d1abSDmitry Osipenko
155*7cb8d1abSDmitry Osipenko submit->num_in_syncobjs = num_in_syncobjs;
156*7cb8d1abSDmitry Osipenko submit->in_syncobjs = syncobjs;
157*7cb8d1abSDmitry Osipenko
158*7cb8d1abSDmitry Osipenko return ret;
159*7cb8d1abSDmitry Osipenko }
160*7cb8d1abSDmitry Osipenko
virtio_gpu_reset_syncobjs(struct drm_syncobj ** syncobjs,u32 nr_syncobjs)161*7cb8d1abSDmitry Osipenko static void virtio_gpu_reset_syncobjs(struct drm_syncobj **syncobjs,
162*7cb8d1abSDmitry Osipenko u32 nr_syncobjs)
163*7cb8d1abSDmitry Osipenko {
164*7cb8d1abSDmitry Osipenko u32 i;
165*7cb8d1abSDmitry Osipenko
166*7cb8d1abSDmitry Osipenko for (i = 0; i < nr_syncobjs; i++) {
167*7cb8d1abSDmitry Osipenko if (syncobjs[i])
168*7cb8d1abSDmitry Osipenko drm_syncobj_replace_fence(syncobjs[i], NULL);
169*7cb8d1abSDmitry Osipenko }
170*7cb8d1abSDmitry Osipenko }
171*7cb8d1abSDmitry Osipenko
172*7cb8d1abSDmitry Osipenko static void
virtio_gpu_free_post_deps(struct virtio_gpu_submit_post_dep * post_deps,u32 nr_syncobjs)173*7cb8d1abSDmitry Osipenko virtio_gpu_free_post_deps(struct virtio_gpu_submit_post_dep *post_deps,
174*7cb8d1abSDmitry Osipenko u32 nr_syncobjs)
175*7cb8d1abSDmitry Osipenko {
176*7cb8d1abSDmitry Osipenko u32 i = nr_syncobjs;
177*7cb8d1abSDmitry Osipenko
178*7cb8d1abSDmitry Osipenko while (i--) {
179*7cb8d1abSDmitry Osipenko kfree(post_deps[i].chain);
180*7cb8d1abSDmitry Osipenko drm_syncobj_put(post_deps[i].syncobj);
181*7cb8d1abSDmitry Osipenko }
182*7cb8d1abSDmitry Osipenko
183*7cb8d1abSDmitry Osipenko kvfree(post_deps);
184*7cb8d1abSDmitry Osipenko }
185*7cb8d1abSDmitry Osipenko
virtio_gpu_parse_post_deps(struct virtio_gpu_submit * submit)186*7cb8d1abSDmitry Osipenko static int virtio_gpu_parse_post_deps(struct virtio_gpu_submit *submit)
187*7cb8d1abSDmitry Osipenko {
188*7cb8d1abSDmitry Osipenko struct drm_virtgpu_execbuffer *exbuf = submit->exbuf;
189*7cb8d1abSDmitry Osipenko struct drm_virtgpu_execbuffer_syncobj syncobj_desc;
190*7cb8d1abSDmitry Osipenko struct virtio_gpu_submit_post_dep *post_deps;
191*7cb8d1abSDmitry Osipenko u32 num_out_syncobjs = exbuf->num_out_syncobjs;
192*7cb8d1abSDmitry Osipenko size_t syncobj_stride = exbuf->syncobj_stride;
193*7cb8d1abSDmitry Osipenko int ret = 0, i;
194*7cb8d1abSDmitry Osipenko
195*7cb8d1abSDmitry Osipenko if (!num_out_syncobjs)
196*7cb8d1abSDmitry Osipenko return 0;
197*7cb8d1abSDmitry Osipenko
198*7cb8d1abSDmitry Osipenko post_deps = kvcalloc(num_out_syncobjs, sizeof(*post_deps), GFP_KERNEL);
199*7cb8d1abSDmitry Osipenko if (!post_deps)
200*7cb8d1abSDmitry Osipenko return -ENOMEM;
201*7cb8d1abSDmitry Osipenko
202*7cb8d1abSDmitry Osipenko for (i = 0; i < num_out_syncobjs; i++) {
203*7cb8d1abSDmitry Osipenko u64 address = exbuf->out_syncobjs + i * syncobj_stride;
204*7cb8d1abSDmitry Osipenko
205*7cb8d1abSDmitry Osipenko memset(&syncobj_desc, 0, sizeof(syncobj_desc));
206*7cb8d1abSDmitry Osipenko
207*7cb8d1abSDmitry Osipenko if (copy_from_user(&syncobj_desc,
208*7cb8d1abSDmitry Osipenko u64_to_user_ptr(address),
209*7cb8d1abSDmitry Osipenko min(syncobj_stride, sizeof(syncobj_desc)))) {
210*7cb8d1abSDmitry Osipenko ret = -EFAULT;
211*7cb8d1abSDmitry Osipenko break;
212*7cb8d1abSDmitry Osipenko }
213*7cb8d1abSDmitry Osipenko
214*7cb8d1abSDmitry Osipenko post_deps[i].point = syncobj_desc.point;
215*7cb8d1abSDmitry Osipenko
216*7cb8d1abSDmitry Osipenko if (syncobj_desc.flags) {
217*7cb8d1abSDmitry Osipenko ret = -EINVAL;
218*7cb8d1abSDmitry Osipenko break;
219*7cb8d1abSDmitry Osipenko }
220*7cb8d1abSDmitry Osipenko
221*7cb8d1abSDmitry Osipenko if (syncobj_desc.point) {
222*7cb8d1abSDmitry Osipenko post_deps[i].chain = dma_fence_chain_alloc();
223*7cb8d1abSDmitry Osipenko if (!post_deps[i].chain) {
224*7cb8d1abSDmitry Osipenko ret = -ENOMEM;
225*7cb8d1abSDmitry Osipenko break;
226*7cb8d1abSDmitry Osipenko }
227*7cb8d1abSDmitry Osipenko }
228*7cb8d1abSDmitry Osipenko
229*7cb8d1abSDmitry Osipenko post_deps[i].syncobj = drm_syncobj_find(submit->file,
230*7cb8d1abSDmitry Osipenko syncobj_desc.handle);
231*7cb8d1abSDmitry Osipenko if (!post_deps[i].syncobj) {
232*7cb8d1abSDmitry Osipenko kfree(post_deps[i].chain);
233*7cb8d1abSDmitry Osipenko ret = -EINVAL;
234*7cb8d1abSDmitry Osipenko break;
235*7cb8d1abSDmitry Osipenko }
236*7cb8d1abSDmitry Osipenko }
237*7cb8d1abSDmitry Osipenko
238*7cb8d1abSDmitry Osipenko if (ret) {
239*7cb8d1abSDmitry Osipenko virtio_gpu_free_post_deps(post_deps, i);
240*7cb8d1abSDmitry Osipenko return ret;
241*7cb8d1abSDmitry Osipenko }
242*7cb8d1abSDmitry Osipenko
243*7cb8d1abSDmitry Osipenko submit->num_out_syncobjs = num_out_syncobjs;
244*7cb8d1abSDmitry Osipenko submit->post_deps = post_deps;
245*7cb8d1abSDmitry Osipenko
246*7cb8d1abSDmitry Osipenko return 0;
247*7cb8d1abSDmitry Osipenko }
248*7cb8d1abSDmitry Osipenko
249*7cb8d1abSDmitry Osipenko static void
virtio_gpu_process_post_deps(struct virtio_gpu_submit * submit)250*7cb8d1abSDmitry Osipenko virtio_gpu_process_post_deps(struct virtio_gpu_submit *submit)
251*7cb8d1abSDmitry Osipenko {
252*7cb8d1abSDmitry Osipenko struct virtio_gpu_submit_post_dep *post_deps = submit->post_deps;
253*7cb8d1abSDmitry Osipenko
254*7cb8d1abSDmitry Osipenko if (post_deps) {
255*7cb8d1abSDmitry Osipenko struct dma_fence *fence = &submit->out_fence->f;
256*7cb8d1abSDmitry Osipenko u32 i;
257*7cb8d1abSDmitry Osipenko
258*7cb8d1abSDmitry Osipenko for (i = 0; i < submit->num_out_syncobjs; i++) {
259*7cb8d1abSDmitry Osipenko if (post_deps[i].chain) {
260*7cb8d1abSDmitry Osipenko drm_syncobj_add_point(post_deps[i].syncobj,
261*7cb8d1abSDmitry Osipenko post_deps[i].chain,
262*7cb8d1abSDmitry Osipenko fence, post_deps[i].point);
263*7cb8d1abSDmitry Osipenko post_deps[i].chain = NULL;
264*7cb8d1abSDmitry Osipenko } else {
265*7cb8d1abSDmitry Osipenko drm_syncobj_replace_fence(post_deps[i].syncobj,
266*7cb8d1abSDmitry Osipenko fence);
267*7cb8d1abSDmitry Osipenko }
268*7cb8d1abSDmitry Osipenko }
269*7cb8d1abSDmitry Osipenko }
270*7cb8d1abSDmitry Osipenko }
271*7cb8d1abSDmitry Osipenko
virtio_gpu_fence_event_create(struct drm_device * dev,struct drm_file * file,struct virtio_gpu_fence * fence,u32 ring_idx)272e4812ab8SDmitry Osipenko static int virtio_gpu_fence_event_create(struct drm_device *dev,
273e4812ab8SDmitry Osipenko struct drm_file *file,
274e4812ab8SDmitry Osipenko struct virtio_gpu_fence *fence,
275e4812ab8SDmitry Osipenko u32 ring_idx)
276e4812ab8SDmitry Osipenko {
277e4812ab8SDmitry Osipenko struct virtio_gpu_fence_event *e = NULL;
278e4812ab8SDmitry Osipenko int ret;
279e4812ab8SDmitry Osipenko
280e4812ab8SDmitry Osipenko e = kzalloc(sizeof(*e), GFP_KERNEL);
281e4812ab8SDmitry Osipenko if (!e)
282e4812ab8SDmitry Osipenko return -ENOMEM;
283e4812ab8SDmitry Osipenko
284e4812ab8SDmitry Osipenko e->event.type = VIRTGPU_EVENT_FENCE_SIGNALED;
285e4812ab8SDmitry Osipenko e->event.length = sizeof(e->event);
286e4812ab8SDmitry Osipenko
287e4812ab8SDmitry Osipenko ret = drm_event_reserve_init(dev, file, &e->base, &e->event);
288e4812ab8SDmitry Osipenko if (ret) {
289e4812ab8SDmitry Osipenko kfree(e);
290e4812ab8SDmitry Osipenko return ret;
291e4812ab8SDmitry Osipenko }
292e4812ab8SDmitry Osipenko
293e4812ab8SDmitry Osipenko fence->e = e;
294e4812ab8SDmitry Osipenko
295e4812ab8SDmitry Osipenko return 0;
296e4812ab8SDmitry Osipenko }
297e4812ab8SDmitry Osipenko
virtio_gpu_init_submit_buflist(struct virtio_gpu_submit * submit)298e4812ab8SDmitry Osipenko static int virtio_gpu_init_submit_buflist(struct virtio_gpu_submit *submit)
299e4812ab8SDmitry Osipenko {
300e4812ab8SDmitry Osipenko struct drm_virtgpu_execbuffer *exbuf = submit->exbuf;
301e4812ab8SDmitry Osipenko u32 *bo_handles;
302e4812ab8SDmitry Osipenko
303e4812ab8SDmitry Osipenko if (!exbuf->num_bo_handles)
304e4812ab8SDmitry Osipenko return 0;
305e4812ab8SDmitry Osipenko
306e4812ab8SDmitry Osipenko bo_handles = kvmalloc_array(exbuf->num_bo_handles, sizeof(*bo_handles),
307e4812ab8SDmitry Osipenko GFP_KERNEL);
308e4812ab8SDmitry Osipenko if (!bo_handles)
309e4812ab8SDmitry Osipenko return -ENOMEM;
310e4812ab8SDmitry Osipenko
311e4812ab8SDmitry Osipenko if (copy_from_user(bo_handles, u64_to_user_ptr(exbuf->bo_handles),
312e4812ab8SDmitry Osipenko exbuf->num_bo_handles * sizeof(*bo_handles))) {
313e4812ab8SDmitry Osipenko kvfree(bo_handles);
314e4812ab8SDmitry Osipenko return -EFAULT;
315e4812ab8SDmitry Osipenko }
316e4812ab8SDmitry Osipenko
317e4812ab8SDmitry Osipenko submit->buflist = virtio_gpu_array_from_handles(submit->file, bo_handles,
318e4812ab8SDmitry Osipenko exbuf->num_bo_handles);
319e4812ab8SDmitry Osipenko if (!submit->buflist) {
320e4812ab8SDmitry Osipenko kvfree(bo_handles);
321e4812ab8SDmitry Osipenko return -ENOENT;
322e4812ab8SDmitry Osipenko }
323e4812ab8SDmitry Osipenko
324e4812ab8SDmitry Osipenko kvfree(bo_handles);
325e4812ab8SDmitry Osipenko
326e4812ab8SDmitry Osipenko return 0;
327e4812ab8SDmitry Osipenko }
328e4812ab8SDmitry Osipenko
virtio_gpu_cleanup_submit(struct virtio_gpu_submit * submit)329e4812ab8SDmitry Osipenko static void virtio_gpu_cleanup_submit(struct virtio_gpu_submit *submit)
330e4812ab8SDmitry Osipenko {
331*7cb8d1abSDmitry Osipenko virtio_gpu_reset_syncobjs(submit->in_syncobjs, submit->num_in_syncobjs);
332*7cb8d1abSDmitry Osipenko virtio_gpu_free_syncobjs(submit->in_syncobjs, submit->num_in_syncobjs);
333*7cb8d1abSDmitry Osipenko virtio_gpu_free_post_deps(submit->post_deps, submit->num_out_syncobjs);
334*7cb8d1abSDmitry Osipenko
335e4812ab8SDmitry Osipenko if (!IS_ERR(submit->buf))
336e4812ab8SDmitry Osipenko kvfree(submit->buf);
337e4812ab8SDmitry Osipenko
338e4812ab8SDmitry Osipenko if (submit->buflist)
339e4812ab8SDmitry Osipenko virtio_gpu_array_put_free(submit->buflist);
340e4812ab8SDmitry Osipenko
341e4812ab8SDmitry Osipenko if (submit->out_fence_fd >= 0)
342e4812ab8SDmitry Osipenko put_unused_fd(submit->out_fence_fd);
343e4812ab8SDmitry Osipenko
344e4812ab8SDmitry Osipenko if (submit->out_fence)
345e4812ab8SDmitry Osipenko dma_fence_put(&submit->out_fence->f);
346e4812ab8SDmitry Osipenko
347e4812ab8SDmitry Osipenko if (submit->sync_file)
348e4812ab8SDmitry Osipenko fput(submit->sync_file->file);
349e4812ab8SDmitry Osipenko }
350e4812ab8SDmitry Osipenko
virtio_gpu_submit(struct virtio_gpu_submit * submit)351e4812ab8SDmitry Osipenko static void virtio_gpu_submit(struct virtio_gpu_submit *submit)
352e4812ab8SDmitry Osipenko {
353e4812ab8SDmitry Osipenko virtio_gpu_cmd_submit(submit->vgdev, submit->buf, submit->exbuf->size,
354e4812ab8SDmitry Osipenko submit->vfpriv->ctx_id, submit->buflist,
355e4812ab8SDmitry Osipenko submit->out_fence);
356e4812ab8SDmitry Osipenko virtio_gpu_notify(submit->vgdev);
357e4812ab8SDmitry Osipenko }
358e4812ab8SDmitry Osipenko
virtio_gpu_complete_submit(struct virtio_gpu_submit * submit)359e4812ab8SDmitry Osipenko static void virtio_gpu_complete_submit(struct virtio_gpu_submit *submit)
360e4812ab8SDmitry Osipenko {
361e4812ab8SDmitry Osipenko submit->buf = NULL;
362e4812ab8SDmitry Osipenko submit->buflist = NULL;
363e4812ab8SDmitry Osipenko submit->sync_file = NULL;
364e4812ab8SDmitry Osipenko submit->out_fence_fd = -1;
365e4812ab8SDmitry Osipenko }
366e4812ab8SDmitry Osipenko
virtio_gpu_init_submit(struct virtio_gpu_submit * submit,struct drm_virtgpu_execbuffer * exbuf,struct drm_device * dev,struct drm_file * file,u64 fence_ctx,u32 ring_idx)367e4812ab8SDmitry Osipenko static int virtio_gpu_init_submit(struct virtio_gpu_submit *submit,
368e4812ab8SDmitry Osipenko struct drm_virtgpu_execbuffer *exbuf,
369e4812ab8SDmitry Osipenko struct drm_device *dev,
370e4812ab8SDmitry Osipenko struct drm_file *file,
371e4812ab8SDmitry Osipenko u64 fence_ctx, u32 ring_idx)
372e4812ab8SDmitry Osipenko {
373e4812ab8SDmitry Osipenko struct virtio_gpu_fpriv *vfpriv = file->driver_priv;
374e4812ab8SDmitry Osipenko struct virtio_gpu_device *vgdev = dev->dev_private;
375e4812ab8SDmitry Osipenko struct virtio_gpu_fence *out_fence;
37670d1ace5SGurchetan Singh bool drm_fence_event;
377e4812ab8SDmitry Osipenko int err;
378e4812ab8SDmitry Osipenko
379e4812ab8SDmitry Osipenko memset(submit, 0, sizeof(*submit));
380e4812ab8SDmitry Osipenko
38170d1ace5SGurchetan Singh if ((exbuf->flags & VIRTGPU_EXECBUF_RING_IDX) &&
38270d1ace5SGurchetan Singh (vfpriv->ring_idx_mask & BIT_ULL(ring_idx)))
38370d1ace5SGurchetan Singh drm_fence_event = true;
38470d1ace5SGurchetan Singh else
38570d1ace5SGurchetan Singh drm_fence_event = false;
386e4812ab8SDmitry Osipenko
38770d1ace5SGurchetan Singh if ((exbuf->flags & VIRTGPU_EXECBUF_FENCE_FD_OUT) ||
388*7cb8d1abSDmitry Osipenko exbuf->num_out_syncobjs ||
38970d1ace5SGurchetan Singh exbuf->num_bo_handles ||
39070d1ace5SGurchetan Singh drm_fence_event)
39170d1ace5SGurchetan Singh out_fence = virtio_gpu_fence_alloc(vgdev, fence_ctx, ring_idx);
39270d1ace5SGurchetan Singh else
39370d1ace5SGurchetan Singh out_fence = NULL;
39470d1ace5SGurchetan Singh
39570d1ace5SGurchetan Singh if (drm_fence_event) {
396e4812ab8SDmitry Osipenko err = virtio_gpu_fence_event_create(dev, file, out_fence, ring_idx);
397e4812ab8SDmitry Osipenko if (err) {
398e4812ab8SDmitry Osipenko dma_fence_put(&out_fence->f);
399e4812ab8SDmitry Osipenko return err;
400e4812ab8SDmitry Osipenko }
40170d1ace5SGurchetan Singh }
402e4812ab8SDmitry Osipenko
403e4812ab8SDmitry Osipenko submit->out_fence = out_fence;
404e4812ab8SDmitry Osipenko submit->fence_ctx = fence_ctx;
405e4812ab8SDmitry Osipenko submit->ring_idx = ring_idx;
406e4812ab8SDmitry Osipenko submit->out_fence_fd = -1;
407e4812ab8SDmitry Osipenko submit->vfpriv = vfpriv;
408e4812ab8SDmitry Osipenko submit->vgdev = vgdev;
409e4812ab8SDmitry Osipenko submit->exbuf = exbuf;
410e4812ab8SDmitry Osipenko submit->file = file;
411e4812ab8SDmitry Osipenko
412e4812ab8SDmitry Osipenko err = virtio_gpu_init_submit_buflist(submit);
413e4812ab8SDmitry Osipenko if (err)
414e4812ab8SDmitry Osipenko return err;
415e4812ab8SDmitry Osipenko
416e4812ab8SDmitry Osipenko submit->buf = vmemdup_user(u64_to_user_ptr(exbuf->command), exbuf->size);
417e4812ab8SDmitry Osipenko if (IS_ERR(submit->buf))
418e4812ab8SDmitry Osipenko return PTR_ERR(submit->buf);
419e4812ab8SDmitry Osipenko
420e4812ab8SDmitry Osipenko if (exbuf->flags & VIRTGPU_EXECBUF_FENCE_FD_OUT) {
421e4812ab8SDmitry Osipenko err = get_unused_fd_flags(O_CLOEXEC);
422e4812ab8SDmitry Osipenko if (err < 0)
423e4812ab8SDmitry Osipenko return err;
424e4812ab8SDmitry Osipenko
425e4812ab8SDmitry Osipenko submit->out_fence_fd = err;
426e4812ab8SDmitry Osipenko
427e4812ab8SDmitry Osipenko submit->sync_file = sync_file_create(&out_fence->f);
428e4812ab8SDmitry Osipenko if (!submit->sync_file)
429e4812ab8SDmitry Osipenko return -ENOMEM;
430e4812ab8SDmitry Osipenko }
431e4812ab8SDmitry Osipenko
432e4812ab8SDmitry Osipenko return 0;
433e4812ab8SDmitry Osipenko }
434e4812ab8SDmitry Osipenko
virtio_gpu_wait_in_fence(struct virtio_gpu_submit * submit)435e4812ab8SDmitry Osipenko static int virtio_gpu_wait_in_fence(struct virtio_gpu_submit *submit)
436e4812ab8SDmitry Osipenko {
437e4812ab8SDmitry Osipenko int ret = 0;
438e4812ab8SDmitry Osipenko
439e4812ab8SDmitry Osipenko if (submit->exbuf->flags & VIRTGPU_EXECBUF_FENCE_FD_IN) {
440e4812ab8SDmitry Osipenko struct dma_fence *in_fence =
441e4812ab8SDmitry Osipenko sync_file_get_fence(submit->exbuf->fence_fd);
442e4812ab8SDmitry Osipenko if (!in_fence)
443e4812ab8SDmitry Osipenko return -EINVAL;
444e4812ab8SDmitry Osipenko
445e4812ab8SDmitry Osipenko /*
446e4812ab8SDmitry Osipenko * Wait if the fence is from a foreign context, or if the
447e4812ab8SDmitry Osipenko * fence array contains any fence from a foreign context.
448e4812ab8SDmitry Osipenko */
449e4812ab8SDmitry Osipenko ret = virtio_gpu_dma_fence_wait(submit, in_fence);
450e4812ab8SDmitry Osipenko
451e4812ab8SDmitry Osipenko dma_fence_put(in_fence);
452e4812ab8SDmitry Osipenko }
453e4812ab8SDmitry Osipenko
454e4812ab8SDmitry Osipenko return ret;
455e4812ab8SDmitry Osipenko }
456e4812ab8SDmitry Osipenko
virtio_gpu_install_out_fence_fd(struct virtio_gpu_submit * submit)457e4812ab8SDmitry Osipenko static void virtio_gpu_install_out_fence_fd(struct virtio_gpu_submit *submit)
458e4812ab8SDmitry Osipenko {
459e4812ab8SDmitry Osipenko if (submit->sync_file) {
460e4812ab8SDmitry Osipenko submit->exbuf->fence_fd = submit->out_fence_fd;
461e4812ab8SDmitry Osipenko fd_install(submit->out_fence_fd, submit->sync_file->file);
462e4812ab8SDmitry Osipenko }
463e4812ab8SDmitry Osipenko }
464e4812ab8SDmitry Osipenko
virtio_gpu_lock_buflist(struct virtio_gpu_submit * submit)465e4812ab8SDmitry Osipenko static int virtio_gpu_lock_buflist(struct virtio_gpu_submit *submit)
466e4812ab8SDmitry Osipenko {
467e4812ab8SDmitry Osipenko if (submit->buflist)
468e4812ab8SDmitry Osipenko return virtio_gpu_array_lock_resv(submit->buflist);
469e4812ab8SDmitry Osipenko
470e4812ab8SDmitry Osipenko return 0;
471e4812ab8SDmitry Osipenko }
472e4812ab8SDmitry Osipenko
virtio_gpu_execbuffer_ioctl(struct drm_device * dev,void * data,struct drm_file * file)473e4812ab8SDmitry Osipenko int virtio_gpu_execbuffer_ioctl(struct drm_device *dev, void *data,
474e4812ab8SDmitry Osipenko struct drm_file *file)
475e4812ab8SDmitry Osipenko {
476e4812ab8SDmitry Osipenko struct virtio_gpu_device *vgdev = dev->dev_private;
477e4812ab8SDmitry Osipenko struct virtio_gpu_fpriv *vfpriv = file->driver_priv;
478e4812ab8SDmitry Osipenko u64 fence_ctx = vgdev->fence_drv.context;
479e4812ab8SDmitry Osipenko struct drm_virtgpu_execbuffer *exbuf = data;
480e4812ab8SDmitry Osipenko struct virtio_gpu_submit submit;
481e4812ab8SDmitry Osipenko u32 ring_idx = 0;
482e4812ab8SDmitry Osipenko int ret = -EINVAL;
483e4812ab8SDmitry Osipenko
484e4812ab8SDmitry Osipenko if (!vgdev->has_virgl_3d)
485e4812ab8SDmitry Osipenko return -ENOSYS;
486e4812ab8SDmitry Osipenko
487e4812ab8SDmitry Osipenko if (exbuf->flags & ~VIRTGPU_EXECBUF_FLAGS)
488e4812ab8SDmitry Osipenko return ret;
489e4812ab8SDmitry Osipenko
490e4812ab8SDmitry Osipenko if (exbuf->flags & VIRTGPU_EXECBUF_RING_IDX) {
491e4812ab8SDmitry Osipenko if (exbuf->ring_idx >= vfpriv->num_rings)
492e4812ab8SDmitry Osipenko return ret;
493e4812ab8SDmitry Osipenko
494e4812ab8SDmitry Osipenko if (!vfpriv->base_fence_ctx)
495e4812ab8SDmitry Osipenko return ret;
496e4812ab8SDmitry Osipenko
497e4812ab8SDmitry Osipenko fence_ctx = vfpriv->base_fence_ctx;
498e4812ab8SDmitry Osipenko ring_idx = exbuf->ring_idx;
499e4812ab8SDmitry Osipenko }
500e4812ab8SDmitry Osipenko
501e4812ab8SDmitry Osipenko virtio_gpu_create_context(dev, file);
502e4812ab8SDmitry Osipenko
503e4812ab8SDmitry Osipenko ret = virtio_gpu_init_submit(&submit, exbuf, dev, file,
504e4812ab8SDmitry Osipenko fence_ctx, ring_idx);
505e4812ab8SDmitry Osipenko if (ret)
506e4812ab8SDmitry Osipenko goto cleanup;
507e4812ab8SDmitry Osipenko
508*7cb8d1abSDmitry Osipenko ret = virtio_gpu_parse_post_deps(&submit);
509*7cb8d1abSDmitry Osipenko if (ret)
510*7cb8d1abSDmitry Osipenko goto cleanup;
511*7cb8d1abSDmitry Osipenko
512*7cb8d1abSDmitry Osipenko ret = virtio_gpu_parse_deps(&submit);
513*7cb8d1abSDmitry Osipenko if (ret)
514*7cb8d1abSDmitry Osipenko goto cleanup;
515*7cb8d1abSDmitry Osipenko
516e4812ab8SDmitry Osipenko /*
517e4812ab8SDmitry Osipenko * Await in-fences in the end of the job submission path to
518e4812ab8SDmitry Osipenko * optimize the path by proceeding directly to the submission
519e4812ab8SDmitry Osipenko * to virtio after the waits.
520e4812ab8SDmitry Osipenko */
521e4812ab8SDmitry Osipenko ret = virtio_gpu_wait_in_fence(&submit);
522e4812ab8SDmitry Osipenko if (ret)
523e4812ab8SDmitry Osipenko goto cleanup;
524e4812ab8SDmitry Osipenko
525e4812ab8SDmitry Osipenko ret = virtio_gpu_lock_buflist(&submit);
526e4812ab8SDmitry Osipenko if (ret)
527e4812ab8SDmitry Osipenko goto cleanup;
528e4812ab8SDmitry Osipenko
529e4812ab8SDmitry Osipenko virtio_gpu_submit(&submit);
530e4812ab8SDmitry Osipenko
531e4812ab8SDmitry Osipenko /*
532e4812ab8SDmitry Osipenko * Set up usr-out data after submitting the job to optimize
533e4812ab8SDmitry Osipenko * the job submission path.
534e4812ab8SDmitry Osipenko */
535e4812ab8SDmitry Osipenko virtio_gpu_install_out_fence_fd(&submit);
536*7cb8d1abSDmitry Osipenko virtio_gpu_process_post_deps(&submit);
537e4812ab8SDmitry Osipenko virtio_gpu_complete_submit(&submit);
538e4812ab8SDmitry Osipenko cleanup:
539e4812ab8SDmitry Osipenko virtio_gpu_cleanup_submit(&submit);
540e4812ab8SDmitry Osipenko
541e4812ab8SDmitry Osipenko return ret;
542e4812ab8SDmitry Osipenko }
543