xref: /openbmc/linux/drivers/staging/media/ipu3/ipu3.c (revision dbdc7237)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2017 - 2018 Intel Corporation
4  * Copyright 2017 Google LLC
5  *
6  * Based on Intel IPU4 driver.
7  *
8  */
9 
10 #include <linux/delay.h>
11 #include <linux/interrupt.h>
12 #include <linux/module.h>
13 #include <linux/pm_runtime.h>
14 
15 #include "ipu3.h"
16 #include "ipu3-dmamap.h"
17 #include "ipu3-mmu.h"
18 
19 #define IMGU_PCI_ID			0x1919
20 #define IMGU_PCI_BAR			0
21 #define IMGU_DMA_MASK			DMA_BIT_MASK(39)
22 #define IMGU_MAX_QUEUE_DEPTH		(2 + 2)
23 
24 /*
25  * pre-allocated buffer size for IMGU dummy buffers. Those
26  * values should be tuned to big enough to avoid buffer
27  * re-allocation when streaming to lower streaming latency.
28  */
29 #define CSS_QUEUE_IN_BUF_SIZE		0
30 #define CSS_QUEUE_PARAMS_BUF_SIZE	0
31 #define CSS_QUEUE_OUT_BUF_SIZE		(4160 * 3120 * 12 / 8)
32 #define CSS_QUEUE_VF_BUF_SIZE		(1920 * 1080 * 12 / 8)
33 #define CSS_QUEUE_STAT_3A_BUF_SIZE	sizeof(struct ipu3_uapi_stats_3a)
34 
35 static const size_t css_queue_buf_size_map[IPU3_CSS_QUEUES] = {
36 	[IPU3_CSS_QUEUE_IN] = CSS_QUEUE_IN_BUF_SIZE,
37 	[IPU3_CSS_QUEUE_PARAMS] = CSS_QUEUE_PARAMS_BUF_SIZE,
38 	[IPU3_CSS_QUEUE_OUT] = CSS_QUEUE_OUT_BUF_SIZE,
39 	[IPU3_CSS_QUEUE_VF] = CSS_QUEUE_VF_BUF_SIZE,
40 	[IPU3_CSS_QUEUE_STAT_3A] = CSS_QUEUE_STAT_3A_BUF_SIZE,
41 };
42 
43 static const struct imgu_node_mapping imgu_node_map[IMGU_NODE_NUM] = {
44 	[IMGU_NODE_IN] = {IPU3_CSS_QUEUE_IN, "input"},
45 	[IMGU_NODE_PARAMS] = {IPU3_CSS_QUEUE_PARAMS, "parameters"},
46 	[IMGU_NODE_OUT] = {IPU3_CSS_QUEUE_OUT, "output"},
47 	[IMGU_NODE_VF] = {IPU3_CSS_QUEUE_VF, "viewfinder"},
48 	[IMGU_NODE_STAT_3A] = {IPU3_CSS_QUEUE_STAT_3A, "3a stat"},
49 };
50 
imgu_node_to_queue(unsigned int node)51 unsigned int imgu_node_to_queue(unsigned int node)
52 {
53 	return imgu_node_map[node].css_queue;
54 }
55 
imgu_map_node(struct imgu_device * imgu,unsigned int css_queue)56 unsigned int imgu_map_node(struct imgu_device *imgu, unsigned int css_queue)
57 {
58 	unsigned int i;
59 
60 	for (i = 0; i < IMGU_NODE_NUM; i++)
61 		if (imgu_node_map[i].css_queue == css_queue)
62 			break;
63 
64 	return i;
65 }
66 
67 /**************** Dummy buffers ****************/
68 
imgu_dummybufs_cleanup(struct imgu_device * imgu,unsigned int pipe)69 static void imgu_dummybufs_cleanup(struct imgu_device *imgu, unsigned int pipe)
70 {
71 	unsigned int i;
72 	struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe];
73 
74 	for (i = 0; i < IPU3_CSS_QUEUES; i++)
75 		imgu_dmamap_free(imgu,
76 				 &imgu_pipe->queues[i].dmap);
77 }
78 
imgu_dummybufs_preallocate(struct imgu_device * imgu,unsigned int pipe)79 static int imgu_dummybufs_preallocate(struct imgu_device *imgu,
80 				      unsigned int pipe)
81 {
82 	unsigned int i;
83 	size_t size;
84 	struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe];
85 
86 	for (i = 0; i < IPU3_CSS_QUEUES; i++) {
87 		size = css_queue_buf_size_map[i];
88 		/*
89 		 * Do not enable dummy buffers for master queue,
90 		 * always require that real buffers from user are
91 		 * available.
92 		 */
93 		if (i == IMGU_QUEUE_MASTER || size == 0)
94 			continue;
95 
96 		if (!imgu_dmamap_alloc(imgu,
97 				       &imgu_pipe->queues[i].dmap, size)) {
98 			imgu_dummybufs_cleanup(imgu, pipe);
99 			return -ENOMEM;
100 		}
101 	}
102 
103 	return 0;
104 }
105 
imgu_dummybufs_init(struct imgu_device * imgu,unsigned int pipe)106 static int imgu_dummybufs_init(struct imgu_device *imgu, unsigned int pipe)
107 {
108 	const struct v4l2_pix_format_mplane *mpix;
109 	const struct v4l2_meta_format	*meta;
110 	unsigned int i, k, node;
111 	size_t size;
112 	struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe];
113 
114 	/* Allocate a dummy buffer for each queue where buffer is optional */
115 	for (i = 0; i < IPU3_CSS_QUEUES; i++) {
116 		node = imgu_map_node(imgu, i);
117 		if (!imgu_pipe->queue_enabled[node] || i == IMGU_QUEUE_MASTER)
118 			continue;
119 
120 		if (!imgu_pipe->nodes[IMGU_NODE_VF].enabled &&
121 		    i == IPU3_CSS_QUEUE_VF)
122 			/*
123 			 * Do not enable dummy buffers for VF if it is not
124 			 * requested by the user.
125 			 */
126 			continue;
127 
128 		meta = &imgu_pipe->nodes[node].vdev_fmt.fmt.meta;
129 		mpix = &imgu_pipe->nodes[node].vdev_fmt.fmt.pix_mp;
130 
131 		if (node == IMGU_NODE_STAT_3A || node == IMGU_NODE_PARAMS)
132 			size = meta->buffersize;
133 		else
134 			size = mpix->plane_fmt[0].sizeimage;
135 
136 		if (imgu_css_dma_buffer_resize(imgu,
137 					       &imgu_pipe->queues[i].dmap,
138 					       size)) {
139 			imgu_dummybufs_cleanup(imgu, pipe);
140 			return -ENOMEM;
141 		}
142 
143 		for (k = 0; k < IMGU_MAX_QUEUE_DEPTH; k++)
144 			imgu_css_buf_init(&imgu_pipe->queues[i].dummybufs[k], i,
145 					  imgu_pipe->queues[i].dmap.daddr);
146 	}
147 
148 	return 0;
149 }
150 
151 /* May be called from atomic context */
imgu_dummybufs_get(struct imgu_device * imgu,int queue,unsigned int pipe)152 static struct imgu_css_buffer *imgu_dummybufs_get(struct imgu_device *imgu,
153 						   int queue, unsigned int pipe)
154 {
155 	unsigned int i;
156 	struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe];
157 
158 	/* dummybufs are not allocated for master q */
159 	if (queue == IPU3_CSS_QUEUE_IN)
160 		return NULL;
161 
162 	if (WARN_ON(!imgu_pipe->queues[queue].dmap.vaddr))
163 		/* Buffer should not be allocated here */
164 		return NULL;
165 
166 	for (i = 0; i < IMGU_MAX_QUEUE_DEPTH; i++)
167 		if (imgu_css_buf_state(&imgu_pipe->queues[queue].dummybufs[i]) !=
168 			IPU3_CSS_BUFFER_QUEUED)
169 			break;
170 
171 	if (i == IMGU_MAX_QUEUE_DEPTH)
172 		return NULL;
173 
174 	imgu_css_buf_init(&imgu_pipe->queues[queue].dummybufs[i], queue,
175 			  imgu_pipe->queues[queue].dmap.daddr);
176 
177 	return &imgu_pipe->queues[queue].dummybufs[i];
178 }
179 
180 /* Check if given buffer is a dummy buffer */
imgu_dummybufs_check(struct imgu_device * imgu,struct imgu_css_buffer * buf,unsigned int pipe)181 static bool imgu_dummybufs_check(struct imgu_device *imgu,
182 				 struct imgu_css_buffer *buf,
183 				 unsigned int pipe)
184 {
185 	unsigned int i;
186 	struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe];
187 
188 	for (i = 0; i < IMGU_MAX_QUEUE_DEPTH; i++)
189 		if (buf == &imgu_pipe->queues[buf->queue].dummybufs[i])
190 			break;
191 
192 	return i < IMGU_MAX_QUEUE_DEPTH;
193 }
194 
imgu_buffer_done(struct imgu_device * imgu,struct vb2_buffer * vb,enum vb2_buffer_state state)195 static void imgu_buffer_done(struct imgu_device *imgu, struct vb2_buffer *vb,
196 			     enum vb2_buffer_state state)
197 {
198 	mutex_lock(&imgu->lock);
199 	imgu_v4l2_buffer_done(vb, state);
200 	mutex_unlock(&imgu->lock);
201 }
202 
imgu_queue_getbuf(struct imgu_device * imgu,unsigned int node,unsigned int pipe)203 static struct imgu_css_buffer *imgu_queue_getbuf(struct imgu_device *imgu,
204 						 unsigned int node,
205 						 unsigned int pipe)
206 {
207 	struct imgu_buffer *buf;
208 	struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe];
209 
210 	if (WARN_ON(node >= IMGU_NODE_NUM))
211 		return NULL;
212 
213 	/* Find first free buffer from the node */
214 	list_for_each_entry(buf, &imgu_pipe->nodes[node].buffers, vid_buf.list) {
215 		if (imgu_css_buf_state(&buf->css_buf) == IPU3_CSS_BUFFER_NEW)
216 			return &buf->css_buf;
217 	}
218 
219 	/* There were no free buffers, try to return a dummy buffer */
220 	return imgu_dummybufs_get(imgu, imgu_node_map[node].css_queue, pipe);
221 }
222 
223 /*
224  * Queue as many buffers to CSS as possible. If all buffers don't fit into
225  * CSS buffer queues, they remain unqueued and will be queued later.
226  */
imgu_queue_buffers(struct imgu_device * imgu,bool initial,unsigned int pipe)227 int imgu_queue_buffers(struct imgu_device *imgu, bool initial, unsigned int pipe)
228 {
229 	unsigned int node;
230 	int r = 0;
231 	struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe];
232 
233 	if (!imgu_css_is_streaming(&imgu->css))
234 		return 0;
235 
236 	dev_dbg(&imgu->pci_dev->dev, "Queue buffers to pipe %d", pipe);
237 	mutex_lock(&imgu->lock);
238 
239 	if (!imgu_css_pipe_queue_empty(&imgu->css, pipe)) {
240 		mutex_unlock(&imgu->lock);
241 		return 0;
242 	}
243 
244 	/* Buffer set is queued to FW only when input buffer is ready */
245 	for (node = IMGU_NODE_NUM - 1;
246 	     imgu_queue_getbuf(imgu, IMGU_NODE_IN, pipe);
247 	     node = node ? node - 1 : IMGU_NODE_NUM - 1) {
248 		if (node == IMGU_NODE_VF &&
249 		    !imgu_pipe->nodes[IMGU_NODE_VF].enabled) {
250 			dev_warn(&imgu->pci_dev->dev,
251 				 "Vf not enabled, ignore queue");
252 			continue;
253 		} else if (node == IMGU_NODE_PARAMS &&
254 			   imgu_pipe->nodes[node].enabled) {
255 			struct vb2_buffer *vb;
256 			struct imgu_vb2_buffer *ivb;
257 
258 			/* No parameters for this frame */
259 			if (list_empty(&imgu_pipe->nodes[node].buffers))
260 				continue;
261 
262 			ivb = list_first_entry(&imgu_pipe->nodes[node].buffers,
263 					       struct imgu_vb2_buffer, list);
264 			list_del(&ivb->list);
265 			vb = &ivb->vbb.vb2_buf;
266 			r = imgu_css_set_parameters(&imgu->css, pipe,
267 						    vb2_plane_vaddr(vb, 0));
268 			if (r) {
269 				vb2_buffer_done(vb, VB2_BUF_STATE_ERROR);
270 				dev_warn(&imgu->pci_dev->dev,
271 					 "set parameters failed.");
272 				continue;
273 			}
274 
275 			vb2_buffer_done(vb, VB2_BUF_STATE_DONE);
276 			dev_dbg(&imgu->pci_dev->dev,
277 				"queue user parameters %d to css.", vb->index);
278 		} else if (imgu_pipe->queue_enabled[node]) {
279 			struct imgu_css_buffer *buf =
280 				imgu_queue_getbuf(imgu, node, pipe);
281 			struct imgu_buffer *ibuf = NULL;
282 			bool dummy;
283 
284 			if (!buf)
285 				break;
286 
287 			r = imgu_css_buf_queue(&imgu->css, pipe, buf);
288 			if (r)
289 				break;
290 			dummy = imgu_dummybufs_check(imgu, buf, pipe);
291 			if (!dummy)
292 				ibuf = container_of(buf, struct imgu_buffer,
293 						    css_buf);
294 			dev_dbg(&imgu->pci_dev->dev,
295 				"queue %s %s buffer %u to css da: 0x%08x\n",
296 				dummy ? "dummy" : "user",
297 				imgu_node_map[node].name,
298 				dummy ? 0 : ibuf->vid_buf.vbb.vb2_buf.index,
299 				(u32)buf->daddr);
300 		}
301 	}
302 	mutex_unlock(&imgu->lock);
303 
304 	if (r && r != -EBUSY)
305 		goto failed;
306 
307 	return 0;
308 
309 failed:
310 	/*
311 	 * On error, mark all buffers as failed which are not
312 	 * yet queued to CSS
313 	 */
314 	dev_err(&imgu->pci_dev->dev,
315 		"failed to queue buffer to CSS on queue %i (%d)\n",
316 		node, r);
317 
318 	if (initial)
319 		/* If we were called from streamon(), no need to finish bufs */
320 		return r;
321 
322 	for (node = 0; node < IMGU_NODE_NUM; node++) {
323 		struct imgu_buffer *buf, *buf0;
324 
325 		if (!imgu_pipe->queue_enabled[node])
326 			continue;	/* Skip disabled queues */
327 
328 		mutex_lock(&imgu->lock);
329 		list_for_each_entry_safe(buf, buf0,
330 					 &imgu_pipe->nodes[node].buffers,
331 					 vid_buf.list) {
332 			if (imgu_css_buf_state(&buf->css_buf) ==
333 			    IPU3_CSS_BUFFER_QUEUED)
334 				continue;	/* Was already queued, skip */
335 
336 			imgu_v4l2_buffer_done(&buf->vid_buf.vbb.vb2_buf,
337 					      VB2_BUF_STATE_ERROR);
338 		}
339 		mutex_unlock(&imgu->lock);
340 	}
341 
342 	return r;
343 }
344 
imgu_powerup(struct imgu_device * imgu)345 static int imgu_powerup(struct imgu_device *imgu)
346 {
347 	int r;
348 	unsigned int pipe;
349 	unsigned int freq = 200;
350 	struct v4l2_mbus_framefmt *fmt;
351 
352 	/* input larger than 2048*1152, ask imgu to work on high freq */
353 	for_each_set_bit(pipe, imgu->css.enabled_pipes, IMGU_MAX_PIPE_NUM) {
354 		fmt = &imgu->imgu_pipe[pipe].nodes[IMGU_NODE_IN].pad_fmt;
355 		dev_dbg(&imgu->pci_dev->dev, "pipe %u input format = %ux%u",
356 			pipe, fmt->width, fmt->height);
357 		if ((fmt->width * fmt->height) >= (2048 * 1152))
358 			freq = 450;
359 	}
360 
361 	r = imgu_css_set_powerup(&imgu->pci_dev->dev, imgu->base, freq);
362 	if (r)
363 		return r;
364 
365 	imgu_mmu_resume(imgu->mmu);
366 	return 0;
367 }
368 
imgu_powerdown(struct imgu_device * imgu)369 static void imgu_powerdown(struct imgu_device *imgu)
370 {
371 	imgu_mmu_suspend(imgu->mmu);
372 	imgu_css_set_powerdown(&imgu->pci_dev->dev, imgu->base);
373 }
374 
imgu_s_stream(struct imgu_device * imgu,int enable)375 int imgu_s_stream(struct imgu_device *imgu, int enable)
376 {
377 	struct device *dev = &imgu->pci_dev->dev;
378 	int r, pipe;
379 
380 	if (!enable) {
381 		/* Stop streaming */
382 		dev_dbg(dev, "stream off\n");
383 		/* Block new buffers to be queued to CSS. */
384 		atomic_set(&imgu->qbuf_barrier, 1);
385 		imgu_css_stop_streaming(&imgu->css);
386 		synchronize_irq(imgu->pci_dev->irq);
387 		atomic_set(&imgu->qbuf_barrier, 0);
388 		imgu_powerdown(imgu);
389 		pm_runtime_put(&imgu->pci_dev->dev);
390 
391 		return 0;
392 	}
393 
394 	/* Set Power */
395 	r = pm_runtime_resume_and_get(dev);
396 	if (r < 0) {
397 		dev_err(dev, "failed to set imgu power\n");
398 		return r;
399 	}
400 
401 	r = imgu_powerup(imgu);
402 	if (r) {
403 		dev_err(dev, "failed to power up imgu\n");
404 		pm_runtime_put(dev);
405 		return r;
406 	}
407 
408 	/* Start CSS streaming */
409 	r = imgu_css_start_streaming(&imgu->css);
410 	if (r) {
411 		dev_err(dev, "failed to start css streaming (%d)", r);
412 		goto fail_start_streaming;
413 	}
414 
415 	for_each_set_bit(pipe, imgu->css.enabled_pipes, IMGU_MAX_PIPE_NUM) {
416 		/* Initialize dummy buffers */
417 		r = imgu_dummybufs_init(imgu, pipe);
418 		if (r) {
419 			dev_err(dev, "failed to initialize dummy buffers (%d)", r);
420 			goto fail_dummybufs;
421 		}
422 
423 		/* Queue as many buffers from queue as possible */
424 		r = imgu_queue_buffers(imgu, true, pipe);
425 		if (r) {
426 			dev_err(dev, "failed to queue initial buffers (%d)", r);
427 			goto fail_queueing;
428 		}
429 	}
430 
431 	return 0;
432 fail_queueing:
433 	for_each_set_bit(pipe, imgu->css.enabled_pipes, IMGU_MAX_PIPE_NUM)
434 		imgu_dummybufs_cleanup(imgu, pipe);
435 fail_dummybufs:
436 	imgu_css_stop_streaming(&imgu->css);
437 fail_start_streaming:
438 	pm_runtime_put(dev);
439 
440 	return r;
441 }
442 
imgu_video_nodes_exit(struct imgu_device * imgu)443 static void imgu_video_nodes_exit(struct imgu_device *imgu)
444 {
445 	int i;
446 
447 	for (i = 0; i < IMGU_MAX_PIPE_NUM; i++)
448 		imgu_dummybufs_cleanup(imgu, i);
449 
450 	imgu_v4l2_unregister(imgu);
451 }
452 
imgu_video_nodes_init(struct imgu_device * imgu)453 static int imgu_video_nodes_init(struct imgu_device *imgu)
454 {
455 	struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES] = { NULL };
456 	struct v4l2_rect *rects[IPU3_CSS_RECTS] = { NULL };
457 	struct imgu_media_pipe *imgu_pipe;
458 	unsigned int i, j;
459 	int r;
460 
461 	imgu->buf_struct_size = sizeof(struct imgu_buffer);
462 
463 	for (j = 0; j < IMGU_MAX_PIPE_NUM; j++) {
464 		imgu_pipe = &imgu->imgu_pipe[j];
465 
466 		for (i = 0; i < IMGU_NODE_NUM; i++) {
467 			imgu_pipe->nodes[i].name = imgu_node_map[i].name;
468 			imgu_pipe->nodes[i].output = i < IMGU_QUEUE_FIRST_INPUT;
469 			imgu_pipe->nodes[i].enabled = false;
470 
471 			if (i != IMGU_NODE_PARAMS && i != IMGU_NODE_STAT_3A)
472 				fmts[imgu_node_map[i].css_queue] =
473 					&imgu_pipe->nodes[i].vdev_fmt.fmt.pix_mp;
474 			atomic_set(&imgu_pipe->nodes[i].sequence, 0);
475 		}
476 	}
477 
478 	r = imgu_v4l2_register(imgu);
479 	if (r)
480 		return r;
481 
482 	/* Set initial formats and initialize formats of video nodes */
483 	for (j = 0; j < IMGU_MAX_PIPE_NUM; j++) {
484 		imgu_pipe = &imgu->imgu_pipe[j];
485 
486 		rects[IPU3_CSS_RECT_EFFECTIVE] = &imgu_pipe->imgu_sd.rect.eff;
487 		rects[IPU3_CSS_RECT_BDS] = &imgu_pipe->imgu_sd.rect.bds;
488 		imgu_css_fmt_set(&imgu->css, fmts, rects, j);
489 
490 		/* Pre-allocate dummy buffers */
491 		r = imgu_dummybufs_preallocate(imgu, j);
492 		if (r) {
493 			dev_err(&imgu->pci_dev->dev,
494 				"failed to pre-allocate dummy buffers (%d)", r);
495 			goto out_cleanup;
496 		}
497 	}
498 
499 	return 0;
500 
501 out_cleanup:
502 	imgu_video_nodes_exit(imgu);
503 
504 	return r;
505 }
506 
507 /**************** PCI interface ****************/
508 
imgu_isr_threaded(int irq,void * imgu_ptr)509 static irqreturn_t imgu_isr_threaded(int irq, void *imgu_ptr)
510 {
511 	struct imgu_device *imgu = imgu_ptr;
512 	struct imgu_media_pipe *imgu_pipe;
513 	int p;
514 
515 	/* Dequeue / queue buffers */
516 	do {
517 		u64 ns = ktime_get_ns();
518 		struct imgu_css_buffer *b;
519 		struct imgu_buffer *buf = NULL;
520 		unsigned int node, pipe;
521 		bool dummy;
522 
523 		do {
524 			mutex_lock(&imgu->lock);
525 			b = imgu_css_buf_dequeue(&imgu->css);
526 			mutex_unlock(&imgu->lock);
527 		} while (PTR_ERR(b) == -EAGAIN);
528 
529 		if (IS_ERR(b)) {
530 			if (PTR_ERR(b) != -EBUSY)	/* All done */
531 				dev_err(&imgu->pci_dev->dev,
532 					"failed to dequeue buffers (%ld)\n",
533 					PTR_ERR(b));
534 			break;
535 		}
536 
537 		node = imgu_map_node(imgu, b->queue);
538 		pipe = b->pipe;
539 		dummy = imgu_dummybufs_check(imgu, b, pipe);
540 		if (!dummy)
541 			buf = container_of(b, struct imgu_buffer, css_buf);
542 		dev_dbg(&imgu->pci_dev->dev,
543 			"dequeue %s %s buffer %d daddr 0x%x from css\n",
544 			dummy ? "dummy" : "user",
545 			imgu_node_map[node].name,
546 			dummy ? 0 : buf->vid_buf.vbb.vb2_buf.index,
547 			(u32)b->daddr);
548 
549 		if (dummy)
550 			/* It was a dummy buffer, skip it */
551 			continue;
552 
553 		/* Fill vb2 buffer entries and tell it's ready */
554 		imgu_pipe = &imgu->imgu_pipe[pipe];
555 		if (!imgu_pipe->nodes[node].output) {
556 			buf->vid_buf.vbb.vb2_buf.timestamp = ns;
557 			buf->vid_buf.vbb.field = V4L2_FIELD_NONE;
558 			buf->vid_buf.vbb.sequence =
559 				atomic_inc_return(
560 				&imgu_pipe->nodes[node].sequence);
561 			dev_dbg(&imgu->pci_dev->dev, "vb2 buffer sequence %d",
562 				buf->vid_buf.vbb.sequence);
563 		}
564 		imgu_buffer_done(imgu, &buf->vid_buf.vbb.vb2_buf,
565 				 imgu_css_buf_state(&buf->css_buf) ==
566 						    IPU3_CSS_BUFFER_DONE ?
567 						    VB2_BUF_STATE_DONE :
568 						    VB2_BUF_STATE_ERROR);
569 		mutex_lock(&imgu->lock);
570 		if (imgu_css_queue_empty(&imgu->css))
571 			wake_up_all(&imgu->buf_drain_wq);
572 		mutex_unlock(&imgu->lock);
573 	} while (1);
574 
575 	/*
576 	 * Try to queue more buffers for CSS.
577 	 * qbuf_barrier is used to disable new buffers
578 	 * to be queued to CSS.
579 	 */
580 	if (!atomic_read(&imgu->qbuf_barrier))
581 		for_each_set_bit(p, imgu->css.enabled_pipes, IMGU_MAX_PIPE_NUM)
582 			imgu_queue_buffers(imgu, false, p);
583 
584 	return IRQ_HANDLED;
585 }
586 
imgu_isr(int irq,void * imgu_ptr)587 static irqreturn_t imgu_isr(int irq, void *imgu_ptr)
588 {
589 	struct imgu_device *imgu = imgu_ptr;
590 
591 	/* acknowledge interruption */
592 	if (imgu_css_irq_ack(&imgu->css) < 0)
593 		return IRQ_NONE;
594 
595 	return IRQ_WAKE_THREAD;
596 }
597 
imgu_pci_config_setup(struct pci_dev * dev)598 static int imgu_pci_config_setup(struct pci_dev *dev)
599 {
600 	u16 pci_command;
601 	int r = pci_enable_msi(dev);
602 
603 	if (r) {
604 		dev_err(&dev->dev, "failed to enable MSI (%d)\n", r);
605 		return r;
606 	}
607 
608 	pci_read_config_word(dev, PCI_COMMAND, &pci_command);
609 	pci_command |= PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
610 			PCI_COMMAND_INTX_DISABLE;
611 	pci_write_config_word(dev, PCI_COMMAND, pci_command);
612 
613 	return 0;
614 }
615 
imgu_pci_probe(struct pci_dev * pci_dev,const struct pci_device_id * id)616 static int imgu_pci_probe(struct pci_dev *pci_dev,
617 			  const struct pci_device_id *id)
618 {
619 	struct imgu_device *imgu;
620 	phys_addr_t phys;
621 	unsigned long phys_len;
622 	void __iomem *const *iomap;
623 	int r;
624 
625 	imgu = devm_kzalloc(&pci_dev->dev, sizeof(*imgu), GFP_KERNEL);
626 	if (!imgu)
627 		return -ENOMEM;
628 
629 	imgu->pci_dev = pci_dev;
630 
631 	r = pcim_enable_device(pci_dev);
632 	if (r) {
633 		dev_err(&pci_dev->dev, "failed to enable device (%d)\n", r);
634 		return r;
635 	}
636 
637 	dev_info(&pci_dev->dev, "device 0x%x (rev: 0x%x)\n",
638 		 pci_dev->device, pci_dev->revision);
639 
640 	phys = pci_resource_start(pci_dev, IMGU_PCI_BAR);
641 	phys_len = pci_resource_len(pci_dev, IMGU_PCI_BAR);
642 
643 	r = pcim_iomap_regions(pci_dev, 1 << IMGU_PCI_BAR, pci_name(pci_dev));
644 	if (r) {
645 		dev_err(&pci_dev->dev, "failed to remap I/O memory (%d)\n", r);
646 		return r;
647 	}
648 	dev_info(&pci_dev->dev, "physical base address %pap, %lu bytes\n",
649 		 &phys, phys_len);
650 
651 	iomap = pcim_iomap_table(pci_dev);
652 	if (!iomap) {
653 		dev_err(&pci_dev->dev, "failed to iomap table\n");
654 		return -ENODEV;
655 	}
656 
657 	imgu->base = iomap[IMGU_PCI_BAR];
658 
659 	pci_set_drvdata(pci_dev, imgu);
660 
661 	pci_set_master(pci_dev);
662 
663 	r = dma_coerce_mask_and_coherent(&pci_dev->dev, IMGU_DMA_MASK);
664 	if (r) {
665 		dev_err(&pci_dev->dev, "failed to set DMA mask (%d)\n", r);
666 		return -ENODEV;
667 	}
668 
669 	r = imgu_pci_config_setup(pci_dev);
670 	if (r)
671 		return r;
672 
673 	mutex_init(&imgu->lock);
674 	mutex_init(&imgu->streaming_lock);
675 	atomic_set(&imgu->qbuf_barrier, 0);
676 	init_waitqueue_head(&imgu->buf_drain_wq);
677 
678 	r = imgu_css_set_powerup(&pci_dev->dev, imgu->base, 200);
679 	if (r) {
680 		dev_err(&pci_dev->dev,
681 			"failed to power up CSS (%d)\n", r);
682 		goto out_mutex_destroy;
683 	}
684 
685 	imgu->mmu = imgu_mmu_init(&pci_dev->dev, imgu->base);
686 	if (IS_ERR(imgu->mmu)) {
687 		r = PTR_ERR(imgu->mmu);
688 		dev_err(&pci_dev->dev, "failed to initialize MMU (%d)\n", r);
689 		goto out_css_powerdown;
690 	}
691 
692 	r = imgu_dmamap_init(imgu);
693 	if (r) {
694 		dev_err(&pci_dev->dev,
695 			"failed to initialize DMA mapping (%d)\n", r);
696 		goto out_mmu_exit;
697 	}
698 
699 	/* ISP programming */
700 	r = imgu_css_init(&pci_dev->dev, &imgu->css, imgu->base, phys_len);
701 	if (r) {
702 		dev_err(&pci_dev->dev, "failed to initialize CSS (%d)\n", r);
703 		goto out_dmamap_exit;
704 	}
705 
706 	/* v4l2 sub-device registration */
707 	r = imgu_video_nodes_init(imgu);
708 	if (r) {
709 		dev_err(&pci_dev->dev, "failed to create V4L2 devices (%d)\n",
710 			r);
711 		goto out_css_cleanup;
712 	}
713 
714 	r = devm_request_threaded_irq(&pci_dev->dev, pci_dev->irq,
715 				      imgu_isr, imgu_isr_threaded,
716 				      IRQF_SHARED, IMGU_NAME, imgu);
717 	if (r) {
718 		dev_err(&pci_dev->dev, "failed to request IRQ (%d)\n", r);
719 		goto out_video_exit;
720 	}
721 
722 	pm_runtime_put_noidle(&pci_dev->dev);
723 	pm_runtime_allow(&pci_dev->dev);
724 
725 	return 0;
726 
727 out_video_exit:
728 	imgu_video_nodes_exit(imgu);
729 out_css_cleanup:
730 	imgu_css_cleanup(&imgu->css);
731 out_dmamap_exit:
732 	imgu_dmamap_exit(imgu);
733 out_mmu_exit:
734 	imgu_mmu_exit(imgu->mmu);
735 out_css_powerdown:
736 	imgu_css_set_powerdown(&pci_dev->dev, imgu->base);
737 out_mutex_destroy:
738 	mutex_destroy(&imgu->streaming_lock);
739 	mutex_destroy(&imgu->lock);
740 
741 	return r;
742 }
743 
imgu_pci_remove(struct pci_dev * pci_dev)744 static void imgu_pci_remove(struct pci_dev *pci_dev)
745 {
746 	struct imgu_device *imgu = pci_get_drvdata(pci_dev);
747 
748 	pm_runtime_forbid(&pci_dev->dev);
749 	pm_runtime_get_noresume(&pci_dev->dev);
750 
751 	imgu_video_nodes_exit(imgu);
752 	imgu_css_cleanup(&imgu->css);
753 	imgu_css_set_powerdown(&pci_dev->dev, imgu->base);
754 	imgu_dmamap_exit(imgu);
755 	imgu_mmu_exit(imgu->mmu);
756 	mutex_destroy(&imgu->streaming_lock);
757 	mutex_destroy(&imgu->lock);
758 }
759 
imgu_suspend(struct device * dev)760 static int __maybe_unused imgu_suspend(struct device *dev)
761 {
762 	struct pci_dev *pci_dev = to_pci_dev(dev);
763 	struct imgu_device *imgu = pci_get_drvdata(pci_dev);
764 
765 	dev_dbg(dev, "enter %s\n", __func__);
766 	imgu->suspend_in_stream = imgu_css_is_streaming(&imgu->css);
767 	if (!imgu->suspend_in_stream)
768 		goto out;
769 	/* Block new buffers to be queued to CSS. */
770 	atomic_set(&imgu->qbuf_barrier, 1);
771 	/*
772 	 * Wait for currently running irq handler to be done so that
773 	 * no new buffers will be queued to fw later.
774 	 */
775 	synchronize_irq(pci_dev->irq);
776 	/* Wait until all buffers in CSS are done. */
777 	if (!wait_event_timeout(imgu->buf_drain_wq,
778 	    imgu_css_queue_empty(&imgu->css), msecs_to_jiffies(1000)))
779 		dev_err(dev, "wait buffer drain timeout.\n");
780 
781 	imgu_css_stop_streaming(&imgu->css);
782 	atomic_set(&imgu->qbuf_barrier, 0);
783 	imgu_powerdown(imgu);
784 	pm_runtime_force_suspend(dev);
785 out:
786 	dev_dbg(dev, "leave %s\n", __func__);
787 	return 0;
788 }
789 
imgu_resume(struct device * dev)790 static int __maybe_unused imgu_resume(struct device *dev)
791 {
792 	struct imgu_device *imgu = dev_get_drvdata(dev);
793 	int r = 0;
794 	unsigned int pipe;
795 
796 	dev_dbg(dev, "enter %s\n", __func__);
797 
798 	if (!imgu->suspend_in_stream)
799 		goto out;
800 
801 	pm_runtime_force_resume(dev);
802 
803 	r = imgu_powerup(imgu);
804 	if (r) {
805 		dev_err(dev, "failed to power up imgu\n");
806 		goto out;
807 	}
808 
809 	/* Start CSS streaming */
810 	r = imgu_css_start_streaming(&imgu->css);
811 	if (r) {
812 		dev_err(dev, "failed to resume css streaming (%d)", r);
813 		goto out;
814 	}
815 
816 	for_each_set_bit(pipe, imgu->css.enabled_pipes, IMGU_MAX_PIPE_NUM) {
817 		r = imgu_queue_buffers(imgu, true, pipe);
818 		if (r)
819 			dev_err(dev, "failed to queue buffers to pipe %d (%d)",
820 				pipe, r);
821 	}
822 
823 out:
824 	dev_dbg(dev, "leave %s\n", __func__);
825 
826 	return r;
827 }
828 
829 /*
830  * PCI rpm framework checks the existence of driver rpm callbacks.
831  * Place a dummy callback here to avoid rpm going into error state.
832  */
imgu_rpm_dummy_cb(struct device * dev)833 static __maybe_unused int imgu_rpm_dummy_cb(struct device *dev)
834 {
835 	return 0;
836 }
837 
838 static const struct dev_pm_ops imgu_pm_ops = {
839 	SET_RUNTIME_PM_OPS(&imgu_rpm_dummy_cb, &imgu_rpm_dummy_cb, NULL)
840 	SET_SYSTEM_SLEEP_PM_OPS(&imgu_suspend, &imgu_resume)
841 };
842 
843 static const struct pci_device_id imgu_pci_tbl[] = {
844 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, IMGU_PCI_ID) },
845 	{ 0, }
846 };
847 
848 MODULE_DEVICE_TABLE(pci, imgu_pci_tbl);
849 
850 static struct pci_driver imgu_pci_driver = {
851 	.name = IMGU_NAME,
852 	.id_table = imgu_pci_tbl,
853 	.probe = imgu_pci_probe,
854 	.remove = imgu_pci_remove,
855 	.driver = {
856 		.pm = &imgu_pm_ops,
857 	},
858 };
859 
860 module_pci_driver(imgu_pci_driver);
861 
862 MODULE_AUTHOR("Tuukka Toivonen <tuukka.toivonen@intel.com>");
863 MODULE_AUTHOR("Tianshu Qiu <tian.shu.qiu@intel.com>");
864 MODULE_AUTHOR("Jian Xu Zheng <jian.xu.zheng@intel.com>");
865 MODULE_AUTHOR("Yuning Pu <yuning.pu@intel.com>");
866 MODULE_AUTHOR("Yong Zhi <yong.zhi@intel.com>");
867 MODULE_LICENSE("GPL v2");
868 MODULE_DESCRIPTION("Intel ipu3_imgu PCI driver");
869