1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (C) 2018 Intel Corporation
3 
4 #include <linux/device.h>
5 #include <linux/iopoll.h>
6 #include <linux/slab.h>
7 
8 #include "ipu3.h"
9 #include "ipu3-css.h"
10 #include "ipu3-css-fw.h"
11 #include "ipu3-css-params.h"
12 #include "ipu3-dmamap.h"
13 #include "ipu3-tables.h"
14 
15 /* IRQ configuration */
16 #define IMGU_IRQCTRL_IRQ_MASK	(IMGU_IRQCTRL_IRQ_SP1 | \
17 				 IMGU_IRQCTRL_IRQ_SP2 | \
18 				 IMGU_IRQCTRL_IRQ_SW_PIN(0) | \
19 				 IMGU_IRQCTRL_IRQ_SW_PIN(1))
20 
21 #define IPU3_CSS_FORMAT_BPP_DEN	50	/* Denominator */
22 
23 /* Some sane limits for resolutions */
24 #define IPU3_CSS_MIN_RES	32
25 #define IPU3_CSS_MAX_H		3136
26 #define IPU3_CSS_MAX_W		4224
27 
28 /* minimal envelope size(GDC in - out) should be 4 */
29 #define MIN_ENVELOPE            4
30 
31 /*
32  * pre-allocated buffer size for CSS ABI, auxiliary frames
33  * after BDS and before GDC. Those values should be tuned
34  * to big enough to avoid buffer re-allocation when
35  * streaming to lower streaming latency.
36  */
37 #define CSS_ABI_SIZE    136
38 #define CSS_BDS_SIZE    (4480 * 3200 * 3)
39 #define CSS_GDC_SIZE    (4224 * 3200 * 12 / 8)
40 
41 #define IPU3_CSS_QUEUE_TO_FLAGS(q)	(1 << (q))
42 #define IPU3_CSS_FORMAT_FL_IN		\
43 			IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_IN)
44 #define IPU3_CSS_FORMAT_FL_OUT		\
45 			IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_OUT)
46 #define IPU3_CSS_FORMAT_FL_VF		\
47 			IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_VF)
48 
49 /* Formats supported by IPU3 Camera Sub System */
50 static const struct imgu_css_format imgu_css_formats[] = {
51 	{
52 		.pixelformat = V4L2_PIX_FMT_NV12,
53 		.colorspace = V4L2_COLORSPACE_SRGB,
54 		.frame_format = IMGU_ABI_FRAME_FORMAT_NV12,
55 		.osys_format = IMGU_ABI_OSYS_FORMAT_NV12,
56 		.osys_tiling = IMGU_ABI_OSYS_TILING_NONE,
57 		.chroma_decim = 4,
58 		.width_align = IPU3_UAPI_ISP_VEC_ELEMS,
59 		.flags = IPU3_CSS_FORMAT_FL_OUT | IPU3_CSS_FORMAT_FL_VF,
60 	}, {
61 		/* Each 32 bytes contains 25 10-bit pixels */
62 		.pixelformat = V4L2_PIX_FMT_IPU3_SBGGR10,
63 		.colorspace = V4L2_COLORSPACE_RAW,
64 		.frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
65 		.bayer_order = IMGU_ABI_BAYER_ORDER_BGGR,
66 		.bit_depth = 10,
67 		.width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
68 		.flags = IPU3_CSS_FORMAT_FL_IN,
69 	}, {
70 		.pixelformat = V4L2_PIX_FMT_IPU3_SGBRG10,
71 		.colorspace = V4L2_COLORSPACE_RAW,
72 		.frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
73 		.bayer_order = IMGU_ABI_BAYER_ORDER_GBRG,
74 		.bit_depth = 10,
75 		.width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
76 		.flags = IPU3_CSS_FORMAT_FL_IN,
77 	}, {
78 		.pixelformat = V4L2_PIX_FMT_IPU3_SGRBG10,
79 		.colorspace = V4L2_COLORSPACE_RAW,
80 		.frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
81 		.bayer_order = IMGU_ABI_BAYER_ORDER_GRBG,
82 		.bit_depth = 10,
83 		.width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
84 		.flags = IPU3_CSS_FORMAT_FL_IN,
85 	}, {
86 		.pixelformat = V4L2_PIX_FMT_IPU3_SRGGB10,
87 		.colorspace = V4L2_COLORSPACE_RAW,
88 		.frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
89 		.bayer_order = IMGU_ABI_BAYER_ORDER_RGGB,
90 		.bit_depth = 10,
91 		.width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
92 		.flags = IPU3_CSS_FORMAT_FL_IN,
93 	},
94 };
95 
96 static const struct {
97 	enum imgu_abi_queue_id qid;
98 	size_t ptr_ofs;
99 } imgu_css_queues[IPU3_CSS_QUEUES] = {
100 	[IPU3_CSS_QUEUE_IN] = {
101 		IMGU_ABI_QUEUE_C_ID,
102 		offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
103 	},
104 	[IPU3_CSS_QUEUE_OUT] = {
105 		IMGU_ABI_QUEUE_D_ID,
106 		offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
107 	},
108 	[IPU3_CSS_QUEUE_VF] = {
109 		IMGU_ABI_QUEUE_E_ID,
110 		offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
111 	},
112 	[IPU3_CSS_QUEUE_STAT_3A] = {
113 		IMGU_ABI_QUEUE_F_ID,
114 		offsetof(struct imgu_abi_buffer, payload.s3a.data_ptr)
115 	},
116 };
117 
118 /* Initialize queue based on given format, adjust format as needed */
imgu_css_queue_init(struct imgu_css_queue * queue,struct v4l2_pix_format_mplane * fmt,u32 flags)119 static int imgu_css_queue_init(struct imgu_css_queue *queue,
120 			       struct v4l2_pix_format_mplane *fmt, u32 flags)
121 {
122 	struct v4l2_pix_format_mplane *const f = &queue->fmt.mpix;
123 	unsigned int i;
124 	u32 sizeimage;
125 
126 	INIT_LIST_HEAD(&queue->bufs);
127 
128 	queue->css_fmt = NULL;	/* Disable */
129 	if (!fmt)
130 		return 0;
131 
132 	for (i = 0; i < ARRAY_SIZE(imgu_css_formats); i++) {
133 		if (!(imgu_css_formats[i].flags & flags))
134 			continue;
135 		queue->css_fmt = &imgu_css_formats[i];
136 		if (imgu_css_formats[i].pixelformat == fmt->pixelformat)
137 			break;
138 	}
139 	if (!queue->css_fmt)
140 		return -EINVAL;	/* Could not find any suitable format */
141 
142 	queue->fmt.mpix = *fmt;
143 
144 	f->width = ALIGN(clamp_t(u32, f->width,
145 				 IPU3_CSS_MIN_RES, IPU3_CSS_MAX_W), 2);
146 	f->height = ALIGN(clamp_t(u32, f->height,
147 				  IPU3_CSS_MIN_RES, IPU3_CSS_MAX_H), 2);
148 	queue->width_pad = ALIGN(f->width, queue->css_fmt->width_align);
149 	f->plane_fmt[0].bytesperline =
150 		imgu_bytesperline(f->width, queue->css_fmt->frame_format);
151 	sizeimage = f->height * f->plane_fmt[0].bytesperline;
152 	if (queue->css_fmt->chroma_decim)
153 		sizeimage += 2 * sizeimage / queue->css_fmt->chroma_decim;
154 
155 	f->plane_fmt[0].sizeimage = sizeimage;
156 	f->field = V4L2_FIELD_NONE;
157 	f->num_planes = 1;
158 	f->colorspace = queue->css_fmt->colorspace;
159 	f->flags = 0;
160 	f->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
161 	f->quantization = V4L2_QUANTIZATION_DEFAULT;
162 	f->xfer_func = V4L2_XFER_FUNC_DEFAULT;
163 	memset(f->reserved, 0, sizeof(f->reserved));
164 
165 	return 0;
166 }
167 
imgu_css_queue_enabled(struct imgu_css_queue * q)168 static bool imgu_css_queue_enabled(struct imgu_css_queue *q)
169 {
170 	return q->css_fmt;
171 }
172 
173 /******************* css hw *******************/
174 
175 /* In the style of writesl() defined in include/asm-generic/io.h */
writes(const void * mem,ssize_t count,void __iomem * addr)176 static inline void writes(const void *mem, ssize_t count, void __iomem *addr)
177 {
178 	if (count >= 4) {
179 		const u32 *buf = mem;
180 
181 		count /= 4;
182 		do {
183 			writel(*buf++, addr);
184 			addr += 4;
185 		} while (--count);
186 	}
187 }
188 
189 /* Wait until register `reg', masked with `mask', becomes `cmp' */
imgu_hw_wait(void __iomem * base,int reg,u32 mask,u32 cmp)190 static int imgu_hw_wait(void __iomem *base, int reg, u32 mask, u32 cmp)
191 {
192 	u32 val;
193 
194 	return readl_poll_timeout(base + reg, val, (val & mask) == cmp,
195 				  1000, 100 * 1000);
196 }
197 
198 /* Initialize the IPU3 CSS hardware and associated h/w blocks */
199 
imgu_css_set_powerup(struct device * dev,void __iomem * base,unsigned int freq)200 int imgu_css_set_powerup(struct device *dev, void __iomem *base,
201 			 unsigned int freq)
202 {
203 	u32 pm_ctrl, state, val;
204 
205 	dev_dbg(dev, "%s with freq %u\n", __func__, freq);
206 	/* Clear the CSS busy signal */
207 	readl(base + IMGU_REG_GP_BUSY);
208 	writel(0, base + IMGU_REG_GP_BUSY);
209 
210 	/* Wait for idle signal */
211 	if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
212 			 IMGU_STATE_IDLE_STS)) {
213 		dev_err(dev, "failed to set CSS idle\n");
214 		goto fail;
215 	}
216 
217 	/* Reset the css */
218 	writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET,
219 	       base + IMGU_REG_PM_CTRL);
220 
221 	usleep_range(200, 300);
222 
223 	/** Prepare CSS */
224 
225 	pm_ctrl = readl(base + IMGU_REG_PM_CTRL);
226 	state = readl(base + IMGU_REG_STATE);
227 
228 	dev_dbg(dev, "CSS pm_ctrl 0x%x state 0x%x (power %s)\n",
229 		pm_ctrl, state, state & IMGU_STATE_POWER_DOWN ? "down" : "up");
230 
231 	/* Power up CSS using wrapper */
232 	if (state & IMGU_STATE_POWER_DOWN) {
233 		writel(IMGU_PM_CTRL_RACE_TO_HALT | IMGU_PM_CTRL_START,
234 		       base + IMGU_REG_PM_CTRL);
235 		if (imgu_hw_wait(base, IMGU_REG_PM_CTRL,
236 				 IMGU_PM_CTRL_START, 0)) {
237 			dev_err(dev, "failed to power up CSS\n");
238 			goto fail;
239 		}
240 		usleep_range(2000, 3000);
241 	} else {
242 		writel(IMGU_PM_CTRL_RACE_TO_HALT, base + IMGU_REG_PM_CTRL);
243 	}
244 
245 	/* Set the busy bit */
246 	writel(readl(base + IMGU_REG_GP_BUSY) | 1, base + IMGU_REG_GP_BUSY);
247 
248 	/* Set CSS clock frequency */
249 	pm_ctrl = readl(base + IMGU_REG_PM_CTRL);
250 	val = pm_ctrl & ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
251 	writel(val, base + IMGU_REG_PM_CTRL);
252 	writel(0, base + IMGU_REG_GP_BUSY);
253 	if (imgu_hw_wait(base, IMGU_REG_STATE,
254 			 IMGU_STATE_PWRDNM_FSM_MASK, 0)) {
255 		dev_err(dev, "failed to pwrdn CSS\n");
256 		goto fail;
257 	}
258 	val = (freq / IMGU_SYSTEM_REQ_FREQ_DIVIDER) & IMGU_SYSTEM_REQ_FREQ_MASK;
259 	writel(val, base + IMGU_REG_SYSTEM_REQ);
260 	writel(1, base + IMGU_REG_GP_BUSY);
261 	writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_HALT,
262 	       base + IMGU_REG_PM_CTRL);
263 	if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS,
264 			 IMGU_STATE_HALT_STS)) {
265 		dev_err(dev, "failed to halt CSS\n");
266 		goto fail;
267 	}
268 
269 	writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_START,
270 	       base + IMGU_REG_PM_CTRL);
271 	if (imgu_hw_wait(base, IMGU_REG_PM_CTRL, IMGU_PM_CTRL_START, 0)) {
272 		dev_err(dev, "failed to start CSS\n");
273 		goto fail;
274 	}
275 	writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_UNHALT,
276 	       base + IMGU_REG_PM_CTRL);
277 
278 	val = readl(base + IMGU_REG_PM_CTRL);	/* get pm_ctrl */
279 	val &= ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
280 	val |= pm_ctrl & (IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
281 	writel(val, base + IMGU_REG_PM_CTRL);
282 
283 	return 0;
284 
285 fail:
286 	imgu_css_set_powerdown(dev, base);
287 	return -EIO;
288 }
289 
imgu_css_set_powerdown(struct device * dev,void __iomem * base)290 void imgu_css_set_powerdown(struct device *dev, void __iomem *base)
291 {
292 	dev_dbg(dev, "%s\n", __func__);
293 	/* wait for cio idle signal */
294 	if (imgu_hw_wait(base, IMGU_REG_CIO_GATE_BURST_STATE,
295 			 IMGU_CIO_GATE_BURST_MASK, 0))
296 		dev_warn(dev, "wait cio gate idle timeout");
297 
298 	/* wait for css idle signal */
299 	if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
300 			 IMGU_STATE_IDLE_STS))
301 		dev_warn(dev, "wait css idle timeout\n");
302 
303 	/* do halt-halted handshake with css */
304 	writel(1, base + IMGU_REG_GP_HALT);
305 	if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS,
306 			 IMGU_STATE_HALT_STS))
307 		dev_warn(dev, "failed to halt css");
308 
309 	/* de-assert the busy bit */
310 	writel(0, base + IMGU_REG_GP_BUSY);
311 }
312 
imgu_css_hw_enable_irq(struct imgu_css * css)313 static void imgu_css_hw_enable_irq(struct imgu_css *css)
314 {
315 	void __iomem *const base = css->base;
316 	u32 val, i;
317 
318 	/* Set up interrupts */
319 
320 	/*
321 	 * Enable IRQ on the SP which signals that SP goes to idle
322 	 * (aka ready state) and set trigger to pulse
323 	 */
324 	val = readl(base + IMGU_REG_SP_CTRL(0)) | IMGU_CTRL_IRQ_READY;
325 	writel(val, base + IMGU_REG_SP_CTRL(0));
326 	writel(val | IMGU_CTRL_IRQ_CLEAR, base + IMGU_REG_SP_CTRL(0));
327 
328 	/* Enable IRQs from the IMGU wrapper */
329 	writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_ENABLE);
330 	/* Clear */
331 	writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_STATUS);
332 
333 	/* Enable IRQs from main IRQ controller */
334 	writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(IMGU_IRQCTRL_MAIN));
335 	writel(0, base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN));
336 	writel(IMGU_IRQCTRL_IRQ_MASK,
337 	       base + IMGU_REG_IRQCTRL_EDGE(IMGU_IRQCTRL_MAIN));
338 	writel(IMGU_IRQCTRL_IRQ_MASK,
339 	       base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN));
340 	writel(IMGU_IRQCTRL_IRQ_MASK,
341 	       base + IMGU_REG_IRQCTRL_CLEAR(IMGU_IRQCTRL_MAIN));
342 	writel(IMGU_IRQCTRL_IRQ_MASK,
343 	       base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN));
344 	/* Wait for write complete */
345 	readl(base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN));
346 
347 	/* Enable IRQs from SP0 and SP1 controllers */
348 	for (i = IMGU_IRQCTRL_SP0; i <= IMGU_IRQCTRL_SP1; i++) {
349 		writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(i));
350 		writel(0, base + IMGU_REG_IRQCTRL_MASK(i));
351 		writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_EDGE(i));
352 		writel(IMGU_IRQCTRL_IRQ_MASK,
353 		       base + IMGU_REG_IRQCTRL_ENABLE(i));
354 		writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_CLEAR(i));
355 		writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_MASK(i));
356 		/* Wait for write complete */
357 		readl(base + IMGU_REG_IRQCTRL_ENABLE(i));
358 	}
359 }
360 
imgu_css_hw_init(struct imgu_css * css)361 static int imgu_css_hw_init(struct imgu_css *css)
362 {
363 	/* For checking that streaming monitor statuses are valid */
364 	static const struct {
365 		u32 reg;
366 		u32 mask;
367 		const char *name;
368 	} stream_monitors[] = {
369 		{
370 			IMGU_REG_GP_SP1_STRMON_STAT,
371 			IMGU_GP_STRMON_STAT_ISP_PORT_SP12ISP,
372 			"ISP0 to SP0"
373 		}, {
374 			IMGU_REG_GP_ISP_STRMON_STAT,
375 			IMGU_GP_STRMON_STAT_SP1_PORT_ISP2SP1,
376 			"SP0 to ISP0"
377 		}, {
378 			IMGU_REG_GP_MOD_STRMON_STAT,
379 			IMGU_GP_STRMON_STAT_MOD_PORT_ISP2DMA,
380 			"ISP0 to DMA0"
381 		}, {
382 			IMGU_REG_GP_ISP_STRMON_STAT,
383 			IMGU_GP_STRMON_STAT_ISP_PORT_DMA2ISP,
384 			"DMA0 to ISP0"
385 		}, {
386 			IMGU_REG_GP_MOD_STRMON_STAT,
387 			IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC,
388 			"ISP0 to GDC0"
389 		}, {
390 			IMGU_REG_GP_MOD_STRMON_STAT,
391 			IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS,
392 			"GDC0 to ISP0"
393 		}, {
394 			IMGU_REG_GP_MOD_STRMON_STAT,
395 			IMGU_GP_STRMON_STAT_MOD_PORT_SP12DMA,
396 			"SP0 to DMA0"
397 		}, {
398 			IMGU_REG_GP_SP1_STRMON_STAT,
399 			IMGU_GP_STRMON_STAT_SP1_PORT_DMA2SP1,
400 			"DMA0 to SP0"
401 		}, {
402 			IMGU_REG_GP_MOD_STRMON_STAT,
403 			IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC,
404 			"SP0 to GDC0"
405 		}, {
406 			IMGU_REG_GP_MOD_STRMON_STAT,
407 			IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS,
408 			"GDC0 to SP0"
409 		},
410 	};
411 
412 	struct device *dev = css->dev;
413 	void __iomem *const base = css->base;
414 	u32 val, i;
415 
416 	/* Set instruction cache address and inv bit for ISP, SP, and SP1 */
417 	for (i = 0; i < IMGU_NUM_SP; i++) {
418 		struct imgu_fw_info *bi =
419 					&css->fwp->binary_header[css->fw_sp[i]];
420 
421 		writel(css->binary[css->fw_sp[i]].daddr,
422 		       base + IMGU_REG_SP_ICACHE_ADDR(bi->type));
423 		writel(readl(base + IMGU_REG_SP_CTRL(bi->type)) |
424 		       IMGU_CTRL_ICACHE_INV,
425 		       base + IMGU_REG_SP_CTRL(bi->type));
426 	}
427 	writel(css->binary[css->fw_bl].daddr, base + IMGU_REG_ISP_ICACHE_ADDR);
428 	writel(readl(base + IMGU_REG_ISP_CTRL) | IMGU_CTRL_ICACHE_INV,
429 	       base + IMGU_REG_ISP_CTRL);
430 
431 	/* Check that IMGU hardware is ready */
432 
433 	if (!(readl(base + IMGU_REG_SP_CTRL(0)) & IMGU_CTRL_IDLE)) {
434 		dev_err(dev, "SP is not idle\n");
435 		return -EIO;
436 	}
437 	if (!(readl(base + IMGU_REG_ISP_CTRL) & IMGU_CTRL_IDLE)) {
438 		dev_err(dev, "ISP is not idle\n");
439 		return -EIO;
440 	}
441 
442 	for (i = 0; i < ARRAY_SIZE(stream_monitors); i++) {
443 		val = readl(base + stream_monitors[i].reg);
444 		if (val & stream_monitors[i].mask) {
445 			dev_err(dev, "error: Stream monitor %s is valid\n",
446 				stream_monitors[i].name);
447 			return -EIO;
448 		}
449 	}
450 
451 	/* Initialize GDC with default values */
452 
453 	for (i = 0; i < ARRAY_SIZE(imgu_css_gdc_lut[0]); i++) {
454 		u32 val0 = imgu_css_gdc_lut[0][i] & IMGU_GDC_LUT_MASK;
455 		u32 val1 = imgu_css_gdc_lut[1][i] & IMGU_GDC_LUT_MASK;
456 		u32 val2 = imgu_css_gdc_lut[2][i] & IMGU_GDC_LUT_MASK;
457 		u32 val3 = imgu_css_gdc_lut[3][i] & IMGU_GDC_LUT_MASK;
458 
459 		writel(val0 | (val1 << 16),
460 		       base + IMGU_REG_GDC_LUT_BASE + i * 8);
461 		writel(val2 | (val3 << 16),
462 		       base + IMGU_REG_GDC_LUT_BASE + i * 8 + 4);
463 	}
464 
465 	return 0;
466 }
467 
468 /* Boot the given IPU3 CSS SP */
imgu_css_hw_start_sp(struct imgu_css * css,int sp)469 static int imgu_css_hw_start_sp(struct imgu_css *css, int sp)
470 {
471 	void __iomem *const base = css->base;
472 	struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
473 	struct imgu_abi_sp_init_dmem_cfg dmem_cfg = {
474 		.ddr_data_addr = css->binary[css->fw_sp[sp]].daddr
475 			+ bi->blob.data_source,
476 		.dmem_data_addr = bi->blob.data_target,
477 		.dmem_bss_addr = bi->blob.bss_target,
478 		.data_size = bi->blob.data_size,
479 		.bss_size = bi->blob.bss_size,
480 		.sp_id = sp,
481 	};
482 
483 	writes(&dmem_cfg, sizeof(dmem_cfg), base +
484 	       IMGU_REG_SP_DMEM_BASE(sp) + bi->info.sp.init_dmem_data);
485 
486 	writel(bi->info.sp.sp_entry, base + IMGU_REG_SP_START_ADDR(sp));
487 
488 	writel(readl(base + IMGU_REG_SP_CTRL(sp))
489 		| IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_SP_CTRL(sp));
490 
491 	if (imgu_hw_wait(css->base, IMGU_REG_SP_DMEM_BASE(sp)
492 			 + bi->info.sp.sw_state,
493 			 ~0, IMGU_ABI_SP_SWSTATE_INITIALIZED))
494 		return -EIO;
495 
496 	return 0;
497 }
498 
499 /* Start the IPU3 CSS ImgU (Imaging Unit) and all the SPs */
imgu_css_hw_start(struct imgu_css * css)500 static int imgu_css_hw_start(struct imgu_css *css)
501 {
502 	static const u32 event_mask =
503 		((1 << IMGU_ABI_EVTTYPE_OUT_FRAME_DONE) |
504 		(1 << IMGU_ABI_EVTTYPE_2ND_OUT_FRAME_DONE) |
505 		(1 << IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE) |
506 		(1 << IMGU_ABI_EVTTYPE_2ND_VF_OUT_FRAME_DONE) |
507 		(1 << IMGU_ABI_EVTTYPE_3A_STATS_DONE) |
508 		(1 << IMGU_ABI_EVTTYPE_DIS_STATS_DONE) |
509 		(1 << IMGU_ABI_EVTTYPE_PIPELINE_DONE) |
510 		(1 << IMGU_ABI_EVTTYPE_FRAME_TAGGED) |
511 		(1 << IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE) |
512 		(1 << IMGU_ABI_EVTTYPE_METADATA_DONE) |
513 		(1 << IMGU_ABI_EVTTYPE_ACC_STAGE_COMPLETE))
514 		<< IMGU_ABI_SP_COMM_EVENT_IRQ_MASK_OR_SHIFT;
515 
516 	void __iomem *const base = css->base;
517 	struct imgu_fw_info *bi, *bl = &css->fwp->binary_header[css->fw_bl];
518 	unsigned int i;
519 
520 	writel(IMGU_TLB_INVALIDATE, base + IMGU_REG_TLB_INVALIDATE);
521 
522 	/* Start bootloader */
523 
524 	writel(IMGU_ABI_BL_SWSTATE_BUSY,
525 	       base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.sw_state);
526 	writel(IMGU_NUM_SP,
527 	       base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.num_dma_cmds);
528 
529 	for (i = 0; i < IMGU_NUM_SP; i++) {
530 		int j = IMGU_NUM_SP - i - 1;	/* load sp1 first, then sp0 */
531 		struct imgu_fw_info *sp =
532 					&css->fwp->binary_header[css->fw_sp[j]];
533 		struct imgu_abi_bl_dma_cmd_entry dma_cmd = {
534 			.src_addr = css->binary[css->fw_sp[j]].daddr
535 				+ sp->blob.text_source,
536 			.size = sp->blob.text_size,
537 			.dst_type = IMGU_ABI_BL_DMACMD_TYPE_SP_PMEM,
538 			.dst_addr = IMGU_SP_PMEM_BASE(j),
539 		};
540 
541 		writes(&dma_cmd, sizeof(dma_cmd),
542 		       base + IMGU_REG_ISP_DMEM_BASE + i * sizeof(dma_cmd) +
543 		       bl->info.bl.dma_cmd_list);
544 	}
545 
546 	writel(bl->info.bl.bl_entry, base + IMGU_REG_ISP_START_ADDR);
547 
548 	writel(readl(base + IMGU_REG_ISP_CTRL)
549 		| IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_ISP_CTRL);
550 	if (imgu_hw_wait(css->base, IMGU_REG_ISP_DMEM_BASE
551 			 + bl->info.bl.sw_state, ~0,
552 			 IMGU_ABI_BL_SWSTATE_OK)) {
553 		dev_err(css->dev, "failed to start bootloader\n");
554 		return -EIO;
555 	}
556 
557 	/* Start ISP */
558 
559 	memset(css->xmem_sp_group_ptrs.vaddr, 0,
560 	       sizeof(struct imgu_abi_sp_group));
561 
562 	bi = &css->fwp->binary_header[css->fw_sp[0]];
563 
564 	writel(css->xmem_sp_group_ptrs.daddr,
565 	       base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.per_frame_data);
566 
567 	writel(IMGU_ABI_SP_SWSTATE_TERMINATED,
568 	       base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state);
569 	writel(1, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb);
570 
571 	if (imgu_css_hw_start_sp(css, 0))
572 		return -EIO;
573 
574 	writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.isp_started);
575 	writel(0, base + IMGU_REG_SP_DMEM_BASE(0) +
576 		bi->info.sp.host_sp_queues_initialized);
577 	writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sleep_mode);
578 	writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb);
579 	writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(0)
580 		+ bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
581 
582 	/* Enable all events for all queues */
583 
584 	for (i = 0; i < IPU3_CSS_PIPE_ID_NUM; i++)
585 		writel(event_mask, base + IMGU_REG_SP_DMEM_BASE(0)
586 			+ bi->info.sp.host_sp_com
587 			+ IMGU_ABI_SP_COMM_EVENT_IRQ_MASK(i));
588 	writel(1, base + IMGU_REG_SP_DMEM_BASE(0) +
589 		bi->info.sp.host_sp_queues_initialized);
590 
591 	/* Start SP1 */
592 
593 	bi = &css->fwp->binary_header[css->fw_sp[1]];
594 
595 	writel(IMGU_ABI_SP_SWSTATE_TERMINATED,
596 	       base + IMGU_REG_SP_DMEM_BASE(1) + bi->info.sp.sw_state);
597 
598 	if (imgu_css_hw_start_sp(css, 1))
599 		return -EIO;
600 
601 	writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(1)
602 		+ bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
603 
604 	return 0;
605 }
606 
imgu_css_hw_stop(struct imgu_css * css)607 static void imgu_css_hw_stop(struct imgu_css *css)
608 {
609 	void __iomem *const base = css->base;
610 	struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]];
611 
612 	/* Stop fw */
613 	writel(IMGU_ABI_SP_COMM_COMMAND_TERMINATE,
614 	       base + IMGU_REG_SP_DMEM_BASE(0) +
615 	       bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
616 	if (imgu_hw_wait(css->base, IMGU_REG_SP_CTRL(0),
617 			 IMGU_CTRL_IDLE, IMGU_CTRL_IDLE))
618 		dev_err(css->dev, "wait sp0 idle timeout.\n");
619 	if (readl(base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state) !=
620 		  IMGU_ABI_SP_SWSTATE_TERMINATED)
621 		dev_err(css->dev, "sp0 is not terminated.\n");
622 	if (imgu_hw_wait(css->base, IMGU_REG_ISP_CTRL,
623 			 IMGU_CTRL_IDLE, IMGU_CTRL_IDLE))
624 		dev_err(css->dev, "wait isp idle timeout\n");
625 }
626 
imgu_css_hw_cleanup(struct imgu_css * css)627 static void imgu_css_hw_cleanup(struct imgu_css *css)
628 {
629 	void __iomem *const base = css->base;
630 
631 	/** Reset CSS **/
632 
633 	/* Clear the CSS busy signal */
634 	readl(base + IMGU_REG_GP_BUSY);
635 	writel(0, base + IMGU_REG_GP_BUSY);
636 
637 	/* Wait for idle signal */
638 	if (imgu_hw_wait(css->base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
639 			 IMGU_STATE_IDLE_STS))
640 		dev_err(css->dev, "failed to shut down hw cleanly\n");
641 
642 	/* Reset the css */
643 	writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET,
644 	       base + IMGU_REG_PM_CTRL);
645 
646 	usleep_range(200, 300);
647 }
648 
imgu_css_pipeline_cleanup(struct imgu_css * css,unsigned int pipe)649 static void imgu_css_pipeline_cleanup(struct imgu_css *css, unsigned int pipe)
650 {
651 	struct imgu_device *imgu = dev_get_drvdata(css->dev);
652 	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
653 	unsigned int i;
654 
655 	imgu_css_pool_cleanup(imgu, &css_pipe->pool.parameter_set_info);
656 	imgu_css_pool_cleanup(imgu, &css_pipe->pool.acc);
657 	imgu_css_pool_cleanup(imgu, &css_pipe->pool.gdc);
658 	imgu_css_pool_cleanup(imgu, &css_pipe->pool.obgrid);
659 
660 	for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
661 		imgu_css_pool_cleanup(imgu, &css_pipe->pool.binary_params_p[i]);
662 }
663 
664 /*
665  * This function initializes various stages of the
666  * IPU3 CSS ISP pipeline
667  */
imgu_css_pipeline_init(struct imgu_css * css,unsigned int pipe)668 static int imgu_css_pipeline_init(struct imgu_css *css, unsigned int pipe)
669 {
670 	static const int BYPC = 2;	/* Bytes per component */
671 	static const struct imgu_abi_buffer_sp buffer_sp_init = {
672 		.buf_src = {.queue_id = IMGU_ABI_QUEUE_EVENT_ID},
673 		.buf_type = IMGU_ABI_BUFFER_TYPE_INVALID,
674 	};
675 
676 	struct imgu_abi_isp_iterator_config *cfg_iter;
677 	struct imgu_abi_isp_ref_config *cfg_ref;
678 	struct imgu_abi_isp_dvs_config *cfg_dvs;
679 	struct imgu_abi_isp_tnr3_config *cfg_tnr;
680 	struct imgu_abi_isp_ref_dmem_state *cfg_ref_state;
681 	struct imgu_abi_isp_tnr3_dmem_state *cfg_tnr_state;
682 
683 	const int stage = 0;
684 	unsigned int i, j;
685 
686 	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
687 	struct imgu_css_queue *css_queue_in =
688 			&css_pipe->queue[IPU3_CSS_QUEUE_IN];
689 	struct imgu_css_queue *css_queue_out =
690 			&css_pipe->queue[IPU3_CSS_QUEUE_OUT];
691 	struct imgu_css_queue *css_queue_vf =
692 			&css_pipe->queue[IPU3_CSS_QUEUE_VF];
693 	const struct imgu_fw_info *bi =
694 			&css->fwp->binary_header[css_pipe->bindex];
695 	const unsigned int stripes = bi->info.isp.sp.iterator.num_stripes;
696 
697 	struct imgu_fw_config_memory_offsets *cofs = (void *)css->fwp +
698 		bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_CONFIG];
699 	struct imgu_fw_state_memory_offsets *sofs = (void *)css->fwp +
700 		bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_STATE];
701 
702 	struct imgu_abi_isp_stage *isp_stage;
703 	struct imgu_abi_sp_stage *sp_stage;
704 	struct imgu_abi_sp_group *sp_group;
705 	struct imgu_abi_frames_sp *frames_sp;
706 	struct imgu_abi_frame_sp *frame_sp;
707 	struct imgu_abi_frame_sp_info *frame_sp_info;
708 
709 	const unsigned int bds_width_pad =
710 				ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width,
711 				      2 * IPU3_UAPI_ISP_VEC_ELEMS);
712 
713 	const enum imgu_abi_memories m0 = IMGU_ABI_MEM_ISP_DMEM0;
714 	enum imgu_abi_param_class cfg = IMGU_ABI_PARAM_CLASS_CONFIG;
715 	void *vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr;
716 
717 	struct imgu_device *imgu = dev_get_drvdata(css->dev);
718 
719 	dev_dbg(css->dev, "%s for pipe %d", __func__, pipe);
720 
721 	/* Configure iterator */
722 
723 	cfg_iter = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
724 					       &cofs->dmem.iterator,
725 					       sizeof(*cfg_iter), vaddr);
726 	if (!cfg_iter)
727 		goto bad_firmware;
728 
729 	frame_sp_info = &cfg_iter->input_info;
730 	frame_sp_info->res.width	= css_queue_in->fmt.mpix.width;
731 	frame_sp_info->res.height	= css_queue_in->fmt.mpix.height;
732 	frame_sp_info->padded_width	= css_queue_in->width_pad;
733 	frame_sp_info->format		= css_queue_in->css_fmt->frame_format;
734 	frame_sp_info->raw_bit_depth	= css_queue_in->css_fmt->bit_depth;
735 	frame_sp_info->raw_bayer_order	= css_queue_in->css_fmt->bayer_order;
736 	frame_sp_info->raw_type		= IMGU_ABI_RAW_TYPE_BAYER;
737 
738 	frame_sp_info = &cfg_iter->internal_info;
739 	frame_sp_info->res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width;
740 	frame_sp_info->res.height = css_pipe->rect[IPU3_CSS_RECT_BDS].height;
741 	frame_sp_info->padded_width	= bds_width_pad;
742 	frame_sp_info->format		= css_queue_out->css_fmt->frame_format;
743 	frame_sp_info->raw_bit_depth	= css_queue_out->css_fmt->bit_depth;
744 	frame_sp_info->raw_bayer_order	= css_queue_out->css_fmt->bayer_order;
745 	frame_sp_info->raw_type		= IMGU_ABI_RAW_TYPE_BAYER;
746 
747 	frame_sp_info = &cfg_iter->output_info;
748 	frame_sp_info->res.width	= css_queue_out->fmt.mpix.width;
749 	frame_sp_info->res.height	= css_queue_out->fmt.mpix.height;
750 	frame_sp_info->padded_width	= css_queue_out->width_pad;
751 	frame_sp_info->format		= css_queue_out->css_fmt->frame_format;
752 	frame_sp_info->raw_bit_depth	= css_queue_out->css_fmt->bit_depth;
753 	frame_sp_info->raw_bayer_order	= css_queue_out->css_fmt->bayer_order;
754 	frame_sp_info->raw_type		= IMGU_ABI_RAW_TYPE_BAYER;
755 
756 	frame_sp_info = &cfg_iter->vf_info;
757 	frame_sp_info->res.width	= css_queue_vf->fmt.mpix.width;
758 	frame_sp_info->res.height	= css_queue_vf->fmt.mpix.height;
759 	frame_sp_info->padded_width	= css_queue_vf->width_pad;
760 	frame_sp_info->format		= css_queue_vf->css_fmt->frame_format;
761 	frame_sp_info->raw_bit_depth	= css_queue_vf->css_fmt->bit_depth;
762 	frame_sp_info->raw_bayer_order	= css_queue_vf->css_fmt->bayer_order;
763 	frame_sp_info->raw_type		= IMGU_ABI_RAW_TYPE_BAYER;
764 
765 	cfg_iter->dvs_envelope.width =
766 				css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width;
767 	cfg_iter->dvs_envelope.height =
768 				css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height;
769 
770 	/* Configure reference (delay) frames */
771 
772 	cfg_ref = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
773 					      &cofs->dmem.ref,
774 					      sizeof(*cfg_ref), vaddr);
775 	if (!cfg_ref)
776 		goto bad_firmware;
777 
778 	cfg_ref->port_b.crop = 0;
779 	cfg_ref->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES / BYPC;
780 	cfg_ref->port_b.width =
781 		css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width;
782 	cfg_ref->port_b.stride =
783 		css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline;
784 	cfg_ref->width_a_over_b =
785 				IPU3_UAPI_ISP_VEC_ELEMS / cfg_ref->port_b.elems;
786 	cfg_ref->dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1;
787 	for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) {
788 		cfg_ref->ref_frame_addr_y[i] =
789 			css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr;
790 		cfg_ref->ref_frame_addr_c[i] =
791 			css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr +
792 			css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline *
793 			css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height;
794 	}
795 	for (; i < IMGU_ABI_FRAMES_REF; i++) {
796 		cfg_ref->ref_frame_addr_y[i] = 0;
797 		cfg_ref->ref_frame_addr_c[i] = 0;
798 	}
799 
800 	/* Configure DVS (digital video stabilization) */
801 
802 	cfg_dvs = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
803 					      &cofs->dmem.dvs, sizeof(*cfg_dvs),
804 					      vaddr);
805 	if (!cfg_dvs)
806 		goto bad_firmware;
807 
808 	cfg_dvs->num_horizontal_blocks =
809 			ALIGN(DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].width,
810 					   IMGU_DVS_BLOCK_W), 2);
811 	cfg_dvs->num_vertical_blocks =
812 			DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].height,
813 				     IMGU_DVS_BLOCK_H);
814 
815 	/* Configure TNR (temporal noise reduction) */
816 
817 	if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) {
818 		cfg_tnr = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
819 						      &cofs->dmem.tnr3,
820 						      sizeof(*cfg_tnr),
821 						      vaddr);
822 		if (!cfg_tnr)
823 			goto bad_firmware;
824 
825 		cfg_tnr->port_b.crop = 0;
826 		cfg_tnr->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES;
827 		cfg_tnr->port_b.width =
828 			css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width;
829 		cfg_tnr->port_b.stride =
830 			css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline;
831 		cfg_tnr->width_a_over_b =
832 			IPU3_UAPI_ISP_VEC_ELEMS / cfg_tnr->port_b.elems;
833 		cfg_tnr->frame_height =
834 			css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height;
835 		cfg_tnr->delay_frame = IPU3_CSS_AUX_FRAMES - 1;
836 		for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
837 			cfg_tnr->frame_addr[i] =
838 				css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR]
839 					.mem[i].daddr;
840 		for (; i < IMGU_ABI_FRAMES_TNR; i++)
841 			cfg_tnr->frame_addr[i] = 0;
842 	}
843 
844 	/* Configure ref dmem state parameters */
845 
846 	cfg = IMGU_ABI_PARAM_CLASS_STATE;
847 	vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr;
848 
849 	cfg_ref_state = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
850 						    &sofs->dmem.ref,
851 						    sizeof(*cfg_ref_state),
852 						    vaddr);
853 	if (!cfg_ref_state)
854 		goto bad_firmware;
855 
856 	cfg_ref_state->ref_in_buf_idx = 0;
857 	cfg_ref_state->ref_out_buf_idx = 1;
858 
859 	/* Configure tnr dmem state parameters */
860 	if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) {
861 		cfg_tnr_state =
862 			imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
863 						    &sofs->dmem.tnr3,
864 						    sizeof(*cfg_tnr_state),
865 						    vaddr);
866 		if (!cfg_tnr_state)
867 			goto bad_firmware;
868 
869 		cfg_tnr_state->in_bufidx = 0;
870 		cfg_tnr_state->out_bufidx = 1;
871 		cfg_tnr_state->bypass_filter = 0;
872 		cfg_tnr_state->total_frame_counter = 0;
873 		for (i = 0; i < IMGU_ABI_BUF_SETS_TNR; i++)
874 			cfg_tnr_state->buffer_frame_counter[i] = 0;
875 	}
876 
877 	/* Configure ISP stage */
878 
879 	isp_stage = css_pipe->xmem_isp_stage_ptrs[pipe][stage].vaddr;
880 	memset(isp_stage, 0, sizeof(*isp_stage));
881 	isp_stage->blob_info = bi->blob;
882 	isp_stage->binary_info = bi->info.isp.sp;
883 	strscpy(isp_stage->binary_name,
884 		(char *)css->fwp + bi->blob.prog_name_offset,
885 		sizeof(isp_stage->binary_name));
886 	isp_stage->mem_initializers = bi->info.isp.sp.mem_initializers;
887 	for (i = IMGU_ABI_PARAM_CLASS_CONFIG; i < IMGU_ABI_PARAM_CLASS_NUM; i++)
888 		for (j = 0; j < IMGU_ABI_NUM_MEMORIES; j++)
889 			isp_stage->mem_initializers.params[i][j].address =
890 					css_pipe->binary_params_cs[i - 1][j].daddr;
891 
892 	/* Configure SP stage */
893 
894 	sp_stage = css_pipe->xmem_sp_stage_ptrs[pipe][stage].vaddr;
895 	memset(sp_stage, 0, sizeof(*sp_stage));
896 
897 	frames_sp = &sp_stage->frames;
898 	frames_sp->in.buf_attr = buffer_sp_init;
899 	for (i = 0; i < IMGU_ABI_BINARY_MAX_OUTPUT_PORTS; i++)
900 		frames_sp->out[i].buf_attr = buffer_sp_init;
901 	frames_sp->out_vf.buf_attr = buffer_sp_init;
902 	frames_sp->s3a_buf = buffer_sp_init;
903 	frames_sp->dvs_buf = buffer_sp_init;
904 
905 	sp_stage->stage_type = IMGU_ABI_STAGE_TYPE_ISP;
906 	sp_stage->num = stage;
907 	sp_stage->isp_online = 0;
908 	sp_stage->isp_copy_vf = 0;
909 	sp_stage->isp_copy_output = 0;
910 
911 	sp_stage->enable.vf_output = css_pipe->vf_output_en;
912 
913 	frames_sp->effective_in_res.width =
914 				css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].width;
915 	frames_sp->effective_in_res.height =
916 				css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].height;
917 
918 	frame_sp = &frames_sp->in;
919 	frame_sp->info.res.width	= css_queue_in->fmt.mpix.width;
920 	frame_sp->info.res.height	= css_queue_in->fmt.mpix.height;
921 	frame_sp->info.padded_width	= css_queue_in->width_pad;
922 	frame_sp->info.format		= css_queue_in->css_fmt->frame_format;
923 	frame_sp->info.raw_bit_depth	= css_queue_in->css_fmt->bit_depth;
924 	frame_sp->info.raw_bayer_order	= css_queue_in->css_fmt->bayer_order;
925 	frame_sp->info.raw_type		= IMGU_ABI_RAW_TYPE_BAYER;
926 	frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_C_ID;
927 	frame_sp->buf_attr.buf_type	= IMGU_ABI_BUFFER_TYPE_INPUT_FRAME;
928 
929 	frame_sp = &frames_sp->out[0];
930 	frame_sp->info.res.width	= css_queue_out->fmt.mpix.width;
931 	frame_sp->info.res.height	= css_queue_out->fmt.mpix.height;
932 	frame_sp->info.padded_width	= css_queue_out->width_pad;
933 	frame_sp->info.format		= css_queue_out->css_fmt->frame_format;
934 	frame_sp->info.raw_bit_depth	= css_queue_out->css_fmt->bit_depth;
935 	frame_sp->info.raw_bayer_order	= css_queue_out->css_fmt->bayer_order;
936 	frame_sp->info.raw_type		= IMGU_ABI_RAW_TYPE_BAYER;
937 	frame_sp->planes.nv.uv.offset	= css_queue_out->width_pad *
938 					  css_queue_out->fmt.mpix.height;
939 	frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_D_ID;
940 	frame_sp->buf_attr.buf_type	= IMGU_ABI_BUFFER_TYPE_OUTPUT_FRAME;
941 
942 	frame_sp = &frames_sp->out[1];
943 	frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_EVENT_ID;
944 
945 	frame_sp_info = &frames_sp->internal_frame_info;
946 	frame_sp_info->res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width;
947 	frame_sp_info->res.height = css_pipe->rect[IPU3_CSS_RECT_BDS].height;
948 	frame_sp_info->padded_width	= bds_width_pad;
949 	frame_sp_info->format		= css_queue_out->css_fmt->frame_format;
950 	frame_sp_info->raw_bit_depth	= css_queue_out->css_fmt->bit_depth;
951 	frame_sp_info->raw_bayer_order	= css_queue_out->css_fmt->bayer_order;
952 	frame_sp_info->raw_type		= IMGU_ABI_RAW_TYPE_BAYER;
953 
954 	frame_sp = &frames_sp->out_vf;
955 	frame_sp->info.res.width	= css_queue_vf->fmt.mpix.width;
956 	frame_sp->info.res.height	= css_queue_vf->fmt.mpix.height;
957 	frame_sp->info.padded_width	= css_queue_vf->width_pad;
958 	frame_sp->info.format		= css_queue_vf->css_fmt->frame_format;
959 	frame_sp->info.raw_bit_depth	= css_queue_vf->css_fmt->bit_depth;
960 	frame_sp->info.raw_bayer_order	= css_queue_vf->css_fmt->bayer_order;
961 	frame_sp->info.raw_type		= IMGU_ABI_RAW_TYPE_BAYER;
962 	frame_sp->planes.yuv.u.offset	= css_queue_vf->width_pad *
963 					  css_queue_vf->fmt.mpix.height;
964 	frame_sp->planes.yuv.v.offset	= css_queue_vf->width_pad *
965 					  css_queue_vf->fmt.mpix.height * 5 / 4;
966 	frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_E_ID;
967 	frame_sp->buf_attr.buf_type	= IMGU_ABI_BUFFER_TYPE_VF_OUTPUT_FRAME;
968 
969 	frames_sp->s3a_buf.buf_src.queue_id = IMGU_ABI_QUEUE_F_ID;
970 	frames_sp->s3a_buf.buf_type	= IMGU_ABI_BUFFER_TYPE_3A_STATISTICS;
971 
972 	frames_sp->dvs_buf.buf_src.queue_id = IMGU_ABI_QUEUE_G_ID;
973 	frames_sp->dvs_buf.buf_type	= IMGU_ABI_BUFFER_TYPE_DIS_STATISTICS;
974 
975 	sp_stage->dvs_envelope.width =
976 				css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width;
977 	sp_stage->dvs_envelope.height =
978 				css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height;
979 
980 	sp_stage->isp_pipe_version =
981 				bi->info.isp.sp.pipeline.isp_pipe_version;
982 	sp_stage->isp_deci_log_factor =
983 			clamp(max(fls(css_pipe->rect[IPU3_CSS_RECT_BDS].width /
984 				      IMGU_MAX_BQ_GRID_WIDTH),
985 				  fls(css_pipe->rect[IPU3_CSS_RECT_BDS].height /
986 				      IMGU_MAX_BQ_GRID_HEIGHT)) - 1, 3, 5);
987 	sp_stage->isp_vf_downscale_bits = 0;
988 	sp_stage->if_config_index = 255;
989 	sp_stage->sp_enable_xnr = 0;
990 	sp_stage->num_stripes = stripes;
991 	sp_stage->enable.s3a = 1;
992 	sp_stage->enable.dvs_stats = 0;
993 
994 	sp_stage->xmem_bin_addr = css->binary[css_pipe->bindex].daddr;
995 	sp_stage->xmem_map_addr = css_pipe->sp_ddr_ptrs.daddr;
996 	sp_stage->isp_stage_addr =
997 		css_pipe->xmem_isp_stage_ptrs[pipe][stage].daddr;
998 
999 	/* Configure SP group */
1000 
1001 	sp_group = css->xmem_sp_group_ptrs.vaddr;
1002 	memset(&sp_group->pipe[pipe], 0, sizeof(struct imgu_abi_sp_pipeline));
1003 
1004 	sp_group->pipe[pipe].num_stages = 1;
1005 	sp_group->pipe[pipe].pipe_id = css_pipe->pipe_id;
1006 	sp_group->pipe[pipe].thread_id = pipe;
1007 	sp_group->pipe[pipe].pipe_num = pipe;
1008 	sp_group->pipe[pipe].num_execs = -1;
1009 	sp_group->pipe[pipe].pipe_qos_config = -1;
1010 	sp_group->pipe[pipe].required_bds_factor = 0;
1011 	sp_group->pipe[pipe].dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1;
1012 	sp_group->pipe[pipe].inout_port_config =
1013 					IMGU_ABI_PORT_CONFIG_TYPE_INPUT_HOST |
1014 					IMGU_ABI_PORT_CONFIG_TYPE_OUTPUT_HOST;
1015 	sp_group->pipe[pipe].scaler_pp_lut = 0;
1016 	sp_group->pipe[pipe].shading.internal_frame_origin_x_bqs_on_sctbl = 0;
1017 	sp_group->pipe[pipe].shading.internal_frame_origin_y_bqs_on_sctbl = 0;
1018 	sp_group->pipe[pipe].sp_stage_addr[stage] =
1019 			css_pipe->xmem_sp_stage_ptrs[pipe][stage].daddr;
1020 	sp_group->pipe[pipe].pipe_config =
1021 			bi->info.isp.sp.enable.params ? (1 << pipe) : 0;
1022 	sp_group->pipe[pipe].pipe_config |= IMGU_ABI_PIPE_CONFIG_ACQUIRE_ISP;
1023 
1024 	/* Initialize parameter pools */
1025 
1026 	if (imgu_css_pool_init(imgu, &css_pipe->pool.parameter_set_info,
1027 			       sizeof(struct imgu_abi_parameter_set_info)) ||
1028 	    imgu_css_pool_init(imgu, &css_pipe->pool.acc,
1029 			       sizeof(struct imgu_abi_acc_param)) ||
1030 	    imgu_css_pool_init(imgu, &css_pipe->pool.gdc,
1031 			       sizeof(struct imgu_abi_gdc_warp_param) *
1032 			       3 * cfg_dvs->num_horizontal_blocks / 2 *
1033 			       cfg_dvs->num_vertical_blocks) ||
1034 	    imgu_css_pool_init(imgu, &css_pipe->pool.obgrid,
1035 			       imgu_css_fw_obgrid_size(
1036 			       &css->fwp->binary_header[css_pipe->bindex])))
1037 		goto out_of_memory;
1038 
1039 	for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1040 		if (imgu_css_pool_init(imgu,
1041 				       &css_pipe->pool.binary_params_p[i],
1042 				       bi->info.isp.sp.mem_initializers.params
1043 				       [IMGU_ABI_PARAM_CLASS_PARAM][i].size))
1044 			goto out_of_memory;
1045 
1046 	return 0;
1047 
1048 bad_firmware:
1049 	imgu_css_pipeline_cleanup(css, pipe);
1050 	return -EPROTO;
1051 
1052 out_of_memory:
1053 	imgu_css_pipeline_cleanup(css, pipe);
1054 	return -ENOMEM;
1055 }
1056 
imgu_css_queue_pos(struct imgu_css * css,int queue,int thread)1057 static u8 imgu_css_queue_pos(struct imgu_css *css, int queue, int thread)
1058 {
1059 	static const unsigned int sp;
1060 	void __iomem *const base = css->base;
1061 	struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1062 	struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1063 						bi->info.sp.host_sp_queue;
1064 
1065 	return queue >= 0 ? readb(&q->host2sp_bufq_info[thread][queue].end) :
1066 			    readb(&q->host2sp_evtq_info.end);
1067 }
1068 
1069 /* Sent data to sp using given buffer queue, or if queue < 0, event queue. */
imgu_css_queue_data(struct imgu_css * css,int queue,int thread,u32 data)1070 static int imgu_css_queue_data(struct imgu_css *css,
1071 			       int queue, int thread, u32 data)
1072 {
1073 	static const unsigned int sp;
1074 	void __iomem *const base = css->base;
1075 	struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1076 	struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1077 						bi->info.sp.host_sp_queue;
1078 	u8 size, start, end, end2;
1079 
1080 	if (queue >= 0) {
1081 		size = readb(&q->host2sp_bufq_info[thread][queue].size);
1082 		start = readb(&q->host2sp_bufq_info[thread][queue].start);
1083 		end = readb(&q->host2sp_bufq_info[thread][queue].end);
1084 	} else {
1085 		size = readb(&q->host2sp_evtq_info.size);
1086 		start = readb(&q->host2sp_evtq_info.start);
1087 		end = readb(&q->host2sp_evtq_info.end);
1088 	}
1089 
1090 	if (size == 0)
1091 		return -EIO;
1092 
1093 	end2 = (end + 1) % size;
1094 	if (end2 == start)
1095 		return -EBUSY;	/* Queue full */
1096 
1097 	if (queue >= 0) {
1098 		writel(data, &q->host2sp_bufq[thread][queue][end]);
1099 		writeb(end2, &q->host2sp_bufq_info[thread][queue].end);
1100 	} else {
1101 		writel(data, &q->host2sp_evtq[end]);
1102 		writeb(end2, &q->host2sp_evtq_info.end);
1103 	}
1104 
1105 	return 0;
1106 }
1107 
1108 /* Receive data using given buffer queue, or if queue < 0, event queue. */
imgu_css_dequeue_data(struct imgu_css * css,int queue,u32 * data)1109 static int imgu_css_dequeue_data(struct imgu_css *css, int queue, u32 *data)
1110 {
1111 	static const unsigned int sp;
1112 	void __iomem *const base = css->base;
1113 	struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1114 	struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1115 						bi->info.sp.host_sp_queue;
1116 	u8 size, start, end, start2;
1117 
1118 	if (queue >= 0) {
1119 		size = readb(&q->sp2host_bufq_info[queue].size);
1120 		start = readb(&q->sp2host_bufq_info[queue].start);
1121 		end = readb(&q->sp2host_bufq_info[queue].end);
1122 	} else {
1123 		size = readb(&q->sp2host_evtq_info.size);
1124 		start = readb(&q->sp2host_evtq_info.start);
1125 		end = readb(&q->sp2host_evtq_info.end);
1126 	}
1127 
1128 	if (size == 0)
1129 		return -EIO;
1130 
1131 	if (end == start)
1132 		return -EBUSY;	/* Queue empty */
1133 
1134 	start2 = (start + 1) % size;
1135 
1136 	if (queue >= 0) {
1137 		*data = readl(&q->sp2host_bufq[queue][start]);
1138 		writeb(start2, &q->sp2host_bufq_info[queue].start);
1139 	} else {
1140 		int r;
1141 
1142 		*data = readl(&q->sp2host_evtq[start]);
1143 		writeb(start2, &q->sp2host_evtq_info.start);
1144 
1145 		/* Acknowledge events dequeued from event queue */
1146 		r = imgu_css_queue_data(css, queue, 0,
1147 					IMGU_ABI_EVENT_EVENT_DEQUEUED);
1148 		if (r < 0)
1149 			return r;
1150 	}
1151 
1152 	return 0;
1153 }
1154 
1155 /* Free binary-specific resources */
imgu_css_binary_cleanup(struct imgu_css * css,unsigned int pipe)1156 static void imgu_css_binary_cleanup(struct imgu_css *css, unsigned int pipe)
1157 {
1158 	struct imgu_device *imgu = dev_get_drvdata(css->dev);
1159 	unsigned int i, j;
1160 
1161 	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1162 
1163 	for (j = 0; j < IMGU_ABI_PARAM_CLASS_NUM - 1; j++)
1164 		for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1165 			imgu_dmamap_free(imgu,
1166 					 &css_pipe->binary_params_cs[j][i]);
1167 
1168 	j = IPU3_CSS_AUX_FRAME_REF;
1169 	for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1170 		imgu_dmamap_free(imgu,
1171 				 &css_pipe->aux_frames[j].mem[i]);
1172 
1173 	j = IPU3_CSS_AUX_FRAME_TNR;
1174 	for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1175 		imgu_dmamap_free(imgu,
1176 				 &css_pipe->aux_frames[j].mem[i]);
1177 }
1178 
imgu_css_binary_preallocate(struct imgu_css * css,unsigned int pipe)1179 static int imgu_css_binary_preallocate(struct imgu_css *css, unsigned int pipe)
1180 {
1181 	struct imgu_device *imgu = dev_get_drvdata(css->dev);
1182 	unsigned int i, j;
1183 
1184 	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1185 
1186 	for (j = IMGU_ABI_PARAM_CLASS_CONFIG;
1187 	     j < IMGU_ABI_PARAM_CLASS_NUM; j++)
1188 		for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1189 			if (!imgu_dmamap_alloc(imgu,
1190 					       &css_pipe->binary_params_cs[j - 1][i],
1191 					       CSS_ABI_SIZE))
1192 				goto out_of_memory;
1193 
1194 	for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1195 		if (!imgu_dmamap_alloc(imgu,
1196 				       &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i],
1197 				       CSS_BDS_SIZE))
1198 			goto out_of_memory;
1199 
1200 	for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1201 		if (!imgu_dmamap_alloc(imgu,
1202 				       &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].mem[i],
1203 				       CSS_GDC_SIZE))
1204 			goto out_of_memory;
1205 
1206 	return 0;
1207 
1208 out_of_memory:
1209 	imgu_css_binary_cleanup(css, pipe);
1210 	return -ENOMEM;
1211 }
1212 
1213 /* allocate binary-specific resources */
imgu_css_binary_setup(struct imgu_css * css,unsigned int pipe)1214 static int imgu_css_binary_setup(struct imgu_css *css, unsigned int pipe)
1215 {
1216 	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1217 	struct imgu_fw_info *bi = &css->fwp->binary_header[css_pipe->bindex];
1218 	struct imgu_device *imgu = dev_get_drvdata(css->dev);
1219 	int i, j, size;
1220 	static const int BYPC = 2;	/* Bytes per component */
1221 	unsigned int w, h;
1222 
1223 	/* Allocate parameter memory blocks for this binary */
1224 
1225 	for (j = IMGU_ABI_PARAM_CLASS_CONFIG; j < IMGU_ABI_PARAM_CLASS_NUM; j++)
1226 		for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) {
1227 			if (imgu_css_dma_buffer_resize(
1228 			    imgu,
1229 			    &css_pipe->binary_params_cs[j - 1][i],
1230 			    bi->info.isp.sp.mem_initializers.params[j][i].size))
1231 				goto out_of_memory;
1232 		}
1233 
1234 	/* Allocate internal frame buffers */
1235 
1236 	/* Reference frames for DVS, FRAME_FORMAT_YUV420_16 */
1237 	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel = BYPC;
1238 	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width =
1239 					css_pipe->rect[IPU3_CSS_RECT_BDS].width;
1240 	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height =
1241 				ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].height,
1242 				      IMGU_DVS_BLOCK_H) + 2 * IMGU_GDC_BUF_Y;
1243 	h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height;
1244 	w = ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width,
1245 		  2 * IPU3_UAPI_ISP_VEC_ELEMS) + 2 * IMGU_GDC_BUF_X;
1246 	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline =
1247 		css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel * w;
1248 	size = w * h * BYPC + (w / 2) * (h / 2) * BYPC * 2;
1249 	for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1250 		if (imgu_css_dma_buffer_resize(
1251 			imgu,
1252 			&css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i],
1253 			size))
1254 			goto out_of_memory;
1255 
1256 	/* TNR frames for temporal noise reduction, FRAME_FORMAT_YUV_LINE */
1257 	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperpixel = 1;
1258 	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width =
1259 			roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].width,
1260 				bi->info.isp.sp.block.block_width *
1261 				IPU3_UAPI_ISP_VEC_ELEMS);
1262 	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height =
1263 			roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].height,
1264 				bi->info.isp.sp.block.output_block_height);
1265 
1266 	w = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width;
1267 	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline = w;
1268 	h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height;
1269 	size = w * ALIGN(h * 3 / 2 + 3, 2);	/* +3 for vf_pp prefetch */
1270 	for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1271 		if (imgu_css_dma_buffer_resize(
1272 			imgu,
1273 			&css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].mem[i],
1274 			size))
1275 			goto out_of_memory;
1276 
1277 	return 0;
1278 
1279 out_of_memory:
1280 	imgu_css_binary_cleanup(css, pipe);
1281 	return -ENOMEM;
1282 }
1283 
imgu_css_start_streaming(struct imgu_css * css)1284 int imgu_css_start_streaming(struct imgu_css *css)
1285 {
1286 	u32 data;
1287 	int r, pipe;
1288 
1289 	if (css->streaming)
1290 		return -EPROTO;
1291 
1292 	for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1293 		r = imgu_css_binary_setup(css, pipe);
1294 		if (r < 0)
1295 			return r;
1296 	}
1297 
1298 	r = imgu_css_hw_init(css);
1299 	if (r < 0)
1300 		return r;
1301 
1302 	r = imgu_css_hw_start(css);
1303 	if (r < 0)
1304 		goto fail;
1305 
1306 	for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1307 		r = imgu_css_pipeline_init(css, pipe);
1308 		if (r < 0)
1309 			goto fail;
1310 	}
1311 
1312 	css->streaming = true;
1313 
1314 	imgu_css_hw_enable_irq(css);
1315 
1316 	/* Initialize parameters to default */
1317 	for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1318 		r = imgu_css_set_parameters(css, pipe, NULL);
1319 		if (r < 0)
1320 			goto fail;
1321 	}
1322 
1323 	while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_A_ID, &data)))
1324 		;
1325 	if (r != -EBUSY)
1326 		goto fail;
1327 
1328 	while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_B_ID, &data)))
1329 		;
1330 	if (r != -EBUSY)
1331 		goto fail;
1332 
1333 	for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1334 		r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
1335 					IMGU_ABI_EVENT_START_STREAM |
1336 					pipe << 16);
1337 		if (r < 0)
1338 			goto fail;
1339 	}
1340 
1341 	return 0;
1342 
1343 fail:
1344 	css->streaming = false;
1345 	imgu_css_hw_cleanup(css);
1346 	for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1347 		imgu_css_pipeline_cleanup(css, pipe);
1348 		imgu_css_binary_cleanup(css, pipe);
1349 	}
1350 
1351 	return r;
1352 }
1353 
imgu_css_stop_streaming(struct imgu_css * css)1354 void imgu_css_stop_streaming(struct imgu_css *css)
1355 {
1356 	struct imgu_css_buffer *b, *b0;
1357 	int q, r, pipe;
1358 
1359 	for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1360 		r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
1361 					IMGU_ABI_EVENT_STOP_STREAM);
1362 		if (r < 0)
1363 			dev_warn(css->dev, "failed on stop stream event\n");
1364 	}
1365 
1366 	if (!css->streaming)
1367 		return;
1368 
1369 	imgu_css_hw_stop(css);
1370 
1371 	imgu_css_hw_cleanup(css);
1372 
1373 	for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1374 		struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1375 
1376 		imgu_css_pipeline_cleanup(css, pipe);
1377 
1378 		spin_lock(&css_pipe->qlock);
1379 		for (q = 0; q < IPU3_CSS_QUEUES; q++)
1380 			list_for_each_entry_safe(b, b0,
1381 						 &css_pipe->queue[q].bufs,
1382 						 list) {
1383 				b->state = IPU3_CSS_BUFFER_FAILED;
1384 				list_del(&b->list);
1385 			}
1386 		spin_unlock(&css_pipe->qlock);
1387 	}
1388 
1389 	css->streaming = false;
1390 }
1391 
imgu_css_pipe_queue_empty(struct imgu_css * css,unsigned int pipe)1392 bool imgu_css_pipe_queue_empty(struct imgu_css *css, unsigned int pipe)
1393 {
1394 	int q;
1395 	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1396 
1397 	spin_lock(&css_pipe->qlock);
1398 	for (q = 0; q < IPU3_CSS_QUEUES; q++)
1399 		if (!list_empty(&css_pipe->queue[q].bufs))
1400 			break;
1401 	spin_unlock(&css_pipe->qlock);
1402 	return (q == IPU3_CSS_QUEUES);
1403 }
1404 
imgu_css_queue_empty(struct imgu_css * css)1405 bool imgu_css_queue_empty(struct imgu_css *css)
1406 {
1407 	unsigned int pipe;
1408 	bool ret = false;
1409 
1410 	for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++)
1411 		ret &= imgu_css_pipe_queue_empty(css, pipe);
1412 
1413 	return ret;
1414 }
1415 
imgu_css_is_streaming(struct imgu_css * css)1416 bool imgu_css_is_streaming(struct imgu_css *css)
1417 {
1418 	return css->streaming;
1419 }
1420 
imgu_css_map_init(struct imgu_css * css,unsigned int pipe)1421 static int imgu_css_map_init(struct imgu_css *css, unsigned int pipe)
1422 {
1423 	struct imgu_device *imgu = dev_get_drvdata(css->dev);
1424 	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1425 	unsigned int p, q, i;
1426 
1427 	/* Allocate and map common structures with imgu hardware */
1428 	for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++)
1429 		for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) {
1430 			if (!imgu_dmamap_alloc(imgu,
1431 					       &css_pipe->xmem_sp_stage_ptrs[p][i],
1432 					       sizeof(struct imgu_abi_sp_stage)))
1433 				return -ENOMEM;
1434 			if (!imgu_dmamap_alloc(imgu,
1435 					       &css_pipe->xmem_isp_stage_ptrs[p][i],
1436 					       sizeof(struct imgu_abi_isp_stage)))
1437 				return -ENOMEM;
1438 		}
1439 
1440 	if (!imgu_dmamap_alloc(imgu, &css_pipe->sp_ddr_ptrs,
1441 			       ALIGN(sizeof(struct imgu_abi_ddr_address_map),
1442 				     IMGU_ABI_ISP_DDR_WORD_BYTES)))
1443 		return -ENOMEM;
1444 
1445 	for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1446 		unsigned int abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]);
1447 
1448 		for (i = 0; i < abi_buf_num; i++)
1449 			if (!imgu_dmamap_alloc(imgu,
1450 					       &css_pipe->abi_buffers[q][i],
1451 					       sizeof(struct imgu_abi_buffer)))
1452 				return -ENOMEM;
1453 	}
1454 
1455 	if (imgu_css_binary_preallocate(css, pipe)) {
1456 		imgu_css_binary_cleanup(css, pipe);
1457 		return -ENOMEM;
1458 	}
1459 
1460 	return 0;
1461 }
1462 
imgu_css_pipe_cleanup(struct imgu_css * css,unsigned int pipe)1463 static void imgu_css_pipe_cleanup(struct imgu_css *css, unsigned int pipe)
1464 {
1465 	struct imgu_device *imgu = dev_get_drvdata(css->dev);
1466 	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1467 	unsigned int p, q, i, abi_buf_num;
1468 
1469 	imgu_css_binary_cleanup(css, pipe);
1470 
1471 	for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1472 		abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]);
1473 		for (i = 0; i < abi_buf_num; i++)
1474 			imgu_dmamap_free(imgu, &css_pipe->abi_buffers[q][i]);
1475 	}
1476 
1477 	for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++)
1478 		for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) {
1479 			imgu_dmamap_free(imgu,
1480 					 &css_pipe->xmem_sp_stage_ptrs[p][i]);
1481 			imgu_dmamap_free(imgu,
1482 					 &css_pipe->xmem_isp_stage_ptrs[p][i]);
1483 		}
1484 
1485 	imgu_dmamap_free(imgu, &css_pipe->sp_ddr_ptrs);
1486 }
1487 
imgu_css_cleanup(struct imgu_css * css)1488 void imgu_css_cleanup(struct imgu_css *css)
1489 {
1490 	struct imgu_device *imgu = dev_get_drvdata(css->dev);
1491 	unsigned int pipe;
1492 
1493 	imgu_css_stop_streaming(css);
1494 	for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++)
1495 		imgu_css_pipe_cleanup(css, pipe);
1496 	imgu_dmamap_free(imgu, &css->xmem_sp_group_ptrs);
1497 	imgu_css_fw_cleanup(css);
1498 }
1499 
imgu_css_init(struct device * dev,struct imgu_css * css,void __iomem * base,int length)1500 int imgu_css_init(struct device *dev, struct imgu_css *css,
1501 		  void __iomem *base, int length)
1502 {
1503 	struct imgu_device *imgu = dev_get_drvdata(dev);
1504 	int r, q, pipe;
1505 
1506 	/* Initialize main data structure */
1507 	css->dev = dev;
1508 	css->base = base;
1509 	css->iomem_length = length;
1510 
1511 	for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++) {
1512 		struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1513 
1514 		css_pipe->vf_output_en = false;
1515 		spin_lock_init(&css_pipe->qlock);
1516 		css_pipe->bindex = IPU3_CSS_DEFAULT_BINARY;
1517 		css_pipe->pipe_id = IPU3_CSS_PIPE_ID_VIDEO;
1518 		for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1519 			r = imgu_css_queue_init(&css_pipe->queue[q], NULL, 0);
1520 			if (r)
1521 				return r;
1522 		}
1523 		r = imgu_css_map_init(css, pipe);
1524 		if (r) {
1525 			imgu_css_cleanup(css);
1526 			return r;
1527 		}
1528 	}
1529 	if (!imgu_dmamap_alloc(imgu, &css->xmem_sp_group_ptrs,
1530 			       sizeof(struct imgu_abi_sp_group)))
1531 		return -ENOMEM;
1532 
1533 	r = imgu_css_fw_init(css);
1534 	if (r)
1535 		return r;
1536 
1537 	return 0;
1538 }
1539 
imgu_css_adjust(u32 res,u32 align)1540 static u32 imgu_css_adjust(u32 res, u32 align)
1541 {
1542 	u32 val = max_t(u32, IPU3_CSS_MIN_RES, res);
1543 
1544 	return DIV_ROUND_CLOSEST(val, align) * align;
1545 }
1546 
1547 /* Select a binary matching the required resolutions and formats */
imgu_css_find_binary(struct imgu_css * css,unsigned int pipe,struct imgu_css_queue queue[IPU3_CSS_QUEUES],struct v4l2_rect rects[IPU3_CSS_RECTS])1548 static int imgu_css_find_binary(struct imgu_css *css,
1549 				unsigned int pipe,
1550 				struct imgu_css_queue queue[IPU3_CSS_QUEUES],
1551 				struct v4l2_rect rects[IPU3_CSS_RECTS])
1552 {
1553 	const int binary_nr = css->fwp->file_header.binary_nr;
1554 	unsigned int binary_mode =
1555 		(css->pipes[pipe].pipe_id == IPU3_CSS_PIPE_ID_CAPTURE) ?
1556 		IA_CSS_BINARY_MODE_PRIMARY : IA_CSS_BINARY_MODE_VIDEO;
1557 	const struct v4l2_pix_format_mplane *in =
1558 					&queue[IPU3_CSS_QUEUE_IN].fmt.mpix;
1559 	const struct v4l2_pix_format_mplane *out =
1560 					&queue[IPU3_CSS_QUEUE_OUT].fmt.mpix;
1561 	const struct v4l2_pix_format_mplane *vf =
1562 					&queue[IPU3_CSS_QUEUE_VF].fmt.mpix;
1563 	u32 stripe_w = 0, stripe_h = 0;
1564 	const char *name;
1565 	int i, j;
1566 
1567 	if (!imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_IN]))
1568 		return -EINVAL;
1569 
1570 	/* Find out the strip size boundary */
1571 	for (i = 0; i < binary_nr; i++) {
1572 		struct imgu_fw_info *bi = &css->fwp->binary_header[i];
1573 
1574 		u32 max_width = bi->info.isp.sp.output.max_width;
1575 		u32 max_height = bi->info.isp.sp.output.max_height;
1576 
1577 		if (bi->info.isp.sp.iterator.num_stripes <= 1) {
1578 			stripe_w = stripe_w ?
1579 				min(stripe_w, max_width) : max_width;
1580 			stripe_h = stripe_h ?
1581 				min(stripe_h, max_height) : max_height;
1582 		}
1583 	}
1584 
1585 	for (i = 0; i < binary_nr; i++) {
1586 		struct imgu_fw_info *bi = &css->fwp->binary_header[i];
1587 		enum imgu_abi_frame_format q_fmt;
1588 
1589 		name = (void *)css->fwp + bi->blob.prog_name_offset;
1590 
1591 		/* Check that binary supports memory-to-memory processing */
1592 		if (bi->info.isp.sp.input.source !=
1593 		    IMGU_ABI_BINARY_INPUT_SOURCE_MEMORY)
1594 			continue;
1595 
1596 		/* Check that binary supports raw10 input */
1597 		if (!bi->info.isp.sp.enable.input_feeder &&
1598 		    !bi->info.isp.sp.enable.input_raw)
1599 			continue;
1600 
1601 		/* Check binary mode */
1602 		if (bi->info.isp.sp.pipeline.mode != binary_mode)
1603 			continue;
1604 
1605 		/* Since input is RGGB bayer, need to process colors */
1606 		if (bi->info.isp.sp.enable.luma_only)
1607 			continue;
1608 
1609 		if (in->width < bi->info.isp.sp.input.min_width ||
1610 		    in->width > bi->info.isp.sp.input.max_width ||
1611 		    in->height < bi->info.isp.sp.input.min_height ||
1612 		    in->height > bi->info.isp.sp.input.max_height)
1613 			continue;
1614 
1615 		if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_OUT])) {
1616 			if (bi->info.isp.num_output_pins <= 0)
1617 				continue;
1618 
1619 			q_fmt = queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format;
1620 			for (j = 0; j < bi->info.isp.num_output_formats; j++)
1621 				if (bi->info.isp.output_formats[j] == q_fmt)
1622 					break;
1623 			if (j >= bi->info.isp.num_output_formats)
1624 				continue;
1625 
1626 			if (out->width < bi->info.isp.sp.output.min_width ||
1627 			    out->width > bi->info.isp.sp.output.max_width ||
1628 			    out->height < bi->info.isp.sp.output.min_height ||
1629 			    out->height > bi->info.isp.sp.output.max_height)
1630 				continue;
1631 
1632 			if (out->width > bi->info.isp.sp.internal.max_width ||
1633 			    out->height > bi->info.isp.sp.internal.max_height)
1634 				continue;
1635 		}
1636 
1637 		if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_VF])) {
1638 			if (bi->info.isp.num_output_pins <= 1)
1639 				continue;
1640 
1641 			q_fmt = queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format;
1642 			for (j = 0; j < bi->info.isp.num_output_formats; j++)
1643 				if (bi->info.isp.output_formats[j] == q_fmt)
1644 					break;
1645 			if (j >= bi->info.isp.num_output_formats)
1646 				continue;
1647 
1648 			if (vf->width < bi->info.isp.sp.output.min_width ||
1649 			    vf->width > bi->info.isp.sp.output.max_width ||
1650 			    vf->height < bi->info.isp.sp.output.min_height ||
1651 			    vf->height > bi->info.isp.sp.output.max_height)
1652 				continue;
1653 		}
1654 
1655 		/* All checks passed, select the binary */
1656 		dev_dbg(css->dev, "using binary %s id = %u\n", name,
1657 			bi->info.isp.sp.id);
1658 		return i;
1659 	}
1660 
1661 	/* Can not find suitable binary for these parameters */
1662 	return -EINVAL;
1663 }
1664 
1665 /*
1666  * Check that there is a binary matching requirements. Parameters may be
1667  * NULL indicating disabled input/output. Return negative if given
1668  * parameters can not be supported or on error, zero or positive indicating
1669  * found binary number. May modify the given parameters if not exact match
1670  * is found.
1671  */
imgu_css_fmt_try(struct imgu_css * css,struct v4l2_pix_format_mplane * fmts[IPU3_CSS_QUEUES],struct v4l2_rect * rects[IPU3_CSS_RECTS],unsigned int pipe)1672 int imgu_css_fmt_try(struct imgu_css *css,
1673 		     struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES],
1674 		     struct v4l2_rect *rects[IPU3_CSS_RECTS],
1675 		     unsigned int pipe)
1676 {
1677 	static const u32 EFF_ALIGN_W = 2;
1678 	static const u32 BDS_ALIGN_W = 4;
1679 	static const u32 OUT_ALIGN_W = 8;
1680 	static const u32 OUT_ALIGN_H = 4;
1681 	static const u32 VF_ALIGN_W  = 2;
1682 	static const char *qnames[IPU3_CSS_QUEUES] = {
1683 		[IPU3_CSS_QUEUE_IN] = "in",
1684 		[IPU3_CSS_QUEUE_PARAMS]    = "params",
1685 		[IPU3_CSS_QUEUE_OUT] = "out",
1686 		[IPU3_CSS_QUEUE_VF] = "vf",
1687 		[IPU3_CSS_QUEUE_STAT_3A]   = "3a",
1688 	};
1689 	static const char *rnames[IPU3_CSS_RECTS] = {
1690 		[IPU3_CSS_RECT_EFFECTIVE] = "effective resolution",
1691 		[IPU3_CSS_RECT_BDS]       = "bayer-domain scaled resolution",
1692 		[IPU3_CSS_RECT_ENVELOPE]  = "DVS envelope size",
1693 		[IPU3_CSS_RECT_GDC]  = "GDC output res",
1694 	};
1695 	struct v4l2_rect r[IPU3_CSS_RECTS] = { };
1696 	struct v4l2_rect *const eff = &r[IPU3_CSS_RECT_EFFECTIVE];
1697 	struct v4l2_rect *const bds = &r[IPU3_CSS_RECT_BDS];
1698 	struct v4l2_rect *const env = &r[IPU3_CSS_RECT_ENVELOPE];
1699 	struct v4l2_rect *const gdc = &r[IPU3_CSS_RECT_GDC];
1700 	struct imgu_css_queue *q;
1701 	struct v4l2_pix_format_mplane *in, *out, *vf;
1702 	int i, s, ret;
1703 
1704 	q = kcalloc(IPU3_CSS_QUEUES, sizeof(struct imgu_css_queue), GFP_KERNEL);
1705 	if (!q)
1706 		return -ENOMEM;
1707 
1708 	in  = &q[IPU3_CSS_QUEUE_IN].fmt.mpix;
1709 	out = &q[IPU3_CSS_QUEUE_OUT].fmt.mpix;
1710 	vf  = &q[IPU3_CSS_QUEUE_VF].fmt.mpix;
1711 
1712 	/* Adjust all formats, get statistics buffer sizes and formats */
1713 	for (i = 0; i < IPU3_CSS_QUEUES; i++) {
1714 		if (fmts[i])
1715 			dev_dbg(css->dev, "%s %s: (%i,%i) fmt 0x%x\n", __func__,
1716 				qnames[i], fmts[i]->width, fmts[i]->height,
1717 				fmts[i]->pixelformat);
1718 		else
1719 			dev_dbg(css->dev, "%s %s: (not set)\n", __func__,
1720 				qnames[i]);
1721 		if (imgu_css_queue_init(&q[i], fmts[i],
1722 					IPU3_CSS_QUEUE_TO_FLAGS(i))) {
1723 			dev_notice(css->dev, "can not initialize queue %s\n",
1724 				   qnames[i]);
1725 			ret = -EINVAL;
1726 			goto out;
1727 		}
1728 	}
1729 	for (i = 0; i < IPU3_CSS_RECTS; i++) {
1730 		if (rects[i]) {
1731 			dev_dbg(css->dev, "%s %s: (%i,%i)\n", __func__,
1732 				rnames[i], rects[i]->width, rects[i]->height);
1733 			r[i].width  = rects[i]->width;
1734 			r[i].height = rects[i]->height;
1735 		} else {
1736 			dev_dbg(css->dev, "%s %s: (not set)\n", __func__,
1737 				rnames[i]);
1738 		}
1739 		/* For now, force known good resolutions */
1740 		r[i].left = 0;
1741 		r[i].top  = 0;
1742 	}
1743 
1744 	/* Always require one input and vf only if out is also enabled */
1745 	if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_IN]) ||
1746 	    !imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) {
1747 		dev_warn(css->dev, "required queues are disabled\n");
1748 		ret = -EINVAL;
1749 		goto out;
1750 	}
1751 
1752 	if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) {
1753 		out->width = in->width;
1754 		out->height = in->height;
1755 	}
1756 	if (eff->width <= 0 || eff->height <= 0) {
1757 		eff->width = in->width;
1758 		eff->height = in->height;
1759 	}
1760 	if (bds->width <= 0 || bds->height <= 0) {
1761 		bds->width = out->width;
1762 		bds->height = out->height;
1763 	}
1764 	if (gdc->width <= 0 || gdc->height <= 0) {
1765 		gdc->width = out->width;
1766 		gdc->height = out->height;
1767 	}
1768 
1769 	in->width   = imgu_css_adjust(in->width, 1);
1770 	in->height  = imgu_css_adjust(in->height, 1);
1771 	eff->width  = imgu_css_adjust(eff->width, EFF_ALIGN_W);
1772 	eff->height = imgu_css_adjust(eff->height, 1);
1773 	bds->width  = imgu_css_adjust(bds->width, BDS_ALIGN_W);
1774 	bds->height = imgu_css_adjust(bds->height, 1);
1775 	gdc->width  = imgu_css_adjust(gdc->width, OUT_ALIGN_W);
1776 	gdc->height = imgu_css_adjust(gdc->height, OUT_ALIGN_H);
1777 	out->width  = imgu_css_adjust(out->width, OUT_ALIGN_W);
1778 	out->height = imgu_css_adjust(out->height, OUT_ALIGN_H);
1779 	vf->width   = imgu_css_adjust(vf->width, VF_ALIGN_W);
1780 	vf->height  = imgu_css_adjust(vf->height, 1);
1781 
1782 	s = (bds->width - gdc->width) / 2;
1783 	env->width = s < MIN_ENVELOPE ? MIN_ENVELOPE : s;
1784 	s = (bds->height - gdc->height) / 2;
1785 	env->height = s < MIN_ENVELOPE ? MIN_ENVELOPE : s;
1786 
1787 	ret = imgu_css_find_binary(css, pipe, q, r);
1788 	if (ret < 0) {
1789 		dev_err(css->dev, "failed to find suitable binary\n");
1790 		ret = -EINVAL;
1791 		goto out;
1792 	}
1793 	css->pipes[pipe].bindex = ret;
1794 
1795 	dev_dbg(css->dev, "Binary index %d for pipe %d found.",
1796 		css->pipes[pipe].bindex, pipe);
1797 
1798 	/* Final adjustment and set back the queried formats */
1799 	for (i = 0; i < IPU3_CSS_QUEUES; i++) {
1800 		if (fmts[i]) {
1801 			if (imgu_css_queue_init(&q[i], &q[i].fmt.mpix,
1802 						IPU3_CSS_QUEUE_TO_FLAGS(i))) {
1803 				dev_err(css->dev,
1804 					"final resolution adjustment failed\n");
1805 				ret = -EINVAL;
1806 				goto out;
1807 			}
1808 			*fmts[i] = q[i].fmt.mpix;
1809 		}
1810 	}
1811 
1812 	for (i = 0; i < IPU3_CSS_RECTS; i++)
1813 		if (rects[i])
1814 			*rects[i] = r[i];
1815 
1816 	dev_dbg(css->dev,
1817 		"in(%u,%u) if(%u,%u) ds(%u,%u) gdc(%u,%u) out(%u,%u) vf(%u,%u)",
1818 		 in->width, in->height, eff->width, eff->height,
1819 		 bds->width, bds->height, gdc->width, gdc->height,
1820 		 out->width, out->height, vf->width, vf->height);
1821 
1822 	ret = 0;
1823 out:
1824 	kfree(q);
1825 	return ret;
1826 }
1827 
imgu_css_fmt_set(struct imgu_css * css,struct v4l2_pix_format_mplane * fmts[IPU3_CSS_QUEUES],struct v4l2_rect * rects[IPU3_CSS_RECTS],unsigned int pipe)1828 int imgu_css_fmt_set(struct imgu_css *css,
1829 		     struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES],
1830 		     struct v4l2_rect *rects[IPU3_CSS_RECTS],
1831 		     unsigned int pipe)
1832 {
1833 	struct v4l2_rect rect_data[IPU3_CSS_RECTS];
1834 	struct v4l2_rect *all_rects[IPU3_CSS_RECTS];
1835 	int i, r;
1836 	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1837 
1838 	for (i = 0; i < IPU3_CSS_RECTS; i++) {
1839 		if (rects[i])
1840 			rect_data[i] = *rects[i];
1841 		else
1842 			memset(&rect_data[i], 0, sizeof(rect_data[i]));
1843 		all_rects[i] = &rect_data[i];
1844 	}
1845 	r = imgu_css_fmt_try(css, fmts, all_rects, pipe);
1846 	if (r < 0)
1847 		return r;
1848 
1849 	for (i = 0; i < IPU3_CSS_QUEUES; i++)
1850 		if (imgu_css_queue_init(&css_pipe->queue[i], fmts[i],
1851 					IPU3_CSS_QUEUE_TO_FLAGS(i)))
1852 			return -EINVAL;
1853 	for (i = 0; i < IPU3_CSS_RECTS; i++) {
1854 		css_pipe->rect[i] = rect_data[i];
1855 		if (rects[i])
1856 			*rects[i] = rect_data[i];
1857 	}
1858 
1859 	return 0;
1860 }
1861 
imgu_css_meta_fmt_set(struct v4l2_meta_format * fmt)1862 int imgu_css_meta_fmt_set(struct v4l2_meta_format *fmt)
1863 {
1864 	switch (fmt->dataformat) {
1865 	case V4L2_META_FMT_IPU3_PARAMS:
1866 		fmt->buffersize = sizeof(struct ipu3_uapi_params);
1867 
1868 		/*
1869 		 * Sanity check for the parameter struct size. This must
1870 		 * not change!
1871 		 */
1872 		BUILD_BUG_ON(sizeof(struct ipu3_uapi_params) != 39328);
1873 
1874 		break;
1875 	case V4L2_META_FMT_IPU3_STAT_3A:
1876 		fmt->buffersize = sizeof(struct ipu3_uapi_stats_3a);
1877 		break;
1878 	default:
1879 		return -EINVAL;
1880 	}
1881 
1882 	return 0;
1883 }
1884 
1885 /*
1886  * Queue given buffer to CSS. imgu_css_buf_prepare() must have been first
1887  * called for the buffer. May be called from interrupt context.
1888  * Returns 0 on success, -EBUSY if the buffer queue is full, or some other
1889  * code on error conditions.
1890  */
imgu_css_buf_queue(struct imgu_css * css,unsigned int pipe,struct imgu_css_buffer * b)1891 int imgu_css_buf_queue(struct imgu_css *css, unsigned int pipe,
1892 		       struct imgu_css_buffer *b)
1893 {
1894 	struct imgu_abi_buffer *abi_buf;
1895 	struct imgu_addr_t *buf_addr;
1896 	u32 data;
1897 	int r;
1898 	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1899 
1900 	if (!css->streaming)
1901 		return -EPROTO;	/* CSS or buffer in wrong state */
1902 
1903 	if (b->queue >= IPU3_CSS_QUEUES || !imgu_css_queues[b->queue].qid)
1904 		return -EINVAL;
1905 
1906 	b->queue_pos = imgu_css_queue_pos(css, imgu_css_queues[b->queue].qid,
1907 					  pipe);
1908 
1909 	if (b->queue_pos >= ARRAY_SIZE(css->pipes[pipe].abi_buffers[b->queue]))
1910 		return -EIO;
1911 	abi_buf = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].vaddr;
1912 
1913 	/* Fill struct abi_buffer for firmware */
1914 	memset(abi_buf, 0, sizeof(*abi_buf));
1915 
1916 	buf_addr = (void *)abi_buf + imgu_css_queues[b->queue].ptr_ofs;
1917 	*(imgu_addr_t *)buf_addr = b->daddr;
1918 
1919 	if (b->queue == IPU3_CSS_QUEUE_STAT_3A)
1920 		abi_buf->payload.s3a.data.dmem.s3a_tbl = b->daddr;
1921 
1922 	if (b->queue == IPU3_CSS_QUEUE_OUT)
1923 		abi_buf->payload.frame.padded_width =
1924 				css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad;
1925 
1926 	if (b->queue == IPU3_CSS_QUEUE_VF)
1927 		abi_buf->payload.frame.padded_width =
1928 					css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad;
1929 
1930 	spin_lock(&css_pipe->qlock);
1931 	list_add_tail(&b->list, &css_pipe->queue[b->queue].bufs);
1932 	spin_unlock(&css_pipe->qlock);
1933 	b->state = IPU3_CSS_BUFFER_QUEUED;
1934 
1935 	data = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].daddr;
1936 	r = imgu_css_queue_data(css, imgu_css_queues[b->queue].qid,
1937 				pipe, data);
1938 	if (r < 0)
1939 		goto queueing_failed;
1940 
1941 	data = IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe,
1942 					      imgu_css_queues[b->queue].qid);
1943 	r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, data);
1944 	if (r < 0)
1945 		goto queueing_failed;
1946 
1947 	dev_dbg(css->dev, "queued buffer %p to css queue %i in pipe %d\n",
1948 		b, b->queue, pipe);
1949 
1950 	return 0;
1951 
1952 queueing_failed:
1953 	b->state = (r == -EBUSY || r == -EAGAIN) ?
1954 		IPU3_CSS_BUFFER_NEW : IPU3_CSS_BUFFER_FAILED;
1955 	list_del(&b->list);
1956 
1957 	return r;
1958 }
1959 
1960 /*
1961  * Get next ready CSS buffer. Returns -EAGAIN in which case the function
1962  * should be called again, or -EBUSY which means that there are no more
1963  * buffers available. May be called from interrupt context.
1964  */
imgu_css_buf_dequeue(struct imgu_css * css)1965 struct imgu_css_buffer *imgu_css_buf_dequeue(struct imgu_css *css)
1966 {
1967 	static const unsigned char evtype_to_queue[] = {
1968 		[IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE] = IPU3_CSS_QUEUE_IN,
1969 		[IMGU_ABI_EVTTYPE_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_OUT,
1970 		[IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_VF,
1971 		[IMGU_ABI_EVTTYPE_3A_STATS_DONE] = IPU3_CSS_QUEUE_STAT_3A,
1972 	};
1973 	struct imgu_css_buffer *b = ERR_PTR(-EAGAIN);
1974 	u32 event, daddr;
1975 	int evtype, pipe, pipeid, queue, qid, r;
1976 	struct imgu_css_pipe *css_pipe;
1977 
1978 	if (!css->streaming)
1979 		return ERR_PTR(-EPROTO);
1980 
1981 	r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event);
1982 	if (r < 0)
1983 		return ERR_PTR(r);
1984 
1985 	evtype = (event & IMGU_ABI_EVTTYPE_EVENT_MASK) >>
1986 		  IMGU_ABI_EVTTYPE_EVENT_SHIFT;
1987 
1988 	switch (evtype) {
1989 	case IMGU_ABI_EVTTYPE_OUT_FRAME_DONE:
1990 	case IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE:
1991 	case IMGU_ABI_EVTTYPE_3A_STATS_DONE:
1992 	case IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE:
1993 		pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >>
1994 			IMGU_ABI_EVTTYPE_PIPE_SHIFT;
1995 		pipeid = (event & IMGU_ABI_EVTTYPE_PIPEID_MASK) >>
1996 			IMGU_ABI_EVTTYPE_PIPEID_SHIFT;
1997 		queue = evtype_to_queue[evtype];
1998 		qid = imgu_css_queues[queue].qid;
1999 
2000 		if (pipe >= IMGU_MAX_PIPE_NUM) {
2001 			dev_err(css->dev, "Invalid pipe: %i\n", pipe);
2002 			return ERR_PTR(-EIO);
2003 		}
2004 
2005 		if (qid >= IMGU_ABI_QUEUE_NUM) {
2006 			dev_err(css->dev, "Invalid qid: %i\n", qid);
2007 			return ERR_PTR(-EIO);
2008 		}
2009 		css_pipe = &css->pipes[pipe];
2010 		dev_dbg(css->dev,
2011 			"event: buffer done 0x%x queue %i pipe %i pipeid %i\n",
2012 			event, queue, pipe, pipeid);
2013 
2014 		r = imgu_css_dequeue_data(css, qid, &daddr);
2015 		if (r < 0) {
2016 			dev_err(css->dev, "failed to dequeue buffer\n");
2017 			/* Force real error, not -EBUSY */
2018 			return ERR_PTR(-EIO);
2019 		}
2020 
2021 		r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2022 					IMGU_ABI_EVENT_BUFFER_DEQUEUED(qid));
2023 		if (r < 0) {
2024 			dev_err(css->dev, "failed to queue event\n");
2025 			return ERR_PTR(-EIO);
2026 		}
2027 
2028 		spin_lock(&css_pipe->qlock);
2029 		if (list_empty(&css_pipe->queue[queue].bufs)) {
2030 			spin_unlock(&css_pipe->qlock);
2031 			dev_err(css->dev, "event on empty queue\n");
2032 			return ERR_PTR(-EIO);
2033 		}
2034 		b = list_first_entry(&css_pipe->queue[queue].bufs,
2035 				     struct imgu_css_buffer, list);
2036 		if (queue != b->queue ||
2037 		    daddr != css_pipe->abi_buffers
2038 			[b->queue][b->queue_pos].daddr) {
2039 			spin_unlock(&css_pipe->qlock);
2040 			dev_err(css->dev, "dequeued bad buffer 0x%x\n", daddr);
2041 			return ERR_PTR(-EIO);
2042 		}
2043 
2044 		dev_dbg(css->dev, "buffer 0x%8x done from pipe %d\n", daddr, pipe);
2045 		b->pipe = pipe;
2046 		b->state = IPU3_CSS_BUFFER_DONE;
2047 		list_del(&b->list);
2048 		spin_unlock(&css_pipe->qlock);
2049 		break;
2050 	case IMGU_ABI_EVTTYPE_PIPELINE_DONE:
2051 		pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >>
2052 			IMGU_ABI_EVTTYPE_PIPE_SHIFT;
2053 		if (pipe >= IMGU_MAX_PIPE_NUM) {
2054 			dev_err(css->dev, "Invalid pipe: %i\n", pipe);
2055 			return ERR_PTR(-EIO);
2056 		}
2057 
2058 		css_pipe = &css->pipes[pipe];
2059 		dev_dbg(css->dev, "event: pipeline done 0x%8x for pipe %d\n",
2060 			event, pipe);
2061 		break;
2062 	case IMGU_ABI_EVTTYPE_TIMER:
2063 		r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event);
2064 		if (r < 0)
2065 			return ERR_PTR(r);
2066 
2067 		if ((event & IMGU_ABI_EVTTYPE_EVENT_MASK) >>
2068 		    IMGU_ABI_EVTTYPE_EVENT_SHIFT == IMGU_ABI_EVTTYPE_TIMER)
2069 			dev_dbg(css->dev, "event: timer\n");
2070 		else
2071 			dev_warn(css->dev, "half of timer event missing\n");
2072 		break;
2073 	case IMGU_ABI_EVTTYPE_FW_WARNING:
2074 		dev_warn(css->dev, "event: firmware warning 0x%x\n", event);
2075 		break;
2076 	case IMGU_ABI_EVTTYPE_FW_ASSERT:
2077 		dev_err(css->dev,
2078 			"event: firmware assert 0x%x module_id %i line_no %i\n",
2079 			event,
2080 			(event & IMGU_ABI_EVTTYPE_MODULEID_MASK) >>
2081 			IMGU_ABI_EVTTYPE_MODULEID_SHIFT,
2082 			swab16((event & IMGU_ABI_EVTTYPE_LINENO_MASK) >>
2083 			       IMGU_ABI_EVTTYPE_LINENO_SHIFT));
2084 		break;
2085 	default:
2086 		dev_warn(css->dev, "received unknown event 0x%x\n", event);
2087 	}
2088 
2089 	return b;
2090 }
2091 
2092 /*
2093  * Get a new set of parameters from pool and initialize them based on
2094  * the parameters params, gdc, and obgrid. Any of these may be NULL,
2095  * in which case the previously set parameters are used.
2096  * If parameters haven't been set previously, initialize from scratch.
2097  *
2098  * Return index to css->parameter_set_info which has the newly created
2099  * parameters or negative value on error.
2100  */
imgu_css_set_parameters(struct imgu_css * css,unsigned int pipe,struct ipu3_uapi_params * set_params)2101 int imgu_css_set_parameters(struct imgu_css *css, unsigned int pipe,
2102 			    struct ipu3_uapi_params *set_params)
2103 {
2104 	static const unsigned int queue_id = IMGU_ABI_QUEUE_A_ID;
2105 	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
2106 	const int stage = 0;
2107 	const struct imgu_fw_info *bi;
2108 	int obgrid_size;
2109 	unsigned int stripes, i;
2110 	struct ipu3_uapi_flags *use = set_params ? &set_params->use : NULL;
2111 
2112 	/* Destination buffers which are filled here */
2113 	struct imgu_abi_parameter_set_info *param_set;
2114 	struct imgu_abi_acc_param *acc = NULL;
2115 	struct imgu_abi_gdc_warp_param *gdc = NULL;
2116 	struct ipu3_uapi_obgrid_param *obgrid = NULL;
2117 	const struct imgu_css_map *map;
2118 	void *vmem0 = NULL;
2119 	void *dmem0 = NULL;
2120 
2121 	enum imgu_abi_memories m;
2122 	int r = -EBUSY;
2123 
2124 	if (!css->streaming)
2125 		return -EPROTO;
2126 
2127 	dev_dbg(css->dev, "%s for pipe %d", __func__, pipe);
2128 
2129 	bi = &css->fwp->binary_header[css_pipe->bindex];
2130 	obgrid_size = imgu_css_fw_obgrid_size(bi);
2131 	stripes = bi->info.isp.sp.iterator.num_stripes ? : 1;
2132 
2133 	imgu_css_pool_get(&css_pipe->pool.parameter_set_info);
2134 	param_set = imgu_css_pool_last(&css_pipe->pool.parameter_set_info,
2135 				       0)->vaddr;
2136 
2137 	/* Get a new acc only if new parameters given, or none yet */
2138 	map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2139 	if (set_params || !map->vaddr) {
2140 		imgu_css_pool_get(&css_pipe->pool.acc);
2141 		map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2142 		acc = map->vaddr;
2143 	}
2144 
2145 	/* Get new VMEM0 only if needed, or none yet */
2146 	m = IMGU_ABI_MEM_ISP_VMEM0;
2147 	map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2148 	if (!map->vaddr || (set_params && (set_params->use.lin_vmem_params ||
2149 					   set_params->use.tnr3_vmem_params ||
2150 					   set_params->use.xnr3_vmem_params))) {
2151 		imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]);
2152 		map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2153 		vmem0 = map->vaddr;
2154 	}
2155 
2156 	/* Get new DMEM0 only if needed, or none yet */
2157 	m = IMGU_ABI_MEM_ISP_DMEM0;
2158 	map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2159 	if (!map->vaddr || (set_params && (set_params->use.tnr3_dmem_params ||
2160 					   set_params->use.xnr3_dmem_params))) {
2161 		imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]);
2162 		map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2163 		dmem0 = map->vaddr;
2164 	}
2165 
2166 	/* Configure acc parameter cluster */
2167 	if (acc) {
2168 		/* get acc_old */
2169 		map = imgu_css_pool_last(&css_pipe->pool.acc, 1);
2170 		/* user acc */
2171 		r = imgu_css_cfg_acc(css, pipe, use, acc, map->vaddr,
2172 			set_params ? &set_params->acc_param : NULL);
2173 		if (r < 0)
2174 			goto fail;
2175 	}
2176 
2177 	/* Configure late binding parameters */
2178 	if (vmem0) {
2179 		m = IMGU_ABI_MEM_ISP_VMEM0;
2180 		map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1);
2181 		r = imgu_css_cfg_vmem0(css, pipe, use, vmem0,
2182 				       map->vaddr, set_params);
2183 		if (r < 0)
2184 			goto fail;
2185 	}
2186 
2187 	if (dmem0) {
2188 		m = IMGU_ABI_MEM_ISP_DMEM0;
2189 		map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1);
2190 		r = imgu_css_cfg_dmem0(css, pipe, use, dmem0,
2191 				       map->vaddr, set_params);
2192 		if (r < 0)
2193 			goto fail;
2194 	}
2195 
2196 	/* Get a new gdc only if a new gdc is given, or none yet */
2197 	if (bi->info.isp.sp.enable.dvs_6axis) {
2198 		unsigned int a = IPU3_CSS_AUX_FRAME_REF;
2199 		unsigned int g = IPU3_CSS_RECT_GDC;
2200 		unsigned int e = IPU3_CSS_RECT_ENVELOPE;
2201 
2202 		map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2203 		if (!map->vaddr) {
2204 			imgu_css_pool_get(&css_pipe->pool.gdc);
2205 			map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2206 			gdc = map->vaddr;
2207 			imgu_css_cfg_gdc_table(map->vaddr,
2208 				css_pipe->aux_frames[a].bytesperline /
2209 				css_pipe->aux_frames[a].bytesperpixel,
2210 				css_pipe->aux_frames[a].height,
2211 				css_pipe->rect[g].width,
2212 				css_pipe->rect[g].height,
2213 				css_pipe->rect[e].width,
2214 				css_pipe->rect[e].height);
2215 		}
2216 	}
2217 
2218 	/* Get a new obgrid only if a new obgrid is given, or none yet */
2219 	map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2220 	if (!map->vaddr || (set_params && set_params->use.obgrid_param)) {
2221 		imgu_css_pool_get(&css_pipe->pool.obgrid);
2222 		map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2223 		obgrid = map->vaddr;
2224 
2225 		/* Configure optical black level grid (obgrid) */
2226 		if (set_params && set_params->use.obgrid_param)
2227 			for (i = 0; i < obgrid_size / sizeof(*obgrid); i++)
2228 				obgrid[i] = set_params->obgrid_param;
2229 		else
2230 			memset(obgrid, 0, obgrid_size);
2231 	}
2232 
2233 	/* Configure parameter set info, queued to `queue_id' */
2234 
2235 	memset(param_set, 0, sizeof(*param_set));
2236 	map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2237 	param_set->mem_map.acc_cluster_params_for_sp = map->daddr;
2238 
2239 	map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2240 	param_set->mem_map.dvs_6axis_params_y = map->daddr;
2241 
2242 	for (i = 0; i < stripes; i++) {
2243 		map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2244 		param_set->mem_map.obgrid_tbl[i] =
2245 			map->daddr + (obgrid_size / stripes) * i;
2246 	}
2247 
2248 	for (m = 0; m < IMGU_ABI_NUM_MEMORIES; m++) {
2249 		map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2250 		param_set->mem_map.isp_mem_param[stage][m] = map->daddr;
2251 	}
2252 
2253 	/* Then queue the new parameter buffer */
2254 	map = imgu_css_pool_last(&css_pipe->pool.parameter_set_info, 0);
2255 	r = imgu_css_queue_data(css, queue_id, pipe, map->daddr);
2256 	if (r < 0)
2257 		goto fail;
2258 
2259 	r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2260 				IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe,
2261 							       queue_id));
2262 	if (r < 0)
2263 		goto fail_no_put;
2264 
2265 	/* Finally dequeue all old parameter buffers */
2266 
2267 	do {
2268 		u32 daddr;
2269 
2270 		r = imgu_css_dequeue_data(css, queue_id, &daddr);
2271 		if (r == -EBUSY)
2272 			break;
2273 		if (r)
2274 			goto fail_no_put;
2275 		r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2276 					IMGU_ABI_EVENT_BUFFER_DEQUEUED
2277 					(queue_id));
2278 		if (r < 0) {
2279 			dev_err(css->dev, "failed to queue parameter event\n");
2280 			goto fail_no_put;
2281 		}
2282 	} while (1);
2283 
2284 	return 0;
2285 
2286 fail:
2287 	/*
2288 	 * A failure, most likely the parameter queue was full.
2289 	 * Return error but continue streaming. User can try submitting new
2290 	 * parameters again later.
2291 	 */
2292 
2293 	imgu_css_pool_put(&css_pipe->pool.parameter_set_info);
2294 	if (acc)
2295 		imgu_css_pool_put(&css_pipe->pool.acc);
2296 	if (gdc)
2297 		imgu_css_pool_put(&css_pipe->pool.gdc);
2298 	if (obgrid)
2299 		imgu_css_pool_put(&css_pipe->pool.obgrid);
2300 	if (vmem0)
2301 		imgu_css_pool_put(
2302 			&css_pipe->pool.binary_params_p
2303 			[IMGU_ABI_MEM_ISP_VMEM0]);
2304 	if (dmem0)
2305 		imgu_css_pool_put(
2306 			&css_pipe->pool.binary_params_p
2307 			[IMGU_ABI_MEM_ISP_DMEM0]);
2308 
2309 fail_no_put:
2310 	return r;
2311 }
2312 
imgu_css_irq_ack(struct imgu_css * css)2313 int imgu_css_irq_ack(struct imgu_css *css)
2314 {
2315 	static const int NUM_SWIRQS = 3;
2316 	struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]];
2317 	void __iomem *const base = css->base;
2318 	u32 irq_status[IMGU_IRQCTRL_NUM];
2319 	int i;
2320 
2321 	u32 imgu_status = readl(base + IMGU_REG_INT_STATUS);
2322 
2323 	writel(imgu_status, base + IMGU_REG_INT_STATUS);
2324 	for (i = 0; i < IMGU_IRQCTRL_NUM; i++)
2325 		irq_status[i] = readl(base + IMGU_REG_IRQCTRL_STATUS(i));
2326 
2327 	for (i = 0; i < NUM_SWIRQS; i++) {
2328 		if (irq_status[IMGU_IRQCTRL_SP0] & IMGU_IRQCTRL_IRQ_SW_PIN(i)) {
2329 			/* SP SW interrupt */
2330 			u32 cnt = readl(base + IMGU_REG_SP_DMEM_BASE(0) +
2331 					bi->info.sp.output);
2332 			u32 val = readl(base + IMGU_REG_SP_DMEM_BASE(0) +
2333 					bi->info.sp.output + 4 + 4 * i);
2334 
2335 			dev_dbg(css->dev, "%s: swirq %i cnt %i val 0x%x\n",
2336 				__func__, i, cnt, val);
2337 		}
2338 	}
2339 
2340 	for (i = IMGU_IRQCTRL_NUM - 1; i >= 0; i--)
2341 		if (irq_status[i]) {
2342 			writel(irq_status[i], base + IMGU_REG_IRQCTRL_CLEAR(i));
2343 			/* Wait for write to complete */
2344 			readl(base + IMGU_REG_IRQCTRL_ENABLE(i));
2345 		}
2346 
2347 	dev_dbg(css->dev, "%s: imgu 0x%x main 0x%x sp0 0x%x sp1 0x%x\n",
2348 		__func__, imgu_status, irq_status[IMGU_IRQCTRL_MAIN],
2349 		irq_status[IMGU_IRQCTRL_SP0], irq_status[IMGU_IRQCTRL_SP1]);
2350 
2351 	if (!imgu_status && !irq_status[IMGU_IRQCTRL_MAIN])
2352 		return -ENOMSG;
2353 
2354 	return 0;
2355 }
2356