1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (C) 2018 Intel Corporation 3 4 #include <linux/device.h> 5 #include <linux/iopoll.h> 6 #include <linux/slab.h> 7 8 #include "ipu3.h" 9 #include "ipu3-css.h" 10 #include "ipu3-css-fw.h" 11 #include "ipu3-css-params.h" 12 #include "ipu3-dmamap.h" 13 #include "ipu3-tables.h" 14 15 /* IRQ configuration */ 16 #define IMGU_IRQCTRL_IRQ_MASK (IMGU_IRQCTRL_IRQ_SP1 | \ 17 IMGU_IRQCTRL_IRQ_SP2 | \ 18 IMGU_IRQCTRL_IRQ_SW_PIN(0) | \ 19 IMGU_IRQCTRL_IRQ_SW_PIN(1)) 20 21 #define IPU3_CSS_FORMAT_BPP_DEN 50 /* Denominator */ 22 23 /* Some sane limits for resolutions */ 24 #define IPU3_CSS_MIN_RES 32 25 #define IPU3_CSS_MAX_H 3136 26 #define IPU3_CSS_MAX_W 4224 27 28 /* minimal envelope size(GDC in - out) should be 4 */ 29 #define MIN_ENVELOPE 4 30 31 /* 32 * pre-allocated buffer size for CSS ABI, auxiliary frames 33 * after BDS and before GDC. Those values should be tuned 34 * to big enough to avoid buffer re-allocation when 35 * streaming to lower streaming latency. 36 */ 37 #define CSS_ABI_SIZE 136 38 #define CSS_BDS_SIZE (4480 * 3200 * 3) 39 #define CSS_GDC_SIZE (4224 * 3200 * 12 / 8) 40 41 #define IPU3_CSS_QUEUE_TO_FLAGS(q) (1 << (q)) 42 #define IPU3_CSS_FORMAT_FL_IN \ 43 IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_IN) 44 #define IPU3_CSS_FORMAT_FL_OUT \ 45 IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_OUT) 46 #define IPU3_CSS_FORMAT_FL_VF \ 47 IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_VF) 48 49 /* Formats supported by IPU3 Camera Sub System */ 50 static const struct imgu_css_format imgu_css_formats[] = { 51 { 52 .pixelformat = V4L2_PIX_FMT_NV12, 53 .colorspace = V4L2_COLORSPACE_SRGB, 54 .frame_format = IMGU_ABI_FRAME_FORMAT_NV12, 55 .osys_format = IMGU_ABI_OSYS_FORMAT_NV12, 56 .osys_tiling = IMGU_ABI_OSYS_TILING_NONE, 57 .chroma_decim = 4, 58 .width_align = IPU3_UAPI_ISP_VEC_ELEMS, 59 .flags = IPU3_CSS_FORMAT_FL_OUT | IPU3_CSS_FORMAT_FL_VF, 60 }, { 61 /* Each 32 bytes contains 25 10-bit pixels */ 62 .pixelformat = V4L2_PIX_FMT_IPU3_SBGGR10, 63 .colorspace = V4L2_COLORSPACE_RAW, 64 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED, 65 .bayer_order = IMGU_ABI_BAYER_ORDER_BGGR, 66 .bit_depth = 10, 67 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS, 68 .flags = IPU3_CSS_FORMAT_FL_IN, 69 }, { 70 .pixelformat = V4L2_PIX_FMT_IPU3_SGBRG10, 71 .colorspace = V4L2_COLORSPACE_RAW, 72 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED, 73 .bayer_order = IMGU_ABI_BAYER_ORDER_GBRG, 74 .bit_depth = 10, 75 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS, 76 .flags = IPU3_CSS_FORMAT_FL_IN, 77 }, { 78 .pixelformat = V4L2_PIX_FMT_IPU3_SGRBG10, 79 .colorspace = V4L2_COLORSPACE_RAW, 80 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED, 81 .bayer_order = IMGU_ABI_BAYER_ORDER_GRBG, 82 .bit_depth = 10, 83 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS, 84 .flags = IPU3_CSS_FORMAT_FL_IN, 85 }, { 86 .pixelformat = V4L2_PIX_FMT_IPU3_SRGGB10, 87 .colorspace = V4L2_COLORSPACE_RAW, 88 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED, 89 .bayer_order = IMGU_ABI_BAYER_ORDER_RGGB, 90 .bit_depth = 10, 91 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS, 92 .flags = IPU3_CSS_FORMAT_FL_IN, 93 }, 94 }; 95 96 static const struct { 97 enum imgu_abi_queue_id qid; 98 size_t ptr_ofs; 99 } imgu_css_queues[IPU3_CSS_QUEUES] = { 100 [IPU3_CSS_QUEUE_IN] = { 101 IMGU_ABI_QUEUE_C_ID, 102 offsetof(struct imgu_abi_buffer, payload.frame.frame_data) 103 }, 104 [IPU3_CSS_QUEUE_OUT] = { 105 IMGU_ABI_QUEUE_D_ID, 106 offsetof(struct imgu_abi_buffer, payload.frame.frame_data) 107 }, 108 [IPU3_CSS_QUEUE_VF] = { 109 IMGU_ABI_QUEUE_E_ID, 110 offsetof(struct imgu_abi_buffer, payload.frame.frame_data) 111 }, 112 [IPU3_CSS_QUEUE_STAT_3A] = { 113 IMGU_ABI_QUEUE_F_ID, 114 offsetof(struct imgu_abi_buffer, payload.s3a.data_ptr) 115 }, 116 }; 117 118 /* Initialize queue based on given format, adjust format as needed */ 119 static int imgu_css_queue_init(struct imgu_css_queue *queue, 120 struct v4l2_pix_format_mplane *fmt, u32 flags) 121 { 122 struct v4l2_pix_format_mplane *const f = &queue->fmt.mpix; 123 unsigned int i; 124 u32 sizeimage; 125 126 INIT_LIST_HEAD(&queue->bufs); 127 128 queue->css_fmt = NULL; /* Disable */ 129 if (!fmt) 130 return 0; 131 132 for (i = 0; i < ARRAY_SIZE(imgu_css_formats); i++) { 133 if (!(imgu_css_formats[i].flags & flags)) 134 continue; 135 queue->css_fmt = &imgu_css_formats[i]; 136 if (imgu_css_formats[i].pixelformat == fmt->pixelformat) 137 break; 138 } 139 if (!queue->css_fmt) 140 return -EINVAL; /* Could not find any suitable format */ 141 142 queue->fmt.mpix = *fmt; 143 144 f->width = ALIGN(clamp_t(u32, f->width, 145 IPU3_CSS_MIN_RES, IPU3_CSS_MAX_W), 2); 146 f->height = ALIGN(clamp_t(u32, f->height, 147 IPU3_CSS_MIN_RES, IPU3_CSS_MAX_H), 2); 148 queue->width_pad = ALIGN(f->width, queue->css_fmt->width_align); 149 f->plane_fmt[0].bytesperline = 150 imgu_bytesperline(f->width, queue->css_fmt->frame_format); 151 sizeimage = f->height * f->plane_fmt[0].bytesperline; 152 if (queue->css_fmt->chroma_decim) 153 sizeimage += 2 * sizeimage / queue->css_fmt->chroma_decim; 154 155 f->plane_fmt[0].sizeimage = sizeimage; 156 f->field = V4L2_FIELD_NONE; 157 f->num_planes = 1; 158 f->colorspace = queue->css_fmt->colorspace; 159 f->flags = 0; 160 f->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT; 161 f->quantization = V4L2_QUANTIZATION_DEFAULT; 162 f->xfer_func = V4L2_XFER_FUNC_DEFAULT; 163 memset(f->reserved, 0, sizeof(f->reserved)); 164 165 return 0; 166 } 167 168 static bool imgu_css_queue_enabled(struct imgu_css_queue *q) 169 { 170 return q->css_fmt; 171 } 172 173 /******************* css hw *******************/ 174 175 /* In the style of writesl() defined in include/asm-generic/io.h */ 176 static inline void writes(const void *mem, ssize_t count, void __iomem *addr) 177 { 178 if (count >= 4) { 179 const u32 *buf = mem; 180 181 count /= 4; 182 do { 183 writel(*buf++, addr); 184 addr += 4; 185 } while (--count); 186 } 187 } 188 189 /* Wait until register `reg', masked with `mask', becomes `cmp' */ 190 static int imgu_hw_wait(void __iomem *base, int reg, u32 mask, u32 cmp) 191 { 192 u32 val; 193 194 return readl_poll_timeout(base + reg, val, (val & mask) == cmp, 195 1000, 100 * 1000); 196 } 197 198 /* Initialize the IPU3 CSS hardware and associated h/w blocks */ 199 200 int imgu_css_set_powerup(struct device *dev, void __iomem *base, 201 unsigned int freq) 202 { 203 u32 pm_ctrl, state, val; 204 205 dev_dbg(dev, "%s with freq %u\n", __func__, freq); 206 /* Clear the CSS busy signal */ 207 readl(base + IMGU_REG_GP_BUSY); 208 writel(0, base + IMGU_REG_GP_BUSY); 209 210 /* Wait for idle signal */ 211 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS, 212 IMGU_STATE_IDLE_STS)) { 213 dev_err(dev, "failed to set CSS idle\n"); 214 goto fail; 215 } 216 217 /* Reset the css */ 218 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET, 219 base + IMGU_REG_PM_CTRL); 220 221 usleep_range(200, 300); 222 223 /** Prepare CSS */ 224 225 pm_ctrl = readl(base + IMGU_REG_PM_CTRL); 226 state = readl(base + IMGU_REG_STATE); 227 228 dev_dbg(dev, "CSS pm_ctrl 0x%x state 0x%x (power %s)\n", 229 pm_ctrl, state, state & IMGU_STATE_POWER_DOWN ? "down" : "up"); 230 231 /* Power up CSS using wrapper */ 232 if (state & IMGU_STATE_POWER_DOWN) { 233 writel(IMGU_PM_CTRL_RACE_TO_HALT | IMGU_PM_CTRL_START, 234 base + IMGU_REG_PM_CTRL); 235 if (imgu_hw_wait(base, IMGU_REG_PM_CTRL, 236 IMGU_PM_CTRL_START, 0)) { 237 dev_err(dev, "failed to power up CSS\n"); 238 goto fail; 239 } 240 usleep_range(2000, 3000); 241 } else { 242 writel(IMGU_PM_CTRL_RACE_TO_HALT, base + IMGU_REG_PM_CTRL); 243 } 244 245 /* Set the busy bit */ 246 writel(readl(base + IMGU_REG_GP_BUSY) | 1, base + IMGU_REG_GP_BUSY); 247 248 /* Set CSS clock frequency */ 249 pm_ctrl = readl(base + IMGU_REG_PM_CTRL); 250 val = pm_ctrl & ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF); 251 writel(val, base + IMGU_REG_PM_CTRL); 252 writel(0, base + IMGU_REG_GP_BUSY); 253 if (imgu_hw_wait(base, IMGU_REG_STATE, 254 IMGU_STATE_PWRDNM_FSM_MASK, 0)) { 255 dev_err(dev, "failed to pwrdn CSS\n"); 256 goto fail; 257 } 258 val = (freq / IMGU_SYSTEM_REQ_FREQ_DIVIDER) & IMGU_SYSTEM_REQ_FREQ_MASK; 259 writel(val, base + IMGU_REG_SYSTEM_REQ); 260 writel(1, base + IMGU_REG_GP_BUSY); 261 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_HALT, 262 base + IMGU_REG_PM_CTRL); 263 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS, 264 IMGU_STATE_HALT_STS)) { 265 dev_err(dev, "failed to halt CSS\n"); 266 goto fail; 267 } 268 269 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_START, 270 base + IMGU_REG_PM_CTRL); 271 if (imgu_hw_wait(base, IMGU_REG_PM_CTRL, IMGU_PM_CTRL_START, 0)) { 272 dev_err(dev, "failed to start CSS\n"); 273 goto fail; 274 } 275 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_UNHALT, 276 base + IMGU_REG_PM_CTRL); 277 278 val = readl(base + IMGU_REG_PM_CTRL); /* get pm_ctrl */ 279 val &= ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF); 280 val |= pm_ctrl & (IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF); 281 writel(val, base + IMGU_REG_PM_CTRL); 282 283 return 0; 284 285 fail: 286 imgu_css_set_powerdown(dev, base); 287 return -EIO; 288 } 289 290 void imgu_css_set_powerdown(struct device *dev, void __iomem *base) 291 { 292 dev_dbg(dev, "%s\n", __func__); 293 /* wait for cio idle signal */ 294 if (imgu_hw_wait(base, IMGU_REG_CIO_GATE_BURST_STATE, 295 IMGU_CIO_GATE_BURST_MASK, 0)) 296 dev_warn(dev, "wait cio gate idle timeout"); 297 298 /* wait for css idle signal */ 299 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS, 300 IMGU_STATE_IDLE_STS)) 301 dev_warn(dev, "wait css idle timeout\n"); 302 303 /* do halt-halted handshake with css */ 304 writel(1, base + IMGU_REG_GP_HALT); 305 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS, 306 IMGU_STATE_HALT_STS)) 307 dev_warn(dev, "failed to halt css"); 308 309 /* de-assert the busy bit */ 310 writel(0, base + IMGU_REG_GP_BUSY); 311 } 312 313 static void imgu_css_hw_enable_irq(struct imgu_css *css) 314 { 315 void __iomem *const base = css->base; 316 u32 val, i; 317 318 /* Set up interrupts */ 319 320 /* 321 * Enable IRQ on the SP which signals that SP goes to idle 322 * (aka ready state) and set trigger to pulse 323 */ 324 val = readl(base + IMGU_REG_SP_CTRL(0)) | IMGU_CTRL_IRQ_READY; 325 writel(val, base + IMGU_REG_SP_CTRL(0)); 326 writel(val | IMGU_CTRL_IRQ_CLEAR, base + IMGU_REG_SP_CTRL(0)); 327 328 /* Enable IRQs from the IMGU wrapper */ 329 writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_ENABLE); 330 /* Clear */ 331 writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_STATUS); 332 333 /* Enable IRQs from main IRQ controller */ 334 writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(IMGU_IRQCTRL_MAIN)); 335 writel(0, base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN)); 336 writel(IMGU_IRQCTRL_IRQ_MASK, 337 base + IMGU_REG_IRQCTRL_EDGE(IMGU_IRQCTRL_MAIN)); 338 writel(IMGU_IRQCTRL_IRQ_MASK, 339 base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN)); 340 writel(IMGU_IRQCTRL_IRQ_MASK, 341 base + IMGU_REG_IRQCTRL_CLEAR(IMGU_IRQCTRL_MAIN)); 342 writel(IMGU_IRQCTRL_IRQ_MASK, 343 base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN)); 344 /* Wait for write complete */ 345 readl(base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN)); 346 347 /* Enable IRQs from SP0 and SP1 controllers */ 348 for (i = IMGU_IRQCTRL_SP0; i <= IMGU_IRQCTRL_SP1; i++) { 349 writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(i)); 350 writel(0, base + IMGU_REG_IRQCTRL_MASK(i)); 351 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_EDGE(i)); 352 writel(IMGU_IRQCTRL_IRQ_MASK, 353 base + IMGU_REG_IRQCTRL_ENABLE(i)); 354 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_CLEAR(i)); 355 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_MASK(i)); 356 /* Wait for write complete */ 357 readl(base + IMGU_REG_IRQCTRL_ENABLE(i)); 358 } 359 } 360 361 static int imgu_css_hw_init(struct imgu_css *css) 362 { 363 /* For checking that streaming monitor statuses are valid */ 364 static const struct { 365 u32 reg; 366 u32 mask; 367 const char *name; 368 } stream_monitors[] = { 369 { 370 IMGU_REG_GP_SP1_STRMON_STAT, 371 IMGU_GP_STRMON_STAT_ISP_PORT_SP12ISP, 372 "ISP0 to SP0" 373 }, { 374 IMGU_REG_GP_ISP_STRMON_STAT, 375 IMGU_GP_STRMON_STAT_SP1_PORT_ISP2SP1, 376 "SP0 to ISP0" 377 }, { 378 IMGU_REG_GP_MOD_STRMON_STAT, 379 IMGU_GP_STRMON_STAT_MOD_PORT_ISP2DMA, 380 "ISP0 to DMA0" 381 }, { 382 IMGU_REG_GP_ISP_STRMON_STAT, 383 IMGU_GP_STRMON_STAT_ISP_PORT_DMA2ISP, 384 "DMA0 to ISP0" 385 }, { 386 IMGU_REG_GP_MOD_STRMON_STAT, 387 IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC, 388 "ISP0 to GDC0" 389 }, { 390 IMGU_REG_GP_MOD_STRMON_STAT, 391 IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS, 392 "GDC0 to ISP0" 393 }, { 394 IMGU_REG_GP_MOD_STRMON_STAT, 395 IMGU_GP_STRMON_STAT_MOD_PORT_SP12DMA, 396 "SP0 to DMA0" 397 }, { 398 IMGU_REG_GP_SP1_STRMON_STAT, 399 IMGU_GP_STRMON_STAT_SP1_PORT_DMA2SP1, 400 "DMA0 to SP0" 401 }, { 402 IMGU_REG_GP_MOD_STRMON_STAT, 403 IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC, 404 "SP0 to GDC0" 405 }, { 406 IMGU_REG_GP_MOD_STRMON_STAT, 407 IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS, 408 "GDC0 to SP0" 409 }, 410 }; 411 412 struct device *dev = css->dev; 413 void __iomem *const base = css->base; 414 u32 val, i; 415 416 /* Set instruction cache address and inv bit for ISP, SP, and SP1 */ 417 for (i = 0; i < IMGU_NUM_SP; i++) { 418 struct imgu_fw_info *bi = 419 &css->fwp->binary_header[css->fw_sp[i]]; 420 421 writel(css->binary[css->fw_sp[i]].daddr, 422 base + IMGU_REG_SP_ICACHE_ADDR(bi->type)); 423 writel(readl(base + IMGU_REG_SP_CTRL(bi->type)) | 424 IMGU_CTRL_ICACHE_INV, 425 base + IMGU_REG_SP_CTRL(bi->type)); 426 } 427 writel(css->binary[css->fw_bl].daddr, base + IMGU_REG_ISP_ICACHE_ADDR); 428 writel(readl(base + IMGU_REG_ISP_CTRL) | IMGU_CTRL_ICACHE_INV, 429 base + IMGU_REG_ISP_CTRL); 430 431 /* Check that IMGU hardware is ready */ 432 433 if (!(readl(base + IMGU_REG_SP_CTRL(0)) & IMGU_CTRL_IDLE)) { 434 dev_err(dev, "SP is not idle\n"); 435 return -EIO; 436 } 437 if (!(readl(base + IMGU_REG_ISP_CTRL) & IMGU_CTRL_IDLE)) { 438 dev_err(dev, "ISP is not idle\n"); 439 return -EIO; 440 } 441 442 for (i = 0; i < ARRAY_SIZE(stream_monitors); i++) { 443 val = readl(base + stream_monitors[i].reg); 444 if (val & stream_monitors[i].mask) { 445 dev_err(dev, "error: Stream monitor %s is valid\n", 446 stream_monitors[i].name); 447 return -EIO; 448 } 449 } 450 451 /* Initialize GDC with default values */ 452 453 for (i = 0; i < ARRAY_SIZE(imgu_css_gdc_lut[0]); i++) { 454 u32 val0 = imgu_css_gdc_lut[0][i] & IMGU_GDC_LUT_MASK; 455 u32 val1 = imgu_css_gdc_lut[1][i] & IMGU_GDC_LUT_MASK; 456 u32 val2 = imgu_css_gdc_lut[2][i] & IMGU_GDC_LUT_MASK; 457 u32 val3 = imgu_css_gdc_lut[3][i] & IMGU_GDC_LUT_MASK; 458 459 writel(val0 | (val1 << 16), 460 base + IMGU_REG_GDC_LUT_BASE + i * 8); 461 writel(val2 | (val3 << 16), 462 base + IMGU_REG_GDC_LUT_BASE + i * 8 + 4); 463 } 464 465 return 0; 466 } 467 468 /* Boot the given IPU3 CSS SP */ 469 static int imgu_css_hw_start_sp(struct imgu_css *css, int sp) 470 { 471 void __iomem *const base = css->base; 472 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]]; 473 struct imgu_abi_sp_init_dmem_cfg dmem_cfg = { 474 .ddr_data_addr = css->binary[css->fw_sp[sp]].daddr 475 + bi->blob.data_source, 476 .dmem_data_addr = bi->blob.data_target, 477 .dmem_bss_addr = bi->blob.bss_target, 478 .data_size = bi->blob.data_size, 479 .bss_size = bi->blob.bss_size, 480 .sp_id = sp, 481 }; 482 483 writes(&dmem_cfg, sizeof(dmem_cfg), base + 484 IMGU_REG_SP_DMEM_BASE(sp) + bi->info.sp.init_dmem_data); 485 486 writel(bi->info.sp.sp_entry, base + IMGU_REG_SP_START_ADDR(sp)); 487 488 writel(readl(base + IMGU_REG_SP_CTRL(sp)) 489 | IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_SP_CTRL(sp)); 490 491 if (imgu_hw_wait(css->base, IMGU_REG_SP_DMEM_BASE(sp) 492 + bi->info.sp.sw_state, 493 ~0, IMGU_ABI_SP_SWSTATE_INITIALIZED)) 494 return -EIO; 495 496 return 0; 497 } 498 499 /* Start the IPU3 CSS ImgU (Imaging Unit) and all the SPs */ 500 static int imgu_css_hw_start(struct imgu_css *css) 501 { 502 static const u32 event_mask = 503 ((1 << IMGU_ABI_EVTTYPE_OUT_FRAME_DONE) | 504 (1 << IMGU_ABI_EVTTYPE_2ND_OUT_FRAME_DONE) | 505 (1 << IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE) | 506 (1 << IMGU_ABI_EVTTYPE_2ND_VF_OUT_FRAME_DONE) | 507 (1 << IMGU_ABI_EVTTYPE_3A_STATS_DONE) | 508 (1 << IMGU_ABI_EVTTYPE_DIS_STATS_DONE) | 509 (1 << IMGU_ABI_EVTTYPE_PIPELINE_DONE) | 510 (1 << IMGU_ABI_EVTTYPE_FRAME_TAGGED) | 511 (1 << IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE) | 512 (1 << IMGU_ABI_EVTTYPE_METADATA_DONE) | 513 (1 << IMGU_ABI_EVTTYPE_ACC_STAGE_COMPLETE)) 514 << IMGU_ABI_SP_COMM_EVENT_IRQ_MASK_OR_SHIFT; 515 516 void __iomem *const base = css->base; 517 struct imgu_fw_info *bi, *bl = &css->fwp->binary_header[css->fw_bl]; 518 unsigned int i; 519 520 writel(IMGU_TLB_INVALIDATE, base + IMGU_REG_TLB_INVALIDATE); 521 522 /* Start bootloader */ 523 524 writel(IMGU_ABI_BL_SWSTATE_BUSY, 525 base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.sw_state); 526 writel(IMGU_NUM_SP, 527 base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.num_dma_cmds); 528 529 for (i = 0; i < IMGU_NUM_SP; i++) { 530 int j = IMGU_NUM_SP - i - 1; /* load sp1 first, then sp0 */ 531 struct imgu_fw_info *sp = 532 &css->fwp->binary_header[css->fw_sp[j]]; 533 struct imgu_abi_bl_dma_cmd_entry dma_cmd = { 534 .src_addr = css->binary[css->fw_sp[j]].daddr 535 + sp->blob.text_source, 536 .size = sp->blob.text_size, 537 .dst_type = IMGU_ABI_BL_DMACMD_TYPE_SP_PMEM, 538 .dst_addr = IMGU_SP_PMEM_BASE(j), 539 }; 540 541 writes(&dma_cmd, sizeof(dma_cmd), 542 base + IMGU_REG_ISP_DMEM_BASE + i * sizeof(dma_cmd) + 543 bl->info.bl.dma_cmd_list); 544 } 545 546 writel(bl->info.bl.bl_entry, base + IMGU_REG_ISP_START_ADDR); 547 548 writel(readl(base + IMGU_REG_ISP_CTRL) 549 | IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_ISP_CTRL); 550 if (imgu_hw_wait(css->base, IMGU_REG_ISP_DMEM_BASE 551 + bl->info.bl.sw_state, ~0, 552 IMGU_ABI_BL_SWSTATE_OK)) { 553 dev_err(css->dev, "failed to start bootloader\n"); 554 return -EIO; 555 } 556 557 /* Start ISP */ 558 559 memset(css->xmem_sp_group_ptrs.vaddr, 0, 560 sizeof(struct imgu_abi_sp_group)); 561 562 bi = &css->fwp->binary_header[css->fw_sp[0]]; 563 564 writel(css->xmem_sp_group_ptrs.daddr, 565 base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.per_frame_data); 566 567 writel(IMGU_ABI_SP_SWSTATE_TERMINATED, 568 base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state); 569 writel(1, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb); 570 571 if (imgu_css_hw_start_sp(css, 0)) 572 return -EIO; 573 574 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.isp_started); 575 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + 576 bi->info.sp.host_sp_queues_initialized); 577 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sleep_mode); 578 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb); 579 writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(0) 580 + bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND); 581 582 /* Enable all events for all queues */ 583 584 for (i = 0; i < IPU3_CSS_PIPE_ID_NUM; i++) 585 writel(event_mask, base + IMGU_REG_SP_DMEM_BASE(0) 586 + bi->info.sp.host_sp_com 587 + IMGU_ABI_SP_COMM_EVENT_IRQ_MASK(i)); 588 writel(1, base + IMGU_REG_SP_DMEM_BASE(0) + 589 bi->info.sp.host_sp_queues_initialized); 590 591 /* Start SP1 */ 592 593 bi = &css->fwp->binary_header[css->fw_sp[1]]; 594 595 writel(IMGU_ABI_SP_SWSTATE_TERMINATED, 596 base + IMGU_REG_SP_DMEM_BASE(1) + bi->info.sp.sw_state); 597 598 if (imgu_css_hw_start_sp(css, 1)) 599 return -EIO; 600 601 writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(1) 602 + bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND); 603 604 return 0; 605 } 606 607 static void imgu_css_hw_stop(struct imgu_css *css) 608 { 609 void __iomem *const base = css->base; 610 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]]; 611 612 /* Stop fw */ 613 writel(IMGU_ABI_SP_COMM_COMMAND_TERMINATE, 614 base + IMGU_REG_SP_DMEM_BASE(0) + 615 bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND); 616 if (imgu_hw_wait(css->base, IMGU_REG_SP_CTRL(0), 617 IMGU_CTRL_IDLE, IMGU_CTRL_IDLE)) 618 dev_err(css->dev, "wait sp0 idle timeout.\n"); 619 if (readl(base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state) != 620 IMGU_ABI_SP_SWSTATE_TERMINATED) 621 dev_err(css->dev, "sp0 is not terminated.\n"); 622 if (imgu_hw_wait(css->base, IMGU_REG_ISP_CTRL, 623 IMGU_CTRL_IDLE, IMGU_CTRL_IDLE)) 624 dev_err(css->dev, "wait isp idle timeout\n"); 625 } 626 627 static void imgu_css_hw_cleanup(struct imgu_css *css) 628 { 629 void __iomem *const base = css->base; 630 631 /** Reset CSS **/ 632 633 /* Clear the CSS busy signal */ 634 readl(base + IMGU_REG_GP_BUSY); 635 writel(0, base + IMGU_REG_GP_BUSY); 636 637 /* Wait for idle signal */ 638 if (imgu_hw_wait(css->base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS, 639 IMGU_STATE_IDLE_STS)) 640 dev_err(css->dev, "failed to shut down hw cleanly\n"); 641 642 /* Reset the css */ 643 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET, 644 base + IMGU_REG_PM_CTRL); 645 646 usleep_range(200, 300); 647 } 648 649 static void imgu_css_pipeline_cleanup(struct imgu_css *css, unsigned int pipe) 650 { 651 struct imgu_device *imgu = dev_get_drvdata(css->dev); 652 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 653 unsigned int i; 654 655 imgu_css_pool_cleanup(imgu, &css_pipe->pool.parameter_set_info); 656 imgu_css_pool_cleanup(imgu, &css_pipe->pool.acc); 657 imgu_css_pool_cleanup(imgu, &css_pipe->pool.gdc); 658 imgu_css_pool_cleanup(imgu, &css_pipe->pool.obgrid); 659 660 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) 661 imgu_css_pool_cleanup(imgu, &css_pipe->pool.binary_params_p[i]); 662 } 663 664 /* 665 * This function initializes various stages of the 666 * IPU3 CSS ISP pipeline 667 */ 668 static int imgu_css_pipeline_init(struct imgu_css *css, unsigned int pipe) 669 { 670 static const int BYPC = 2; /* Bytes per component */ 671 static const struct imgu_abi_buffer_sp buffer_sp_init = { 672 .buf_src = {.queue_id = IMGU_ABI_QUEUE_EVENT_ID}, 673 .buf_type = IMGU_ABI_BUFFER_TYPE_INVALID, 674 }; 675 676 struct imgu_abi_isp_iterator_config *cfg_iter; 677 struct imgu_abi_isp_ref_config *cfg_ref; 678 struct imgu_abi_isp_dvs_config *cfg_dvs; 679 struct imgu_abi_isp_tnr3_config *cfg_tnr; 680 struct imgu_abi_isp_ref_dmem_state *cfg_ref_state; 681 struct imgu_abi_isp_tnr3_dmem_state *cfg_tnr_state; 682 683 const int stage = 0; 684 unsigned int i, j; 685 686 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 687 struct imgu_css_queue *css_queue_in = 688 &css_pipe->queue[IPU3_CSS_QUEUE_IN]; 689 struct imgu_css_queue *css_queue_out = 690 &css_pipe->queue[IPU3_CSS_QUEUE_OUT]; 691 struct imgu_css_queue *css_queue_vf = 692 &css_pipe->queue[IPU3_CSS_QUEUE_VF]; 693 const struct imgu_fw_info *bi = 694 &css->fwp->binary_header[css_pipe->bindex]; 695 const unsigned int stripes = bi->info.isp.sp.iterator.num_stripes; 696 697 struct imgu_fw_config_memory_offsets *cofs = (void *)css->fwp + 698 bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_CONFIG]; 699 struct imgu_fw_state_memory_offsets *sofs = (void *)css->fwp + 700 bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_STATE]; 701 702 struct imgu_abi_isp_stage *isp_stage; 703 struct imgu_abi_sp_stage *sp_stage; 704 struct imgu_abi_sp_group *sp_group; 705 struct imgu_abi_frames_sp *frames_sp; 706 struct imgu_abi_frame_sp *frame_sp; 707 struct imgu_abi_frame_sp_info *frame_sp_info; 708 709 const unsigned int bds_width_pad = 710 ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width, 711 2 * IPU3_UAPI_ISP_VEC_ELEMS); 712 713 const enum imgu_abi_memories m0 = IMGU_ABI_MEM_ISP_DMEM0; 714 enum imgu_abi_param_class cfg = IMGU_ABI_PARAM_CLASS_CONFIG; 715 void *vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr; 716 717 struct imgu_device *imgu = dev_get_drvdata(css->dev); 718 719 dev_dbg(css->dev, "%s for pipe %d", __func__, pipe); 720 721 /* Configure iterator */ 722 723 cfg_iter = imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 724 &cofs->dmem.iterator, 725 sizeof(*cfg_iter), vaddr); 726 if (!cfg_iter) 727 goto bad_firmware; 728 729 frame_sp_info = &cfg_iter->input_info; 730 frame_sp_info->res.width = css_queue_in->fmt.mpix.width; 731 frame_sp_info->res.height = css_queue_in->fmt.mpix.height; 732 frame_sp_info->padded_width = css_queue_in->width_pad; 733 frame_sp_info->format = css_queue_in->css_fmt->frame_format; 734 frame_sp_info->raw_bit_depth = css_queue_in->css_fmt->bit_depth; 735 frame_sp_info->raw_bayer_order = css_queue_in->css_fmt->bayer_order; 736 frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER; 737 738 frame_sp_info = &cfg_iter->internal_info; 739 frame_sp_info->res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width; 740 frame_sp_info->res.height = css_pipe->rect[IPU3_CSS_RECT_BDS].height; 741 frame_sp_info->padded_width = bds_width_pad; 742 frame_sp_info->format = css_queue_out->css_fmt->frame_format; 743 frame_sp_info->raw_bit_depth = css_queue_out->css_fmt->bit_depth; 744 frame_sp_info->raw_bayer_order = css_queue_out->css_fmt->bayer_order; 745 frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER; 746 747 frame_sp_info = &cfg_iter->output_info; 748 frame_sp_info->res.width = css_queue_out->fmt.mpix.width; 749 frame_sp_info->res.height = css_queue_out->fmt.mpix.height; 750 frame_sp_info->padded_width = css_queue_out->width_pad; 751 frame_sp_info->format = css_queue_out->css_fmt->frame_format; 752 frame_sp_info->raw_bit_depth = css_queue_out->css_fmt->bit_depth; 753 frame_sp_info->raw_bayer_order = css_queue_out->css_fmt->bayer_order; 754 frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER; 755 756 frame_sp_info = &cfg_iter->vf_info; 757 frame_sp_info->res.width = css_queue_vf->fmt.mpix.width; 758 frame_sp_info->res.height = css_queue_vf->fmt.mpix.height; 759 frame_sp_info->padded_width = css_queue_vf->width_pad; 760 frame_sp_info->format = css_queue_vf->css_fmt->frame_format; 761 frame_sp_info->raw_bit_depth = css_queue_vf->css_fmt->bit_depth; 762 frame_sp_info->raw_bayer_order = css_queue_vf->css_fmt->bayer_order; 763 frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER; 764 765 cfg_iter->dvs_envelope.width = 766 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width; 767 cfg_iter->dvs_envelope.height = 768 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height; 769 770 /* Configure reference (delay) frames */ 771 772 cfg_ref = imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 773 &cofs->dmem.ref, 774 sizeof(*cfg_ref), vaddr); 775 if (!cfg_ref) 776 goto bad_firmware; 777 778 cfg_ref->port_b.crop = 0; 779 cfg_ref->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES / BYPC; 780 cfg_ref->port_b.width = 781 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width; 782 cfg_ref->port_b.stride = 783 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline; 784 cfg_ref->width_a_over_b = 785 IPU3_UAPI_ISP_VEC_ELEMS / cfg_ref->port_b.elems; 786 cfg_ref->dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1; 787 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) { 788 cfg_ref->ref_frame_addr_y[i] = 789 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr; 790 cfg_ref->ref_frame_addr_c[i] = 791 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr + 792 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline * 793 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height; 794 } 795 for (; i < IMGU_ABI_FRAMES_REF; i++) { 796 cfg_ref->ref_frame_addr_y[i] = 0; 797 cfg_ref->ref_frame_addr_c[i] = 0; 798 } 799 800 /* Configure DVS (digital video stabilization) */ 801 802 cfg_dvs = imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 803 &cofs->dmem.dvs, sizeof(*cfg_dvs), 804 vaddr); 805 if (!cfg_dvs) 806 goto bad_firmware; 807 808 cfg_dvs->num_horizontal_blocks = 809 ALIGN(DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].width, 810 IMGU_DVS_BLOCK_W), 2); 811 cfg_dvs->num_vertical_blocks = 812 DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].height, 813 IMGU_DVS_BLOCK_H); 814 815 /* Configure TNR (temporal noise reduction) */ 816 817 if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) { 818 cfg_tnr = imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 819 &cofs->dmem.tnr3, 820 sizeof(*cfg_tnr), 821 vaddr); 822 if (!cfg_tnr) 823 goto bad_firmware; 824 825 cfg_tnr->port_b.crop = 0; 826 cfg_tnr->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES; 827 cfg_tnr->port_b.width = 828 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width; 829 cfg_tnr->port_b.stride = 830 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline; 831 cfg_tnr->width_a_over_b = 832 IPU3_UAPI_ISP_VEC_ELEMS / cfg_tnr->port_b.elems; 833 cfg_tnr->frame_height = 834 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height; 835 cfg_tnr->delay_frame = IPU3_CSS_AUX_FRAMES - 1; 836 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 837 cfg_tnr->frame_addr[i] = 838 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR] 839 .mem[i].daddr; 840 for (; i < IMGU_ABI_FRAMES_TNR; i++) 841 cfg_tnr->frame_addr[i] = 0; 842 } 843 844 /* Configure ref dmem state parameters */ 845 846 cfg = IMGU_ABI_PARAM_CLASS_STATE; 847 vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr; 848 849 cfg_ref_state = imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 850 &sofs->dmem.ref, 851 sizeof(*cfg_ref_state), 852 vaddr); 853 if (!cfg_ref_state) 854 goto bad_firmware; 855 856 cfg_ref_state->ref_in_buf_idx = 0; 857 cfg_ref_state->ref_out_buf_idx = 1; 858 859 /* Configure tnr dmem state parameters */ 860 if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) { 861 cfg_tnr_state = 862 imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 863 &sofs->dmem.tnr3, 864 sizeof(*cfg_tnr_state), 865 vaddr); 866 if (!cfg_tnr_state) 867 goto bad_firmware; 868 869 cfg_tnr_state->in_bufidx = 0; 870 cfg_tnr_state->out_bufidx = 1; 871 cfg_tnr_state->bypass_filter = 0; 872 cfg_tnr_state->total_frame_counter = 0; 873 for (i = 0; i < IMGU_ABI_BUF_SETS_TNR; i++) 874 cfg_tnr_state->buffer_frame_counter[i] = 0; 875 } 876 877 /* Configure ISP stage */ 878 879 isp_stage = css_pipe->xmem_isp_stage_ptrs[pipe][stage].vaddr; 880 memset(isp_stage, 0, sizeof(*isp_stage)); 881 isp_stage->blob_info = bi->blob; 882 isp_stage->binary_info = bi->info.isp.sp; 883 strscpy(isp_stage->binary_name, 884 (char *)css->fwp + bi->blob.prog_name_offset, 885 sizeof(isp_stage->binary_name)); 886 isp_stage->mem_initializers = bi->info.isp.sp.mem_initializers; 887 for (i = IMGU_ABI_PARAM_CLASS_CONFIG; i < IMGU_ABI_PARAM_CLASS_NUM; i++) 888 for (j = 0; j < IMGU_ABI_NUM_MEMORIES; j++) 889 isp_stage->mem_initializers.params[i][j].address = 890 css_pipe->binary_params_cs[i - 1][j].daddr; 891 892 /* Configure SP stage */ 893 894 sp_stage = css_pipe->xmem_sp_stage_ptrs[pipe][stage].vaddr; 895 memset(sp_stage, 0, sizeof(*sp_stage)); 896 897 frames_sp = &sp_stage->frames; 898 frames_sp->in.buf_attr = buffer_sp_init; 899 for (i = 0; i < IMGU_ABI_BINARY_MAX_OUTPUT_PORTS; i++) 900 frames_sp->out[i].buf_attr = buffer_sp_init; 901 frames_sp->out_vf.buf_attr = buffer_sp_init; 902 frames_sp->s3a_buf = buffer_sp_init; 903 frames_sp->dvs_buf = buffer_sp_init; 904 905 sp_stage->stage_type = IMGU_ABI_STAGE_TYPE_ISP; 906 sp_stage->num = stage; 907 sp_stage->isp_online = 0; 908 sp_stage->isp_copy_vf = 0; 909 sp_stage->isp_copy_output = 0; 910 911 sp_stage->enable.vf_output = css_pipe->vf_output_en; 912 913 frames_sp->effective_in_res.width = 914 css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].width; 915 frames_sp->effective_in_res.height = 916 css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].height; 917 918 frame_sp = &frames_sp->in; 919 frame_sp->info.res.width = css_queue_in->fmt.mpix.width; 920 frame_sp->info.res.height = css_queue_in->fmt.mpix.height; 921 frame_sp->info.padded_width = css_queue_in->width_pad; 922 frame_sp->info.format = css_queue_in->css_fmt->frame_format; 923 frame_sp->info.raw_bit_depth = css_queue_in->css_fmt->bit_depth; 924 frame_sp->info.raw_bayer_order = css_queue_in->css_fmt->bayer_order; 925 frame_sp->info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; 926 frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_C_ID; 927 frame_sp->buf_attr.buf_type = IMGU_ABI_BUFFER_TYPE_INPUT_FRAME; 928 929 frame_sp = &frames_sp->out[0]; 930 frame_sp->info.res.width = css_queue_out->fmt.mpix.width; 931 frame_sp->info.res.height = css_queue_out->fmt.mpix.height; 932 frame_sp->info.padded_width = css_queue_out->width_pad; 933 frame_sp->info.format = css_queue_out->css_fmt->frame_format; 934 frame_sp->info.raw_bit_depth = css_queue_out->css_fmt->bit_depth; 935 frame_sp->info.raw_bayer_order = css_queue_out->css_fmt->bayer_order; 936 frame_sp->info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; 937 frame_sp->planes.nv.uv.offset = css_queue_out->width_pad * 938 css_queue_out->fmt.mpix.height; 939 frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_D_ID; 940 frame_sp->buf_attr.buf_type = IMGU_ABI_BUFFER_TYPE_OUTPUT_FRAME; 941 942 frame_sp = &frames_sp->out[1]; 943 frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_EVENT_ID; 944 945 frame_sp_info = &frames_sp->internal_frame_info; 946 frame_sp_info->res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width; 947 frame_sp_info->res.height = css_pipe->rect[IPU3_CSS_RECT_BDS].height; 948 frame_sp_info->padded_width = bds_width_pad; 949 frame_sp_info->format = css_queue_out->css_fmt->frame_format; 950 frame_sp_info->raw_bit_depth = css_queue_out->css_fmt->bit_depth; 951 frame_sp_info->raw_bayer_order = css_queue_out->css_fmt->bayer_order; 952 frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER; 953 954 frame_sp = &frames_sp->out_vf; 955 frame_sp->info.res.width = css_queue_vf->fmt.mpix.width; 956 frame_sp->info.res.height = css_queue_vf->fmt.mpix.height; 957 frame_sp->info.padded_width = css_queue_vf->width_pad; 958 frame_sp->info.format = css_queue_vf->css_fmt->frame_format; 959 frame_sp->info.raw_bit_depth = css_queue_vf->css_fmt->bit_depth; 960 frame_sp->info.raw_bayer_order = css_queue_vf->css_fmt->bayer_order; 961 frame_sp->info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; 962 frame_sp->planes.yuv.u.offset = css_queue_vf->width_pad * 963 css_queue_vf->fmt.mpix.height; 964 frame_sp->planes.yuv.v.offset = css_queue_vf->width_pad * 965 css_queue_vf->fmt.mpix.height * 5 / 4; 966 frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_E_ID; 967 frame_sp->buf_attr.buf_type = IMGU_ABI_BUFFER_TYPE_VF_OUTPUT_FRAME; 968 969 frames_sp->s3a_buf.buf_src.queue_id = IMGU_ABI_QUEUE_F_ID; 970 frames_sp->s3a_buf.buf_type = IMGU_ABI_BUFFER_TYPE_3A_STATISTICS; 971 972 frames_sp->dvs_buf.buf_src.queue_id = IMGU_ABI_QUEUE_G_ID; 973 frames_sp->dvs_buf.buf_type = IMGU_ABI_BUFFER_TYPE_DIS_STATISTICS; 974 975 sp_stage->dvs_envelope.width = 976 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width; 977 sp_stage->dvs_envelope.height = 978 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height; 979 980 sp_stage->isp_pipe_version = 981 bi->info.isp.sp.pipeline.isp_pipe_version; 982 sp_stage->isp_deci_log_factor = 983 clamp(max(fls(css_pipe->rect[IPU3_CSS_RECT_BDS].width / 984 IMGU_MAX_BQ_GRID_WIDTH), 985 fls(css_pipe->rect[IPU3_CSS_RECT_BDS].height / 986 IMGU_MAX_BQ_GRID_HEIGHT)) - 1, 3, 5); 987 sp_stage->isp_vf_downscale_bits = 0; 988 sp_stage->if_config_index = 255; 989 sp_stage->sp_enable_xnr = 0; 990 sp_stage->num_stripes = stripes; 991 sp_stage->enable.s3a = 1; 992 sp_stage->enable.dvs_stats = 0; 993 994 sp_stage->xmem_bin_addr = css->binary[css_pipe->bindex].daddr; 995 sp_stage->xmem_map_addr = css_pipe->sp_ddr_ptrs.daddr; 996 sp_stage->isp_stage_addr = 997 css_pipe->xmem_isp_stage_ptrs[pipe][stage].daddr; 998 999 /* Configure SP group */ 1000 1001 sp_group = css->xmem_sp_group_ptrs.vaddr; 1002 memset(&sp_group->pipe[pipe], 0, sizeof(struct imgu_abi_sp_pipeline)); 1003 1004 sp_group->pipe[pipe].num_stages = 1; 1005 sp_group->pipe[pipe].pipe_id = css_pipe->pipe_id; 1006 sp_group->pipe[pipe].thread_id = pipe; 1007 sp_group->pipe[pipe].pipe_num = pipe; 1008 sp_group->pipe[pipe].num_execs = -1; 1009 sp_group->pipe[pipe].pipe_qos_config = -1; 1010 sp_group->pipe[pipe].required_bds_factor = 0; 1011 sp_group->pipe[pipe].dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1; 1012 sp_group->pipe[pipe].inout_port_config = 1013 IMGU_ABI_PORT_CONFIG_TYPE_INPUT_HOST | 1014 IMGU_ABI_PORT_CONFIG_TYPE_OUTPUT_HOST; 1015 sp_group->pipe[pipe].scaler_pp_lut = 0; 1016 sp_group->pipe[pipe].shading.internal_frame_origin_x_bqs_on_sctbl = 0; 1017 sp_group->pipe[pipe].shading.internal_frame_origin_y_bqs_on_sctbl = 0; 1018 sp_group->pipe[pipe].sp_stage_addr[stage] = 1019 css_pipe->xmem_sp_stage_ptrs[pipe][stage].daddr; 1020 sp_group->pipe[pipe].pipe_config = 1021 bi->info.isp.sp.enable.params ? (1 << pipe) : 0; 1022 sp_group->pipe[pipe].pipe_config |= IMGU_ABI_PIPE_CONFIG_ACQUIRE_ISP; 1023 1024 /* Initialize parameter pools */ 1025 1026 if (imgu_css_pool_init(imgu, &css_pipe->pool.parameter_set_info, 1027 sizeof(struct imgu_abi_parameter_set_info)) || 1028 imgu_css_pool_init(imgu, &css_pipe->pool.acc, 1029 sizeof(struct imgu_abi_acc_param)) || 1030 imgu_css_pool_init(imgu, &css_pipe->pool.gdc, 1031 sizeof(struct imgu_abi_gdc_warp_param) * 1032 3 * cfg_dvs->num_horizontal_blocks / 2 * 1033 cfg_dvs->num_vertical_blocks) || 1034 imgu_css_pool_init(imgu, &css_pipe->pool.obgrid, 1035 imgu_css_fw_obgrid_size( 1036 &css->fwp->binary_header[css_pipe->bindex]))) 1037 goto out_of_memory; 1038 1039 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) 1040 if (imgu_css_pool_init(imgu, 1041 &css_pipe->pool.binary_params_p[i], 1042 bi->info.isp.sp.mem_initializers.params 1043 [IMGU_ABI_PARAM_CLASS_PARAM][i].size)) 1044 goto out_of_memory; 1045 1046 return 0; 1047 1048 bad_firmware: 1049 imgu_css_pipeline_cleanup(css, pipe); 1050 return -EPROTO; 1051 1052 out_of_memory: 1053 imgu_css_pipeline_cleanup(css, pipe); 1054 return -ENOMEM; 1055 } 1056 1057 static u8 imgu_css_queue_pos(struct imgu_css *css, int queue, int thread) 1058 { 1059 static const unsigned int sp; 1060 void __iomem *const base = css->base; 1061 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]]; 1062 struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) + 1063 bi->info.sp.host_sp_queue; 1064 1065 return queue >= 0 ? readb(&q->host2sp_bufq_info[thread][queue].end) : 1066 readb(&q->host2sp_evtq_info.end); 1067 } 1068 1069 /* Sent data to sp using given buffer queue, or if queue < 0, event queue. */ 1070 static int imgu_css_queue_data(struct imgu_css *css, 1071 int queue, int thread, u32 data) 1072 { 1073 static const unsigned int sp; 1074 void __iomem *const base = css->base; 1075 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]]; 1076 struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) + 1077 bi->info.sp.host_sp_queue; 1078 u8 size, start, end, end2; 1079 1080 if (queue >= 0) { 1081 size = readb(&q->host2sp_bufq_info[thread][queue].size); 1082 start = readb(&q->host2sp_bufq_info[thread][queue].start); 1083 end = readb(&q->host2sp_bufq_info[thread][queue].end); 1084 } else { 1085 size = readb(&q->host2sp_evtq_info.size); 1086 start = readb(&q->host2sp_evtq_info.start); 1087 end = readb(&q->host2sp_evtq_info.end); 1088 } 1089 1090 if (size == 0) 1091 return -EIO; 1092 1093 end2 = (end + 1) % size; 1094 if (end2 == start) 1095 return -EBUSY; /* Queue full */ 1096 1097 if (queue >= 0) { 1098 writel(data, &q->host2sp_bufq[thread][queue][end]); 1099 writeb(end2, &q->host2sp_bufq_info[thread][queue].end); 1100 } else { 1101 writel(data, &q->host2sp_evtq[end]); 1102 writeb(end2, &q->host2sp_evtq_info.end); 1103 } 1104 1105 return 0; 1106 } 1107 1108 /* Receive data using given buffer queue, or if queue < 0, event queue. */ 1109 static int imgu_css_dequeue_data(struct imgu_css *css, int queue, u32 *data) 1110 { 1111 static const unsigned int sp; 1112 void __iomem *const base = css->base; 1113 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]]; 1114 struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) + 1115 bi->info.sp.host_sp_queue; 1116 u8 size, start, end, start2; 1117 1118 if (queue >= 0) { 1119 size = readb(&q->sp2host_bufq_info[queue].size); 1120 start = readb(&q->sp2host_bufq_info[queue].start); 1121 end = readb(&q->sp2host_bufq_info[queue].end); 1122 } else { 1123 size = readb(&q->sp2host_evtq_info.size); 1124 start = readb(&q->sp2host_evtq_info.start); 1125 end = readb(&q->sp2host_evtq_info.end); 1126 } 1127 1128 if (size == 0) 1129 return -EIO; 1130 1131 if (end == start) 1132 return -EBUSY; /* Queue empty */ 1133 1134 start2 = (start + 1) % size; 1135 1136 if (queue >= 0) { 1137 *data = readl(&q->sp2host_bufq[queue][start]); 1138 writeb(start2, &q->sp2host_bufq_info[queue].start); 1139 } else { 1140 int r; 1141 1142 *data = readl(&q->sp2host_evtq[start]); 1143 writeb(start2, &q->sp2host_evtq_info.start); 1144 1145 /* Acknowledge events dequeued from event queue */ 1146 r = imgu_css_queue_data(css, queue, 0, 1147 IMGU_ABI_EVENT_EVENT_DEQUEUED); 1148 if (r < 0) 1149 return r; 1150 } 1151 1152 return 0; 1153 } 1154 1155 /* Free binary-specific resources */ 1156 static void imgu_css_binary_cleanup(struct imgu_css *css, unsigned int pipe) 1157 { 1158 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1159 unsigned int i, j; 1160 1161 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1162 1163 for (j = 0; j < IMGU_ABI_PARAM_CLASS_NUM - 1; j++) 1164 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) 1165 imgu_dmamap_free(imgu, 1166 &css_pipe->binary_params_cs[j][i]); 1167 1168 j = IPU3_CSS_AUX_FRAME_REF; 1169 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1170 imgu_dmamap_free(imgu, 1171 &css_pipe->aux_frames[j].mem[i]); 1172 1173 j = IPU3_CSS_AUX_FRAME_TNR; 1174 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1175 imgu_dmamap_free(imgu, 1176 &css_pipe->aux_frames[j].mem[i]); 1177 } 1178 1179 static int imgu_css_binary_preallocate(struct imgu_css *css, unsigned int pipe) 1180 { 1181 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1182 unsigned int i, j; 1183 1184 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1185 1186 for (j = IMGU_ABI_PARAM_CLASS_CONFIG; 1187 j < IMGU_ABI_PARAM_CLASS_NUM; j++) 1188 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) 1189 if (!imgu_dmamap_alloc(imgu, 1190 &css_pipe->binary_params_cs[j - 1][i], 1191 CSS_ABI_SIZE)) 1192 goto out_of_memory; 1193 1194 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1195 if (!imgu_dmamap_alloc(imgu, 1196 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i], 1197 CSS_BDS_SIZE)) 1198 goto out_of_memory; 1199 1200 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1201 if (!imgu_dmamap_alloc(imgu, 1202 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].mem[i], 1203 CSS_GDC_SIZE)) 1204 goto out_of_memory; 1205 1206 return 0; 1207 1208 out_of_memory: 1209 imgu_css_binary_cleanup(css, pipe); 1210 return -ENOMEM; 1211 } 1212 1213 /* allocate binary-specific resources */ 1214 static int imgu_css_binary_setup(struct imgu_css *css, unsigned int pipe) 1215 { 1216 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1217 struct imgu_fw_info *bi = &css->fwp->binary_header[css_pipe->bindex]; 1218 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1219 int i, j, size; 1220 static const int BYPC = 2; /* Bytes per component */ 1221 unsigned int w, h; 1222 1223 /* Allocate parameter memory blocks for this binary */ 1224 1225 for (j = IMGU_ABI_PARAM_CLASS_CONFIG; j < IMGU_ABI_PARAM_CLASS_NUM; j++) 1226 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) { 1227 if (imgu_css_dma_buffer_resize( 1228 imgu, 1229 &css_pipe->binary_params_cs[j - 1][i], 1230 bi->info.isp.sp.mem_initializers.params[j][i].size)) 1231 goto out_of_memory; 1232 } 1233 1234 /* Allocate internal frame buffers */ 1235 1236 /* Reference frames for DVS, FRAME_FORMAT_YUV420_16 */ 1237 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel = BYPC; 1238 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width = 1239 css_pipe->rect[IPU3_CSS_RECT_BDS].width; 1240 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height = 1241 ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].height, 1242 IMGU_DVS_BLOCK_H) + 2 * IMGU_GDC_BUF_Y; 1243 h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height; 1244 w = ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width, 1245 2 * IPU3_UAPI_ISP_VEC_ELEMS) + 2 * IMGU_GDC_BUF_X; 1246 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline = 1247 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel * w; 1248 size = w * h * BYPC + (w / 2) * (h / 2) * BYPC * 2; 1249 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1250 if (imgu_css_dma_buffer_resize( 1251 imgu, 1252 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i], 1253 size)) 1254 goto out_of_memory; 1255 1256 /* TNR frames for temporal noise reduction, FRAME_FORMAT_YUV_LINE */ 1257 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperpixel = 1; 1258 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width = 1259 roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].width, 1260 bi->info.isp.sp.block.block_width * 1261 IPU3_UAPI_ISP_VEC_ELEMS); 1262 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height = 1263 roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].height, 1264 bi->info.isp.sp.block.output_block_height); 1265 1266 w = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width; 1267 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline = w; 1268 h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height; 1269 size = w * ALIGN(h * 3 / 2 + 3, 2); /* +3 for vf_pp prefetch */ 1270 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1271 if (imgu_css_dma_buffer_resize( 1272 imgu, 1273 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].mem[i], 1274 size)) 1275 goto out_of_memory; 1276 1277 return 0; 1278 1279 out_of_memory: 1280 imgu_css_binary_cleanup(css, pipe); 1281 return -ENOMEM; 1282 } 1283 1284 int imgu_css_start_streaming(struct imgu_css *css) 1285 { 1286 u32 data; 1287 int r, pipe; 1288 1289 if (css->streaming) 1290 return -EPROTO; 1291 1292 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1293 r = imgu_css_binary_setup(css, pipe); 1294 if (r < 0) 1295 return r; 1296 } 1297 1298 r = imgu_css_hw_init(css); 1299 if (r < 0) 1300 return r; 1301 1302 r = imgu_css_hw_start(css); 1303 if (r < 0) 1304 goto fail; 1305 1306 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1307 r = imgu_css_pipeline_init(css, pipe); 1308 if (r < 0) 1309 goto fail; 1310 } 1311 1312 css->streaming = true; 1313 1314 imgu_css_hw_enable_irq(css); 1315 1316 /* Initialize parameters to default */ 1317 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1318 r = imgu_css_set_parameters(css, pipe, NULL); 1319 if (r < 0) 1320 goto fail; 1321 } 1322 1323 while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_A_ID, &data))) 1324 ; 1325 if (r != -EBUSY) 1326 goto fail; 1327 1328 while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_B_ID, &data))) 1329 ; 1330 if (r != -EBUSY) 1331 goto fail; 1332 1333 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1334 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, 1335 IMGU_ABI_EVENT_START_STREAM | 1336 pipe << 16); 1337 if (r < 0) 1338 goto fail; 1339 } 1340 1341 return 0; 1342 1343 fail: 1344 css->streaming = false; 1345 imgu_css_hw_cleanup(css); 1346 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1347 imgu_css_pipeline_cleanup(css, pipe); 1348 imgu_css_binary_cleanup(css, pipe); 1349 } 1350 1351 return r; 1352 } 1353 1354 void imgu_css_stop_streaming(struct imgu_css *css) 1355 { 1356 struct imgu_css_buffer *b, *b0; 1357 int q, r, pipe; 1358 1359 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1360 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, 1361 IMGU_ABI_EVENT_STOP_STREAM); 1362 if (r < 0) 1363 dev_warn(css->dev, "failed on stop stream event\n"); 1364 } 1365 1366 if (!css->streaming) 1367 return; 1368 1369 imgu_css_hw_stop(css); 1370 1371 imgu_css_hw_cleanup(css); 1372 1373 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1374 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1375 1376 imgu_css_pipeline_cleanup(css, pipe); 1377 1378 spin_lock(&css_pipe->qlock); 1379 for (q = 0; q < IPU3_CSS_QUEUES; q++) 1380 list_for_each_entry_safe(b, b0, 1381 &css_pipe->queue[q].bufs, 1382 list) { 1383 b->state = IPU3_CSS_BUFFER_FAILED; 1384 list_del(&b->list); 1385 } 1386 spin_unlock(&css_pipe->qlock); 1387 } 1388 1389 css->streaming = false; 1390 } 1391 1392 bool imgu_css_pipe_queue_empty(struct imgu_css *css, unsigned int pipe) 1393 { 1394 int q; 1395 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1396 1397 spin_lock(&css_pipe->qlock); 1398 for (q = 0; q < IPU3_CSS_QUEUES; q++) 1399 if (!list_empty(&css_pipe->queue[q].bufs)) 1400 break; 1401 spin_unlock(&css_pipe->qlock); 1402 return (q == IPU3_CSS_QUEUES); 1403 } 1404 1405 bool imgu_css_queue_empty(struct imgu_css *css) 1406 { 1407 unsigned int pipe; 1408 bool ret = false; 1409 1410 for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++) 1411 ret &= imgu_css_pipe_queue_empty(css, pipe); 1412 1413 return ret; 1414 } 1415 1416 bool imgu_css_is_streaming(struct imgu_css *css) 1417 { 1418 return css->streaming; 1419 } 1420 1421 static int imgu_css_map_init(struct imgu_css *css, unsigned int pipe) 1422 { 1423 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1424 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1425 unsigned int p, q, i; 1426 1427 /* Allocate and map common structures with imgu hardware */ 1428 for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++) 1429 for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) { 1430 if (!imgu_dmamap_alloc(imgu, 1431 &css_pipe->xmem_sp_stage_ptrs[p][i], 1432 sizeof(struct imgu_abi_sp_stage))) 1433 return -ENOMEM; 1434 if (!imgu_dmamap_alloc(imgu, 1435 &css_pipe->xmem_isp_stage_ptrs[p][i], 1436 sizeof(struct imgu_abi_isp_stage))) 1437 return -ENOMEM; 1438 } 1439 1440 if (!imgu_dmamap_alloc(imgu, &css_pipe->sp_ddr_ptrs, 1441 ALIGN(sizeof(struct imgu_abi_ddr_address_map), 1442 IMGU_ABI_ISP_DDR_WORD_BYTES))) 1443 return -ENOMEM; 1444 1445 for (q = 0; q < IPU3_CSS_QUEUES; q++) { 1446 unsigned int abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]); 1447 1448 for (i = 0; i < abi_buf_num; i++) 1449 if (!imgu_dmamap_alloc(imgu, 1450 &css_pipe->abi_buffers[q][i], 1451 sizeof(struct imgu_abi_buffer))) 1452 return -ENOMEM; 1453 } 1454 1455 if (imgu_css_binary_preallocate(css, pipe)) { 1456 imgu_css_binary_cleanup(css, pipe); 1457 return -ENOMEM; 1458 } 1459 1460 return 0; 1461 } 1462 1463 static void imgu_css_pipe_cleanup(struct imgu_css *css, unsigned int pipe) 1464 { 1465 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1466 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1467 unsigned int p, q, i, abi_buf_num; 1468 1469 imgu_css_binary_cleanup(css, pipe); 1470 1471 for (q = 0; q < IPU3_CSS_QUEUES; q++) { 1472 abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]); 1473 for (i = 0; i < abi_buf_num; i++) 1474 imgu_dmamap_free(imgu, &css_pipe->abi_buffers[q][i]); 1475 } 1476 1477 for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++) 1478 for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) { 1479 imgu_dmamap_free(imgu, 1480 &css_pipe->xmem_sp_stage_ptrs[p][i]); 1481 imgu_dmamap_free(imgu, 1482 &css_pipe->xmem_isp_stage_ptrs[p][i]); 1483 } 1484 1485 imgu_dmamap_free(imgu, &css_pipe->sp_ddr_ptrs); 1486 } 1487 1488 void imgu_css_cleanup(struct imgu_css *css) 1489 { 1490 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1491 unsigned int pipe; 1492 1493 imgu_css_stop_streaming(css); 1494 for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++) 1495 imgu_css_pipe_cleanup(css, pipe); 1496 imgu_dmamap_free(imgu, &css->xmem_sp_group_ptrs); 1497 imgu_css_fw_cleanup(css); 1498 } 1499 1500 int imgu_css_init(struct device *dev, struct imgu_css *css, 1501 void __iomem *base, int length) 1502 { 1503 struct imgu_device *imgu = dev_get_drvdata(dev); 1504 int r, q, pipe; 1505 1506 /* Initialize main data structure */ 1507 css->dev = dev; 1508 css->base = base; 1509 css->iomem_length = length; 1510 1511 for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++) { 1512 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1513 1514 css_pipe->vf_output_en = false; 1515 spin_lock_init(&css_pipe->qlock); 1516 css_pipe->bindex = IPU3_CSS_DEFAULT_BINARY; 1517 css_pipe->pipe_id = IPU3_CSS_PIPE_ID_VIDEO; 1518 for (q = 0; q < IPU3_CSS_QUEUES; q++) { 1519 r = imgu_css_queue_init(&css_pipe->queue[q], NULL, 0); 1520 if (r) 1521 return r; 1522 } 1523 r = imgu_css_map_init(css, pipe); 1524 if (r) { 1525 imgu_css_cleanup(css); 1526 return r; 1527 } 1528 } 1529 if (!imgu_dmamap_alloc(imgu, &css->xmem_sp_group_ptrs, 1530 sizeof(struct imgu_abi_sp_group))) 1531 return -ENOMEM; 1532 1533 r = imgu_css_fw_init(css); 1534 if (r) 1535 return r; 1536 1537 return 0; 1538 } 1539 1540 static u32 imgu_css_adjust(u32 res, u32 align) 1541 { 1542 u32 val = max_t(u32, IPU3_CSS_MIN_RES, res); 1543 1544 return DIV_ROUND_CLOSEST(val, align) * align; 1545 } 1546 1547 /* Select a binary matching the required resolutions and formats */ 1548 static int imgu_css_find_binary(struct imgu_css *css, 1549 unsigned int pipe, 1550 struct imgu_css_queue queue[IPU3_CSS_QUEUES], 1551 struct v4l2_rect rects[IPU3_CSS_RECTS]) 1552 { 1553 const int binary_nr = css->fwp->file_header.binary_nr; 1554 unsigned int binary_mode = 1555 (css->pipes[pipe].pipe_id == IPU3_CSS_PIPE_ID_CAPTURE) ? 1556 IA_CSS_BINARY_MODE_PRIMARY : IA_CSS_BINARY_MODE_VIDEO; 1557 const struct v4l2_pix_format_mplane *in = 1558 &queue[IPU3_CSS_QUEUE_IN].fmt.mpix; 1559 const struct v4l2_pix_format_mplane *out = 1560 &queue[IPU3_CSS_QUEUE_OUT].fmt.mpix; 1561 const struct v4l2_pix_format_mplane *vf = 1562 &queue[IPU3_CSS_QUEUE_VF].fmt.mpix; 1563 u32 stripe_w = 0, stripe_h = 0; 1564 const char *name; 1565 int i, j; 1566 1567 if (!imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_IN])) 1568 return -EINVAL; 1569 1570 /* Find out the strip size boundary */ 1571 for (i = 0; i < binary_nr; i++) { 1572 struct imgu_fw_info *bi = &css->fwp->binary_header[i]; 1573 1574 u32 max_width = bi->info.isp.sp.output.max_width; 1575 u32 max_height = bi->info.isp.sp.output.max_height; 1576 1577 if (bi->info.isp.sp.iterator.num_stripes <= 1) { 1578 stripe_w = stripe_w ? 1579 min(stripe_w, max_width) : max_width; 1580 stripe_h = stripe_h ? 1581 min(stripe_h, max_height) : max_height; 1582 } 1583 } 1584 1585 for (i = 0; i < binary_nr; i++) { 1586 struct imgu_fw_info *bi = &css->fwp->binary_header[i]; 1587 enum imgu_abi_frame_format q_fmt; 1588 1589 name = (void *)css->fwp + bi->blob.prog_name_offset; 1590 1591 /* Check that binary supports memory-to-memory processing */ 1592 if (bi->info.isp.sp.input.source != 1593 IMGU_ABI_BINARY_INPUT_SOURCE_MEMORY) 1594 continue; 1595 1596 /* Check that binary supports raw10 input */ 1597 if (!bi->info.isp.sp.enable.input_feeder && 1598 !bi->info.isp.sp.enable.input_raw) 1599 continue; 1600 1601 /* Check binary mode */ 1602 if (bi->info.isp.sp.pipeline.mode != binary_mode) 1603 continue; 1604 1605 /* Since input is RGGB bayer, need to process colors */ 1606 if (bi->info.isp.sp.enable.luma_only) 1607 continue; 1608 1609 if (in->width < bi->info.isp.sp.input.min_width || 1610 in->width > bi->info.isp.sp.input.max_width || 1611 in->height < bi->info.isp.sp.input.min_height || 1612 in->height > bi->info.isp.sp.input.max_height) 1613 continue; 1614 1615 if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_OUT])) { 1616 if (bi->info.isp.num_output_pins <= 0) 1617 continue; 1618 1619 q_fmt = queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format; 1620 for (j = 0; j < bi->info.isp.num_output_formats; j++) 1621 if (bi->info.isp.output_formats[j] == q_fmt) 1622 break; 1623 if (j >= bi->info.isp.num_output_formats) 1624 continue; 1625 1626 if (out->width < bi->info.isp.sp.output.min_width || 1627 out->width > bi->info.isp.sp.output.max_width || 1628 out->height < bi->info.isp.sp.output.min_height || 1629 out->height > bi->info.isp.sp.output.max_height) 1630 continue; 1631 1632 if (out->width > bi->info.isp.sp.internal.max_width || 1633 out->height > bi->info.isp.sp.internal.max_height) 1634 continue; 1635 } 1636 1637 if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_VF])) { 1638 if (bi->info.isp.num_output_pins <= 1) 1639 continue; 1640 1641 q_fmt = queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format; 1642 for (j = 0; j < bi->info.isp.num_output_formats; j++) 1643 if (bi->info.isp.output_formats[j] == q_fmt) 1644 break; 1645 if (j >= bi->info.isp.num_output_formats) 1646 continue; 1647 1648 if (vf->width < bi->info.isp.sp.output.min_width || 1649 vf->width > bi->info.isp.sp.output.max_width || 1650 vf->height < bi->info.isp.sp.output.min_height || 1651 vf->height > bi->info.isp.sp.output.max_height) 1652 continue; 1653 } 1654 1655 /* All checks passed, select the binary */ 1656 dev_dbg(css->dev, "using binary %s id = %u\n", name, 1657 bi->info.isp.sp.id); 1658 return i; 1659 } 1660 1661 /* Can not find suitable binary for these parameters */ 1662 return -EINVAL; 1663 } 1664 1665 /* 1666 * Check that there is a binary matching requirements. Parameters may be 1667 * NULL indicating disabled input/output. Return negative if given 1668 * parameters can not be supported or on error, zero or positive indicating 1669 * found binary number. May modify the given parameters if not exact match 1670 * is found. 1671 */ 1672 int imgu_css_fmt_try(struct imgu_css *css, 1673 struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES], 1674 struct v4l2_rect *rects[IPU3_CSS_RECTS], 1675 unsigned int pipe) 1676 { 1677 static const u32 EFF_ALIGN_W = 2; 1678 static const u32 BDS_ALIGN_W = 4; 1679 static const u32 OUT_ALIGN_W = 8; 1680 static const u32 OUT_ALIGN_H = 4; 1681 static const u32 VF_ALIGN_W = 2; 1682 static const char *qnames[IPU3_CSS_QUEUES] = { 1683 [IPU3_CSS_QUEUE_IN] = "in", 1684 [IPU3_CSS_QUEUE_PARAMS] = "params", 1685 [IPU3_CSS_QUEUE_OUT] = "out", 1686 [IPU3_CSS_QUEUE_VF] = "vf", 1687 [IPU3_CSS_QUEUE_STAT_3A] = "3a", 1688 }; 1689 static const char *rnames[IPU3_CSS_RECTS] = { 1690 [IPU3_CSS_RECT_EFFECTIVE] = "effective resolution", 1691 [IPU3_CSS_RECT_BDS] = "bayer-domain scaled resolution", 1692 [IPU3_CSS_RECT_ENVELOPE] = "DVS envelope size", 1693 [IPU3_CSS_RECT_GDC] = "GDC output res", 1694 }; 1695 struct v4l2_rect r[IPU3_CSS_RECTS] = { }; 1696 struct v4l2_rect *const eff = &r[IPU3_CSS_RECT_EFFECTIVE]; 1697 struct v4l2_rect *const bds = &r[IPU3_CSS_RECT_BDS]; 1698 struct v4l2_rect *const env = &r[IPU3_CSS_RECT_ENVELOPE]; 1699 struct v4l2_rect *const gdc = &r[IPU3_CSS_RECT_GDC]; 1700 struct imgu_css_queue *q; 1701 struct v4l2_pix_format_mplane *in, *out, *vf; 1702 int i, s, ret; 1703 1704 q = kcalloc(IPU3_CSS_QUEUES, sizeof(struct imgu_css_queue), GFP_KERNEL); 1705 if (!q) 1706 return -ENOMEM; 1707 1708 in = &q[IPU3_CSS_QUEUE_IN].fmt.mpix; 1709 out = &q[IPU3_CSS_QUEUE_OUT].fmt.mpix; 1710 vf = &q[IPU3_CSS_QUEUE_VF].fmt.mpix; 1711 1712 /* Adjust all formats, get statistics buffer sizes and formats */ 1713 for (i = 0; i < IPU3_CSS_QUEUES; i++) { 1714 if (fmts[i]) 1715 dev_dbg(css->dev, "%s %s: (%i,%i) fmt 0x%x\n", __func__, 1716 qnames[i], fmts[i]->width, fmts[i]->height, 1717 fmts[i]->pixelformat); 1718 else 1719 dev_dbg(css->dev, "%s %s: (not set)\n", __func__, 1720 qnames[i]); 1721 if (imgu_css_queue_init(&q[i], fmts[i], 1722 IPU3_CSS_QUEUE_TO_FLAGS(i))) { 1723 dev_notice(css->dev, "can not initialize queue %s\n", 1724 qnames[i]); 1725 ret = -EINVAL; 1726 goto out; 1727 } 1728 } 1729 for (i = 0; i < IPU3_CSS_RECTS; i++) { 1730 if (rects[i]) { 1731 dev_dbg(css->dev, "%s %s: (%i,%i)\n", __func__, 1732 rnames[i], rects[i]->width, rects[i]->height); 1733 r[i].width = rects[i]->width; 1734 r[i].height = rects[i]->height; 1735 } else { 1736 dev_dbg(css->dev, "%s %s: (not set)\n", __func__, 1737 rnames[i]); 1738 } 1739 /* For now, force known good resolutions */ 1740 r[i].left = 0; 1741 r[i].top = 0; 1742 } 1743 1744 /* Always require one input and vf only if out is also enabled */ 1745 if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_IN]) || 1746 !imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) { 1747 dev_warn(css->dev, "required queues are disabled\n"); 1748 ret = -EINVAL; 1749 goto out; 1750 } 1751 1752 if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) { 1753 out->width = in->width; 1754 out->height = in->height; 1755 } 1756 if (eff->width <= 0 || eff->height <= 0) { 1757 eff->width = in->width; 1758 eff->height = in->height; 1759 } 1760 if (bds->width <= 0 || bds->height <= 0) { 1761 bds->width = out->width; 1762 bds->height = out->height; 1763 } 1764 if (gdc->width <= 0 || gdc->height <= 0) { 1765 gdc->width = out->width; 1766 gdc->height = out->height; 1767 } 1768 1769 in->width = imgu_css_adjust(in->width, 1); 1770 in->height = imgu_css_adjust(in->height, 1); 1771 eff->width = imgu_css_adjust(eff->width, EFF_ALIGN_W); 1772 eff->height = imgu_css_adjust(eff->height, 1); 1773 bds->width = imgu_css_adjust(bds->width, BDS_ALIGN_W); 1774 bds->height = imgu_css_adjust(bds->height, 1); 1775 gdc->width = imgu_css_adjust(gdc->width, OUT_ALIGN_W); 1776 gdc->height = imgu_css_adjust(gdc->height, OUT_ALIGN_H); 1777 out->width = imgu_css_adjust(out->width, OUT_ALIGN_W); 1778 out->height = imgu_css_adjust(out->height, OUT_ALIGN_H); 1779 vf->width = imgu_css_adjust(vf->width, VF_ALIGN_W); 1780 vf->height = imgu_css_adjust(vf->height, 1); 1781 1782 s = (bds->width - gdc->width) / 2; 1783 env->width = s < MIN_ENVELOPE ? MIN_ENVELOPE : s; 1784 s = (bds->height - gdc->height) / 2; 1785 env->height = s < MIN_ENVELOPE ? MIN_ENVELOPE : s; 1786 1787 ret = imgu_css_find_binary(css, pipe, q, r); 1788 if (ret < 0) { 1789 dev_err(css->dev, "failed to find suitable binary\n"); 1790 ret = -EINVAL; 1791 goto out; 1792 } 1793 css->pipes[pipe].bindex = ret; 1794 1795 dev_dbg(css->dev, "Binary index %d for pipe %d found.", 1796 css->pipes[pipe].bindex, pipe); 1797 1798 /* Final adjustment and set back the queried formats */ 1799 for (i = 0; i < IPU3_CSS_QUEUES; i++) { 1800 if (fmts[i]) { 1801 if (imgu_css_queue_init(&q[i], &q[i].fmt.mpix, 1802 IPU3_CSS_QUEUE_TO_FLAGS(i))) { 1803 dev_err(css->dev, 1804 "final resolution adjustment failed\n"); 1805 ret = -EINVAL; 1806 goto out; 1807 } 1808 *fmts[i] = q[i].fmt.mpix; 1809 } 1810 } 1811 1812 for (i = 0; i < IPU3_CSS_RECTS; i++) 1813 if (rects[i]) 1814 *rects[i] = r[i]; 1815 1816 dev_dbg(css->dev, 1817 "in(%u,%u) if(%u,%u) ds(%u,%u) gdc(%u,%u) out(%u,%u) vf(%u,%u)", 1818 in->width, in->height, eff->width, eff->height, 1819 bds->width, bds->height, gdc->width, gdc->height, 1820 out->width, out->height, vf->width, vf->height); 1821 1822 ret = 0; 1823 out: 1824 kfree(q); 1825 return ret; 1826 } 1827 1828 int imgu_css_fmt_set(struct imgu_css *css, 1829 struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES], 1830 struct v4l2_rect *rects[IPU3_CSS_RECTS], 1831 unsigned int pipe) 1832 { 1833 struct v4l2_rect rect_data[IPU3_CSS_RECTS]; 1834 struct v4l2_rect *all_rects[IPU3_CSS_RECTS]; 1835 int i, r; 1836 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1837 1838 for (i = 0; i < IPU3_CSS_RECTS; i++) { 1839 if (rects[i]) 1840 rect_data[i] = *rects[i]; 1841 else 1842 memset(&rect_data[i], 0, sizeof(rect_data[i])); 1843 all_rects[i] = &rect_data[i]; 1844 } 1845 r = imgu_css_fmt_try(css, fmts, all_rects, pipe); 1846 if (r < 0) 1847 return r; 1848 1849 for (i = 0; i < IPU3_CSS_QUEUES; i++) 1850 if (imgu_css_queue_init(&css_pipe->queue[i], fmts[i], 1851 IPU3_CSS_QUEUE_TO_FLAGS(i))) 1852 return -EINVAL; 1853 for (i = 0; i < IPU3_CSS_RECTS; i++) { 1854 css_pipe->rect[i] = rect_data[i]; 1855 if (rects[i]) 1856 *rects[i] = rect_data[i]; 1857 } 1858 1859 return 0; 1860 } 1861 1862 int imgu_css_meta_fmt_set(struct v4l2_meta_format *fmt) 1863 { 1864 switch (fmt->dataformat) { 1865 case V4L2_META_FMT_IPU3_PARAMS: 1866 fmt->buffersize = sizeof(struct ipu3_uapi_params); 1867 1868 /* 1869 * Sanity check for the parameter struct size. This must 1870 * not change! 1871 */ 1872 BUILD_BUG_ON(sizeof(struct ipu3_uapi_params) != 39328); 1873 1874 break; 1875 case V4L2_META_FMT_IPU3_STAT_3A: 1876 fmt->buffersize = sizeof(struct ipu3_uapi_stats_3a); 1877 break; 1878 default: 1879 return -EINVAL; 1880 } 1881 1882 return 0; 1883 } 1884 1885 /* 1886 * Queue given buffer to CSS. imgu_css_buf_prepare() must have been first 1887 * called for the buffer. May be called from interrupt context. 1888 * Returns 0 on success, -EBUSY if the buffer queue is full, or some other 1889 * code on error conditions. 1890 */ 1891 int imgu_css_buf_queue(struct imgu_css *css, unsigned int pipe, 1892 struct imgu_css_buffer *b) 1893 { 1894 struct imgu_abi_buffer *abi_buf; 1895 struct imgu_addr_t *buf_addr; 1896 u32 data; 1897 int r; 1898 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1899 1900 if (!css->streaming) 1901 return -EPROTO; /* CSS or buffer in wrong state */ 1902 1903 if (b->queue >= IPU3_CSS_QUEUES || !imgu_css_queues[b->queue].qid) 1904 return -EINVAL; 1905 1906 b->queue_pos = imgu_css_queue_pos(css, imgu_css_queues[b->queue].qid, 1907 pipe); 1908 1909 if (b->queue_pos >= ARRAY_SIZE(css->pipes[pipe].abi_buffers[b->queue])) 1910 return -EIO; 1911 abi_buf = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].vaddr; 1912 1913 /* Fill struct abi_buffer for firmware */ 1914 memset(abi_buf, 0, sizeof(*abi_buf)); 1915 1916 buf_addr = (void *)abi_buf + imgu_css_queues[b->queue].ptr_ofs; 1917 *(imgu_addr_t *)buf_addr = b->daddr; 1918 1919 if (b->queue == IPU3_CSS_QUEUE_STAT_3A) 1920 abi_buf->payload.s3a.data.dmem.s3a_tbl = b->daddr; 1921 1922 if (b->queue == IPU3_CSS_QUEUE_OUT) 1923 abi_buf->payload.frame.padded_width = 1924 css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad; 1925 1926 if (b->queue == IPU3_CSS_QUEUE_VF) 1927 abi_buf->payload.frame.padded_width = 1928 css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad; 1929 1930 spin_lock(&css_pipe->qlock); 1931 list_add_tail(&b->list, &css_pipe->queue[b->queue].bufs); 1932 spin_unlock(&css_pipe->qlock); 1933 b->state = IPU3_CSS_BUFFER_QUEUED; 1934 1935 data = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].daddr; 1936 r = imgu_css_queue_data(css, imgu_css_queues[b->queue].qid, 1937 pipe, data); 1938 if (r < 0) 1939 goto queueing_failed; 1940 1941 data = IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe, 1942 imgu_css_queues[b->queue].qid); 1943 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, data); 1944 if (r < 0) 1945 goto queueing_failed; 1946 1947 dev_dbg(css->dev, "queued buffer %p to css queue %i in pipe %d\n", 1948 b, b->queue, pipe); 1949 1950 return 0; 1951 1952 queueing_failed: 1953 b->state = (r == -EBUSY || r == -EAGAIN) ? 1954 IPU3_CSS_BUFFER_NEW : IPU3_CSS_BUFFER_FAILED; 1955 list_del(&b->list); 1956 1957 return r; 1958 } 1959 1960 /* 1961 * Get next ready CSS buffer. Returns -EAGAIN in which case the function 1962 * should be called again, or -EBUSY which means that there are no more 1963 * buffers available. May be called from interrupt context. 1964 */ 1965 struct imgu_css_buffer *imgu_css_buf_dequeue(struct imgu_css *css) 1966 { 1967 static const unsigned char evtype_to_queue[] = { 1968 [IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE] = IPU3_CSS_QUEUE_IN, 1969 [IMGU_ABI_EVTTYPE_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_OUT, 1970 [IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_VF, 1971 [IMGU_ABI_EVTTYPE_3A_STATS_DONE] = IPU3_CSS_QUEUE_STAT_3A, 1972 }; 1973 struct imgu_css_buffer *b = ERR_PTR(-EAGAIN); 1974 u32 event, daddr; 1975 int evtype, pipe, pipeid, queue, qid, r; 1976 struct imgu_css_pipe *css_pipe; 1977 1978 if (!css->streaming) 1979 return ERR_PTR(-EPROTO); 1980 1981 r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event); 1982 if (r < 0) 1983 return ERR_PTR(r); 1984 1985 evtype = (event & IMGU_ABI_EVTTYPE_EVENT_MASK) >> 1986 IMGU_ABI_EVTTYPE_EVENT_SHIFT; 1987 1988 switch (evtype) { 1989 case IMGU_ABI_EVTTYPE_OUT_FRAME_DONE: 1990 case IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE: 1991 case IMGU_ABI_EVTTYPE_3A_STATS_DONE: 1992 case IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE: 1993 pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >> 1994 IMGU_ABI_EVTTYPE_PIPE_SHIFT; 1995 pipeid = (event & IMGU_ABI_EVTTYPE_PIPEID_MASK) >> 1996 IMGU_ABI_EVTTYPE_PIPEID_SHIFT; 1997 queue = evtype_to_queue[evtype]; 1998 qid = imgu_css_queues[queue].qid; 1999 2000 if (pipe >= IMGU_MAX_PIPE_NUM) { 2001 dev_err(css->dev, "Invalid pipe: %i\n", pipe); 2002 return ERR_PTR(-EIO); 2003 } 2004 2005 if (qid >= IMGU_ABI_QUEUE_NUM) { 2006 dev_err(css->dev, "Invalid qid: %i\n", qid); 2007 return ERR_PTR(-EIO); 2008 } 2009 css_pipe = &css->pipes[pipe]; 2010 dev_dbg(css->dev, 2011 "event: buffer done 0x%x queue %i pipe %i pipeid %i\n", 2012 event, queue, pipe, pipeid); 2013 2014 r = imgu_css_dequeue_data(css, qid, &daddr); 2015 if (r < 0) { 2016 dev_err(css->dev, "failed to dequeue buffer\n"); 2017 /* Force real error, not -EBUSY */ 2018 return ERR_PTR(-EIO); 2019 } 2020 2021 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, 2022 IMGU_ABI_EVENT_BUFFER_DEQUEUED(qid)); 2023 if (r < 0) { 2024 dev_err(css->dev, "failed to queue event\n"); 2025 return ERR_PTR(-EIO); 2026 } 2027 2028 spin_lock(&css_pipe->qlock); 2029 if (list_empty(&css_pipe->queue[queue].bufs)) { 2030 spin_unlock(&css_pipe->qlock); 2031 dev_err(css->dev, "event on empty queue\n"); 2032 return ERR_PTR(-EIO); 2033 } 2034 b = list_first_entry(&css_pipe->queue[queue].bufs, 2035 struct imgu_css_buffer, list); 2036 if (queue != b->queue || 2037 daddr != css_pipe->abi_buffers 2038 [b->queue][b->queue_pos].daddr) { 2039 spin_unlock(&css_pipe->qlock); 2040 dev_err(css->dev, "dequeued bad buffer 0x%x\n", daddr); 2041 return ERR_PTR(-EIO); 2042 } 2043 2044 dev_dbg(css->dev, "buffer 0x%8x done from pipe %d\n", daddr, pipe); 2045 b->pipe = pipe; 2046 b->state = IPU3_CSS_BUFFER_DONE; 2047 list_del(&b->list); 2048 spin_unlock(&css_pipe->qlock); 2049 break; 2050 case IMGU_ABI_EVTTYPE_PIPELINE_DONE: 2051 pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >> 2052 IMGU_ABI_EVTTYPE_PIPE_SHIFT; 2053 if (pipe >= IMGU_MAX_PIPE_NUM) { 2054 dev_err(css->dev, "Invalid pipe: %i\n", pipe); 2055 return ERR_PTR(-EIO); 2056 } 2057 2058 css_pipe = &css->pipes[pipe]; 2059 dev_dbg(css->dev, "event: pipeline done 0x%8x for pipe %d\n", 2060 event, pipe); 2061 break; 2062 case IMGU_ABI_EVTTYPE_TIMER: 2063 r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event); 2064 if (r < 0) 2065 return ERR_PTR(r); 2066 2067 if ((event & IMGU_ABI_EVTTYPE_EVENT_MASK) >> 2068 IMGU_ABI_EVTTYPE_EVENT_SHIFT == IMGU_ABI_EVTTYPE_TIMER) 2069 dev_dbg(css->dev, "event: timer\n"); 2070 else 2071 dev_warn(css->dev, "half of timer event missing\n"); 2072 break; 2073 case IMGU_ABI_EVTTYPE_FW_WARNING: 2074 dev_warn(css->dev, "event: firmware warning 0x%x\n", event); 2075 break; 2076 case IMGU_ABI_EVTTYPE_FW_ASSERT: 2077 dev_err(css->dev, 2078 "event: firmware assert 0x%x module_id %i line_no %i\n", 2079 event, 2080 (event & IMGU_ABI_EVTTYPE_MODULEID_MASK) >> 2081 IMGU_ABI_EVTTYPE_MODULEID_SHIFT, 2082 swab16((event & IMGU_ABI_EVTTYPE_LINENO_MASK) >> 2083 IMGU_ABI_EVTTYPE_LINENO_SHIFT)); 2084 break; 2085 default: 2086 dev_warn(css->dev, "received unknown event 0x%x\n", event); 2087 } 2088 2089 return b; 2090 } 2091 2092 /* 2093 * Get a new set of parameters from pool and initialize them based on 2094 * the parameters params, gdc, and obgrid. Any of these may be NULL, 2095 * in which case the previously set parameters are used. 2096 * If parameters haven't been set previously, initialize from scratch. 2097 * 2098 * Return index to css->parameter_set_info which has the newly created 2099 * parameters or negative value on error. 2100 */ 2101 int imgu_css_set_parameters(struct imgu_css *css, unsigned int pipe, 2102 struct ipu3_uapi_params *set_params) 2103 { 2104 static const unsigned int queue_id = IMGU_ABI_QUEUE_A_ID; 2105 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 2106 const int stage = 0; 2107 const struct imgu_fw_info *bi; 2108 int obgrid_size; 2109 unsigned int stripes, i; 2110 struct ipu3_uapi_flags *use = set_params ? &set_params->use : NULL; 2111 2112 /* Destination buffers which are filled here */ 2113 struct imgu_abi_parameter_set_info *param_set; 2114 struct imgu_abi_acc_param *acc = NULL; 2115 struct imgu_abi_gdc_warp_param *gdc = NULL; 2116 struct ipu3_uapi_obgrid_param *obgrid = NULL; 2117 const struct imgu_css_map *map; 2118 void *vmem0 = NULL; 2119 void *dmem0 = NULL; 2120 2121 enum imgu_abi_memories m; 2122 int r = -EBUSY; 2123 2124 if (!css->streaming) 2125 return -EPROTO; 2126 2127 dev_dbg(css->dev, "%s for pipe %d", __func__, pipe); 2128 2129 bi = &css->fwp->binary_header[css_pipe->bindex]; 2130 obgrid_size = imgu_css_fw_obgrid_size(bi); 2131 stripes = bi->info.isp.sp.iterator.num_stripes ? : 1; 2132 2133 imgu_css_pool_get(&css_pipe->pool.parameter_set_info); 2134 param_set = imgu_css_pool_last(&css_pipe->pool.parameter_set_info, 2135 0)->vaddr; 2136 2137 /* Get a new acc only if new parameters given, or none yet */ 2138 map = imgu_css_pool_last(&css_pipe->pool.acc, 0); 2139 if (set_params || !map->vaddr) { 2140 imgu_css_pool_get(&css_pipe->pool.acc); 2141 map = imgu_css_pool_last(&css_pipe->pool.acc, 0); 2142 acc = map->vaddr; 2143 } 2144 2145 /* Get new VMEM0 only if needed, or none yet */ 2146 m = IMGU_ABI_MEM_ISP_VMEM0; 2147 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0); 2148 if (!map->vaddr || (set_params && (set_params->use.lin_vmem_params || 2149 set_params->use.tnr3_vmem_params || 2150 set_params->use.xnr3_vmem_params))) { 2151 imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]); 2152 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0); 2153 vmem0 = map->vaddr; 2154 } 2155 2156 /* Get new DMEM0 only if needed, or none yet */ 2157 m = IMGU_ABI_MEM_ISP_DMEM0; 2158 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0); 2159 if (!map->vaddr || (set_params && (set_params->use.tnr3_dmem_params || 2160 set_params->use.xnr3_dmem_params))) { 2161 imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]); 2162 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0); 2163 dmem0 = map->vaddr; 2164 } 2165 2166 /* Configure acc parameter cluster */ 2167 if (acc) { 2168 /* get acc_old */ 2169 map = imgu_css_pool_last(&css_pipe->pool.acc, 1); 2170 /* user acc */ 2171 r = imgu_css_cfg_acc(css, pipe, use, acc, map->vaddr, 2172 set_params ? &set_params->acc_param : NULL); 2173 if (r < 0) 2174 goto fail; 2175 } 2176 2177 /* Configure late binding parameters */ 2178 if (vmem0) { 2179 m = IMGU_ABI_MEM_ISP_VMEM0; 2180 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1); 2181 r = imgu_css_cfg_vmem0(css, pipe, use, vmem0, 2182 map->vaddr, set_params); 2183 if (r < 0) 2184 goto fail; 2185 } 2186 2187 if (dmem0) { 2188 m = IMGU_ABI_MEM_ISP_DMEM0; 2189 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1); 2190 r = imgu_css_cfg_dmem0(css, pipe, use, dmem0, 2191 map->vaddr, set_params); 2192 if (r < 0) 2193 goto fail; 2194 } 2195 2196 /* Get a new gdc only if a new gdc is given, or none yet */ 2197 if (bi->info.isp.sp.enable.dvs_6axis) { 2198 unsigned int a = IPU3_CSS_AUX_FRAME_REF; 2199 unsigned int g = IPU3_CSS_RECT_GDC; 2200 unsigned int e = IPU3_CSS_RECT_ENVELOPE; 2201 2202 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0); 2203 if (!map->vaddr) { 2204 imgu_css_pool_get(&css_pipe->pool.gdc); 2205 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0); 2206 gdc = map->vaddr; 2207 imgu_css_cfg_gdc_table(map->vaddr, 2208 css_pipe->aux_frames[a].bytesperline / 2209 css_pipe->aux_frames[a].bytesperpixel, 2210 css_pipe->aux_frames[a].height, 2211 css_pipe->rect[g].width, 2212 css_pipe->rect[g].height, 2213 css_pipe->rect[e].width, 2214 css_pipe->rect[e].height); 2215 } 2216 } 2217 2218 /* Get a new obgrid only if a new obgrid is given, or none yet */ 2219 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0); 2220 if (!map->vaddr || (set_params && set_params->use.obgrid_param)) { 2221 imgu_css_pool_get(&css_pipe->pool.obgrid); 2222 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0); 2223 obgrid = map->vaddr; 2224 2225 /* Configure optical black level grid (obgrid) */ 2226 if (set_params && set_params->use.obgrid_param) 2227 for (i = 0; i < obgrid_size / sizeof(*obgrid); i++) 2228 obgrid[i] = set_params->obgrid_param; 2229 else 2230 memset(obgrid, 0, obgrid_size); 2231 } 2232 2233 /* Configure parameter set info, queued to `queue_id' */ 2234 2235 memset(param_set, 0, sizeof(*param_set)); 2236 map = imgu_css_pool_last(&css_pipe->pool.acc, 0); 2237 param_set->mem_map.acc_cluster_params_for_sp = map->daddr; 2238 2239 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0); 2240 param_set->mem_map.dvs_6axis_params_y = map->daddr; 2241 2242 for (i = 0; i < stripes; i++) { 2243 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0); 2244 param_set->mem_map.obgrid_tbl[i] = 2245 map->daddr + (obgrid_size / stripes) * i; 2246 } 2247 2248 for (m = 0; m < IMGU_ABI_NUM_MEMORIES; m++) { 2249 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0); 2250 param_set->mem_map.isp_mem_param[stage][m] = map->daddr; 2251 } 2252 2253 /* Then queue the new parameter buffer */ 2254 map = imgu_css_pool_last(&css_pipe->pool.parameter_set_info, 0); 2255 r = imgu_css_queue_data(css, queue_id, pipe, map->daddr); 2256 if (r < 0) 2257 goto fail; 2258 2259 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, 2260 IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe, 2261 queue_id)); 2262 if (r < 0) 2263 goto fail_no_put; 2264 2265 /* Finally dequeue all old parameter buffers */ 2266 2267 do { 2268 u32 daddr; 2269 2270 r = imgu_css_dequeue_data(css, queue_id, &daddr); 2271 if (r == -EBUSY) 2272 break; 2273 if (r) 2274 goto fail_no_put; 2275 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, 2276 IMGU_ABI_EVENT_BUFFER_DEQUEUED 2277 (queue_id)); 2278 if (r < 0) { 2279 dev_err(css->dev, "failed to queue parameter event\n"); 2280 goto fail_no_put; 2281 } 2282 } while (1); 2283 2284 return 0; 2285 2286 fail: 2287 /* 2288 * A failure, most likely the parameter queue was full. 2289 * Return error but continue streaming. User can try submitting new 2290 * parameters again later. 2291 */ 2292 2293 imgu_css_pool_put(&css_pipe->pool.parameter_set_info); 2294 if (acc) 2295 imgu_css_pool_put(&css_pipe->pool.acc); 2296 if (gdc) 2297 imgu_css_pool_put(&css_pipe->pool.gdc); 2298 if (obgrid) 2299 imgu_css_pool_put(&css_pipe->pool.obgrid); 2300 if (vmem0) 2301 imgu_css_pool_put( 2302 &css_pipe->pool.binary_params_p 2303 [IMGU_ABI_MEM_ISP_VMEM0]); 2304 if (dmem0) 2305 imgu_css_pool_put( 2306 &css_pipe->pool.binary_params_p 2307 [IMGU_ABI_MEM_ISP_DMEM0]); 2308 2309 fail_no_put: 2310 return r; 2311 } 2312 2313 int imgu_css_irq_ack(struct imgu_css *css) 2314 { 2315 static const int NUM_SWIRQS = 3; 2316 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]]; 2317 void __iomem *const base = css->base; 2318 u32 irq_status[IMGU_IRQCTRL_NUM]; 2319 int i; 2320 2321 u32 imgu_status = readl(base + IMGU_REG_INT_STATUS); 2322 2323 writel(imgu_status, base + IMGU_REG_INT_STATUS); 2324 for (i = 0; i < IMGU_IRQCTRL_NUM; i++) 2325 irq_status[i] = readl(base + IMGU_REG_IRQCTRL_STATUS(i)); 2326 2327 for (i = 0; i < NUM_SWIRQS; i++) { 2328 if (irq_status[IMGU_IRQCTRL_SP0] & IMGU_IRQCTRL_IRQ_SW_PIN(i)) { 2329 /* SP SW interrupt */ 2330 u32 cnt = readl(base + IMGU_REG_SP_DMEM_BASE(0) + 2331 bi->info.sp.output); 2332 u32 val = readl(base + IMGU_REG_SP_DMEM_BASE(0) + 2333 bi->info.sp.output + 4 + 4 * i); 2334 2335 dev_dbg(css->dev, "%s: swirq %i cnt %i val 0x%x\n", 2336 __func__, i, cnt, val); 2337 } 2338 } 2339 2340 for (i = IMGU_IRQCTRL_NUM - 1; i >= 0; i--) 2341 if (irq_status[i]) { 2342 writel(irq_status[i], base + IMGU_REG_IRQCTRL_CLEAR(i)); 2343 /* Wait for write to complete */ 2344 readl(base + IMGU_REG_IRQCTRL_ENABLE(i)); 2345 } 2346 2347 dev_dbg(css->dev, "%s: imgu 0x%x main 0x%x sp0 0x%x sp1 0x%x\n", 2348 __func__, imgu_status, irq_status[IMGU_IRQCTRL_MAIN], 2349 irq_status[IMGU_IRQCTRL_SP0], irq_status[IMGU_IRQCTRL_SP1]); 2350 2351 if (!imgu_status && !irq_status[IMGU_IRQCTRL_MAIN]) 2352 return -ENOMSG; 2353 2354 return 0; 2355 } 2356