1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (C) 2018 Intel Corporation 3 4 #include <linux/device.h> 5 #include <linux/iopoll.h> 6 #include <linux/slab.h> 7 8 #include "ipu3-css.h" 9 #include "ipu3-css-fw.h" 10 #include "ipu3-css-params.h" 11 #include "ipu3-dmamap.h" 12 #include "ipu3-tables.h" 13 14 /* IRQ configuration */ 15 #define IMGU_IRQCTRL_IRQ_MASK (IMGU_IRQCTRL_IRQ_SP1 | \ 16 IMGU_IRQCTRL_IRQ_SP2 | \ 17 IMGU_IRQCTRL_IRQ_SW_PIN(0) | \ 18 IMGU_IRQCTRL_IRQ_SW_PIN(1)) 19 20 #define IPU3_CSS_FORMAT_BPP_DEN 50 /* Denominator */ 21 22 /* Some sane limits for resolutions */ 23 #define IPU3_CSS_MIN_RES 32 24 #define IPU3_CSS_MAX_H 3136 25 #define IPU3_CSS_MAX_W 4224 26 27 /* filter size from graph settings is fixed as 4 */ 28 #define FILTER_SIZE 4 29 #define MIN_ENVELOPE 8 30 31 /* 32 * pre-allocated buffer size for CSS ABI, auxiliary frames 33 * after BDS and before GDC. Those values should be tuned 34 * to big enough to avoid buffer re-allocation when 35 * streaming to lower streaming latency. 36 */ 37 #define CSS_ABI_SIZE 136 38 #define CSS_BDS_SIZE (4480 * 3200 * 3) 39 #define CSS_GDC_SIZE (4224 * 3200 * 12 / 8) 40 41 #define IPU3_CSS_QUEUE_TO_FLAGS(q) (1 << (q)) 42 #define IPU3_CSS_FORMAT_FL_IN \ 43 IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_IN) 44 #define IPU3_CSS_FORMAT_FL_OUT \ 45 IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_OUT) 46 #define IPU3_CSS_FORMAT_FL_VF \ 47 IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_VF) 48 49 /* Formats supported by IPU3 Camera Sub System */ 50 static const struct imgu_css_format imgu_css_formats[] = { 51 { 52 .pixelformat = V4L2_PIX_FMT_NV12, 53 .colorspace = V4L2_COLORSPACE_SRGB, 54 .frame_format = IMGU_ABI_FRAME_FORMAT_NV12, 55 .osys_format = IMGU_ABI_OSYS_FORMAT_NV12, 56 .osys_tiling = IMGU_ABI_OSYS_TILING_NONE, 57 .bytesperpixel_num = 1 * IPU3_CSS_FORMAT_BPP_DEN, 58 .chroma_decim = 4, 59 .width_align = IPU3_UAPI_ISP_VEC_ELEMS, 60 .flags = IPU3_CSS_FORMAT_FL_OUT | IPU3_CSS_FORMAT_FL_VF, 61 }, { 62 /* Each 32 bytes contains 25 10-bit pixels */ 63 .pixelformat = V4L2_PIX_FMT_IPU3_SBGGR10, 64 .colorspace = V4L2_COLORSPACE_RAW, 65 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED, 66 .bayer_order = IMGU_ABI_BAYER_ORDER_BGGR, 67 .bit_depth = 10, 68 .bytesperpixel_num = 64, 69 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS, 70 .flags = IPU3_CSS_FORMAT_FL_IN, 71 }, { 72 .pixelformat = V4L2_PIX_FMT_IPU3_SGBRG10, 73 .colorspace = V4L2_COLORSPACE_RAW, 74 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED, 75 .bayer_order = IMGU_ABI_BAYER_ORDER_GBRG, 76 .bit_depth = 10, 77 .bytesperpixel_num = 64, 78 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS, 79 .flags = IPU3_CSS_FORMAT_FL_IN, 80 }, { 81 .pixelformat = V4L2_PIX_FMT_IPU3_SGRBG10, 82 .colorspace = V4L2_COLORSPACE_RAW, 83 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED, 84 .bayer_order = IMGU_ABI_BAYER_ORDER_GRBG, 85 .bit_depth = 10, 86 .bytesperpixel_num = 64, 87 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS, 88 .flags = IPU3_CSS_FORMAT_FL_IN, 89 }, { 90 .pixelformat = V4L2_PIX_FMT_IPU3_SRGGB10, 91 .colorspace = V4L2_COLORSPACE_RAW, 92 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED, 93 .bayer_order = IMGU_ABI_BAYER_ORDER_RGGB, 94 .bit_depth = 10, 95 .bytesperpixel_num = 64, 96 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS, 97 .flags = IPU3_CSS_FORMAT_FL_IN, 98 }, 99 }; 100 101 static const struct { 102 enum imgu_abi_queue_id qid; 103 size_t ptr_ofs; 104 } imgu_css_queues[IPU3_CSS_QUEUES] = { 105 [IPU3_CSS_QUEUE_IN] = { 106 IMGU_ABI_QUEUE_C_ID, 107 offsetof(struct imgu_abi_buffer, payload.frame.frame_data) 108 }, 109 [IPU3_CSS_QUEUE_OUT] = { 110 IMGU_ABI_QUEUE_D_ID, 111 offsetof(struct imgu_abi_buffer, payload.frame.frame_data) 112 }, 113 [IPU3_CSS_QUEUE_VF] = { 114 IMGU_ABI_QUEUE_E_ID, 115 offsetof(struct imgu_abi_buffer, payload.frame.frame_data) 116 }, 117 [IPU3_CSS_QUEUE_STAT_3A] = { 118 IMGU_ABI_QUEUE_F_ID, 119 offsetof(struct imgu_abi_buffer, payload.s3a.data_ptr) 120 }, 121 }; 122 123 /* Initialize queue based on given format, adjust format as needed */ 124 static int imgu_css_queue_init(struct imgu_css_queue *queue, 125 struct v4l2_pix_format_mplane *fmt, u32 flags) 126 { 127 struct v4l2_pix_format_mplane *const f = &queue->fmt.mpix; 128 unsigned int i; 129 u32 sizeimage; 130 131 INIT_LIST_HEAD(&queue->bufs); 132 133 queue->css_fmt = NULL; /* Disable */ 134 if (!fmt) 135 return 0; 136 137 for (i = 0; i < ARRAY_SIZE(imgu_css_formats); i++) { 138 if (!(imgu_css_formats[i].flags & flags)) 139 continue; 140 queue->css_fmt = &imgu_css_formats[i]; 141 if (imgu_css_formats[i].pixelformat == fmt->pixelformat) 142 break; 143 } 144 if (!queue->css_fmt) 145 return -EINVAL; /* Could not find any suitable format */ 146 147 queue->fmt.mpix = *fmt; 148 149 f->width = ALIGN(clamp_t(u32, f->width, 150 IPU3_CSS_MIN_RES, IPU3_CSS_MAX_W), 2); 151 f->height = ALIGN(clamp_t(u32, f->height, 152 IPU3_CSS_MIN_RES, IPU3_CSS_MAX_H), 2); 153 queue->width_pad = ALIGN(f->width, queue->css_fmt->width_align); 154 if (queue->css_fmt->frame_format != IMGU_ABI_FRAME_FORMAT_RAW_PACKED) 155 f->plane_fmt[0].bytesperline = DIV_ROUND_UP(queue->width_pad * 156 queue->css_fmt->bytesperpixel_num, 157 IPU3_CSS_FORMAT_BPP_DEN); 158 else 159 /* For packed raw, alignment for bpl is by 50 to the width */ 160 f->plane_fmt[0].bytesperline = 161 DIV_ROUND_UP(f->width, 162 IPU3_CSS_FORMAT_BPP_DEN) * 163 queue->css_fmt->bytesperpixel_num; 164 165 sizeimage = f->height * f->plane_fmt[0].bytesperline; 166 if (queue->css_fmt->chroma_decim) 167 sizeimage += 2 * sizeimage / queue->css_fmt->chroma_decim; 168 169 f->plane_fmt[0].sizeimage = sizeimage; 170 f->field = V4L2_FIELD_NONE; 171 f->num_planes = 1; 172 f->colorspace = queue->css_fmt->colorspace; 173 f->flags = 0; 174 f->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT; 175 f->quantization = V4L2_QUANTIZATION_DEFAULT; 176 f->xfer_func = V4L2_XFER_FUNC_DEFAULT; 177 memset(f->reserved, 0, sizeof(f->reserved)); 178 179 return 0; 180 } 181 182 static bool imgu_css_queue_enabled(struct imgu_css_queue *q) 183 { 184 return q->css_fmt; 185 } 186 187 /******************* css hw *******************/ 188 189 /* In the style of writesl() defined in include/asm-generic/io.h */ 190 static inline void writes(const void *mem, ssize_t count, void __iomem *addr) 191 { 192 if (count >= 4) { 193 const u32 *buf = mem; 194 195 count /= 4; 196 do { 197 writel(*buf++, addr); 198 addr += 4; 199 } while (--count); 200 } 201 } 202 203 /* Wait until register `reg', masked with `mask', becomes `cmp' */ 204 static int imgu_hw_wait(void __iomem *base, int reg, u32 mask, u32 cmp) 205 { 206 u32 val; 207 208 return readl_poll_timeout(base + reg, val, (val & mask) == cmp, 209 1000, 100 * 1000); 210 } 211 212 /* Initialize the IPU3 CSS hardware and associated h/w blocks */ 213 214 int imgu_css_set_powerup(struct device *dev, void __iomem *base) 215 { 216 static const unsigned int freq = 450; 217 u32 pm_ctrl, state, val; 218 219 dev_dbg(dev, "%s\n", __func__); 220 /* Clear the CSS busy signal */ 221 readl(base + IMGU_REG_GP_BUSY); 222 writel(0, base + IMGU_REG_GP_BUSY); 223 224 /* Wait for idle signal */ 225 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS, 226 IMGU_STATE_IDLE_STS)) { 227 dev_err(dev, "failed to set CSS idle\n"); 228 goto fail; 229 } 230 231 /* Reset the css */ 232 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET, 233 base + IMGU_REG_PM_CTRL); 234 235 usleep_range(200, 300); 236 237 /** Prepare CSS */ 238 239 pm_ctrl = readl(base + IMGU_REG_PM_CTRL); 240 state = readl(base + IMGU_REG_STATE); 241 242 dev_dbg(dev, "CSS pm_ctrl 0x%x state 0x%x (power %s)\n", 243 pm_ctrl, state, state & IMGU_STATE_POWER_DOWN ? "down" : "up"); 244 245 /* Power up CSS using wrapper */ 246 if (state & IMGU_STATE_POWER_DOWN) { 247 writel(IMGU_PM_CTRL_RACE_TO_HALT | IMGU_PM_CTRL_START, 248 base + IMGU_REG_PM_CTRL); 249 if (imgu_hw_wait(base, IMGU_REG_PM_CTRL, 250 IMGU_PM_CTRL_START, 0)) { 251 dev_err(dev, "failed to power up CSS\n"); 252 goto fail; 253 } 254 usleep_range(2000, 3000); 255 } else { 256 writel(IMGU_PM_CTRL_RACE_TO_HALT, base + IMGU_REG_PM_CTRL); 257 } 258 259 /* Set the busy bit */ 260 writel(readl(base + IMGU_REG_GP_BUSY) | 1, base + IMGU_REG_GP_BUSY); 261 262 /* Set CSS clock frequency */ 263 pm_ctrl = readl(base + IMGU_REG_PM_CTRL); 264 val = pm_ctrl & ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF); 265 writel(val, base + IMGU_REG_PM_CTRL); 266 writel(0, base + IMGU_REG_GP_BUSY); 267 if (imgu_hw_wait(base, IMGU_REG_STATE, 268 IMGU_STATE_PWRDNM_FSM_MASK, 0)) { 269 dev_err(dev, "failed to pwrdn CSS\n"); 270 goto fail; 271 } 272 val = (freq / IMGU_SYSTEM_REQ_FREQ_DIVIDER) & IMGU_SYSTEM_REQ_FREQ_MASK; 273 writel(val, base + IMGU_REG_SYSTEM_REQ); 274 writel(1, base + IMGU_REG_GP_BUSY); 275 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_HALT, 276 base + IMGU_REG_PM_CTRL); 277 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS, 278 IMGU_STATE_HALT_STS)) { 279 dev_err(dev, "failed to halt CSS\n"); 280 goto fail; 281 } 282 283 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_START, 284 base + IMGU_REG_PM_CTRL); 285 if (imgu_hw_wait(base, IMGU_REG_PM_CTRL, IMGU_PM_CTRL_START, 0)) { 286 dev_err(dev, "failed to start CSS\n"); 287 goto fail; 288 } 289 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_UNHALT, 290 base + IMGU_REG_PM_CTRL); 291 292 val = readl(base + IMGU_REG_PM_CTRL); /* get pm_ctrl */ 293 val &= ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF); 294 val |= pm_ctrl & (IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF); 295 writel(val, base + IMGU_REG_PM_CTRL); 296 297 return 0; 298 299 fail: 300 imgu_css_set_powerdown(dev, base); 301 return -EIO; 302 } 303 304 void imgu_css_set_powerdown(struct device *dev, void __iomem *base) 305 { 306 dev_dbg(dev, "%s\n", __func__); 307 /* wait for cio idle signal */ 308 if (imgu_hw_wait(base, IMGU_REG_CIO_GATE_BURST_STATE, 309 IMGU_CIO_GATE_BURST_MASK, 0)) 310 dev_warn(dev, "wait cio gate idle timeout"); 311 312 /* wait for css idle signal */ 313 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS, 314 IMGU_STATE_IDLE_STS)) 315 dev_warn(dev, "wait css idle timeout\n"); 316 317 /* do halt-halted handshake with css */ 318 writel(1, base + IMGU_REG_GP_HALT); 319 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS, 320 IMGU_STATE_HALT_STS)) 321 dev_warn(dev, "failed to halt css"); 322 323 /* de-assert the busy bit */ 324 writel(0, base + IMGU_REG_GP_BUSY); 325 } 326 327 static void imgu_css_hw_enable_irq(struct imgu_css *css) 328 { 329 void __iomem *const base = css->base; 330 u32 val, i; 331 332 /* Set up interrupts */ 333 334 /* 335 * Enable IRQ on the SP which signals that SP goes to idle 336 * (aka ready state) and set trigger to pulse 337 */ 338 val = readl(base + IMGU_REG_SP_CTRL(0)) | IMGU_CTRL_IRQ_READY; 339 writel(val, base + IMGU_REG_SP_CTRL(0)); 340 writel(val | IMGU_CTRL_IRQ_CLEAR, base + IMGU_REG_SP_CTRL(0)); 341 342 /* Enable IRQs from the IMGU wrapper */ 343 writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_ENABLE); 344 /* Clear */ 345 writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_STATUS); 346 347 /* Enable IRQs from main IRQ controller */ 348 writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(IMGU_IRQCTRL_MAIN)); 349 writel(0, base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN)); 350 writel(IMGU_IRQCTRL_IRQ_MASK, 351 base + IMGU_REG_IRQCTRL_EDGE(IMGU_IRQCTRL_MAIN)); 352 writel(IMGU_IRQCTRL_IRQ_MASK, 353 base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN)); 354 writel(IMGU_IRQCTRL_IRQ_MASK, 355 base + IMGU_REG_IRQCTRL_CLEAR(IMGU_IRQCTRL_MAIN)); 356 writel(IMGU_IRQCTRL_IRQ_MASK, 357 base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN)); 358 /* Wait for write complete */ 359 readl(base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN)); 360 361 /* Enable IRQs from SP0 and SP1 controllers */ 362 for (i = IMGU_IRQCTRL_SP0; i <= IMGU_IRQCTRL_SP1; i++) { 363 writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(i)); 364 writel(0, base + IMGU_REG_IRQCTRL_MASK(i)); 365 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_EDGE(i)); 366 writel(IMGU_IRQCTRL_IRQ_MASK, 367 base + IMGU_REG_IRQCTRL_ENABLE(i)); 368 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_CLEAR(i)); 369 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_MASK(i)); 370 /* Wait for write complete */ 371 readl(base + IMGU_REG_IRQCTRL_ENABLE(i)); 372 } 373 } 374 375 static int imgu_css_hw_init(struct imgu_css *css) 376 { 377 /* For checking that streaming monitor statuses are valid */ 378 static const struct { 379 u32 reg; 380 u32 mask; 381 const char *name; 382 } stream_monitors[] = { 383 { 384 IMGU_REG_GP_SP1_STRMON_STAT, 385 IMGU_GP_STRMON_STAT_ISP_PORT_SP12ISP, 386 "ISP0 to SP0" 387 }, { 388 IMGU_REG_GP_ISP_STRMON_STAT, 389 IMGU_GP_STRMON_STAT_SP1_PORT_ISP2SP1, 390 "SP0 to ISP0" 391 }, { 392 IMGU_REG_GP_MOD_STRMON_STAT, 393 IMGU_GP_STRMON_STAT_MOD_PORT_ISP2DMA, 394 "ISP0 to DMA0" 395 }, { 396 IMGU_REG_GP_ISP_STRMON_STAT, 397 IMGU_GP_STRMON_STAT_ISP_PORT_DMA2ISP, 398 "DMA0 to ISP0" 399 }, { 400 IMGU_REG_GP_MOD_STRMON_STAT, 401 IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC, 402 "ISP0 to GDC0" 403 }, { 404 IMGU_REG_GP_MOD_STRMON_STAT, 405 IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS, 406 "GDC0 to ISP0" 407 }, { 408 IMGU_REG_GP_MOD_STRMON_STAT, 409 IMGU_GP_STRMON_STAT_MOD_PORT_SP12DMA, 410 "SP0 to DMA0" 411 }, { 412 IMGU_REG_GP_SP1_STRMON_STAT, 413 IMGU_GP_STRMON_STAT_SP1_PORT_DMA2SP1, 414 "DMA0 to SP0" 415 }, { 416 IMGU_REG_GP_MOD_STRMON_STAT, 417 IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC, 418 "SP0 to GDC0" 419 }, { 420 IMGU_REG_GP_MOD_STRMON_STAT, 421 IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS, 422 "GDC0 to SP0" 423 }, 424 }; 425 426 struct device *dev = css->dev; 427 void __iomem *const base = css->base; 428 u32 val, i; 429 430 /* Set instruction cache address and inv bit for ISP, SP, and SP1 */ 431 for (i = 0; i < IMGU_NUM_SP; i++) { 432 struct imgu_fw_info *bi = 433 &css->fwp->binary_header[css->fw_sp[i]]; 434 435 writel(css->binary[css->fw_sp[i]].daddr, 436 base + IMGU_REG_SP_ICACHE_ADDR(bi->type)); 437 writel(readl(base + IMGU_REG_SP_CTRL(bi->type)) | 438 IMGU_CTRL_ICACHE_INV, 439 base + IMGU_REG_SP_CTRL(bi->type)); 440 } 441 writel(css->binary[css->fw_bl].daddr, base + IMGU_REG_ISP_ICACHE_ADDR); 442 writel(readl(base + IMGU_REG_ISP_CTRL) | IMGU_CTRL_ICACHE_INV, 443 base + IMGU_REG_ISP_CTRL); 444 445 /* Check that IMGU hardware is ready */ 446 447 if (!(readl(base + IMGU_REG_SP_CTRL(0)) & IMGU_CTRL_IDLE)) { 448 dev_err(dev, "SP is not idle\n"); 449 return -EIO; 450 } 451 if (!(readl(base + IMGU_REG_ISP_CTRL) & IMGU_CTRL_IDLE)) { 452 dev_err(dev, "ISP is not idle\n"); 453 return -EIO; 454 } 455 456 for (i = 0; i < ARRAY_SIZE(stream_monitors); i++) { 457 val = readl(base + stream_monitors[i].reg); 458 if (val & stream_monitors[i].mask) { 459 dev_err(dev, "error: Stream monitor %s is valid\n", 460 stream_monitors[i].name); 461 return -EIO; 462 } 463 } 464 465 /* Initialize GDC with default values */ 466 467 for (i = 0; i < ARRAY_SIZE(imgu_css_gdc_lut[0]); i++) { 468 u32 val0 = imgu_css_gdc_lut[0][i] & IMGU_GDC_LUT_MASK; 469 u32 val1 = imgu_css_gdc_lut[1][i] & IMGU_GDC_LUT_MASK; 470 u32 val2 = imgu_css_gdc_lut[2][i] & IMGU_GDC_LUT_MASK; 471 u32 val3 = imgu_css_gdc_lut[3][i] & IMGU_GDC_LUT_MASK; 472 473 writel(val0 | (val1 << 16), 474 base + IMGU_REG_GDC_LUT_BASE + i * 8); 475 writel(val2 | (val3 << 16), 476 base + IMGU_REG_GDC_LUT_BASE + i * 8 + 4); 477 } 478 479 return 0; 480 } 481 482 /* Boot the given IPU3 CSS SP */ 483 static int imgu_css_hw_start_sp(struct imgu_css *css, int sp) 484 { 485 void __iomem *const base = css->base; 486 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]]; 487 struct imgu_abi_sp_init_dmem_cfg dmem_cfg = { 488 .ddr_data_addr = css->binary[css->fw_sp[sp]].daddr 489 + bi->blob.data_source, 490 .dmem_data_addr = bi->blob.data_target, 491 .dmem_bss_addr = bi->blob.bss_target, 492 .data_size = bi->blob.data_size, 493 .bss_size = bi->blob.bss_size, 494 .sp_id = sp, 495 }; 496 497 writes(&dmem_cfg, sizeof(dmem_cfg), base + 498 IMGU_REG_SP_DMEM_BASE(sp) + bi->info.sp.init_dmem_data); 499 500 writel(bi->info.sp.sp_entry, base + IMGU_REG_SP_START_ADDR(sp)); 501 502 writel(readl(base + IMGU_REG_SP_CTRL(sp)) 503 | IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_SP_CTRL(sp)); 504 505 if (imgu_hw_wait(css->base, IMGU_REG_SP_DMEM_BASE(sp) 506 + bi->info.sp.sw_state, 507 ~0, IMGU_ABI_SP_SWSTATE_INITIALIZED)) 508 return -EIO; 509 510 return 0; 511 } 512 513 /* Start the IPU3 CSS ImgU (Imaging Unit) and all the SPs */ 514 static int imgu_css_hw_start(struct imgu_css *css) 515 { 516 static const u32 event_mask = 517 ((1 << IMGU_ABI_EVTTYPE_OUT_FRAME_DONE) | 518 (1 << IMGU_ABI_EVTTYPE_2ND_OUT_FRAME_DONE) | 519 (1 << IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE) | 520 (1 << IMGU_ABI_EVTTYPE_2ND_VF_OUT_FRAME_DONE) | 521 (1 << IMGU_ABI_EVTTYPE_3A_STATS_DONE) | 522 (1 << IMGU_ABI_EVTTYPE_DIS_STATS_DONE) | 523 (1 << IMGU_ABI_EVTTYPE_PIPELINE_DONE) | 524 (1 << IMGU_ABI_EVTTYPE_FRAME_TAGGED) | 525 (1 << IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE) | 526 (1 << IMGU_ABI_EVTTYPE_METADATA_DONE) | 527 (1 << IMGU_ABI_EVTTYPE_ACC_STAGE_COMPLETE)) 528 << IMGU_ABI_SP_COMM_EVENT_IRQ_MASK_OR_SHIFT; 529 530 void __iomem *const base = css->base; 531 struct imgu_fw_info *bi, *bl = &css->fwp->binary_header[css->fw_bl]; 532 unsigned int i; 533 534 writel(IMGU_TLB_INVALIDATE, base + IMGU_REG_TLB_INVALIDATE); 535 536 /* Start bootloader */ 537 538 writel(IMGU_ABI_BL_SWSTATE_BUSY, 539 base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.sw_state); 540 writel(IMGU_NUM_SP, 541 base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.num_dma_cmds); 542 543 for (i = 0; i < IMGU_NUM_SP; i++) { 544 int j = IMGU_NUM_SP - i - 1; /* load sp1 first, then sp0 */ 545 struct imgu_fw_info *sp = 546 &css->fwp->binary_header[css->fw_sp[j]]; 547 struct imgu_abi_bl_dma_cmd_entry dma_cmd = { 548 .src_addr = css->binary[css->fw_sp[j]].daddr 549 + sp->blob.text_source, 550 .size = sp->blob.text_size, 551 .dst_type = IMGU_ABI_BL_DMACMD_TYPE_SP_PMEM, 552 .dst_addr = IMGU_SP_PMEM_BASE(j), 553 }; 554 555 writes(&dma_cmd, sizeof(dma_cmd), 556 base + IMGU_REG_ISP_DMEM_BASE + i * sizeof(dma_cmd) + 557 bl->info.bl.dma_cmd_list); 558 } 559 560 writel(bl->info.bl.bl_entry, base + IMGU_REG_ISP_START_ADDR); 561 562 writel(readl(base + IMGU_REG_ISP_CTRL) 563 | IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_ISP_CTRL); 564 if (imgu_hw_wait(css->base, IMGU_REG_ISP_DMEM_BASE 565 + bl->info.bl.sw_state, ~0, 566 IMGU_ABI_BL_SWSTATE_OK)) { 567 dev_err(css->dev, "failed to start bootloader\n"); 568 return -EIO; 569 } 570 571 /* Start ISP */ 572 573 memset(css->xmem_sp_group_ptrs.vaddr, 0, 574 sizeof(struct imgu_abi_sp_group)); 575 576 bi = &css->fwp->binary_header[css->fw_sp[0]]; 577 578 writel(css->xmem_sp_group_ptrs.daddr, 579 base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.per_frame_data); 580 581 writel(IMGU_ABI_SP_SWSTATE_TERMINATED, 582 base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state); 583 writel(1, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb); 584 585 if (imgu_css_hw_start_sp(css, 0)) 586 return -EIO; 587 588 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.isp_started); 589 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + 590 bi->info.sp.host_sp_queues_initialized); 591 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sleep_mode); 592 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb); 593 writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(0) 594 + bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND); 595 596 /* Enable all events for all queues */ 597 598 for (i = 0; i < IPU3_CSS_PIPE_ID_NUM; i++) 599 writel(event_mask, base + IMGU_REG_SP_DMEM_BASE(0) 600 + bi->info.sp.host_sp_com 601 + IMGU_ABI_SP_COMM_EVENT_IRQ_MASK(i)); 602 writel(1, base + IMGU_REG_SP_DMEM_BASE(0) + 603 bi->info.sp.host_sp_queues_initialized); 604 605 /* Start SP1 */ 606 607 bi = &css->fwp->binary_header[css->fw_sp[1]]; 608 609 writel(IMGU_ABI_SP_SWSTATE_TERMINATED, 610 base + IMGU_REG_SP_DMEM_BASE(1) + bi->info.sp.sw_state); 611 612 if (imgu_css_hw_start_sp(css, 1)) 613 return -EIO; 614 615 writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(1) 616 + bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND); 617 618 return 0; 619 } 620 621 static void imgu_css_hw_stop(struct imgu_css *css) 622 { 623 void __iomem *const base = css->base; 624 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]]; 625 626 /* Stop fw */ 627 writel(IMGU_ABI_SP_COMM_COMMAND_TERMINATE, 628 base + IMGU_REG_SP_DMEM_BASE(0) + 629 bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND); 630 if (imgu_hw_wait(css->base, IMGU_REG_SP_CTRL(0), 631 IMGU_CTRL_IDLE, IMGU_CTRL_IDLE)) 632 dev_err(css->dev, "wait sp0 idle timeout.\n"); 633 if (readl(base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state) != 634 IMGU_ABI_SP_SWSTATE_TERMINATED) 635 dev_err(css->dev, "sp0 is not terminated.\n"); 636 if (imgu_hw_wait(css->base, IMGU_REG_ISP_CTRL, 637 IMGU_CTRL_IDLE, IMGU_CTRL_IDLE)) 638 dev_err(css->dev, "wait isp idle timeout\n"); 639 } 640 641 static void imgu_css_hw_cleanup(struct imgu_css *css) 642 { 643 void __iomem *const base = css->base; 644 645 /** Reset CSS **/ 646 647 /* Clear the CSS busy signal */ 648 readl(base + IMGU_REG_GP_BUSY); 649 writel(0, base + IMGU_REG_GP_BUSY); 650 651 /* Wait for idle signal */ 652 if (imgu_hw_wait(css->base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS, 653 IMGU_STATE_IDLE_STS)) 654 dev_err(css->dev, "failed to shut down hw cleanly\n"); 655 656 /* Reset the css */ 657 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET, 658 base + IMGU_REG_PM_CTRL); 659 660 usleep_range(200, 300); 661 } 662 663 static void imgu_css_pipeline_cleanup(struct imgu_css *css, unsigned int pipe) 664 { 665 struct imgu_device *imgu = dev_get_drvdata(css->dev); 666 unsigned int i; 667 668 imgu_css_pool_cleanup(imgu, 669 &css->pipes[pipe].pool.parameter_set_info); 670 imgu_css_pool_cleanup(imgu, &css->pipes[pipe].pool.acc); 671 imgu_css_pool_cleanup(imgu, &css->pipes[pipe].pool.gdc); 672 imgu_css_pool_cleanup(imgu, &css->pipes[pipe].pool.obgrid); 673 674 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) 675 imgu_css_pool_cleanup(imgu, 676 &css->pipes[pipe].pool.binary_params_p[i]); 677 } 678 679 /* 680 * This function initializes various stages of the 681 * IPU3 CSS ISP pipeline 682 */ 683 static int imgu_css_pipeline_init(struct imgu_css *css, unsigned int pipe) 684 { 685 static const int BYPC = 2; /* Bytes per component */ 686 static const struct imgu_abi_buffer_sp buffer_sp_init = { 687 .buf_src = {.queue_id = IMGU_ABI_QUEUE_EVENT_ID}, 688 .buf_type = IMGU_ABI_BUFFER_TYPE_INVALID, 689 }; 690 691 struct imgu_abi_isp_iterator_config *cfg_iter; 692 struct imgu_abi_isp_ref_config *cfg_ref; 693 struct imgu_abi_isp_dvs_config *cfg_dvs; 694 struct imgu_abi_isp_tnr3_config *cfg_tnr; 695 struct imgu_abi_isp_ref_dmem_state *cfg_ref_state; 696 struct imgu_abi_isp_tnr3_dmem_state *cfg_tnr_state; 697 698 const int stage = 0; 699 unsigned int i, j; 700 701 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 702 const struct imgu_fw_info *bi = 703 &css->fwp->binary_header[css_pipe->bindex]; 704 const unsigned int stripes = bi->info.isp.sp.iterator.num_stripes; 705 706 struct imgu_fw_config_memory_offsets *cofs = (void *)css->fwp + 707 bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_CONFIG]; 708 struct imgu_fw_state_memory_offsets *sofs = (void *)css->fwp + 709 bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_STATE]; 710 711 struct imgu_abi_isp_stage *isp_stage; 712 struct imgu_abi_sp_stage *sp_stage; 713 struct imgu_abi_sp_group *sp_group; 714 715 const unsigned int bds_width_pad = 716 ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width, 717 2 * IPU3_UAPI_ISP_VEC_ELEMS); 718 719 const enum imgu_abi_memories m0 = IMGU_ABI_MEM_ISP_DMEM0; 720 enum imgu_abi_param_class cfg = IMGU_ABI_PARAM_CLASS_CONFIG; 721 void *vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr; 722 723 struct imgu_device *imgu = dev_get_drvdata(css->dev); 724 725 dev_dbg(css->dev, "%s for pipe %d", __func__, pipe); 726 727 /* Configure iterator */ 728 729 cfg_iter = imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 730 &cofs->dmem.iterator, 731 sizeof(*cfg_iter), vaddr); 732 if (!cfg_iter) 733 goto bad_firmware; 734 735 cfg_iter->input_info.res.width = 736 css_pipe->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.width; 737 cfg_iter->input_info.res.height = 738 css_pipe->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.height; 739 cfg_iter->input_info.padded_width = 740 css_pipe->queue[IPU3_CSS_QUEUE_IN].width_pad; 741 cfg_iter->input_info.format = 742 css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->frame_format; 743 cfg_iter->input_info.raw_bit_depth = 744 css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->bit_depth; 745 cfg_iter->input_info.raw_bayer_order = 746 css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->bayer_order; 747 cfg_iter->input_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; 748 749 cfg_iter->internal_info.res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width; 750 cfg_iter->internal_info.res.height = 751 css_pipe->rect[IPU3_CSS_RECT_BDS].height; 752 cfg_iter->internal_info.padded_width = bds_width_pad; 753 cfg_iter->internal_info.format = 754 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format; 755 cfg_iter->internal_info.raw_bit_depth = 756 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth; 757 cfg_iter->internal_info.raw_bayer_order = 758 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order; 759 cfg_iter->internal_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; 760 761 cfg_iter->output_info.res.width = 762 css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.width; 763 cfg_iter->output_info.res.height = 764 css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.height; 765 cfg_iter->output_info.padded_width = 766 css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad; 767 cfg_iter->output_info.format = 768 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format; 769 cfg_iter->output_info.raw_bit_depth = 770 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth; 771 cfg_iter->output_info.raw_bayer_order = 772 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order; 773 cfg_iter->output_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; 774 775 cfg_iter->vf_info.res.width = 776 css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.width; 777 cfg_iter->vf_info.res.height = 778 css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height; 779 cfg_iter->vf_info.padded_width = 780 css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad; 781 cfg_iter->vf_info.format = 782 css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format; 783 cfg_iter->vf_info.raw_bit_depth = 784 css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->bit_depth; 785 cfg_iter->vf_info.raw_bayer_order = 786 css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->bayer_order; 787 cfg_iter->vf_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; 788 789 cfg_iter->dvs_envelope.width = css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width; 790 cfg_iter->dvs_envelope.height = 791 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height; 792 793 /* Configure reference (delay) frames */ 794 795 cfg_ref = imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 796 &cofs->dmem.ref, 797 sizeof(*cfg_ref), vaddr); 798 if (!cfg_ref) 799 goto bad_firmware; 800 801 cfg_ref->port_b.crop = 0; 802 cfg_ref->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES / BYPC; 803 cfg_ref->port_b.width = 804 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width; 805 cfg_ref->port_b.stride = 806 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline; 807 cfg_ref->width_a_over_b = 808 IPU3_UAPI_ISP_VEC_ELEMS / cfg_ref->port_b.elems; 809 cfg_ref->dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1; 810 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) { 811 cfg_ref->ref_frame_addr_y[i] = 812 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr; 813 cfg_ref->ref_frame_addr_c[i] = 814 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr + 815 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline * 816 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height; 817 } 818 for (; i < IMGU_ABI_FRAMES_REF; i++) { 819 cfg_ref->ref_frame_addr_y[i] = 0; 820 cfg_ref->ref_frame_addr_c[i] = 0; 821 } 822 823 /* Configure DVS (digital video stabilization) */ 824 825 cfg_dvs = imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 826 &cofs->dmem.dvs, sizeof(*cfg_dvs), 827 vaddr); 828 if (!cfg_dvs) 829 goto bad_firmware; 830 831 cfg_dvs->num_horizontal_blocks = 832 ALIGN(DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].width, 833 IMGU_DVS_BLOCK_W), 2); 834 cfg_dvs->num_vertical_blocks = 835 DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].height, 836 IMGU_DVS_BLOCK_H); 837 838 /* Configure TNR (temporal noise reduction) */ 839 840 if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) { 841 cfg_tnr = imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 842 &cofs->dmem.tnr3, 843 sizeof(*cfg_tnr), 844 vaddr); 845 if (!cfg_tnr) 846 goto bad_firmware; 847 848 cfg_tnr->port_b.crop = 0; 849 cfg_tnr->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES; 850 cfg_tnr->port_b.width = 851 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width; 852 cfg_tnr->port_b.stride = 853 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline; 854 cfg_tnr->width_a_over_b = 855 IPU3_UAPI_ISP_VEC_ELEMS / cfg_tnr->port_b.elems; 856 cfg_tnr->frame_height = 857 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height; 858 cfg_tnr->delay_frame = IPU3_CSS_AUX_FRAMES - 1; 859 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 860 cfg_tnr->frame_addr[i] = 861 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR] 862 .mem[i].daddr; 863 for (; i < IMGU_ABI_FRAMES_TNR; i++) 864 cfg_tnr->frame_addr[i] = 0; 865 } 866 867 /* Configure ref dmem state parameters */ 868 869 cfg = IMGU_ABI_PARAM_CLASS_STATE; 870 vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr; 871 872 cfg_ref_state = imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 873 &sofs->dmem.ref, 874 sizeof(*cfg_ref_state), 875 vaddr); 876 if (!cfg_ref_state) 877 goto bad_firmware; 878 879 cfg_ref_state->ref_in_buf_idx = 0; 880 cfg_ref_state->ref_out_buf_idx = 1; 881 882 /* Configure tnr dmem state parameters */ 883 if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) { 884 cfg_tnr_state = 885 imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 886 &sofs->dmem.tnr3, 887 sizeof(*cfg_tnr_state), 888 vaddr); 889 if (!cfg_tnr_state) 890 goto bad_firmware; 891 892 cfg_tnr_state->in_bufidx = 0; 893 cfg_tnr_state->out_bufidx = 1; 894 cfg_tnr_state->bypass_filter = 0; 895 cfg_tnr_state->total_frame_counter = 0; 896 for (i = 0; i < IMGU_ABI_BUF_SETS_TNR; i++) 897 cfg_tnr_state->buffer_frame_counter[i] = 0; 898 } 899 900 /* Configure ISP stage */ 901 902 isp_stage = css_pipe->xmem_isp_stage_ptrs[pipe][stage].vaddr; 903 memset(isp_stage, 0, sizeof(*isp_stage)); 904 isp_stage->blob_info = bi->blob; 905 isp_stage->binary_info = bi->info.isp.sp; 906 strscpy(isp_stage->binary_name, 907 (char *)css->fwp + bi->blob.prog_name_offset, 908 sizeof(isp_stage->binary_name)); 909 isp_stage->mem_initializers = bi->info.isp.sp.mem_initializers; 910 for (i = IMGU_ABI_PARAM_CLASS_CONFIG; i < IMGU_ABI_PARAM_CLASS_NUM; i++) 911 for (j = 0; j < IMGU_ABI_NUM_MEMORIES; j++) 912 isp_stage->mem_initializers.params[i][j].address = 913 css_pipe->binary_params_cs[i - 1][j].daddr; 914 915 /* Configure SP stage */ 916 917 sp_stage = css_pipe->xmem_sp_stage_ptrs[pipe][stage].vaddr; 918 memset(sp_stage, 0, sizeof(*sp_stage)); 919 920 sp_stage->frames.in.buf_attr = buffer_sp_init; 921 for (i = 0; i < IMGU_ABI_BINARY_MAX_OUTPUT_PORTS; i++) 922 sp_stage->frames.out[i].buf_attr = buffer_sp_init; 923 sp_stage->frames.out_vf.buf_attr = buffer_sp_init; 924 sp_stage->frames.s3a_buf = buffer_sp_init; 925 sp_stage->frames.dvs_buf = buffer_sp_init; 926 927 sp_stage->stage_type = IMGU_ABI_STAGE_TYPE_ISP; 928 sp_stage->num = stage; 929 sp_stage->isp_online = 0; 930 sp_stage->isp_copy_vf = 0; 931 sp_stage->isp_copy_output = 0; 932 933 sp_stage->enable.vf_output = css_pipe->vf_output_en; 934 935 sp_stage->frames.effective_in_res.width = 936 css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].width; 937 sp_stage->frames.effective_in_res.height = 938 css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].height; 939 sp_stage->frames.in.info.res.width = 940 css_pipe->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.width; 941 sp_stage->frames.in.info.res.height = 942 css_pipe->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.height; 943 sp_stage->frames.in.info.padded_width = 944 css_pipe->queue[IPU3_CSS_QUEUE_IN].width_pad; 945 sp_stage->frames.in.info.format = 946 css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->frame_format; 947 sp_stage->frames.in.info.raw_bit_depth = 948 css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->bit_depth; 949 sp_stage->frames.in.info.raw_bayer_order = 950 css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->bayer_order; 951 sp_stage->frames.in.info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; 952 sp_stage->frames.in.buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_C_ID; 953 sp_stage->frames.in.buf_attr.buf_type = 954 IMGU_ABI_BUFFER_TYPE_INPUT_FRAME; 955 956 sp_stage->frames.out[0].info.res.width = 957 css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.width; 958 sp_stage->frames.out[0].info.res.height = 959 css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.height; 960 sp_stage->frames.out[0].info.padded_width = 961 css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad; 962 sp_stage->frames.out[0].info.format = 963 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format; 964 sp_stage->frames.out[0].info.raw_bit_depth = 965 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth; 966 sp_stage->frames.out[0].info.raw_bayer_order = 967 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order; 968 sp_stage->frames.out[0].info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; 969 sp_stage->frames.out[0].planes.nv.uv.offset = 970 css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad * 971 css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.height; 972 sp_stage->frames.out[0].buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_D_ID; 973 sp_stage->frames.out[0].buf_attr.buf_type = 974 IMGU_ABI_BUFFER_TYPE_OUTPUT_FRAME; 975 976 sp_stage->frames.out[1].buf_attr.buf_src.queue_id = 977 IMGU_ABI_QUEUE_EVENT_ID; 978 979 sp_stage->frames.internal_frame_info.res.width = 980 css_pipe->rect[IPU3_CSS_RECT_BDS].width; 981 sp_stage->frames.internal_frame_info.res.height = 982 css_pipe->rect[IPU3_CSS_RECT_BDS].height; 983 sp_stage->frames.internal_frame_info.padded_width = bds_width_pad; 984 985 sp_stage->frames.internal_frame_info.format = 986 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format; 987 sp_stage->frames.internal_frame_info.raw_bit_depth = 988 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth; 989 sp_stage->frames.internal_frame_info.raw_bayer_order = 990 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order; 991 sp_stage->frames.internal_frame_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; 992 993 sp_stage->frames.out_vf.info.res.width = 994 css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.width; 995 sp_stage->frames.out_vf.info.res.height = 996 css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height; 997 sp_stage->frames.out_vf.info.padded_width = 998 css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad; 999 sp_stage->frames.out_vf.info.format = 1000 css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format; 1001 sp_stage->frames.out_vf.info.raw_bit_depth = 1002 css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->bit_depth; 1003 sp_stage->frames.out_vf.info.raw_bayer_order = 1004 css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->bayer_order; 1005 sp_stage->frames.out_vf.info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; 1006 sp_stage->frames.out_vf.planes.yuv.u.offset = 1007 css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad * 1008 css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height; 1009 sp_stage->frames.out_vf.planes.yuv.v.offset = 1010 css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad * 1011 css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height * 5 / 4; 1012 sp_stage->frames.out_vf.buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_E_ID; 1013 sp_stage->frames.out_vf.buf_attr.buf_type = 1014 IMGU_ABI_BUFFER_TYPE_VF_OUTPUT_FRAME; 1015 1016 sp_stage->frames.s3a_buf.buf_src.queue_id = IMGU_ABI_QUEUE_F_ID; 1017 sp_stage->frames.s3a_buf.buf_type = IMGU_ABI_BUFFER_TYPE_3A_STATISTICS; 1018 1019 sp_stage->frames.dvs_buf.buf_src.queue_id = IMGU_ABI_QUEUE_G_ID; 1020 sp_stage->frames.dvs_buf.buf_type = IMGU_ABI_BUFFER_TYPE_DIS_STATISTICS; 1021 1022 sp_stage->dvs_envelope.width = css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width; 1023 sp_stage->dvs_envelope.height = 1024 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height; 1025 1026 sp_stage->isp_pipe_version = 1027 bi->info.isp.sp.pipeline.isp_pipe_version; 1028 sp_stage->isp_deci_log_factor = 1029 clamp(max(fls(css_pipe->rect[IPU3_CSS_RECT_BDS].width / 1030 IMGU_MAX_BQ_GRID_WIDTH), 1031 fls(css_pipe->rect[IPU3_CSS_RECT_BDS].height / 1032 IMGU_MAX_BQ_GRID_HEIGHT)) - 1, 3, 5); 1033 sp_stage->isp_vf_downscale_bits = 0; 1034 sp_stage->if_config_index = 255; 1035 sp_stage->sp_enable_xnr = 0; 1036 sp_stage->num_stripes = stripes; 1037 sp_stage->enable.s3a = 1; 1038 sp_stage->enable.dvs_stats = 0; 1039 1040 sp_stage->xmem_bin_addr = css->binary[css_pipe->bindex].daddr; 1041 sp_stage->xmem_map_addr = css_pipe->sp_ddr_ptrs.daddr; 1042 sp_stage->isp_stage_addr = 1043 css_pipe->xmem_isp_stage_ptrs[pipe][stage].daddr; 1044 1045 /* Configure SP group */ 1046 1047 sp_group = css->xmem_sp_group_ptrs.vaddr; 1048 memset(&sp_group->pipe[pipe], 0, sizeof(struct imgu_abi_sp_pipeline)); 1049 1050 sp_group->pipe[pipe].num_stages = 1; 1051 sp_group->pipe[pipe].pipe_id = css_pipe->pipe_id; 1052 sp_group->pipe[pipe].thread_id = pipe; 1053 sp_group->pipe[pipe].pipe_num = pipe; 1054 sp_group->pipe[pipe].num_execs = -1; 1055 sp_group->pipe[pipe].pipe_qos_config = -1; 1056 sp_group->pipe[pipe].required_bds_factor = 0; 1057 sp_group->pipe[pipe].dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1; 1058 sp_group->pipe[pipe].inout_port_config = 1059 IMGU_ABI_PORT_CONFIG_TYPE_INPUT_HOST | 1060 IMGU_ABI_PORT_CONFIG_TYPE_OUTPUT_HOST; 1061 sp_group->pipe[pipe].scaler_pp_lut = 0; 1062 sp_group->pipe[pipe].shading.internal_frame_origin_x_bqs_on_sctbl = 0; 1063 sp_group->pipe[pipe].shading.internal_frame_origin_y_bqs_on_sctbl = 0; 1064 sp_group->pipe[pipe].sp_stage_addr[stage] = 1065 css_pipe->xmem_sp_stage_ptrs[pipe][stage].daddr; 1066 sp_group->pipe[pipe].pipe_config = 1067 bi->info.isp.sp.enable.params ? (1 << pipe) : 0; 1068 sp_group->pipe[pipe].pipe_config |= IMGU_ABI_PIPE_CONFIG_ACQUIRE_ISP; 1069 1070 /* Initialize parameter pools */ 1071 1072 if (imgu_css_pool_init(imgu, &css_pipe->pool.parameter_set_info, 1073 sizeof(struct imgu_abi_parameter_set_info)) || 1074 imgu_css_pool_init(imgu, &css_pipe->pool.acc, 1075 sizeof(struct imgu_abi_acc_param)) || 1076 imgu_css_pool_init(imgu, &css_pipe->pool.gdc, 1077 sizeof(struct imgu_abi_gdc_warp_param) * 1078 3 * cfg_dvs->num_horizontal_blocks / 2 * 1079 cfg_dvs->num_vertical_blocks) || 1080 imgu_css_pool_init(imgu, &css_pipe->pool.obgrid, 1081 imgu_css_fw_obgrid_size( 1082 &css->fwp->binary_header[css_pipe->bindex]))) 1083 goto out_of_memory; 1084 1085 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) 1086 if (imgu_css_pool_init(imgu, 1087 &css_pipe->pool.binary_params_p[i], 1088 bi->info.isp.sp.mem_initializers.params 1089 [IMGU_ABI_PARAM_CLASS_PARAM][i].size)) 1090 goto out_of_memory; 1091 1092 return 0; 1093 1094 bad_firmware: 1095 imgu_css_pipeline_cleanup(css, pipe); 1096 return -EPROTO; 1097 1098 out_of_memory: 1099 imgu_css_pipeline_cleanup(css, pipe); 1100 return -ENOMEM; 1101 } 1102 1103 static u8 imgu_css_queue_pos(struct imgu_css *css, int queue, int thread) 1104 { 1105 static const unsigned int sp; 1106 void __iomem *const base = css->base; 1107 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]]; 1108 struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) + 1109 bi->info.sp.host_sp_queue; 1110 1111 return queue >= 0 ? readb(&q->host2sp_bufq_info[thread][queue].end) : 1112 readb(&q->host2sp_evtq_info.end); 1113 } 1114 1115 /* Sent data to sp using given buffer queue, or if queue < 0, event queue. */ 1116 static int imgu_css_queue_data(struct imgu_css *css, 1117 int queue, int thread, u32 data) 1118 { 1119 static const unsigned int sp; 1120 void __iomem *const base = css->base; 1121 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]]; 1122 struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) + 1123 bi->info.sp.host_sp_queue; 1124 u8 size, start, end, end2; 1125 1126 if (queue >= 0) { 1127 size = readb(&q->host2sp_bufq_info[thread][queue].size); 1128 start = readb(&q->host2sp_bufq_info[thread][queue].start); 1129 end = readb(&q->host2sp_bufq_info[thread][queue].end); 1130 } else { 1131 size = readb(&q->host2sp_evtq_info.size); 1132 start = readb(&q->host2sp_evtq_info.start); 1133 end = readb(&q->host2sp_evtq_info.end); 1134 } 1135 1136 if (size == 0) 1137 return -EIO; 1138 1139 end2 = (end + 1) % size; 1140 if (end2 == start) 1141 return -EBUSY; /* Queue full */ 1142 1143 if (queue >= 0) { 1144 writel(data, &q->host2sp_bufq[thread][queue][end]); 1145 writeb(end2, &q->host2sp_bufq_info[thread][queue].end); 1146 } else { 1147 writel(data, &q->host2sp_evtq[end]); 1148 writeb(end2, &q->host2sp_evtq_info.end); 1149 } 1150 1151 return 0; 1152 } 1153 1154 /* Receive data using given buffer queue, or if queue < 0, event queue. */ 1155 static int imgu_css_dequeue_data(struct imgu_css *css, int queue, u32 *data) 1156 { 1157 static const unsigned int sp; 1158 void __iomem *const base = css->base; 1159 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]]; 1160 struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) + 1161 bi->info.sp.host_sp_queue; 1162 u8 size, start, end, start2; 1163 1164 if (queue >= 0) { 1165 size = readb(&q->sp2host_bufq_info[queue].size); 1166 start = readb(&q->sp2host_bufq_info[queue].start); 1167 end = readb(&q->sp2host_bufq_info[queue].end); 1168 } else { 1169 size = readb(&q->sp2host_evtq_info.size); 1170 start = readb(&q->sp2host_evtq_info.start); 1171 end = readb(&q->sp2host_evtq_info.end); 1172 } 1173 1174 if (size == 0) 1175 return -EIO; 1176 1177 if (end == start) 1178 return -EBUSY; /* Queue empty */ 1179 1180 start2 = (start + 1) % size; 1181 1182 if (queue >= 0) { 1183 *data = readl(&q->sp2host_bufq[queue][start]); 1184 writeb(start2, &q->sp2host_bufq_info[queue].start); 1185 } else { 1186 int r; 1187 1188 *data = readl(&q->sp2host_evtq[start]); 1189 writeb(start2, &q->sp2host_evtq_info.start); 1190 1191 /* Acknowledge events dequeued from event queue */ 1192 r = imgu_css_queue_data(css, queue, 0, 1193 IMGU_ABI_EVENT_EVENT_DEQUEUED); 1194 if (r < 0) 1195 return r; 1196 } 1197 1198 return 0; 1199 } 1200 1201 /* Free binary-specific resources */ 1202 static void imgu_css_binary_cleanup(struct imgu_css *css, unsigned int pipe) 1203 { 1204 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1205 unsigned int i, j; 1206 1207 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1208 1209 for (j = 0; j < IMGU_ABI_PARAM_CLASS_NUM - 1; j++) 1210 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) 1211 imgu_dmamap_free(imgu, 1212 &css_pipe->binary_params_cs[j][i]); 1213 1214 j = IPU3_CSS_AUX_FRAME_REF; 1215 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1216 imgu_dmamap_free(imgu, 1217 &css_pipe->aux_frames[j].mem[i]); 1218 1219 j = IPU3_CSS_AUX_FRAME_TNR; 1220 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1221 imgu_dmamap_free(imgu, 1222 &css_pipe->aux_frames[j].mem[i]); 1223 } 1224 1225 static int imgu_css_binary_preallocate(struct imgu_css *css, unsigned int pipe) 1226 { 1227 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1228 unsigned int i, j; 1229 1230 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1231 1232 for (j = IMGU_ABI_PARAM_CLASS_CONFIG; 1233 j < IMGU_ABI_PARAM_CLASS_NUM; j++) 1234 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) 1235 if (!imgu_dmamap_alloc(imgu, 1236 &css_pipe->binary_params_cs[j - 1][i], 1237 CSS_ABI_SIZE)) 1238 goto out_of_memory; 1239 1240 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1241 if (!imgu_dmamap_alloc(imgu, 1242 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF]. 1243 mem[i], CSS_BDS_SIZE)) 1244 goto out_of_memory; 1245 1246 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1247 if (!imgu_dmamap_alloc(imgu, 1248 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR]. 1249 mem[i], CSS_GDC_SIZE)) 1250 goto out_of_memory; 1251 1252 return 0; 1253 1254 out_of_memory: 1255 imgu_css_binary_cleanup(css, pipe); 1256 return -ENOMEM; 1257 } 1258 1259 /* allocate binary-specific resources */ 1260 static int imgu_css_binary_setup(struct imgu_css *css, unsigned int pipe) 1261 { 1262 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1263 struct imgu_fw_info *bi = &css->fwp->binary_header[css_pipe->bindex]; 1264 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1265 int i, j, size; 1266 static const int BYPC = 2; /* Bytes per component */ 1267 unsigned int w, h; 1268 1269 /* Allocate parameter memory blocks for this binary */ 1270 1271 for (j = IMGU_ABI_PARAM_CLASS_CONFIG; j < IMGU_ABI_PARAM_CLASS_NUM; j++) 1272 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) { 1273 if (imgu_css_dma_buffer_resize( 1274 imgu, 1275 &css_pipe->binary_params_cs[j - 1][i], 1276 bi->info.isp.sp.mem_initializers.params[j][i].size)) 1277 goto out_of_memory; 1278 } 1279 1280 /* Allocate internal frame buffers */ 1281 1282 /* Reference frames for DVS, FRAME_FORMAT_YUV420_16 */ 1283 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel = BYPC; 1284 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width = 1285 css_pipe->rect[IPU3_CSS_RECT_BDS].width; 1286 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height = 1287 ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].height, 1288 IMGU_DVS_BLOCK_H) + 2 * IMGU_GDC_BUF_Y; 1289 h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height; 1290 w = ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width, 1291 2 * IPU3_UAPI_ISP_VEC_ELEMS) + 2 * IMGU_GDC_BUF_X; 1292 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline = 1293 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel * w; 1294 size = w * h * BYPC + (w / 2) * (h / 2) * BYPC * 2; 1295 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1296 if (imgu_css_dma_buffer_resize( 1297 imgu, 1298 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i], 1299 size)) 1300 goto out_of_memory; 1301 1302 /* TNR frames for temporal noise reduction, FRAME_FORMAT_YUV_LINE */ 1303 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperpixel = 1; 1304 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width = 1305 roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].width, 1306 bi->info.isp.sp.block.block_width * 1307 IPU3_UAPI_ISP_VEC_ELEMS); 1308 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height = 1309 roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].height, 1310 bi->info.isp.sp.block.output_block_height); 1311 1312 w = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width; 1313 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline = w; 1314 h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height; 1315 size = w * ALIGN(h * 3 / 2 + 3, 2); /* +3 for vf_pp prefetch */ 1316 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1317 if (imgu_css_dma_buffer_resize( 1318 imgu, 1319 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].mem[i], 1320 size)) 1321 goto out_of_memory; 1322 1323 return 0; 1324 1325 out_of_memory: 1326 imgu_css_binary_cleanup(css, pipe); 1327 return -ENOMEM; 1328 } 1329 1330 int imgu_css_start_streaming(struct imgu_css *css) 1331 { 1332 u32 data; 1333 int r, pipe; 1334 1335 if (css->streaming) 1336 return -EPROTO; 1337 1338 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1339 r = imgu_css_binary_setup(css, pipe); 1340 if (r < 0) 1341 return r; 1342 } 1343 1344 r = imgu_css_hw_init(css); 1345 if (r < 0) 1346 return r; 1347 1348 r = imgu_css_hw_start(css); 1349 if (r < 0) 1350 goto fail; 1351 1352 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1353 r = imgu_css_pipeline_init(css, pipe); 1354 if (r < 0) 1355 goto fail; 1356 } 1357 1358 css->streaming = true; 1359 1360 imgu_css_hw_enable_irq(css); 1361 1362 /* Initialize parameters to default */ 1363 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1364 r = imgu_css_set_parameters(css, pipe, NULL); 1365 if (r < 0) 1366 goto fail; 1367 } 1368 1369 while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_A_ID, &data))) 1370 ; 1371 if (r != -EBUSY) 1372 goto fail; 1373 1374 while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_B_ID, &data))) 1375 ; 1376 if (r != -EBUSY) 1377 goto fail; 1378 1379 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1380 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, 1381 IMGU_ABI_EVENT_START_STREAM | 1382 pipe << 16); 1383 if (r < 0) 1384 goto fail; 1385 } 1386 1387 return 0; 1388 1389 fail: 1390 css->streaming = false; 1391 imgu_css_hw_cleanup(css); 1392 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1393 imgu_css_pipeline_cleanup(css, pipe); 1394 imgu_css_binary_cleanup(css, pipe); 1395 } 1396 1397 return r; 1398 } 1399 1400 void imgu_css_stop_streaming(struct imgu_css *css) 1401 { 1402 struct imgu_css_buffer *b, *b0; 1403 int q, r, pipe; 1404 1405 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1406 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, 1407 IMGU_ABI_EVENT_STOP_STREAM); 1408 if (r < 0) 1409 dev_warn(css->dev, "failed on stop stream event\n"); 1410 } 1411 1412 if (!css->streaming) 1413 return; 1414 1415 imgu_css_hw_stop(css); 1416 1417 imgu_css_hw_cleanup(css); 1418 1419 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1420 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1421 1422 imgu_css_pipeline_cleanup(css, pipe); 1423 1424 spin_lock(&css_pipe->qlock); 1425 for (q = 0; q < IPU3_CSS_QUEUES; q++) 1426 list_for_each_entry_safe(b, b0, 1427 &css_pipe->queue[q].bufs, 1428 list) { 1429 b->state = IPU3_CSS_BUFFER_FAILED; 1430 list_del(&b->list); 1431 } 1432 spin_unlock(&css_pipe->qlock); 1433 } 1434 1435 css->streaming = false; 1436 } 1437 1438 bool imgu_css_pipe_queue_empty(struct imgu_css *css, unsigned int pipe) 1439 { 1440 int q; 1441 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1442 1443 spin_lock(&css_pipe->qlock); 1444 for (q = 0; q < IPU3_CSS_QUEUES; q++) 1445 if (!list_empty(&css_pipe->queue[q].bufs)) 1446 break; 1447 spin_unlock(&css_pipe->qlock); 1448 return (q == IPU3_CSS_QUEUES); 1449 } 1450 1451 bool imgu_css_queue_empty(struct imgu_css *css) 1452 { 1453 unsigned int pipe; 1454 bool ret = 0; 1455 1456 for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++) 1457 ret &= imgu_css_pipe_queue_empty(css, pipe); 1458 1459 return ret; 1460 } 1461 1462 bool imgu_css_is_streaming(struct imgu_css *css) 1463 { 1464 return css->streaming; 1465 } 1466 1467 static int imgu_css_map_init(struct imgu_css *css, unsigned int pipe) 1468 { 1469 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1470 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1471 unsigned int p, q, i; 1472 1473 /* Allocate and map common structures with imgu hardware */ 1474 for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++) 1475 for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) { 1476 if (!imgu_dmamap_alloc(imgu, 1477 &css_pipe-> 1478 xmem_sp_stage_ptrs[p][i], 1479 sizeof(struct imgu_abi_sp_stage))) 1480 return -ENOMEM; 1481 if (!imgu_dmamap_alloc(imgu, 1482 &css_pipe-> 1483 xmem_isp_stage_ptrs[p][i], 1484 sizeof(struct imgu_abi_isp_stage))) 1485 return -ENOMEM; 1486 } 1487 1488 if (!imgu_dmamap_alloc(imgu, &css_pipe->sp_ddr_ptrs, 1489 ALIGN(sizeof(struct imgu_abi_ddr_address_map), 1490 IMGU_ABI_ISP_DDR_WORD_BYTES))) 1491 return -ENOMEM; 1492 1493 for (q = 0; q < IPU3_CSS_QUEUES; q++) { 1494 unsigned int abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]); 1495 1496 for (i = 0; i < abi_buf_num; i++) 1497 if (!imgu_dmamap_alloc(imgu, 1498 &css_pipe->abi_buffers[q][i], 1499 sizeof(struct imgu_abi_buffer))) 1500 return -ENOMEM; 1501 } 1502 1503 if (imgu_css_binary_preallocate(css, pipe)) { 1504 imgu_css_binary_cleanup(css, pipe); 1505 return -ENOMEM; 1506 } 1507 1508 return 0; 1509 } 1510 1511 static void imgu_css_pipe_cleanup(struct imgu_css *css, unsigned int pipe) 1512 { 1513 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1514 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1515 unsigned int p, q, i, abi_buf_num; 1516 1517 imgu_css_binary_cleanup(css, pipe); 1518 1519 for (q = 0; q < IPU3_CSS_QUEUES; q++) { 1520 abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]); 1521 for (i = 0; i < abi_buf_num; i++) 1522 imgu_dmamap_free(imgu, &css_pipe->abi_buffers[q][i]); 1523 } 1524 1525 for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++) 1526 for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) { 1527 imgu_dmamap_free(imgu, 1528 &css_pipe->xmem_sp_stage_ptrs[p][i]); 1529 imgu_dmamap_free(imgu, 1530 &css_pipe->xmem_isp_stage_ptrs[p][i]); 1531 } 1532 1533 imgu_dmamap_free(imgu, &css_pipe->sp_ddr_ptrs); 1534 } 1535 1536 void imgu_css_cleanup(struct imgu_css *css) 1537 { 1538 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1539 unsigned int pipe; 1540 1541 imgu_css_stop_streaming(css); 1542 for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++) 1543 imgu_css_pipe_cleanup(css, pipe); 1544 imgu_dmamap_free(imgu, &css->xmem_sp_group_ptrs); 1545 imgu_css_fw_cleanup(css); 1546 } 1547 1548 int imgu_css_init(struct device *dev, struct imgu_css *css, 1549 void __iomem *base, int length) 1550 { 1551 struct imgu_device *imgu = dev_get_drvdata(dev); 1552 int r, q, pipe; 1553 1554 /* Initialize main data structure */ 1555 css->dev = dev; 1556 css->base = base; 1557 css->iomem_length = length; 1558 1559 for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++) { 1560 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1561 1562 css_pipe->vf_output_en = false; 1563 spin_lock_init(&css_pipe->qlock); 1564 css_pipe->bindex = IPU3_CSS_DEFAULT_BINARY; 1565 css_pipe->pipe_id = IPU3_CSS_PIPE_ID_VIDEO; 1566 for (q = 0; q < IPU3_CSS_QUEUES; q++) { 1567 r = imgu_css_queue_init(&css_pipe->queue[q], NULL, 0); 1568 if (r) 1569 return r; 1570 } 1571 r = imgu_css_map_init(css, pipe); 1572 if (r) { 1573 imgu_css_cleanup(css); 1574 return r; 1575 } 1576 } 1577 if (!imgu_dmamap_alloc(imgu, &css->xmem_sp_group_ptrs, 1578 sizeof(struct imgu_abi_sp_group))) 1579 return -ENOMEM; 1580 1581 r = imgu_css_fw_init(css); 1582 if (r) 1583 return r; 1584 1585 return 0; 1586 } 1587 1588 static u32 imgu_css_adjust(u32 res, u32 align) 1589 { 1590 u32 val = max_t(u32, IPU3_CSS_MIN_RES, res); 1591 1592 return DIV_ROUND_CLOSEST(val, align) * align; 1593 } 1594 1595 /* Select a binary matching the required resolutions and formats */ 1596 static int imgu_css_find_binary(struct imgu_css *css, 1597 unsigned int pipe, 1598 struct imgu_css_queue queue[IPU3_CSS_QUEUES], 1599 struct v4l2_rect rects[IPU3_CSS_RECTS]) 1600 { 1601 const int binary_nr = css->fwp->file_header.binary_nr; 1602 unsigned int binary_mode = 1603 (css->pipes[pipe].pipe_id == IPU3_CSS_PIPE_ID_CAPTURE) ? 1604 IA_CSS_BINARY_MODE_PRIMARY : IA_CSS_BINARY_MODE_VIDEO; 1605 const struct v4l2_pix_format_mplane *in = 1606 &queue[IPU3_CSS_QUEUE_IN].fmt.mpix; 1607 const struct v4l2_pix_format_mplane *out = 1608 &queue[IPU3_CSS_QUEUE_OUT].fmt.mpix; 1609 const struct v4l2_pix_format_mplane *vf = 1610 &queue[IPU3_CSS_QUEUE_VF].fmt.mpix; 1611 u32 stripe_w = 0, stripe_h = 0; 1612 const char *name; 1613 int i, j; 1614 1615 if (!imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_IN])) 1616 return -EINVAL; 1617 1618 /* Find out the strip size boundary */ 1619 for (i = 0; i < binary_nr; i++) { 1620 struct imgu_fw_info *bi = &css->fwp->binary_header[i]; 1621 1622 u32 max_width = bi->info.isp.sp.output.max_width; 1623 u32 max_height = bi->info.isp.sp.output.max_height; 1624 1625 if (bi->info.isp.sp.iterator.num_stripes <= 1) { 1626 stripe_w = stripe_w ? 1627 min(stripe_w, max_width) : max_width; 1628 stripe_h = stripe_h ? 1629 min(stripe_h, max_height) : max_height; 1630 } 1631 } 1632 1633 for (i = 0; i < binary_nr; i++) { 1634 struct imgu_fw_info *bi = &css->fwp->binary_header[i]; 1635 enum imgu_abi_frame_format q_fmt; 1636 1637 name = (void *)css->fwp + bi->blob.prog_name_offset; 1638 1639 /* Check that binary supports memory-to-memory processing */ 1640 if (bi->info.isp.sp.input.source != 1641 IMGU_ABI_BINARY_INPUT_SOURCE_MEMORY) 1642 continue; 1643 1644 /* Check that binary supports raw10 input */ 1645 if (!bi->info.isp.sp.enable.input_feeder && 1646 !bi->info.isp.sp.enable.input_raw) 1647 continue; 1648 1649 /* Check binary mode */ 1650 if (bi->info.isp.sp.pipeline.mode != binary_mode) 1651 continue; 1652 1653 /* Since input is RGGB bayer, need to process colors */ 1654 if (bi->info.isp.sp.enable.luma_only) 1655 continue; 1656 1657 if (in->width < bi->info.isp.sp.input.min_width || 1658 in->width > bi->info.isp.sp.input.max_width || 1659 in->height < bi->info.isp.sp.input.min_height || 1660 in->height > bi->info.isp.sp.input.max_height) 1661 continue; 1662 1663 if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_OUT])) { 1664 if (bi->info.isp.num_output_pins <= 0) 1665 continue; 1666 1667 q_fmt = queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format; 1668 for (j = 0; j < bi->info.isp.num_output_formats; j++) 1669 if (bi->info.isp.output_formats[j] == q_fmt) 1670 break; 1671 if (j >= bi->info.isp.num_output_formats) 1672 continue; 1673 1674 if (out->width < bi->info.isp.sp.output.min_width || 1675 out->width > bi->info.isp.sp.output.max_width || 1676 out->height < bi->info.isp.sp.output.min_height || 1677 out->height > bi->info.isp.sp.output.max_height) 1678 continue; 1679 1680 if (out->width > bi->info.isp.sp.internal.max_width || 1681 out->height > bi->info.isp.sp.internal.max_height) 1682 continue; 1683 } 1684 1685 if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_VF])) { 1686 if (bi->info.isp.num_output_pins <= 1) 1687 continue; 1688 1689 q_fmt = queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format; 1690 for (j = 0; j < bi->info.isp.num_output_formats; j++) 1691 if (bi->info.isp.output_formats[j] == q_fmt) 1692 break; 1693 if (j >= bi->info.isp.num_output_formats) 1694 continue; 1695 1696 if (vf->width < bi->info.isp.sp.output.min_width || 1697 vf->width > bi->info.isp.sp.output.max_width || 1698 vf->height < bi->info.isp.sp.output.min_height || 1699 vf->height > bi->info.isp.sp.output.max_height) 1700 continue; 1701 } 1702 1703 /* All checks passed, select the binary */ 1704 dev_dbg(css->dev, "using binary %s id = %u\n", name, 1705 bi->info.isp.sp.id); 1706 return i; 1707 } 1708 1709 /* Can not find suitable binary for these parameters */ 1710 return -EINVAL; 1711 } 1712 1713 /* 1714 * Check that there is a binary matching requirements. Parameters may be 1715 * NULL indicating disabled input/output. Return negative if given 1716 * parameters can not be supported or on error, zero or positive indicating 1717 * found binary number. May modify the given parameters if not exact match 1718 * is found. 1719 */ 1720 int imgu_css_fmt_try(struct imgu_css *css, 1721 struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES], 1722 struct v4l2_rect *rects[IPU3_CSS_RECTS], 1723 unsigned int pipe) 1724 { 1725 static const u32 EFF_ALIGN_W = 2; 1726 static const u32 BDS_ALIGN_W = 4; 1727 static const u32 OUT_ALIGN_W = 8; 1728 static const u32 OUT_ALIGN_H = 4; 1729 static const u32 VF_ALIGN_W = 2; 1730 static const char *qnames[IPU3_CSS_QUEUES] = { 1731 [IPU3_CSS_QUEUE_IN] = "in", 1732 [IPU3_CSS_QUEUE_PARAMS] = "params", 1733 [IPU3_CSS_QUEUE_OUT] = "out", 1734 [IPU3_CSS_QUEUE_VF] = "vf", 1735 [IPU3_CSS_QUEUE_STAT_3A] = "3a", 1736 }; 1737 static const char *rnames[IPU3_CSS_RECTS] = { 1738 [IPU3_CSS_RECT_EFFECTIVE] = "effective resolution", 1739 [IPU3_CSS_RECT_BDS] = "bayer-domain scaled resolution", 1740 [IPU3_CSS_RECT_ENVELOPE] = "DVS envelope size", 1741 [IPU3_CSS_RECT_GDC] = "GDC output res", 1742 }; 1743 struct v4l2_rect r[IPU3_CSS_RECTS] = { }; 1744 struct v4l2_rect *const eff = &r[IPU3_CSS_RECT_EFFECTIVE]; 1745 struct v4l2_rect *const bds = &r[IPU3_CSS_RECT_BDS]; 1746 struct v4l2_rect *const env = &r[IPU3_CSS_RECT_ENVELOPE]; 1747 struct v4l2_rect *const gdc = &r[IPU3_CSS_RECT_GDC]; 1748 struct imgu_css_queue *q; 1749 struct v4l2_pix_format_mplane *in, *out, *vf; 1750 int i, s, ret; 1751 1752 q = kcalloc(IPU3_CSS_QUEUES, sizeof(struct imgu_css_queue), GFP_KERNEL); 1753 if (!q) 1754 return -ENOMEM; 1755 1756 in = &q[IPU3_CSS_QUEUE_IN].fmt.mpix; 1757 out = &q[IPU3_CSS_QUEUE_OUT].fmt.mpix; 1758 vf = &q[IPU3_CSS_QUEUE_VF].fmt.mpix; 1759 1760 /* Adjust all formats, get statistics buffer sizes and formats */ 1761 for (i = 0; i < IPU3_CSS_QUEUES; i++) { 1762 if (fmts[i]) 1763 dev_dbg(css->dev, "%s %s: (%i,%i) fmt 0x%x\n", __func__, 1764 qnames[i], fmts[i]->width, fmts[i]->height, 1765 fmts[i]->pixelformat); 1766 else 1767 dev_dbg(css->dev, "%s %s: (not set)\n", __func__, 1768 qnames[i]); 1769 if (imgu_css_queue_init(&q[i], fmts[i], 1770 IPU3_CSS_QUEUE_TO_FLAGS(i))) { 1771 dev_notice(css->dev, "can not initialize queue %s\n", 1772 qnames[i]); 1773 ret = -EINVAL; 1774 goto out; 1775 } 1776 } 1777 for (i = 0; i < IPU3_CSS_RECTS; i++) { 1778 if (rects[i]) { 1779 dev_dbg(css->dev, "%s %s: (%i,%i)\n", __func__, 1780 rnames[i], rects[i]->width, rects[i]->height); 1781 r[i].width = rects[i]->width; 1782 r[i].height = rects[i]->height; 1783 } else { 1784 dev_dbg(css->dev, "%s %s: (not set)\n", __func__, 1785 rnames[i]); 1786 } 1787 /* For now, force known good resolutions */ 1788 r[i].left = 0; 1789 r[i].top = 0; 1790 } 1791 1792 /* Always require one input and vf only if out is also enabled */ 1793 if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_IN]) || 1794 !imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) { 1795 dev_warn(css->dev, "required queues are disabled\n"); 1796 ret = -EINVAL; 1797 goto out; 1798 } 1799 1800 if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) { 1801 out->width = in->width; 1802 out->height = in->height; 1803 } 1804 if (eff->width <= 0 || eff->height <= 0) { 1805 eff->width = in->width; 1806 eff->height = in->height; 1807 } 1808 if (bds->width <= 0 || bds->height <= 0) { 1809 bds->width = out->width; 1810 bds->height = out->height; 1811 } 1812 if (gdc->width <= 0 || gdc->height <= 0) { 1813 gdc->width = out->width; 1814 gdc->height = out->height; 1815 } 1816 1817 in->width = imgu_css_adjust(in->width, 1); 1818 in->height = imgu_css_adjust(in->height, 1); 1819 eff->width = imgu_css_adjust(eff->width, EFF_ALIGN_W); 1820 eff->height = imgu_css_adjust(eff->height, 1); 1821 bds->width = imgu_css_adjust(bds->width, BDS_ALIGN_W); 1822 bds->height = imgu_css_adjust(bds->height, 1); 1823 gdc->width = imgu_css_adjust(gdc->width, OUT_ALIGN_W); 1824 gdc->height = imgu_css_adjust(gdc->height, OUT_ALIGN_H); 1825 out->width = imgu_css_adjust(out->width, OUT_ALIGN_W); 1826 out->height = imgu_css_adjust(out->height, OUT_ALIGN_H); 1827 vf->width = imgu_css_adjust(vf->width, VF_ALIGN_W); 1828 vf->height = imgu_css_adjust(vf->height, 1); 1829 1830 s = (bds->width - gdc->width) / 2 - FILTER_SIZE; 1831 env->width = s < MIN_ENVELOPE ? MIN_ENVELOPE : s; 1832 s = (bds->height - gdc->height) / 2 - FILTER_SIZE; 1833 env->height = s < MIN_ENVELOPE ? MIN_ENVELOPE : s; 1834 1835 ret = imgu_css_find_binary(css, pipe, q, r); 1836 if (ret < 0) { 1837 dev_err(css->dev, "failed to find suitable binary\n"); 1838 ret = -EINVAL; 1839 goto out; 1840 } 1841 css->pipes[pipe].bindex = ret; 1842 1843 dev_dbg(css->dev, "Binary index %d for pipe %d found.", 1844 css->pipes[pipe].bindex, pipe); 1845 1846 /* Final adjustment and set back the queried formats */ 1847 for (i = 0; i < IPU3_CSS_QUEUES; i++) { 1848 if (fmts[i]) { 1849 if (imgu_css_queue_init(&q[i], &q[i].fmt.mpix, 1850 IPU3_CSS_QUEUE_TO_FLAGS(i))) { 1851 dev_err(css->dev, 1852 "final resolution adjustment failed\n"); 1853 ret = -EINVAL; 1854 goto out; 1855 } 1856 *fmts[i] = q[i].fmt.mpix; 1857 } 1858 } 1859 1860 for (i = 0; i < IPU3_CSS_RECTS; i++) 1861 if (rects[i]) 1862 *rects[i] = r[i]; 1863 1864 dev_dbg(css->dev, 1865 "in(%u,%u) if(%u,%u) ds(%u,%u) gdc(%u,%u) out(%u,%u) vf(%u,%u)", 1866 in->width, in->height, eff->width, eff->height, 1867 bds->width, bds->height, gdc->width, gdc->height, 1868 out->width, out->height, vf->width, vf->height); 1869 1870 ret = 0; 1871 out: 1872 kfree(q); 1873 return ret; 1874 } 1875 1876 int imgu_css_fmt_set(struct imgu_css *css, 1877 struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES], 1878 struct v4l2_rect *rects[IPU3_CSS_RECTS], 1879 unsigned int pipe) 1880 { 1881 struct v4l2_rect rect_data[IPU3_CSS_RECTS]; 1882 struct v4l2_rect *all_rects[IPU3_CSS_RECTS]; 1883 int i, r; 1884 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1885 1886 for (i = 0; i < IPU3_CSS_RECTS; i++) { 1887 if (rects[i]) 1888 rect_data[i] = *rects[i]; 1889 else 1890 memset(&rect_data[i], 0, sizeof(rect_data[i])); 1891 all_rects[i] = &rect_data[i]; 1892 } 1893 r = imgu_css_fmt_try(css, fmts, all_rects, pipe); 1894 if (r < 0) 1895 return r; 1896 1897 for (i = 0; i < IPU3_CSS_QUEUES; i++) 1898 if (imgu_css_queue_init(&css_pipe->queue[i], fmts[i], 1899 IPU3_CSS_QUEUE_TO_FLAGS(i))) 1900 return -EINVAL; 1901 for (i = 0; i < IPU3_CSS_RECTS; i++) { 1902 css_pipe->rect[i] = rect_data[i]; 1903 if (rects[i]) 1904 *rects[i] = rect_data[i]; 1905 } 1906 1907 return 0; 1908 } 1909 1910 int imgu_css_meta_fmt_set(struct v4l2_meta_format *fmt) 1911 { 1912 switch (fmt->dataformat) { 1913 case V4L2_META_FMT_IPU3_PARAMS: 1914 fmt->buffersize = sizeof(struct ipu3_uapi_params); 1915 break; 1916 case V4L2_META_FMT_IPU3_STAT_3A: 1917 fmt->buffersize = sizeof(struct ipu3_uapi_stats_3a); 1918 break; 1919 default: 1920 return -EINVAL; 1921 } 1922 1923 return 0; 1924 } 1925 1926 /* 1927 * Queue given buffer to CSS. imgu_css_buf_prepare() must have been first 1928 * called for the buffer. May be called from interrupt context. 1929 * Returns 0 on success, -EBUSY if the buffer queue is full, or some other 1930 * code on error conditions. 1931 */ 1932 int imgu_css_buf_queue(struct imgu_css *css, unsigned int pipe, 1933 struct imgu_css_buffer *b) 1934 { 1935 struct imgu_abi_buffer *abi_buf; 1936 struct imgu_addr_t *buf_addr; 1937 u32 data; 1938 int r; 1939 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1940 1941 if (!css->streaming) 1942 return -EPROTO; /* CSS or buffer in wrong state */ 1943 1944 if (b->queue >= IPU3_CSS_QUEUES || !imgu_css_queues[b->queue].qid) 1945 return -EINVAL; 1946 1947 b->queue_pos = imgu_css_queue_pos(css, imgu_css_queues[b->queue].qid, 1948 pipe); 1949 1950 if (b->queue_pos >= ARRAY_SIZE(css->pipes[pipe].abi_buffers[b->queue])) 1951 return -EIO; 1952 abi_buf = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].vaddr; 1953 1954 /* Fill struct abi_buffer for firmware */ 1955 memset(abi_buf, 0, sizeof(*abi_buf)); 1956 1957 buf_addr = (void *)abi_buf + imgu_css_queues[b->queue].ptr_ofs; 1958 *(imgu_addr_t *)buf_addr = b->daddr; 1959 1960 if (b->queue == IPU3_CSS_QUEUE_STAT_3A) 1961 abi_buf->payload.s3a.data.dmem.s3a_tbl = b->daddr; 1962 1963 if (b->queue == IPU3_CSS_QUEUE_OUT) 1964 abi_buf->payload.frame.padded_width = 1965 css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad; 1966 1967 if (b->queue == IPU3_CSS_QUEUE_VF) 1968 abi_buf->payload.frame.padded_width = 1969 css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad; 1970 1971 spin_lock(&css_pipe->qlock); 1972 list_add_tail(&b->list, &css_pipe->queue[b->queue].bufs); 1973 spin_unlock(&css_pipe->qlock); 1974 b->state = IPU3_CSS_BUFFER_QUEUED; 1975 1976 data = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].daddr; 1977 r = imgu_css_queue_data(css, imgu_css_queues[b->queue].qid, 1978 pipe, data); 1979 if (r < 0) 1980 goto queueing_failed; 1981 1982 data = IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe, 1983 imgu_css_queues[b->queue].qid); 1984 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, data); 1985 if (r < 0) 1986 goto queueing_failed; 1987 1988 dev_dbg(css->dev, "queued buffer %p to css queue %i in pipe %d\n", 1989 b, b->queue, pipe); 1990 1991 return 0; 1992 1993 queueing_failed: 1994 b->state = (r == -EBUSY || r == -EAGAIN) ? 1995 IPU3_CSS_BUFFER_NEW : IPU3_CSS_BUFFER_FAILED; 1996 list_del(&b->list); 1997 1998 return r; 1999 } 2000 2001 /* 2002 * Get next ready CSS buffer. Returns -EAGAIN in which case the function 2003 * should be called again, or -EBUSY which means that there are no more 2004 * buffers available. May be called from interrupt context. 2005 */ 2006 struct imgu_css_buffer *imgu_css_buf_dequeue(struct imgu_css *css) 2007 { 2008 static const unsigned char evtype_to_queue[] = { 2009 [IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE] = IPU3_CSS_QUEUE_IN, 2010 [IMGU_ABI_EVTTYPE_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_OUT, 2011 [IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_VF, 2012 [IMGU_ABI_EVTTYPE_3A_STATS_DONE] = IPU3_CSS_QUEUE_STAT_3A, 2013 }; 2014 struct imgu_css_buffer *b = ERR_PTR(-EAGAIN); 2015 u32 event, daddr; 2016 int evtype, pipe, pipeid, queue, qid, r; 2017 struct imgu_css_pipe *css_pipe; 2018 2019 if (!css->streaming) 2020 return ERR_PTR(-EPROTO); 2021 2022 r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event); 2023 if (r < 0) 2024 return ERR_PTR(r); 2025 2026 evtype = (event & IMGU_ABI_EVTTYPE_EVENT_MASK) >> 2027 IMGU_ABI_EVTTYPE_EVENT_SHIFT; 2028 2029 switch (evtype) { 2030 case IMGU_ABI_EVTTYPE_OUT_FRAME_DONE: 2031 case IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE: 2032 case IMGU_ABI_EVTTYPE_3A_STATS_DONE: 2033 case IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE: 2034 pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >> 2035 IMGU_ABI_EVTTYPE_PIPE_SHIFT; 2036 pipeid = (event & IMGU_ABI_EVTTYPE_PIPEID_MASK) >> 2037 IMGU_ABI_EVTTYPE_PIPEID_SHIFT; 2038 queue = evtype_to_queue[evtype]; 2039 qid = imgu_css_queues[queue].qid; 2040 2041 if (pipe >= IMGU_MAX_PIPE_NUM) { 2042 dev_err(css->dev, "Invalid pipe: %i\n", pipe); 2043 return ERR_PTR(-EIO); 2044 } 2045 2046 if (qid >= IMGU_ABI_QUEUE_NUM) { 2047 dev_err(css->dev, "Invalid qid: %i\n", qid); 2048 return ERR_PTR(-EIO); 2049 } 2050 css_pipe = &css->pipes[pipe]; 2051 dev_dbg(css->dev, 2052 "event: buffer done 0x%x queue %i pipe %i pipeid %i\n", 2053 event, queue, pipe, pipeid); 2054 2055 r = imgu_css_dequeue_data(css, qid, &daddr); 2056 if (r < 0) { 2057 dev_err(css->dev, "failed to dequeue buffer\n"); 2058 /* Force real error, not -EBUSY */ 2059 return ERR_PTR(-EIO); 2060 } 2061 2062 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, 2063 IMGU_ABI_EVENT_BUFFER_DEQUEUED(qid)); 2064 if (r < 0) { 2065 dev_err(css->dev, "failed to queue event\n"); 2066 return ERR_PTR(-EIO); 2067 } 2068 2069 spin_lock(&css_pipe->qlock); 2070 if (list_empty(&css_pipe->queue[queue].bufs)) { 2071 spin_unlock(&css_pipe->qlock); 2072 dev_err(css->dev, "event on empty queue\n"); 2073 return ERR_PTR(-EIO); 2074 } 2075 b = list_first_entry(&css_pipe->queue[queue].bufs, 2076 struct imgu_css_buffer, list); 2077 if (queue != b->queue || 2078 daddr != css_pipe->abi_buffers 2079 [b->queue][b->queue_pos].daddr) { 2080 spin_unlock(&css_pipe->qlock); 2081 dev_err(css->dev, "dequeued bad buffer 0x%x\n", daddr); 2082 return ERR_PTR(-EIO); 2083 } 2084 2085 dev_dbg(css->dev, "buffer 0x%8x done from pipe %d\n", daddr, pipe); 2086 b->pipe = pipe; 2087 b->state = IPU3_CSS_BUFFER_DONE; 2088 list_del(&b->list); 2089 spin_unlock(&css_pipe->qlock); 2090 break; 2091 case IMGU_ABI_EVTTYPE_PIPELINE_DONE: 2092 pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >> 2093 IMGU_ABI_EVTTYPE_PIPE_SHIFT; 2094 if (pipe >= IMGU_MAX_PIPE_NUM) { 2095 dev_err(css->dev, "Invalid pipe: %i\n", pipe); 2096 return ERR_PTR(-EIO); 2097 } 2098 2099 css_pipe = &css->pipes[pipe]; 2100 dev_dbg(css->dev, "event: pipeline done 0x%8x for pipe %d\n", 2101 event, pipe); 2102 break; 2103 case IMGU_ABI_EVTTYPE_TIMER: 2104 r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event); 2105 if (r < 0) 2106 return ERR_PTR(r); 2107 2108 if ((event & IMGU_ABI_EVTTYPE_EVENT_MASK) >> 2109 IMGU_ABI_EVTTYPE_EVENT_SHIFT == IMGU_ABI_EVTTYPE_TIMER) 2110 dev_dbg(css->dev, "event: timer\n"); 2111 else 2112 dev_warn(css->dev, "half of timer event missing\n"); 2113 break; 2114 case IMGU_ABI_EVTTYPE_FW_WARNING: 2115 dev_warn(css->dev, "event: firmware warning 0x%x\n", event); 2116 break; 2117 case IMGU_ABI_EVTTYPE_FW_ASSERT: 2118 dev_err(css->dev, 2119 "event: firmware assert 0x%x module_id %i line_no %i\n", 2120 event, 2121 (event & IMGU_ABI_EVTTYPE_MODULEID_MASK) >> 2122 IMGU_ABI_EVTTYPE_MODULEID_SHIFT, 2123 swab16((event & IMGU_ABI_EVTTYPE_LINENO_MASK) >> 2124 IMGU_ABI_EVTTYPE_LINENO_SHIFT)); 2125 break; 2126 default: 2127 dev_warn(css->dev, "received unknown event 0x%x\n", event); 2128 } 2129 2130 return b; 2131 } 2132 2133 /* 2134 * Get a new set of parameters from pool and initialize them based on 2135 * the parameters params, gdc, and obgrid. Any of these may be NULL, 2136 * in which case the previously set parameters are used. 2137 * If parameters haven't been set previously, initialize from scratch. 2138 * 2139 * Return index to css->parameter_set_info which has the newly created 2140 * parameters or negative value on error. 2141 */ 2142 int imgu_css_set_parameters(struct imgu_css *css, unsigned int pipe, 2143 struct ipu3_uapi_params *set_params) 2144 { 2145 static const unsigned int queue_id = IMGU_ABI_QUEUE_A_ID; 2146 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 2147 const int stage = 0; 2148 const struct imgu_fw_info *bi; 2149 int obgrid_size; 2150 unsigned int stripes, i; 2151 struct ipu3_uapi_flags *use = set_params ? &set_params->use : NULL; 2152 2153 /* Destination buffers which are filled here */ 2154 struct imgu_abi_parameter_set_info *param_set; 2155 struct imgu_abi_acc_param *acc = NULL; 2156 struct imgu_abi_gdc_warp_param *gdc = NULL; 2157 struct ipu3_uapi_obgrid_param *obgrid = NULL; 2158 const struct imgu_css_map *map; 2159 void *vmem0 = NULL; 2160 void *dmem0 = NULL; 2161 2162 enum imgu_abi_memories m; 2163 int r = -EBUSY; 2164 2165 if (!css->streaming) 2166 return -EPROTO; 2167 2168 dev_dbg(css->dev, "%s for pipe %d", __func__, pipe); 2169 2170 bi = &css->fwp->binary_header[css_pipe->bindex]; 2171 obgrid_size = imgu_css_fw_obgrid_size(bi); 2172 stripes = bi->info.isp.sp.iterator.num_stripes ? : 1; 2173 2174 imgu_css_pool_get(&css_pipe->pool.parameter_set_info); 2175 param_set = imgu_css_pool_last(&css_pipe->pool.parameter_set_info, 2176 0)->vaddr; 2177 2178 /* Get a new acc only if new parameters given, or none yet */ 2179 map = imgu_css_pool_last(&css_pipe->pool.acc, 0); 2180 if (set_params || !map->vaddr) { 2181 imgu_css_pool_get(&css_pipe->pool.acc); 2182 map = imgu_css_pool_last(&css_pipe->pool.acc, 0); 2183 acc = map->vaddr; 2184 } 2185 2186 /* Get new VMEM0 only if needed, or none yet */ 2187 m = IMGU_ABI_MEM_ISP_VMEM0; 2188 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0); 2189 if (!map->vaddr || (set_params && (set_params->use.lin_vmem_params || 2190 set_params->use.tnr3_vmem_params || 2191 set_params->use.xnr3_vmem_params))) { 2192 imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]); 2193 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0); 2194 vmem0 = map->vaddr; 2195 } 2196 2197 /* Get new DMEM0 only if needed, or none yet */ 2198 m = IMGU_ABI_MEM_ISP_DMEM0; 2199 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0); 2200 if (!map->vaddr || (set_params && (set_params->use.tnr3_dmem_params || 2201 set_params->use.xnr3_dmem_params))) { 2202 imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]); 2203 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0); 2204 dmem0 = map->vaddr; 2205 } 2206 2207 /* Configure acc parameter cluster */ 2208 if (acc) { 2209 /* get acc_old */ 2210 map = imgu_css_pool_last(&css_pipe->pool.acc, 1); 2211 /* user acc */ 2212 r = imgu_css_cfg_acc(css, pipe, use, acc, map->vaddr, 2213 set_params ? &set_params->acc_param : NULL); 2214 if (r < 0) 2215 goto fail; 2216 } 2217 2218 /* Configure late binding parameters */ 2219 if (vmem0) { 2220 m = IMGU_ABI_MEM_ISP_VMEM0; 2221 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1); 2222 r = imgu_css_cfg_vmem0(css, pipe, use, vmem0, 2223 map->vaddr, set_params); 2224 if (r < 0) 2225 goto fail; 2226 } 2227 2228 if (dmem0) { 2229 m = IMGU_ABI_MEM_ISP_DMEM0; 2230 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1); 2231 r = imgu_css_cfg_dmem0(css, pipe, use, dmem0, 2232 map->vaddr, set_params); 2233 if (r < 0) 2234 goto fail; 2235 } 2236 2237 /* Get a new gdc only if a new gdc is given, or none yet */ 2238 if (bi->info.isp.sp.enable.dvs_6axis) { 2239 unsigned int a = IPU3_CSS_AUX_FRAME_REF; 2240 unsigned int g = IPU3_CSS_RECT_GDC; 2241 unsigned int e = IPU3_CSS_RECT_ENVELOPE; 2242 2243 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0); 2244 if (!map->vaddr) { 2245 imgu_css_pool_get(&css_pipe->pool.gdc); 2246 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0); 2247 gdc = map->vaddr; 2248 imgu_css_cfg_gdc_table(map->vaddr, 2249 css_pipe->aux_frames[a].bytesperline / 2250 css_pipe->aux_frames[a].bytesperpixel, 2251 css_pipe->aux_frames[a].height, 2252 css_pipe->rect[g].width, 2253 css_pipe->rect[g].height, 2254 css_pipe->rect[e].width + FILTER_SIZE, 2255 css_pipe->rect[e].height + 2256 FILTER_SIZE); 2257 } 2258 } 2259 2260 /* Get a new obgrid only if a new obgrid is given, or none yet */ 2261 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0); 2262 if (!map->vaddr || (set_params && set_params->use.obgrid_param)) { 2263 imgu_css_pool_get(&css_pipe->pool.obgrid); 2264 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0); 2265 obgrid = map->vaddr; 2266 2267 /* Configure optical black level grid (obgrid) */ 2268 if (set_params && set_params->use.obgrid_param) 2269 for (i = 0; i < obgrid_size / sizeof(*obgrid); i++) 2270 obgrid[i] = set_params->obgrid_param; 2271 else 2272 memset(obgrid, 0, obgrid_size); 2273 } 2274 2275 /* Configure parameter set info, queued to `queue_id' */ 2276 2277 memset(param_set, 0, sizeof(*param_set)); 2278 map = imgu_css_pool_last(&css_pipe->pool.acc, 0); 2279 param_set->mem_map.acc_cluster_params_for_sp = map->daddr; 2280 2281 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0); 2282 param_set->mem_map.dvs_6axis_params_y = map->daddr; 2283 2284 for (i = 0; i < stripes; i++) { 2285 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0); 2286 param_set->mem_map.obgrid_tbl[i] = 2287 map->daddr + (obgrid_size / stripes) * i; 2288 } 2289 2290 for (m = 0; m < IMGU_ABI_NUM_MEMORIES; m++) { 2291 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0); 2292 param_set->mem_map.isp_mem_param[stage][m] = map->daddr; 2293 } 2294 2295 /* Then queue the new parameter buffer */ 2296 map = imgu_css_pool_last(&css_pipe->pool.parameter_set_info, 0); 2297 r = imgu_css_queue_data(css, queue_id, pipe, map->daddr); 2298 if (r < 0) 2299 goto fail; 2300 2301 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, 2302 IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe, 2303 queue_id)); 2304 if (r < 0) 2305 goto fail_no_put; 2306 2307 /* Finally dequeue all old parameter buffers */ 2308 2309 do { 2310 u32 daddr; 2311 2312 r = imgu_css_dequeue_data(css, queue_id, &daddr); 2313 if (r == -EBUSY) 2314 break; 2315 if (r) 2316 goto fail_no_put; 2317 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, 2318 IMGU_ABI_EVENT_BUFFER_DEQUEUED 2319 (queue_id)); 2320 if (r < 0) { 2321 dev_err(css->dev, "failed to queue parameter event\n"); 2322 goto fail_no_put; 2323 } 2324 } while (1); 2325 2326 return 0; 2327 2328 fail: 2329 /* 2330 * A failure, most likely the parameter queue was full. 2331 * Return error but continue streaming. User can try submitting new 2332 * parameters again later. 2333 */ 2334 2335 imgu_css_pool_put(&css_pipe->pool.parameter_set_info); 2336 if (acc) 2337 imgu_css_pool_put(&css_pipe->pool.acc); 2338 if (gdc) 2339 imgu_css_pool_put(&css_pipe->pool.gdc); 2340 if (obgrid) 2341 imgu_css_pool_put(&css_pipe->pool.obgrid); 2342 if (vmem0) 2343 imgu_css_pool_put( 2344 &css_pipe->pool.binary_params_p 2345 [IMGU_ABI_MEM_ISP_VMEM0]); 2346 if (dmem0) 2347 imgu_css_pool_put( 2348 &css_pipe->pool.binary_params_p 2349 [IMGU_ABI_MEM_ISP_DMEM0]); 2350 2351 fail_no_put: 2352 return r; 2353 } 2354 2355 int imgu_css_irq_ack(struct imgu_css *css) 2356 { 2357 static const int NUM_SWIRQS = 3; 2358 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]]; 2359 void __iomem *const base = css->base; 2360 u32 irq_status[IMGU_IRQCTRL_NUM]; 2361 int i; 2362 2363 u32 imgu_status = readl(base + IMGU_REG_INT_STATUS); 2364 2365 writel(imgu_status, base + IMGU_REG_INT_STATUS); 2366 for (i = 0; i < IMGU_IRQCTRL_NUM; i++) 2367 irq_status[i] = readl(base + IMGU_REG_IRQCTRL_STATUS(i)); 2368 2369 for (i = 0; i < NUM_SWIRQS; i++) { 2370 if (irq_status[IMGU_IRQCTRL_SP0] & IMGU_IRQCTRL_IRQ_SW_PIN(i)) { 2371 /* SP SW interrupt */ 2372 u32 cnt = readl(base + IMGU_REG_SP_DMEM_BASE(0) + 2373 bi->info.sp.output); 2374 u32 val = readl(base + IMGU_REG_SP_DMEM_BASE(0) + 2375 bi->info.sp.output + 4 + 4 * i); 2376 2377 dev_dbg(css->dev, "%s: swirq %i cnt %i val 0x%x\n", 2378 __func__, i, cnt, val); 2379 } 2380 } 2381 2382 for (i = IMGU_IRQCTRL_NUM - 1; i >= 0; i--) 2383 if (irq_status[i]) { 2384 writel(irq_status[i], base + IMGU_REG_IRQCTRL_CLEAR(i)); 2385 /* Wait for write to complete */ 2386 readl(base + IMGU_REG_IRQCTRL_ENABLE(i)); 2387 } 2388 2389 dev_dbg(css->dev, "%s: imgu 0x%x main 0x%x sp0 0x%x sp1 0x%x\n", 2390 __func__, imgu_status, irq_status[IMGU_IRQCTRL_MAIN], 2391 irq_status[IMGU_IRQCTRL_SP0], irq_status[IMGU_IRQCTRL_SP1]); 2392 2393 if (!imgu_status && !irq_status[IMGU_IRQCTRL_MAIN]) 2394 return -ENOMSG; 2395 2396 return 0; 2397 } 2398