1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2012-2016 Mentor Graphics Inc. 4 * 5 * Queued image conversion support, with tiling and rotation. 6 */ 7 8 #include <linux/interrupt.h> 9 #include <linux/dma-mapping.h> 10 #include <video/imx-ipu-image-convert.h> 11 #include "ipu-prv.h" 12 13 /* 14 * The IC Resizer has a restriction that the output frame from the 15 * resizer must be 1024 or less in both width (pixels) and height 16 * (lines). 17 * 18 * The image converter attempts to split up a conversion when 19 * the desired output (converted) frame resolution exceeds the 20 * IC resizer limit of 1024 in either dimension. 21 * 22 * If either dimension of the output frame exceeds the limit, the 23 * dimension is split into 1, 2, or 4 equal stripes, for a maximum 24 * of 4*4 or 16 tiles. A conversion is then carried out for each 25 * tile (but taking care to pass the full frame stride length to 26 * the DMA channel's parameter memory!). IDMA double-buffering is used 27 * to convert each tile back-to-back when possible (see note below 28 * when double_buffering boolean is set). 29 * 30 * Note that the input frame must be split up into the same number 31 * of tiles as the output frame: 32 * 33 * +---------+-----+ 34 * +-----+---+ | A | B | 35 * | A | B | | | | 36 * +-----+---+ --> +---------+-----+ 37 * | C | D | | C | D | 38 * +-----+---+ | | | 39 * +---------+-----+ 40 * 41 * Clockwise 90° rotations are handled by first rescaling into a 42 * reusable temporary tile buffer and then rotating with the 8x8 43 * block rotator, writing to the correct destination: 44 * 45 * +-----+-----+ 46 * | | | 47 * +-----+---+ +---------+ | C | A | 48 * | A | B | | A,B, | | | | | 49 * +-----+---+ --> | C,D | | --> | | | 50 * | C | D | +---------+ +-----+-----+ 51 * +-----+---+ | D | B | 52 * | | | 53 * +-----+-----+ 54 * 55 * If the 8x8 block rotator is used, horizontal or vertical flipping 56 * is done during the rotation step, otherwise flipping is done 57 * during the scaling step. 58 * With rotation or flipping, tile order changes between input and 59 * output image. Tiles are numbered row major from top left to bottom 60 * right for both input and output image. 61 */ 62 63 #define MAX_STRIPES_W 4 64 #define MAX_STRIPES_H 4 65 #define MAX_TILES (MAX_STRIPES_W * MAX_STRIPES_H) 66 67 #define MIN_W 16 68 #define MIN_H 8 69 #define MAX_W 4096 70 #define MAX_H 4096 71 72 enum ipu_image_convert_type { 73 IMAGE_CONVERT_IN = 0, 74 IMAGE_CONVERT_OUT, 75 }; 76 77 struct ipu_image_convert_dma_buf { 78 void *virt; 79 dma_addr_t phys; 80 unsigned long len; 81 }; 82 83 struct ipu_image_convert_dma_chan { 84 int in; 85 int out; 86 int rot_in; 87 int rot_out; 88 int vdi_in_p; 89 int vdi_in; 90 int vdi_in_n; 91 }; 92 93 /* dimensions of one tile */ 94 struct ipu_image_tile { 95 u32 width; 96 u32 height; 97 u32 left; 98 u32 top; 99 /* size and strides are in bytes */ 100 u32 size; 101 u32 stride; 102 u32 rot_stride; 103 /* start Y or packed offset of this tile */ 104 u32 offset; 105 /* offset from start to tile in U plane, for planar formats */ 106 u32 u_off; 107 /* offset from start to tile in V plane, for planar formats */ 108 u32 v_off; 109 }; 110 111 struct ipu_image_convert_image { 112 struct ipu_image base; 113 enum ipu_image_convert_type type; 114 115 const struct ipu_image_pixfmt *fmt; 116 unsigned int stride; 117 118 /* # of rows (horizontal stripes) if dest height is > 1024 */ 119 unsigned int num_rows; 120 /* # of columns (vertical stripes) if dest width is > 1024 */ 121 unsigned int num_cols; 122 123 struct ipu_image_tile tile[MAX_TILES]; 124 }; 125 126 struct ipu_image_pixfmt { 127 u32 fourcc; /* V4L2 fourcc */ 128 int bpp; /* total bpp */ 129 int uv_width_dec; /* decimation in width for U/V planes */ 130 int uv_height_dec; /* decimation in height for U/V planes */ 131 bool planar; /* planar format */ 132 bool uv_swapped; /* U and V planes are swapped */ 133 bool uv_packed; /* partial planar (U and V in same plane) */ 134 }; 135 136 struct ipu_image_convert_ctx; 137 struct ipu_image_convert_chan; 138 struct ipu_image_convert_priv; 139 140 struct ipu_image_convert_ctx { 141 struct ipu_image_convert_chan *chan; 142 143 ipu_image_convert_cb_t complete; 144 void *complete_context; 145 146 /* Source/destination image data and rotation mode */ 147 struct ipu_image_convert_image in; 148 struct ipu_image_convert_image out; 149 struct ipu_ic_csc csc; 150 enum ipu_rotate_mode rot_mode; 151 u32 downsize_coeff_h; 152 u32 downsize_coeff_v; 153 u32 image_resize_coeff_h; 154 u32 image_resize_coeff_v; 155 u32 resize_coeffs_h[MAX_STRIPES_W]; 156 u32 resize_coeffs_v[MAX_STRIPES_H]; 157 158 /* intermediate buffer for rotation */ 159 struct ipu_image_convert_dma_buf rot_intermediate[2]; 160 161 /* current buffer number for double buffering */ 162 int cur_buf_num; 163 164 bool aborting; 165 struct completion aborted; 166 167 /* can we use double-buffering for this conversion operation? */ 168 bool double_buffering; 169 /* num_rows * num_cols */ 170 unsigned int num_tiles; 171 /* next tile to process */ 172 unsigned int next_tile; 173 /* where to place converted tile in dest image */ 174 unsigned int out_tile_map[MAX_TILES]; 175 176 struct list_head list; 177 }; 178 179 struct ipu_image_convert_chan { 180 struct ipu_image_convert_priv *priv; 181 182 enum ipu_ic_task ic_task; 183 const struct ipu_image_convert_dma_chan *dma_ch; 184 185 struct ipu_ic *ic; 186 struct ipuv3_channel *in_chan; 187 struct ipuv3_channel *out_chan; 188 struct ipuv3_channel *rotation_in_chan; 189 struct ipuv3_channel *rotation_out_chan; 190 191 /* the IPU end-of-frame irqs */ 192 int out_eof_irq; 193 int rot_out_eof_irq; 194 195 spinlock_t irqlock; 196 197 /* list of convert contexts */ 198 struct list_head ctx_list; 199 /* queue of conversion runs */ 200 struct list_head pending_q; 201 /* queue of completed runs */ 202 struct list_head done_q; 203 204 /* the current conversion run */ 205 struct ipu_image_convert_run *current_run; 206 }; 207 208 struct ipu_image_convert_priv { 209 struct ipu_image_convert_chan chan[IC_NUM_TASKS]; 210 struct ipu_soc *ipu; 211 }; 212 213 static const struct ipu_image_convert_dma_chan 214 image_convert_dma_chan[IC_NUM_TASKS] = { 215 [IC_TASK_VIEWFINDER] = { 216 .in = IPUV3_CHANNEL_MEM_IC_PRP_VF, 217 .out = IPUV3_CHANNEL_IC_PRP_VF_MEM, 218 .rot_in = IPUV3_CHANNEL_MEM_ROT_VF, 219 .rot_out = IPUV3_CHANNEL_ROT_VF_MEM, 220 .vdi_in_p = IPUV3_CHANNEL_MEM_VDI_PREV, 221 .vdi_in = IPUV3_CHANNEL_MEM_VDI_CUR, 222 .vdi_in_n = IPUV3_CHANNEL_MEM_VDI_NEXT, 223 }, 224 [IC_TASK_POST_PROCESSOR] = { 225 .in = IPUV3_CHANNEL_MEM_IC_PP, 226 .out = IPUV3_CHANNEL_IC_PP_MEM, 227 .rot_in = IPUV3_CHANNEL_MEM_ROT_PP, 228 .rot_out = IPUV3_CHANNEL_ROT_PP_MEM, 229 }, 230 }; 231 232 static const struct ipu_image_pixfmt image_convert_formats[] = { 233 { 234 .fourcc = V4L2_PIX_FMT_RGB565, 235 .bpp = 16, 236 }, { 237 .fourcc = V4L2_PIX_FMT_RGB24, 238 .bpp = 24, 239 }, { 240 .fourcc = V4L2_PIX_FMT_BGR24, 241 .bpp = 24, 242 }, { 243 .fourcc = V4L2_PIX_FMT_RGB32, 244 .bpp = 32, 245 }, { 246 .fourcc = V4L2_PIX_FMT_BGR32, 247 .bpp = 32, 248 }, { 249 .fourcc = V4L2_PIX_FMT_XRGB32, 250 .bpp = 32, 251 }, { 252 .fourcc = V4L2_PIX_FMT_XBGR32, 253 .bpp = 32, 254 }, { 255 .fourcc = V4L2_PIX_FMT_BGRX32, 256 .bpp = 32, 257 }, { 258 .fourcc = V4L2_PIX_FMT_RGBX32, 259 .bpp = 32, 260 }, { 261 .fourcc = V4L2_PIX_FMT_YUYV, 262 .bpp = 16, 263 .uv_width_dec = 2, 264 .uv_height_dec = 1, 265 }, { 266 .fourcc = V4L2_PIX_FMT_UYVY, 267 .bpp = 16, 268 .uv_width_dec = 2, 269 .uv_height_dec = 1, 270 }, { 271 .fourcc = V4L2_PIX_FMT_YUV420, 272 .bpp = 12, 273 .planar = true, 274 .uv_width_dec = 2, 275 .uv_height_dec = 2, 276 }, { 277 .fourcc = V4L2_PIX_FMT_YVU420, 278 .bpp = 12, 279 .planar = true, 280 .uv_width_dec = 2, 281 .uv_height_dec = 2, 282 .uv_swapped = true, 283 }, { 284 .fourcc = V4L2_PIX_FMT_NV12, 285 .bpp = 12, 286 .planar = true, 287 .uv_width_dec = 2, 288 .uv_height_dec = 2, 289 .uv_packed = true, 290 }, { 291 .fourcc = V4L2_PIX_FMT_YUV422P, 292 .bpp = 16, 293 .planar = true, 294 .uv_width_dec = 2, 295 .uv_height_dec = 1, 296 }, { 297 .fourcc = V4L2_PIX_FMT_NV16, 298 .bpp = 16, 299 .planar = true, 300 .uv_width_dec = 2, 301 .uv_height_dec = 1, 302 .uv_packed = true, 303 }, 304 }; 305 306 static const struct ipu_image_pixfmt *get_format(u32 fourcc) 307 { 308 const struct ipu_image_pixfmt *ret = NULL; 309 unsigned int i; 310 311 for (i = 0; i < ARRAY_SIZE(image_convert_formats); i++) { 312 if (image_convert_formats[i].fourcc == fourcc) { 313 ret = &image_convert_formats[i]; 314 break; 315 } 316 } 317 318 return ret; 319 } 320 321 static void dump_format(struct ipu_image_convert_ctx *ctx, 322 struct ipu_image_convert_image *ic_image) 323 { 324 struct ipu_image_convert_chan *chan = ctx->chan; 325 struct ipu_image_convert_priv *priv = chan->priv; 326 327 dev_dbg(priv->ipu->dev, 328 "task %u: ctx %p: %s format: %dx%d (%dx%d tiles), %c%c%c%c\n", 329 chan->ic_task, ctx, 330 ic_image->type == IMAGE_CONVERT_OUT ? "Output" : "Input", 331 ic_image->base.pix.width, ic_image->base.pix.height, 332 ic_image->num_cols, ic_image->num_rows, 333 ic_image->fmt->fourcc & 0xff, 334 (ic_image->fmt->fourcc >> 8) & 0xff, 335 (ic_image->fmt->fourcc >> 16) & 0xff, 336 (ic_image->fmt->fourcc >> 24) & 0xff); 337 } 338 339 int ipu_image_convert_enum_format(int index, u32 *fourcc) 340 { 341 const struct ipu_image_pixfmt *fmt; 342 343 if (index >= (int)ARRAY_SIZE(image_convert_formats)) 344 return -EINVAL; 345 346 /* Format found */ 347 fmt = &image_convert_formats[index]; 348 *fourcc = fmt->fourcc; 349 return 0; 350 } 351 EXPORT_SYMBOL_GPL(ipu_image_convert_enum_format); 352 353 static void free_dma_buf(struct ipu_image_convert_priv *priv, 354 struct ipu_image_convert_dma_buf *buf) 355 { 356 if (buf->virt) 357 dma_free_coherent(priv->ipu->dev, 358 buf->len, buf->virt, buf->phys); 359 buf->virt = NULL; 360 buf->phys = 0; 361 } 362 363 static int alloc_dma_buf(struct ipu_image_convert_priv *priv, 364 struct ipu_image_convert_dma_buf *buf, 365 int size) 366 { 367 buf->len = PAGE_ALIGN(size); 368 buf->virt = dma_alloc_coherent(priv->ipu->dev, buf->len, &buf->phys, 369 GFP_DMA | GFP_KERNEL); 370 if (!buf->virt) { 371 dev_err(priv->ipu->dev, "failed to alloc dma buffer\n"); 372 return -ENOMEM; 373 } 374 375 return 0; 376 } 377 378 static inline int num_stripes(int dim) 379 { 380 return (dim - 1) / 1024 + 1; 381 } 382 383 /* 384 * Calculate downsizing coefficients, which are the same for all tiles, 385 * and bilinear resizing coefficients, which are used to find the best 386 * seam positions. 387 */ 388 static int calc_image_resize_coefficients(struct ipu_image_convert_ctx *ctx, 389 struct ipu_image *in, 390 struct ipu_image *out) 391 { 392 u32 downsized_width = in->rect.width; 393 u32 downsized_height = in->rect.height; 394 u32 downsize_coeff_v = 0; 395 u32 downsize_coeff_h = 0; 396 u32 resized_width = out->rect.width; 397 u32 resized_height = out->rect.height; 398 u32 resize_coeff_h; 399 u32 resize_coeff_v; 400 401 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 402 resized_width = out->rect.height; 403 resized_height = out->rect.width; 404 } 405 406 /* Do not let invalid input lead to an endless loop below */ 407 if (WARN_ON(resized_width == 0 || resized_height == 0)) 408 return -EINVAL; 409 410 while (downsized_width > 1024 || 411 downsized_width >= resized_width * 2) { 412 downsized_width >>= 1; 413 downsize_coeff_h++; 414 } 415 416 while (downsized_height > 1024 || 417 downsized_height >= resized_height * 2) { 418 downsized_height >>= 1; 419 downsize_coeff_v++; 420 } 421 422 /* 423 * Calculate the bilinear resizing coefficients that could be used if 424 * we were converting with a single tile. The bottom right output pixel 425 * should sample as close as possible to the bottom right input pixel 426 * out of the decimator, but not overshoot it: 427 */ 428 resize_coeff_h = 8192 * (downsized_width - 1) / (resized_width - 1); 429 resize_coeff_v = 8192 * (downsized_height - 1) / (resized_height - 1); 430 431 dev_dbg(ctx->chan->priv->ipu->dev, 432 "%s: hscale: >>%u, *8192/%u vscale: >>%u, *8192/%u, %ux%u tiles\n", 433 __func__, downsize_coeff_h, resize_coeff_h, downsize_coeff_v, 434 resize_coeff_v, ctx->in.num_cols, ctx->in.num_rows); 435 436 if (downsize_coeff_h > 2 || downsize_coeff_v > 2 || 437 resize_coeff_h > 0x3fff || resize_coeff_v > 0x3fff) 438 return -EINVAL; 439 440 ctx->downsize_coeff_h = downsize_coeff_h; 441 ctx->downsize_coeff_v = downsize_coeff_v; 442 ctx->image_resize_coeff_h = resize_coeff_h; 443 ctx->image_resize_coeff_v = resize_coeff_v; 444 445 return 0; 446 } 447 448 #define round_closest(x, y) round_down((x) + (y)/2, (y)) 449 450 /* 451 * Find the best aligned seam position in the inverval [out_start, out_end]. 452 * Rotation and image offsets are out of scope. 453 * 454 * @out_start: start of inverval, must be within 1024 pixels / lines 455 * of out_end 456 * @out_end: end of interval, smaller than or equal to out_edge 457 * @in_edge: input right / bottom edge 458 * @out_edge: output right / bottom edge 459 * @in_align: input alignment, either horizontal 8-byte line start address 460 * alignment, or pixel alignment due to image format 461 * @out_align: output alignment, either horizontal 8-byte line start address 462 * alignment, or pixel alignment due to image format or rotator 463 * block size 464 * @in_burst: horizontal input burst size in case of horizontal flip 465 * @out_burst: horizontal output burst size or rotator block size 466 * @downsize_coeff: downsizing section coefficient 467 * @resize_coeff: main processing section resizing coefficient 468 * @_in_seam: aligned input seam position return value 469 * @_out_seam: aligned output seam position return value 470 */ 471 static void find_best_seam(struct ipu_image_convert_ctx *ctx, 472 unsigned int out_start, 473 unsigned int out_end, 474 unsigned int in_edge, 475 unsigned int out_edge, 476 unsigned int in_align, 477 unsigned int out_align, 478 unsigned int in_burst, 479 unsigned int out_burst, 480 unsigned int downsize_coeff, 481 unsigned int resize_coeff, 482 u32 *_in_seam, 483 u32 *_out_seam) 484 { 485 struct device *dev = ctx->chan->priv->ipu->dev; 486 unsigned int out_pos; 487 /* Input / output seam position candidates */ 488 unsigned int out_seam = 0; 489 unsigned int in_seam = 0; 490 unsigned int min_diff = UINT_MAX; 491 492 /* 493 * Output tiles must start at a multiple of 8 bytes horizontally and 494 * possibly at an even line horizontally depending on the pixel format. 495 * Only consider output aligned positions for the seam. 496 */ 497 out_start = round_up(out_start, out_align); 498 for (out_pos = out_start; out_pos < out_end; out_pos += out_align) { 499 unsigned int in_pos; 500 unsigned int in_pos_aligned; 501 unsigned int abs_diff; 502 503 /* 504 * Tiles in the right row / bottom column may not be allowed to 505 * overshoot horizontally / vertically. out_burst may be the 506 * actual DMA burst size, or the rotator block size. 507 */ 508 if ((out_burst > 1) && (out_edge - out_pos) % out_burst) 509 continue; 510 511 /* 512 * Input sample position, corresponding to out_pos, 19.13 fixed 513 * point. 514 */ 515 in_pos = (out_pos * resize_coeff) << downsize_coeff; 516 /* 517 * The closest input sample position that we could actually 518 * start the input tile at, 19.13 fixed point. 519 */ 520 in_pos_aligned = round_closest(in_pos, 8192U * in_align); 521 522 if ((in_burst > 1) && 523 (in_edge - in_pos_aligned / 8192U) % in_burst) 524 continue; 525 526 if (in_pos < in_pos_aligned) 527 abs_diff = in_pos_aligned - in_pos; 528 else 529 abs_diff = in_pos - in_pos_aligned; 530 531 if (abs_diff < min_diff) { 532 in_seam = in_pos_aligned; 533 out_seam = out_pos; 534 min_diff = abs_diff; 535 } 536 } 537 538 *_out_seam = out_seam; 539 /* Convert 19.13 fixed point to integer seam position */ 540 *_in_seam = DIV_ROUND_CLOSEST(in_seam, 8192U); 541 542 dev_dbg(dev, "%s: out_seam %u(%u) in [%u, %u], in_seam %u(%u) diff %u.%03u\n", 543 __func__, out_seam, out_align, out_start, out_end, 544 *_in_seam, in_align, min_diff / 8192, 545 DIV_ROUND_CLOSEST(min_diff % 8192 * 1000, 8192)); 546 } 547 548 /* 549 * Tile left edges are required to be aligned to multiples of 8 bytes 550 * by the IDMAC. 551 */ 552 static inline u32 tile_left_align(const struct ipu_image_pixfmt *fmt) 553 { 554 if (fmt->planar) 555 return fmt->uv_packed ? 8 : 8 * fmt->uv_width_dec; 556 else 557 return fmt->bpp == 32 ? 2 : fmt->bpp == 16 ? 4 : 8; 558 } 559 560 /* 561 * Tile top edge alignment is only limited by chroma subsampling. 562 */ 563 static inline u32 tile_top_align(const struct ipu_image_pixfmt *fmt) 564 { 565 return fmt->uv_height_dec > 1 ? 2 : 1; 566 } 567 568 static inline u32 tile_width_align(enum ipu_image_convert_type type, 569 const struct ipu_image_pixfmt *fmt, 570 enum ipu_rotate_mode rot_mode) 571 { 572 if (type == IMAGE_CONVERT_IN) { 573 /* 574 * The IC burst reads 8 pixels at a time. Reading beyond the 575 * end of the line is usually acceptable. Those pixels are 576 * ignored, unless the IC has to write the scaled line in 577 * reverse. 578 */ 579 return (!ipu_rot_mode_is_irt(rot_mode) && 580 (rot_mode & IPU_ROT_BIT_HFLIP)) ? 8 : 2; 581 } 582 583 /* 584 * Align to 16x16 pixel blocks for planar 4:2:0 chroma subsampled 585 * formats to guarantee 8-byte aligned line start addresses in the 586 * chroma planes when IRT is used. Align to 8x8 pixel IRT block size 587 * for all other formats. 588 */ 589 return (ipu_rot_mode_is_irt(rot_mode) && 590 fmt->planar && !fmt->uv_packed) ? 591 8 * fmt->uv_width_dec : 8; 592 } 593 594 static inline u32 tile_height_align(enum ipu_image_convert_type type, 595 const struct ipu_image_pixfmt *fmt, 596 enum ipu_rotate_mode rot_mode) 597 { 598 if (type == IMAGE_CONVERT_IN || !ipu_rot_mode_is_irt(rot_mode)) 599 return 2; 600 601 /* 602 * Align to 16x16 pixel blocks for planar 4:2:0 chroma subsampled 603 * formats to guarantee 8-byte aligned line start addresses in the 604 * chroma planes when IRT is used. Align to 8x8 pixel IRT block size 605 * for all other formats. 606 */ 607 return (fmt->planar && !fmt->uv_packed) ? 8 * fmt->uv_width_dec : 8; 608 } 609 610 /* 611 * Fill in left position and width and for all tiles in an input column, and 612 * for all corresponding output tiles. If the 90° rotator is used, the output 613 * tiles are in a row, and output tile top position and height are set. 614 */ 615 static void fill_tile_column(struct ipu_image_convert_ctx *ctx, 616 unsigned int col, 617 struct ipu_image_convert_image *in, 618 unsigned int in_left, unsigned int in_width, 619 struct ipu_image_convert_image *out, 620 unsigned int out_left, unsigned int out_width) 621 { 622 unsigned int row, tile_idx; 623 struct ipu_image_tile *in_tile, *out_tile; 624 625 for (row = 0; row < in->num_rows; row++) { 626 tile_idx = in->num_cols * row + col; 627 in_tile = &in->tile[tile_idx]; 628 out_tile = &out->tile[ctx->out_tile_map[tile_idx]]; 629 630 in_tile->left = in_left; 631 in_tile->width = in_width; 632 633 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 634 out_tile->top = out_left; 635 out_tile->height = out_width; 636 } else { 637 out_tile->left = out_left; 638 out_tile->width = out_width; 639 } 640 } 641 } 642 643 /* 644 * Fill in top position and height and for all tiles in an input row, and 645 * for all corresponding output tiles. If the 90° rotator is used, the output 646 * tiles are in a column, and output tile left position and width are set. 647 */ 648 static void fill_tile_row(struct ipu_image_convert_ctx *ctx, unsigned int row, 649 struct ipu_image_convert_image *in, 650 unsigned int in_top, unsigned int in_height, 651 struct ipu_image_convert_image *out, 652 unsigned int out_top, unsigned int out_height) 653 { 654 unsigned int col, tile_idx; 655 struct ipu_image_tile *in_tile, *out_tile; 656 657 for (col = 0; col < in->num_cols; col++) { 658 tile_idx = in->num_cols * row + col; 659 in_tile = &in->tile[tile_idx]; 660 out_tile = &out->tile[ctx->out_tile_map[tile_idx]]; 661 662 in_tile->top = in_top; 663 in_tile->height = in_height; 664 665 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 666 out_tile->left = out_top; 667 out_tile->width = out_height; 668 } else { 669 out_tile->top = out_top; 670 out_tile->height = out_height; 671 } 672 } 673 } 674 675 /* 676 * Find the best horizontal and vertical seam positions to split into tiles. 677 * Minimize the fractional part of the input sampling position for the 678 * top / left pixels of each tile. 679 */ 680 static void find_seams(struct ipu_image_convert_ctx *ctx, 681 struct ipu_image_convert_image *in, 682 struct ipu_image_convert_image *out) 683 { 684 struct device *dev = ctx->chan->priv->ipu->dev; 685 unsigned int resized_width = out->base.rect.width; 686 unsigned int resized_height = out->base.rect.height; 687 unsigned int col; 688 unsigned int row; 689 unsigned int in_left_align = tile_left_align(in->fmt); 690 unsigned int in_top_align = tile_top_align(in->fmt); 691 unsigned int out_left_align = tile_left_align(out->fmt); 692 unsigned int out_top_align = tile_top_align(out->fmt); 693 unsigned int out_width_align = tile_width_align(out->type, out->fmt, 694 ctx->rot_mode); 695 unsigned int out_height_align = tile_height_align(out->type, out->fmt, 696 ctx->rot_mode); 697 unsigned int in_right = in->base.rect.width; 698 unsigned int in_bottom = in->base.rect.height; 699 unsigned int out_right = out->base.rect.width; 700 unsigned int out_bottom = out->base.rect.height; 701 unsigned int flipped_out_left; 702 unsigned int flipped_out_top; 703 704 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 705 /* Switch width/height and align top left to IRT block size */ 706 resized_width = out->base.rect.height; 707 resized_height = out->base.rect.width; 708 out_left_align = out_height_align; 709 out_top_align = out_width_align; 710 out_width_align = out_left_align; 711 out_height_align = out_top_align; 712 out_right = out->base.rect.height; 713 out_bottom = out->base.rect.width; 714 } 715 716 for (col = in->num_cols - 1; col > 0; col--) { 717 bool allow_in_overshoot = ipu_rot_mode_is_irt(ctx->rot_mode) || 718 !(ctx->rot_mode & IPU_ROT_BIT_HFLIP); 719 bool allow_out_overshoot = (col < in->num_cols - 1) && 720 !(ctx->rot_mode & IPU_ROT_BIT_HFLIP); 721 unsigned int out_start; 722 unsigned int out_end; 723 unsigned int in_left; 724 unsigned int out_left; 725 726 /* 727 * Align input width to burst length if the scaling step flips 728 * horizontally. 729 */ 730 731 /* Start within 1024 pixels of the right edge */ 732 out_start = max_t(int, 0, out_right - 1024); 733 /* End before having to add more columns to the left */ 734 out_end = min_t(unsigned int, out_right, col * 1024); 735 736 find_best_seam(ctx, out_start, out_end, 737 in_right, out_right, 738 in_left_align, out_left_align, 739 allow_in_overshoot ? 1 : 8 /* burst length */, 740 allow_out_overshoot ? 1 : out_width_align, 741 ctx->downsize_coeff_h, ctx->image_resize_coeff_h, 742 &in_left, &out_left); 743 744 if (ctx->rot_mode & IPU_ROT_BIT_HFLIP) 745 flipped_out_left = resized_width - out_right; 746 else 747 flipped_out_left = out_left; 748 749 fill_tile_column(ctx, col, in, in_left, in_right - in_left, 750 out, flipped_out_left, out_right - out_left); 751 752 dev_dbg(dev, "%s: col %u: %u, %u -> %u, %u\n", __func__, col, 753 in_left, in_right - in_left, 754 flipped_out_left, out_right - out_left); 755 756 in_right = in_left; 757 out_right = out_left; 758 } 759 760 flipped_out_left = (ctx->rot_mode & IPU_ROT_BIT_HFLIP) ? 761 resized_width - out_right : 0; 762 763 fill_tile_column(ctx, 0, in, 0, in_right, 764 out, flipped_out_left, out_right); 765 766 dev_dbg(dev, "%s: col 0: 0, %u -> %u, %u\n", __func__, 767 in_right, flipped_out_left, out_right); 768 769 for (row = in->num_rows - 1; row > 0; row--) { 770 bool allow_overshoot = row < in->num_rows - 1; 771 unsigned int out_start; 772 unsigned int out_end; 773 unsigned int in_top; 774 unsigned int out_top; 775 776 /* Start within 1024 lines of the bottom edge */ 777 out_start = max_t(int, 0, out_bottom - 1024); 778 /* End before having to add more rows above */ 779 out_end = min_t(unsigned int, out_bottom, row * 1024); 780 781 find_best_seam(ctx, out_start, out_end, 782 in_bottom, out_bottom, 783 in_top_align, out_top_align, 784 1, allow_overshoot ? 1 : out_height_align, 785 ctx->downsize_coeff_v, ctx->image_resize_coeff_v, 786 &in_top, &out_top); 787 788 if ((ctx->rot_mode & IPU_ROT_BIT_VFLIP) ^ 789 ipu_rot_mode_is_irt(ctx->rot_mode)) 790 flipped_out_top = resized_height - out_bottom; 791 else 792 flipped_out_top = out_top; 793 794 fill_tile_row(ctx, row, in, in_top, in_bottom - in_top, 795 out, flipped_out_top, out_bottom - out_top); 796 797 dev_dbg(dev, "%s: row %u: %u, %u -> %u, %u\n", __func__, row, 798 in_top, in_bottom - in_top, 799 flipped_out_top, out_bottom - out_top); 800 801 in_bottom = in_top; 802 out_bottom = out_top; 803 } 804 805 if ((ctx->rot_mode & IPU_ROT_BIT_VFLIP) ^ 806 ipu_rot_mode_is_irt(ctx->rot_mode)) 807 flipped_out_top = resized_height - out_bottom; 808 else 809 flipped_out_top = 0; 810 811 fill_tile_row(ctx, 0, in, 0, in_bottom, 812 out, flipped_out_top, out_bottom); 813 814 dev_dbg(dev, "%s: row 0: 0, %u -> %u, %u\n", __func__, 815 in_bottom, flipped_out_top, out_bottom); 816 } 817 818 static void calc_tile_dimensions(struct ipu_image_convert_ctx *ctx, 819 struct ipu_image_convert_image *image) 820 { 821 struct ipu_image_convert_chan *chan = ctx->chan; 822 struct ipu_image_convert_priv *priv = chan->priv; 823 unsigned int i; 824 825 for (i = 0; i < ctx->num_tiles; i++) { 826 struct ipu_image_tile *tile; 827 const unsigned int row = i / image->num_cols; 828 const unsigned int col = i % image->num_cols; 829 830 if (image->type == IMAGE_CONVERT_OUT) 831 tile = &image->tile[ctx->out_tile_map[i]]; 832 else 833 tile = &image->tile[i]; 834 835 tile->size = ((tile->height * image->fmt->bpp) >> 3) * 836 tile->width; 837 838 if (image->fmt->planar) { 839 tile->stride = tile->width; 840 tile->rot_stride = tile->height; 841 } else { 842 tile->stride = 843 (image->fmt->bpp * tile->width) >> 3; 844 tile->rot_stride = 845 (image->fmt->bpp * tile->height) >> 3; 846 } 847 848 dev_dbg(priv->ipu->dev, 849 "task %u: ctx %p: %s@[%u,%u]: %ux%u@%u,%u\n", 850 chan->ic_task, ctx, 851 image->type == IMAGE_CONVERT_IN ? "Input" : "Output", 852 row, col, 853 tile->width, tile->height, tile->left, tile->top); 854 } 855 } 856 857 /* 858 * Use the rotation transformation to find the tile coordinates 859 * (row, col) of a tile in the destination frame that corresponds 860 * to the given tile coordinates of a source frame. The destination 861 * coordinate is then converted to a tile index. 862 */ 863 static int transform_tile_index(struct ipu_image_convert_ctx *ctx, 864 int src_row, int src_col) 865 { 866 struct ipu_image_convert_chan *chan = ctx->chan; 867 struct ipu_image_convert_priv *priv = chan->priv; 868 struct ipu_image_convert_image *s_image = &ctx->in; 869 struct ipu_image_convert_image *d_image = &ctx->out; 870 int dst_row, dst_col; 871 872 /* with no rotation it's a 1:1 mapping */ 873 if (ctx->rot_mode == IPU_ROTATE_NONE) 874 return src_row * s_image->num_cols + src_col; 875 876 /* 877 * before doing the transform, first we have to translate 878 * source row,col for an origin in the center of s_image 879 */ 880 src_row = src_row * 2 - (s_image->num_rows - 1); 881 src_col = src_col * 2 - (s_image->num_cols - 1); 882 883 /* do the rotation transform */ 884 if (ctx->rot_mode & IPU_ROT_BIT_90) { 885 dst_col = -src_row; 886 dst_row = src_col; 887 } else { 888 dst_col = src_col; 889 dst_row = src_row; 890 } 891 892 /* apply flip */ 893 if (ctx->rot_mode & IPU_ROT_BIT_HFLIP) 894 dst_col = -dst_col; 895 if (ctx->rot_mode & IPU_ROT_BIT_VFLIP) 896 dst_row = -dst_row; 897 898 dev_dbg(priv->ipu->dev, "task %u: ctx %p: [%d,%d] --> [%d,%d]\n", 899 chan->ic_task, ctx, src_col, src_row, dst_col, dst_row); 900 901 /* 902 * finally translate dest row,col using an origin in upper 903 * left of d_image 904 */ 905 dst_row += d_image->num_rows - 1; 906 dst_col += d_image->num_cols - 1; 907 dst_row /= 2; 908 dst_col /= 2; 909 910 return dst_row * d_image->num_cols + dst_col; 911 } 912 913 /* 914 * Fill the out_tile_map[] with transformed destination tile indeces. 915 */ 916 static void calc_out_tile_map(struct ipu_image_convert_ctx *ctx) 917 { 918 struct ipu_image_convert_image *s_image = &ctx->in; 919 unsigned int row, col, tile = 0; 920 921 for (row = 0; row < s_image->num_rows; row++) { 922 for (col = 0; col < s_image->num_cols; col++) { 923 ctx->out_tile_map[tile] = 924 transform_tile_index(ctx, row, col); 925 tile++; 926 } 927 } 928 } 929 930 static int calc_tile_offsets_planar(struct ipu_image_convert_ctx *ctx, 931 struct ipu_image_convert_image *image) 932 { 933 struct ipu_image_convert_chan *chan = ctx->chan; 934 struct ipu_image_convert_priv *priv = chan->priv; 935 const struct ipu_image_pixfmt *fmt = image->fmt; 936 unsigned int row, col, tile = 0; 937 u32 H, top, y_stride, uv_stride; 938 u32 uv_row_off, uv_col_off, uv_off, u_off, v_off, tmp; 939 u32 y_row_off, y_col_off, y_off; 940 u32 y_size, uv_size; 941 942 /* setup some convenience vars */ 943 H = image->base.pix.height; 944 945 y_stride = image->stride; 946 uv_stride = y_stride / fmt->uv_width_dec; 947 if (fmt->uv_packed) 948 uv_stride *= 2; 949 950 y_size = H * y_stride; 951 uv_size = y_size / (fmt->uv_width_dec * fmt->uv_height_dec); 952 953 for (row = 0; row < image->num_rows; row++) { 954 top = image->tile[tile].top; 955 y_row_off = top * y_stride; 956 uv_row_off = (top * uv_stride) / fmt->uv_height_dec; 957 958 for (col = 0; col < image->num_cols; col++) { 959 y_col_off = image->tile[tile].left; 960 uv_col_off = y_col_off / fmt->uv_width_dec; 961 if (fmt->uv_packed) 962 uv_col_off *= 2; 963 964 y_off = y_row_off + y_col_off; 965 uv_off = uv_row_off + uv_col_off; 966 967 u_off = y_size - y_off + uv_off; 968 v_off = (fmt->uv_packed) ? 0 : u_off + uv_size; 969 if (fmt->uv_swapped) { 970 tmp = u_off; 971 u_off = v_off; 972 v_off = tmp; 973 } 974 975 image->tile[tile].offset = y_off; 976 image->tile[tile].u_off = u_off; 977 image->tile[tile++].v_off = v_off; 978 979 if ((y_off & 0x7) || (u_off & 0x7) || (v_off & 0x7)) { 980 dev_err(priv->ipu->dev, 981 "task %u: ctx %p: %s@[%d,%d]: " 982 "y_off %08x, u_off %08x, v_off %08x\n", 983 chan->ic_task, ctx, 984 image->type == IMAGE_CONVERT_IN ? 985 "Input" : "Output", row, col, 986 y_off, u_off, v_off); 987 return -EINVAL; 988 } 989 } 990 } 991 992 return 0; 993 } 994 995 static int calc_tile_offsets_packed(struct ipu_image_convert_ctx *ctx, 996 struct ipu_image_convert_image *image) 997 { 998 struct ipu_image_convert_chan *chan = ctx->chan; 999 struct ipu_image_convert_priv *priv = chan->priv; 1000 const struct ipu_image_pixfmt *fmt = image->fmt; 1001 unsigned int row, col, tile = 0; 1002 u32 bpp, stride, offset; 1003 u32 row_off, col_off; 1004 1005 /* setup some convenience vars */ 1006 stride = image->stride; 1007 bpp = fmt->bpp; 1008 1009 for (row = 0; row < image->num_rows; row++) { 1010 row_off = image->tile[tile].top * stride; 1011 1012 for (col = 0; col < image->num_cols; col++) { 1013 col_off = (image->tile[tile].left * bpp) >> 3; 1014 1015 offset = row_off + col_off; 1016 1017 image->tile[tile].offset = offset; 1018 image->tile[tile].u_off = 0; 1019 image->tile[tile++].v_off = 0; 1020 1021 if (offset & 0x7) { 1022 dev_err(priv->ipu->dev, 1023 "task %u: ctx %p: %s@[%d,%d]: " 1024 "phys %08x\n", 1025 chan->ic_task, ctx, 1026 image->type == IMAGE_CONVERT_IN ? 1027 "Input" : "Output", row, col, 1028 row_off + col_off); 1029 return -EINVAL; 1030 } 1031 } 1032 } 1033 1034 return 0; 1035 } 1036 1037 static int calc_tile_offsets(struct ipu_image_convert_ctx *ctx, 1038 struct ipu_image_convert_image *image) 1039 { 1040 if (image->fmt->planar) 1041 return calc_tile_offsets_planar(ctx, image); 1042 1043 return calc_tile_offsets_packed(ctx, image); 1044 } 1045 1046 /* 1047 * Calculate the resizing ratio for the IC main processing section given input 1048 * size, fixed downsizing coefficient, and output size. 1049 * Either round to closest for the next tile's first pixel to minimize seams 1050 * and distortion (for all but right column / bottom row), or round down to 1051 * avoid sampling beyond the edges of the input image for this tile's last 1052 * pixel. 1053 * Returns the resizing coefficient, resizing ratio is 8192.0 / resize_coeff. 1054 */ 1055 static u32 calc_resize_coeff(u32 input_size, u32 downsize_coeff, 1056 u32 output_size, bool allow_overshoot) 1057 { 1058 u32 downsized = input_size >> downsize_coeff; 1059 1060 if (allow_overshoot) 1061 return DIV_ROUND_CLOSEST(8192 * downsized, output_size); 1062 else 1063 return 8192 * (downsized - 1) / (output_size - 1); 1064 } 1065 1066 /* 1067 * Slightly modify resize coefficients per tile to hide the bilinear 1068 * interpolator reset at tile borders, shifting the right / bottom edge 1069 * by up to a half input pixel. This removes noticeable seams between 1070 * tiles at higher upscaling factors. 1071 */ 1072 static void calc_tile_resize_coefficients(struct ipu_image_convert_ctx *ctx) 1073 { 1074 struct ipu_image_convert_chan *chan = ctx->chan; 1075 struct ipu_image_convert_priv *priv = chan->priv; 1076 struct ipu_image_tile *in_tile, *out_tile; 1077 unsigned int col, row, tile_idx; 1078 unsigned int last_output; 1079 1080 for (col = 0; col < ctx->in.num_cols; col++) { 1081 bool closest = (col < ctx->in.num_cols - 1) && 1082 !(ctx->rot_mode & IPU_ROT_BIT_HFLIP); 1083 u32 resized_width; 1084 u32 resize_coeff_h; 1085 1086 tile_idx = col; 1087 in_tile = &ctx->in.tile[tile_idx]; 1088 out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]]; 1089 1090 if (ipu_rot_mode_is_irt(ctx->rot_mode)) 1091 resized_width = out_tile->height; 1092 else 1093 resized_width = out_tile->width; 1094 1095 resize_coeff_h = calc_resize_coeff(in_tile->width, 1096 ctx->downsize_coeff_h, 1097 resized_width, closest); 1098 1099 dev_dbg(priv->ipu->dev, "%s: column %u hscale: *8192/%u\n", 1100 __func__, col, resize_coeff_h); 1101 1102 1103 for (row = 0; row < ctx->in.num_rows; row++) { 1104 tile_idx = row * ctx->in.num_cols + col; 1105 in_tile = &ctx->in.tile[tile_idx]; 1106 out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]]; 1107 1108 /* 1109 * With the horizontal scaling factor known, round up 1110 * resized width (output width or height) to burst size. 1111 */ 1112 if (ipu_rot_mode_is_irt(ctx->rot_mode)) 1113 out_tile->height = round_up(resized_width, 8); 1114 else 1115 out_tile->width = round_up(resized_width, 8); 1116 1117 /* 1118 * Calculate input width from the last accessed input 1119 * pixel given resized width and scaling coefficients. 1120 * Round up to burst size. 1121 */ 1122 last_output = round_up(resized_width, 8) - 1; 1123 if (closest) 1124 last_output++; 1125 in_tile->width = round_up( 1126 (DIV_ROUND_UP(last_output * resize_coeff_h, 1127 8192) + 1) 1128 << ctx->downsize_coeff_h, 8); 1129 } 1130 1131 ctx->resize_coeffs_h[col] = resize_coeff_h; 1132 } 1133 1134 for (row = 0; row < ctx->in.num_rows; row++) { 1135 bool closest = (row < ctx->in.num_rows - 1) && 1136 !(ctx->rot_mode & IPU_ROT_BIT_VFLIP); 1137 u32 resized_height; 1138 u32 resize_coeff_v; 1139 1140 tile_idx = row * ctx->in.num_cols; 1141 in_tile = &ctx->in.tile[tile_idx]; 1142 out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]]; 1143 1144 if (ipu_rot_mode_is_irt(ctx->rot_mode)) 1145 resized_height = out_tile->width; 1146 else 1147 resized_height = out_tile->height; 1148 1149 resize_coeff_v = calc_resize_coeff(in_tile->height, 1150 ctx->downsize_coeff_v, 1151 resized_height, closest); 1152 1153 dev_dbg(priv->ipu->dev, "%s: row %u vscale: *8192/%u\n", 1154 __func__, row, resize_coeff_v); 1155 1156 for (col = 0; col < ctx->in.num_cols; col++) { 1157 tile_idx = row * ctx->in.num_cols + col; 1158 in_tile = &ctx->in.tile[tile_idx]; 1159 out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]]; 1160 1161 /* 1162 * With the vertical scaling factor known, round up 1163 * resized height (output width or height) to IDMAC 1164 * limitations. 1165 */ 1166 if (ipu_rot_mode_is_irt(ctx->rot_mode)) 1167 out_tile->width = round_up(resized_height, 2); 1168 else 1169 out_tile->height = round_up(resized_height, 2); 1170 1171 /* 1172 * Calculate input width from the last accessed input 1173 * pixel given resized height and scaling coefficients. 1174 * Align to IDMAC restrictions. 1175 */ 1176 last_output = round_up(resized_height, 2) - 1; 1177 if (closest) 1178 last_output++; 1179 in_tile->height = round_up( 1180 (DIV_ROUND_UP(last_output * resize_coeff_v, 1181 8192) + 1) 1182 << ctx->downsize_coeff_v, 2); 1183 } 1184 1185 ctx->resize_coeffs_v[row] = resize_coeff_v; 1186 } 1187 } 1188 1189 /* 1190 * return the number of runs in given queue (pending_q or done_q) 1191 * for this context. hold irqlock when calling. 1192 */ 1193 static int get_run_count(struct ipu_image_convert_ctx *ctx, 1194 struct list_head *q) 1195 { 1196 struct ipu_image_convert_run *run; 1197 int count = 0; 1198 1199 lockdep_assert_held(&ctx->chan->irqlock); 1200 1201 list_for_each_entry(run, q, list) { 1202 if (run->ctx == ctx) 1203 count++; 1204 } 1205 1206 return count; 1207 } 1208 1209 static void convert_stop(struct ipu_image_convert_run *run) 1210 { 1211 struct ipu_image_convert_ctx *ctx = run->ctx; 1212 struct ipu_image_convert_chan *chan = ctx->chan; 1213 struct ipu_image_convert_priv *priv = chan->priv; 1214 1215 dev_dbg(priv->ipu->dev, "%s: task %u: stopping ctx %p run %p\n", 1216 __func__, chan->ic_task, ctx, run); 1217 1218 /* disable IC tasks and the channels */ 1219 ipu_ic_task_disable(chan->ic); 1220 ipu_idmac_disable_channel(chan->in_chan); 1221 ipu_idmac_disable_channel(chan->out_chan); 1222 1223 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 1224 ipu_idmac_disable_channel(chan->rotation_in_chan); 1225 ipu_idmac_disable_channel(chan->rotation_out_chan); 1226 ipu_idmac_unlink(chan->out_chan, chan->rotation_in_chan); 1227 } 1228 1229 ipu_ic_disable(chan->ic); 1230 } 1231 1232 static void init_idmac_channel(struct ipu_image_convert_ctx *ctx, 1233 struct ipuv3_channel *channel, 1234 struct ipu_image_convert_image *image, 1235 enum ipu_rotate_mode rot_mode, 1236 bool rot_swap_width_height, 1237 unsigned int tile) 1238 { 1239 struct ipu_image_convert_chan *chan = ctx->chan; 1240 unsigned int burst_size; 1241 u32 width, height, stride; 1242 dma_addr_t addr0, addr1 = 0; 1243 struct ipu_image tile_image; 1244 unsigned int tile_idx[2]; 1245 1246 if (image->type == IMAGE_CONVERT_OUT) { 1247 tile_idx[0] = ctx->out_tile_map[tile]; 1248 tile_idx[1] = ctx->out_tile_map[1]; 1249 } else { 1250 tile_idx[0] = tile; 1251 tile_idx[1] = 1; 1252 } 1253 1254 if (rot_swap_width_height) { 1255 width = image->tile[tile_idx[0]].height; 1256 height = image->tile[tile_idx[0]].width; 1257 stride = image->tile[tile_idx[0]].rot_stride; 1258 addr0 = ctx->rot_intermediate[0].phys; 1259 if (ctx->double_buffering) 1260 addr1 = ctx->rot_intermediate[1].phys; 1261 } else { 1262 width = image->tile[tile_idx[0]].width; 1263 height = image->tile[tile_idx[0]].height; 1264 stride = image->stride; 1265 addr0 = image->base.phys0 + 1266 image->tile[tile_idx[0]].offset; 1267 if (ctx->double_buffering) 1268 addr1 = image->base.phys0 + 1269 image->tile[tile_idx[1]].offset; 1270 } 1271 1272 ipu_cpmem_zero(channel); 1273 1274 memset(&tile_image, 0, sizeof(tile_image)); 1275 tile_image.pix.width = tile_image.rect.width = width; 1276 tile_image.pix.height = tile_image.rect.height = height; 1277 tile_image.pix.bytesperline = stride; 1278 tile_image.pix.pixelformat = image->fmt->fourcc; 1279 tile_image.phys0 = addr0; 1280 tile_image.phys1 = addr1; 1281 if (image->fmt->planar && !rot_swap_width_height) { 1282 tile_image.u_offset = image->tile[tile_idx[0]].u_off; 1283 tile_image.v_offset = image->tile[tile_idx[0]].v_off; 1284 } 1285 1286 ipu_cpmem_set_image(channel, &tile_image); 1287 1288 if (rot_mode) 1289 ipu_cpmem_set_rotation(channel, rot_mode); 1290 1291 /* 1292 * Skip writing U and V components to odd rows in the output 1293 * channels for planar 4:2:0. 1294 */ 1295 if ((channel == chan->out_chan || 1296 channel == chan->rotation_out_chan) && 1297 image->fmt->planar && image->fmt->uv_height_dec == 2) 1298 ipu_cpmem_skip_odd_chroma_rows(channel); 1299 1300 if (channel == chan->rotation_in_chan || 1301 channel == chan->rotation_out_chan) { 1302 burst_size = 8; 1303 ipu_cpmem_set_block_mode(channel); 1304 } else 1305 burst_size = (width % 16) ? 8 : 16; 1306 1307 ipu_cpmem_set_burstsize(channel, burst_size); 1308 1309 ipu_ic_task_idma_init(chan->ic, channel, width, height, 1310 burst_size, rot_mode); 1311 1312 /* 1313 * Setting a non-zero AXI ID collides with the PRG AXI snooping, so 1314 * only do this when there is no PRG present. 1315 */ 1316 if (!channel->ipu->prg_priv) 1317 ipu_cpmem_set_axi_id(channel, 1); 1318 1319 ipu_idmac_set_double_buffer(channel, ctx->double_buffering); 1320 } 1321 1322 static int convert_start(struct ipu_image_convert_run *run, unsigned int tile) 1323 { 1324 struct ipu_image_convert_ctx *ctx = run->ctx; 1325 struct ipu_image_convert_chan *chan = ctx->chan; 1326 struct ipu_image_convert_priv *priv = chan->priv; 1327 struct ipu_image_convert_image *s_image = &ctx->in; 1328 struct ipu_image_convert_image *d_image = &ctx->out; 1329 unsigned int dst_tile = ctx->out_tile_map[tile]; 1330 unsigned int dest_width, dest_height; 1331 unsigned int col, row; 1332 u32 rsc; 1333 int ret; 1334 1335 dev_dbg(priv->ipu->dev, "%s: task %u: starting ctx %p run %p tile %u -> %u\n", 1336 __func__, chan->ic_task, ctx, run, tile, dst_tile); 1337 1338 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 1339 /* swap width/height for resizer */ 1340 dest_width = d_image->tile[dst_tile].height; 1341 dest_height = d_image->tile[dst_tile].width; 1342 } else { 1343 dest_width = d_image->tile[dst_tile].width; 1344 dest_height = d_image->tile[dst_tile].height; 1345 } 1346 1347 row = tile / s_image->num_cols; 1348 col = tile % s_image->num_cols; 1349 1350 rsc = (ctx->downsize_coeff_v << 30) | 1351 (ctx->resize_coeffs_v[row] << 16) | 1352 (ctx->downsize_coeff_h << 14) | 1353 (ctx->resize_coeffs_h[col]); 1354 1355 dev_dbg(priv->ipu->dev, "%s: %ux%u -> %ux%u (rsc = 0x%x)\n", 1356 __func__, s_image->tile[tile].width, 1357 s_image->tile[tile].height, dest_width, dest_height, rsc); 1358 1359 /* setup the IC resizer and CSC */ 1360 ret = ipu_ic_task_init_rsc(chan->ic, &ctx->csc, 1361 s_image->tile[tile].width, 1362 s_image->tile[tile].height, 1363 dest_width, 1364 dest_height, 1365 rsc); 1366 if (ret) { 1367 dev_err(priv->ipu->dev, "ipu_ic_task_init failed, %d\n", ret); 1368 return ret; 1369 } 1370 1371 /* init the source MEM-->IC PP IDMAC channel */ 1372 init_idmac_channel(ctx, chan->in_chan, s_image, 1373 IPU_ROTATE_NONE, false, tile); 1374 1375 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 1376 /* init the IC PP-->MEM IDMAC channel */ 1377 init_idmac_channel(ctx, chan->out_chan, d_image, 1378 IPU_ROTATE_NONE, true, tile); 1379 1380 /* init the MEM-->IC PP ROT IDMAC channel */ 1381 init_idmac_channel(ctx, chan->rotation_in_chan, d_image, 1382 ctx->rot_mode, true, tile); 1383 1384 /* init the destination IC PP ROT-->MEM IDMAC channel */ 1385 init_idmac_channel(ctx, chan->rotation_out_chan, d_image, 1386 IPU_ROTATE_NONE, false, tile); 1387 1388 /* now link IC PP-->MEM to MEM-->IC PP ROT */ 1389 ipu_idmac_link(chan->out_chan, chan->rotation_in_chan); 1390 } else { 1391 /* init the destination IC PP-->MEM IDMAC channel */ 1392 init_idmac_channel(ctx, chan->out_chan, d_image, 1393 ctx->rot_mode, false, tile); 1394 } 1395 1396 /* enable the IC */ 1397 ipu_ic_enable(chan->ic); 1398 1399 /* set buffers ready */ 1400 ipu_idmac_select_buffer(chan->in_chan, 0); 1401 ipu_idmac_select_buffer(chan->out_chan, 0); 1402 if (ipu_rot_mode_is_irt(ctx->rot_mode)) 1403 ipu_idmac_select_buffer(chan->rotation_out_chan, 0); 1404 if (ctx->double_buffering) { 1405 ipu_idmac_select_buffer(chan->in_chan, 1); 1406 ipu_idmac_select_buffer(chan->out_chan, 1); 1407 if (ipu_rot_mode_is_irt(ctx->rot_mode)) 1408 ipu_idmac_select_buffer(chan->rotation_out_chan, 1); 1409 } 1410 1411 /* enable the channels! */ 1412 ipu_idmac_enable_channel(chan->in_chan); 1413 ipu_idmac_enable_channel(chan->out_chan); 1414 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 1415 ipu_idmac_enable_channel(chan->rotation_in_chan); 1416 ipu_idmac_enable_channel(chan->rotation_out_chan); 1417 } 1418 1419 ipu_ic_task_enable(chan->ic); 1420 1421 ipu_cpmem_dump(chan->in_chan); 1422 ipu_cpmem_dump(chan->out_chan); 1423 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 1424 ipu_cpmem_dump(chan->rotation_in_chan); 1425 ipu_cpmem_dump(chan->rotation_out_chan); 1426 } 1427 1428 ipu_dump(priv->ipu); 1429 1430 return 0; 1431 } 1432 1433 /* hold irqlock when calling */ 1434 static int do_run(struct ipu_image_convert_run *run) 1435 { 1436 struct ipu_image_convert_ctx *ctx = run->ctx; 1437 struct ipu_image_convert_chan *chan = ctx->chan; 1438 1439 lockdep_assert_held(&chan->irqlock); 1440 1441 ctx->in.base.phys0 = run->in_phys; 1442 ctx->out.base.phys0 = run->out_phys; 1443 1444 ctx->cur_buf_num = 0; 1445 ctx->next_tile = 1; 1446 1447 /* remove run from pending_q and set as current */ 1448 list_del(&run->list); 1449 chan->current_run = run; 1450 1451 return convert_start(run, 0); 1452 } 1453 1454 /* hold irqlock when calling */ 1455 static void run_next(struct ipu_image_convert_chan *chan) 1456 { 1457 struct ipu_image_convert_priv *priv = chan->priv; 1458 struct ipu_image_convert_run *run, *tmp; 1459 int ret; 1460 1461 lockdep_assert_held(&chan->irqlock); 1462 1463 list_for_each_entry_safe(run, tmp, &chan->pending_q, list) { 1464 /* skip contexts that are aborting */ 1465 if (run->ctx->aborting) { 1466 dev_dbg(priv->ipu->dev, 1467 "%s: task %u: skipping aborting ctx %p run %p\n", 1468 __func__, chan->ic_task, run->ctx, run); 1469 continue; 1470 } 1471 1472 ret = do_run(run); 1473 if (!ret) 1474 break; 1475 1476 /* 1477 * something went wrong with start, add the run 1478 * to done q and continue to the next run in the 1479 * pending q. 1480 */ 1481 run->status = ret; 1482 list_add_tail(&run->list, &chan->done_q); 1483 chan->current_run = NULL; 1484 } 1485 } 1486 1487 static void empty_done_q(struct ipu_image_convert_chan *chan) 1488 { 1489 struct ipu_image_convert_priv *priv = chan->priv; 1490 struct ipu_image_convert_run *run; 1491 unsigned long flags; 1492 1493 spin_lock_irqsave(&chan->irqlock, flags); 1494 1495 while (!list_empty(&chan->done_q)) { 1496 run = list_entry(chan->done_q.next, 1497 struct ipu_image_convert_run, 1498 list); 1499 1500 list_del(&run->list); 1501 1502 dev_dbg(priv->ipu->dev, 1503 "%s: task %u: completing ctx %p run %p with %d\n", 1504 __func__, chan->ic_task, run->ctx, run, run->status); 1505 1506 /* call the completion callback and free the run */ 1507 spin_unlock_irqrestore(&chan->irqlock, flags); 1508 run->ctx->complete(run, run->ctx->complete_context); 1509 spin_lock_irqsave(&chan->irqlock, flags); 1510 } 1511 1512 spin_unlock_irqrestore(&chan->irqlock, flags); 1513 } 1514 1515 /* 1516 * the bottom half thread clears out the done_q, calling the 1517 * completion handler for each. 1518 */ 1519 static irqreturn_t do_bh(int irq, void *dev_id) 1520 { 1521 struct ipu_image_convert_chan *chan = dev_id; 1522 struct ipu_image_convert_priv *priv = chan->priv; 1523 struct ipu_image_convert_ctx *ctx; 1524 unsigned long flags; 1525 1526 dev_dbg(priv->ipu->dev, "%s: task %u: enter\n", __func__, 1527 chan->ic_task); 1528 1529 empty_done_q(chan); 1530 1531 spin_lock_irqsave(&chan->irqlock, flags); 1532 1533 /* 1534 * the done_q is cleared out, signal any contexts 1535 * that are aborting that abort can complete. 1536 */ 1537 list_for_each_entry(ctx, &chan->ctx_list, list) { 1538 if (ctx->aborting) { 1539 dev_dbg(priv->ipu->dev, 1540 "%s: task %u: signaling abort for ctx %p\n", 1541 __func__, chan->ic_task, ctx); 1542 complete_all(&ctx->aborted); 1543 } 1544 } 1545 1546 spin_unlock_irqrestore(&chan->irqlock, flags); 1547 1548 dev_dbg(priv->ipu->dev, "%s: task %u: exit\n", __func__, 1549 chan->ic_task); 1550 1551 return IRQ_HANDLED; 1552 } 1553 1554 static bool ic_settings_changed(struct ipu_image_convert_ctx *ctx) 1555 { 1556 unsigned int cur_tile = ctx->next_tile - 1; 1557 unsigned int next_tile = ctx->next_tile; 1558 1559 if (ctx->resize_coeffs_h[cur_tile % ctx->in.num_cols] != 1560 ctx->resize_coeffs_h[next_tile % ctx->in.num_cols] || 1561 ctx->resize_coeffs_v[cur_tile / ctx->in.num_cols] != 1562 ctx->resize_coeffs_v[next_tile / ctx->in.num_cols] || 1563 ctx->in.tile[cur_tile].width != ctx->in.tile[next_tile].width || 1564 ctx->in.tile[cur_tile].height != ctx->in.tile[next_tile].height || 1565 ctx->out.tile[cur_tile].width != ctx->out.tile[next_tile].width || 1566 ctx->out.tile[cur_tile].height != ctx->out.tile[next_tile].height) 1567 return true; 1568 1569 return false; 1570 } 1571 1572 /* hold irqlock when calling */ 1573 static irqreturn_t do_irq(struct ipu_image_convert_run *run) 1574 { 1575 struct ipu_image_convert_ctx *ctx = run->ctx; 1576 struct ipu_image_convert_chan *chan = ctx->chan; 1577 struct ipu_image_tile *src_tile, *dst_tile; 1578 struct ipu_image_convert_image *s_image = &ctx->in; 1579 struct ipu_image_convert_image *d_image = &ctx->out; 1580 struct ipuv3_channel *outch; 1581 unsigned int dst_idx; 1582 1583 lockdep_assert_held(&chan->irqlock); 1584 1585 outch = ipu_rot_mode_is_irt(ctx->rot_mode) ? 1586 chan->rotation_out_chan : chan->out_chan; 1587 1588 /* 1589 * It is difficult to stop the channel DMA before the channels 1590 * enter the paused state. Without double-buffering the channels 1591 * are always in a paused state when the EOF irq occurs, so it 1592 * is safe to stop the channels now. For double-buffering we 1593 * just ignore the abort until the operation completes, when it 1594 * is safe to shut down. 1595 */ 1596 if (ctx->aborting && !ctx->double_buffering) { 1597 convert_stop(run); 1598 run->status = -EIO; 1599 goto done; 1600 } 1601 1602 if (ctx->next_tile == ctx->num_tiles) { 1603 /* 1604 * the conversion is complete 1605 */ 1606 convert_stop(run); 1607 run->status = 0; 1608 goto done; 1609 } 1610 1611 /* 1612 * not done, place the next tile buffers. 1613 */ 1614 if (!ctx->double_buffering) { 1615 if (ic_settings_changed(ctx)) { 1616 convert_stop(run); 1617 convert_start(run, ctx->next_tile); 1618 } else { 1619 src_tile = &s_image->tile[ctx->next_tile]; 1620 dst_idx = ctx->out_tile_map[ctx->next_tile]; 1621 dst_tile = &d_image->tile[dst_idx]; 1622 1623 ipu_cpmem_set_buffer(chan->in_chan, 0, 1624 s_image->base.phys0 + 1625 src_tile->offset); 1626 ipu_cpmem_set_buffer(outch, 0, 1627 d_image->base.phys0 + 1628 dst_tile->offset); 1629 if (s_image->fmt->planar) 1630 ipu_cpmem_set_uv_offset(chan->in_chan, 1631 src_tile->u_off, 1632 src_tile->v_off); 1633 if (d_image->fmt->planar) 1634 ipu_cpmem_set_uv_offset(outch, 1635 dst_tile->u_off, 1636 dst_tile->v_off); 1637 1638 ipu_idmac_select_buffer(chan->in_chan, 0); 1639 ipu_idmac_select_buffer(outch, 0); 1640 } 1641 } else if (ctx->next_tile < ctx->num_tiles - 1) { 1642 1643 src_tile = &s_image->tile[ctx->next_tile + 1]; 1644 dst_idx = ctx->out_tile_map[ctx->next_tile + 1]; 1645 dst_tile = &d_image->tile[dst_idx]; 1646 1647 ipu_cpmem_set_buffer(chan->in_chan, ctx->cur_buf_num, 1648 s_image->base.phys0 + src_tile->offset); 1649 ipu_cpmem_set_buffer(outch, ctx->cur_buf_num, 1650 d_image->base.phys0 + dst_tile->offset); 1651 1652 ipu_idmac_select_buffer(chan->in_chan, ctx->cur_buf_num); 1653 ipu_idmac_select_buffer(outch, ctx->cur_buf_num); 1654 1655 ctx->cur_buf_num ^= 1; 1656 } 1657 1658 ctx->next_tile++; 1659 return IRQ_HANDLED; 1660 done: 1661 list_add_tail(&run->list, &chan->done_q); 1662 chan->current_run = NULL; 1663 run_next(chan); 1664 return IRQ_WAKE_THREAD; 1665 } 1666 1667 static irqreturn_t norotate_irq(int irq, void *data) 1668 { 1669 struct ipu_image_convert_chan *chan = data; 1670 struct ipu_image_convert_ctx *ctx; 1671 struct ipu_image_convert_run *run; 1672 unsigned long flags; 1673 irqreturn_t ret; 1674 1675 spin_lock_irqsave(&chan->irqlock, flags); 1676 1677 /* get current run and its context */ 1678 run = chan->current_run; 1679 if (!run) { 1680 ret = IRQ_NONE; 1681 goto out; 1682 } 1683 1684 ctx = run->ctx; 1685 1686 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 1687 /* this is a rotation operation, just ignore */ 1688 spin_unlock_irqrestore(&chan->irqlock, flags); 1689 return IRQ_HANDLED; 1690 } 1691 1692 ret = do_irq(run); 1693 out: 1694 spin_unlock_irqrestore(&chan->irqlock, flags); 1695 return ret; 1696 } 1697 1698 static irqreturn_t rotate_irq(int irq, void *data) 1699 { 1700 struct ipu_image_convert_chan *chan = data; 1701 struct ipu_image_convert_priv *priv = chan->priv; 1702 struct ipu_image_convert_ctx *ctx; 1703 struct ipu_image_convert_run *run; 1704 unsigned long flags; 1705 irqreturn_t ret; 1706 1707 spin_lock_irqsave(&chan->irqlock, flags); 1708 1709 /* get current run and its context */ 1710 run = chan->current_run; 1711 if (!run) { 1712 ret = IRQ_NONE; 1713 goto out; 1714 } 1715 1716 ctx = run->ctx; 1717 1718 if (!ipu_rot_mode_is_irt(ctx->rot_mode)) { 1719 /* this was NOT a rotation operation, shouldn't happen */ 1720 dev_err(priv->ipu->dev, "Unexpected rotation interrupt\n"); 1721 spin_unlock_irqrestore(&chan->irqlock, flags); 1722 return IRQ_HANDLED; 1723 } 1724 1725 ret = do_irq(run); 1726 out: 1727 spin_unlock_irqrestore(&chan->irqlock, flags); 1728 return ret; 1729 } 1730 1731 /* 1732 * try to force the completion of runs for this ctx. Called when 1733 * abort wait times out in ipu_image_convert_abort(). 1734 */ 1735 static void force_abort(struct ipu_image_convert_ctx *ctx) 1736 { 1737 struct ipu_image_convert_chan *chan = ctx->chan; 1738 struct ipu_image_convert_run *run; 1739 unsigned long flags; 1740 1741 spin_lock_irqsave(&chan->irqlock, flags); 1742 1743 run = chan->current_run; 1744 if (run && run->ctx == ctx) { 1745 convert_stop(run); 1746 run->status = -EIO; 1747 list_add_tail(&run->list, &chan->done_q); 1748 chan->current_run = NULL; 1749 run_next(chan); 1750 } 1751 1752 spin_unlock_irqrestore(&chan->irqlock, flags); 1753 1754 empty_done_q(chan); 1755 } 1756 1757 static void release_ipu_resources(struct ipu_image_convert_chan *chan) 1758 { 1759 if (chan->out_eof_irq >= 0) 1760 free_irq(chan->out_eof_irq, chan); 1761 if (chan->rot_out_eof_irq >= 0) 1762 free_irq(chan->rot_out_eof_irq, chan); 1763 1764 if (!IS_ERR_OR_NULL(chan->in_chan)) 1765 ipu_idmac_put(chan->in_chan); 1766 if (!IS_ERR_OR_NULL(chan->out_chan)) 1767 ipu_idmac_put(chan->out_chan); 1768 if (!IS_ERR_OR_NULL(chan->rotation_in_chan)) 1769 ipu_idmac_put(chan->rotation_in_chan); 1770 if (!IS_ERR_OR_NULL(chan->rotation_out_chan)) 1771 ipu_idmac_put(chan->rotation_out_chan); 1772 if (!IS_ERR_OR_NULL(chan->ic)) 1773 ipu_ic_put(chan->ic); 1774 1775 chan->in_chan = chan->out_chan = chan->rotation_in_chan = 1776 chan->rotation_out_chan = NULL; 1777 chan->out_eof_irq = chan->rot_out_eof_irq = -1; 1778 } 1779 1780 static int get_ipu_resources(struct ipu_image_convert_chan *chan) 1781 { 1782 const struct ipu_image_convert_dma_chan *dma = chan->dma_ch; 1783 struct ipu_image_convert_priv *priv = chan->priv; 1784 int ret; 1785 1786 /* get IC */ 1787 chan->ic = ipu_ic_get(priv->ipu, chan->ic_task); 1788 if (IS_ERR(chan->ic)) { 1789 dev_err(priv->ipu->dev, "could not acquire IC\n"); 1790 ret = PTR_ERR(chan->ic); 1791 goto err; 1792 } 1793 1794 /* get IDMAC channels */ 1795 chan->in_chan = ipu_idmac_get(priv->ipu, dma->in); 1796 chan->out_chan = ipu_idmac_get(priv->ipu, dma->out); 1797 if (IS_ERR(chan->in_chan) || IS_ERR(chan->out_chan)) { 1798 dev_err(priv->ipu->dev, "could not acquire idmac channels\n"); 1799 ret = -EBUSY; 1800 goto err; 1801 } 1802 1803 chan->rotation_in_chan = ipu_idmac_get(priv->ipu, dma->rot_in); 1804 chan->rotation_out_chan = ipu_idmac_get(priv->ipu, dma->rot_out); 1805 if (IS_ERR(chan->rotation_in_chan) || IS_ERR(chan->rotation_out_chan)) { 1806 dev_err(priv->ipu->dev, 1807 "could not acquire idmac rotation channels\n"); 1808 ret = -EBUSY; 1809 goto err; 1810 } 1811 1812 /* acquire the EOF interrupts */ 1813 chan->out_eof_irq = ipu_idmac_channel_irq(priv->ipu, 1814 chan->out_chan, 1815 IPU_IRQ_EOF); 1816 1817 ret = request_threaded_irq(chan->out_eof_irq, norotate_irq, do_bh, 1818 0, "ipu-ic", chan); 1819 if (ret < 0) { 1820 dev_err(priv->ipu->dev, "could not acquire irq %d\n", 1821 chan->out_eof_irq); 1822 chan->out_eof_irq = -1; 1823 goto err; 1824 } 1825 1826 chan->rot_out_eof_irq = ipu_idmac_channel_irq(priv->ipu, 1827 chan->rotation_out_chan, 1828 IPU_IRQ_EOF); 1829 1830 ret = request_threaded_irq(chan->rot_out_eof_irq, rotate_irq, do_bh, 1831 0, "ipu-ic", chan); 1832 if (ret < 0) { 1833 dev_err(priv->ipu->dev, "could not acquire irq %d\n", 1834 chan->rot_out_eof_irq); 1835 chan->rot_out_eof_irq = -1; 1836 goto err; 1837 } 1838 1839 return 0; 1840 err: 1841 release_ipu_resources(chan); 1842 return ret; 1843 } 1844 1845 static int fill_image(struct ipu_image_convert_ctx *ctx, 1846 struct ipu_image_convert_image *ic_image, 1847 struct ipu_image *image, 1848 enum ipu_image_convert_type type) 1849 { 1850 struct ipu_image_convert_priv *priv = ctx->chan->priv; 1851 1852 ic_image->base = *image; 1853 ic_image->type = type; 1854 1855 ic_image->fmt = get_format(image->pix.pixelformat); 1856 if (!ic_image->fmt) { 1857 dev_err(priv->ipu->dev, "pixelformat not supported for %s\n", 1858 type == IMAGE_CONVERT_OUT ? "Output" : "Input"); 1859 return -EINVAL; 1860 } 1861 1862 if (ic_image->fmt->planar) 1863 ic_image->stride = ic_image->base.pix.width; 1864 else 1865 ic_image->stride = ic_image->base.pix.bytesperline; 1866 1867 return 0; 1868 } 1869 1870 /* borrowed from drivers/media/v4l2-core/v4l2-common.c */ 1871 static unsigned int clamp_align(unsigned int x, unsigned int min, 1872 unsigned int max, unsigned int align) 1873 { 1874 /* Bits that must be zero to be aligned */ 1875 unsigned int mask = ~((1 << align) - 1); 1876 1877 /* Clamp to aligned min and max */ 1878 x = clamp(x, (min + ~mask) & mask, max & mask); 1879 1880 /* Round to nearest aligned value */ 1881 if (align) 1882 x = (x + (1 << (align - 1))) & mask; 1883 1884 return x; 1885 } 1886 1887 /* Adjusts input/output images to IPU restrictions */ 1888 void ipu_image_convert_adjust(struct ipu_image *in, struct ipu_image *out, 1889 enum ipu_rotate_mode rot_mode) 1890 { 1891 const struct ipu_image_pixfmt *infmt, *outfmt; 1892 u32 w_align_out, h_align_out; 1893 u32 w_align_in, h_align_in; 1894 1895 infmt = get_format(in->pix.pixelformat); 1896 outfmt = get_format(out->pix.pixelformat); 1897 1898 /* set some default pixel formats if needed */ 1899 if (!infmt) { 1900 in->pix.pixelformat = V4L2_PIX_FMT_RGB24; 1901 infmt = get_format(V4L2_PIX_FMT_RGB24); 1902 } 1903 if (!outfmt) { 1904 out->pix.pixelformat = V4L2_PIX_FMT_RGB24; 1905 outfmt = get_format(V4L2_PIX_FMT_RGB24); 1906 } 1907 1908 /* image converter does not handle fields */ 1909 in->pix.field = out->pix.field = V4L2_FIELD_NONE; 1910 1911 /* resizer cannot downsize more than 4:1 */ 1912 if (ipu_rot_mode_is_irt(rot_mode)) { 1913 out->pix.height = max_t(__u32, out->pix.height, 1914 in->pix.width / 4); 1915 out->pix.width = max_t(__u32, out->pix.width, 1916 in->pix.height / 4); 1917 } else { 1918 out->pix.width = max_t(__u32, out->pix.width, 1919 in->pix.width / 4); 1920 out->pix.height = max_t(__u32, out->pix.height, 1921 in->pix.height / 4); 1922 } 1923 1924 /* align input width/height */ 1925 w_align_in = ilog2(tile_width_align(IMAGE_CONVERT_IN, infmt, 1926 rot_mode)); 1927 h_align_in = ilog2(tile_height_align(IMAGE_CONVERT_IN, infmt, 1928 rot_mode)); 1929 in->pix.width = clamp_align(in->pix.width, MIN_W, MAX_W, 1930 w_align_in); 1931 in->pix.height = clamp_align(in->pix.height, MIN_H, MAX_H, 1932 h_align_in); 1933 1934 /* align output width/height */ 1935 w_align_out = ilog2(tile_width_align(IMAGE_CONVERT_OUT, outfmt, 1936 rot_mode)); 1937 h_align_out = ilog2(tile_height_align(IMAGE_CONVERT_OUT, outfmt, 1938 rot_mode)); 1939 out->pix.width = clamp_align(out->pix.width, MIN_W, MAX_W, 1940 w_align_out); 1941 out->pix.height = clamp_align(out->pix.height, MIN_H, MAX_H, 1942 h_align_out); 1943 1944 /* set input/output strides and image sizes */ 1945 in->pix.bytesperline = infmt->planar ? 1946 clamp_align(in->pix.width, 2 << w_align_in, MAX_W, 1947 w_align_in) : 1948 clamp_align((in->pix.width * infmt->bpp) >> 3, 1949 ((2 << w_align_in) * infmt->bpp) >> 3, 1950 (MAX_W * infmt->bpp) >> 3, 1951 w_align_in); 1952 in->pix.sizeimage = infmt->planar ? 1953 (in->pix.height * in->pix.bytesperline * infmt->bpp) >> 3 : 1954 in->pix.height * in->pix.bytesperline; 1955 out->pix.bytesperline = outfmt->planar ? out->pix.width : 1956 (out->pix.width * outfmt->bpp) >> 3; 1957 out->pix.sizeimage = outfmt->planar ? 1958 (out->pix.height * out->pix.bytesperline * outfmt->bpp) >> 3 : 1959 out->pix.height * out->pix.bytesperline; 1960 } 1961 EXPORT_SYMBOL_GPL(ipu_image_convert_adjust); 1962 1963 /* 1964 * this is used by ipu_image_convert_prepare() to verify set input and 1965 * output images are valid before starting the conversion. Clients can 1966 * also call it before calling ipu_image_convert_prepare(). 1967 */ 1968 int ipu_image_convert_verify(struct ipu_image *in, struct ipu_image *out, 1969 enum ipu_rotate_mode rot_mode) 1970 { 1971 struct ipu_image testin, testout; 1972 1973 testin = *in; 1974 testout = *out; 1975 1976 ipu_image_convert_adjust(&testin, &testout, rot_mode); 1977 1978 if (testin.pix.width != in->pix.width || 1979 testin.pix.height != in->pix.height || 1980 testout.pix.width != out->pix.width || 1981 testout.pix.height != out->pix.height) 1982 return -EINVAL; 1983 1984 return 0; 1985 } 1986 EXPORT_SYMBOL_GPL(ipu_image_convert_verify); 1987 1988 /* 1989 * Call ipu_image_convert_prepare() to prepare for the conversion of 1990 * given images and rotation mode. Returns a new conversion context. 1991 */ 1992 struct ipu_image_convert_ctx * 1993 ipu_image_convert_prepare(struct ipu_soc *ipu, enum ipu_ic_task ic_task, 1994 struct ipu_image *in, struct ipu_image *out, 1995 enum ipu_rotate_mode rot_mode, 1996 ipu_image_convert_cb_t complete, 1997 void *complete_context) 1998 { 1999 struct ipu_image_convert_priv *priv = ipu->image_convert_priv; 2000 struct ipu_image_convert_image *s_image, *d_image; 2001 struct ipu_image_convert_chan *chan; 2002 struct ipu_image_convert_ctx *ctx; 2003 unsigned long flags; 2004 unsigned int i; 2005 bool get_res; 2006 int ret; 2007 2008 if (!in || !out || !complete || 2009 (ic_task != IC_TASK_VIEWFINDER && 2010 ic_task != IC_TASK_POST_PROCESSOR)) 2011 return ERR_PTR(-EINVAL); 2012 2013 /* verify the in/out images before continuing */ 2014 ret = ipu_image_convert_verify(in, out, rot_mode); 2015 if (ret) { 2016 dev_err(priv->ipu->dev, "%s: in/out formats invalid\n", 2017 __func__); 2018 return ERR_PTR(ret); 2019 } 2020 2021 chan = &priv->chan[ic_task]; 2022 2023 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 2024 if (!ctx) 2025 return ERR_PTR(-ENOMEM); 2026 2027 dev_dbg(priv->ipu->dev, "%s: task %u: ctx %p\n", __func__, 2028 chan->ic_task, ctx); 2029 2030 ctx->chan = chan; 2031 init_completion(&ctx->aborted); 2032 2033 s_image = &ctx->in; 2034 d_image = &ctx->out; 2035 2036 /* set tiling and rotation */ 2037 d_image->num_rows = num_stripes(out->pix.height); 2038 d_image->num_cols = num_stripes(out->pix.width); 2039 if (ipu_rot_mode_is_irt(rot_mode)) { 2040 s_image->num_rows = d_image->num_cols; 2041 s_image->num_cols = d_image->num_rows; 2042 } else { 2043 s_image->num_rows = d_image->num_rows; 2044 s_image->num_cols = d_image->num_cols; 2045 } 2046 2047 ctx->num_tiles = d_image->num_cols * d_image->num_rows; 2048 ctx->rot_mode = rot_mode; 2049 2050 ret = fill_image(ctx, s_image, in, IMAGE_CONVERT_IN); 2051 if (ret) 2052 goto out_free; 2053 ret = fill_image(ctx, d_image, out, IMAGE_CONVERT_OUT); 2054 if (ret) 2055 goto out_free; 2056 2057 ret = calc_image_resize_coefficients(ctx, in, out); 2058 if (ret) 2059 goto out_free; 2060 2061 calc_out_tile_map(ctx); 2062 2063 find_seams(ctx, s_image, d_image); 2064 2065 calc_tile_dimensions(ctx, s_image); 2066 ret = calc_tile_offsets(ctx, s_image); 2067 if (ret) 2068 goto out_free; 2069 2070 calc_tile_dimensions(ctx, d_image); 2071 ret = calc_tile_offsets(ctx, d_image); 2072 if (ret) 2073 goto out_free; 2074 2075 calc_tile_resize_coefficients(ctx); 2076 2077 ret = ipu_ic_calc_csc(&ctx->csc, 2078 s_image->base.pix.ycbcr_enc, 2079 s_image->base.pix.quantization, 2080 ipu_pixelformat_to_colorspace(s_image->fmt->fourcc), 2081 d_image->base.pix.ycbcr_enc, 2082 d_image->base.pix.quantization, 2083 ipu_pixelformat_to_colorspace(d_image->fmt->fourcc)); 2084 if (ret) 2085 goto out_free; 2086 2087 dump_format(ctx, s_image); 2088 dump_format(ctx, d_image); 2089 2090 ctx->complete = complete; 2091 ctx->complete_context = complete_context; 2092 2093 /* 2094 * Can we use double-buffering for this operation? If there is 2095 * only one tile (the whole image can be converted in a single 2096 * operation) there's no point in using double-buffering. Also, 2097 * the IPU's IDMAC channels allow only a single U and V plane 2098 * offset shared between both buffers, but these offsets change 2099 * for every tile, and therefore would have to be updated for 2100 * each buffer which is not possible. So double-buffering is 2101 * impossible when either the source or destination images are 2102 * a planar format (YUV420, YUV422P, etc.). Further, differently 2103 * sized tiles or different resizing coefficients per tile 2104 * prevent double-buffering as well. 2105 */ 2106 ctx->double_buffering = (ctx->num_tiles > 1 && 2107 !s_image->fmt->planar && 2108 !d_image->fmt->planar); 2109 for (i = 1; i < ctx->num_tiles; i++) { 2110 if (ctx->in.tile[i].width != ctx->in.tile[0].width || 2111 ctx->in.tile[i].height != ctx->in.tile[0].height || 2112 ctx->out.tile[i].width != ctx->out.tile[0].width || 2113 ctx->out.tile[i].height != ctx->out.tile[0].height) { 2114 ctx->double_buffering = false; 2115 break; 2116 } 2117 } 2118 for (i = 1; i < ctx->in.num_cols; i++) { 2119 if (ctx->resize_coeffs_h[i] != ctx->resize_coeffs_h[0]) { 2120 ctx->double_buffering = false; 2121 break; 2122 } 2123 } 2124 for (i = 1; i < ctx->in.num_rows; i++) { 2125 if (ctx->resize_coeffs_v[i] != ctx->resize_coeffs_v[0]) { 2126 ctx->double_buffering = false; 2127 break; 2128 } 2129 } 2130 2131 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 2132 unsigned long intermediate_size = d_image->tile[0].size; 2133 2134 for (i = 1; i < ctx->num_tiles; i++) { 2135 if (d_image->tile[i].size > intermediate_size) 2136 intermediate_size = d_image->tile[i].size; 2137 } 2138 2139 ret = alloc_dma_buf(priv, &ctx->rot_intermediate[0], 2140 intermediate_size); 2141 if (ret) 2142 goto out_free; 2143 if (ctx->double_buffering) { 2144 ret = alloc_dma_buf(priv, 2145 &ctx->rot_intermediate[1], 2146 intermediate_size); 2147 if (ret) 2148 goto out_free_dmabuf0; 2149 } 2150 } 2151 2152 spin_lock_irqsave(&chan->irqlock, flags); 2153 2154 get_res = list_empty(&chan->ctx_list); 2155 2156 list_add_tail(&ctx->list, &chan->ctx_list); 2157 2158 spin_unlock_irqrestore(&chan->irqlock, flags); 2159 2160 if (get_res) { 2161 ret = get_ipu_resources(chan); 2162 if (ret) 2163 goto out_free_dmabuf1; 2164 } 2165 2166 return ctx; 2167 2168 out_free_dmabuf1: 2169 free_dma_buf(priv, &ctx->rot_intermediate[1]); 2170 spin_lock_irqsave(&chan->irqlock, flags); 2171 list_del(&ctx->list); 2172 spin_unlock_irqrestore(&chan->irqlock, flags); 2173 out_free_dmabuf0: 2174 free_dma_buf(priv, &ctx->rot_intermediate[0]); 2175 out_free: 2176 kfree(ctx); 2177 return ERR_PTR(ret); 2178 } 2179 EXPORT_SYMBOL_GPL(ipu_image_convert_prepare); 2180 2181 /* 2182 * Carry out a single image conversion run. Only the physaddr's of the input 2183 * and output image buffers are needed. The conversion context must have 2184 * been created previously with ipu_image_convert_prepare(). 2185 */ 2186 int ipu_image_convert_queue(struct ipu_image_convert_run *run) 2187 { 2188 struct ipu_image_convert_chan *chan; 2189 struct ipu_image_convert_priv *priv; 2190 struct ipu_image_convert_ctx *ctx; 2191 unsigned long flags; 2192 int ret = 0; 2193 2194 if (!run || !run->ctx || !run->in_phys || !run->out_phys) 2195 return -EINVAL; 2196 2197 ctx = run->ctx; 2198 chan = ctx->chan; 2199 priv = chan->priv; 2200 2201 dev_dbg(priv->ipu->dev, "%s: task %u: ctx %p run %p\n", __func__, 2202 chan->ic_task, ctx, run); 2203 2204 INIT_LIST_HEAD(&run->list); 2205 2206 spin_lock_irqsave(&chan->irqlock, flags); 2207 2208 if (ctx->aborting) { 2209 ret = -EIO; 2210 goto unlock; 2211 } 2212 2213 list_add_tail(&run->list, &chan->pending_q); 2214 2215 if (!chan->current_run) { 2216 ret = do_run(run); 2217 if (ret) 2218 chan->current_run = NULL; 2219 } 2220 unlock: 2221 spin_unlock_irqrestore(&chan->irqlock, flags); 2222 return ret; 2223 } 2224 EXPORT_SYMBOL_GPL(ipu_image_convert_queue); 2225 2226 /* Abort any active or pending conversions for this context */ 2227 static void __ipu_image_convert_abort(struct ipu_image_convert_ctx *ctx) 2228 { 2229 struct ipu_image_convert_chan *chan = ctx->chan; 2230 struct ipu_image_convert_priv *priv = chan->priv; 2231 struct ipu_image_convert_run *run, *active_run, *tmp; 2232 unsigned long flags; 2233 int run_count, ret; 2234 2235 spin_lock_irqsave(&chan->irqlock, flags); 2236 2237 /* move all remaining pending runs in this context to done_q */ 2238 list_for_each_entry_safe(run, tmp, &chan->pending_q, list) { 2239 if (run->ctx != ctx) 2240 continue; 2241 run->status = -EIO; 2242 list_move_tail(&run->list, &chan->done_q); 2243 } 2244 2245 run_count = get_run_count(ctx, &chan->done_q); 2246 active_run = (chan->current_run && chan->current_run->ctx == ctx) ? 2247 chan->current_run : NULL; 2248 2249 if (active_run) 2250 reinit_completion(&ctx->aborted); 2251 2252 ctx->aborting = true; 2253 2254 spin_unlock_irqrestore(&chan->irqlock, flags); 2255 2256 if (!run_count && !active_run) { 2257 dev_dbg(priv->ipu->dev, 2258 "%s: task %u: no abort needed for ctx %p\n", 2259 __func__, chan->ic_task, ctx); 2260 return; 2261 } 2262 2263 if (!active_run) { 2264 empty_done_q(chan); 2265 return; 2266 } 2267 2268 dev_dbg(priv->ipu->dev, 2269 "%s: task %u: wait for completion: %d runs\n", 2270 __func__, chan->ic_task, run_count); 2271 2272 ret = wait_for_completion_timeout(&ctx->aborted, 2273 msecs_to_jiffies(10000)); 2274 if (ret == 0) { 2275 dev_warn(priv->ipu->dev, "%s: timeout\n", __func__); 2276 force_abort(ctx); 2277 } 2278 } 2279 2280 void ipu_image_convert_abort(struct ipu_image_convert_ctx *ctx) 2281 { 2282 __ipu_image_convert_abort(ctx); 2283 ctx->aborting = false; 2284 } 2285 EXPORT_SYMBOL_GPL(ipu_image_convert_abort); 2286 2287 /* Unprepare image conversion context */ 2288 void ipu_image_convert_unprepare(struct ipu_image_convert_ctx *ctx) 2289 { 2290 struct ipu_image_convert_chan *chan = ctx->chan; 2291 struct ipu_image_convert_priv *priv = chan->priv; 2292 unsigned long flags; 2293 bool put_res; 2294 2295 /* make sure no runs are hanging around */ 2296 __ipu_image_convert_abort(ctx); 2297 2298 dev_dbg(priv->ipu->dev, "%s: task %u: removing ctx %p\n", __func__, 2299 chan->ic_task, ctx); 2300 2301 spin_lock_irqsave(&chan->irqlock, flags); 2302 2303 list_del(&ctx->list); 2304 2305 put_res = list_empty(&chan->ctx_list); 2306 2307 spin_unlock_irqrestore(&chan->irqlock, flags); 2308 2309 if (put_res) 2310 release_ipu_resources(chan); 2311 2312 free_dma_buf(priv, &ctx->rot_intermediate[1]); 2313 free_dma_buf(priv, &ctx->rot_intermediate[0]); 2314 2315 kfree(ctx); 2316 } 2317 EXPORT_SYMBOL_GPL(ipu_image_convert_unprepare); 2318 2319 /* 2320 * "Canned" asynchronous single image conversion. Allocates and returns 2321 * a new conversion run. On successful return the caller must free the 2322 * run and call ipu_image_convert_unprepare() after conversion completes. 2323 */ 2324 struct ipu_image_convert_run * 2325 ipu_image_convert(struct ipu_soc *ipu, enum ipu_ic_task ic_task, 2326 struct ipu_image *in, struct ipu_image *out, 2327 enum ipu_rotate_mode rot_mode, 2328 ipu_image_convert_cb_t complete, 2329 void *complete_context) 2330 { 2331 struct ipu_image_convert_ctx *ctx; 2332 struct ipu_image_convert_run *run; 2333 int ret; 2334 2335 ctx = ipu_image_convert_prepare(ipu, ic_task, in, out, rot_mode, 2336 complete, complete_context); 2337 if (IS_ERR(ctx)) 2338 return ERR_CAST(ctx); 2339 2340 run = kzalloc(sizeof(*run), GFP_KERNEL); 2341 if (!run) { 2342 ipu_image_convert_unprepare(ctx); 2343 return ERR_PTR(-ENOMEM); 2344 } 2345 2346 run->ctx = ctx; 2347 run->in_phys = in->phys0; 2348 run->out_phys = out->phys0; 2349 2350 ret = ipu_image_convert_queue(run); 2351 if (ret) { 2352 ipu_image_convert_unprepare(ctx); 2353 kfree(run); 2354 return ERR_PTR(ret); 2355 } 2356 2357 return run; 2358 } 2359 EXPORT_SYMBOL_GPL(ipu_image_convert); 2360 2361 /* "Canned" synchronous single image conversion */ 2362 static void image_convert_sync_complete(struct ipu_image_convert_run *run, 2363 void *data) 2364 { 2365 struct completion *comp = data; 2366 2367 complete(comp); 2368 } 2369 2370 int ipu_image_convert_sync(struct ipu_soc *ipu, enum ipu_ic_task ic_task, 2371 struct ipu_image *in, struct ipu_image *out, 2372 enum ipu_rotate_mode rot_mode) 2373 { 2374 struct ipu_image_convert_run *run; 2375 struct completion comp; 2376 int ret; 2377 2378 init_completion(&comp); 2379 2380 run = ipu_image_convert(ipu, ic_task, in, out, rot_mode, 2381 image_convert_sync_complete, &comp); 2382 if (IS_ERR(run)) 2383 return PTR_ERR(run); 2384 2385 ret = wait_for_completion_timeout(&comp, msecs_to_jiffies(10000)); 2386 ret = (ret == 0) ? -ETIMEDOUT : 0; 2387 2388 ipu_image_convert_unprepare(run->ctx); 2389 kfree(run); 2390 2391 return ret; 2392 } 2393 EXPORT_SYMBOL_GPL(ipu_image_convert_sync); 2394 2395 int ipu_image_convert_init(struct ipu_soc *ipu, struct device *dev) 2396 { 2397 struct ipu_image_convert_priv *priv; 2398 int i; 2399 2400 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); 2401 if (!priv) 2402 return -ENOMEM; 2403 2404 ipu->image_convert_priv = priv; 2405 priv->ipu = ipu; 2406 2407 for (i = 0; i < IC_NUM_TASKS; i++) { 2408 struct ipu_image_convert_chan *chan = &priv->chan[i]; 2409 2410 chan->ic_task = i; 2411 chan->priv = priv; 2412 chan->dma_ch = &image_convert_dma_chan[i]; 2413 chan->out_eof_irq = -1; 2414 chan->rot_out_eof_irq = -1; 2415 2416 spin_lock_init(&chan->irqlock); 2417 INIT_LIST_HEAD(&chan->ctx_list); 2418 INIT_LIST_HEAD(&chan->pending_q); 2419 INIT_LIST_HEAD(&chan->done_q); 2420 } 2421 2422 return 0; 2423 } 2424 2425 void ipu_image_convert_exit(struct ipu_soc *ipu) 2426 { 2427 } 2428