1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2012-2016 Mentor Graphics Inc. 4 * 5 * Queued image conversion support, with tiling and rotation. 6 */ 7 8 #include <linux/interrupt.h> 9 #include <linux/dma-mapping.h> 10 #include <video/imx-ipu-image-convert.h> 11 #include "ipu-prv.h" 12 13 /* 14 * The IC Resizer has a restriction that the output frame from the 15 * resizer must be 1024 or less in both width (pixels) and height 16 * (lines). 17 * 18 * The image converter attempts to split up a conversion when 19 * the desired output (converted) frame resolution exceeds the 20 * IC resizer limit of 1024 in either dimension. 21 * 22 * If either dimension of the output frame exceeds the limit, the 23 * dimension is split into 1, 2, or 4 equal stripes, for a maximum 24 * of 4*4 or 16 tiles. A conversion is then carried out for each 25 * tile (but taking care to pass the full frame stride length to 26 * the DMA channel's parameter memory!). IDMA double-buffering is used 27 * to convert each tile back-to-back when possible (see note below 28 * when double_buffering boolean is set). 29 * 30 * Note that the input frame must be split up into the same number 31 * of tiles as the output frame: 32 * 33 * +---------+-----+ 34 * +-----+---+ | A | B | 35 * | A | B | | | | 36 * +-----+---+ --> +---------+-----+ 37 * | C | D | | C | D | 38 * +-----+---+ | | | 39 * +---------+-----+ 40 * 41 * Clockwise 90° rotations are handled by first rescaling into a 42 * reusable temporary tile buffer and then rotating with the 8x8 43 * block rotator, writing to the correct destination: 44 * 45 * +-----+-----+ 46 * | | | 47 * +-----+---+ +---------+ | C | A | 48 * | A | B | | A,B, | | | | | 49 * +-----+---+ --> | C,D | | --> | | | 50 * | C | D | +---------+ +-----+-----+ 51 * +-----+---+ | D | B | 52 * | | | 53 * +-----+-----+ 54 * 55 * If the 8x8 block rotator is used, horizontal or vertical flipping 56 * is done during the rotation step, otherwise flipping is done 57 * during the scaling step. 58 * With rotation or flipping, tile order changes between input and 59 * output image. Tiles are numbered row major from top left to bottom 60 * right for both input and output image. 61 */ 62 63 #define MAX_STRIPES_W 4 64 #define MAX_STRIPES_H 4 65 #define MAX_TILES (MAX_STRIPES_W * MAX_STRIPES_H) 66 67 #define MIN_W 16 68 #define MIN_H 8 69 #define MAX_W 4096 70 #define MAX_H 4096 71 72 enum ipu_image_convert_type { 73 IMAGE_CONVERT_IN = 0, 74 IMAGE_CONVERT_OUT, 75 }; 76 77 struct ipu_image_convert_dma_buf { 78 void *virt; 79 dma_addr_t phys; 80 unsigned long len; 81 }; 82 83 struct ipu_image_convert_dma_chan { 84 int in; 85 int out; 86 int rot_in; 87 int rot_out; 88 int vdi_in_p; 89 int vdi_in; 90 int vdi_in_n; 91 }; 92 93 /* dimensions of one tile */ 94 struct ipu_image_tile { 95 u32 width; 96 u32 height; 97 u32 left; 98 u32 top; 99 /* size and strides are in bytes */ 100 u32 size; 101 u32 stride; 102 u32 rot_stride; 103 /* start Y or packed offset of this tile */ 104 u32 offset; 105 /* offset from start to tile in U plane, for planar formats */ 106 u32 u_off; 107 /* offset from start to tile in V plane, for planar formats */ 108 u32 v_off; 109 }; 110 111 struct ipu_image_convert_image { 112 struct ipu_image base; 113 enum ipu_image_convert_type type; 114 115 const struct ipu_image_pixfmt *fmt; 116 unsigned int stride; 117 118 /* # of rows (horizontal stripes) if dest height is > 1024 */ 119 unsigned int num_rows; 120 /* # of columns (vertical stripes) if dest width is > 1024 */ 121 unsigned int num_cols; 122 123 struct ipu_image_tile tile[MAX_TILES]; 124 }; 125 126 struct ipu_image_pixfmt { 127 u32 fourcc; /* V4L2 fourcc */ 128 int bpp; /* total bpp */ 129 int uv_width_dec; /* decimation in width for U/V planes */ 130 int uv_height_dec; /* decimation in height for U/V planes */ 131 bool planar; /* planar format */ 132 bool uv_swapped; /* U and V planes are swapped */ 133 bool uv_packed; /* partial planar (U and V in same plane) */ 134 }; 135 136 struct ipu_image_convert_ctx; 137 struct ipu_image_convert_chan; 138 struct ipu_image_convert_priv; 139 140 struct ipu_image_convert_ctx { 141 struct ipu_image_convert_chan *chan; 142 143 ipu_image_convert_cb_t complete; 144 void *complete_context; 145 146 /* Source/destination image data and rotation mode */ 147 struct ipu_image_convert_image in; 148 struct ipu_image_convert_image out; 149 struct ipu_ic_csc csc; 150 enum ipu_rotate_mode rot_mode; 151 u32 downsize_coeff_h; 152 u32 downsize_coeff_v; 153 u32 image_resize_coeff_h; 154 u32 image_resize_coeff_v; 155 u32 resize_coeffs_h[MAX_STRIPES_W]; 156 u32 resize_coeffs_v[MAX_STRIPES_H]; 157 158 /* intermediate buffer for rotation */ 159 struct ipu_image_convert_dma_buf rot_intermediate[2]; 160 161 /* current buffer number for double buffering */ 162 int cur_buf_num; 163 164 bool aborting; 165 struct completion aborted; 166 167 /* can we use double-buffering for this conversion operation? */ 168 bool double_buffering; 169 /* num_rows * num_cols */ 170 unsigned int num_tiles; 171 /* next tile to process */ 172 unsigned int next_tile; 173 /* where to place converted tile in dest image */ 174 unsigned int out_tile_map[MAX_TILES]; 175 176 struct list_head list; 177 }; 178 179 struct ipu_image_convert_chan { 180 struct ipu_image_convert_priv *priv; 181 182 enum ipu_ic_task ic_task; 183 const struct ipu_image_convert_dma_chan *dma_ch; 184 185 struct ipu_ic *ic; 186 struct ipuv3_channel *in_chan; 187 struct ipuv3_channel *out_chan; 188 struct ipuv3_channel *rotation_in_chan; 189 struct ipuv3_channel *rotation_out_chan; 190 191 /* the IPU end-of-frame irqs */ 192 int out_eof_irq; 193 int rot_out_eof_irq; 194 195 spinlock_t irqlock; 196 197 /* list of convert contexts */ 198 struct list_head ctx_list; 199 /* queue of conversion runs */ 200 struct list_head pending_q; 201 /* queue of completed runs */ 202 struct list_head done_q; 203 204 /* the current conversion run */ 205 struct ipu_image_convert_run *current_run; 206 }; 207 208 struct ipu_image_convert_priv { 209 struct ipu_image_convert_chan chan[IC_NUM_TASKS]; 210 struct ipu_soc *ipu; 211 }; 212 213 static const struct ipu_image_convert_dma_chan 214 image_convert_dma_chan[IC_NUM_TASKS] = { 215 [IC_TASK_VIEWFINDER] = { 216 .in = IPUV3_CHANNEL_MEM_IC_PRP_VF, 217 .out = IPUV3_CHANNEL_IC_PRP_VF_MEM, 218 .rot_in = IPUV3_CHANNEL_MEM_ROT_VF, 219 .rot_out = IPUV3_CHANNEL_ROT_VF_MEM, 220 .vdi_in_p = IPUV3_CHANNEL_MEM_VDI_PREV, 221 .vdi_in = IPUV3_CHANNEL_MEM_VDI_CUR, 222 .vdi_in_n = IPUV3_CHANNEL_MEM_VDI_NEXT, 223 }, 224 [IC_TASK_POST_PROCESSOR] = { 225 .in = IPUV3_CHANNEL_MEM_IC_PP, 226 .out = IPUV3_CHANNEL_IC_PP_MEM, 227 .rot_in = IPUV3_CHANNEL_MEM_ROT_PP, 228 .rot_out = IPUV3_CHANNEL_ROT_PP_MEM, 229 }, 230 }; 231 232 static const struct ipu_image_pixfmt image_convert_formats[] = { 233 { 234 .fourcc = V4L2_PIX_FMT_RGB565, 235 .bpp = 16, 236 }, { 237 .fourcc = V4L2_PIX_FMT_RGB24, 238 .bpp = 24, 239 }, { 240 .fourcc = V4L2_PIX_FMT_BGR24, 241 .bpp = 24, 242 }, { 243 .fourcc = V4L2_PIX_FMT_RGB32, 244 .bpp = 32, 245 }, { 246 .fourcc = V4L2_PIX_FMT_BGR32, 247 .bpp = 32, 248 }, { 249 .fourcc = V4L2_PIX_FMT_XRGB32, 250 .bpp = 32, 251 }, { 252 .fourcc = V4L2_PIX_FMT_XBGR32, 253 .bpp = 32, 254 }, { 255 .fourcc = V4L2_PIX_FMT_BGRX32, 256 .bpp = 32, 257 }, { 258 .fourcc = V4L2_PIX_FMT_RGBX32, 259 .bpp = 32, 260 }, { 261 .fourcc = V4L2_PIX_FMT_YUYV, 262 .bpp = 16, 263 .uv_width_dec = 2, 264 .uv_height_dec = 1, 265 }, { 266 .fourcc = V4L2_PIX_FMT_UYVY, 267 .bpp = 16, 268 .uv_width_dec = 2, 269 .uv_height_dec = 1, 270 }, { 271 .fourcc = V4L2_PIX_FMT_YUV420, 272 .bpp = 12, 273 .planar = true, 274 .uv_width_dec = 2, 275 .uv_height_dec = 2, 276 }, { 277 .fourcc = V4L2_PIX_FMT_YVU420, 278 .bpp = 12, 279 .planar = true, 280 .uv_width_dec = 2, 281 .uv_height_dec = 2, 282 .uv_swapped = true, 283 }, { 284 .fourcc = V4L2_PIX_FMT_NV12, 285 .bpp = 12, 286 .planar = true, 287 .uv_width_dec = 2, 288 .uv_height_dec = 2, 289 .uv_packed = true, 290 }, { 291 .fourcc = V4L2_PIX_FMT_YUV422P, 292 .bpp = 16, 293 .planar = true, 294 .uv_width_dec = 2, 295 .uv_height_dec = 1, 296 }, { 297 .fourcc = V4L2_PIX_FMT_NV16, 298 .bpp = 16, 299 .planar = true, 300 .uv_width_dec = 2, 301 .uv_height_dec = 1, 302 .uv_packed = true, 303 }, 304 }; 305 306 static const struct ipu_image_pixfmt *get_format(u32 fourcc) 307 { 308 const struct ipu_image_pixfmt *ret = NULL; 309 unsigned int i; 310 311 for (i = 0; i < ARRAY_SIZE(image_convert_formats); i++) { 312 if (image_convert_formats[i].fourcc == fourcc) { 313 ret = &image_convert_formats[i]; 314 break; 315 } 316 } 317 318 return ret; 319 } 320 321 static void dump_format(struct ipu_image_convert_ctx *ctx, 322 struct ipu_image_convert_image *ic_image) 323 { 324 struct ipu_image_convert_chan *chan = ctx->chan; 325 struct ipu_image_convert_priv *priv = chan->priv; 326 327 dev_dbg(priv->ipu->dev, 328 "task %u: ctx %p: %s format: %dx%d (%dx%d tiles), %c%c%c%c\n", 329 chan->ic_task, ctx, 330 ic_image->type == IMAGE_CONVERT_OUT ? "Output" : "Input", 331 ic_image->base.pix.width, ic_image->base.pix.height, 332 ic_image->num_cols, ic_image->num_rows, 333 ic_image->fmt->fourcc & 0xff, 334 (ic_image->fmt->fourcc >> 8) & 0xff, 335 (ic_image->fmt->fourcc >> 16) & 0xff, 336 (ic_image->fmt->fourcc >> 24) & 0xff); 337 } 338 339 int ipu_image_convert_enum_format(int index, u32 *fourcc) 340 { 341 const struct ipu_image_pixfmt *fmt; 342 343 if (index >= (int)ARRAY_SIZE(image_convert_formats)) 344 return -EINVAL; 345 346 /* Format found */ 347 fmt = &image_convert_formats[index]; 348 *fourcc = fmt->fourcc; 349 return 0; 350 } 351 EXPORT_SYMBOL_GPL(ipu_image_convert_enum_format); 352 353 static void free_dma_buf(struct ipu_image_convert_priv *priv, 354 struct ipu_image_convert_dma_buf *buf) 355 { 356 if (buf->virt) 357 dma_free_coherent(priv->ipu->dev, 358 buf->len, buf->virt, buf->phys); 359 buf->virt = NULL; 360 buf->phys = 0; 361 } 362 363 static int alloc_dma_buf(struct ipu_image_convert_priv *priv, 364 struct ipu_image_convert_dma_buf *buf, 365 int size) 366 { 367 buf->len = PAGE_ALIGN(size); 368 buf->virt = dma_alloc_coherent(priv->ipu->dev, buf->len, &buf->phys, 369 GFP_DMA | GFP_KERNEL); 370 if (!buf->virt) { 371 dev_err(priv->ipu->dev, "failed to alloc dma buffer\n"); 372 return -ENOMEM; 373 } 374 375 return 0; 376 } 377 378 static inline int num_stripes(int dim) 379 { 380 return (dim - 1) / 1024 + 1; 381 } 382 383 /* 384 * Calculate downsizing coefficients, which are the same for all tiles, 385 * and bilinear resizing coefficients, which are used to find the best 386 * seam positions. 387 */ 388 static int calc_image_resize_coefficients(struct ipu_image_convert_ctx *ctx, 389 struct ipu_image *in, 390 struct ipu_image *out) 391 { 392 u32 downsized_width = in->rect.width; 393 u32 downsized_height = in->rect.height; 394 u32 downsize_coeff_v = 0; 395 u32 downsize_coeff_h = 0; 396 u32 resized_width = out->rect.width; 397 u32 resized_height = out->rect.height; 398 u32 resize_coeff_h; 399 u32 resize_coeff_v; 400 401 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 402 resized_width = out->rect.height; 403 resized_height = out->rect.width; 404 } 405 406 /* Do not let invalid input lead to an endless loop below */ 407 if (WARN_ON(resized_width == 0 || resized_height == 0)) 408 return -EINVAL; 409 410 while (downsized_width > 1024 || 411 downsized_width >= resized_width * 2) { 412 downsized_width >>= 1; 413 downsize_coeff_h++; 414 } 415 416 while (downsized_height > 1024 || 417 downsized_height >= resized_height * 2) { 418 downsized_height >>= 1; 419 downsize_coeff_v++; 420 } 421 422 /* 423 * Calculate the bilinear resizing coefficients that could be used if 424 * we were converting with a single tile. The bottom right output pixel 425 * should sample as close as possible to the bottom right input pixel 426 * out of the decimator, but not overshoot it: 427 */ 428 resize_coeff_h = 8192 * (downsized_width - 1) / (resized_width - 1); 429 resize_coeff_v = 8192 * (downsized_height - 1) / (resized_height - 1); 430 431 dev_dbg(ctx->chan->priv->ipu->dev, 432 "%s: hscale: >>%u, *8192/%u vscale: >>%u, *8192/%u, %ux%u tiles\n", 433 __func__, downsize_coeff_h, resize_coeff_h, downsize_coeff_v, 434 resize_coeff_v, ctx->in.num_cols, ctx->in.num_rows); 435 436 if (downsize_coeff_h > 2 || downsize_coeff_v > 2 || 437 resize_coeff_h > 0x3fff || resize_coeff_v > 0x3fff) 438 return -EINVAL; 439 440 ctx->downsize_coeff_h = downsize_coeff_h; 441 ctx->downsize_coeff_v = downsize_coeff_v; 442 ctx->image_resize_coeff_h = resize_coeff_h; 443 ctx->image_resize_coeff_v = resize_coeff_v; 444 445 return 0; 446 } 447 448 #define round_closest(x, y) round_down((x) + (y)/2, (y)) 449 450 /* 451 * Find the best aligned seam position for the given column / row index. 452 * Rotation and image offsets are out of scope. 453 * 454 * @index: column / row index, used to calculate valid interval 455 * @in_edge: input right / bottom edge 456 * @out_edge: output right / bottom edge 457 * @in_align: input alignment, either horizontal 8-byte line start address 458 * alignment, or pixel alignment due to image format 459 * @out_align: output alignment, either horizontal 8-byte line start address 460 * alignment, or pixel alignment due to image format or rotator 461 * block size 462 * @in_burst: horizontal input burst size in case of horizontal flip 463 * @out_burst: horizontal output burst size or rotator block size 464 * @downsize_coeff: downsizing section coefficient 465 * @resize_coeff: main processing section resizing coefficient 466 * @_in_seam: aligned input seam position return value 467 * @_out_seam: aligned output seam position return value 468 */ 469 static void find_best_seam(struct ipu_image_convert_ctx *ctx, 470 unsigned int index, 471 unsigned int in_edge, 472 unsigned int out_edge, 473 unsigned int in_align, 474 unsigned int out_align, 475 unsigned int in_burst, 476 unsigned int out_burst, 477 unsigned int downsize_coeff, 478 unsigned int resize_coeff, 479 u32 *_in_seam, 480 u32 *_out_seam) 481 { 482 struct device *dev = ctx->chan->priv->ipu->dev; 483 unsigned int out_pos; 484 /* Input / output seam position candidates */ 485 unsigned int out_seam = 0; 486 unsigned int in_seam = 0; 487 unsigned int min_diff = UINT_MAX; 488 unsigned int out_start; 489 unsigned int out_end; 490 491 /* Start within 1024 pixels of the right / bottom edge */ 492 out_start = max_t(int, index * out_align, out_edge - 1024); 493 /* End before having to add more columns to the left / rows above */ 494 out_end = min_t(unsigned int, out_edge, index * 1024 + 1); 495 496 /* 497 * Output tiles must start at a multiple of 8 bytes horizontally and 498 * possibly at an even line horizontally depending on the pixel format. 499 * Only consider output aligned positions for the seam. 500 */ 501 out_start = round_up(out_start, out_align); 502 for (out_pos = out_start; out_pos < out_end; out_pos += out_align) { 503 unsigned int in_pos; 504 unsigned int in_pos_aligned; 505 unsigned int abs_diff; 506 507 /* 508 * Tiles in the right row / bottom column may not be allowed to 509 * overshoot horizontally / vertically. out_burst may be the 510 * actual DMA burst size, or the rotator block size. 511 */ 512 if ((out_burst > 1) && (out_edge - out_pos) % out_burst) 513 continue; 514 515 /* 516 * Input sample position, corresponding to out_pos, 19.13 fixed 517 * point. 518 */ 519 in_pos = (out_pos * resize_coeff) << downsize_coeff; 520 /* 521 * The closest input sample position that we could actually 522 * start the input tile at, 19.13 fixed point. 523 */ 524 in_pos_aligned = round_closest(in_pos, 8192U * in_align); 525 526 if ((in_burst > 1) && 527 (in_edge - in_pos_aligned / 8192U) % in_burst) 528 continue; 529 530 if (in_pos < in_pos_aligned) 531 abs_diff = in_pos_aligned - in_pos; 532 else 533 abs_diff = in_pos - in_pos_aligned; 534 535 if (abs_diff < min_diff) { 536 in_seam = in_pos_aligned; 537 out_seam = out_pos; 538 min_diff = abs_diff; 539 } 540 } 541 542 *_out_seam = out_seam; 543 /* Convert 19.13 fixed point to integer seam position */ 544 *_in_seam = DIV_ROUND_CLOSEST(in_seam, 8192U); 545 546 dev_dbg(dev, "%s: out_seam %u(%u) in [%u, %u], in_seam %u(%u) diff %u.%03u\n", 547 __func__, out_seam, out_align, out_start, out_end, 548 *_in_seam, in_align, min_diff / 8192, 549 DIV_ROUND_CLOSEST(min_diff % 8192 * 1000, 8192)); 550 } 551 552 /* 553 * Tile left edges are required to be aligned to multiples of 8 bytes 554 * by the IDMAC. 555 */ 556 static inline u32 tile_left_align(const struct ipu_image_pixfmt *fmt) 557 { 558 if (fmt->planar) 559 return fmt->uv_packed ? 8 : 8 * fmt->uv_width_dec; 560 else 561 return fmt->bpp == 32 ? 2 : fmt->bpp == 16 ? 4 : 8; 562 } 563 564 /* 565 * Tile top edge alignment is only limited by chroma subsampling. 566 */ 567 static inline u32 tile_top_align(const struct ipu_image_pixfmt *fmt) 568 { 569 return fmt->uv_height_dec > 1 ? 2 : 1; 570 } 571 572 static inline u32 tile_width_align(enum ipu_image_convert_type type, 573 const struct ipu_image_pixfmt *fmt, 574 enum ipu_rotate_mode rot_mode) 575 { 576 if (type == IMAGE_CONVERT_IN) { 577 /* 578 * The IC burst reads 8 pixels at a time. Reading beyond the 579 * end of the line is usually acceptable. Those pixels are 580 * ignored, unless the IC has to write the scaled line in 581 * reverse. 582 */ 583 return (!ipu_rot_mode_is_irt(rot_mode) && 584 (rot_mode & IPU_ROT_BIT_HFLIP)) ? 8 : 2; 585 } 586 587 /* 588 * Align to 16x16 pixel blocks for planar 4:2:0 chroma subsampled 589 * formats to guarantee 8-byte aligned line start addresses in the 590 * chroma planes when IRT is used. Align to 8x8 pixel IRT block size 591 * for all other formats. 592 */ 593 return (ipu_rot_mode_is_irt(rot_mode) && 594 fmt->planar && !fmt->uv_packed) ? 595 8 * fmt->uv_width_dec : 8; 596 } 597 598 static inline u32 tile_height_align(enum ipu_image_convert_type type, 599 const struct ipu_image_pixfmt *fmt, 600 enum ipu_rotate_mode rot_mode) 601 { 602 if (type == IMAGE_CONVERT_IN || !ipu_rot_mode_is_irt(rot_mode)) 603 return 2; 604 605 /* 606 * Align to 16x16 pixel blocks for planar 4:2:0 chroma subsampled 607 * formats to guarantee 8-byte aligned line start addresses in the 608 * chroma planes when IRT is used. Align to 8x8 pixel IRT block size 609 * for all other formats. 610 */ 611 return (fmt->planar && !fmt->uv_packed) ? 8 * fmt->uv_width_dec : 8; 612 } 613 614 /* 615 * Fill in left position and width and for all tiles in an input column, and 616 * for all corresponding output tiles. If the 90° rotator is used, the output 617 * tiles are in a row, and output tile top position and height are set. 618 */ 619 static void fill_tile_column(struct ipu_image_convert_ctx *ctx, 620 unsigned int col, 621 struct ipu_image_convert_image *in, 622 unsigned int in_left, unsigned int in_width, 623 struct ipu_image_convert_image *out, 624 unsigned int out_left, unsigned int out_width) 625 { 626 unsigned int row, tile_idx; 627 struct ipu_image_tile *in_tile, *out_tile; 628 629 for (row = 0; row < in->num_rows; row++) { 630 tile_idx = in->num_cols * row + col; 631 in_tile = &in->tile[tile_idx]; 632 out_tile = &out->tile[ctx->out_tile_map[tile_idx]]; 633 634 in_tile->left = in_left; 635 in_tile->width = in_width; 636 637 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 638 out_tile->top = out_left; 639 out_tile->height = out_width; 640 } else { 641 out_tile->left = out_left; 642 out_tile->width = out_width; 643 } 644 } 645 } 646 647 /* 648 * Fill in top position and height and for all tiles in an input row, and 649 * for all corresponding output tiles. If the 90° rotator is used, the output 650 * tiles are in a column, and output tile left position and width are set. 651 */ 652 static void fill_tile_row(struct ipu_image_convert_ctx *ctx, unsigned int row, 653 struct ipu_image_convert_image *in, 654 unsigned int in_top, unsigned int in_height, 655 struct ipu_image_convert_image *out, 656 unsigned int out_top, unsigned int out_height) 657 { 658 unsigned int col, tile_idx; 659 struct ipu_image_tile *in_tile, *out_tile; 660 661 for (col = 0; col < in->num_cols; col++) { 662 tile_idx = in->num_cols * row + col; 663 in_tile = &in->tile[tile_idx]; 664 out_tile = &out->tile[ctx->out_tile_map[tile_idx]]; 665 666 in_tile->top = in_top; 667 in_tile->height = in_height; 668 669 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 670 out_tile->left = out_top; 671 out_tile->width = out_height; 672 } else { 673 out_tile->top = out_top; 674 out_tile->height = out_height; 675 } 676 } 677 } 678 679 /* 680 * Find the best horizontal and vertical seam positions to split into tiles. 681 * Minimize the fractional part of the input sampling position for the 682 * top / left pixels of each tile. 683 */ 684 static void find_seams(struct ipu_image_convert_ctx *ctx, 685 struct ipu_image_convert_image *in, 686 struct ipu_image_convert_image *out) 687 { 688 struct device *dev = ctx->chan->priv->ipu->dev; 689 unsigned int resized_width = out->base.rect.width; 690 unsigned int resized_height = out->base.rect.height; 691 unsigned int col; 692 unsigned int row; 693 unsigned int in_left_align = tile_left_align(in->fmt); 694 unsigned int in_top_align = tile_top_align(in->fmt); 695 unsigned int out_left_align = tile_left_align(out->fmt); 696 unsigned int out_top_align = tile_top_align(out->fmt); 697 unsigned int out_width_align = tile_width_align(out->type, out->fmt, 698 ctx->rot_mode); 699 unsigned int out_height_align = tile_height_align(out->type, out->fmt, 700 ctx->rot_mode); 701 unsigned int in_right = in->base.rect.width; 702 unsigned int in_bottom = in->base.rect.height; 703 unsigned int out_right = out->base.rect.width; 704 unsigned int out_bottom = out->base.rect.height; 705 unsigned int flipped_out_left; 706 unsigned int flipped_out_top; 707 708 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 709 /* Switch width/height and align top left to IRT block size */ 710 resized_width = out->base.rect.height; 711 resized_height = out->base.rect.width; 712 out_left_align = out_height_align; 713 out_top_align = out_width_align; 714 out_width_align = out_left_align; 715 out_height_align = out_top_align; 716 out_right = out->base.rect.height; 717 out_bottom = out->base.rect.width; 718 } 719 720 for (col = in->num_cols - 1; col > 0; col--) { 721 bool allow_in_overshoot = ipu_rot_mode_is_irt(ctx->rot_mode) || 722 !(ctx->rot_mode & IPU_ROT_BIT_HFLIP); 723 bool allow_out_overshoot = (col < in->num_cols - 1) && 724 !(ctx->rot_mode & IPU_ROT_BIT_HFLIP); 725 unsigned int in_left; 726 unsigned int out_left; 727 728 /* 729 * Align input width to burst length if the scaling step flips 730 * horizontally. 731 */ 732 733 find_best_seam(ctx, col, 734 in_right, out_right, 735 in_left_align, out_left_align, 736 allow_in_overshoot ? 1 : 8 /* burst length */, 737 allow_out_overshoot ? 1 : out_width_align, 738 ctx->downsize_coeff_h, ctx->image_resize_coeff_h, 739 &in_left, &out_left); 740 741 if (ctx->rot_mode & IPU_ROT_BIT_HFLIP) 742 flipped_out_left = resized_width - out_right; 743 else 744 flipped_out_left = out_left; 745 746 fill_tile_column(ctx, col, in, in_left, in_right - in_left, 747 out, flipped_out_left, out_right - out_left); 748 749 dev_dbg(dev, "%s: col %u: %u, %u -> %u, %u\n", __func__, col, 750 in_left, in_right - in_left, 751 flipped_out_left, out_right - out_left); 752 753 in_right = in_left; 754 out_right = out_left; 755 } 756 757 flipped_out_left = (ctx->rot_mode & IPU_ROT_BIT_HFLIP) ? 758 resized_width - out_right : 0; 759 760 fill_tile_column(ctx, 0, in, 0, in_right, 761 out, flipped_out_left, out_right); 762 763 dev_dbg(dev, "%s: col 0: 0, %u -> %u, %u\n", __func__, 764 in_right, flipped_out_left, out_right); 765 766 for (row = in->num_rows - 1; row > 0; row--) { 767 bool allow_overshoot = row < in->num_rows - 1; 768 unsigned int in_top; 769 unsigned int out_top; 770 771 find_best_seam(ctx, row, 772 in_bottom, out_bottom, 773 in_top_align, out_top_align, 774 1, allow_overshoot ? 1 : out_height_align, 775 ctx->downsize_coeff_v, ctx->image_resize_coeff_v, 776 &in_top, &out_top); 777 778 if ((ctx->rot_mode & IPU_ROT_BIT_VFLIP) ^ 779 ipu_rot_mode_is_irt(ctx->rot_mode)) 780 flipped_out_top = resized_height - out_bottom; 781 else 782 flipped_out_top = out_top; 783 784 fill_tile_row(ctx, row, in, in_top, in_bottom - in_top, 785 out, flipped_out_top, out_bottom - out_top); 786 787 dev_dbg(dev, "%s: row %u: %u, %u -> %u, %u\n", __func__, row, 788 in_top, in_bottom - in_top, 789 flipped_out_top, out_bottom - out_top); 790 791 in_bottom = in_top; 792 out_bottom = out_top; 793 } 794 795 if ((ctx->rot_mode & IPU_ROT_BIT_VFLIP) ^ 796 ipu_rot_mode_is_irt(ctx->rot_mode)) 797 flipped_out_top = resized_height - out_bottom; 798 else 799 flipped_out_top = 0; 800 801 fill_tile_row(ctx, 0, in, 0, in_bottom, 802 out, flipped_out_top, out_bottom); 803 804 dev_dbg(dev, "%s: row 0: 0, %u -> %u, %u\n", __func__, 805 in_bottom, flipped_out_top, out_bottom); 806 } 807 808 static void calc_tile_dimensions(struct ipu_image_convert_ctx *ctx, 809 struct ipu_image_convert_image *image) 810 { 811 struct ipu_image_convert_chan *chan = ctx->chan; 812 struct ipu_image_convert_priv *priv = chan->priv; 813 unsigned int i; 814 815 for (i = 0; i < ctx->num_tiles; i++) { 816 struct ipu_image_tile *tile; 817 const unsigned int row = i / image->num_cols; 818 const unsigned int col = i % image->num_cols; 819 820 if (image->type == IMAGE_CONVERT_OUT) 821 tile = &image->tile[ctx->out_tile_map[i]]; 822 else 823 tile = &image->tile[i]; 824 825 tile->size = ((tile->height * image->fmt->bpp) >> 3) * 826 tile->width; 827 828 if (image->fmt->planar) { 829 tile->stride = tile->width; 830 tile->rot_stride = tile->height; 831 } else { 832 tile->stride = 833 (image->fmt->bpp * tile->width) >> 3; 834 tile->rot_stride = 835 (image->fmt->bpp * tile->height) >> 3; 836 } 837 838 dev_dbg(priv->ipu->dev, 839 "task %u: ctx %p: %s@[%u,%u]: %ux%u@%u,%u\n", 840 chan->ic_task, ctx, 841 image->type == IMAGE_CONVERT_IN ? "Input" : "Output", 842 row, col, 843 tile->width, tile->height, tile->left, tile->top); 844 } 845 } 846 847 /* 848 * Use the rotation transformation to find the tile coordinates 849 * (row, col) of a tile in the destination frame that corresponds 850 * to the given tile coordinates of a source frame. The destination 851 * coordinate is then converted to a tile index. 852 */ 853 static int transform_tile_index(struct ipu_image_convert_ctx *ctx, 854 int src_row, int src_col) 855 { 856 struct ipu_image_convert_chan *chan = ctx->chan; 857 struct ipu_image_convert_priv *priv = chan->priv; 858 struct ipu_image_convert_image *s_image = &ctx->in; 859 struct ipu_image_convert_image *d_image = &ctx->out; 860 int dst_row, dst_col; 861 862 /* with no rotation it's a 1:1 mapping */ 863 if (ctx->rot_mode == IPU_ROTATE_NONE) 864 return src_row * s_image->num_cols + src_col; 865 866 /* 867 * before doing the transform, first we have to translate 868 * source row,col for an origin in the center of s_image 869 */ 870 src_row = src_row * 2 - (s_image->num_rows - 1); 871 src_col = src_col * 2 - (s_image->num_cols - 1); 872 873 /* do the rotation transform */ 874 if (ctx->rot_mode & IPU_ROT_BIT_90) { 875 dst_col = -src_row; 876 dst_row = src_col; 877 } else { 878 dst_col = src_col; 879 dst_row = src_row; 880 } 881 882 /* apply flip */ 883 if (ctx->rot_mode & IPU_ROT_BIT_HFLIP) 884 dst_col = -dst_col; 885 if (ctx->rot_mode & IPU_ROT_BIT_VFLIP) 886 dst_row = -dst_row; 887 888 dev_dbg(priv->ipu->dev, "task %u: ctx %p: [%d,%d] --> [%d,%d]\n", 889 chan->ic_task, ctx, src_col, src_row, dst_col, dst_row); 890 891 /* 892 * finally translate dest row,col using an origin in upper 893 * left of d_image 894 */ 895 dst_row += d_image->num_rows - 1; 896 dst_col += d_image->num_cols - 1; 897 dst_row /= 2; 898 dst_col /= 2; 899 900 return dst_row * d_image->num_cols + dst_col; 901 } 902 903 /* 904 * Fill the out_tile_map[] with transformed destination tile indeces. 905 */ 906 static void calc_out_tile_map(struct ipu_image_convert_ctx *ctx) 907 { 908 struct ipu_image_convert_image *s_image = &ctx->in; 909 unsigned int row, col, tile = 0; 910 911 for (row = 0; row < s_image->num_rows; row++) { 912 for (col = 0; col < s_image->num_cols; col++) { 913 ctx->out_tile_map[tile] = 914 transform_tile_index(ctx, row, col); 915 tile++; 916 } 917 } 918 } 919 920 static int calc_tile_offsets_planar(struct ipu_image_convert_ctx *ctx, 921 struct ipu_image_convert_image *image) 922 { 923 struct ipu_image_convert_chan *chan = ctx->chan; 924 struct ipu_image_convert_priv *priv = chan->priv; 925 const struct ipu_image_pixfmt *fmt = image->fmt; 926 unsigned int row, col, tile = 0; 927 u32 H, top, y_stride, uv_stride; 928 u32 uv_row_off, uv_col_off, uv_off, u_off, v_off, tmp; 929 u32 y_row_off, y_col_off, y_off; 930 u32 y_size, uv_size; 931 932 /* setup some convenience vars */ 933 H = image->base.pix.height; 934 935 y_stride = image->stride; 936 uv_stride = y_stride / fmt->uv_width_dec; 937 if (fmt->uv_packed) 938 uv_stride *= 2; 939 940 y_size = H * y_stride; 941 uv_size = y_size / (fmt->uv_width_dec * fmt->uv_height_dec); 942 943 for (row = 0; row < image->num_rows; row++) { 944 top = image->tile[tile].top; 945 y_row_off = top * y_stride; 946 uv_row_off = (top * uv_stride) / fmt->uv_height_dec; 947 948 for (col = 0; col < image->num_cols; col++) { 949 y_col_off = image->tile[tile].left; 950 uv_col_off = y_col_off / fmt->uv_width_dec; 951 if (fmt->uv_packed) 952 uv_col_off *= 2; 953 954 y_off = y_row_off + y_col_off; 955 uv_off = uv_row_off + uv_col_off; 956 957 u_off = y_size - y_off + uv_off; 958 v_off = (fmt->uv_packed) ? 0 : u_off + uv_size; 959 if (fmt->uv_swapped) { 960 tmp = u_off; 961 u_off = v_off; 962 v_off = tmp; 963 } 964 965 image->tile[tile].offset = y_off; 966 image->tile[tile].u_off = u_off; 967 image->tile[tile++].v_off = v_off; 968 969 if ((y_off & 0x7) || (u_off & 0x7) || (v_off & 0x7)) { 970 dev_err(priv->ipu->dev, 971 "task %u: ctx %p: %s@[%d,%d]: " 972 "y_off %08x, u_off %08x, v_off %08x\n", 973 chan->ic_task, ctx, 974 image->type == IMAGE_CONVERT_IN ? 975 "Input" : "Output", row, col, 976 y_off, u_off, v_off); 977 return -EINVAL; 978 } 979 } 980 } 981 982 return 0; 983 } 984 985 static int calc_tile_offsets_packed(struct ipu_image_convert_ctx *ctx, 986 struct ipu_image_convert_image *image) 987 { 988 struct ipu_image_convert_chan *chan = ctx->chan; 989 struct ipu_image_convert_priv *priv = chan->priv; 990 const struct ipu_image_pixfmt *fmt = image->fmt; 991 unsigned int row, col, tile = 0; 992 u32 bpp, stride, offset; 993 u32 row_off, col_off; 994 995 /* setup some convenience vars */ 996 stride = image->stride; 997 bpp = fmt->bpp; 998 999 for (row = 0; row < image->num_rows; row++) { 1000 row_off = image->tile[tile].top * stride; 1001 1002 for (col = 0; col < image->num_cols; col++) { 1003 col_off = (image->tile[tile].left * bpp) >> 3; 1004 1005 offset = row_off + col_off; 1006 1007 image->tile[tile].offset = offset; 1008 image->tile[tile].u_off = 0; 1009 image->tile[tile++].v_off = 0; 1010 1011 if (offset & 0x7) { 1012 dev_err(priv->ipu->dev, 1013 "task %u: ctx %p: %s@[%d,%d]: " 1014 "phys %08x\n", 1015 chan->ic_task, ctx, 1016 image->type == IMAGE_CONVERT_IN ? 1017 "Input" : "Output", row, col, 1018 row_off + col_off); 1019 return -EINVAL; 1020 } 1021 } 1022 } 1023 1024 return 0; 1025 } 1026 1027 static int calc_tile_offsets(struct ipu_image_convert_ctx *ctx, 1028 struct ipu_image_convert_image *image) 1029 { 1030 if (image->fmt->planar) 1031 return calc_tile_offsets_planar(ctx, image); 1032 1033 return calc_tile_offsets_packed(ctx, image); 1034 } 1035 1036 /* 1037 * Calculate the resizing ratio for the IC main processing section given input 1038 * size, fixed downsizing coefficient, and output size. 1039 * Either round to closest for the next tile's first pixel to minimize seams 1040 * and distortion (for all but right column / bottom row), or round down to 1041 * avoid sampling beyond the edges of the input image for this tile's last 1042 * pixel. 1043 * Returns the resizing coefficient, resizing ratio is 8192.0 / resize_coeff. 1044 */ 1045 static u32 calc_resize_coeff(u32 input_size, u32 downsize_coeff, 1046 u32 output_size, bool allow_overshoot) 1047 { 1048 u32 downsized = input_size >> downsize_coeff; 1049 1050 if (allow_overshoot) 1051 return DIV_ROUND_CLOSEST(8192 * downsized, output_size); 1052 else 1053 return 8192 * (downsized - 1) / (output_size - 1); 1054 } 1055 1056 /* 1057 * Slightly modify resize coefficients per tile to hide the bilinear 1058 * interpolator reset at tile borders, shifting the right / bottom edge 1059 * by up to a half input pixel. This removes noticeable seams between 1060 * tiles at higher upscaling factors. 1061 */ 1062 static void calc_tile_resize_coefficients(struct ipu_image_convert_ctx *ctx) 1063 { 1064 struct ipu_image_convert_chan *chan = ctx->chan; 1065 struct ipu_image_convert_priv *priv = chan->priv; 1066 struct ipu_image_tile *in_tile, *out_tile; 1067 unsigned int col, row, tile_idx; 1068 unsigned int last_output; 1069 1070 for (col = 0; col < ctx->in.num_cols; col++) { 1071 bool closest = (col < ctx->in.num_cols - 1) && 1072 !(ctx->rot_mode & IPU_ROT_BIT_HFLIP); 1073 u32 resized_width; 1074 u32 resize_coeff_h; 1075 1076 tile_idx = col; 1077 in_tile = &ctx->in.tile[tile_idx]; 1078 out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]]; 1079 1080 if (ipu_rot_mode_is_irt(ctx->rot_mode)) 1081 resized_width = out_tile->height; 1082 else 1083 resized_width = out_tile->width; 1084 1085 resize_coeff_h = calc_resize_coeff(in_tile->width, 1086 ctx->downsize_coeff_h, 1087 resized_width, closest); 1088 1089 dev_dbg(priv->ipu->dev, "%s: column %u hscale: *8192/%u\n", 1090 __func__, col, resize_coeff_h); 1091 1092 1093 for (row = 0; row < ctx->in.num_rows; row++) { 1094 tile_idx = row * ctx->in.num_cols + col; 1095 in_tile = &ctx->in.tile[tile_idx]; 1096 out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]]; 1097 1098 /* 1099 * With the horizontal scaling factor known, round up 1100 * resized width (output width or height) to burst size. 1101 */ 1102 if (ipu_rot_mode_is_irt(ctx->rot_mode)) 1103 out_tile->height = round_up(resized_width, 8); 1104 else 1105 out_tile->width = round_up(resized_width, 8); 1106 1107 /* 1108 * Calculate input width from the last accessed input 1109 * pixel given resized width and scaling coefficients. 1110 * Round up to burst size. 1111 */ 1112 last_output = round_up(resized_width, 8) - 1; 1113 if (closest) 1114 last_output++; 1115 in_tile->width = round_up( 1116 (DIV_ROUND_UP(last_output * resize_coeff_h, 1117 8192) + 1) 1118 << ctx->downsize_coeff_h, 8); 1119 } 1120 1121 ctx->resize_coeffs_h[col] = resize_coeff_h; 1122 } 1123 1124 for (row = 0; row < ctx->in.num_rows; row++) { 1125 bool closest = (row < ctx->in.num_rows - 1) && 1126 !(ctx->rot_mode & IPU_ROT_BIT_VFLIP); 1127 u32 resized_height; 1128 u32 resize_coeff_v; 1129 1130 tile_idx = row * ctx->in.num_cols; 1131 in_tile = &ctx->in.tile[tile_idx]; 1132 out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]]; 1133 1134 if (ipu_rot_mode_is_irt(ctx->rot_mode)) 1135 resized_height = out_tile->width; 1136 else 1137 resized_height = out_tile->height; 1138 1139 resize_coeff_v = calc_resize_coeff(in_tile->height, 1140 ctx->downsize_coeff_v, 1141 resized_height, closest); 1142 1143 dev_dbg(priv->ipu->dev, "%s: row %u vscale: *8192/%u\n", 1144 __func__, row, resize_coeff_v); 1145 1146 for (col = 0; col < ctx->in.num_cols; col++) { 1147 tile_idx = row * ctx->in.num_cols + col; 1148 in_tile = &ctx->in.tile[tile_idx]; 1149 out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]]; 1150 1151 /* 1152 * With the vertical scaling factor known, round up 1153 * resized height (output width or height) to IDMAC 1154 * limitations. 1155 */ 1156 if (ipu_rot_mode_is_irt(ctx->rot_mode)) 1157 out_tile->width = round_up(resized_height, 2); 1158 else 1159 out_tile->height = round_up(resized_height, 2); 1160 1161 /* 1162 * Calculate input width from the last accessed input 1163 * pixel given resized height and scaling coefficients. 1164 * Align to IDMAC restrictions. 1165 */ 1166 last_output = round_up(resized_height, 2) - 1; 1167 if (closest) 1168 last_output++; 1169 in_tile->height = round_up( 1170 (DIV_ROUND_UP(last_output * resize_coeff_v, 1171 8192) + 1) 1172 << ctx->downsize_coeff_v, 2); 1173 } 1174 1175 ctx->resize_coeffs_v[row] = resize_coeff_v; 1176 } 1177 } 1178 1179 /* 1180 * return the number of runs in given queue (pending_q or done_q) 1181 * for this context. hold irqlock when calling. 1182 */ 1183 static int get_run_count(struct ipu_image_convert_ctx *ctx, 1184 struct list_head *q) 1185 { 1186 struct ipu_image_convert_run *run; 1187 int count = 0; 1188 1189 lockdep_assert_held(&ctx->chan->irqlock); 1190 1191 list_for_each_entry(run, q, list) { 1192 if (run->ctx == ctx) 1193 count++; 1194 } 1195 1196 return count; 1197 } 1198 1199 static void convert_stop(struct ipu_image_convert_run *run) 1200 { 1201 struct ipu_image_convert_ctx *ctx = run->ctx; 1202 struct ipu_image_convert_chan *chan = ctx->chan; 1203 struct ipu_image_convert_priv *priv = chan->priv; 1204 1205 dev_dbg(priv->ipu->dev, "%s: task %u: stopping ctx %p run %p\n", 1206 __func__, chan->ic_task, ctx, run); 1207 1208 /* disable IC tasks and the channels */ 1209 ipu_ic_task_disable(chan->ic); 1210 ipu_idmac_disable_channel(chan->in_chan); 1211 ipu_idmac_disable_channel(chan->out_chan); 1212 1213 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 1214 ipu_idmac_disable_channel(chan->rotation_in_chan); 1215 ipu_idmac_disable_channel(chan->rotation_out_chan); 1216 ipu_idmac_unlink(chan->out_chan, chan->rotation_in_chan); 1217 } 1218 1219 ipu_ic_disable(chan->ic); 1220 } 1221 1222 static void init_idmac_channel(struct ipu_image_convert_ctx *ctx, 1223 struct ipuv3_channel *channel, 1224 struct ipu_image_convert_image *image, 1225 enum ipu_rotate_mode rot_mode, 1226 bool rot_swap_width_height, 1227 unsigned int tile) 1228 { 1229 struct ipu_image_convert_chan *chan = ctx->chan; 1230 unsigned int burst_size; 1231 u32 width, height, stride; 1232 dma_addr_t addr0, addr1 = 0; 1233 struct ipu_image tile_image; 1234 unsigned int tile_idx[2]; 1235 1236 if (image->type == IMAGE_CONVERT_OUT) { 1237 tile_idx[0] = ctx->out_tile_map[tile]; 1238 tile_idx[1] = ctx->out_tile_map[1]; 1239 } else { 1240 tile_idx[0] = tile; 1241 tile_idx[1] = 1; 1242 } 1243 1244 if (rot_swap_width_height) { 1245 width = image->tile[tile_idx[0]].height; 1246 height = image->tile[tile_idx[0]].width; 1247 stride = image->tile[tile_idx[0]].rot_stride; 1248 addr0 = ctx->rot_intermediate[0].phys; 1249 if (ctx->double_buffering) 1250 addr1 = ctx->rot_intermediate[1].phys; 1251 } else { 1252 width = image->tile[tile_idx[0]].width; 1253 height = image->tile[tile_idx[0]].height; 1254 stride = image->stride; 1255 addr0 = image->base.phys0 + 1256 image->tile[tile_idx[0]].offset; 1257 if (ctx->double_buffering) 1258 addr1 = image->base.phys0 + 1259 image->tile[tile_idx[1]].offset; 1260 } 1261 1262 ipu_cpmem_zero(channel); 1263 1264 memset(&tile_image, 0, sizeof(tile_image)); 1265 tile_image.pix.width = tile_image.rect.width = width; 1266 tile_image.pix.height = tile_image.rect.height = height; 1267 tile_image.pix.bytesperline = stride; 1268 tile_image.pix.pixelformat = image->fmt->fourcc; 1269 tile_image.phys0 = addr0; 1270 tile_image.phys1 = addr1; 1271 if (image->fmt->planar && !rot_swap_width_height) { 1272 tile_image.u_offset = image->tile[tile_idx[0]].u_off; 1273 tile_image.v_offset = image->tile[tile_idx[0]].v_off; 1274 } 1275 1276 ipu_cpmem_set_image(channel, &tile_image); 1277 1278 if (rot_mode) 1279 ipu_cpmem_set_rotation(channel, rot_mode); 1280 1281 /* 1282 * Skip writing U and V components to odd rows in the output 1283 * channels for planar 4:2:0. 1284 */ 1285 if ((channel == chan->out_chan || 1286 channel == chan->rotation_out_chan) && 1287 image->fmt->planar && image->fmt->uv_height_dec == 2) 1288 ipu_cpmem_skip_odd_chroma_rows(channel); 1289 1290 if (channel == chan->rotation_in_chan || 1291 channel == chan->rotation_out_chan) { 1292 burst_size = 8; 1293 ipu_cpmem_set_block_mode(channel); 1294 } else 1295 burst_size = (width % 16) ? 8 : 16; 1296 1297 ipu_cpmem_set_burstsize(channel, burst_size); 1298 1299 ipu_ic_task_idma_init(chan->ic, channel, width, height, 1300 burst_size, rot_mode); 1301 1302 /* 1303 * Setting a non-zero AXI ID collides with the PRG AXI snooping, so 1304 * only do this when there is no PRG present. 1305 */ 1306 if (!channel->ipu->prg_priv) 1307 ipu_cpmem_set_axi_id(channel, 1); 1308 1309 ipu_idmac_set_double_buffer(channel, ctx->double_buffering); 1310 } 1311 1312 static int convert_start(struct ipu_image_convert_run *run, unsigned int tile) 1313 { 1314 struct ipu_image_convert_ctx *ctx = run->ctx; 1315 struct ipu_image_convert_chan *chan = ctx->chan; 1316 struct ipu_image_convert_priv *priv = chan->priv; 1317 struct ipu_image_convert_image *s_image = &ctx->in; 1318 struct ipu_image_convert_image *d_image = &ctx->out; 1319 unsigned int dst_tile = ctx->out_tile_map[tile]; 1320 unsigned int dest_width, dest_height; 1321 unsigned int col, row; 1322 u32 rsc; 1323 int ret; 1324 1325 dev_dbg(priv->ipu->dev, "%s: task %u: starting ctx %p run %p tile %u -> %u\n", 1326 __func__, chan->ic_task, ctx, run, tile, dst_tile); 1327 1328 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 1329 /* swap width/height for resizer */ 1330 dest_width = d_image->tile[dst_tile].height; 1331 dest_height = d_image->tile[dst_tile].width; 1332 } else { 1333 dest_width = d_image->tile[dst_tile].width; 1334 dest_height = d_image->tile[dst_tile].height; 1335 } 1336 1337 row = tile / s_image->num_cols; 1338 col = tile % s_image->num_cols; 1339 1340 rsc = (ctx->downsize_coeff_v << 30) | 1341 (ctx->resize_coeffs_v[row] << 16) | 1342 (ctx->downsize_coeff_h << 14) | 1343 (ctx->resize_coeffs_h[col]); 1344 1345 dev_dbg(priv->ipu->dev, "%s: %ux%u -> %ux%u (rsc = 0x%x)\n", 1346 __func__, s_image->tile[tile].width, 1347 s_image->tile[tile].height, dest_width, dest_height, rsc); 1348 1349 /* setup the IC resizer and CSC */ 1350 ret = ipu_ic_task_init_rsc(chan->ic, &ctx->csc, 1351 s_image->tile[tile].width, 1352 s_image->tile[tile].height, 1353 dest_width, 1354 dest_height, 1355 rsc); 1356 if (ret) { 1357 dev_err(priv->ipu->dev, "ipu_ic_task_init failed, %d\n", ret); 1358 return ret; 1359 } 1360 1361 /* init the source MEM-->IC PP IDMAC channel */ 1362 init_idmac_channel(ctx, chan->in_chan, s_image, 1363 IPU_ROTATE_NONE, false, tile); 1364 1365 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 1366 /* init the IC PP-->MEM IDMAC channel */ 1367 init_idmac_channel(ctx, chan->out_chan, d_image, 1368 IPU_ROTATE_NONE, true, tile); 1369 1370 /* init the MEM-->IC PP ROT IDMAC channel */ 1371 init_idmac_channel(ctx, chan->rotation_in_chan, d_image, 1372 ctx->rot_mode, true, tile); 1373 1374 /* init the destination IC PP ROT-->MEM IDMAC channel */ 1375 init_idmac_channel(ctx, chan->rotation_out_chan, d_image, 1376 IPU_ROTATE_NONE, false, tile); 1377 1378 /* now link IC PP-->MEM to MEM-->IC PP ROT */ 1379 ipu_idmac_link(chan->out_chan, chan->rotation_in_chan); 1380 } else { 1381 /* init the destination IC PP-->MEM IDMAC channel */ 1382 init_idmac_channel(ctx, chan->out_chan, d_image, 1383 ctx->rot_mode, false, tile); 1384 } 1385 1386 /* enable the IC */ 1387 ipu_ic_enable(chan->ic); 1388 1389 /* set buffers ready */ 1390 ipu_idmac_select_buffer(chan->in_chan, 0); 1391 ipu_idmac_select_buffer(chan->out_chan, 0); 1392 if (ipu_rot_mode_is_irt(ctx->rot_mode)) 1393 ipu_idmac_select_buffer(chan->rotation_out_chan, 0); 1394 if (ctx->double_buffering) { 1395 ipu_idmac_select_buffer(chan->in_chan, 1); 1396 ipu_idmac_select_buffer(chan->out_chan, 1); 1397 if (ipu_rot_mode_is_irt(ctx->rot_mode)) 1398 ipu_idmac_select_buffer(chan->rotation_out_chan, 1); 1399 } 1400 1401 /* enable the channels! */ 1402 ipu_idmac_enable_channel(chan->in_chan); 1403 ipu_idmac_enable_channel(chan->out_chan); 1404 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 1405 ipu_idmac_enable_channel(chan->rotation_in_chan); 1406 ipu_idmac_enable_channel(chan->rotation_out_chan); 1407 } 1408 1409 ipu_ic_task_enable(chan->ic); 1410 1411 ipu_cpmem_dump(chan->in_chan); 1412 ipu_cpmem_dump(chan->out_chan); 1413 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 1414 ipu_cpmem_dump(chan->rotation_in_chan); 1415 ipu_cpmem_dump(chan->rotation_out_chan); 1416 } 1417 1418 ipu_dump(priv->ipu); 1419 1420 return 0; 1421 } 1422 1423 /* hold irqlock when calling */ 1424 static int do_run(struct ipu_image_convert_run *run) 1425 { 1426 struct ipu_image_convert_ctx *ctx = run->ctx; 1427 struct ipu_image_convert_chan *chan = ctx->chan; 1428 1429 lockdep_assert_held(&chan->irqlock); 1430 1431 ctx->in.base.phys0 = run->in_phys; 1432 ctx->out.base.phys0 = run->out_phys; 1433 1434 ctx->cur_buf_num = 0; 1435 ctx->next_tile = 1; 1436 1437 /* remove run from pending_q and set as current */ 1438 list_del(&run->list); 1439 chan->current_run = run; 1440 1441 return convert_start(run, 0); 1442 } 1443 1444 /* hold irqlock when calling */ 1445 static void run_next(struct ipu_image_convert_chan *chan) 1446 { 1447 struct ipu_image_convert_priv *priv = chan->priv; 1448 struct ipu_image_convert_run *run, *tmp; 1449 int ret; 1450 1451 lockdep_assert_held(&chan->irqlock); 1452 1453 list_for_each_entry_safe(run, tmp, &chan->pending_q, list) { 1454 /* skip contexts that are aborting */ 1455 if (run->ctx->aborting) { 1456 dev_dbg(priv->ipu->dev, 1457 "%s: task %u: skipping aborting ctx %p run %p\n", 1458 __func__, chan->ic_task, run->ctx, run); 1459 continue; 1460 } 1461 1462 ret = do_run(run); 1463 if (!ret) 1464 break; 1465 1466 /* 1467 * something went wrong with start, add the run 1468 * to done q and continue to the next run in the 1469 * pending q. 1470 */ 1471 run->status = ret; 1472 list_add_tail(&run->list, &chan->done_q); 1473 chan->current_run = NULL; 1474 } 1475 } 1476 1477 static void empty_done_q(struct ipu_image_convert_chan *chan) 1478 { 1479 struct ipu_image_convert_priv *priv = chan->priv; 1480 struct ipu_image_convert_run *run; 1481 unsigned long flags; 1482 1483 spin_lock_irqsave(&chan->irqlock, flags); 1484 1485 while (!list_empty(&chan->done_q)) { 1486 run = list_entry(chan->done_q.next, 1487 struct ipu_image_convert_run, 1488 list); 1489 1490 list_del(&run->list); 1491 1492 dev_dbg(priv->ipu->dev, 1493 "%s: task %u: completing ctx %p run %p with %d\n", 1494 __func__, chan->ic_task, run->ctx, run, run->status); 1495 1496 /* call the completion callback and free the run */ 1497 spin_unlock_irqrestore(&chan->irqlock, flags); 1498 run->ctx->complete(run, run->ctx->complete_context); 1499 spin_lock_irqsave(&chan->irqlock, flags); 1500 } 1501 1502 spin_unlock_irqrestore(&chan->irqlock, flags); 1503 } 1504 1505 /* 1506 * the bottom half thread clears out the done_q, calling the 1507 * completion handler for each. 1508 */ 1509 static irqreturn_t do_bh(int irq, void *dev_id) 1510 { 1511 struct ipu_image_convert_chan *chan = dev_id; 1512 struct ipu_image_convert_priv *priv = chan->priv; 1513 struct ipu_image_convert_ctx *ctx; 1514 unsigned long flags; 1515 1516 dev_dbg(priv->ipu->dev, "%s: task %u: enter\n", __func__, 1517 chan->ic_task); 1518 1519 empty_done_q(chan); 1520 1521 spin_lock_irqsave(&chan->irqlock, flags); 1522 1523 /* 1524 * the done_q is cleared out, signal any contexts 1525 * that are aborting that abort can complete. 1526 */ 1527 list_for_each_entry(ctx, &chan->ctx_list, list) { 1528 if (ctx->aborting) { 1529 dev_dbg(priv->ipu->dev, 1530 "%s: task %u: signaling abort for ctx %p\n", 1531 __func__, chan->ic_task, ctx); 1532 complete_all(&ctx->aborted); 1533 } 1534 } 1535 1536 spin_unlock_irqrestore(&chan->irqlock, flags); 1537 1538 dev_dbg(priv->ipu->dev, "%s: task %u: exit\n", __func__, 1539 chan->ic_task); 1540 1541 return IRQ_HANDLED; 1542 } 1543 1544 static bool ic_settings_changed(struct ipu_image_convert_ctx *ctx) 1545 { 1546 unsigned int cur_tile = ctx->next_tile - 1; 1547 unsigned int next_tile = ctx->next_tile; 1548 1549 if (ctx->resize_coeffs_h[cur_tile % ctx->in.num_cols] != 1550 ctx->resize_coeffs_h[next_tile % ctx->in.num_cols] || 1551 ctx->resize_coeffs_v[cur_tile / ctx->in.num_cols] != 1552 ctx->resize_coeffs_v[next_tile / ctx->in.num_cols] || 1553 ctx->in.tile[cur_tile].width != ctx->in.tile[next_tile].width || 1554 ctx->in.tile[cur_tile].height != ctx->in.tile[next_tile].height || 1555 ctx->out.tile[cur_tile].width != ctx->out.tile[next_tile].width || 1556 ctx->out.tile[cur_tile].height != ctx->out.tile[next_tile].height) 1557 return true; 1558 1559 return false; 1560 } 1561 1562 /* hold irqlock when calling */ 1563 static irqreturn_t do_irq(struct ipu_image_convert_run *run) 1564 { 1565 struct ipu_image_convert_ctx *ctx = run->ctx; 1566 struct ipu_image_convert_chan *chan = ctx->chan; 1567 struct ipu_image_tile *src_tile, *dst_tile; 1568 struct ipu_image_convert_image *s_image = &ctx->in; 1569 struct ipu_image_convert_image *d_image = &ctx->out; 1570 struct ipuv3_channel *outch; 1571 unsigned int dst_idx; 1572 1573 lockdep_assert_held(&chan->irqlock); 1574 1575 outch = ipu_rot_mode_is_irt(ctx->rot_mode) ? 1576 chan->rotation_out_chan : chan->out_chan; 1577 1578 /* 1579 * It is difficult to stop the channel DMA before the channels 1580 * enter the paused state. Without double-buffering the channels 1581 * are always in a paused state when the EOF irq occurs, so it 1582 * is safe to stop the channels now. For double-buffering we 1583 * just ignore the abort until the operation completes, when it 1584 * is safe to shut down. 1585 */ 1586 if (ctx->aborting && !ctx->double_buffering) { 1587 convert_stop(run); 1588 run->status = -EIO; 1589 goto done; 1590 } 1591 1592 if (ctx->next_tile == ctx->num_tiles) { 1593 /* 1594 * the conversion is complete 1595 */ 1596 convert_stop(run); 1597 run->status = 0; 1598 goto done; 1599 } 1600 1601 /* 1602 * not done, place the next tile buffers. 1603 */ 1604 if (!ctx->double_buffering) { 1605 if (ic_settings_changed(ctx)) { 1606 convert_stop(run); 1607 convert_start(run, ctx->next_tile); 1608 } else { 1609 src_tile = &s_image->tile[ctx->next_tile]; 1610 dst_idx = ctx->out_tile_map[ctx->next_tile]; 1611 dst_tile = &d_image->tile[dst_idx]; 1612 1613 ipu_cpmem_set_buffer(chan->in_chan, 0, 1614 s_image->base.phys0 + 1615 src_tile->offset); 1616 ipu_cpmem_set_buffer(outch, 0, 1617 d_image->base.phys0 + 1618 dst_tile->offset); 1619 if (s_image->fmt->planar) 1620 ipu_cpmem_set_uv_offset(chan->in_chan, 1621 src_tile->u_off, 1622 src_tile->v_off); 1623 if (d_image->fmt->planar) 1624 ipu_cpmem_set_uv_offset(outch, 1625 dst_tile->u_off, 1626 dst_tile->v_off); 1627 1628 ipu_idmac_select_buffer(chan->in_chan, 0); 1629 ipu_idmac_select_buffer(outch, 0); 1630 } 1631 } else if (ctx->next_tile < ctx->num_tiles - 1) { 1632 1633 src_tile = &s_image->tile[ctx->next_tile + 1]; 1634 dst_idx = ctx->out_tile_map[ctx->next_tile + 1]; 1635 dst_tile = &d_image->tile[dst_idx]; 1636 1637 ipu_cpmem_set_buffer(chan->in_chan, ctx->cur_buf_num, 1638 s_image->base.phys0 + src_tile->offset); 1639 ipu_cpmem_set_buffer(outch, ctx->cur_buf_num, 1640 d_image->base.phys0 + dst_tile->offset); 1641 1642 ipu_idmac_select_buffer(chan->in_chan, ctx->cur_buf_num); 1643 ipu_idmac_select_buffer(outch, ctx->cur_buf_num); 1644 1645 ctx->cur_buf_num ^= 1; 1646 } 1647 1648 ctx->next_tile++; 1649 return IRQ_HANDLED; 1650 done: 1651 list_add_tail(&run->list, &chan->done_q); 1652 chan->current_run = NULL; 1653 run_next(chan); 1654 return IRQ_WAKE_THREAD; 1655 } 1656 1657 static irqreturn_t norotate_irq(int irq, void *data) 1658 { 1659 struct ipu_image_convert_chan *chan = data; 1660 struct ipu_image_convert_ctx *ctx; 1661 struct ipu_image_convert_run *run; 1662 unsigned long flags; 1663 irqreturn_t ret; 1664 1665 spin_lock_irqsave(&chan->irqlock, flags); 1666 1667 /* get current run and its context */ 1668 run = chan->current_run; 1669 if (!run) { 1670 ret = IRQ_NONE; 1671 goto out; 1672 } 1673 1674 ctx = run->ctx; 1675 1676 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 1677 /* this is a rotation operation, just ignore */ 1678 spin_unlock_irqrestore(&chan->irqlock, flags); 1679 return IRQ_HANDLED; 1680 } 1681 1682 ret = do_irq(run); 1683 out: 1684 spin_unlock_irqrestore(&chan->irqlock, flags); 1685 return ret; 1686 } 1687 1688 static irqreturn_t rotate_irq(int irq, void *data) 1689 { 1690 struct ipu_image_convert_chan *chan = data; 1691 struct ipu_image_convert_priv *priv = chan->priv; 1692 struct ipu_image_convert_ctx *ctx; 1693 struct ipu_image_convert_run *run; 1694 unsigned long flags; 1695 irqreturn_t ret; 1696 1697 spin_lock_irqsave(&chan->irqlock, flags); 1698 1699 /* get current run and its context */ 1700 run = chan->current_run; 1701 if (!run) { 1702 ret = IRQ_NONE; 1703 goto out; 1704 } 1705 1706 ctx = run->ctx; 1707 1708 if (!ipu_rot_mode_is_irt(ctx->rot_mode)) { 1709 /* this was NOT a rotation operation, shouldn't happen */ 1710 dev_err(priv->ipu->dev, "Unexpected rotation interrupt\n"); 1711 spin_unlock_irqrestore(&chan->irqlock, flags); 1712 return IRQ_HANDLED; 1713 } 1714 1715 ret = do_irq(run); 1716 out: 1717 spin_unlock_irqrestore(&chan->irqlock, flags); 1718 return ret; 1719 } 1720 1721 /* 1722 * try to force the completion of runs for this ctx. Called when 1723 * abort wait times out in ipu_image_convert_abort(). 1724 */ 1725 static void force_abort(struct ipu_image_convert_ctx *ctx) 1726 { 1727 struct ipu_image_convert_chan *chan = ctx->chan; 1728 struct ipu_image_convert_run *run; 1729 unsigned long flags; 1730 1731 spin_lock_irqsave(&chan->irqlock, flags); 1732 1733 run = chan->current_run; 1734 if (run && run->ctx == ctx) { 1735 convert_stop(run); 1736 run->status = -EIO; 1737 list_add_tail(&run->list, &chan->done_q); 1738 chan->current_run = NULL; 1739 run_next(chan); 1740 } 1741 1742 spin_unlock_irqrestore(&chan->irqlock, flags); 1743 1744 empty_done_q(chan); 1745 } 1746 1747 static void release_ipu_resources(struct ipu_image_convert_chan *chan) 1748 { 1749 if (chan->out_eof_irq >= 0) 1750 free_irq(chan->out_eof_irq, chan); 1751 if (chan->rot_out_eof_irq >= 0) 1752 free_irq(chan->rot_out_eof_irq, chan); 1753 1754 if (!IS_ERR_OR_NULL(chan->in_chan)) 1755 ipu_idmac_put(chan->in_chan); 1756 if (!IS_ERR_OR_NULL(chan->out_chan)) 1757 ipu_idmac_put(chan->out_chan); 1758 if (!IS_ERR_OR_NULL(chan->rotation_in_chan)) 1759 ipu_idmac_put(chan->rotation_in_chan); 1760 if (!IS_ERR_OR_NULL(chan->rotation_out_chan)) 1761 ipu_idmac_put(chan->rotation_out_chan); 1762 if (!IS_ERR_OR_NULL(chan->ic)) 1763 ipu_ic_put(chan->ic); 1764 1765 chan->in_chan = chan->out_chan = chan->rotation_in_chan = 1766 chan->rotation_out_chan = NULL; 1767 chan->out_eof_irq = chan->rot_out_eof_irq = -1; 1768 } 1769 1770 static int get_ipu_resources(struct ipu_image_convert_chan *chan) 1771 { 1772 const struct ipu_image_convert_dma_chan *dma = chan->dma_ch; 1773 struct ipu_image_convert_priv *priv = chan->priv; 1774 int ret; 1775 1776 /* get IC */ 1777 chan->ic = ipu_ic_get(priv->ipu, chan->ic_task); 1778 if (IS_ERR(chan->ic)) { 1779 dev_err(priv->ipu->dev, "could not acquire IC\n"); 1780 ret = PTR_ERR(chan->ic); 1781 goto err; 1782 } 1783 1784 /* get IDMAC channels */ 1785 chan->in_chan = ipu_idmac_get(priv->ipu, dma->in); 1786 chan->out_chan = ipu_idmac_get(priv->ipu, dma->out); 1787 if (IS_ERR(chan->in_chan) || IS_ERR(chan->out_chan)) { 1788 dev_err(priv->ipu->dev, "could not acquire idmac channels\n"); 1789 ret = -EBUSY; 1790 goto err; 1791 } 1792 1793 chan->rotation_in_chan = ipu_idmac_get(priv->ipu, dma->rot_in); 1794 chan->rotation_out_chan = ipu_idmac_get(priv->ipu, dma->rot_out); 1795 if (IS_ERR(chan->rotation_in_chan) || IS_ERR(chan->rotation_out_chan)) { 1796 dev_err(priv->ipu->dev, 1797 "could not acquire idmac rotation channels\n"); 1798 ret = -EBUSY; 1799 goto err; 1800 } 1801 1802 /* acquire the EOF interrupts */ 1803 chan->out_eof_irq = ipu_idmac_channel_irq(priv->ipu, 1804 chan->out_chan, 1805 IPU_IRQ_EOF); 1806 1807 ret = request_threaded_irq(chan->out_eof_irq, norotate_irq, do_bh, 1808 0, "ipu-ic", chan); 1809 if (ret < 0) { 1810 dev_err(priv->ipu->dev, "could not acquire irq %d\n", 1811 chan->out_eof_irq); 1812 chan->out_eof_irq = -1; 1813 goto err; 1814 } 1815 1816 chan->rot_out_eof_irq = ipu_idmac_channel_irq(priv->ipu, 1817 chan->rotation_out_chan, 1818 IPU_IRQ_EOF); 1819 1820 ret = request_threaded_irq(chan->rot_out_eof_irq, rotate_irq, do_bh, 1821 0, "ipu-ic", chan); 1822 if (ret < 0) { 1823 dev_err(priv->ipu->dev, "could not acquire irq %d\n", 1824 chan->rot_out_eof_irq); 1825 chan->rot_out_eof_irq = -1; 1826 goto err; 1827 } 1828 1829 return 0; 1830 err: 1831 release_ipu_resources(chan); 1832 return ret; 1833 } 1834 1835 static int fill_image(struct ipu_image_convert_ctx *ctx, 1836 struct ipu_image_convert_image *ic_image, 1837 struct ipu_image *image, 1838 enum ipu_image_convert_type type) 1839 { 1840 struct ipu_image_convert_priv *priv = ctx->chan->priv; 1841 1842 ic_image->base = *image; 1843 ic_image->type = type; 1844 1845 ic_image->fmt = get_format(image->pix.pixelformat); 1846 if (!ic_image->fmt) { 1847 dev_err(priv->ipu->dev, "pixelformat not supported for %s\n", 1848 type == IMAGE_CONVERT_OUT ? "Output" : "Input"); 1849 return -EINVAL; 1850 } 1851 1852 if (ic_image->fmt->planar) 1853 ic_image->stride = ic_image->base.pix.width; 1854 else 1855 ic_image->stride = ic_image->base.pix.bytesperline; 1856 1857 return 0; 1858 } 1859 1860 /* borrowed from drivers/media/v4l2-core/v4l2-common.c */ 1861 static unsigned int clamp_align(unsigned int x, unsigned int min, 1862 unsigned int max, unsigned int align) 1863 { 1864 /* Bits that must be zero to be aligned */ 1865 unsigned int mask = ~((1 << align) - 1); 1866 1867 /* Clamp to aligned min and max */ 1868 x = clamp(x, (min + ~mask) & mask, max & mask); 1869 1870 /* Round to nearest aligned value */ 1871 if (align) 1872 x = (x + (1 << (align - 1))) & mask; 1873 1874 return x; 1875 } 1876 1877 /* Adjusts input/output images to IPU restrictions */ 1878 void ipu_image_convert_adjust(struct ipu_image *in, struct ipu_image *out, 1879 enum ipu_rotate_mode rot_mode) 1880 { 1881 const struct ipu_image_pixfmt *infmt, *outfmt; 1882 u32 w_align_out, h_align_out; 1883 u32 w_align_in, h_align_in; 1884 1885 infmt = get_format(in->pix.pixelformat); 1886 outfmt = get_format(out->pix.pixelformat); 1887 1888 /* set some default pixel formats if needed */ 1889 if (!infmt) { 1890 in->pix.pixelformat = V4L2_PIX_FMT_RGB24; 1891 infmt = get_format(V4L2_PIX_FMT_RGB24); 1892 } 1893 if (!outfmt) { 1894 out->pix.pixelformat = V4L2_PIX_FMT_RGB24; 1895 outfmt = get_format(V4L2_PIX_FMT_RGB24); 1896 } 1897 1898 /* image converter does not handle fields */ 1899 in->pix.field = out->pix.field = V4L2_FIELD_NONE; 1900 1901 /* resizer cannot downsize more than 4:1 */ 1902 if (ipu_rot_mode_is_irt(rot_mode)) { 1903 out->pix.height = max_t(__u32, out->pix.height, 1904 in->pix.width / 4); 1905 out->pix.width = max_t(__u32, out->pix.width, 1906 in->pix.height / 4); 1907 } else { 1908 out->pix.width = max_t(__u32, out->pix.width, 1909 in->pix.width / 4); 1910 out->pix.height = max_t(__u32, out->pix.height, 1911 in->pix.height / 4); 1912 } 1913 1914 /* align input width/height */ 1915 w_align_in = ilog2(tile_width_align(IMAGE_CONVERT_IN, infmt, 1916 rot_mode)); 1917 h_align_in = ilog2(tile_height_align(IMAGE_CONVERT_IN, infmt, 1918 rot_mode)); 1919 in->pix.width = clamp_align(in->pix.width, MIN_W, MAX_W, 1920 w_align_in); 1921 in->pix.height = clamp_align(in->pix.height, MIN_H, MAX_H, 1922 h_align_in); 1923 1924 /* align output width/height */ 1925 w_align_out = ilog2(tile_width_align(IMAGE_CONVERT_OUT, outfmt, 1926 rot_mode)); 1927 h_align_out = ilog2(tile_height_align(IMAGE_CONVERT_OUT, outfmt, 1928 rot_mode)); 1929 out->pix.width = clamp_align(out->pix.width, MIN_W, MAX_W, 1930 w_align_out); 1931 out->pix.height = clamp_align(out->pix.height, MIN_H, MAX_H, 1932 h_align_out); 1933 1934 /* set input/output strides and image sizes */ 1935 in->pix.bytesperline = infmt->planar ? 1936 clamp_align(in->pix.width, 2 << w_align_in, MAX_W, 1937 w_align_in) : 1938 clamp_align((in->pix.width * infmt->bpp) >> 3, 1939 ((2 << w_align_in) * infmt->bpp) >> 3, 1940 (MAX_W * infmt->bpp) >> 3, 1941 w_align_in); 1942 in->pix.sizeimage = infmt->planar ? 1943 (in->pix.height * in->pix.bytesperline * infmt->bpp) >> 3 : 1944 in->pix.height * in->pix.bytesperline; 1945 out->pix.bytesperline = outfmt->planar ? out->pix.width : 1946 (out->pix.width * outfmt->bpp) >> 3; 1947 out->pix.sizeimage = outfmt->planar ? 1948 (out->pix.height * out->pix.bytesperline * outfmt->bpp) >> 3 : 1949 out->pix.height * out->pix.bytesperline; 1950 } 1951 EXPORT_SYMBOL_GPL(ipu_image_convert_adjust); 1952 1953 /* 1954 * this is used by ipu_image_convert_prepare() to verify set input and 1955 * output images are valid before starting the conversion. Clients can 1956 * also call it before calling ipu_image_convert_prepare(). 1957 */ 1958 int ipu_image_convert_verify(struct ipu_image *in, struct ipu_image *out, 1959 enum ipu_rotate_mode rot_mode) 1960 { 1961 struct ipu_image testin, testout; 1962 1963 testin = *in; 1964 testout = *out; 1965 1966 ipu_image_convert_adjust(&testin, &testout, rot_mode); 1967 1968 if (testin.pix.width != in->pix.width || 1969 testin.pix.height != in->pix.height || 1970 testout.pix.width != out->pix.width || 1971 testout.pix.height != out->pix.height) 1972 return -EINVAL; 1973 1974 return 0; 1975 } 1976 EXPORT_SYMBOL_GPL(ipu_image_convert_verify); 1977 1978 /* 1979 * Call ipu_image_convert_prepare() to prepare for the conversion of 1980 * given images and rotation mode. Returns a new conversion context. 1981 */ 1982 struct ipu_image_convert_ctx * 1983 ipu_image_convert_prepare(struct ipu_soc *ipu, enum ipu_ic_task ic_task, 1984 struct ipu_image *in, struct ipu_image *out, 1985 enum ipu_rotate_mode rot_mode, 1986 ipu_image_convert_cb_t complete, 1987 void *complete_context) 1988 { 1989 struct ipu_image_convert_priv *priv = ipu->image_convert_priv; 1990 struct ipu_image_convert_image *s_image, *d_image; 1991 struct ipu_image_convert_chan *chan; 1992 struct ipu_image_convert_ctx *ctx; 1993 unsigned long flags; 1994 unsigned int i; 1995 bool get_res; 1996 int ret; 1997 1998 if (!in || !out || !complete || 1999 (ic_task != IC_TASK_VIEWFINDER && 2000 ic_task != IC_TASK_POST_PROCESSOR)) 2001 return ERR_PTR(-EINVAL); 2002 2003 /* verify the in/out images before continuing */ 2004 ret = ipu_image_convert_verify(in, out, rot_mode); 2005 if (ret) { 2006 dev_err(priv->ipu->dev, "%s: in/out formats invalid\n", 2007 __func__); 2008 return ERR_PTR(ret); 2009 } 2010 2011 chan = &priv->chan[ic_task]; 2012 2013 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 2014 if (!ctx) 2015 return ERR_PTR(-ENOMEM); 2016 2017 dev_dbg(priv->ipu->dev, "%s: task %u: ctx %p\n", __func__, 2018 chan->ic_task, ctx); 2019 2020 ctx->chan = chan; 2021 init_completion(&ctx->aborted); 2022 2023 s_image = &ctx->in; 2024 d_image = &ctx->out; 2025 2026 /* set tiling and rotation */ 2027 d_image->num_rows = num_stripes(out->pix.height); 2028 d_image->num_cols = num_stripes(out->pix.width); 2029 if (ipu_rot_mode_is_irt(rot_mode)) { 2030 s_image->num_rows = d_image->num_cols; 2031 s_image->num_cols = d_image->num_rows; 2032 } else { 2033 s_image->num_rows = d_image->num_rows; 2034 s_image->num_cols = d_image->num_cols; 2035 } 2036 2037 ctx->num_tiles = d_image->num_cols * d_image->num_rows; 2038 ctx->rot_mode = rot_mode; 2039 2040 ret = fill_image(ctx, s_image, in, IMAGE_CONVERT_IN); 2041 if (ret) 2042 goto out_free; 2043 ret = fill_image(ctx, d_image, out, IMAGE_CONVERT_OUT); 2044 if (ret) 2045 goto out_free; 2046 2047 ret = calc_image_resize_coefficients(ctx, in, out); 2048 if (ret) 2049 goto out_free; 2050 2051 calc_out_tile_map(ctx); 2052 2053 find_seams(ctx, s_image, d_image); 2054 2055 calc_tile_dimensions(ctx, s_image); 2056 ret = calc_tile_offsets(ctx, s_image); 2057 if (ret) 2058 goto out_free; 2059 2060 calc_tile_dimensions(ctx, d_image); 2061 ret = calc_tile_offsets(ctx, d_image); 2062 if (ret) 2063 goto out_free; 2064 2065 calc_tile_resize_coefficients(ctx); 2066 2067 ret = ipu_ic_calc_csc(&ctx->csc, 2068 s_image->base.pix.ycbcr_enc, 2069 s_image->base.pix.quantization, 2070 ipu_pixelformat_to_colorspace(s_image->fmt->fourcc), 2071 d_image->base.pix.ycbcr_enc, 2072 d_image->base.pix.quantization, 2073 ipu_pixelformat_to_colorspace(d_image->fmt->fourcc)); 2074 if (ret) 2075 goto out_free; 2076 2077 dump_format(ctx, s_image); 2078 dump_format(ctx, d_image); 2079 2080 ctx->complete = complete; 2081 ctx->complete_context = complete_context; 2082 2083 /* 2084 * Can we use double-buffering for this operation? If there is 2085 * only one tile (the whole image can be converted in a single 2086 * operation) there's no point in using double-buffering. Also, 2087 * the IPU's IDMAC channels allow only a single U and V plane 2088 * offset shared between both buffers, but these offsets change 2089 * for every tile, and therefore would have to be updated for 2090 * each buffer which is not possible. So double-buffering is 2091 * impossible when either the source or destination images are 2092 * a planar format (YUV420, YUV422P, etc.). Further, differently 2093 * sized tiles or different resizing coefficients per tile 2094 * prevent double-buffering as well. 2095 */ 2096 ctx->double_buffering = (ctx->num_tiles > 1 && 2097 !s_image->fmt->planar && 2098 !d_image->fmt->planar); 2099 for (i = 1; i < ctx->num_tiles; i++) { 2100 if (ctx->in.tile[i].width != ctx->in.tile[0].width || 2101 ctx->in.tile[i].height != ctx->in.tile[0].height || 2102 ctx->out.tile[i].width != ctx->out.tile[0].width || 2103 ctx->out.tile[i].height != ctx->out.tile[0].height) { 2104 ctx->double_buffering = false; 2105 break; 2106 } 2107 } 2108 for (i = 1; i < ctx->in.num_cols; i++) { 2109 if (ctx->resize_coeffs_h[i] != ctx->resize_coeffs_h[0]) { 2110 ctx->double_buffering = false; 2111 break; 2112 } 2113 } 2114 for (i = 1; i < ctx->in.num_rows; i++) { 2115 if (ctx->resize_coeffs_v[i] != ctx->resize_coeffs_v[0]) { 2116 ctx->double_buffering = false; 2117 break; 2118 } 2119 } 2120 2121 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 2122 unsigned long intermediate_size = d_image->tile[0].size; 2123 2124 for (i = 1; i < ctx->num_tiles; i++) { 2125 if (d_image->tile[i].size > intermediate_size) 2126 intermediate_size = d_image->tile[i].size; 2127 } 2128 2129 ret = alloc_dma_buf(priv, &ctx->rot_intermediate[0], 2130 intermediate_size); 2131 if (ret) 2132 goto out_free; 2133 if (ctx->double_buffering) { 2134 ret = alloc_dma_buf(priv, 2135 &ctx->rot_intermediate[1], 2136 intermediate_size); 2137 if (ret) 2138 goto out_free_dmabuf0; 2139 } 2140 } 2141 2142 spin_lock_irqsave(&chan->irqlock, flags); 2143 2144 get_res = list_empty(&chan->ctx_list); 2145 2146 list_add_tail(&ctx->list, &chan->ctx_list); 2147 2148 spin_unlock_irqrestore(&chan->irqlock, flags); 2149 2150 if (get_res) { 2151 ret = get_ipu_resources(chan); 2152 if (ret) 2153 goto out_free_dmabuf1; 2154 } 2155 2156 return ctx; 2157 2158 out_free_dmabuf1: 2159 free_dma_buf(priv, &ctx->rot_intermediate[1]); 2160 spin_lock_irqsave(&chan->irqlock, flags); 2161 list_del(&ctx->list); 2162 spin_unlock_irqrestore(&chan->irqlock, flags); 2163 out_free_dmabuf0: 2164 free_dma_buf(priv, &ctx->rot_intermediate[0]); 2165 out_free: 2166 kfree(ctx); 2167 return ERR_PTR(ret); 2168 } 2169 EXPORT_SYMBOL_GPL(ipu_image_convert_prepare); 2170 2171 /* 2172 * Carry out a single image conversion run. Only the physaddr's of the input 2173 * and output image buffers are needed. The conversion context must have 2174 * been created previously with ipu_image_convert_prepare(). 2175 */ 2176 int ipu_image_convert_queue(struct ipu_image_convert_run *run) 2177 { 2178 struct ipu_image_convert_chan *chan; 2179 struct ipu_image_convert_priv *priv; 2180 struct ipu_image_convert_ctx *ctx; 2181 unsigned long flags; 2182 int ret = 0; 2183 2184 if (!run || !run->ctx || !run->in_phys || !run->out_phys) 2185 return -EINVAL; 2186 2187 ctx = run->ctx; 2188 chan = ctx->chan; 2189 priv = chan->priv; 2190 2191 dev_dbg(priv->ipu->dev, "%s: task %u: ctx %p run %p\n", __func__, 2192 chan->ic_task, ctx, run); 2193 2194 INIT_LIST_HEAD(&run->list); 2195 2196 spin_lock_irqsave(&chan->irqlock, flags); 2197 2198 if (ctx->aborting) { 2199 ret = -EIO; 2200 goto unlock; 2201 } 2202 2203 list_add_tail(&run->list, &chan->pending_q); 2204 2205 if (!chan->current_run) { 2206 ret = do_run(run); 2207 if (ret) 2208 chan->current_run = NULL; 2209 } 2210 unlock: 2211 spin_unlock_irqrestore(&chan->irqlock, flags); 2212 return ret; 2213 } 2214 EXPORT_SYMBOL_GPL(ipu_image_convert_queue); 2215 2216 /* Abort any active or pending conversions for this context */ 2217 static void __ipu_image_convert_abort(struct ipu_image_convert_ctx *ctx) 2218 { 2219 struct ipu_image_convert_chan *chan = ctx->chan; 2220 struct ipu_image_convert_priv *priv = chan->priv; 2221 struct ipu_image_convert_run *run, *active_run, *tmp; 2222 unsigned long flags; 2223 int run_count, ret; 2224 2225 spin_lock_irqsave(&chan->irqlock, flags); 2226 2227 /* move all remaining pending runs in this context to done_q */ 2228 list_for_each_entry_safe(run, tmp, &chan->pending_q, list) { 2229 if (run->ctx != ctx) 2230 continue; 2231 run->status = -EIO; 2232 list_move_tail(&run->list, &chan->done_q); 2233 } 2234 2235 run_count = get_run_count(ctx, &chan->done_q); 2236 active_run = (chan->current_run && chan->current_run->ctx == ctx) ? 2237 chan->current_run : NULL; 2238 2239 if (active_run) 2240 reinit_completion(&ctx->aborted); 2241 2242 ctx->aborting = true; 2243 2244 spin_unlock_irqrestore(&chan->irqlock, flags); 2245 2246 if (!run_count && !active_run) { 2247 dev_dbg(priv->ipu->dev, 2248 "%s: task %u: no abort needed for ctx %p\n", 2249 __func__, chan->ic_task, ctx); 2250 return; 2251 } 2252 2253 if (!active_run) { 2254 empty_done_q(chan); 2255 return; 2256 } 2257 2258 dev_dbg(priv->ipu->dev, 2259 "%s: task %u: wait for completion: %d runs\n", 2260 __func__, chan->ic_task, run_count); 2261 2262 ret = wait_for_completion_timeout(&ctx->aborted, 2263 msecs_to_jiffies(10000)); 2264 if (ret == 0) { 2265 dev_warn(priv->ipu->dev, "%s: timeout\n", __func__); 2266 force_abort(ctx); 2267 } 2268 } 2269 2270 void ipu_image_convert_abort(struct ipu_image_convert_ctx *ctx) 2271 { 2272 __ipu_image_convert_abort(ctx); 2273 ctx->aborting = false; 2274 } 2275 EXPORT_SYMBOL_GPL(ipu_image_convert_abort); 2276 2277 /* Unprepare image conversion context */ 2278 void ipu_image_convert_unprepare(struct ipu_image_convert_ctx *ctx) 2279 { 2280 struct ipu_image_convert_chan *chan = ctx->chan; 2281 struct ipu_image_convert_priv *priv = chan->priv; 2282 unsigned long flags; 2283 bool put_res; 2284 2285 /* make sure no runs are hanging around */ 2286 __ipu_image_convert_abort(ctx); 2287 2288 dev_dbg(priv->ipu->dev, "%s: task %u: removing ctx %p\n", __func__, 2289 chan->ic_task, ctx); 2290 2291 spin_lock_irqsave(&chan->irqlock, flags); 2292 2293 list_del(&ctx->list); 2294 2295 put_res = list_empty(&chan->ctx_list); 2296 2297 spin_unlock_irqrestore(&chan->irqlock, flags); 2298 2299 if (put_res) 2300 release_ipu_resources(chan); 2301 2302 free_dma_buf(priv, &ctx->rot_intermediate[1]); 2303 free_dma_buf(priv, &ctx->rot_intermediate[0]); 2304 2305 kfree(ctx); 2306 } 2307 EXPORT_SYMBOL_GPL(ipu_image_convert_unprepare); 2308 2309 /* 2310 * "Canned" asynchronous single image conversion. Allocates and returns 2311 * a new conversion run. On successful return the caller must free the 2312 * run and call ipu_image_convert_unprepare() after conversion completes. 2313 */ 2314 struct ipu_image_convert_run * 2315 ipu_image_convert(struct ipu_soc *ipu, enum ipu_ic_task ic_task, 2316 struct ipu_image *in, struct ipu_image *out, 2317 enum ipu_rotate_mode rot_mode, 2318 ipu_image_convert_cb_t complete, 2319 void *complete_context) 2320 { 2321 struct ipu_image_convert_ctx *ctx; 2322 struct ipu_image_convert_run *run; 2323 int ret; 2324 2325 ctx = ipu_image_convert_prepare(ipu, ic_task, in, out, rot_mode, 2326 complete, complete_context); 2327 if (IS_ERR(ctx)) 2328 return ERR_CAST(ctx); 2329 2330 run = kzalloc(sizeof(*run), GFP_KERNEL); 2331 if (!run) { 2332 ipu_image_convert_unprepare(ctx); 2333 return ERR_PTR(-ENOMEM); 2334 } 2335 2336 run->ctx = ctx; 2337 run->in_phys = in->phys0; 2338 run->out_phys = out->phys0; 2339 2340 ret = ipu_image_convert_queue(run); 2341 if (ret) { 2342 ipu_image_convert_unprepare(ctx); 2343 kfree(run); 2344 return ERR_PTR(ret); 2345 } 2346 2347 return run; 2348 } 2349 EXPORT_SYMBOL_GPL(ipu_image_convert); 2350 2351 /* "Canned" synchronous single image conversion */ 2352 static void image_convert_sync_complete(struct ipu_image_convert_run *run, 2353 void *data) 2354 { 2355 struct completion *comp = data; 2356 2357 complete(comp); 2358 } 2359 2360 int ipu_image_convert_sync(struct ipu_soc *ipu, enum ipu_ic_task ic_task, 2361 struct ipu_image *in, struct ipu_image *out, 2362 enum ipu_rotate_mode rot_mode) 2363 { 2364 struct ipu_image_convert_run *run; 2365 struct completion comp; 2366 int ret; 2367 2368 init_completion(&comp); 2369 2370 run = ipu_image_convert(ipu, ic_task, in, out, rot_mode, 2371 image_convert_sync_complete, &comp); 2372 if (IS_ERR(run)) 2373 return PTR_ERR(run); 2374 2375 ret = wait_for_completion_timeout(&comp, msecs_to_jiffies(10000)); 2376 ret = (ret == 0) ? -ETIMEDOUT : 0; 2377 2378 ipu_image_convert_unprepare(run->ctx); 2379 kfree(run); 2380 2381 return ret; 2382 } 2383 EXPORT_SYMBOL_GPL(ipu_image_convert_sync); 2384 2385 int ipu_image_convert_init(struct ipu_soc *ipu, struct device *dev) 2386 { 2387 struct ipu_image_convert_priv *priv; 2388 int i; 2389 2390 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); 2391 if (!priv) 2392 return -ENOMEM; 2393 2394 ipu->image_convert_priv = priv; 2395 priv->ipu = ipu; 2396 2397 for (i = 0; i < IC_NUM_TASKS; i++) { 2398 struct ipu_image_convert_chan *chan = &priv->chan[i]; 2399 2400 chan->ic_task = i; 2401 chan->priv = priv; 2402 chan->dma_ch = &image_convert_dma_chan[i]; 2403 chan->out_eof_irq = -1; 2404 chan->rot_out_eof_irq = -1; 2405 2406 spin_lock_init(&chan->irqlock); 2407 INIT_LIST_HEAD(&chan->ctx_list); 2408 INIT_LIST_HEAD(&chan->pending_q); 2409 INIT_LIST_HEAD(&chan->done_q); 2410 } 2411 2412 return 0; 2413 } 2414 2415 void ipu_image_convert_exit(struct ipu_soc *ipu) 2416 { 2417 } 2418