1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2012-2016 Mentor Graphics Inc. 4 * 5 * Queued image conversion support, with tiling and rotation. 6 */ 7 8 #include <linux/interrupt.h> 9 #include <linux/dma-mapping.h> 10 #include <video/imx-ipu-image-convert.h> 11 #include "ipu-prv.h" 12 13 /* 14 * The IC Resizer has a restriction that the output frame from the 15 * resizer must be 1024 or less in both width (pixels) and height 16 * (lines). 17 * 18 * The image converter attempts to split up a conversion when 19 * the desired output (converted) frame resolution exceeds the 20 * IC resizer limit of 1024 in either dimension. 21 * 22 * If either dimension of the output frame exceeds the limit, the 23 * dimension is split into 1, 2, or 4 equal stripes, for a maximum 24 * of 4*4 or 16 tiles. A conversion is then carried out for each 25 * tile (but taking care to pass the full frame stride length to 26 * the DMA channel's parameter memory!). IDMA double-buffering is used 27 * to convert each tile back-to-back when possible (see note below 28 * when double_buffering boolean is set). 29 * 30 * Note that the input frame must be split up into the same number 31 * of tiles as the output frame: 32 * 33 * +---------+-----+ 34 * +-----+---+ | A | B | 35 * | A | B | | | | 36 * +-----+---+ --> +---------+-----+ 37 * | C | D | | C | D | 38 * +-----+---+ | | | 39 * +---------+-----+ 40 * 41 * Clockwise 90° rotations are handled by first rescaling into a 42 * reusable temporary tile buffer and then rotating with the 8x8 43 * block rotator, writing to the correct destination: 44 * 45 * +-----+-----+ 46 * | | | 47 * +-----+---+ +---------+ | C | A | 48 * | A | B | | A,B, | | | | | 49 * +-----+---+ --> | C,D | | --> | | | 50 * | C | D | +---------+ +-----+-----+ 51 * +-----+---+ | D | B | 52 * | | | 53 * +-----+-----+ 54 * 55 * If the 8x8 block rotator is used, horizontal or vertical flipping 56 * is done during the rotation step, otherwise flipping is done 57 * during the scaling step. 58 * With rotation or flipping, tile order changes between input and 59 * output image. Tiles are numbered row major from top left to bottom 60 * right for both input and output image. 61 */ 62 63 #define MAX_STRIPES_W 4 64 #define MAX_STRIPES_H 4 65 #define MAX_TILES (MAX_STRIPES_W * MAX_STRIPES_H) 66 67 #define MIN_W 16 68 #define MIN_H 8 69 #define MAX_W 4096 70 #define MAX_H 4096 71 72 enum ipu_image_convert_type { 73 IMAGE_CONVERT_IN = 0, 74 IMAGE_CONVERT_OUT, 75 }; 76 77 struct ipu_image_convert_dma_buf { 78 void *virt; 79 dma_addr_t phys; 80 unsigned long len; 81 }; 82 83 struct ipu_image_convert_dma_chan { 84 int in; 85 int out; 86 int rot_in; 87 int rot_out; 88 int vdi_in_p; 89 int vdi_in; 90 int vdi_in_n; 91 }; 92 93 /* dimensions of one tile */ 94 struct ipu_image_tile { 95 u32 width; 96 u32 height; 97 u32 left; 98 u32 top; 99 /* size and strides are in bytes */ 100 u32 size; 101 u32 stride; 102 u32 rot_stride; 103 /* start Y or packed offset of this tile */ 104 u32 offset; 105 /* offset from start to tile in U plane, for planar formats */ 106 u32 u_off; 107 /* offset from start to tile in V plane, for planar formats */ 108 u32 v_off; 109 }; 110 111 struct ipu_image_convert_image { 112 struct ipu_image base; 113 enum ipu_image_convert_type type; 114 115 const struct ipu_image_pixfmt *fmt; 116 unsigned int stride; 117 118 /* # of rows (horizontal stripes) if dest height is > 1024 */ 119 unsigned int num_rows; 120 /* # of columns (vertical stripes) if dest width is > 1024 */ 121 unsigned int num_cols; 122 123 struct ipu_image_tile tile[MAX_TILES]; 124 }; 125 126 struct ipu_image_pixfmt { 127 u32 fourcc; /* V4L2 fourcc */ 128 int bpp; /* total bpp */ 129 int uv_width_dec; /* decimation in width for U/V planes */ 130 int uv_height_dec; /* decimation in height for U/V planes */ 131 bool planar; /* planar format */ 132 bool uv_swapped; /* U and V planes are swapped */ 133 bool uv_packed; /* partial planar (U and V in same plane) */ 134 }; 135 136 struct ipu_image_convert_ctx; 137 struct ipu_image_convert_chan; 138 struct ipu_image_convert_priv; 139 140 struct ipu_image_convert_ctx { 141 struct ipu_image_convert_chan *chan; 142 143 ipu_image_convert_cb_t complete; 144 void *complete_context; 145 146 /* Source/destination image data and rotation mode */ 147 struct ipu_image_convert_image in; 148 struct ipu_image_convert_image out; 149 enum ipu_rotate_mode rot_mode; 150 u32 downsize_coeff_h; 151 u32 downsize_coeff_v; 152 u32 image_resize_coeff_h; 153 u32 image_resize_coeff_v; 154 u32 resize_coeffs_h[MAX_STRIPES_W]; 155 u32 resize_coeffs_v[MAX_STRIPES_H]; 156 157 /* intermediate buffer for rotation */ 158 struct ipu_image_convert_dma_buf rot_intermediate[2]; 159 160 /* current buffer number for double buffering */ 161 int cur_buf_num; 162 163 bool aborting; 164 struct completion aborted; 165 166 /* can we use double-buffering for this conversion operation? */ 167 bool double_buffering; 168 /* num_rows * num_cols */ 169 unsigned int num_tiles; 170 /* next tile to process */ 171 unsigned int next_tile; 172 /* where to place converted tile in dest image */ 173 unsigned int out_tile_map[MAX_TILES]; 174 175 struct list_head list; 176 }; 177 178 struct ipu_image_convert_chan { 179 struct ipu_image_convert_priv *priv; 180 181 enum ipu_ic_task ic_task; 182 const struct ipu_image_convert_dma_chan *dma_ch; 183 184 struct ipu_ic *ic; 185 struct ipuv3_channel *in_chan; 186 struct ipuv3_channel *out_chan; 187 struct ipuv3_channel *rotation_in_chan; 188 struct ipuv3_channel *rotation_out_chan; 189 190 /* the IPU end-of-frame irqs */ 191 int out_eof_irq; 192 int rot_out_eof_irq; 193 194 spinlock_t irqlock; 195 196 /* list of convert contexts */ 197 struct list_head ctx_list; 198 /* queue of conversion runs */ 199 struct list_head pending_q; 200 /* queue of completed runs */ 201 struct list_head done_q; 202 203 /* the current conversion run */ 204 struct ipu_image_convert_run *current_run; 205 }; 206 207 struct ipu_image_convert_priv { 208 struct ipu_image_convert_chan chan[IC_NUM_TASKS]; 209 struct ipu_soc *ipu; 210 }; 211 212 static const struct ipu_image_convert_dma_chan 213 image_convert_dma_chan[IC_NUM_TASKS] = { 214 [IC_TASK_VIEWFINDER] = { 215 .in = IPUV3_CHANNEL_MEM_IC_PRP_VF, 216 .out = IPUV3_CHANNEL_IC_PRP_VF_MEM, 217 .rot_in = IPUV3_CHANNEL_MEM_ROT_VF, 218 .rot_out = IPUV3_CHANNEL_ROT_VF_MEM, 219 .vdi_in_p = IPUV3_CHANNEL_MEM_VDI_PREV, 220 .vdi_in = IPUV3_CHANNEL_MEM_VDI_CUR, 221 .vdi_in_n = IPUV3_CHANNEL_MEM_VDI_NEXT, 222 }, 223 [IC_TASK_POST_PROCESSOR] = { 224 .in = IPUV3_CHANNEL_MEM_IC_PP, 225 .out = IPUV3_CHANNEL_IC_PP_MEM, 226 .rot_in = IPUV3_CHANNEL_MEM_ROT_PP, 227 .rot_out = IPUV3_CHANNEL_ROT_PP_MEM, 228 }, 229 }; 230 231 static const struct ipu_image_pixfmt image_convert_formats[] = { 232 { 233 .fourcc = V4L2_PIX_FMT_RGB565, 234 .bpp = 16, 235 }, { 236 .fourcc = V4L2_PIX_FMT_RGB24, 237 .bpp = 24, 238 }, { 239 .fourcc = V4L2_PIX_FMT_BGR24, 240 .bpp = 24, 241 }, { 242 .fourcc = V4L2_PIX_FMT_RGB32, 243 .bpp = 32, 244 }, { 245 .fourcc = V4L2_PIX_FMT_BGR32, 246 .bpp = 32, 247 }, { 248 .fourcc = V4L2_PIX_FMT_XRGB32, 249 .bpp = 32, 250 }, { 251 .fourcc = V4L2_PIX_FMT_XBGR32, 252 .bpp = 32, 253 }, { 254 .fourcc = V4L2_PIX_FMT_YUYV, 255 .bpp = 16, 256 .uv_width_dec = 2, 257 .uv_height_dec = 1, 258 }, { 259 .fourcc = V4L2_PIX_FMT_UYVY, 260 .bpp = 16, 261 .uv_width_dec = 2, 262 .uv_height_dec = 1, 263 }, { 264 .fourcc = V4L2_PIX_FMT_YUV420, 265 .bpp = 12, 266 .planar = true, 267 .uv_width_dec = 2, 268 .uv_height_dec = 2, 269 }, { 270 .fourcc = V4L2_PIX_FMT_YVU420, 271 .bpp = 12, 272 .planar = true, 273 .uv_width_dec = 2, 274 .uv_height_dec = 2, 275 .uv_swapped = true, 276 }, { 277 .fourcc = V4L2_PIX_FMT_NV12, 278 .bpp = 12, 279 .planar = true, 280 .uv_width_dec = 2, 281 .uv_height_dec = 2, 282 .uv_packed = true, 283 }, { 284 .fourcc = V4L2_PIX_FMT_YUV422P, 285 .bpp = 16, 286 .planar = true, 287 .uv_width_dec = 2, 288 .uv_height_dec = 1, 289 }, { 290 .fourcc = V4L2_PIX_FMT_NV16, 291 .bpp = 16, 292 .planar = true, 293 .uv_width_dec = 2, 294 .uv_height_dec = 1, 295 .uv_packed = true, 296 }, 297 }; 298 299 static const struct ipu_image_pixfmt *get_format(u32 fourcc) 300 { 301 const struct ipu_image_pixfmt *ret = NULL; 302 unsigned int i; 303 304 for (i = 0; i < ARRAY_SIZE(image_convert_formats); i++) { 305 if (image_convert_formats[i].fourcc == fourcc) { 306 ret = &image_convert_formats[i]; 307 break; 308 } 309 } 310 311 return ret; 312 } 313 314 static void dump_format(struct ipu_image_convert_ctx *ctx, 315 struct ipu_image_convert_image *ic_image) 316 { 317 struct ipu_image_convert_chan *chan = ctx->chan; 318 struct ipu_image_convert_priv *priv = chan->priv; 319 320 dev_dbg(priv->ipu->dev, 321 "task %u: ctx %p: %s format: %dx%d (%dx%d tiles), %c%c%c%c\n", 322 chan->ic_task, ctx, 323 ic_image->type == IMAGE_CONVERT_OUT ? "Output" : "Input", 324 ic_image->base.pix.width, ic_image->base.pix.height, 325 ic_image->num_cols, ic_image->num_rows, 326 ic_image->fmt->fourcc & 0xff, 327 (ic_image->fmt->fourcc >> 8) & 0xff, 328 (ic_image->fmt->fourcc >> 16) & 0xff, 329 (ic_image->fmt->fourcc >> 24) & 0xff); 330 } 331 332 int ipu_image_convert_enum_format(int index, u32 *fourcc) 333 { 334 const struct ipu_image_pixfmt *fmt; 335 336 if (index >= (int)ARRAY_SIZE(image_convert_formats)) 337 return -EINVAL; 338 339 /* Format found */ 340 fmt = &image_convert_formats[index]; 341 *fourcc = fmt->fourcc; 342 return 0; 343 } 344 EXPORT_SYMBOL_GPL(ipu_image_convert_enum_format); 345 346 static void free_dma_buf(struct ipu_image_convert_priv *priv, 347 struct ipu_image_convert_dma_buf *buf) 348 { 349 if (buf->virt) 350 dma_free_coherent(priv->ipu->dev, 351 buf->len, buf->virt, buf->phys); 352 buf->virt = NULL; 353 buf->phys = 0; 354 } 355 356 static int alloc_dma_buf(struct ipu_image_convert_priv *priv, 357 struct ipu_image_convert_dma_buf *buf, 358 int size) 359 { 360 buf->len = PAGE_ALIGN(size); 361 buf->virt = dma_alloc_coherent(priv->ipu->dev, buf->len, &buf->phys, 362 GFP_DMA | GFP_KERNEL); 363 if (!buf->virt) { 364 dev_err(priv->ipu->dev, "failed to alloc dma buffer\n"); 365 return -ENOMEM; 366 } 367 368 return 0; 369 } 370 371 static inline int num_stripes(int dim) 372 { 373 return (dim - 1) / 1024 + 1; 374 } 375 376 /* 377 * Calculate downsizing coefficients, which are the same for all tiles, 378 * and bilinear resizing coefficients, which are used to find the best 379 * seam positions. 380 */ 381 static int calc_image_resize_coefficients(struct ipu_image_convert_ctx *ctx, 382 struct ipu_image *in, 383 struct ipu_image *out) 384 { 385 u32 downsized_width = in->rect.width; 386 u32 downsized_height = in->rect.height; 387 u32 downsize_coeff_v = 0; 388 u32 downsize_coeff_h = 0; 389 u32 resized_width = out->rect.width; 390 u32 resized_height = out->rect.height; 391 u32 resize_coeff_h; 392 u32 resize_coeff_v; 393 394 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 395 resized_width = out->rect.height; 396 resized_height = out->rect.width; 397 } 398 399 /* Do not let invalid input lead to an endless loop below */ 400 if (WARN_ON(resized_width == 0 || resized_height == 0)) 401 return -EINVAL; 402 403 while (downsized_width > 1024 || 404 downsized_width >= resized_width * 2) { 405 downsized_width >>= 1; 406 downsize_coeff_h++; 407 } 408 409 while (downsized_height > 1024 || 410 downsized_height >= resized_height * 2) { 411 downsized_height >>= 1; 412 downsize_coeff_v++; 413 } 414 415 /* 416 * Calculate the bilinear resizing coefficients that could be used if 417 * we were converting with a single tile. The bottom right output pixel 418 * should sample as close as possible to the bottom right input pixel 419 * out of the decimator, but not overshoot it: 420 */ 421 resize_coeff_h = 8192 * (downsized_width - 1) / (resized_width - 1); 422 resize_coeff_v = 8192 * (downsized_height - 1) / (resized_height - 1); 423 424 dev_dbg(ctx->chan->priv->ipu->dev, 425 "%s: hscale: >>%u, *8192/%u vscale: >>%u, *8192/%u, %ux%u tiles\n", 426 __func__, downsize_coeff_h, resize_coeff_h, downsize_coeff_v, 427 resize_coeff_v, ctx->in.num_cols, ctx->in.num_rows); 428 429 if (downsize_coeff_h > 2 || downsize_coeff_v > 2 || 430 resize_coeff_h > 0x3fff || resize_coeff_v > 0x3fff) 431 return -EINVAL; 432 433 ctx->downsize_coeff_h = downsize_coeff_h; 434 ctx->downsize_coeff_v = downsize_coeff_v; 435 ctx->image_resize_coeff_h = resize_coeff_h; 436 ctx->image_resize_coeff_v = resize_coeff_v; 437 438 return 0; 439 } 440 441 #define round_closest(x, y) round_down((x) + (y)/2, (y)) 442 443 /* 444 * Find the best aligned seam position in the inverval [out_start, out_end]. 445 * Rotation and image offsets are out of scope. 446 * 447 * @out_start: start of inverval, must be within 1024 pixels / lines 448 * of out_end 449 * @out_end: end of interval, smaller than or equal to out_edge 450 * @in_edge: input right / bottom edge 451 * @out_edge: output right / bottom edge 452 * @in_align: input alignment, either horizontal 8-byte line start address 453 * alignment, or pixel alignment due to image format 454 * @out_align: output alignment, either horizontal 8-byte line start address 455 * alignment, or pixel alignment due to image format or rotator 456 * block size 457 * @in_burst: horizontal input burst size in case of horizontal flip 458 * @out_burst: horizontal output burst size or rotator block size 459 * @downsize_coeff: downsizing section coefficient 460 * @resize_coeff: main processing section resizing coefficient 461 * @_in_seam: aligned input seam position return value 462 * @_out_seam: aligned output seam position return value 463 */ 464 static void find_best_seam(struct ipu_image_convert_ctx *ctx, 465 unsigned int out_start, 466 unsigned int out_end, 467 unsigned int in_edge, 468 unsigned int out_edge, 469 unsigned int in_align, 470 unsigned int out_align, 471 unsigned int in_burst, 472 unsigned int out_burst, 473 unsigned int downsize_coeff, 474 unsigned int resize_coeff, 475 u32 *_in_seam, 476 u32 *_out_seam) 477 { 478 struct device *dev = ctx->chan->priv->ipu->dev; 479 unsigned int out_pos; 480 /* Input / output seam position candidates */ 481 unsigned int out_seam = 0; 482 unsigned int in_seam = 0; 483 unsigned int min_diff = UINT_MAX; 484 485 /* 486 * Output tiles must start at a multiple of 8 bytes horizontally and 487 * possibly at an even line horizontally depending on the pixel format. 488 * Only consider output aligned positions for the seam. 489 */ 490 out_start = round_up(out_start, out_align); 491 for (out_pos = out_start; out_pos < out_end; out_pos += out_align) { 492 unsigned int in_pos; 493 unsigned int in_pos_aligned; 494 unsigned int abs_diff; 495 496 /* 497 * Tiles in the right row / bottom column may not be allowed to 498 * overshoot horizontally / vertically. out_burst may be the 499 * actual DMA burst size, or the rotator block size. 500 */ 501 if ((out_burst > 1) && (out_edge - out_pos) % out_burst) 502 continue; 503 504 /* 505 * Input sample position, corresponding to out_pos, 19.13 fixed 506 * point. 507 */ 508 in_pos = (out_pos * resize_coeff) << downsize_coeff; 509 /* 510 * The closest input sample position that we could actually 511 * start the input tile at, 19.13 fixed point. 512 */ 513 in_pos_aligned = round_closest(in_pos, 8192U * in_align); 514 515 if ((in_burst > 1) && 516 (in_edge - in_pos_aligned / 8192U) % in_burst) 517 continue; 518 519 if (in_pos < in_pos_aligned) 520 abs_diff = in_pos_aligned - in_pos; 521 else 522 abs_diff = in_pos - in_pos_aligned; 523 524 if (abs_diff < min_diff) { 525 in_seam = in_pos_aligned; 526 out_seam = out_pos; 527 min_diff = abs_diff; 528 } 529 } 530 531 *_out_seam = out_seam; 532 /* Convert 19.13 fixed point to integer seam position */ 533 *_in_seam = DIV_ROUND_CLOSEST(in_seam, 8192U); 534 535 dev_dbg(dev, "%s: out_seam %u(%u) in [%u, %u], in_seam %u(%u) diff %u.%03u\n", 536 __func__, out_seam, out_align, out_start, out_end, 537 *_in_seam, in_align, min_diff / 8192, 538 DIV_ROUND_CLOSEST(min_diff % 8192 * 1000, 8192)); 539 } 540 541 /* 542 * Tile left edges are required to be aligned to multiples of 8 bytes 543 * by the IDMAC. 544 */ 545 static inline u32 tile_left_align(const struct ipu_image_pixfmt *fmt) 546 { 547 if (fmt->planar) 548 return fmt->uv_packed ? 8 : 8 * fmt->uv_width_dec; 549 else 550 return fmt->bpp == 32 ? 2 : fmt->bpp == 16 ? 4 : 8; 551 } 552 553 /* 554 * Tile top edge alignment is only limited by chroma subsampling. 555 */ 556 static inline u32 tile_top_align(const struct ipu_image_pixfmt *fmt) 557 { 558 return fmt->uv_height_dec > 1 ? 2 : 1; 559 } 560 561 static inline u32 tile_width_align(enum ipu_image_convert_type type, 562 const struct ipu_image_pixfmt *fmt, 563 enum ipu_rotate_mode rot_mode) 564 { 565 if (type == IMAGE_CONVERT_IN) { 566 /* 567 * The IC burst reads 8 pixels at a time. Reading beyond the 568 * end of the line is usually acceptable. Those pixels are 569 * ignored, unless the IC has to write the scaled line in 570 * reverse. 571 */ 572 return (!ipu_rot_mode_is_irt(rot_mode) && 573 (rot_mode & IPU_ROT_BIT_HFLIP)) ? 8 : 2; 574 } 575 576 /* 577 * Align to 16x16 pixel blocks for planar 4:2:0 chroma subsampled 578 * formats to guarantee 8-byte aligned line start addresses in the 579 * chroma planes when IRT is used. Align to 8x8 pixel IRT block size 580 * for all other formats. 581 */ 582 return (ipu_rot_mode_is_irt(rot_mode) && 583 fmt->planar && !fmt->uv_packed) ? 584 8 * fmt->uv_width_dec : 8; 585 } 586 587 static inline u32 tile_height_align(enum ipu_image_convert_type type, 588 const struct ipu_image_pixfmt *fmt, 589 enum ipu_rotate_mode rot_mode) 590 { 591 if (type == IMAGE_CONVERT_IN || !ipu_rot_mode_is_irt(rot_mode)) 592 return 2; 593 594 /* 595 * Align to 16x16 pixel blocks for planar 4:2:0 chroma subsampled 596 * formats to guarantee 8-byte aligned line start addresses in the 597 * chroma planes when IRT is used. Align to 8x8 pixel IRT block size 598 * for all other formats. 599 */ 600 return (fmt->planar && !fmt->uv_packed) ? 8 * fmt->uv_width_dec : 8; 601 } 602 603 /* 604 * Fill in left position and width and for all tiles in an input column, and 605 * for all corresponding output tiles. If the 90° rotator is used, the output 606 * tiles are in a row, and output tile top position and height are set. 607 */ 608 static void fill_tile_column(struct ipu_image_convert_ctx *ctx, 609 unsigned int col, 610 struct ipu_image_convert_image *in, 611 unsigned int in_left, unsigned int in_width, 612 struct ipu_image_convert_image *out, 613 unsigned int out_left, unsigned int out_width) 614 { 615 unsigned int row, tile_idx; 616 struct ipu_image_tile *in_tile, *out_tile; 617 618 for (row = 0; row < in->num_rows; row++) { 619 tile_idx = in->num_cols * row + col; 620 in_tile = &in->tile[tile_idx]; 621 out_tile = &out->tile[ctx->out_tile_map[tile_idx]]; 622 623 in_tile->left = in_left; 624 in_tile->width = in_width; 625 626 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 627 out_tile->top = out_left; 628 out_tile->height = out_width; 629 } else { 630 out_tile->left = out_left; 631 out_tile->width = out_width; 632 } 633 } 634 } 635 636 /* 637 * Fill in top position and height and for all tiles in an input row, and 638 * for all corresponding output tiles. If the 90° rotator is used, the output 639 * tiles are in a column, and output tile left position and width are set. 640 */ 641 static void fill_tile_row(struct ipu_image_convert_ctx *ctx, unsigned int row, 642 struct ipu_image_convert_image *in, 643 unsigned int in_top, unsigned int in_height, 644 struct ipu_image_convert_image *out, 645 unsigned int out_top, unsigned int out_height) 646 { 647 unsigned int col, tile_idx; 648 struct ipu_image_tile *in_tile, *out_tile; 649 650 for (col = 0; col < in->num_cols; col++) { 651 tile_idx = in->num_cols * row + col; 652 in_tile = &in->tile[tile_idx]; 653 out_tile = &out->tile[ctx->out_tile_map[tile_idx]]; 654 655 in_tile->top = in_top; 656 in_tile->height = in_height; 657 658 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 659 out_tile->left = out_top; 660 out_tile->width = out_height; 661 } else { 662 out_tile->top = out_top; 663 out_tile->height = out_height; 664 } 665 } 666 } 667 668 /* 669 * Find the best horizontal and vertical seam positions to split into tiles. 670 * Minimize the fractional part of the input sampling position for the 671 * top / left pixels of each tile. 672 */ 673 static void find_seams(struct ipu_image_convert_ctx *ctx, 674 struct ipu_image_convert_image *in, 675 struct ipu_image_convert_image *out) 676 { 677 struct device *dev = ctx->chan->priv->ipu->dev; 678 unsigned int resized_width = out->base.rect.width; 679 unsigned int resized_height = out->base.rect.height; 680 unsigned int col; 681 unsigned int row; 682 unsigned int in_left_align = tile_left_align(in->fmt); 683 unsigned int in_top_align = tile_top_align(in->fmt); 684 unsigned int out_left_align = tile_left_align(out->fmt); 685 unsigned int out_top_align = tile_top_align(out->fmt); 686 unsigned int out_width_align = tile_width_align(out->type, out->fmt, 687 ctx->rot_mode); 688 unsigned int out_height_align = tile_height_align(out->type, out->fmt, 689 ctx->rot_mode); 690 unsigned int in_right = in->base.rect.width; 691 unsigned int in_bottom = in->base.rect.height; 692 unsigned int out_right = out->base.rect.width; 693 unsigned int out_bottom = out->base.rect.height; 694 unsigned int flipped_out_left; 695 unsigned int flipped_out_top; 696 697 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 698 /* Switch width/height and align top left to IRT block size */ 699 resized_width = out->base.rect.height; 700 resized_height = out->base.rect.width; 701 out_left_align = out_height_align; 702 out_top_align = out_width_align; 703 out_width_align = out_left_align; 704 out_height_align = out_top_align; 705 out_right = out->base.rect.height; 706 out_bottom = out->base.rect.width; 707 } 708 709 for (col = in->num_cols - 1; col > 0; col--) { 710 bool allow_in_overshoot = ipu_rot_mode_is_irt(ctx->rot_mode) || 711 !(ctx->rot_mode & IPU_ROT_BIT_HFLIP); 712 bool allow_out_overshoot = (col < in->num_cols - 1) && 713 !(ctx->rot_mode & IPU_ROT_BIT_HFLIP); 714 unsigned int out_start; 715 unsigned int out_end; 716 unsigned int in_left; 717 unsigned int out_left; 718 719 /* 720 * Align input width to burst length if the scaling step flips 721 * horizontally. 722 */ 723 724 /* Start within 1024 pixels of the right edge */ 725 out_start = max_t(int, 0, out_right - 1024); 726 /* End before having to add more columns to the left */ 727 out_end = min_t(unsigned int, out_right, col * 1024); 728 729 find_best_seam(ctx, out_start, out_end, 730 in_right, out_right, 731 in_left_align, out_left_align, 732 allow_in_overshoot ? 1 : 8 /* burst length */, 733 allow_out_overshoot ? 1 : out_width_align, 734 ctx->downsize_coeff_h, ctx->image_resize_coeff_h, 735 &in_left, &out_left); 736 737 if (ctx->rot_mode & IPU_ROT_BIT_HFLIP) 738 flipped_out_left = resized_width - out_right; 739 else 740 flipped_out_left = out_left; 741 742 fill_tile_column(ctx, col, in, in_left, in_right - in_left, 743 out, flipped_out_left, out_right - out_left); 744 745 dev_dbg(dev, "%s: col %u: %u, %u -> %u, %u\n", __func__, col, 746 in_left, in_right - in_left, 747 flipped_out_left, out_right - out_left); 748 749 in_right = in_left; 750 out_right = out_left; 751 } 752 753 flipped_out_left = (ctx->rot_mode & IPU_ROT_BIT_HFLIP) ? 754 resized_width - out_right : 0; 755 756 fill_tile_column(ctx, 0, in, 0, in_right, 757 out, flipped_out_left, out_right); 758 759 dev_dbg(dev, "%s: col 0: 0, %u -> %u, %u\n", __func__, 760 in_right, flipped_out_left, out_right); 761 762 for (row = in->num_rows - 1; row > 0; row--) { 763 bool allow_overshoot = row < in->num_rows - 1; 764 unsigned int out_start; 765 unsigned int out_end; 766 unsigned int in_top; 767 unsigned int out_top; 768 769 /* Start within 1024 lines of the bottom edge */ 770 out_start = max_t(int, 0, out_bottom - 1024); 771 /* End before having to add more rows above */ 772 out_end = min_t(unsigned int, out_bottom, row * 1024); 773 774 find_best_seam(ctx, out_start, out_end, 775 in_bottom, out_bottom, 776 in_top_align, out_top_align, 777 1, allow_overshoot ? 1 : out_height_align, 778 ctx->downsize_coeff_v, ctx->image_resize_coeff_v, 779 &in_top, &out_top); 780 781 if ((ctx->rot_mode & IPU_ROT_BIT_VFLIP) ^ 782 ipu_rot_mode_is_irt(ctx->rot_mode)) 783 flipped_out_top = resized_height - out_bottom; 784 else 785 flipped_out_top = out_top; 786 787 fill_tile_row(ctx, row, in, in_top, in_bottom - in_top, 788 out, flipped_out_top, out_bottom - out_top); 789 790 dev_dbg(dev, "%s: row %u: %u, %u -> %u, %u\n", __func__, row, 791 in_top, in_bottom - in_top, 792 flipped_out_top, out_bottom - out_top); 793 794 in_bottom = in_top; 795 out_bottom = out_top; 796 } 797 798 if ((ctx->rot_mode & IPU_ROT_BIT_VFLIP) ^ 799 ipu_rot_mode_is_irt(ctx->rot_mode)) 800 flipped_out_top = resized_height - out_bottom; 801 else 802 flipped_out_top = 0; 803 804 fill_tile_row(ctx, 0, in, 0, in_bottom, 805 out, flipped_out_top, out_bottom); 806 807 dev_dbg(dev, "%s: row 0: 0, %u -> %u, %u\n", __func__, 808 in_bottom, flipped_out_top, out_bottom); 809 } 810 811 static void calc_tile_dimensions(struct ipu_image_convert_ctx *ctx, 812 struct ipu_image_convert_image *image) 813 { 814 struct ipu_image_convert_chan *chan = ctx->chan; 815 struct ipu_image_convert_priv *priv = chan->priv; 816 unsigned int i; 817 818 for (i = 0; i < ctx->num_tiles; i++) { 819 struct ipu_image_tile *tile; 820 const unsigned int row = i / image->num_cols; 821 const unsigned int col = i % image->num_cols; 822 823 if (image->type == IMAGE_CONVERT_OUT) 824 tile = &image->tile[ctx->out_tile_map[i]]; 825 else 826 tile = &image->tile[i]; 827 828 tile->size = ((tile->height * image->fmt->bpp) >> 3) * 829 tile->width; 830 831 if (image->fmt->planar) { 832 tile->stride = tile->width; 833 tile->rot_stride = tile->height; 834 } else { 835 tile->stride = 836 (image->fmt->bpp * tile->width) >> 3; 837 tile->rot_stride = 838 (image->fmt->bpp * tile->height) >> 3; 839 } 840 841 dev_dbg(priv->ipu->dev, 842 "task %u: ctx %p: %s@[%u,%u]: %ux%u@%u,%u\n", 843 chan->ic_task, ctx, 844 image->type == IMAGE_CONVERT_IN ? "Input" : "Output", 845 row, col, 846 tile->width, tile->height, tile->left, tile->top); 847 } 848 } 849 850 /* 851 * Use the rotation transformation to find the tile coordinates 852 * (row, col) of a tile in the destination frame that corresponds 853 * to the given tile coordinates of a source frame. The destination 854 * coordinate is then converted to a tile index. 855 */ 856 static int transform_tile_index(struct ipu_image_convert_ctx *ctx, 857 int src_row, int src_col) 858 { 859 struct ipu_image_convert_chan *chan = ctx->chan; 860 struct ipu_image_convert_priv *priv = chan->priv; 861 struct ipu_image_convert_image *s_image = &ctx->in; 862 struct ipu_image_convert_image *d_image = &ctx->out; 863 int dst_row, dst_col; 864 865 /* with no rotation it's a 1:1 mapping */ 866 if (ctx->rot_mode == IPU_ROTATE_NONE) 867 return src_row * s_image->num_cols + src_col; 868 869 /* 870 * before doing the transform, first we have to translate 871 * source row,col for an origin in the center of s_image 872 */ 873 src_row = src_row * 2 - (s_image->num_rows - 1); 874 src_col = src_col * 2 - (s_image->num_cols - 1); 875 876 /* do the rotation transform */ 877 if (ctx->rot_mode & IPU_ROT_BIT_90) { 878 dst_col = -src_row; 879 dst_row = src_col; 880 } else { 881 dst_col = src_col; 882 dst_row = src_row; 883 } 884 885 /* apply flip */ 886 if (ctx->rot_mode & IPU_ROT_BIT_HFLIP) 887 dst_col = -dst_col; 888 if (ctx->rot_mode & IPU_ROT_BIT_VFLIP) 889 dst_row = -dst_row; 890 891 dev_dbg(priv->ipu->dev, "task %u: ctx %p: [%d,%d] --> [%d,%d]\n", 892 chan->ic_task, ctx, src_col, src_row, dst_col, dst_row); 893 894 /* 895 * finally translate dest row,col using an origin in upper 896 * left of d_image 897 */ 898 dst_row += d_image->num_rows - 1; 899 dst_col += d_image->num_cols - 1; 900 dst_row /= 2; 901 dst_col /= 2; 902 903 return dst_row * d_image->num_cols + dst_col; 904 } 905 906 /* 907 * Fill the out_tile_map[] with transformed destination tile indeces. 908 */ 909 static void calc_out_tile_map(struct ipu_image_convert_ctx *ctx) 910 { 911 struct ipu_image_convert_image *s_image = &ctx->in; 912 unsigned int row, col, tile = 0; 913 914 for (row = 0; row < s_image->num_rows; row++) { 915 for (col = 0; col < s_image->num_cols; col++) { 916 ctx->out_tile_map[tile] = 917 transform_tile_index(ctx, row, col); 918 tile++; 919 } 920 } 921 } 922 923 static int calc_tile_offsets_planar(struct ipu_image_convert_ctx *ctx, 924 struct ipu_image_convert_image *image) 925 { 926 struct ipu_image_convert_chan *chan = ctx->chan; 927 struct ipu_image_convert_priv *priv = chan->priv; 928 const struct ipu_image_pixfmt *fmt = image->fmt; 929 unsigned int row, col, tile = 0; 930 u32 H, top, y_stride, uv_stride; 931 u32 uv_row_off, uv_col_off, uv_off, u_off, v_off, tmp; 932 u32 y_row_off, y_col_off, y_off; 933 u32 y_size, uv_size; 934 935 /* setup some convenience vars */ 936 H = image->base.pix.height; 937 938 y_stride = image->stride; 939 uv_stride = y_stride / fmt->uv_width_dec; 940 if (fmt->uv_packed) 941 uv_stride *= 2; 942 943 y_size = H * y_stride; 944 uv_size = y_size / (fmt->uv_width_dec * fmt->uv_height_dec); 945 946 for (row = 0; row < image->num_rows; row++) { 947 top = image->tile[tile].top; 948 y_row_off = top * y_stride; 949 uv_row_off = (top * uv_stride) / fmt->uv_height_dec; 950 951 for (col = 0; col < image->num_cols; col++) { 952 y_col_off = image->tile[tile].left; 953 uv_col_off = y_col_off / fmt->uv_width_dec; 954 if (fmt->uv_packed) 955 uv_col_off *= 2; 956 957 y_off = y_row_off + y_col_off; 958 uv_off = uv_row_off + uv_col_off; 959 960 u_off = y_size - y_off + uv_off; 961 v_off = (fmt->uv_packed) ? 0 : u_off + uv_size; 962 if (fmt->uv_swapped) { 963 tmp = u_off; 964 u_off = v_off; 965 v_off = tmp; 966 } 967 968 image->tile[tile].offset = y_off; 969 image->tile[tile].u_off = u_off; 970 image->tile[tile++].v_off = v_off; 971 972 if ((y_off & 0x7) || (u_off & 0x7) || (v_off & 0x7)) { 973 dev_err(priv->ipu->dev, 974 "task %u: ctx %p: %s@[%d,%d]: " 975 "y_off %08x, u_off %08x, v_off %08x\n", 976 chan->ic_task, ctx, 977 image->type == IMAGE_CONVERT_IN ? 978 "Input" : "Output", row, col, 979 y_off, u_off, v_off); 980 return -EINVAL; 981 } 982 } 983 } 984 985 return 0; 986 } 987 988 static int calc_tile_offsets_packed(struct ipu_image_convert_ctx *ctx, 989 struct ipu_image_convert_image *image) 990 { 991 struct ipu_image_convert_chan *chan = ctx->chan; 992 struct ipu_image_convert_priv *priv = chan->priv; 993 const struct ipu_image_pixfmt *fmt = image->fmt; 994 unsigned int row, col, tile = 0; 995 u32 bpp, stride, offset; 996 u32 row_off, col_off; 997 998 /* setup some convenience vars */ 999 stride = image->stride; 1000 bpp = fmt->bpp; 1001 1002 for (row = 0; row < image->num_rows; row++) { 1003 row_off = image->tile[tile].top * stride; 1004 1005 for (col = 0; col < image->num_cols; col++) { 1006 col_off = (image->tile[tile].left * bpp) >> 3; 1007 1008 offset = row_off + col_off; 1009 1010 image->tile[tile].offset = offset; 1011 image->tile[tile].u_off = 0; 1012 image->tile[tile++].v_off = 0; 1013 1014 if (offset & 0x7) { 1015 dev_err(priv->ipu->dev, 1016 "task %u: ctx %p: %s@[%d,%d]: " 1017 "phys %08x\n", 1018 chan->ic_task, ctx, 1019 image->type == IMAGE_CONVERT_IN ? 1020 "Input" : "Output", row, col, 1021 row_off + col_off); 1022 return -EINVAL; 1023 } 1024 } 1025 } 1026 1027 return 0; 1028 } 1029 1030 static int calc_tile_offsets(struct ipu_image_convert_ctx *ctx, 1031 struct ipu_image_convert_image *image) 1032 { 1033 if (image->fmt->planar) 1034 return calc_tile_offsets_planar(ctx, image); 1035 1036 return calc_tile_offsets_packed(ctx, image); 1037 } 1038 1039 /* 1040 * Calculate the resizing ratio for the IC main processing section given input 1041 * size, fixed downsizing coefficient, and output size. 1042 * Either round to closest for the next tile's first pixel to minimize seams 1043 * and distortion (for all but right column / bottom row), or round down to 1044 * avoid sampling beyond the edges of the input image for this tile's last 1045 * pixel. 1046 * Returns the resizing coefficient, resizing ratio is 8192.0 / resize_coeff. 1047 */ 1048 static u32 calc_resize_coeff(u32 input_size, u32 downsize_coeff, 1049 u32 output_size, bool allow_overshoot) 1050 { 1051 u32 downsized = input_size >> downsize_coeff; 1052 1053 if (allow_overshoot) 1054 return DIV_ROUND_CLOSEST(8192 * downsized, output_size); 1055 else 1056 return 8192 * (downsized - 1) / (output_size - 1); 1057 } 1058 1059 /* 1060 * Slightly modify resize coefficients per tile to hide the bilinear 1061 * interpolator reset at tile borders, shifting the right / bottom edge 1062 * by up to a half input pixel. This removes noticeable seams between 1063 * tiles at higher upscaling factors. 1064 */ 1065 static void calc_tile_resize_coefficients(struct ipu_image_convert_ctx *ctx) 1066 { 1067 struct ipu_image_convert_chan *chan = ctx->chan; 1068 struct ipu_image_convert_priv *priv = chan->priv; 1069 struct ipu_image_tile *in_tile, *out_tile; 1070 unsigned int col, row, tile_idx; 1071 unsigned int last_output; 1072 1073 for (col = 0; col < ctx->in.num_cols; col++) { 1074 bool closest = (col < ctx->in.num_cols - 1) && 1075 !(ctx->rot_mode & IPU_ROT_BIT_HFLIP); 1076 u32 resized_width; 1077 u32 resize_coeff_h; 1078 1079 tile_idx = col; 1080 in_tile = &ctx->in.tile[tile_idx]; 1081 out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]]; 1082 1083 if (ipu_rot_mode_is_irt(ctx->rot_mode)) 1084 resized_width = out_tile->height; 1085 else 1086 resized_width = out_tile->width; 1087 1088 resize_coeff_h = calc_resize_coeff(in_tile->width, 1089 ctx->downsize_coeff_h, 1090 resized_width, closest); 1091 1092 dev_dbg(priv->ipu->dev, "%s: column %u hscale: *8192/%u\n", 1093 __func__, col, resize_coeff_h); 1094 1095 1096 for (row = 0; row < ctx->in.num_rows; row++) { 1097 tile_idx = row * ctx->in.num_cols + col; 1098 in_tile = &ctx->in.tile[tile_idx]; 1099 out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]]; 1100 1101 /* 1102 * With the horizontal scaling factor known, round up 1103 * resized width (output width or height) to burst size. 1104 */ 1105 if (ipu_rot_mode_is_irt(ctx->rot_mode)) 1106 out_tile->height = round_up(resized_width, 8); 1107 else 1108 out_tile->width = round_up(resized_width, 8); 1109 1110 /* 1111 * Calculate input width from the last accessed input 1112 * pixel given resized width and scaling coefficients. 1113 * Round up to burst size. 1114 */ 1115 last_output = round_up(resized_width, 8) - 1; 1116 if (closest) 1117 last_output++; 1118 in_tile->width = round_up( 1119 (DIV_ROUND_UP(last_output * resize_coeff_h, 1120 8192) + 1) 1121 << ctx->downsize_coeff_h, 8); 1122 } 1123 1124 ctx->resize_coeffs_h[col] = resize_coeff_h; 1125 } 1126 1127 for (row = 0; row < ctx->in.num_rows; row++) { 1128 bool closest = (row < ctx->in.num_rows - 1) && 1129 !(ctx->rot_mode & IPU_ROT_BIT_VFLIP); 1130 u32 resized_height; 1131 u32 resize_coeff_v; 1132 1133 tile_idx = row * ctx->in.num_cols; 1134 in_tile = &ctx->in.tile[tile_idx]; 1135 out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]]; 1136 1137 if (ipu_rot_mode_is_irt(ctx->rot_mode)) 1138 resized_height = out_tile->width; 1139 else 1140 resized_height = out_tile->height; 1141 1142 resize_coeff_v = calc_resize_coeff(in_tile->height, 1143 ctx->downsize_coeff_v, 1144 resized_height, closest); 1145 1146 dev_dbg(priv->ipu->dev, "%s: row %u vscale: *8192/%u\n", 1147 __func__, row, resize_coeff_v); 1148 1149 for (col = 0; col < ctx->in.num_cols; col++) { 1150 tile_idx = row * ctx->in.num_cols + col; 1151 in_tile = &ctx->in.tile[tile_idx]; 1152 out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]]; 1153 1154 /* 1155 * With the vertical scaling factor known, round up 1156 * resized height (output width or height) to IDMAC 1157 * limitations. 1158 */ 1159 if (ipu_rot_mode_is_irt(ctx->rot_mode)) 1160 out_tile->width = round_up(resized_height, 2); 1161 else 1162 out_tile->height = round_up(resized_height, 2); 1163 1164 /* 1165 * Calculate input width from the last accessed input 1166 * pixel given resized height and scaling coefficients. 1167 * Align to IDMAC restrictions. 1168 */ 1169 last_output = round_up(resized_height, 2) - 1; 1170 if (closest) 1171 last_output++; 1172 in_tile->height = round_up( 1173 (DIV_ROUND_UP(last_output * resize_coeff_v, 1174 8192) + 1) 1175 << ctx->downsize_coeff_v, 2); 1176 } 1177 1178 ctx->resize_coeffs_v[row] = resize_coeff_v; 1179 } 1180 } 1181 1182 /* 1183 * return the number of runs in given queue (pending_q or done_q) 1184 * for this context. hold irqlock when calling. 1185 */ 1186 static int get_run_count(struct ipu_image_convert_ctx *ctx, 1187 struct list_head *q) 1188 { 1189 struct ipu_image_convert_run *run; 1190 int count = 0; 1191 1192 lockdep_assert_held(&ctx->chan->irqlock); 1193 1194 list_for_each_entry(run, q, list) { 1195 if (run->ctx == ctx) 1196 count++; 1197 } 1198 1199 return count; 1200 } 1201 1202 static void convert_stop(struct ipu_image_convert_run *run) 1203 { 1204 struct ipu_image_convert_ctx *ctx = run->ctx; 1205 struct ipu_image_convert_chan *chan = ctx->chan; 1206 struct ipu_image_convert_priv *priv = chan->priv; 1207 1208 dev_dbg(priv->ipu->dev, "%s: task %u: stopping ctx %p run %p\n", 1209 __func__, chan->ic_task, ctx, run); 1210 1211 /* disable IC tasks and the channels */ 1212 ipu_ic_task_disable(chan->ic); 1213 ipu_idmac_disable_channel(chan->in_chan); 1214 ipu_idmac_disable_channel(chan->out_chan); 1215 1216 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 1217 ipu_idmac_disable_channel(chan->rotation_in_chan); 1218 ipu_idmac_disable_channel(chan->rotation_out_chan); 1219 ipu_idmac_unlink(chan->out_chan, chan->rotation_in_chan); 1220 } 1221 1222 ipu_ic_disable(chan->ic); 1223 } 1224 1225 static void init_idmac_channel(struct ipu_image_convert_ctx *ctx, 1226 struct ipuv3_channel *channel, 1227 struct ipu_image_convert_image *image, 1228 enum ipu_rotate_mode rot_mode, 1229 bool rot_swap_width_height, 1230 unsigned int tile) 1231 { 1232 struct ipu_image_convert_chan *chan = ctx->chan; 1233 unsigned int burst_size; 1234 u32 width, height, stride; 1235 dma_addr_t addr0, addr1 = 0; 1236 struct ipu_image tile_image; 1237 unsigned int tile_idx[2]; 1238 1239 if (image->type == IMAGE_CONVERT_OUT) { 1240 tile_idx[0] = ctx->out_tile_map[tile]; 1241 tile_idx[1] = ctx->out_tile_map[1]; 1242 } else { 1243 tile_idx[0] = tile; 1244 tile_idx[1] = 1; 1245 } 1246 1247 if (rot_swap_width_height) { 1248 width = image->tile[tile_idx[0]].height; 1249 height = image->tile[tile_idx[0]].width; 1250 stride = image->tile[tile_idx[0]].rot_stride; 1251 addr0 = ctx->rot_intermediate[0].phys; 1252 if (ctx->double_buffering) 1253 addr1 = ctx->rot_intermediate[1].phys; 1254 } else { 1255 width = image->tile[tile_idx[0]].width; 1256 height = image->tile[tile_idx[0]].height; 1257 stride = image->stride; 1258 addr0 = image->base.phys0 + 1259 image->tile[tile_idx[0]].offset; 1260 if (ctx->double_buffering) 1261 addr1 = image->base.phys0 + 1262 image->tile[tile_idx[1]].offset; 1263 } 1264 1265 ipu_cpmem_zero(channel); 1266 1267 memset(&tile_image, 0, sizeof(tile_image)); 1268 tile_image.pix.width = tile_image.rect.width = width; 1269 tile_image.pix.height = tile_image.rect.height = height; 1270 tile_image.pix.bytesperline = stride; 1271 tile_image.pix.pixelformat = image->fmt->fourcc; 1272 tile_image.phys0 = addr0; 1273 tile_image.phys1 = addr1; 1274 if (image->fmt->planar && !rot_swap_width_height) { 1275 tile_image.u_offset = image->tile[tile_idx[0]].u_off; 1276 tile_image.v_offset = image->tile[tile_idx[0]].v_off; 1277 } 1278 1279 ipu_cpmem_set_image(channel, &tile_image); 1280 1281 if (rot_mode) 1282 ipu_cpmem_set_rotation(channel, rot_mode); 1283 1284 if (channel == chan->rotation_in_chan || 1285 channel == chan->rotation_out_chan) { 1286 burst_size = 8; 1287 ipu_cpmem_set_block_mode(channel); 1288 } else 1289 burst_size = (width % 16) ? 8 : 16; 1290 1291 ipu_cpmem_set_burstsize(channel, burst_size); 1292 1293 ipu_ic_task_idma_init(chan->ic, channel, width, height, 1294 burst_size, rot_mode); 1295 1296 /* 1297 * Setting a non-zero AXI ID collides with the PRG AXI snooping, so 1298 * only do this when there is no PRG present. 1299 */ 1300 if (!channel->ipu->prg_priv) 1301 ipu_cpmem_set_axi_id(channel, 1); 1302 1303 ipu_idmac_set_double_buffer(channel, ctx->double_buffering); 1304 } 1305 1306 static int convert_start(struct ipu_image_convert_run *run, unsigned int tile) 1307 { 1308 struct ipu_image_convert_ctx *ctx = run->ctx; 1309 struct ipu_image_convert_chan *chan = ctx->chan; 1310 struct ipu_image_convert_priv *priv = chan->priv; 1311 struct ipu_image_convert_image *s_image = &ctx->in; 1312 struct ipu_image_convert_image *d_image = &ctx->out; 1313 enum ipu_color_space src_cs, dest_cs; 1314 unsigned int dst_tile = ctx->out_tile_map[tile]; 1315 unsigned int dest_width, dest_height; 1316 unsigned int col, row; 1317 u32 rsc; 1318 int ret; 1319 1320 dev_dbg(priv->ipu->dev, "%s: task %u: starting ctx %p run %p tile %u -> %u\n", 1321 __func__, chan->ic_task, ctx, run, tile, dst_tile); 1322 1323 src_cs = ipu_pixelformat_to_colorspace(s_image->fmt->fourcc); 1324 dest_cs = ipu_pixelformat_to_colorspace(d_image->fmt->fourcc); 1325 1326 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 1327 /* swap width/height for resizer */ 1328 dest_width = d_image->tile[dst_tile].height; 1329 dest_height = d_image->tile[dst_tile].width; 1330 } else { 1331 dest_width = d_image->tile[dst_tile].width; 1332 dest_height = d_image->tile[dst_tile].height; 1333 } 1334 1335 row = tile / s_image->num_cols; 1336 col = tile % s_image->num_cols; 1337 1338 rsc = (ctx->downsize_coeff_v << 30) | 1339 (ctx->resize_coeffs_v[row] << 16) | 1340 (ctx->downsize_coeff_h << 14) | 1341 (ctx->resize_coeffs_h[col]); 1342 1343 dev_dbg(priv->ipu->dev, "%s: %ux%u -> %ux%u (rsc = 0x%x)\n", 1344 __func__, s_image->tile[tile].width, 1345 s_image->tile[tile].height, dest_width, dest_height, rsc); 1346 1347 /* setup the IC resizer and CSC */ 1348 ret = ipu_ic_task_init_rsc(chan->ic, 1349 s_image->tile[tile].width, 1350 s_image->tile[tile].height, 1351 dest_width, 1352 dest_height, 1353 src_cs, dest_cs, 1354 rsc); 1355 if (ret) { 1356 dev_err(priv->ipu->dev, "ipu_ic_task_init failed, %d\n", ret); 1357 return ret; 1358 } 1359 1360 /* init the source MEM-->IC PP IDMAC channel */ 1361 init_idmac_channel(ctx, chan->in_chan, s_image, 1362 IPU_ROTATE_NONE, false, tile); 1363 1364 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 1365 /* init the IC PP-->MEM IDMAC channel */ 1366 init_idmac_channel(ctx, chan->out_chan, d_image, 1367 IPU_ROTATE_NONE, true, tile); 1368 1369 /* init the MEM-->IC PP ROT IDMAC channel */ 1370 init_idmac_channel(ctx, chan->rotation_in_chan, d_image, 1371 ctx->rot_mode, true, tile); 1372 1373 /* init the destination IC PP ROT-->MEM IDMAC channel */ 1374 init_idmac_channel(ctx, chan->rotation_out_chan, d_image, 1375 IPU_ROTATE_NONE, false, tile); 1376 1377 /* now link IC PP-->MEM to MEM-->IC PP ROT */ 1378 ipu_idmac_link(chan->out_chan, chan->rotation_in_chan); 1379 } else { 1380 /* init the destination IC PP-->MEM IDMAC channel */ 1381 init_idmac_channel(ctx, chan->out_chan, d_image, 1382 ctx->rot_mode, false, tile); 1383 } 1384 1385 /* enable the IC */ 1386 ipu_ic_enable(chan->ic); 1387 1388 /* set buffers ready */ 1389 ipu_idmac_select_buffer(chan->in_chan, 0); 1390 ipu_idmac_select_buffer(chan->out_chan, 0); 1391 if (ipu_rot_mode_is_irt(ctx->rot_mode)) 1392 ipu_idmac_select_buffer(chan->rotation_out_chan, 0); 1393 if (ctx->double_buffering) { 1394 ipu_idmac_select_buffer(chan->in_chan, 1); 1395 ipu_idmac_select_buffer(chan->out_chan, 1); 1396 if (ipu_rot_mode_is_irt(ctx->rot_mode)) 1397 ipu_idmac_select_buffer(chan->rotation_out_chan, 1); 1398 } 1399 1400 /* enable the channels! */ 1401 ipu_idmac_enable_channel(chan->in_chan); 1402 ipu_idmac_enable_channel(chan->out_chan); 1403 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 1404 ipu_idmac_enable_channel(chan->rotation_in_chan); 1405 ipu_idmac_enable_channel(chan->rotation_out_chan); 1406 } 1407 1408 ipu_ic_task_enable(chan->ic); 1409 1410 ipu_cpmem_dump(chan->in_chan); 1411 ipu_cpmem_dump(chan->out_chan); 1412 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 1413 ipu_cpmem_dump(chan->rotation_in_chan); 1414 ipu_cpmem_dump(chan->rotation_out_chan); 1415 } 1416 1417 ipu_dump(priv->ipu); 1418 1419 return 0; 1420 } 1421 1422 /* hold irqlock when calling */ 1423 static int do_run(struct ipu_image_convert_run *run) 1424 { 1425 struct ipu_image_convert_ctx *ctx = run->ctx; 1426 struct ipu_image_convert_chan *chan = ctx->chan; 1427 1428 lockdep_assert_held(&chan->irqlock); 1429 1430 ctx->in.base.phys0 = run->in_phys; 1431 ctx->out.base.phys0 = run->out_phys; 1432 1433 ctx->cur_buf_num = 0; 1434 ctx->next_tile = 1; 1435 1436 /* remove run from pending_q and set as current */ 1437 list_del(&run->list); 1438 chan->current_run = run; 1439 1440 return convert_start(run, 0); 1441 } 1442 1443 /* hold irqlock when calling */ 1444 static void run_next(struct ipu_image_convert_chan *chan) 1445 { 1446 struct ipu_image_convert_priv *priv = chan->priv; 1447 struct ipu_image_convert_run *run, *tmp; 1448 int ret; 1449 1450 lockdep_assert_held(&chan->irqlock); 1451 1452 list_for_each_entry_safe(run, tmp, &chan->pending_q, list) { 1453 /* skip contexts that are aborting */ 1454 if (run->ctx->aborting) { 1455 dev_dbg(priv->ipu->dev, 1456 "%s: task %u: skipping aborting ctx %p run %p\n", 1457 __func__, chan->ic_task, run->ctx, run); 1458 continue; 1459 } 1460 1461 ret = do_run(run); 1462 if (!ret) 1463 break; 1464 1465 /* 1466 * something went wrong with start, add the run 1467 * to done q and continue to the next run in the 1468 * pending q. 1469 */ 1470 run->status = ret; 1471 list_add_tail(&run->list, &chan->done_q); 1472 chan->current_run = NULL; 1473 } 1474 } 1475 1476 static void empty_done_q(struct ipu_image_convert_chan *chan) 1477 { 1478 struct ipu_image_convert_priv *priv = chan->priv; 1479 struct ipu_image_convert_run *run; 1480 unsigned long flags; 1481 1482 spin_lock_irqsave(&chan->irqlock, flags); 1483 1484 while (!list_empty(&chan->done_q)) { 1485 run = list_entry(chan->done_q.next, 1486 struct ipu_image_convert_run, 1487 list); 1488 1489 list_del(&run->list); 1490 1491 dev_dbg(priv->ipu->dev, 1492 "%s: task %u: completing ctx %p run %p with %d\n", 1493 __func__, chan->ic_task, run->ctx, run, run->status); 1494 1495 /* call the completion callback and free the run */ 1496 spin_unlock_irqrestore(&chan->irqlock, flags); 1497 run->ctx->complete(run, run->ctx->complete_context); 1498 spin_lock_irqsave(&chan->irqlock, flags); 1499 } 1500 1501 spin_unlock_irqrestore(&chan->irqlock, flags); 1502 } 1503 1504 /* 1505 * the bottom half thread clears out the done_q, calling the 1506 * completion handler for each. 1507 */ 1508 static irqreturn_t do_bh(int irq, void *dev_id) 1509 { 1510 struct ipu_image_convert_chan *chan = dev_id; 1511 struct ipu_image_convert_priv *priv = chan->priv; 1512 struct ipu_image_convert_ctx *ctx; 1513 unsigned long flags; 1514 1515 dev_dbg(priv->ipu->dev, "%s: task %u: enter\n", __func__, 1516 chan->ic_task); 1517 1518 empty_done_q(chan); 1519 1520 spin_lock_irqsave(&chan->irqlock, flags); 1521 1522 /* 1523 * the done_q is cleared out, signal any contexts 1524 * that are aborting that abort can complete. 1525 */ 1526 list_for_each_entry(ctx, &chan->ctx_list, list) { 1527 if (ctx->aborting) { 1528 dev_dbg(priv->ipu->dev, 1529 "%s: task %u: signaling abort for ctx %p\n", 1530 __func__, chan->ic_task, ctx); 1531 complete_all(&ctx->aborted); 1532 } 1533 } 1534 1535 spin_unlock_irqrestore(&chan->irqlock, flags); 1536 1537 dev_dbg(priv->ipu->dev, "%s: task %u: exit\n", __func__, 1538 chan->ic_task); 1539 1540 return IRQ_HANDLED; 1541 } 1542 1543 static bool ic_settings_changed(struct ipu_image_convert_ctx *ctx) 1544 { 1545 unsigned int cur_tile = ctx->next_tile - 1; 1546 unsigned int next_tile = ctx->next_tile; 1547 1548 if (ctx->resize_coeffs_h[cur_tile % ctx->in.num_cols] != 1549 ctx->resize_coeffs_h[next_tile % ctx->in.num_cols] || 1550 ctx->resize_coeffs_v[cur_tile / ctx->in.num_cols] != 1551 ctx->resize_coeffs_v[next_tile / ctx->in.num_cols] || 1552 ctx->in.tile[cur_tile].width != ctx->in.tile[next_tile].width || 1553 ctx->in.tile[cur_tile].height != ctx->in.tile[next_tile].height || 1554 ctx->out.tile[cur_tile].width != ctx->out.tile[next_tile].width || 1555 ctx->out.tile[cur_tile].height != ctx->out.tile[next_tile].height) 1556 return true; 1557 1558 return false; 1559 } 1560 1561 /* hold irqlock when calling */ 1562 static irqreturn_t do_irq(struct ipu_image_convert_run *run) 1563 { 1564 struct ipu_image_convert_ctx *ctx = run->ctx; 1565 struct ipu_image_convert_chan *chan = ctx->chan; 1566 struct ipu_image_tile *src_tile, *dst_tile; 1567 struct ipu_image_convert_image *s_image = &ctx->in; 1568 struct ipu_image_convert_image *d_image = &ctx->out; 1569 struct ipuv3_channel *outch; 1570 unsigned int dst_idx; 1571 1572 lockdep_assert_held(&chan->irqlock); 1573 1574 outch = ipu_rot_mode_is_irt(ctx->rot_mode) ? 1575 chan->rotation_out_chan : chan->out_chan; 1576 1577 /* 1578 * It is difficult to stop the channel DMA before the channels 1579 * enter the paused state. Without double-buffering the channels 1580 * are always in a paused state when the EOF irq occurs, so it 1581 * is safe to stop the channels now. For double-buffering we 1582 * just ignore the abort until the operation completes, when it 1583 * is safe to shut down. 1584 */ 1585 if (ctx->aborting && !ctx->double_buffering) { 1586 convert_stop(run); 1587 run->status = -EIO; 1588 goto done; 1589 } 1590 1591 if (ctx->next_tile == ctx->num_tiles) { 1592 /* 1593 * the conversion is complete 1594 */ 1595 convert_stop(run); 1596 run->status = 0; 1597 goto done; 1598 } 1599 1600 /* 1601 * not done, place the next tile buffers. 1602 */ 1603 if (!ctx->double_buffering) { 1604 if (ic_settings_changed(ctx)) { 1605 convert_stop(run); 1606 convert_start(run, ctx->next_tile); 1607 } else { 1608 src_tile = &s_image->tile[ctx->next_tile]; 1609 dst_idx = ctx->out_tile_map[ctx->next_tile]; 1610 dst_tile = &d_image->tile[dst_idx]; 1611 1612 ipu_cpmem_set_buffer(chan->in_chan, 0, 1613 s_image->base.phys0 + 1614 src_tile->offset); 1615 ipu_cpmem_set_buffer(outch, 0, 1616 d_image->base.phys0 + 1617 dst_tile->offset); 1618 if (s_image->fmt->planar) 1619 ipu_cpmem_set_uv_offset(chan->in_chan, 1620 src_tile->u_off, 1621 src_tile->v_off); 1622 if (d_image->fmt->planar) 1623 ipu_cpmem_set_uv_offset(outch, 1624 dst_tile->u_off, 1625 dst_tile->v_off); 1626 1627 ipu_idmac_select_buffer(chan->in_chan, 0); 1628 ipu_idmac_select_buffer(outch, 0); 1629 } 1630 } else if (ctx->next_tile < ctx->num_tiles - 1) { 1631 1632 src_tile = &s_image->tile[ctx->next_tile + 1]; 1633 dst_idx = ctx->out_tile_map[ctx->next_tile + 1]; 1634 dst_tile = &d_image->tile[dst_idx]; 1635 1636 ipu_cpmem_set_buffer(chan->in_chan, ctx->cur_buf_num, 1637 s_image->base.phys0 + src_tile->offset); 1638 ipu_cpmem_set_buffer(outch, ctx->cur_buf_num, 1639 d_image->base.phys0 + dst_tile->offset); 1640 1641 ipu_idmac_select_buffer(chan->in_chan, ctx->cur_buf_num); 1642 ipu_idmac_select_buffer(outch, ctx->cur_buf_num); 1643 1644 ctx->cur_buf_num ^= 1; 1645 } 1646 1647 ctx->next_tile++; 1648 return IRQ_HANDLED; 1649 done: 1650 list_add_tail(&run->list, &chan->done_q); 1651 chan->current_run = NULL; 1652 run_next(chan); 1653 return IRQ_WAKE_THREAD; 1654 } 1655 1656 static irqreturn_t norotate_irq(int irq, void *data) 1657 { 1658 struct ipu_image_convert_chan *chan = data; 1659 struct ipu_image_convert_ctx *ctx; 1660 struct ipu_image_convert_run *run; 1661 unsigned long flags; 1662 irqreturn_t ret; 1663 1664 spin_lock_irqsave(&chan->irqlock, flags); 1665 1666 /* get current run and its context */ 1667 run = chan->current_run; 1668 if (!run) { 1669 ret = IRQ_NONE; 1670 goto out; 1671 } 1672 1673 ctx = run->ctx; 1674 1675 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 1676 /* this is a rotation operation, just ignore */ 1677 spin_unlock_irqrestore(&chan->irqlock, flags); 1678 return IRQ_HANDLED; 1679 } 1680 1681 ret = do_irq(run); 1682 out: 1683 spin_unlock_irqrestore(&chan->irqlock, flags); 1684 return ret; 1685 } 1686 1687 static irqreturn_t rotate_irq(int irq, void *data) 1688 { 1689 struct ipu_image_convert_chan *chan = data; 1690 struct ipu_image_convert_priv *priv = chan->priv; 1691 struct ipu_image_convert_ctx *ctx; 1692 struct ipu_image_convert_run *run; 1693 unsigned long flags; 1694 irqreturn_t ret; 1695 1696 spin_lock_irqsave(&chan->irqlock, flags); 1697 1698 /* get current run and its context */ 1699 run = chan->current_run; 1700 if (!run) { 1701 ret = IRQ_NONE; 1702 goto out; 1703 } 1704 1705 ctx = run->ctx; 1706 1707 if (!ipu_rot_mode_is_irt(ctx->rot_mode)) { 1708 /* this was NOT a rotation operation, shouldn't happen */ 1709 dev_err(priv->ipu->dev, "Unexpected rotation interrupt\n"); 1710 spin_unlock_irqrestore(&chan->irqlock, flags); 1711 return IRQ_HANDLED; 1712 } 1713 1714 ret = do_irq(run); 1715 out: 1716 spin_unlock_irqrestore(&chan->irqlock, flags); 1717 return ret; 1718 } 1719 1720 /* 1721 * try to force the completion of runs for this ctx. Called when 1722 * abort wait times out in ipu_image_convert_abort(). 1723 */ 1724 static void force_abort(struct ipu_image_convert_ctx *ctx) 1725 { 1726 struct ipu_image_convert_chan *chan = ctx->chan; 1727 struct ipu_image_convert_run *run; 1728 unsigned long flags; 1729 1730 spin_lock_irqsave(&chan->irqlock, flags); 1731 1732 run = chan->current_run; 1733 if (run && run->ctx == ctx) { 1734 convert_stop(run); 1735 run->status = -EIO; 1736 list_add_tail(&run->list, &chan->done_q); 1737 chan->current_run = NULL; 1738 run_next(chan); 1739 } 1740 1741 spin_unlock_irqrestore(&chan->irqlock, flags); 1742 1743 empty_done_q(chan); 1744 } 1745 1746 static void release_ipu_resources(struct ipu_image_convert_chan *chan) 1747 { 1748 if (chan->out_eof_irq >= 0) 1749 free_irq(chan->out_eof_irq, chan); 1750 if (chan->rot_out_eof_irq >= 0) 1751 free_irq(chan->rot_out_eof_irq, chan); 1752 1753 if (!IS_ERR_OR_NULL(chan->in_chan)) 1754 ipu_idmac_put(chan->in_chan); 1755 if (!IS_ERR_OR_NULL(chan->out_chan)) 1756 ipu_idmac_put(chan->out_chan); 1757 if (!IS_ERR_OR_NULL(chan->rotation_in_chan)) 1758 ipu_idmac_put(chan->rotation_in_chan); 1759 if (!IS_ERR_OR_NULL(chan->rotation_out_chan)) 1760 ipu_idmac_put(chan->rotation_out_chan); 1761 if (!IS_ERR_OR_NULL(chan->ic)) 1762 ipu_ic_put(chan->ic); 1763 1764 chan->in_chan = chan->out_chan = chan->rotation_in_chan = 1765 chan->rotation_out_chan = NULL; 1766 chan->out_eof_irq = chan->rot_out_eof_irq = -1; 1767 } 1768 1769 static int get_ipu_resources(struct ipu_image_convert_chan *chan) 1770 { 1771 const struct ipu_image_convert_dma_chan *dma = chan->dma_ch; 1772 struct ipu_image_convert_priv *priv = chan->priv; 1773 int ret; 1774 1775 /* get IC */ 1776 chan->ic = ipu_ic_get(priv->ipu, chan->ic_task); 1777 if (IS_ERR(chan->ic)) { 1778 dev_err(priv->ipu->dev, "could not acquire IC\n"); 1779 ret = PTR_ERR(chan->ic); 1780 goto err; 1781 } 1782 1783 /* get IDMAC channels */ 1784 chan->in_chan = ipu_idmac_get(priv->ipu, dma->in); 1785 chan->out_chan = ipu_idmac_get(priv->ipu, dma->out); 1786 if (IS_ERR(chan->in_chan) || IS_ERR(chan->out_chan)) { 1787 dev_err(priv->ipu->dev, "could not acquire idmac channels\n"); 1788 ret = -EBUSY; 1789 goto err; 1790 } 1791 1792 chan->rotation_in_chan = ipu_idmac_get(priv->ipu, dma->rot_in); 1793 chan->rotation_out_chan = ipu_idmac_get(priv->ipu, dma->rot_out); 1794 if (IS_ERR(chan->rotation_in_chan) || IS_ERR(chan->rotation_out_chan)) { 1795 dev_err(priv->ipu->dev, 1796 "could not acquire idmac rotation channels\n"); 1797 ret = -EBUSY; 1798 goto err; 1799 } 1800 1801 /* acquire the EOF interrupts */ 1802 chan->out_eof_irq = ipu_idmac_channel_irq(priv->ipu, 1803 chan->out_chan, 1804 IPU_IRQ_EOF); 1805 1806 ret = request_threaded_irq(chan->out_eof_irq, norotate_irq, do_bh, 1807 0, "ipu-ic", chan); 1808 if (ret < 0) { 1809 dev_err(priv->ipu->dev, "could not acquire irq %d\n", 1810 chan->out_eof_irq); 1811 chan->out_eof_irq = -1; 1812 goto err; 1813 } 1814 1815 chan->rot_out_eof_irq = ipu_idmac_channel_irq(priv->ipu, 1816 chan->rotation_out_chan, 1817 IPU_IRQ_EOF); 1818 1819 ret = request_threaded_irq(chan->rot_out_eof_irq, rotate_irq, do_bh, 1820 0, "ipu-ic", chan); 1821 if (ret < 0) { 1822 dev_err(priv->ipu->dev, "could not acquire irq %d\n", 1823 chan->rot_out_eof_irq); 1824 chan->rot_out_eof_irq = -1; 1825 goto err; 1826 } 1827 1828 return 0; 1829 err: 1830 release_ipu_resources(chan); 1831 return ret; 1832 } 1833 1834 static int fill_image(struct ipu_image_convert_ctx *ctx, 1835 struct ipu_image_convert_image *ic_image, 1836 struct ipu_image *image, 1837 enum ipu_image_convert_type type) 1838 { 1839 struct ipu_image_convert_priv *priv = ctx->chan->priv; 1840 1841 ic_image->base = *image; 1842 ic_image->type = type; 1843 1844 ic_image->fmt = get_format(image->pix.pixelformat); 1845 if (!ic_image->fmt) { 1846 dev_err(priv->ipu->dev, "pixelformat not supported for %s\n", 1847 type == IMAGE_CONVERT_OUT ? "Output" : "Input"); 1848 return -EINVAL; 1849 } 1850 1851 if (ic_image->fmt->planar) 1852 ic_image->stride = ic_image->base.pix.width; 1853 else 1854 ic_image->stride = ic_image->base.pix.bytesperline; 1855 1856 return 0; 1857 } 1858 1859 /* borrowed from drivers/media/v4l2-core/v4l2-common.c */ 1860 static unsigned int clamp_align(unsigned int x, unsigned int min, 1861 unsigned int max, unsigned int align) 1862 { 1863 /* Bits that must be zero to be aligned */ 1864 unsigned int mask = ~((1 << align) - 1); 1865 1866 /* Clamp to aligned min and max */ 1867 x = clamp(x, (min + ~mask) & mask, max & mask); 1868 1869 /* Round to nearest aligned value */ 1870 if (align) 1871 x = (x + (1 << (align - 1))) & mask; 1872 1873 return x; 1874 } 1875 1876 /* Adjusts input/output images to IPU restrictions */ 1877 void ipu_image_convert_adjust(struct ipu_image *in, struct ipu_image *out, 1878 enum ipu_rotate_mode rot_mode) 1879 { 1880 const struct ipu_image_pixfmt *infmt, *outfmt; 1881 u32 w_align_out, h_align_out; 1882 u32 w_align_in, h_align_in; 1883 1884 infmt = get_format(in->pix.pixelformat); 1885 outfmt = get_format(out->pix.pixelformat); 1886 1887 /* set some default pixel formats if needed */ 1888 if (!infmt) { 1889 in->pix.pixelformat = V4L2_PIX_FMT_RGB24; 1890 infmt = get_format(V4L2_PIX_FMT_RGB24); 1891 } 1892 if (!outfmt) { 1893 out->pix.pixelformat = V4L2_PIX_FMT_RGB24; 1894 outfmt = get_format(V4L2_PIX_FMT_RGB24); 1895 } 1896 1897 /* image converter does not handle fields */ 1898 in->pix.field = out->pix.field = V4L2_FIELD_NONE; 1899 1900 /* resizer cannot downsize more than 4:1 */ 1901 if (ipu_rot_mode_is_irt(rot_mode)) { 1902 out->pix.height = max_t(__u32, out->pix.height, 1903 in->pix.width / 4); 1904 out->pix.width = max_t(__u32, out->pix.width, 1905 in->pix.height / 4); 1906 } else { 1907 out->pix.width = max_t(__u32, out->pix.width, 1908 in->pix.width / 4); 1909 out->pix.height = max_t(__u32, out->pix.height, 1910 in->pix.height / 4); 1911 } 1912 1913 /* align input width/height */ 1914 w_align_in = ilog2(tile_width_align(IMAGE_CONVERT_IN, infmt, 1915 rot_mode)); 1916 h_align_in = ilog2(tile_height_align(IMAGE_CONVERT_IN, infmt, 1917 rot_mode)); 1918 in->pix.width = clamp_align(in->pix.width, MIN_W, MAX_W, 1919 w_align_in); 1920 in->pix.height = clamp_align(in->pix.height, MIN_H, MAX_H, 1921 h_align_in); 1922 1923 /* align output width/height */ 1924 w_align_out = ilog2(tile_width_align(IMAGE_CONVERT_OUT, outfmt, 1925 rot_mode)); 1926 h_align_out = ilog2(tile_height_align(IMAGE_CONVERT_OUT, outfmt, 1927 rot_mode)); 1928 out->pix.width = clamp_align(out->pix.width, MIN_W, MAX_W, 1929 w_align_out); 1930 out->pix.height = clamp_align(out->pix.height, MIN_H, MAX_H, 1931 h_align_out); 1932 1933 /* set input/output strides and image sizes */ 1934 in->pix.bytesperline = infmt->planar ? 1935 clamp_align(in->pix.width, 2 << w_align_in, MAX_W, 1936 w_align_in) : 1937 clamp_align((in->pix.width * infmt->bpp) >> 3, 1938 ((2 << w_align_in) * infmt->bpp) >> 3, 1939 (MAX_W * infmt->bpp) >> 3, 1940 w_align_in); 1941 in->pix.sizeimage = infmt->planar ? 1942 (in->pix.height * in->pix.bytesperline * infmt->bpp) >> 3 : 1943 in->pix.height * in->pix.bytesperline; 1944 out->pix.bytesperline = outfmt->planar ? out->pix.width : 1945 (out->pix.width * outfmt->bpp) >> 3; 1946 out->pix.sizeimage = outfmt->planar ? 1947 (out->pix.height * out->pix.bytesperline * outfmt->bpp) >> 3 : 1948 out->pix.height * out->pix.bytesperline; 1949 } 1950 EXPORT_SYMBOL_GPL(ipu_image_convert_adjust); 1951 1952 /* 1953 * this is used by ipu_image_convert_prepare() to verify set input and 1954 * output images are valid before starting the conversion. Clients can 1955 * also call it before calling ipu_image_convert_prepare(). 1956 */ 1957 int ipu_image_convert_verify(struct ipu_image *in, struct ipu_image *out, 1958 enum ipu_rotate_mode rot_mode) 1959 { 1960 struct ipu_image testin, testout; 1961 1962 testin = *in; 1963 testout = *out; 1964 1965 ipu_image_convert_adjust(&testin, &testout, rot_mode); 1966 1967 if (testin.pix.width != in->pix.width || 1968 testin.pix.height != in->pix.height || 1969 testout.pix.width != out->pix.width || 1970 testout.pix.height != out->pix.height) 1971 return -EINVAL; 1972 1973 return 0; 1974 } 1975 EXPORT_SYMBOL_GPL(ipu_image_convert_verify); 1976 1977 /* 1978 * Call ipu_image_convert_prepare() to prepare for the conversion of 1979 * given images and rotation mode. Returns a new conversion context. 1980 */ 1981 struct ipu_image_convert_ctx * 1982 ipu_image_convert_prepare(struct ipu_soc *ipu, enum ipu_ic_task ic_task, 1983 struct ipu_image *in, struct ipu_image *out, 1984 enum ipu_rotate_mode rot_mode, 1985 ipu_image_convert_cb_t complete, 1986 void *complete_context) 1987 { 1988 struct ipu_image_convert_priv *priv = ipu->image_convert_priv; 1989 struct ipu_image_convert_image *s_image, *d_image; 1990 struct ipu_image_convert_chan *chan; 1991 struct ipu_image_convert_ctx *ctx; 1992 unsigned long flags; 1993 unsigned int i; 1994 bool get_res; 1995 int ret; 1996 1997 if (!in || !out || !complete || 1998 (ic_task != IC_TASK_VIEWFINDER && 1999 ic_task != IC_TASK_POST_PROCESSOR)) 2000 return ERR_PTR(-EINVAL); 2001 2002 /* verify the in/out images before continuing */ 2003 ret = ipu_image_convert_verify(in, out, rot_mode); 2004 if (ret) { 2005 dev_err(priv->ipu->dev, "%s: in/out formats invalid\n", 2006 __func__); 2007 return ERR_PTR(ret); 2008 } 2009 2010 chan = &priv->chan[ic_task]; 2011 2012 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 2013 if (!ctx) 2014 return ERR_PTR(-ENOMEM); 2015 2016 dev_dbg(priv->ipu->dev, "%s: task %u: ctx %p\n", __func__, 2017 chan->ic_task, ctx); 2018 2019 ctx->chan = chan; 2020 init_completion(&ctx->aborted); 2021 2022 s_image = &ctx->in; 2023 d_image = &ctx->out; 2024 2025 /* set tiling and rotation */ 2026 d_image->num_rows = num_stripes(out->pix.height); 2027 d_image->num_cols = num_stripes(out->pix.width); 2028 if (ipu_rot_mode_is_irt(rot_mode)) { 2029 s_image->num_rows = d_image->num_cols; 2030 s_image->num_cols = d_image->num_rows; 2031 } else { 2032 s_image->num_rows = d_image->num_rows; 2033 s_image->num_cols = d_image->num_cols; 2034 } 2035 2036 ctx->num_tiles = d_image->num_cols * d_image->num_rows; 2037 ctx->rot_mode = rot_mode; 2038 2039 ret = fill_image(ctx, s_image, in, IMAGE_CONVERT_IN); 2040 if (ret) 2041 goto out_free; 2042 ret = fill_image(ctx, d_image, out, IMAGE_CONVERT_OUT); 2043 if (ret) 2044 goto out_free; 2045 2046 ret = calc_image_resize_coefficients(ctx, in, out); 2047 if (ret) 2048 goto out_free; 2049 2050 calc_out_tile_map(ctx); 2051 2052 find_seams(ctx, s_image, d_image); 2053 2054 calc_tile_dimensions(ctx, s_image); 2055 ret = calc_tile_offsets(ctx, s_image); 2056 if (ret) 2057 goto out_free; 2058 2059 calc_tile_dimensions(ctx, d_image); 2060 ret = calc_tile_offsets(ctx, d_image); 2061 if (ret) 2062 goto out_free; 2063 2064 calc_tile_resize_coefficients(ctx); 2065 2066 dump_format(ctx, s_image); 2067 dump_format(ctx, d_image); 2068 2069 ctx->complete = complete; 2070 ctx->complete_context = complete_context; 2071 2072 /* 2073 * Can we use double-buffering for this operation? If there is 2074 * only one tile (the whole image can be converted in a single 2075 * operation) there's no point in using double-buffering. Also, 2076 * the IPU's IDMAC channels allow only a single U and V plane 2077 * offset shared between both buffers, but these offsets change 2078 * for every tile, and therefore would have to be updated for 2079 * each buffer which is not possible. So double-buffering is 2080 * impossible when either the source or destination images are 2081 * a planar format (YUV420, YUV422P, etc.). Further, differently 2082 * sized tiles or different resizing coefficients per tile 2083 * prevent double-buffering as well. 2084 */ 2085 ctx->double_buffering = (ctx->num_tiles > 1 && 2086 !s_image->fmt->planar && 2087 !d_image->fmt->planar); 2088 for (i = 1; i < ctx->num_tiles; i++) { 2089 if (ctx->in.tile[i].width != ctx->in.tile[0].width || 2090 ctx->in.tile[i].height != ctx->in.tile[0].height || 2091 ctx->out.tile[i].width != ctx->out.tile[0].width || 2092 ctx->out.tile[i].height != ctx->out.tile[0].height) { 2093 ctx->double_buffering = false; 2094 break; 2095 } 2096 } 2097 for (i = 1; i < ctx->in.num_cols; i++) { 2098 if (ctx->resize_coeffs_h[i] != ctx->resize_coeffs_h[0]) { 2099 ctx->double_buffering = false; 2100 break; 2101 } 2102 } 2103 for (i = 1; i < ctx->in.num_rows; i++) { 2104 if (ctx->resize_coeffs_v[i] != ctx->resize_coeffs_v[0]) { 2105 ctx->double_buffering = false; 2106 break; 2107 } 2108 } 2109 2110 if (ipu_rot_mode_is_irt(ctx->rot_mode)) { 2111 unsigned long intermediate_size = d_image->tile[0].size; 2112 2113 for (i = 1; i < ctx->num_tiles; i++) { 2114 if (d_image->tile[i].size > intermediate_size) 2115 intermediate_size = d_image->tile[i].size; 2116 } 2117 2118 ret = alloc_dma_buf(priv, &ctx->rot_intermediate[0], 2119 intermediate_size); 2120 if (ret) 2121 goto out_free; 2122 if (ctx->double_buffering) { 2123 ret = alloc_dma_buf(priv, 2124 &ctx->rot_intermediate[1], 2125 intermediate_size); 2126 if (ret) 2127 goto out_free_dmabuf0; 2128 } 2129 } 2130 2131 spin_lock_irqsave(&chan->irqlock, flags); 2132 2133 get_res = list_empty(&chan->ctx_list); 2134 2135 list_add_tail(&ctx->list, &chan->ctx_list); 2136 2137 spin_unlock_irqrestore(&chan->irqlock, flags); 2138 2139 if (get_res) { 2140 ret = get_ipu_resources(chan); 2141 if (ret) 2142 goto out_free_dmabuf1; 2143 } 2144 2145 return ctx; 2146 2147 out_free_dmabuf1: 2148 free_dma_buf(priv, &ctx->rot_intermediate[1]); 2149 spin_lock_irqsave(&chan->irqlock, flags); 2150 list_del(&ctx->list); 2151 spin_unlock_irqrestore(&chan->irqlock, flags); 2152 out_free_dmabuf0: 2153 free_dma_buf(priv, &ctx->rot_intermediate[0]); 2154 out_free: 2155 kfree(ctx); 2156 return ERR_PTR(ret); 2157 } 2158 EXPORT_SYMBOL_GPL(ipu_image_convert_prepare); 2159 2160 /* 2161 * Carry out a single image conversion run. Only the physaddr's of the input 2162 * and output image buffers are needed. The conversion context must have 2163 * been created previously with ipu_image_convert_prepare(). 2164 */ 2165 int ipu_image_convert_queue(struct ipu_image_convert_run *run) 2166 { 2167 struct ipu_image_convert_chan *chan; 2168 struct ipu_image_convert_priv *priv; 2169 struct ipu_image_convert_ctx *ctx; 2170 unsigned long flags; 2171 int ret = 0; 2172 2173 if (!run || !run->ctx || !run->in_phys || !run->out_phys) 2174 return -EINVAL; 2175 2176 ctx = run->ctx; 2177 chan = ctx->chan; 2178 priv = chan->priv; 2179 2180 dev_dbg(priv->ipu->dev, "%s: task %u: ctx %p run %p\n", __func__, 2181 chan->ic_task, ctx, run); 2182 2183 INIT_LIST_HEAD(&run->list); 2184 2185 spin_lock_irqsave(&chan->irqlock, flags); 2186 2187 if (ctx->aborting) { 2188 ret = -EIO; 2189 goto unlock; 2190 } 2191 2192 list_add_tail(&run->list, &chan->pending_q); 2193 2194 if (!chan->current_run) { 2195 ret = do_run(run); 2196 if (ret) 2197 chan->current_run = NULL; 2198 } 2199 unlock: 2200 spin_unlock_irqrestore(&chan->irqlock, flags); 2201 return ret; 2202 } 2203 EXPORT_SYMBOL_GPL(ipu_image_convert_queue); 2204 2205 /* Abort any active or pending conversions for this context */ 2206 static void __ipu_image_convert_abort(struct ipu_image_convert_ctx *ctx) 2207 { 2208 struct ipu_image_convert_chan *chan = ctx->chan; 2209 struct ipu_image_convert_priv *priv = chan->priv; 2210 struct ipu_image_convert_run *run, *active_run, *tmp; 2211 unsigned long flags; 2212 int run_count, ret; 2213 2214 spin_lock_irqsave(&chan->irqlock, flags); 2215 2216 /* move all remaining pending runs in this context to done_q */ 2217 list_for_each_entry_safe(run, tmp, &chan->pending_q, list) { 2218 if (run->ctx != ctx) 2219 continue; 2220 run->status = -EIO; 2221 list_move_tail(&run->list, &chan->done_q); 2222 } 2223 2224 run_count = get_run_count(ctx, &chan->done_q); 2225 active_run = (chan->current_run && chan->current_run->ctx == ctx) ? 2226 chan->current_run : NULL; 2227 2228 if (active_run) 2229 reinit_completion(&ctx->aborted); 2230 2231 ctx->aborting = true; 2232 2233 spin_unlock_irqrestore(&chan->irqlock, flags); 2234 2235 if (!run_count && !active_run) { 2236 dev_dbg(priv->ipu->dev, 2237 "%s: task %u: no abort needed for ctx %p\n", 2238 __func__, chan->ic_task, ctx); 2239 return; 2240 } 2241 2242 if (!active_run) { 2243 empty_done_q(chan); 2244 return; 2245 } 2246 2247 dev_dbg(priv->ipu->dev, 2248 "%s: task %u: wait for completion: %d runs\n", 2249 __func__, chan->ic_task, run_count); 2250 2251 ret = wait_for_completion_timeout(&ctx->aborted, 2252 msecs_to_jiffies(10000)); 2253 if (ret == 0) { 2254 dev_warn(priv->ipu->dev, "%s: timeout\n", __func__); 2255 force_abort(ctx); 2256 } 2257 } 2258 2259 void ipu_image_convert_abort(struct ipu_image_convert_ctx *ctx) 2260 { 2261 __ipu_image_convert_abort(ctx); 2262 ctx->aborting = false; 2263 } 2264 EXPORT_SYMBOL_GPL(ipu_image_convert_abort); 2265 2266 /* Unprepare image conversion context */ 2267 void ipu_image_convert_unprepare(struct ipu_image_convert_ctx *ctx) 2268 { 2269 struct ipu_image_convert_chan *chan = ctx->chan; 2270 struct ipu_image_convert_priv *priv = chan->priv; 2271 unsigned long flags; 2272 bool put_res; 2273 2274 /* make sure no runs are hanging around */ 2275 __ipu_image_convert_abort(ctx); 2276 2277 dev_dbg(priv->ipu->dev, "%s: task %u: removing ctx %p\n", __func__, 2278 chan->ic_task, ctx); 2279 2280 spin_lock_irqsave(&chan->irqlock, flags); 2281 2282 list_del(&ctx->list); 2283 2284 put_res = list_empty(&chan->ctx_list); 2285 2286 spin_unlock_irqrestore(&chan->irqlock, flags); 2287 2288 if (put_res) 2289 release_ipu_resources(chan); 2290 2291 free_dma_buf(priv, &ctx->rot_intermediate[1]); 2292 free_dma_buf(priv, &ctx->rot_intermediate[0]); 2293 2294 kfree(ctx); 2295 } 2296 EXPORT_SYMBOL_GPL(ipu_image_convert_unprepare); 2297 2298 /* 2299 * "Canned" asynchronous single image conversion. Allocates and returns 2300 * a new conversion run. On successful return the caller must free the 2301 * run and call ipu_image_convert_unprepare() after conversion completes. 2302 */ 2303 struct ipu_image_convert_run * 2304 ipu_image_convert(struct ipu_soc *ipu, enum ipu_ic_task ic_task, 2305 struct ipu_image *in, struct ipu_image *out, 2306 enum ipu_rotate_mode rot_mode, 2307 ipu_image_convert_cb_t complete, 2308 void *complete_context) 2309 { 2310 struct ipu_image_convert_ctx *ctx; 2311 struct ipu_image_convert_run *run; 2312 int ret; 2313 2314 ctx = ipu_image_convert_prepare(ipu, ic_task, in, out, rot_mode, 2315 complete, complete_context); 2316 if (IS_ERR(ctx)) 2317 return ERR_CAST(ctx); 2318 2319 run = kzalloc(sizeof(*run), GFP_KERNEL); 2320 if (!run) { 2321 ipu_image_convert_unprepare(ctx); 2322 return ERR_PTR(-ENOMEM); 2323 } 2324 2325 run->ctx = ctx; 2326 run->in_phys = in->phys0; 2327 run->out_phys = out->phys0; 2328 2329 ret = ipu_image_convert_queue(run); 2330 if (ret) { 2331 ipu_image_convert_unprepare(ctx); 2332 kfree(run); 2333 return ERR_PTR(ret); 2334 } 2335 2336 return run; 2337 } 2338 EXPORT_SYMBOL_GPL(ipu_image_convert); 2339 2340 /* "Canned" synchronous single image conversion */ 2341 static void image_convert_sync_complete(struct ipu_image_convert_run *run, 2342 void *data) 2343 { 2344 struct completion *comp = data; 2345 2346 complete(comp); 2347 } 2348 2349 int ipu_image_convert_sync(struct ipu_soc *ipu, enum ipu_ic_task ic_task, 2350 struct ipu_image *in, struct ipu_image *out, 2351 enum ipu_rotate_mode rot_mode) 2352 { 2353 struct ipu_image_convert_run *run; 2354 struct completion comp; 2355 int ret; 2356 2357 init_completion(&comp); 2358 2359 run = ipu_image_convert(ipu, ic_task, in, out, rot_mode, 2360 image_convert_sync_complete, &comp); 2361 if (IS_ERR(run)) 2362 return PTR_ERR(run); 2363 2364 ret = wait_for_completion_timeout(&comp, msecs_to_jiffies(10000)); 2365 ret = (ret == 0) ? -ETIMEDOUT : 0; 2366 2367 ipu_image_convert_unprepare(run->ctx); 2368 kfree(run); 2369 2370 return ret; 2371 } 2372 EXPORT_SYMBOL_GPL(ipu_image_convert_sync); 2373 2374 int ipu_image_convert_init(struct ipu_soc *ipu, struct device *dev) 2375 { 2376 struct ipu_image_convert_priv *priv; 2377 int i; 2378 2379 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); 2380 if (!priv) 2381 return -ENOMEM; 2382 2383 ipu->image_convert_priv = priv; 2384 priv->ipu = ipu; 2385 2386 for (i = 0; i < IC_NUM_TASKS; i++) { 2387 struct ipu_image_convert_chan *chan = &priv->chan[i]; 2388 2389 chan->ic_task = i; 2390 chan->priv = priv; 2391 chan->dma_ch = &image_convert_dma_chan[i]; 2392 chan->out_eof_irq = -1; 2393 chan->rot_out_eof_irq = -1; 2394 2395 spin_lock_init(&chan->irqlock); 2396 INIT_LIST_HEAD(&chan->ctx_list); 2397 INIT_LIST_HEAD(&chan->pending_q); 2398 INIT_LIST_HEAD(&chan->done_q); 2399 } 2400 2401 return 0; 2402 } 2403 2404 void ipu_image_convert_exit(struct ipu_soc *ipu) 2405 { 2406 } 2407