ipu-image-convert.c - drivers/gpu/ipu-v3/ipu-image-convert.c - Linux diff v6.13.7

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Copyright (C) 2012-2016 Mentor Graphics Inc.
   4 *
   5 * Queued image conversion support, with tiling and rotation.
   6 */
   7
   8#include <linux/interrupt.h>
   9#include <linux/dma-mapping.h>
  10#include <linux/math.h>
  11
  12#include <video/imx-ipu-image-convert.h>
  13
  14#include "ipu-prv.h"
  15
  16/*
  17 * The IC Resizer has a restriction that the output frame from the
  18 * resizer must be 1024 or less in both width (pixels) and height
  19 * (lines).
  20 *
  21 * The image converter attempts to split up a conversion when
  22 * the desired output (converted) frame resolution exceeds the
  23 * IC resizer limit of 1024 in either dimension.
  24 *
  25 * If either dimension of the output frame exceeds the limit, the
  26 * dimension is split into 1, 2, or 4 equal stripes, for a maximum
  27 * of 4*4 or 16 tiles. A conversion is then carried out for each
  28 * tile (but taking care to pass the full frame stride length to
  29 * the DMA channel's parameter memory!). IDMA double-buffering is used
  30 * to convert each tile back-to-back when possible (see note below
  31 * when double_buffering boolean is set).
  32 *
  33 * Note that the input frame must be split up into the same number
  34 * of tiles as the output frame:
  35 *
  36 *                       +---------+-----+
  37 *   +-----+---+         |  A      | B   |
  38 *   | A   | B |         |         |     |
  39 *   +-----+---+   -->   +---------+-----+
  40 *   | C   | D |         |  C      | D   |
  41 *   +-----+---+         |         |     |
  42 *                       +---------+-----+
  43 *
  44 * Clockwise 90° rotations are handled by first rescaling into a
  45 * reusable temporary tile buffer and then rotating with the 8x8
  46 * block rotator, writing to the correct destination:
  47 *
  48 *                                         +-----+-----+
  49 *                                         |     |     |
  50 *   +-----+---+         +---------+       | C   | A   |
  51 *   | A   | B |         | A,B, |  |       |     |     |
  52 *   +-----+---+   -->   | C,D  |  |  -->  |     |     |
  53 *   | C   | D |         +---------+       +-----+-----+
  54 *   +-----+---+                           | D   | B   |
  55 *                                         |     |     |
  56 *                                         +-----+-----+
  57 *
  58 * If the 8x8 block rotator is used, horizontal or vertical flipping
  59 * is done during the rotation step, otherwise flipping is done
  60 * during the scaling step.
  61 * With rotation or flipping, tile order changes between input and
  62 * output image. Tiles are numbered row major from top left to bottom
  63 * right for both input and output image.
  64 */
  65
  66#define MAX_STRIPES_W    4
  67#define MAX_STRIPES_H    4
  68#define MAX_TILES (MAX_STRIPES_W * MAX_STRIPES_H)
  69
  70#define MIN_W     16
  71#define MIN_H     8
  72#define MAX_W     4096
  73#define MAX_H     4096
  74
  75enum ipu_image_convert_type {
  76	IMAGE_CONVERT_IN = 0,
  77	IMAGE_CONVERT_OUT,
  78};
  79
  80struct ipu_image_convert_dma_buf {
  81	void          *virt;
  82	dma_addr_t    phys;
  83	unsigned long len;
  84};
  85
  86struct ipu_image_convert_dma_chan {
  87	int in;
  88	int out;
  89	int rot_in;
  90	int rot_out;
  91	int vdi_in_p;
  92	int vdi_in;
  93	int vdi_in_n;
  94};
  95
  96/* dimensions of one tile */
  97struct ipu_image_tile {
  98	u32 width;
  99	u32 height;
 100	u32 left;
 101	u32 top;
 102	/* size and strides are in bytes */
 103	u32 size;
 104	u32 stride;
 105	u32 rot_stride;
 106	/* start Y or packed offset of this tile */
 107	u32 offset;
 108	/* offset from start to tile in U plane, for planar formats */
 109	u32 u_off;
 110	/* offset from start to tile in V plane, for planar formats */
 111	u32 v_off;
 112};
 113
 114struct ipu_image_convert_image {
 115	struct ipu_image base;
 116	enum ipu_image_convert_type type;
 117
 118	const struct ipu_image_pixfmt *fmt;
 119	unsigned int stride;
 120
 121	/* # of rows (horizontal stripes) if dest height is > 1024 */
 122	unsigned int num_rows;
 123	/* # of columns (vertical stripes) if dest width is > 1024 */
 124	unsigned int num_cols;
 125
 126	struct ipu_image_tile tile[MAX_TILES];
 127};
 128
 129struct ipu_image_pixfmt {
 130	u32	fourcc;        /* V4L2 fourcc */
 131	int     bpp;           /* total bpp */
 132	int     uv_width_dec;  /* decimation in width for U/V planes */
 133	int     uv_height_dec; /* decimation in height for U/V planes */
 134	bool    planar;        /* planar format */
 135	bool    uv_swapped;    /* U and V planes are swapped */
 136	bool    uv_packed;     /* partial planar (U and V in same plane) */
 137};
 138
 139struct ipu_image_convert_ctx;
 140struct ipu_image_convert_chan;
 141struct ipu_image_convert_priv;
 142
 143enum eof_irq_mask {
 144	EOF_IRQ_IN      = BIT(0),
 145	EOF_IRQ_ROT_IN  = BIT(1),
 146	EOF_IRQ_OUT     = BIT(2),
 147	EOF_IRQ_ROT_OUT = BIT(3),
 148};
 149
 150#define EOF_IRQ_COMPLETE (EOF_IRQ_IN | EOF_IRQ_OUT)
 151#define EOF_IRQ_ROT_COMPLETE (EOF_IRQ_IN | EOF_IRQ_OUT |	\
 152			      EOF_IRQ_ROT_IN | EOF_IRQ_ROT_OUT)
 153
 154struct ipu_image_convert_ctx {
 155	struct ipu_image_convert_chan *chan;
 156
 157	ipu_image_convert_cb_t complete;
 158	void *complete_context;
 159
 160	/* Source/destination image data and rotation mode */
 161	struct ipu_image_convert_image in;
 162	struct ipu_image_convert_image out;
 163	struct ipu_ic_csc csc;
 164	enum ipu_rotate_mode rot_mode;
 165	u32 downsize_coeff_h;
 166	u32 downsize_coeff_v;
 167	u32 image_resize_coeff_h;
 168	u32 image_resize_coeff_v;
 169	u32 resize_coeffs_h[MAX_STRIPES_W];
 170	u32 resize_coeffs_v[MAX_STRIPES_H];
 171
 172	/* intermediate buffer for rotation */
 173	struct ipu_image_convert_dma_buf rot_intermediate[2];
 174
 175	/* current buffer number for double buffering */
 176	int cur_buf_num;
 177
 178	bool aborting;
 179	struct completion aborted;
 180
 181	/* can we use double-buffering for this conversion operation? */
 182	bool double_buffering;
 183	/* num_rows * num_cols */
 184	unsigned int num_tiles;
 185	/* next tile to process */
 186	unsigned int next_tile;
 187	/* where to place converted tile in dest image */
 188	unsigned int out_tile_map[MAX_TILES];
 189
 190	/* mask of completed EOF irqs at every tile conversion */
 191	enum eof_irq_mask eof_mask;
 192
 193	struct list_head list;
 194};
 195
 196struct ipu_image_convert_chan {
 197	struct ipu_image_convert_priv *priv;
 198
 199	enum ipu_ic_task ic_task;
 200	const struct ipu_image_convert_dma_chan *dma_ch;
 201
 202	struct ipu_ic *ic;
 203	struct ipuv3_channel *in_chan;
 204	struct ipuv3_channel *out_chan;
 205	struct ipuv3_channel *rotation_in_chan;
 206	struct ipuv3_channel *rotation_out_chan;
 207
 208	/* the IPU end-of-frame irqs */
 209	int in_eof_irq;
 210	int rot_in_eof_irq;
 211	int out_eof_irq;
 212	int rot_out_eof_irq;
 213
 214	spinlock_t irqlock;
 215
 216	/* list of convert contexts */
 217	struct list_head ctx_list;
 218	/* queue of conversion runs */
 219	struct list_head pending_q;
 220	/* queue of completed runs */
 221	struct list_head done_q;
 222
 223	/* the current conversion run */
 224	struct ipu_image_convert_run *current_run;
 225};
 226
 227struct ipu_image_convert_priv {
 228	struct ipu_image_convert_chan chan[IC_NUM_TASKS];
 229	struct ipu_soc *ipu;
 230};
 231
 232static const struct ipu_image_convert_dma_chan
 233image_convert_dma_chan[IC_NUM_TASKS] = {
 234	[IC_TASK_VIEWFINDER] = {
 235		.in = IPUV3_CHANNEL_MEM_IC_PRP_VF,
 236		.out = IPUV3_CHANNEL_IC_PRP_VF_MEM,
 237		.rot_in = IPUV3_CHANNEL_MEM_ROT_VF,
 238		.rot_out = IPUV3_CHANNEL_ROT_VF_MEM,
 239		.vdi_in_p = IPUV3_CHANNEL_MEM_VDI_PREV,
 240		.vdi_in = IPUV3_CHANNEL_MEM_VDI_CUR,
 241		.vdi_in_n = IPUV3_CHANNEL_MEM_VDI_NEXT,
 242	},
 243	[IC_TASK_POST_PROCESSOR] = {
 244		.in = IPUV3_CHANNEL_MEM_IC_PP,
 245		.out = IPUV3_CHANNEL_IC_PP_MEM,
 246		.rot_in = IPUV3_CHANNEL_MEM_ROT_PP,
 247		.rot_out = IPUV3_CHANNEL_ROT_PP_MEM,
 248	},
 249};
 250
 251static const struct ipu_image_pixfmt image_convert_formats[] = {
 252	{
 253		.fourcc	= V4L2_PIX_FMT_RGB565,
 254		.bpp    = 16,
 255	}, {
 256		.fourcc	= V4L2_PIX_FMT_RGB24,
 257		.bpp    = 24,
 258	}, {
 259		.fourcc	= V4L2_PIX_FMT_BGR24,
 260		.bpp    = 24,
 261	}, {
 262		.fourcc	= V4L2_PIX_FMT_RGB32,
 263		.bpp    = 32,
 264	}, {
 265		.fourcc	= V4L2_PIX_FMT_BGR32,
 266		.bpp    = 32,
 267	}, {
 268		.fourcc	= V4L2_PIX_FMT_XRGB32,
 269		.bpp    = 32,
 270	}, {
 271		.fourcc	= V4L2_PIX_FMT_XBGR32,
 272		.bpp    = 32,
 273	}, {
 274		.fourcc	= V4L2_PIX_FMT_BGRX32,
 275		.bpp    = 32,
 276	}, {
 277		.fourcc	= V4L2_PIX_FMT_RGBX32,
 278		.bpp    = 32,
 279	}, {
 280		.fourcc	= V4L2_PIX_FMT_YUYV,
 281		.bpp    = 16,
 282		.uv_width_dec = 2,
 283		.uv_height_dec = 1,
 284	}, {
 285		.fourcc	= V4L2_PIX_FMT_UYVY,
 286		.bpp    = 16,
 287		.uv_width_dec = 2,
 288		.uv_height_dec = 1,
 289	}, {
 290		.fourcc	= V4L2_PIX_FMT_YUV420,
 291		.bpp    = 12,
 292		.planar = true,
 293		.uv_width_dec = 2,
 294		.uv_height_dec = 2,
 295	}, {
 296		.fourcc	= V4L2_PIX_FMT_YVU420,
 297		.bpp    = 12,
 298		.planar = true,
 299		.uv_width_dec = 2,
 300		.uv_height_dec = 2,
 301		.uv_swapped = true,
 302	}, {
 303		.fourcc = V4L2_PIX_FMT_NV12,
 304		.bpp    = 12,
 305		.planar = true,
 306		.uv_width_dec = 2,
 307		.uv_height_dec = 2,
 308		.uv_packed = true,
 309	}, {
 310		.fourcc = V4L2_PIX_FMT_YUV422P,
 311		.bpp    = 16,
 312		.planar = true,
 313		.uv_width_dec = 2,
 314		.uv_height_dec = 1,
 315	}, {
 316		.fourcc = V4L2_PIX_FMT_NV16,
 317		.bpp    = 16,
 318		.planar = true,
 319		.uv_width_dec = 2,
 320		.uv_height_dec = 1,
 321		.uv_packed = true,
 322	},
 323};
 324
 325static const struct ipu_image_pixfmt *get_format(u32 fourcc)
 326{
 327	const struct ipu_image_pixfmt *ret = NULL;
 328	unsigned int i;
 329
 330	for (i = 0; i < ARRAY_SIZE(image_convert_formats); i++) {
 331		if (image_convert_formats[i].fourcc == fourcc) {
 332			ret = &image_convert_formats[i];
 333			break;
 334		}
 335	}
 336
 337	return ret;
 338}
 339
 340static void dump_format(struct ipu_image_convert_ctx *ctx,
 341			struct ipu_image_convert_image *ic_image)
 342{
 343	struct ipu_image_convert_chan *chan = ctx->chan;
 344	struct ipu_image_convert_priv *priv = chan->priv;
 345
 346	dev_dbg(priv->ipu->dev,
 347		"task %u: ctx %p: %s format: %dx%d (%dx%d tiles), %c%c%c%c\n",
 348		chan->ic_task, ctx,
 349		ic_image->type == IMAGE_CONVERT_OUT ? "Output" : "Input",
 350		ic_image->base.pix.width, ic_image->base.pix.height,
 351		ic_image->num_cols, ic_image->num_rows,
 352		ic_image->fmt->fourcc & 0xff,
 353		(ic_image->fmt->fourcc >> 8) & 0xff,
 354		(ic_image->fmt->fourcc >> 16) & 0xff,
 355		(ic_image->fmt->fourcc >> 24) & 0xff);
 356}
 357
 358int ipu_image_convert_enum_format(int index, u32 *fourcc)
 359{
 360	const struct ipu_image_pixfmt *fmt;
 361
 362	if (index >= (int)ARRAY_SIZE(image_convert_formats))
 363		return -EINVAL;
 364
 365	/* Format found */
 366	fmt = &image_convert_formats[index];
 367	*fourcc = fmt->fourcc;
 368	return 0;
 369}
 370EXPORT_SYMBOL_GPL(ipu_image_convert_enum_format);
 371
 372static void free_dma_buf(struct ipu_image_convert_priv *priv,
 373			 struct ipu_image_convert_dma_buf *buf)
 374{
 375	if (buf->virt)
 376		dma_free_coherent(priv->ipu->dev,
 377				  buf->len, buf->virt, buf->phys);
 378	buf->virt = NULL;
 379	buf->phys = 0;
 380}
 381
 382static int alloc_dma_buf(struct ipu_image_convert_priv *priv,
 383			 struct ipu_image_convert_dma_buf *buf,
 384			 int size)
 385{
 386	buf->len = PAGE_ALIGN(size);
 387	buf->virt = dma_alloc_coherent(priv->ipu->dev, buf->len, &buf->phys,
 388				       GFP_DMA | GFP_KERNEL);
 389	if (!buf->virt) {
 390		dev_err(priv->ipu->dev, "failed to alloc dma buffer\n");
 391		return -ENOMEM;
 392	}
 393
 394	return 0;
 395}
 396
 397static inline int num_stripes(int dim)
 398{
 399	return (dim - 1) / 1024 + 1;
 400}
 401
 402/*
 403 * Calculate downsizing coefficients, which are the same for all tiles,
 404 * and initial bilinear resizing coefficients, which are used to find the
 405 * best seam positions.
 406 * Also determine the number of tiles necessary to guarantee that no tile
 407 * is larger than 1024 pixels in either dimension at the output and between
 408 * IC downsizing and main processing sections.
 409 */
 410static int calc_image_resize_coefficients(struct ipu_image_convert_ctx *ctx,
 411					  struct ipu_image *in,
 412					  struct ipu_image *out)
 413{
 414	u32 downsized_width = in->rect.width;
 415	u32 downsized_height = in->rect.height;
 416	u32 downsize_coeff_v = 0;
 417	u32 downsize_coeff_h = 0;
 418	u32 resized_width = out->rect.width;
 419	u32 resized_height = out->rect.height;
 420	u32 resize_coeff_h;
 421	u32 resize_coeff_v;
 422	u32 cols;
 423	u32 rows;
 424
 425	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
 426		resized_width = out->rect.height;
 427		resized_height = out->rect.width;
 428	}
 429
 430	/* Do not let invalid input lead to an endless loop below */
 431	if (WARN_ON(resized_width == 0 || resized_height == 0))
 432		return -EINVAL;
 433
 434	while (downsized_width >= resized_width * 2) {
 435		downsized_width >>= 1;
 436		downsize_coeff_h++;
 437	}
 438
 439	while (downsized_height >= resized_height * 2) {
 440		downsized_height >>= 1;
 441		downsize_coeff_v++;
 442	}
 443
 444	/*
 445	 * Calculate the bilinear resizing coefficients that could be used if
 446	 * we were converting with a single tile. The bottom right output pixel
 447	 * should sample as close as possible to the bottom right input pixel
 448	 * out of the decimator, but not overshoot it:
 449	 */
 450	resize_coeff_h = 8192 * (downsized_width - 1) / (resized_width - 1);
 451	resize_coeff_v = 8192 * (downsized_height - 1) / (resized_height - 1);
 452
 453	/*
 454	 * Both the output of the IC downsizing section before being passed to
 455	 * the IC main processing section and the final output of the IC main
 456	 * processing section must be <= 1024 pixels in both dimensions.
 457	 */
 458	cols = num_stripes(max_t(u32, downsized_width, resized_width));
 459	rows = num_stripes(max_t(u32, downsized_height, resized_height));
 460
 461	dev_dbg(ctx->chan->priv->ipu->dev,
 462		"%s: hscale: >>%u, *8192/%u vscale: >>%u, *8192/%u, %ux%u tiles\n",
 463		__func__, downsize_coeff_h, resize_coeff_h, downsize_coeff_v,
 464		resize_coeff_v, cols, rows);
 465
 466	if (downsize_coeff_h > 2 || downsize_coeff_v  > 2 ||
 467	    resize_coeff_h > 0x3fff || resize_coeff_v > 0x3fff)
 468		return -EINVAL;
 469
 470	ctx->downsize_coeff_h = downsize_coeff_h;
 471	ctx->downsize_coeff_v = downsize_coeff_v;
 472	ctx->image_resize_coeff_h = resize_coeff_h;
 473	ctx->image_resize_coeff_v = resize_coeff_v;
 474	ctx->in.num_cols = cols;
 475	ctx->in.num_rows = rows;
 476
 477	return 0;
 478}
 479
 480#define round_closest(x, y) round_down((x) + (y)/2, (y))
 481
 482/*
 483 * Find the best aligned seam position for the given column / row index.
 484 * Rotation and image offsets are out of scope.
 485 *
 486 * @index: column / row index, used to calculate valid interval
 487 * @in_edge: input right / bottom edge
 488 * @out_edge: output right / bottom edge
 489 * @in_align: input alignment, either horizontal 8-byte line start address
 490 *            alignment, or pixel alignment due to image format
 491 * @out_align: output alignment, either horizontal 8-byte line start address
 492 *             alignment, or pixel alignment due to image format or rotator
 493 *             block size
 494 * @in_burst: horizontal input burst size in case of horizontal flip
 495 * @out_burst: horizontal output burst size or rotator block size
 496 * @downsize_coeff: downsizing section coefficient
 497 * @resize_coeff: main processing section resizing coefficient
 498 * @_in_seam: aligned input seam position return value
 499 * @_out_seam: aligned output seam position return value
 500 */
 501static void find_best_seam(struct ipu_image_convert_ctx *ctx,
 502			   unsigned int index,
 503			   unsigned int in_edge,
 504			   unsigned int out_edge,
 505			   unsigned int in_align,
 506			   unsigned int out_align,
 507			   unsigned int in_burst,
 508			   unsigned int out_burst,
 509			   unsigned int downsize_coeff,
 510			   unsigned int resize_coeff,
 511			   u32 *_in_seam,
 512			   u32 *_out_seam)
 513{
 514	struct device *dev = ctx->chan->priv->ipu->dev;
 515	unsigned int out_pos;
 516	/* Input / output seam position candidates */
 517	unsigned int out_seam = 0;
 518	unsigned int in_seam = 0;
 519	unsigned int min_diff = UINT_MAX;
 520	unsigned int out_start;
 521	unsigned int out_end;
 522	unsigned int in_start;
 523	unsigned int in_end;
 524
 525	/* Start within 1024 pixels of the right / bottom edge */
 526	out_start = max_t(int, index * out_align, out_edge - 1024);
 527	/* End before having to add more columns to the left / rows above */
 528	out_end = min_t(unsigned int, out_edge, index * 1024 + 1);
 529
 530	/*
 531	 * Limit input seam position to make sure that the downsized input tile
 532	 * to the right or bottom does not exceed 1024 pixels.
 533	 */
 534	in_start = max_t(int, index * in_align,
 535			 in_edge - (1024 << downsize_coeff));
 536	in_end = min_t(unsigned int, in_edge,
 537		       index * (1024 << downsize_coeff) + 1);
 538
 539	/*
 540	 * Output tiles must start at a multiple of 8 bytes horizontally and
 541	 * possibly at an even line horizontally depending on the pixel format.
 542	 * Only consider output aligned positions for the seam.
 543	 */
 544	out_start = round_up(out_start, out_align);
 545	for (out_pos = out_start; out_pos < out_end; out_pos += out_align) {
 546		unsigned int in_pos;
 547		unsigned int in_pos_aligned;
 548		unsigned int in_pos_rounded;
 549		unsigned int diff;
 550
 551		/*
 552		 * Tiles in the right row / bottom column may not be allowed to
 553		 * overshoot horizontally / vertically. out_burst may be the
 554		 * actual DMA burst size, or the rotator block size.
 555		 */
 556		if ((out_burst > 1) && (out_edge - out_pos) % out_burst)
 557			continue;
 558
 559		/*
 560		 * Input sample position, corresponding to out_pos, 19.13 fixed
 561		 * point.
 562		 */
 563		in_pos = (out_pos * resize_coeff) << downsize_coeff;
 564		/*
 565		 * The closest input sample position that we could actually
 566		 * start the input tile at, 19.13 fixed point.
 567		 */
 568		in_pos_aligned = round_closest(in_pos, 8192U * in_align);
 569		/* Convert 19.13 fixed point to integer */
 570		in_pos_rounded = in_pos_aligned / 8192U;
 571
 572		if (in_pos_rounded < in_start)
 573			continue;
 574		if (in_pos_rounded >= in_end)
 575			break;
 576
 577		if ((in_burst > 1) &&
 578		    (in_edge - in_pos_rounded) % in_burst)
 579			continue;
 580
 581		diff = abs_diff(in_pos, in_pos_aligned);
 582		if (diff < min_diff) {
 
 
 
 
 583			in_seam = in_pos_rounded;
 584			out_seam = out_pos;
 585			min_diff = diff;
 586		}
 587	}
 588
 589	*_out_seam = out_seam;
 590	*_in_seam = in_seam;
 591
 592	dev_dbg(dev, "%s: out_seam %u(%u) in [%u, %u], in_seam %u(%u) in [%u, %u] diff %u.%03u\n",
 593		__func__, out_seam, out_align, out_start, out_end,
 594		in_seam, in_align, in_start, in_end, min_diff / 8192,
 595		DIV_ROUND_CLOSEST(min_diff % 8192 * 1000, 8192));
 596}
 597
 598/*
 599 * Tile left edges are required to be aligned to multiples of 8 bytes
 600 * by the IDMAC.
 601 */
 602static inline u32 tile_left_align(const struct ipu_image_pixfmt *fmt)
 603{
 604	if (fmt->planar)
 605		return fmt->uv_packed ? 8 : 8 * fmt->uv_width_dec;
 606	else
 607		return fmt->bpp == 32 ? 2 : fmt->bpp == 16 ? 4 : 8;
 608}
 609
 610/*
 611 * Tile top edge alignment is only limited by chroma subsampling.
 612 */
 613static inline u32 tile_top_align(const struct ipu_image_pixfmt *fmt)
 614{
 615	return fmt->uv_height_dec > 1 ? 2 : 1;
 616}
 617
 618static inline u32 tile_width_align(enum ipu_image_convert_type type,
 619				   const struct ipu_image_pixfmt *fmt,
 620				   enum ipu_rotate_mode rot_mode)
 621{
 622	if (type == IMAGE_CONVERT_IN) {
 623		/*
 624		 * The IC burst reads 8 pixels at a time. Reading beyond the
 625		 * end of the line is usually acceptable. Those pixels are
 626		 * ignored, unless the IC has to write the scaled line in
 627		 * reverse.
 628		 */
 629		return (!ipu_rot_mode_is_irt(rot_mode) &&
 630			(rot_mode & IPU_ROT_BIT_HFLIP)) ? 8 : 2;
 631	}
 632
 633	/*
 634	 * Align to 16x16 pixel blocks for planar 4:2:0 chroma subsampled
 635	 * formats to guarantee 8-byte aligned line start addresses in the
 636	 * chroma planes when IRT is used. Align to 8x8 pixel IRT block size
 637	 * for all other formats.
 638	 */
 639	return (ipu_rot_mode_is_irt(rot_mode) &&
 640		fmt->planar && !fmt->uv_packed) ?
 641		8 * fmt->uv_width_dec : 8;
 642}
 643
 644static inline u32 tile_height_align(enum ipu_image_convert_type type,
 645				    const struct ipu_image_pixfmt *fmt,
 646				    enum ipu_rotate_mode rot_mode)
 647{
 648	if (type == IMAGE_CONVERT_IN || !ipu_rot_mode_is_irt(rot_mode))
 649		return 2;
 650
 651	/*
 652	 * Align to 16x16 pixel blocks for planar 4:2:0 chroma subsampled
 653	 * formats to guarantee 8-byte aligned line start addresses in the
 654	 * chroma planes when IRT is used. Align to 8x8 pixel IRT block size
 655	 * for all other formats.
 656	 */
 657	return (fmt->planar && !fmt->uv_packed) ? 8 * fmt->uv_width_dec : 8;
 658}
 659
 660/*
 661 * Fill in left position and width and for all tiles in an input column, and
 662 * for all corresponding output tiles. If the 90° rotator is used, the output
 663 * tiles are in a row, and output tile top position and height are set.
 664 */
 665static void fill_tile_column(struct ipu_image_convert_ctx *ctx,
 666			     unsigned int col,
 667			     struct ipu_image_convert_image *in,
 668			     unsigned int in_left, unsigned int in_width,
 669			     struct ipu_image_convert_image *out,
 670			     unsigned int out_left, unsigned int out_width)
 671{
 672	unsigned int row, tile_idx;
 673	struct ipu_image_tile *in_tile, *out_tile;
 674
 675	for (row = 0; row < in->num_rows; row++) {
 676		tile_idx = in->num_cols * row + col;
 677		in_tile = &in->tile[tile_idx];
 678		out_tile = &out->tile[ctx->out_tile_map[tile_idx]];
 679
 680		in_tile->left = in_left;
 681		in_tile->width = in_width;
 682
 683		if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
 684			out_tile->top = out_left;
 685			out_tile->height = out_width;
 686		} else {
 687			out_tile->left = out_left;
 688			out_tile->width = out_width;
 689		}
 690	}
 691}
 692
 693/*
 694 * Fill in top position and height and for all tiles in an input row, and
 695 * for all corresponding output tiles. If the 90° rotator is used, the output
 696 * tiles are in a column, and output tile left position and width are set.
 697 */
 698static void fill_tile_row(struct ipu_image_convert_ctx *ctx, unsigned int row,
 699			  struct ipu_image_convert_image *in,
 700			  unsigned int in_top, unsigned int in_height,
 701			  struct ipu_image_convert_image *out,
 702			  unsigned int out_top, unsigned int out_height)
 703{
 704	unsigned int col, tile_idx;
 705	struct ipu_image_tile *in_tile, *out_tile;
 706
 707	for (col = 0; col < in->num_cols; col++) {
 708		tile_idx = in->num_cols * row + col;
 709		in_tile = &in->tile[tile_idx];
 710		out_tile = &out->tile[ctx->out_tile_map[tile_idx]];
 711
 712		in_tile->top = in_top;
 713		in_tile->height = in_height;
 714
 715		if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
 716			out_tile->left = out_top;
 717			out_tile->width = out_height;
 718		} else {
 719			out_tile->top = out_top;
 720			out_tile->height = out_height;
 721		}
 722	}
 723}
 724
 725/*
 726 * Find the best horizontal and vertical seam positions to split into tiles.
 727 * Minimize the fractional part of the input sampling position for the
 728 * top / left pixels of each tile.
 729 */
 730static void find_seams(struct ipu_image_convert_ctx *ctx,
 731		       struct ipu_image_convert_image *in,
 732		       struct ipu_image_convert_image *out)
 733{
 734	struct device *dev = ctx->chan->priv->ipu->dev;
 735	unsigned int resized_width = out->base.rect.width;
 736	unsigned int resized_height = out->base.rect.height;
 737	unsigned int col;
 738	unsigned int row;
 739	unsigned int in_left_align = tile_left_align(in->fmt);
 740	unsigned int in_top_align = tile_top_align(in->fmt);
 741	unsigned int out_left_align = tile_left_align(out->fmt);
 742	unsigned int out_top_align = tile_top_align(out->fmt);
 743	unsigned int out_width_align = tile_width_align(out->type, out->fmt,
 744							ctx->rot_mode);
 745	unsigned int out_height_align = tile_height_align(out->type, out->fmt,
 746							  ctx->rot_mode);
 747	unsigned int in_right = in->base.rect.width;
 748	unsigned int in_bottom = in->base.rect.height;
 749	unsigned int out_right = out->base.rect.width;
 750	unsigned int out_bottom = out->base.rect.height;
 751	unsigned int flipped_out_left;
 752	unsigned int flipped_out_top;
 753
 754	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
 755		/* Switch width/height and align top left to IRT block size */
 756		resized_width = out->base.rect.height;
 757		resized_height = out->base.rect.width;
 758		out_left_align = out_height_align;
 759		out_top_align = out_width_align;
 760		out_width_align = out_left_align;
 761		out_height_align = out_top_align;
 762		out_right = out->base.rect.height;
 763		out_bottom = out->base.rect.width;
 764	}
 765
 766	for (col = in->num_cols - 1; col > 0; col--) {
 767		bool allow_in_overshoot = ipu_rot_mode_is_irt(ctx->rot_mode) ||
 768					  !(ctx->rot_mode & IPU_ROT_BIT_HFLIP);
 769		bool allow_out_overshoot = (col < in->num_cols - 1) &&
 770					   !(ctx->rot_mode & IPU_ROT_BIT_HFLIP);
 771		unsigned int in_left;
 772		unsigned int out_left;
 773
 774		/*
 775		 * Align input width to burst length if the scaling step flips
 776		 * horizontally.
 777		 */
 778
 779		find_best_seam(ctx, col,
 780			       in_right, out_right,
 781			       in_left_align, out_left_align,
 782			       allow_in_overshoot ? 1 : 8 /* burst length */,
 783			       allow_out_overshoot ? 1 : out_width_align,
 784			       ctx->downsize_coeff_h, ctx->image_resize_coeff_h,
 785			       &in_left, &out_left);
 786
 787		if (ctx->rot_mode & IPU_ROT_BIT_HFLIP)
 788			flipped_out_left = resized_width - out_right;
 789		else
 790			flipped_out_left = out_left;
 791
 792		fill_tile_column(ctx, col, in, in_left, in_right - in_left,
 793				 out, flipped_out_left, out_right - out_left);
 794
 795		dev_dbg(dev, "%s: col %u: %u, %u -> %u, %u\n", __func__, col,
 796			in_left, in_right - in_left,
 797			flipped_out_left, out_right - out_left);
 798
 799		in_right = in_left;
 800		out_right = out_left;
 801	}
 802
 803	flipped_out_left = (ctx->rot_mode & IPU_ROT_BIT_HFLIP) ?
 804			   resized_width - out_right : 0;
 805
 806	fill_tile_column(ctx, 0, in, 0, in_right,
 807			 out, flipped_out_left, out_right);
 808
 809	dev_dbg(dev, "%s: col 0: 0, %u -> %u, %u\n", __func__,
 810		in_right, flipped_out_left, out_right);
 811
 812	for (row = in->num_rows - 1; row > 0; row--) {
 813		bool allow_overshoot = row < in->num_rows - 1;
 814		unsigned int in_top;
 815		unsigned int out_top;
 816
 817		find_best_seam(ctx, row,
 818			       in_bottom, out_bottom,
 819			       in_top_align, out_top_align,
 820			       1, allow_overshoot ? 1 : out_height_align,
 821			       ctx->downsize_coeff_v, ctx->image_resize_coeff_v,
 822			       &in_top, &out_top);
 823
 824		if ((ctx->rot_mode & IPU_ROT_BIT_VFLIP) ^
 825		    ipu_rot_mode_is_irt(ctx->rot_mode))
 826			flipped_out_top = resized_height - out_bottom;
 827		else
 828			flipped_out_top = out_top;
 829
 830		fill_tile_row(ctx, row, in, in_top, in_bottom - in_top,
 831			      out, flipped_out_top, out_bottom - out_top);
 832
 833		dev_dbg(dev, "%s: row %u: %u, %u -> %u, %u\n", __func__, row,
 834			in_top, in_bottom - in_top,
 835			flipped_out_top, out_bottom - out_top);
 836
 837		in_bottom = in_top;
 838		out_bottom = out_top;
 839	}
 840
 841	if ((ctx->rot_mode & IPU_ROT_BIT_VFLIP) ^
 842	    ipu_rot_mode_is_irt(ctx->rot_mode))
 843		flipped_out_top = resized_height - out_bottom;
 844	else
 845		flipped_out_top = 0;
 846
 847	fill_tile_row(ctx, 0, in, 0, in_bottom,
 848		      out, flipped_out_top, out_bottom);
 849
 850	dev_dbg(dev, "%s: row 0: 0, %u -> %u, %u\n", __func__,
 851		in_bottom, flipped_out_top, out_bottom);
 852}
 853
 854static int calc_tile_dimensions(struct ipu_image_convert_ctx *ctx,
 855				struct ipu_image_convert_image *image)
 856{
 857	struct ipu_image_convert_chan *chan = ctx->chan;
 858	struct ipu_image_convert_priv *priv = chan->priv;
 859	unsigned int max_width = 1024;
 860	unsigned int max_height = 1024;
 861	unsigned int i;
 862
 863	if (image->type == IMAGE_CONVERT_IN) {
 864		/* Up to 4096x4096 input tile size */
 865		max_width <<= ctx->downsize_coeff_h;
 866		max_height <<= ctx->downsize_coeff_v;
 867	}
 868
 869	for (i = 0; i < ctx->num_tiles; i++) {
 870		struct ipu_image_tile *tile;
 871		const unsigned int row = i / image->num_cols;
 872		const unsigned int col = i % image->num_cols;
 873
 874		if (image->type == IMAGE_CONVERT_OUT)
 875			tile = &image->tile[ctx->out_tile_map[i]];
 876		else
 877			tile = &image->tile[i];
 878
 879		tile->size = ((tile->height * image->fmt->bpp) >> 3) *
 880			tile->width;
 881
 882		if (image->fmt->planar) {
 883			tile->stride = tile->width;
 884			tile->rot_stride = tile->height;
 885		} else {
 886			tile->stride =
 887				(image->fmt->bpp * tile->width) >> 3;
 888			tile->rot_stride =
 889				(image->fmt->bpp * tile->height) >> 3;
 890		}
 891
 892		dev_dbg(priv->ipu->dev,
 893			"task %u: ctx %p: %s@[%u,%u]: %ux%u@%u,%u\n",
 894			chan->ic_task, ctx,
 895			image->type == IMAGE_CONVERT_IN ? "Input" : "Output",
 896			row, col,
 897			tile->width, tile->height, tile->left, tile->top);
 898
 899		if (!tile->width || tile->width > max_width ||
 900		    !tile->height || tile->height > max_height) {
 901			dev_err(priv->ipu->dev, "invalid %s tile size: %ux%u\n",
 902				image->type == IMAGE_CONVERT_IN ? "input" :
 903				"output", tile->width, tile->height);
 904			return -EINVAL;
 905		}
 906	}
 907
 908	return 0;
 909}
 910
 911/*
 912 * Use the rotation transformation to find the tile coordinates
 913 * (row, col) of a tile in the destination frame that corresponds
 914 * to the given tile coordinates of a source frame. The destination
 915 * coordinate is then converted to a tile index.
 916 */
 917static int transform_tile_index(struct ipu_image_convert_ctx *ctx,
 918				int src_row, int src_col)
 919{
 920	struct ipu_image_convert_chan *chan = ctx->chan;
 921	struct ipu_image_convert_priv *priv = chan->priv;
 922	struct ipu_image_convert_image *s_image = &ctx->in;
 923	struct ipu_image_convert_image *d_image = &ctx->out;
 924	int dst_row, dst_col;
 925
 926	/* with no rotation it's a 1:1 mapping */
 927	if (ctx->rot_mode == IPU_ROTATE_NONE)
 928		return src_row * s_image->num_cols + src_col;
 929
 930	/*
 931	 * before doing the transform, first we have to translate
 932	 * source row,col for an origin in the center of s_image
 933	 */
 934	src_row = src_row * 2 - (s_image->num_rows - 1);
 935	src_col = src_col * 2 - (s_image->num_cols - 1);
 936
 937	/* do the rotation transform */
 938	if (ctx->rot_mode & IPU_ROT_BIT_90) {
 939		dst_col = -src_row;
 940		dst_row = src_col;
 941	} else {
 942		dst_col = src_col;
 943		dst_row = src_row;
 944	}
 945
 946	/* apply flip */
 947	if (ctx->rot_mode & IPU_ROT_BIT_HFLIP)
 948		dst_col = -dst_col;
 949	if (ctx->rot_mode & IPU_ROT_BIT_VFLIP)
 950		dst_row = -dst_row;
 951
 952	dev_dbg(priv->ipu->dev, "task %u: ctx %p: [%d,%d] --> [%d,%d]\n",
 953		chan->ic_task, ctx, src_col, src_row, dst_col, dst_row);
 954
 955	/*
 956	 * finally translate dest row,col using an origin in upper
 957	 * left of d_image
 958	 */
 959	dst_row += d_image->num_rows - 1;
 960	dst_col += d_image->num_cols - 1;
 961	dst_row /= 2;
 962	dst_col /= 2;
 963
 964	return dst_row * d_image->num_cols + dst_col;
 965}
 966
 967/*
 968 * Fill the out_tile_map[] with transformed destination tile indeces.
 969 */
 970static void calc_out_tile_map(struct ipu_image_convert_ctx *ctx)
 971{
 972	struct ipu_image_convert_image *s_image = &ctx->in;
 973	unsigned int row, col, tile = 0;
 974
 975	for (row = 0; row < s_image->num_rows; row++) {
 976		for (col = 0; col < s_image->num_cols; col++) {
 977			ctx->out_tile_map[tile] =
 978				transform_tile_index(ctx, row, col);
 979			tile++;
 980		}
 981	}
 982}
 983
 984static int calc_tile_offsets_planar(struct ipu_image_convert_ctx *ctx,
 985				    struct ipu_image_convert_image *image)
 986{
 987	struct ipu_image_convert_chan *chan = ctx->chan;
 988	struct ipu_image_convert_priv *priv = chan->priv;
 989	const struct ipu_image_pixfmt *fmt = image->fmt;
 990	unsigned int row, col, tile = 0;
 991	u32 H, top, y_stride, uv_stride;
 992	u32 uv_row_off, uv_col_off, uv_off, u_off, v_off;
 993	u32 y_row_off, y_col_off, y_off;
 994	u32 y_size, uv_size;
 995
 996	/* setup some convenience vars */
 997	H = image->base.pix.height;
 998
 999	y_stride = image->stride;
1000	uv_stride = y_stride / fmt->uv_width_dec;
1001	if (fmt->uv_packed)
1002		uv_stride *= 2;
1003
1004	y_size = H * y_stride;
1005	uv_size = y_size / (fmt->uv_width_dec * fmt->uv_height_dec);
1006
1007	for (row = 0; row < image->num_rows; row++) {
1008		top = image->tile[tile].top;
1009		y_row_off = top * y_stride;
1010		uv_row_off = (top * uv_stride) / fmt->uv_height_dec;
1011
1012		for (col = 0; col < image->num_cols; col++) {
1013			y_col_off = image->tile[tile].left;
1014			uv_col_off = y_col_off / fmt->uv_width_dec;
1015			if (fmt->uv_packed)
1016				uv_col_off *= 2;
1017
1018			y_off = y_row_off + y_col_off;
1019			uv_off = uv_row_off + uv_col_off;
1020
1021			u_off = y_size - y_off + uv_off;
1022			v_off = (fmt->uv_packed) ? 0 : u_off + uv_size;
1023			if (fmt->uv_swapped)
1024				swap(u_off, v_off);
1025
1026			image->tile[tile].offset = y_off;
1027			image->tile[tile].u_off = u_off;
1028			image->tile[tile++].v_off = v_off;
1029
1030			if ((y_off & 0x7) || (u_off & 0x7) || (v_off & 0x7)) {
1031				dev_err(priv->ipu->dev,
1032					"task %u: ctx %p: %s@[%d,%d]: "
1033					"y_off %08x, u_off %08x, v_off %08x\n",
1034					chan->ic_task, ctx,
1035					image->type == IMAGE_CONVERT_IN ?
1036					"Input" : "Output", row, col,
1037					y_off, u_off, v_off);
1038				return -EINVAL;
1039			}
1040		}
1041	}
1042
1043	return 0;
1044}
1045
1046static int calc_tile_offsets_packed(struct ipu_image_convert_ctx *ctx,
1047				    struct ipu_image_convert_image *image)
1048{
1049	struct ipu_image_convert_chan *chan = ctx->chan;
1050	struct ipu_image_convert_priv *priv = chan->priv;
1051	const struct ipu_image_pixfmt *fmt = image->fmt;
1052	unsigned int row, col, tile = 0;
1053	u32 bpp, stride, offset;
1054	u32 row_off, col_off;
1055
1056	/* setup some convenience vars */
1057	stride = image->stride;
1058	bpp = fmt->bpp;
1059
1060	for (row = 0; row < image->num_rows; row++) {
1061		row_off = image->tile[tile].top * stride;
1062
1063		for (col = 0; col < image->num_cols; col++) {
1064			col_off = (image->tile[tile].left * bpp) >> 3;
1065
1066			offset = row_off + col_off;
1067
1068			image->tile[tile].offset = offset;
1069			image->tile[tile].u_off = 0;
1070			image->tile[tile++].v_off = 0;
1071
1072			if (offset & 0x7) {
1073				dev_err(priv->ipu->dev,
1074					"task %u: ctx %p: %s@[%d,%d]: "
1075					"phys %08x\n",
1076					chan->ic_task, ctx,
1077					image->type == IMAGE_CONVERT_IN ?
1078					"Input" : "Output", row, col,
1079					row_off + col_off);
1080				return -EINVAL;
1081			}
1082		}
1083	}
1084
1085	return 0;
1086}
1087
1088static int calc_tile_offsets(struct ipu_image_convert_ctx *ctx,
1089			      struct ipu_image_convert_image *image)
1090{
1091	if (image->fmt->planar)
1092		return calc_tile_offsets_planar(ctx, image);
1093
1094	return calc_tile_offsets_packed(ctx, image);
1095}
1096
1097/*
1098 * Calculate the resizing ratio for the IC main processing section given input
1099 * size, fixed downsizing coefficient, and output size.
1100 * Either round to closest for the next tile's first pixel to minimize seams
1101 * and distortion (for all but right column / bottom row), or round down to
1102 * avoid sampling beyond the edges of the input image for this tile's last
1103 * pixel.
1104 * Returns the resizing coefficient, resizing ratio is 8192.0 / resize_coeff.
1105 */
1106static u32 calc_resize_coeff(u32 input_size, u32 downsize_coeff,
1107			     u32 output_size, bool allow_overshoot)
1108{
1109	u32 downsized = input_size >> downsize_coeff;
1110
1111	if (allow_overshoot)
1112		return DIV_ROUND_CLOSEST(8192 * downsized, output_size);
1113	else
1114		return 8192 * (downsized - 1) / (output_size - 1);
1115}
1116
1117/*
1118 * Slightly modify resize coefficients per tile to hide the bilinear
1119 * interpolator reset at tile borders, shifting the right / bottom edge
1120 * by up to a half input pixel. This removes noticeable seams between
1121 * tiles at higher upscaling factors.
1122 */
1123static void calc_tile_resize_coefficients(struct ipu_image_convert_ctx *ctx)
1124{
1125	struct ipu_image_convert_chan *chan = ctx->chan;
1126	struct ipu_image_convert_priv *priv = chan->priv;
1127	struct ipu_image_tile *in_tile, *out_tile;
1128	unsigned int col, row, tile_idx;
1129	unsigned int last_output;
1130
1131	for (col = 0; col < ctx->in.num_cols; col++) {
1132		bool closest = (col < ctx->in.num_cols - 1) &&
1133			       !(ctx->rot_mode & IPU_ROT_BIT_HFLIP);
1134		u32 resized_width;
1135		u32 resize_coeff_h;
1136		u32 in_width;
1137
1138		tile_idx = col;
1139		in_tile = &ctx->in.tile[tile_idx];
1140		out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]];
1141
1142		if (ipu_rot_mode_is_irt(ctx->rot_mode))
1143			resized_width = out_tile->height;
1144		else
1145			resized_width = out_tile->width;
1146
1147		resize_coeff_h = calc_resize_coeff(in_tile->width,
1148						   ctx->downsize_coeff_h,
1149						   resized_width, closest);
1150
1151		dev_dbg(priv->ipu->dev, "%s: column %u hscale: *8192/%u\n",
1152			__func__, col, resize_coeff_h);
1153
1154		/*
1155		 * With the horizontal scaling factor known, round up resized
1156		 * width (output width or height) to burst size.
1157		 */
1158		resized_width = round_up(resized_width, 8);
1159
1160		/*
1161		 * Calculate input width from the last accessed input pixel
1162		 * given resized width and scaling coefficients. Round up to
1163		 * burst size.
1164		 */
1165		last_output = resized_width - 1;
1166		if (closest && ((last_output * resize_coeff_h) % 8192))
1167			last_output++;
1168		in_width = round_up(
1169			(DIV_ROUND_UP(last_output * resize_coeff_h, 8192) + 1)
1170			<< ctx->downsize_coeff_h, 8);
1171
1172		for (row = 0; row < ctx->in.num_rows; row++) {
1173			tile_idx = row * ctx->in.num_cols + col;
1174			in_tile = &ctx->in.tile[tile_idx];
1175			out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]];
1176
1177			if (ipu_rot_mode_is_irt(ctx->rot_mode))
1178				out_tile->height = resized_width;
1179			else
1180				out_tile->width = resized_width;
1181
1182			in_tile->width = in_width;
1183		}
1184
1185		ctx->resize_coeffs_h[col] = resize_coeff_h;
1186	}
1187
1188	for (row = 0; row < ctx->in.num_rows; row++) {
1189		bool closest = (row < ctx->in.num_rows - 1) &&
1190			       !(ctx->rot_mode & IPU_ROT_BIT_VFLIP);
1191		u32 resized_height;
1192		u32 resize_coeff_v;
1193		u32 in_height;
1194
1195		tile_idx = row * ctx->in.num_cols;
1196		in_tile = &ctx->in.tile[tile_idx];
1197		out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]];
1198
1199		if (ipu_rot_mode_is_irt(ctx->rot_mode))
1200			resized_height = out_tile->width;
1201		else
1202			resized_height = out_tile->height;
1203
1204		resize_coeff_v = calc_resize_coeff(in_tile->height,
1205						   ctx->downsize_coeff_v,
1206						   resized_height, closest);
1207
1208		dev_dbg(priv->ipu->dev, "%s: row %u vscale: *8192/%u\n",
1209			__func__, row, resize_coeff_v);
1210
1211		/*
1212		 * With the vertical scaling factor known, round up resized
1213		 * height (output width or height) to IDMAC limitations.
1214		 */
1215		resized_height = round_up(resized_height, 2);
1216
1217		/*
1218		 * Calculate input width from the last accessed input pixel
1219		 * given resized height and scaling coefficients. Align to
1220		 * IDMAC restrictions.
1221		 */
1222		last_output = resized_height - 1;
1223		if (closest && ((last_output * resize_coeff_v) % 8192))
1224			last_output++;
1225		in_height = round_up(
1226			(DIV_ROUND_UP(last_output * resize_coeff_v, 8192) + 1)
1227			<< ctx->downsize_coeff_v, 2);
1228
1229		for (col = 0; col < ctx->in.num_cols; col++) {
1230			tile_idx = row * ctx->in.num_cols + col;
1231			in_tile = &ctx->in.tile[tile_idx];
1232			out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]];
1233
1234			if (ipu_rot_mode_is_irt(ctx->rot_mode))
1235				out_tile->width = resized_height;
1236			else
1237				out_tile->height = resized_height;
1238
1239			in_tile->height = in_height;
1240		}
1241
1242		ctx->resize_coeffs_v[row] = resize_coeff_v;
1243	}
1244}
1245
1246/*
1247 * return the number of runs in given queue (pending_q or done_q)
1248 * for this context. hold irqlock when calling.
1249 */
1250static int get_run_count(struct ipu_image_convert_ctx *ctx,
1251			 struct list_head *q)
1252{
1253	struct ipu_image_convert_run *run;
1254	int count = 0;
1255
1256	lockdep_assert_held(&ctx->chan->irqlock);
1257
1258	list_for_each_entry(run, q, list) {
1259		if (run->ctx == ctx)
1260			count++;
1261	}
1262
1263	return count;
1264}
1265
1266static void convert_stop(struct ipu_image_convert_run *run)
1267{
1268	struct ipu_image_convert_ctx *ctx = run->ctx;
1269	struct ipu_image_convert_chan *chan = ctx->chan;
1270	struct ipu_image_convert_priv *priv = chan->priv;
1271
1272	dev_dbg(priv->ipu->dev, "%s: task %u: stopping ctx %p run %p\n",
1273		__func__, chan->ic_task, ctx, run);
1274
1275	/* disable IC tasks and the channels */
1276	ipu_ic_task_disable(chan->ic);
1277	ipu_idmac_disable_channel(chan->in_chan);
1278	ipu_idmac_disable_channel(chan->out_chan);
1279
1280	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
1281		ipu_idmac_disable_channel(chan->rotation_in_chan);
1282		ipu_idmac_disable_channel(chan->rotation_out_chan);
1283		ipu_idmac_unlink(chan->out_chan, chan->rotation_in_chan);
1284	}
1285
1286	ipu_ic_disable(chan->ic);
1287}
1288
1289static void init_idmac_channel(struct ipu_image_convert_ctx *ctx,
1290			       struct ipuv3_channel *channel,
1291			       struct ipu_image_convert_image *image,
1292			       enum ipu_rotate_mode rot_mode,
1293			       bool rot_swap_width_height,
1294			       unsigned int tile)
1295{
1296	struct ipu_image_convert_chan *chan = ctx->chan;
1297	unsigned int burst_size;
1298	u32 width, height, stride;
1299	dma_addr_t addr0, addr1 = 0;
1300	struct ipu_image tile_image;
1301	unsigned int tile_idx[2];
1302
1303	if (image->type == IMAGE_CONVERT_OUT) {
1304		tile_idx[0] = ctx->out_tile_map[tile];
1305		tile_idx[1] = ctx->out_tile_map[1];
1306	} else {
1307		tile_idx[0] = tile;
1308		tile_idx[1] = 1;
1309	}
1310
1311	if (rot_swap_width_height) {
1312		width = image->tile[tile_idx[0]].height;
1313		height = image->tile[tile_idx[0]].width;
1314		stride = image->tile[tile_idx[0]].rot_stride;
1315		addr0 = ctx->rot_intermediate[0].phys;
1316		if (ctx->double_buffering)
1317			addr1 = ctx->rot_intermediate[1].phys;
1318	} else {
1319		width = image->tile[tile_idx[0]].width;
1320		height = image->tile[tile_idx[0]].height;
1321		stride = image->stride;
1322		addr0 = image->base.phys0 +
1323			image->tile[tile_idx[0]].offset;
1324		if (ctx->double_buffering)
1325			addr1 = image->base.phys0 +
1326				image->tile[tile_idx[1]].offset;
1327	}
1328
1329	ipu_cpmem_zero(channel);
1330
1331	memset(&tile_image, 0, sizeof(tile_image));
1332	tile_image.pix.width = tile_image.rect.width = width;
1333	tile_image.pix.height = tile_image.rect.height = height;
1334	tile_image.pix.bytesperline = stride;
1335	tile_image.pix.pixelformat =  image->fmt->fourcc;
1336	tile_image.phys0 = addr0;
1337	tile_image.phys1 = addr1;
1338	if (image->fmt->planar && !rot_swap_width_height) {
1339		tile_image.u_offset = image->tile[tile_idx[0]].u_off;
1340		tile_image.v_offset = image->tile[tile_idx[0]].v_off;
1341	}
1342
1343	ipu_cpmem_set_image(channel, &tile_image);
1344
1345	if (rot_mode)
1346		ipu_cpmem_set_rotation(channel, rot_mode);
1347
1348	/*
1349	 * Skip writing U and V components to odd rows in the output
1350	 * channels for planar 4:2:0.
1351	 */
1352	if ((channel == chan->out_chan ||
1353	     channel == chan->rotation_out_chan) &&
1354	    image->fmt->planar && image->fmt->uv_height_dec == 2)
1355		ipu_cpmem_skip_odd_chroma_rows(channel);
1356
1357	if (channel == chan->rotation_in_chan ||
1358	    channel == chan->rotation_out_chan) {
1359		burst_size = 8;
1360		ipu_cpmem_set_block_mode(channel);
1361	} else
1362		burst_size = (width % 16) ? 8 : 16;
1363
1364	ipu_cpmem_set_burstsize(channel, burst_size);
1365
1366	ipu_ic_task_idma_init(chan->ic, channel, width, height,
1367			      burst_size, rot_mode);
1368
1369	/*
1370	 * Setting a non-zero AXI ID collides with the PRG AXI snooping, so
1371	 * only do this when there is no PRG present.
1372	 */
1373	if (!channel->ipu->prg_priv)
1374		ipu_cpmem_set_axi_id(channel, 1);
1375
1376	ipu_idmac_set_double_buffer(channel, ctx->double_buffering);
1377}
1378
1379static int convert_start(struct ipu_image_convert_run *run, unsigned int tile)
1380{
1381	struct ipu_image_convert_ctx *ctx = run->ctx;
1382	struct ipu_image_convert_chan *chan = ctx->chan;
1383	struct ipu_image_convert_priv *priv = chan->priv;
1384	struct ipu_image_convert_image *s_image = &ctx->in;
1385	struct ipu_image_convert_image *d_image = &ctx->out;
1386	unsigned int dst_tile = ctx->out_tile_map[tile];
1387	unsigned int dest_width, dest_height;
1388	unsigned int col, row;
1389	u32 rsc;
1390	int ret;
1391
1392	dev_dbg(priv->ipu->dev, "%s: task %u: starting ctx %p run %p tile %u -> %u\n",
1393		__func__, chan->ic_task, ctx, run, tile, dst_tile);
1394
1395	/* clear EOF irq mask */
1396	ctx->eof_mask = 0;
1397
1398	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
1399		/* swap width/height for resizer */
1400		dest_width = d_image->tile[dst_tile].height;
1401		dest_height = d_image->tile[dst_tile].width;
1402	} else {
1403		dest_width = d_image->tile[dst_tile].width;
1404		dest_height = d_image->tile[dst_tile].height;
1405	}
1406
1407	row = tile / s_image->num_cols;
1408	col = tile % s_image->num_cols;
1409
1410	rsc =  (ctx->downsize_coeff_v << 30) |
1411	       (ctx->resize_coeffs_v[row] << 16) |
1412	       (ctx->downsize_coeff_h << 14) |
1413	       (ctx->resize_coeffs_h[col]);
1414
1415	dev_dbg(priv->ipu->dev, "%s: %ux%u -> %ux%u (rsc = 0x%x)\n",
1416		__func__, s_image->tile[tile].width,
1417		s_image->tile[tile].height, dest_width, dest_height, rsc);
1418
1419	/* setup the IC resizer and CSC */
1420	ret = ipu_ic_task_init_rsc(chan->ic, &ctx->csc,
1421				   s_image->tile[tile].width,
1422				   s_image->tile[tile].height,
1423				   dest_width,
1424				   dest_height,
1425				   rsc);
1426	if (ret) {
1427		dev_err(priv->ipu->dev, "ipu_ic_task_init failed, %d\n", ret);
1428		return ret;
1429	}
1430
1431	/* init the source MEM-->IC PP IDMAC channel */
1432	init_idmac_channel(ctx, chan->in_chan, s_image,
1433			   IPU_ROTATE_NONE, false, tile);
1434
1435	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
1436		/* init the IC PP-->MEM IDMAC channel */
1437		init_idmac_channel(ctx, chan->out_chan, d_image,
1438				   IPU_ROTATE_NONE, true, tile);
1439
1440		/* init the MEM-->IC PP ROT IDMAC channel */
1441		init_idmac_channel(ctx, chan->rotation_in_chan, d_image,
1442				   ctx->rot_mode, true, tile);
1443
1444		/* init the destination IC PP ROT-->MEM IDMAC channel */
1445		init_idmac_channel(ctx, chan->rotation_out_chan, d_image,
1446				   IPU_ROTATE_NONE, false, tile);
1447
1448		/* now link IC PP-->MEM to MEM-->IC PP ROT */
1449		ipu_idmac_link(chan->out_chan, chan->rotation_in_chan);
1450	} else {
1451		/* init the destination IC PP-->MEM IDMAC channel */
1452		init_idmac_channel(ctx, chan->out_chan, d_image,
1453				   ctx->rot_mode, false, tile);
1454	}
1455
1456	/* enable the IC */
1457	ipu_ic_enable(chan->ic);
1458
1459	/* set buffers ready */
1460	ipu_idmac_select_buffer(chan->in_chan, 0);
1461	ipu_idmac_select_buffer(chan->out_chan, 0);
1462	if (ipu_rot_mode_is_irt(ctx->rot_mode))
1463		ipu_idmac_select_buffer(chan->rotation_out_chan, 0);
1464	if (ctx->double_buffering) {
1465		ipu_idmac_select_buffer(chan->in_chan, 1);
1466		ipu_idmac_select_buffer(chan->out_chan, 1);
1467		if (ipu_rot_mode_is_irt(ctx->rot_mode))
1468			ipu_idmac_select_buffer(chan->rotation_out_chan, 1);
1469	}
1470
1471	/* enable the channels! */
1472	ipu_idmac_enable_channel(chan->in_chan);
1473	ipu_idmac_enable_channel(chan->out_chan);
1474	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
1475		ipu_idmac_enable_channel(chan->rotation_in_chan);
1476		ipu_idmac_enable_channel(chan->rotation_out_chan);
1477	}
1478
1479	ipu_ic_task_enable(chan->ic);
1480
1481	ipu_cpmem_dump(chan->in_chan);
1482	ipu_cpmem_dump(chan->out_chan);
1483	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
1484		ipu_cpmem_dump(chan->rotation_in_chan);
1485		ipu_cpmem_dump(chan->rotation_out_chan);
1486	}
1487
1488	ipu_dump(priv->ipu);
1489
1490	return 0;
1491}
1492
1493/* hold irqlock when calling */
1494static int do_run(struct ipu_image_convert_run *run)
1495{
1496	struct ipu_image_convert_ctx *ctx = run->ctx;
1497	struct ipu_image_convert_chan *chan = ctx->chan;
1498
1499	lockdep_assert_held(&chan->irqlock);
1500
1501	ctx->in.base.phys0 = run->in_phys;
1502	ctx->out.base.phys0 = run->out_phys;
1503
1504	ctx->cur_buf_num = 0;
1505	ctx->next_tile = 1;
1506
1507	/* remove run from pending_q and set as current */
1508	list_del(&run->list);
1509	chan->current_run = run;
1510
1511	return convert_start(run, 0);
1512}
1513
1514/* hold irqlock when calling */
1515static void run_next(struct ipu_image_convert_chan *chan)
1516{
1517	struct ipu_image_convert_priv *priv = chan->priv;
1518	struct ipu_image_convert_run *run, *tmp;
1519	int ret;
1520
1521	lockdep_assert_held(&chan->irqlock);
1522
1523	list_for_each_entry_safe(run, tmp, &chan->pending_q, list) {
1524		/* skip contexts that are aborting */
1525		if (run->ctx->aborting) {
1526			dev_dbg(priv->ipu->dev,
1527				"%s: task %u: skipping aborting ctx %p run %p\n",
1528				__func__, chan->ic_task, run->ctx, run);
1529			continue;
1530		}
1531
1532		ret = do_run(run);
1533		if (!ret)
1534			break;
1535
1536		/*
1537		 * something went wrong with start, add the run
1538		 * to done q and continue to the next run in the
1539		 * pending q.
1540		 */
1541		run->status = ret;
1542		list_add_tail(&run->list, &chan->done_q);
1543		chan->current_run = NULL;
1544	}
1545}
1546
1547static void empty_done_q(struct ipu_image_convert_chan *chan)
1548{
1549	struct ipu_image_convert_priv *priv = chan->priv;
1550	struct ipu_image_convert_run *run;
1551	unsigned long flags;
1552
1553	spin_lock_irqsave(&chan->irqlock, flags);
1554
1555	while (!list_empty(&chan->done_q)) {
1556		run = list_entry(chan->done_q.next,
1557				 struct ipu_image_convert_run,
1558				 list);
1559
1560		list_del(&run->list);
1561
1562		dev_dbg(priv->ipu->dev,
1563			"%s: task %u: completing ctx %p run %p with %d\n",
1564			__func__, chan->ic_task, run->ctx, run, run->status);
1565
1566		/* call the completion callback and free the run */
1567		spin_unlock_irqrestore(&chan->irqlock, flags);
1568		run->ctx->complete(run, run->ctx->complete_context);
1569		spin_lock_irqsave(&chan->irqlock, flags);
1570	}
1571
1572	spin_unlock_irqrestore(&chan->irqlock, flags);
1573}
1574
1575/*
1576 * the bottom half thread clears out the done_q, calling the
1577 * completion handler for each.
1578 */
1579static irqreturn_t do_bh(int irq, void *dev_id)
1580{
1581	struct ipu_image_convert_chan *chan = dev_id;
1582	struct ipu_image_convert_priv *priv = chan->priv;
1583	struct ipu_image_convert_ctx *ctx;
1584	unsigned long flags;
1585
1586	dev_dbg(priv->ipu->dev, "%s: task %u: enter\n", __func__,
1587		chan->ic_task);
1588
1589	empty_done_q(chan);
1590
1591	spin_lock_irqsave(&chan->irqlock, flags);
1592
1593	/*
1594	 * the done_q is cleared out, signal any contexts
1595	 * that are aborting that abort can complete.
1596	 */
1597	list_for_each_entry(ctx, &chan->ctx_list, list) {
1598		if (ctx->aborting) {
1599			dev_dbg(priv->ipu->dev,
1600				"%s: task %u: signaling abort for ctx %p\n",
1601				__func__, chan->ic_task, ctx);
1602			complete_all(&ctx->aborted);
1603		}
1604	}
1605
1606	spin_unlock_irqrestore(&chan->irqlock, flags);
1607
1608	dev_dbg(priv->ipu->dev, "%s: task %u: exit\n", __func__,
1609		chan->ic_task);
1610
1611	return IRQ_HANDLED;
1612}
1613
1614static bool ic_settings_changed(struct ipu_image_convert_ctx *ctx)
1615{
1616	unsigned int cur_tile = ctx->next_tile - 1;
1617	unsigned int next_tile = ctx->next_tile;
1618
1619	if (ctx->resize_coeffs_h[cur_tile % ctx->in.num_cols] !=
1620	    ctx->resize_coeffs_h[next_tile % ctx->in.num_cols] ||
1621	    ctx->resize_coeffs_v[cur_tile / ctx->in.num_cols] !=
1622	    ctx->resize_coeffs_v[next_tile / ctx->in.num_cols] ||
1623	    ctx->in.tile[cur_tile].width != ctx->in.tile[next_tile].width ||
1624	    ctx->in.tile[cur_tile].height != ctx->in.tile[next_tile].height ||
1625	    ctx->out.tile[cur_tile].width != ctx->out.tile[next_tile].width ||
1626	    ctx->out.tile[cur_tile].height != ctx->out.tile[next_tile].height)
1627		return true;
1628
1629	return false;
1630}
1631
1632/* hold irqlock when calling */
1633static irqreturn_t do_tile_complete(struct ipu_image_convert_run *run)
1634{
1635	struct ipu_image_convert_ctx *ctx = run->ctx;
1636	struct ipu_image_convert_chan *chan = ctx->chan;
1637	struct ipu_image_tile *src_tile, *dst_tile;
1638	struct ipu_image_convert_image *s_image = &ctx->in;
1639	struct ipu_image_convert_image *d_image = &ctx->out;
1640	struct ipuv3_channel *outch;
1641	unsigned int dst_idx;
1642
1643	lockdep_assert_held(&chan->irqlock);
1644
1645	outch = ipu_rot_mode_is_irt(ctx->rot_mode) ?
1646		chan->rotation_out_chan : chan->out_chan;
1647
1648	/*
1649	 * It is difficult to stop the channel DMA before the channels
1650	 * enter the paused state. Without double-buffering the channels
1651	 * are always in a paused state when the EOF irq occurs, so it
1652	 * is safe to stop the channels now. For double-buffering we
1653	 * just ignore the abort until the operation completes, when it
1654	 * is safe to shut down.
1655	 */
1656	if (ctx->aborting && !ctx->double_buffering) {
1657		convert_stop(run);
1658		run->status = -EIO;
1659		goto done;
1660	}
1661
1662	if (ctx->next_tile == ctx->num_tiles) {
1663		/*
1664		 * the conversion is complete
1665		 */
1666		convert_stop(run);
1667		run->status = 0;
1668		goto done;
1669	}
1670
1671	/*
1672	 * not done, place the next tile buffers.
1673	 */
1674	if (!ctx->double_buffering) {
1675		if (ic_settings_changed(ctx)) {
1676			convert_stop(run);
1677			convert_start(run, ctx->next_tile);
1678		} else {
1679			src_tile = &s_image->tile[ctx->next_tile];
1680			dst_idx = ctx->out_tile_map[ctx->next_tile];
1681			dst_tile = &d_image->tile[dst_idx];
1682
1683			ipu_cpmem_set_buffer(chan->in_chan, 0,
1684					     s_image->base.phys0 +
1685					     src_tile->offset);
1686			ipu_cpmem_set_buffer(outch, 0,
1687					     d_image->base.phys0 +
1688					     dst_tile->offset);
1689			if (s_image->fmt->planar)
1690				ipu_cpmem_set_uv_offset(chan->in_chan,
1691							src_tile->u_off,
1692							src_tile->v_off);
1693			if (d_image->fmt->planar)
1694				ipu_cpmem_set_uv_offset(outch,
1695							dst_tile->u_off,
1696							dst_tile->v_off);
1697
1698			ipu_idmac_select_buffer(chan->in_chan, 0);
1699			ipu_idmac_select_buffer(outch, 0);
1700		}
1701	} else if (ctx->next_tile < ctx->num_tiles - 1) {
1702
1703		src_tile = &s_image->tile[ctx->next_tile + 1];
1704		dst_idx = ctx->out_tile_map[ctx->next_tile + 1];
1705		dst_tile = &d_image->tile[dst_idx];
1706
1707		ipu_cpmem_set_buffer(chan->in_chan, ctx->cur_buf_num,
1708				     s_image->base.phys0 + src_tile->offset);
1709		ipu_cpmem_set_buffer(outch, ctx->cur_buf_num,
1710				     d_image->base.phys0 + dst_tile->offset);
1711
1712		ipu_idmac_select_buffer(chan->in_chan, ctx->cur_buf_num);
1713		ipu_idmac_select_buffer(outch, ctx->cur_buf_num);
1714
1715		ctx->cur_buf_num ^= 1;
1716	}
1717
1718	ctx->eof_mask = 0; /* clear EOF irq mask for next tile */
1719	ctx->next_tile++;
1720	return IRQ_HANDLED;
1721done:
1722	list_add_tail(&run->list, &chan->done_q);
1723	chan->current_run = NULL;
1724	run_next(chan);
1725	return IRQ_WAKE_THREAD;
1726}
1727
1728static irqreturn_t eof_irq(int irq, void *data)
1729{
1730	struct ipu_image_convert_chan *chan = data;
1731	struct ipu_image_convert_priv *priv = chan->priv;
1732	struct ipu_image_convert_ctx *ctx;
1733	struct ipu_image_convert_run *run;
1734	irqreturn_t ret = IRQ_HANDLED;
1735	bool tile_complete = false;
1736	unsigned long flags;
1737
1738	spin_lock_irqsave(&chan->irqlock, flags);
1739
1740	/* get current run and its context */
1741	run = chan->current_run;
1742	if (!run) {
1743		ret = IRQ_NONE;
1744		goto out;
1745	}
1746
1747	ctx = run->ctx;
1748
1749	if (irq == chan->in_eof_irq) {
1750		ctx->eof_mask |= EOF_IRQ_IN;
1751	} else if (irq == chan->out_eof_irq) {
1752		ctx->eof_mask |= EOF_IRQ_OUT;
1753	} else if (irq == chan->rot_in_eof_irq ||
1754		   irq == chan->rot_out_eof_irq) {
1755		if (!ipu_rot_mode_is_irt(ctx->rot_mode)) {
1756			/* this was NOT a rotation op, shouldn't happen */
1757			dev_err(priv->ipu->dev,
1758				"Unexpected rotation interrupt\n");
1759			goto out;
1760		}
1761		ctx->eof_mask |= (irq == chan->rot_in_eof_irq) ?
1762			EOF_IRQ_ROT_IN : EOF_IRQ_ROT_OUT;
1763	} else {
1764		dev_err(priv->ipu->dev, "Received unknown irq %d\n", irq);
1765		ret = IRQ_NONE;
1766		goto out;
1767	}
1768
1769	if (ipu_rot_mode_is_irt(ctx->rot_mode))
1770		tile_complete =	(ctx->eof_mask == EOF_IRQ_ROT_COMPLETE);
1771	else
1772		tile_complete = (ctx->eof_mask == EOF_IRQ_COMPLETE);
1773
1774	if (tile_complete)
1775		ret = do_tile_complete(run);
1776out:
1777	spin_unlock_irqrestore(&chan->irqlock, flags);
1778	return ret;
1779}
1780
1781/*
1782 * try to force the completion of runs for this ctx. Called when
1783 * abort wait times out in ipu_image_convert_abort().
1784 */
1785static void force_abort(struct ipu_image_convert_ctx *ctx)
1786{
1787	struct ipu_image_convert_chan *chan = ctx->chan;
1788	struct ipu_image_convert_run *run;
1789	unsigned long flags;
1790
1791	spin_lock_irqsave(&chan->irqlock, flags);
1792
1793	run = chan->current_run;
1794	if (run && run->ctx == ctx) {
1795		convert_stop(run);
1796		run->status = -EIO;
1797		list_add_tail(&run->list, &chan->done_q);
1798		chan->current_run = NULL;
1799		run_next(chan);
1800	}
1801
1802	spin_unlock_irqrestore(&chan->irqlock, flags);
1803
1804	empty_done_q(chan);
1805}
1806
1807static void release_ipu_resources(struct ipu_image_convert_chan *chan)
1808{
1809	if (chan->in_eof_irq >= 0)
1810		free_irq(chan->in_eof_irq, chan);
1811	if (chan->rot_in_eof_irq >= 0)
1812		free_irq(chan->rot_in_eof_irq, chan);
1813	if (chan->out_eof_irq >= 0)
1814		free_irq(chan->out_eof_irq, chan);
1815	if (chan->rot_out_eof_irq >= 0)
1816		free_irq(chan->rot_out_eof_irq, chan);
1817
1818	if (!IS_ERR_OR_NULL(chan->in_chan))
1819		ipu_idmac_put(chan->in_chan);
1820	if (!IS_ERR_OR_NULL(chan->out_chan))
1821		ipu_idmac_put(chan->out_chan);
1822	if (!IS_ERR_OR_NULL(chan->rotation_in_chan))
1823		ipu_idmac_put(chan->rotation_in_chan);
1824	if (!IS_ERR_OR_NULL(chan->rotation_out_chan))
1825		ipu_idmac_put(chan->rotation_out_chan);
1826	if (!IS_ERR_OR_NULL(chan->ic))
1827		ipu_ic_put(chan->ic);
1828
1829	chan->in_chan = chan->out_chan = chan->rotation_in_chan =
1830		chan->rotation_out_chan = NULL;
1831	chan->in_eof_irq = -1;
1832	chan->rot_in_eof_irq = -1;
1833	chan->out_eof_irq = -1;
1834	chan->rot_out_eof_irq = -1;
1835}
1836
1837static int get_eof_irq(struct ipu_image_convert_chan *chan,
1838		       struct ipuv3_channel *channel)
1839{
1840	struct ipu_image_convert_priv *priv = chan->priv;
1841	int ret, irq;
1842
1843	irq = ipu_idmac_channel_irq(priv->ipu, channel, IPU_IRQ_EOF);
1844
1845	ret = request_threaded_irq(irq, eof_irq, do_bh, 0, "ipu-ic", chan);
1846	if (ret < 0) {
1847		dev_err(priv->ipu->dev, "could not acquire irq %d\n", irq);
1848		return ret;
1849	}
1850
1851	return irq;
1852}
1853
1854static int get_ipu_resources(struct ipu_image_convert_chan *chan)
1855{
1856	const struct ipu_image_convert_dma_chan *dma = chan->dma_ch;
1857	struct ipu_image_convert_priv *priv = chan->priv;
1858	int ret;
1859
1860	/* get IC */
1861	chan->ic = ipu_ic_get(priv->ipu, chan->ic_task);
1862	if (IS_ERR(chan->ic)) {
1863		dev_err(priv->ipu->dev, "could not acquire IC\n");
1864		ret = PTR_ERR(chan->ic);
1865		goto err;
1866	}
1867
1868	/* get IDMAC channels */
1869	chan->in_chan = ipu_idmac_get(priv->ipu, dma->in);
1870	chan->out_chan = ipu_idmac_get(priv->ipu, dma->out);
1871	if (IS_ERR(chan->in_chan) || IS_ERR(chan->out_chan)) {
1872		dev_err(priv->ipu->dev, "could not acquire idmac channels\n");
1873		ret = -EBUSY;
1874		goto err;
1875	}
1876
1877	chan->rotation_in_chan = ipu_idmac_get(priv->ipu, dma->rot_in);
1878	chan->rotation_out_chan = ipu_idmac_get(priv->ipu, dma->rot_out);
1879	if (IS_ERR(chan->rotation_in_chan) || IS_ERR(chan->rotation_out_chan)) {
1880		dev_err(priv->ipu->dev,
1881			"could not acquire idmac rotation channels\n");
1882		ret = -EBUSY;
1883		goto err;
1884	}
1885
1886	/* acquire the EOF interrupts */
1887	ret = get_eof_irq(chan, chan->in_chan);
1888	if (ret < 0) {
1889		chan->in_eof_irq = -1;
1890		goto err;
1891	}
1892	chan->in_eof_irq = ret;
1893
1894	ret = get_eof_irq(chan, chan->rotation_in_chan);
1895	if (ret < 0) {
1896		chan->rot_in_eof_irq = -1;
1897		goto err;
1898	}
1899	chan->rot_in_eof_irq = ret;
1900
1901	ret = get_eof_irq(chan, chan->out_chan);
1902	if (ret < 0) {
1903		chan->out_eof_irq = -1;
1904		goto err;
1905	}
1906	chan->out_eof_irq = ret;
1907
1908	ret = get_eof_irq(chan, chan->rotation_out_chan);
1909	if (ret < 0) {
1910		chan->rot_out_eof_irq = -1;
1911		goto err;
1912	}
1913	chan->rot_out_eof_irq = ret;
1914
1915	return 0;
1916err:
1917	release_ipu_resources(chan);
1918	return ret;
1919}
1920
1921static int fill_image(struct ipu_image_convert_ctx *ctx,
1922		      struct ipu_image_convert_image *ic_image,
1923		      struct ipu_image *image,
1924		      enum ipu_image_convert_type type)
1925{
1926	struct ipu_image_convert_priv *priv = ctx->chan->priv;
1927
1928	ic_image->base = *image;
1929	ic_image->type = type;
1930
1931	ic_image->fmt = get_format(image->pix.pixelformat);
1932	if (!ic_image->fmt) {
1933		dev_err(priv->ipu->dev, "pixelformat not supported for %s\n",
1934			type == IMAGE_CONVERT_OUT ? "Output" : "Input");
1935		return -EINVAL;
1936	}
1937
1938	if (ic_image->fmt->planar)
1939		ic_image->stride = ic_image->base.pix.width;
1940	else
1941		ic_image->stride  = ic_image->base.pix.bytesperline;
1942
1943	return 0;
1944}
1945
1946/* borrowed from drivers/media/v4l2-core/v4l2-common.c */
1947static unsigned int clamp_align(unsigned int x, unsigned int min,
1948				unsigned int max, unsigned int align)
1949{
1950	/* Bits that must be zero to be aligned */
1951	unsigned int mask = ~((1 << align) - 1);
1952
1953	/* Clamp to aligned min and max */
1954	x = clamp(x, (min + ~mask) & mask, max & mask);
1955
1956	/* Round to nearest aligned value */
1957	if (align)
1958		x = (x + (1 << (align - 1))) & mask;
1959
1960	return x;
1961}
1962
1963/* Adjusts input/output images to IPU restrictions */
1964void ipu_image_convert_adjust(struct ipu_image *in, struct ipu_image *out,
1965			      enum ipu_rotate_mode rot_mode)
1966{
1967	const struct ipu_image_pixfmt *infmt, *outfmt;
1968	u32 w_align_out, h_align_out;
1969	u32 w_align_in, h_align_in;
1970
1971	infmt = get_format(in->pix.pixelformat);
1972	outfmt = get_format(out->pix.pixelformat);
1973
1974	/* set some default pixel formats if needed */
1975	if (!infmt) {
1976		in->pix.pixelformat = V4L2_PIX_FMT_RGB24;
1977		infmt = get_format(V4L2_PIX_FMT_RGB24);
1978	}
1979	if (!outfmt) {
1980		out->pix.pixelformat = V4L2_PIX_FMT_RGB24;
1981		outfmt = get_format(V4L2_PIX_FMT_RGB24);
1982	}
1983
1984	/* image converter does not handle fields */
1985	in->pix.field = out->pix.field = V4L2_FIELD_NONE;
1986
1987	/* resizer cannot downsize more than 4:1 */
1988	if (ipu_rot_mode_is_irt(rot_mode)) {
1989		out->pix.height = max_t(__u32, out->pix.height,
1990					in->pix.width / 4);
1991		out->pix.width = max_t(__u32, out->pix.width,
1992				       in->pix.height / 4);
1993	} else {
1994		out->pix.width = max_t(__u32, out->pix.width,
1995				       in->pix.width / 4);
1996		out->pix.height = max_t(__u32, out->pix.height,
1997					in->pix.height / 4);
1998	}
1999
2000	/* align input width/height */
2001	w_align_in = ilog2(tile_width_align(IMAGE_CONVERT_IN, infmt,
2002					    rot_mode));
2003	h_align_in = ilog2(tile_height_align(IMAGE_CONVERT_IN, infmt,
2004					     rot_mode));
2005	in->pix.width = clamp_align(in->pix.width, MIN_W, MAX_W,
2006				    w_align_in);
2007	in->pix.height = clamp_align(in->pix.height, MIN_H, MAX_H,
2008				     h_align_in);
2009
2010	/* align output width/height */
2011	w_align_out = ilog2(tile_width_align(IMAGE_CONVERT_OUT, outfmt,
2012					     rot_mode));
2013	h_align_out = ilog2(tile_height_align(IMAGE_CONVERT_OUT, outfmt,
2014					      rot_mode));
2015	out->pix.width = clamp_align(out->pix.width, MIN_W, MAX_W,
2016				     w_align_out);
2017	out->pix.height = clamp_align(out->pix.height, MIN_H, MAX_H,
2018				      h_align_out);
2019
2020	/* set input/output strides and image sizes */
2021	in->pix.bytesperline = infmt->planar ?
2022		clamp_align(in->pix.width, 2 << w_align_in, MAX_W,
2023			    w_align_in) :
2024		clamp_align((in->pix.width * infmt->bpp) >> 3,
2025			    ((2 << w_align_in) * infmt->bpp) >> 3,
2026			    (MAX_W * infmt->bpp) >> 3,
2027			    w_align_in);
2028	in->pix.sizeimage = infmt->planar ?
2029		(in->pix.height * in->pix.bytesperline * infmt->bpp) >> 3 :
2030		in->pix.height * in->pix.bytesperline;
2031	out->pix.bytesperline = outfmt->planar ? out->pix.width :
2032		(out->pix.width * outfmt->bpp) >> 3;
2033	out->pix.sizeimage = outfmt->planar ?
2034		(out->pix.height * out->pix.bytesperline * outfmt->bpp) >> 3 :
2035		out->pix.height * out->pix.bytesperline;
2036}
2037EXPORT_SYMBOL_GPL(ipu_image_convert_adjust);
2038
2039/*
2040 * this is used by ipu_image_convert_prepare() to verify set input and
2041 * output images are valid before starting the conversion. Clients can
2042 * also call it before calling ipu_image_convert_prepare().
2043 */
2044int ipu_image_convert_verify(struct ipu_image *in, struct ipu_image *out,
2045			     enum ipu_rotate_mode rot_mode)
2046{
2047	struct ipu_image testin, testout;
2048
2049	testin = *in;
2050	testout = *out;
2051
2052	ipu_image_convert_adjust(&testin, &testout, rot_mode);
2053
2054	if (testin.pix.width != in->pix.width ||
2055	    testin.pix.height != in->pix.height ||
2056	    testout.pix.width != out->pix.width ||
2057	    testout.pix.height != out->pix.height)
2058		return -EINVAL;
2059
2060	return 0;
2061}
2062EXPORT_SYMBOL_GPL(ipu_image_convert_verify);
2063
2064/*
2065 * Call ipu_image_convert_prepare() to prepare for the conversion of
2066 * given images and rotation mode. Returns a new conversion context.
2067 */
2068struct ipu_image_convert_ctx *
2069ipu_image_convert_prepare(struct ipu_soc *ipu, enum ipu_ic_task ic_task,
2070			  struct ipu_image *in, struct ipu_image *out,
2071			  enum ipu_rotate_mode rot_mode,
2072			  ipu_image_convert_cb_t complete,
2073			  void *complete_context)
2074{
2075	struct ipu_image_convert_priv *priv = ipu->image_convert_priv;
2076	struct ipu_image_convert_image *s_image, *d_image;
2077	struct ipu_image_convert_chan *chan;
2078	struct ipu_image_convert_ctx *ctx;
2079	unsigned long flags;
2080	unsigned int i;
2081	bool get_res;
2082	int ret;
2083
2084	if (!in || !out || !complete ||
2085	    (ic_task != IC_TASK_VIEWFINDER &&
2086	     ic_task != IC_TASK_POST_PROCESSOR))
2087		return ERR_PTR(-EINVAL);
2088
2089	/* verify the in/out images before continuing */
2090	ret = ipu_image_convert_verify(in, out, rot_mode);
2091	if (ret) {
2092		dev_err(priv->ipu->dev, "%s: in/out formats invalid\n",
2093			__func__);
2094		return ERR_PTR(ret);
2095	}
2096
2097	chan = &priv->chan[ic_task];
2098
2099	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
2100	if (!ctx)
2101		return ERR_PTR(-ENOMEM);
2102
2103	dev_dbg(priv->ipu->dev, "%s: task %u: ctx %p\n", __func__,
2104		chan->ic_task, ctx);
2105
2106	ctx->chan = chan;
2107	init_completion(&ctx->aborted);
2108
2109	ctx->rot_mode = rot_mode;
2110
2111	/* Sets ctx->in.num_rows/cols as well */
2112	ret = calc_image_resize_coefficients(ctx, in, out);
2113	if (ret)
2114		goto out_free;
2115
2116	s_image = &ctx->in;
2117	d_image = &ctx->out;
2118
2119	/* set tiling and rotation */
2120	if (ipu_rot_mode_is_irt(rot_mode)) {
2121		d_image->num_rows = s_image->num_cols;
2122		d_image->num_cols = s_image->num_rows;
2123	} else {
2124		d_image->num_rows = s_image->num_rows;
2125		d_image->num_cols = s_image->num_cols;
2126	}
2127
2128	ctx->num_tiles = d_image->num_cols * d_image->num_rows;
2129
2130	ret = fill_image(ctx, s_image, in, IMAGE_CONVERT_IN);
2131	if (ret)
2132		goto out_free;
2133	ret = fill_image(ctx, d_image, out, IMAGE_CONVERT_OUT);
2134	if (ret)
2135		goto out_free;
2136
2137	calc_out_tile_map(ctx);
2138
2139	find_seams(ctx, s_image, d_image);
2140
2141	ret = calc_tile_dimensions(ctx, s_image);
2142	if (ret)
2143		goto out_free;
2144
2145	ret = calc_tile_offsets(ctx, s_image);
2146	if (ret)
2147		goto out_free;
2148
2149	calc_tile_dimensions(ctx, d_image);
2150	ret = calc_tile_offsets(ctx, d_image);
2151	if (ret)
2152		goto out_free;
2153
2154	calc_tile_resize_coefficients(ctx);
2155
2156	ret = ipu_ic_calc_csc(&ctx->csc,
2157			s_image->base.pix.ycbcr_enc,
2158			s_image->base.pix.quantization,
2159			ipu_pixelformat_to_colorspace(s_image->fmt->fourcc),
2160			d_image->base.pix.ycbcr_enc,
2161			d_image->base.pix.quantization,
2162			ipu_pixelformat_to_colorspace(d_image->fmt->fourcc));
2163	if (ret)
2164		goto out_free;
2165
2166	dump_format(ctx, s_image);
2167	dump_format(ctx, d_image);
2168
2169	ctx->complete = complete;
2170	ctx->complete_context = complete_context;
2171
2172	/*
2173	 * Can we use double-buffering for this operation? If there is
2174	 * only one tile (the whole image can be converted in a single
2175	 * operation) there's no point in using double-buffering. Also,
2176	 * the IPU's IDMAC channels allow only a single U and V plane
2177	 * offset shared between both buffers, but these offsets change
2178	 * for every tile, and therefore would have to be updated for
2179	 * each buffer which is not possible. So double-buffering is
2180	 * impossible when either the source or destination images are
2181	 * a planar format (YUV420, YUV422P, etc.). Further, differently
2182	 * sized tiles or different resizing coefficients per tile
2183	 * prevent double-buffering as well.
2184	 */
2185	ctx->double_buffering = (ctx->num_tiles > 1 &&
2186				 !s_image->fmt->planar &&
2187				 !d_image->fmt->planar);
2188	for (i = 1; i < ctx->num_tiles; i++) {
2189		if (ctx->in.tile[i].width != ctx->in.tile[0].width ||
2190		    ctx->in.tile[i].height != ctx->in.tile[0].height ||
2191		    ctx->out.tile[i].width != ctx->out.tile[0].width ||
2192		    ctx->out.tile[i].height != ctx->out.tile[0].height) {
2193			ctx->double_buffering = false;
2194			break;
2195		}
2196	}
2197	for (i = 1; i < ctx->in.num_cols; i++) {
2198		if (ctx->resize_coeffs_h[i] != ctx->resize_coeffs_h[0]) {
2199			ctx->double_buffering = false;
2200			break;
2201		}
2202	}
2203	for (i = 1; i < ctx->in.num_rows; i++) {
2204		if (ctx->resize_coeffs_v[i] != ctx->resize_coeffs_v[0]) {
2205			ctx->double_buffering = false;
2206			break;
2207		}
2208	}
2209
2210	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
2211		unsigned long intermediate_size = d_image->tile[0].size;
2212
2213		for (i = 1; i < ctx->num_tiles; i++) {
2214			if (d_image->tile[i].size > intermediate_size)
2215				intermediate_size = d_image->tile[i].size;
2216		}
2217
2218		ret = alloc_dma_buf(priv, &ctx->rot_intermediate[0],
2219				    intermediate_size);
2220		if (ret)
2221			goto out_free;
2222		if (ctx->double_buffering) {
2223			ret = alloc_dma_buf(priv,
2224					    &ctx->rot_intermediate[1],
2225					    intermediate_size);
2226			if (ret)
2227				goto out_free_dmabuf0;
2228		}
2229	}
2230
2231	spin_lock_irqsave(&chan->irqlock, flags);
2232
2233	get_res = list_empty(&chan->ctx_list);
2234
2235	list_add_tail(&ctx->list, &chan->ctx_list);
2236
2237	spin_unlock_irqrestore(&chan->irqlock, flags);
2238
2239	if (get_res) {
2240		ret = get_ipu_resources(chan);
2241		if (ret)
2242			goto out_free_dmabuf1;
2243	}
2244
2245	return ctx;
2246
2247out_free_dmabuf1:
2248	free_dma_buf(priv, &ctx->rot_intermediate[1]);
2249	spin_lock_irqsave(&chan->irqlock, flags);
2250	list_del(&ctx->list);
2251	spin_unlock_irqrestore(&chan->irqlock, flags);
2252out_free_dmabuf0:
2253	free_dma_buf(priv, &ctx->rot_intermediate[0]);
2254out_free:
2255	kfree(ctx);
2256	return ERR_PTR(ret);
2257}
2258EXPORT_SYMBOL_GPL(ipu_image_convert_prepare);
2259
2260/*
2261 * Carry out a single image conversion run. Only the physaddr's of the input
2262 * and output image buffers are needed. The conversion context must have
2263 * been created previously with ipu_image_convert_prepare().
2264 */
2265int ipu_image_convert_queue(struct ipu_image_convert_run *run)
2266{
2267	struct ipu_image_convert_chan *chan;
2268	struct ipu_image_convert_priv *priv;
2269	struct ipu_image_convert_ctx *ctx;
2270	unsigned long flags;
2271	int ret = 0;
2272
2273	if (!run || !run->ctx || !run->in_phys || !run->out_phys)
2274		return -EINVAL;
2275
2276	ctx = run->ctx;
2277	chan = ctx->chan;
2278	priv = chan->priv;
2279
2280	dev_dbg(priv->ipu->dev, "%s: task %u: ctx %p run %p\n", __func__,
2281		chan->ic_task, ctx, run);
2282
2283	INIT_LIST_HEAD(&run->list);
2284
2285	spin_lock_irqsave(&chan->irqlock, flags);
2286
2287	if (ctx->aborting) {
2288		ret = -EIO;
2289		goto unlock;
2290	}
2291
2292	list_add_tail(&run->list, &chan->pending_q);
2293
2294	if (!chan->current_run) {
2295		ret = do_run(run);
2296		if (ret)
2297			chan->current_run = NULL;
2298	}
2299unlock:
2300	spin_unlock_irqrestore(&chan->irqlock, flags);
2301	return ret;
2302}
2303EXPORT_SYMBOL_GPL(ipu_image_convert_queue);
2304
2305/* Abort any active or pending conversions for this context */
2306static void __ipu_image_convert_abort(struct ipu_image_convert_ctx *ctx)
2307{
2308	struct ipu_image_convert_chan *chan = ctx->chan;
2309	struct ipu_image_convert_priv *priv = chan->priv;
2310	struct ipu_image_convert_run *run, *active_run, *tmp;
2311	unsigned long flags;
2312	int run_count, ret;
2313
2314	spin_lock_irqsave(&chan->irqlock, flags);
2315
2316	/* move all remaining pending runs in this context to done_q */
2317	list_for_each_entry_safe(run, tmp, &chan->pending_q, list) {
2318		if (run->ctx != ctx)
2319			continue;
2320		run->status = -EIO;
2321		list_move_tail(&run->list, &chan->done_q);
2322	}
2323
2324	run_count = get_run_count(ctx, &chan->done_q);
2325	active_run = (chan->current_run && chan->current_run->ctx == ctx) ?
2326		chan->current_run : NULL;
2327
2328	if (active_run)
2329		reinit_completion(&ctx->aborted);
2330
2331	ctx->aborting = true;
2332
2333	spin_unlock_irqrestore(&chan->irqlock, flags);
2334
2335	if (!run_count && !active_run) {
2336		dev_dbg(priv->ipu->dev,
2337			"%s: task %u: no abort needed for ctx %p\n",
2338			__func__, chan->ic_task, ctx);
2339		return;
2340	}
2341
2342	if (!active_run) {
2343		empty_done_q(chan);
2344		return;
2345	}
2346
2347	dev_dbg(priv->ipu->dev,
2348		"%s: task %u: wait for completion: %d runs\n",
2349		__func__, chan->ic_task, run_count);
2350
2351	ret = wait_for_completion_timeout(&ctx->aborted,
2352					  msecs_to_jiffies(10000));
2353	if (ret == 0) {
2354		dev_warn(priv->ipu->dev, "%s: timeout\n", __func__);
2355		force_abort(ctx);
2356	}
2357}
2358
2359void ipu_image_convert_abort(struct ipu_image_convert_ctx *ctx)
2360{
2361	__ipu_image_convert_abort(ctx);
2362	ctx->aborting = false;
2363}
2364EXPORT_SYMBOL_GPL(ipu_image_convert_abort);
2365
2366/* Unprepare image conversion context */
2367void ipu_image_convert_unprepare(struct ipu_image_convert_ctx *ctx)
2368{
2369	struct ipu_image_convert_chan *chan = ctx->chan;
2370	struct ipu_image_convert_priv *priv = chan->priv;
2371	unsigned long flags;
2372	bool put_res;
2373
2374	/* make sure no runs are hanging around */
2375	__ipu_image_convert_abort(ctx);
2376
2377	dev_dbg(priv->ipu->dev, "%s: task %u: removing ctx %p\n", __func__,
2378		chan->ic_task, ctx);
2379
2380	spin_lock_irqsave(&chan->irqlock, flags);
2381
2382	list_del(&ctx->list);
2383
2384	put_res = list_empty(&chan->ctx_list);
2385
2386	spin_unlock_irqrestore(&chan->irqlock, flags);
2387
2388	if (put_res)
2389		release_ipu_resources(chan);
2390
2391	free_dma_buf(priv, &ctx->rot_intermediate[1]);
2392	free_dma_buf(priv, &ctx->rot_intermediate[0]);
2393
2394	kfree(ctx);
2395}
2396EXPORT_SYMBOL_GPL(ipu_image_convert_unprepare);
2397
2398/*
2399 * "Canned" asynchronous single image conversion. Allocates and returns
2400 * a new conversion run.  On successful return the caller must free the
2401 * run and call ipu_image_convert_unprepare() after conversion completes.
2402 */
2403struct ipu_image_convert_run *
2404ipu_image_convert(struct ipu_soc *ipu, enum ipu_ic_task ic_task,
2405		  struct ipu_image *in, struct ipu_image *out,
2406		  enum ipu_rotate_mode rot_mode,
2407		  ipu_image_convert_cb_t complete,
2408		  void *complete_context)
2409{
2410	struct ipu_image_convert_ctx *ctx;
2411	struct ipu_image_convert_run *run;
2412	int ret;
2413
2414	ctx = ipu_image_convert_prepare(ipu, ic_task, in, out, rot_mode,
2415					complete, complete_context);
2416	if (IS_ERR(ctx))
2417		return ERR_CAST(ctx);
2418
2419	run = kzalloc(sizeof(*run), GFP_KERNEL);
2420	if (!run) {
2421		ipu_image_convert_unprepare(ctx);
2422		return ERR_PTR(-ENOMEM);
2423	}
2424
2425	run->ctx = ctx;
2426	run->in_phys = in->phys0;
2427	run->out_phys = out->phys0;
2428
2429	ret = ipu_image_convert_queue(run);
2430	if (ret) {
2431		ipu_image_convert_unprepare(ctx);
2432		kfree(run);
2433		return ERR_PTR(ret);
2434	}
2435
2436	return run;
2437}
2438EXPORT_SYMBOL_GPL(ipu_image_convert);
2439
2440/* "Canned" synchronous single image conversion */
2441static void image_convert_sync_complete(struct ipu_image_convert_run *run,
2442					void *data)
2443{
2444	struct completion *comp = data;
2445
2446	complete(comp);
2447}
2448
2449int ipu_image_convert_sync(struct ipu_soc *ipu, enum ipu_ic_task ic_task,
2450			   struct ipu_image *in, struct ipu_image *out,
2451			   enum ipu_rotate_mode rot_mode)
2452{
2453	struct ipu_image_convert_run *run;
2454	struct completion comp;
2455	int ret;
2456
2457	init_completion(&comp);
2458
2459	run = ipu_image_convert(ipu, ic_task, in, out, rot_mode,
2460				image_convert_sync_complete, &comp);
2461	if (IS_ERR(run))
2462		return PTR_ERR(run);
2463
2464	ret = wait_for_completion_timeout(&comp, msecs_to_jiffies(10000));
2465	ret = (ret == 0) ? -ETIMEDOUT : 0;
2466
2467	ipu_image_convert_unprepare(run->ctx);
2468	kfree(run);
2469
2470	return ret;
2471}
2472EXPORT_SYMBOL_GPL(ipu_image_convert_sync);
2473
2474int ipu_image_convert_init(struct ipu_soc *ipu, struct device *dev)
2475{
2476	struct ipu_image_convert_priv *priv;
2477	int i;
2478
2479	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
2480	if (!priv)
2481		return -ENOMEM;
2482
2483	ipu->image_convert_priv = priv;
2484	priv->ipu = ipu;
2485
2486	for (i = 0; i < IC_NUM_TASKS; i++) {
2487		struct ipu_image_convert_chan *chan = &priv->chan[i];
2488
2489		chan->ic_task = i;
2490		chan->priv = priv;
2491		chan->dma_ch = &image_convert_dma_chan[i];
2492		chan->in_eof_irq = -1;
2493		chan->rot_in_eof_irq = -1;
2494		chan->out_eof_irq = -1;
2495		chan->rot_out_eof_irq = -1;
2496
2497		spin_lock_init(&chan->irqlock);
2498		INIT_LIST_HEAD(&chan->ctx_list);
2499		INIT_LIST_HEAD(&chan->pending_q);
2500		INIT_LIST_HEAD(&chan->done_q);
2501	}
2502
2503	return 0;
2504}
2505
2506void ipu_image_convert_exit(struct ipu_soc *ipu)
2507{
2508}

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Copyright (C) 2012-2016 Mentor Graphics Inc.
   4 *
   5 * Queued image conversion support, with tiling and rotation.
   6 */
   7
   8#include <linux/interrupt.h>
   9#include <linux/dma-mapping.h>
 
 
  10#include <video/imx-ipu-image-convert.h>
 
  11#include "ipu-prv.h"
  12
  13/*
  14 * The IC Resizer has a restriction that the output frame from the
  15 * resizer must be 1024 or less in both width (pixels) and height
  16 * (lines).
  17 *
  18 * The image converter attempts to split up a conversion when
  19 * the desired output (converted) frame resolution exceeds the
  20 * IC resizer limit of 1024 in either dimension.
  21 *
  22 * If either dimension of the output frame exceeds the limit, the
  23 * dimension is split into 1, 2, or 4 equal stripes, for a maximum
  24 * of 4*4 or 16 tiles. A conversion is then carried out for each
  25 * tile (but taking care to pass the full frame stride length to
  26 * the DMA channel's parameter memory!). IDMA double-buffering is used
  27 * to convert each tile back-to-back when possible (see note below
  28 * when double_buffering boolean is set).
  29 *
  30 * Note that the input frame must be split up into the same number
  31 * of tiles as the output frame:
  32 *
  33 *                       +---------+-----+
  34 *   +-----+---+         |  A      | B   |
  35 *   | A   | B |         |         |     |
  36 *   +-----+---+   -->   +---------+-----+
  37 *   | C   | D |         |  C      | D   |
  38 *   +-----+---+         |         |     |
  39 *                       +---------+-----+
  40 *
  41 * Clockwise 90° rotations are handled by first rescaling into a
  42 * reusable temporary tile buffer and then rotating with the 8x8
  43 * block rotator, writing to the correct destination:
  44 *
  45 *                                         +-----+-----+
  46 *                                         |     |     |
  47 *   +-----+---+         +---------+       | C   | A   |
  48 *   | A   | B |         | A,B, |  |       |     |     |
  49 *   +-----+---+   -->   | C,D  |  |  -->  |     |     |
  50 *   | C   | D |         +---------+       +-----+-----+
  51 *   +-----+---+                           | D   | B   |
  52 *                                         |     |     |
  53 *                                         +-----+-----+
  54 *
  55 * If the 8x8 block rotator is used, horizontal or vertical flipping
  56 * is done during the rotation step, otherwise flipping is done
  57 * during the scaling step.
  58 * With rotation or flipping, tile order changes between input and
  59 * output image. Tiles are numbered row major from top left to bottom
  60 * right for both input and output image.
  61 */
  62
  63#define MAX_STRIPES_W    4
  64#define MAX_STRIPES_H    4
  65#define MAX_TILES (MAX_STRIPES_W * MAX_STRIPES_H)
  66
  67#define MIN_W     16
  68#define MIN_H     8
  69#define MAX_W     4096
  70#define MAX_H     4096
  71
  72enum ipu_image_convert_type {
  73	IMAGE_CONVERT_IN = 0,
  74	IMAGE_CONVERT_OUT,
  75};
  76
  77struct ipu_image_convert_dma_buf {
  78	void          *virt;
  79	dma_addr_t    phys;
  80	unsigned long len;
  81};
  82
  83struct ipu_image_convert_dma_chan {
  84	int in;
  85	int out;
  86	int rot_in;
  87	int rot_out;
  88	int vdi_in_p;
  89	int vdi_in;
  90	int vdi_in_n;
  91};
  92
  93/* dimensions of one tile */
  94struct ipu_image_tile {
  95	u32 width;
  96	u32 height;
  97	u32 left;
  98	u32 top;
  99	/* size and strides are in bytes */
 100	u32 size;
 101	u32 stride;
 102	u32 rot_stride;
 103	/* start Y or packed offset of this tile */
 104	u32 offset;
 105	/* offset from start to tile in U plane, for planar formats */
 106	u32 u_off;
 107	/* offset from start to tile in V plane, for planar formats */
 108	u32 v_off;
 109};
 110
 111struct ipu_image_convert_image {
 112	struct ipu_image base;
 113	enum ipu_image_convert_type type;
 114
 115	const struct ipu_image_pixfmt *fmt;
 116	unsigned int stride;
 117
 118	/* # of rows (horizontal stripes) if dest height is > 1024 */
 119	unsigned int num_rows;
 120	/* # of columns (vertical stripes) if dest width is > 1024 */
 121	unsigned int num_cols;
 122
 123	struct ipu_image_tile tile[MAX_TILES];
 124};
 125
 126struct ipu_image_pixfmt {
 127	u32	fourcc;        /* V4L2 fourcc */
 128	int     bpp;           /* total bpp */
 129	int     uv_width_dec;  /* decimation in width for U/V planes */
 130	int     uv_height_dec; /* decimation in height for U/V planes */
 131	bool    planar;        /* planar format */
 132	bool    uv_swapped;    /* U and V planes are swapped */
 133	bool    uv_packed;     /* partial planar (U and V in same plane) */
 134};
 135
 136struct ipu_image_convert_ctx;
 137struct ipu_image_convert_chan;
 138struct ipu_image_convert_priv;
 139
 140enum eof_irq_mask {
 141	EOF_IRQ_IN      = BIT(0),
 142	EOF_IRQ_ROT_IN  = BIT(1),
 143	EOF_IRQ_OUT     = BIT(2),
 144	EOF_IRQ_ROT_OUT = BIT(3),
 145};
 146
 147#define EOF_IRQ_COMPLETE (EOF_IRQ_IN | EOF_IRQ_OUT)
 148#define EOF_IRQ_ROT_COMPLETE (EOF_IRQ_IN | EOF_IRQ_OUT |	\
 149			      EOF_IRQ_ROT_IN | EOF_IRQ_ROT_OUT)
 150
 151struct ipu_image_convert_ctx {
 152	struct ipu_image_convert_chan *chan;
 153
 154	ipu_image_convert_cb_t complete;
 155	void *complete_context;
 156
 157	/* Source/destination image data and rotation mode */
 158	struct ipu_image_convert_image in;
 159	struct ipu_image_convert_image out;
 160	struct ipu_ic_csc csc;
 161	enum ipu_rotate_mode rot_mode;
 162	u32 downsize_coeff_h;
 163	u32 downsize_coeff_v;
 164	u32 image_resize_coeff_h;
 165	u32 image_resize_coeff_v;
 166	u32 resize_coeffs_h[MAX_STRIPES_W];
 167	u32 resize_coeffs_v[MAX_STRIPES_H];
 168
 169	/* intermediate buffer for rotation */
 170	struct ipu_image_convert_dma_buf rot_intermediate[2];
 171
 172	/* current buffer number for double buffering */
 173	int cur_buf_num;
 174
 175	bool aborting;
 176	struct completion aborted;
 177
 178	/* can we use double-buffering for this conversion operation? */
 179	bool double_buffering;
 180	/* num_rows * num_cols */
 181	unsigned int num_tiles;
 182	/* next tile to process */
 183	unsigned int next_tile;
 184	/* where to place converted tile in dest image */
 185	unsigned int out_tile_map[MAX_TILES];
 186
 187	/* mask of completed EOF irqs at every tile conversion */
 188	enum eof_irq_mask eof_mask;
 189
 190	struct list_head list;
 191};
 192
 193struct ipu_image_convert_chan {
 194	struct ipu_image_convert_priv *priv;
 195
 196	enum ipu_ic_task ic_task;
 197	const struct ipu_image_convert_dma_chan *dma_ch;
 198
 199	struct ipu_ic *ic;
 200	struct ipuv3_channel *in_chan;
 201	struct ipuv3_channel *out_chan;
 202	struct ipuv3_channel *rotation_in_chan;
 203	struct ipuv3_channel *rotation_out_chan;
 204
 205	/* the IPU end-of-frame irqs */
 206	int in_eof_irq;
 207	int rot_in_eof_irq;
 208	int out_eof_irq;
 209	int rot_out_eof_irq;
 210
 211	spinlock_t irqlock;
 212
 213	/* list of convert contexts */
 214	struct list_head ctx_list;
 215	/* queue of conversion runs */
 216	struct list_head pending_q;
 217	/* queue of completed runs */
 218	struct list_head done_q;
 219
 220	/* the current conversion run */
 221	struct ipu_image_convert_run *current_run;
 222};
 223
 224struct ipu_image_convert_priv {
 225	struct ipu_image_convert_chan chan[IC_NUM_TASKS];
 226	struct ipu_soc *ipu;
 227};
 228
 229static const struct ipu_image_convert_dma_chan
 230image_convert_dma_chan[IC_NUM_TASKS] = {
 231	[IC_TASK_VIEWFINDER] = {
 232		.in = IPUV3_CHANNEL_MEM_IC_PRP_VF,
 233		.out = IPUV3_CHANNEL_IC_PRP_VF_MEM,
 234		.rot_in = IPUV3_CHANNEL_MEM_ROT_VF,
 235		.rot_out = IPUV3_CHANNEL_ROT_VF_MEM,
 236		.vdi_in_p = IPUV3_CHANNEL_MEM_VDI_PREV,
 237		.vdi_in = IPUV3_CHANNEL_MEM_VDI_CUR,
 238		.vdi_in_n = IPUV3_CHANNEL_MEM_VDI_NEXT,
 239	},
 240	[IC_TASK_POST_PROCESSOR] = {
 241		.in = IPUV3_CHANNEL_MEM_IC_PP,
 242		.out = IPUV3_CHANNEL_IC_PP_MEM,
 243		.rot_in = IPUV3_CHANNEL_MEM_ROT_PP,
 244		.rot_out = IPUV3_CHANNEL_ROT_PP_MEM,
 245	},
 246};
 247
 248static const struct ipu_image_pixfmt image_convert_formats[] = {
 249	{
 250		.fourcc	= V4L2_PIX_FMT_RGB565,
 251		.bpp    = 16,
 252	}, {
 253		.fourcc	= V4L2_PIX_FMT_RGB24,
 254		.bpp    = 24,
 255	}, {
 256		.fourcc	= V4L2_PIX_FMT_BGR24,
 257		.bpp    = 24,
 258	}, {
 259		.fourcc	= V4L2_PIX_FMT_RGB32,
 260		.bpp    = 32,
 261	}, {
 262		.fourcc	= V4L2_PIX_FMT_BGR32,
 263		.bpp    = 32,
 264	}, {
 265		.fourcc	= V4L2_PIX_FMT_XRGB32,
 266		.bpp    = 32,
 267	}, {
 268		.fourcc	= V4L2_PIX_FMT_XBGR32,
 269		.bpp    = 32,
 270	}, {
 271		.fourcc	= V4L2_PIX_FMT_BGRX32,
 272		.bpp    = 32,
 273	}, {
 274		.fourcc	= V4L2_PIX_FMT_RGBX32,
 275		.bpp    = 32,
 276	}, {
 277		.fourcc	= V4L2_PIX_FMT_YUYV,
 278		.bpp    = 16,
 279		.uv_width_dec = 2,
 280		.uv_height_dec = 1,
 281	}, {
 282		.fourcc	= V4L2_PIX_FMT_UYVY,
 283		.bpp    = 16,
 284		.uv_width_dec = 2,
 285		.uv_height_dec = 1,
 286	}, {
 287		.fourcc	= V4L2_PIX_FMT_YUV420,
 288		.bpp    = 12,
 289		.planar = true,
 290		.uv_width_dec = 2,
 291		.uv_height_dec = 2,
 292	}, {
 293		.fourcc	= V4L2_PIX_FMT_YVU420,
 294		.bpp    = 12,
 295		.planar = true,
 296		.uv_width_dec = 2,
 297		.uv_height_dec = 2,
 298		.uv_swapped = true,
 299	}, {
 300		.fourcc = V4L2_PIX_FMT_NV12,
 301		.bpp    = 12,
 302		.planar = true,
 303		.uv_width_dec = 2,
 304		.uv_height_dec = 2,
 305		.uv_packed = true,
 306	}, {
 307		.fourcc = V4L2_PIX_FMT_YUV422P,
 308		.bpp    = 16,
 309		.planar = true,
 310		.uv_width_dec = 2,
 311		.uv_height_dec = 1,
 312	}, {
 313		.fourcc = V4L2_PIX_FMT_NV16,
 314		.bpp    = 16,
 315		.planar = true,
 316		.uv_width_dec = 2,
 317		.uv_height_dec = 1,
 318		.uv_packed = true,
 319	},
 320};
 321
 322static const struct ipu_image_pixfmt *get_format(u32 fourcc)
 323{
 324	const struct ipu_image_pixfmt *ret = NULL;
 325	unsigned int i;
 326
 327	for (i = 0; i < ARRAY_SIZE(image_convert_formats); i++) {
 328		if (image_convert_formats[i].fourcc == fourcc) {
 329			ret = &image_convert_formats[i];
 330			break;
 331		}
 332	}
 333
 334	return ret;
 335}
 336
 337static void dump_format(struct ipu_image_convert_ctx *ctx,
 338			struct ipu_image_convert_image *ic_image)
 339{
 340	struct ipu_image_convert_chan *chan = ctx->chan;
 341	struct ipu_image_convert_priv *priv = chan->priv;
 342
 343	dev_dbg(priv->ipu->dev,
 344		"task %u: ctx %p: %s format: %dx%d (%dx%d tiles), %c%c%c%c\n",
 345		chan->ic_task, ctx,
 346		ic_image->type == IMAGE_CONVERT_OUT ? "Output" : "Input",
 347		ic_image->base.pix.width, ic_image->base.pix.height,
 348		ic_image->num_cols, ic_image->num_rows,
 349		ic_image->fmt->fourcc & 0xff,
 350		(ic_image->fmt->fourcc >> 8) & 0xff,
 351		(ic_image->fmt->fourcc >> 16) & 0xff,
 352		(ic_image->fmt->fourcc >> 24) & 0xff);
 353}
 354
 355int ipu_image_convert_enum_format(int index, u32 *fourcc)
 356{
 357	const struct ipu_image_pixfmt *fmt;
 358
 359	if (index >= (int)ARRAY_SIZE(image_convert_formats))
 360		return -EINVAL;
 361
 362	/* Format found */
 363	fmt = &image_convert_formats[index];
 364	*fourcc = fmt->fourcc;
 365	return 0;
 366}
 367EXPORT_SYMBOL_GPL(ipu_image_convert_enum_format);
 368
 369static void free_dma_buf(struct ipu_image_convert_priv *priv,
 370			 struct ipu_image_convert_dma_buf *buf)
 371{
 372	if (buf->virt)
 373		dma_free_coherent(priv->ipu->dev,
 374				  buf->len, buf->virt, buf->phys);
 375	buf->virt = NULL;
 376	buf->phys = 0;
 377}
 378
 379static int alloc_dma_buf(struct ipu_image_convert_priv *priv,
 380			 struct ipu_image_convert_dma_buf *buf,
 381			 int size)
 382{
 383	buf->len = PAGE_ALIGN(size);
 384	buf->virt = dma_alloc_coherent(priv->ipu->dev, buf->len, &buf->phys,
 385				       GFP_DMA | GFP_KERNEL);
 386	if (!buf->virt) {
 387		dev_err(priv->ipu->dev, "failed to alloc dma buffer\n");
 388		return -ENOMEM;
 389	}
 390
 391	return 0;
 392}
 393
 394static inline int num_stripes(int dim)
 395{
 396	return (dim - 1) / 1024 + 1;
 397}
 398
 399/*
 400 * Calculate downsizing coefficients, which are the same for all tiles,
 401 * and initial bilinear resizing coefficients, which are used to find the
 402 * best seam positions.
 403 * Also determine the number of tiles necessary to guarantee that no tile
 404 * is larger than 1024 pixels in either dimension at the output and between
 405 * IC downsizing and main processing sections.
 406 */
 407static int calc_image_resize_coefficients(struct ipu_image_convert_ctx *ctx,
 408					  struct ipu_image *in,
 409					  struct ipu_image *out)
 410{
 411	u32 downsized_width = in->rect.width;
 412	u32 downsized_height = in->rect.height;
 413	u32 downsize_coeff_v = 0;
 414	u32 downsize_coeff_h = 0;
 415	u32 resized_width = out->rect.width;
 416	u32 resized_height = out->rect.height;
 417	u32 resize_coeff_h;
 418	u32 resize_coeff_v;
 419	u32 cols;
 420	u32 rows;
 421
 422	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
 423		resized_width = out->rect.height;
 424		resized_height = out->rect.width;
 425	}
 426
 427	/* Do not let invalid input lead to an endless loop below */
 428	if (WARN_ON(resized_width == 0 || resized_height == 0))
 429		return -EINVAL;
 430
 431	while (downsized_width >= resized_width * 2) {
 432		downsized_width >>= 1;
 433		downsize_coeff_h++;
 434	}
 435
 436	while (downsized_height >= resized_height * 2) {
 437		downsized_height >>= 1;
 438		downsize_coeff_v++;
 439	}
 440
 441	/*
 442	 * Calculate the bilinear resizing coefficients that could be used if
 443	 * we were converting with a single tile. The bottom right output pixel
 444	 * should sample as close as possible to the bottom right input pixel
 445	 * out of the decimator, but not overshoot it:
 446	 */
 447	resize_coeff_h = 8192 * (downsized_width - 1) / (resized_width - 1);
 448	resize_coeff_v = 8192 * (downsized_height - 1) / (resized_height - 1);
 449
 450	/*
 451	 * Both the output of the IC downsizing section before being passed to
 452	 * the IC main processing section and the final output of the IC main
 453	 * processing section must be <= 1024 pixels in both dimensions.
 454	 */
 455	cols = num_stripes(max_t(u32, downsized_width, resized_width));
 456	rows = num_stripes(max_t(u32, downsized_height, resized_height));
 457
 458	dev_dbg(ctx->chan->priv->ipu->dev,
 459		"%s: hscale: >>%u, *8192/%u vscale: >>%u, *8192/%u, %ux%u tiles\n",
 460		__func__, downsize_coeff_h, resize_coeff_h, downsize_coeff_v,
 461		resize_coeff_v, cols, rows);
 462
 463	if (downsize_coeff_h > 2 || downsize_coeff_v  > 2 ||
 464	    resize_coeff_h > 0x3fff || resize_coeff_v > 0x3fff)
 465		return -EINVAL;
 466
 467	ctx->downsize_coeff_h = downsize_coeff_h;
 468	ctx->downsize_coeff_v = downsize_coeff_v;
 469	ctx->image_resize_coeff_h = resize_coeff_h;
 470	ctx->image_resize_coeff_v = resize_coeff_v;
 471	ctx->in.num_cols = cols;
 472	ctx->in.num_rows = rows;
 473
 474	return 0;
 475}
 476
 477#define round_closest(x, y) round_down((x) + (y)/2, (y))
 478
 479/*
 480 * Find the best aligned seam position for the given column / row index.
 481 * Rotation and image offsets are out of scope.
 482 *
 483 * @index: column / row index, used to calculate valid interval
 484 * @in_edge: input right / bottom edge
 485 * @out_edge: output right / bottom edge
 486 * @in_align: input alignment, either horizontal 8-byte line start address
 487 *            alignment, or pixel alignment due to image format
 488 * @out_align: output alignment, either horizontal 8-byte line start address
 489 *             alignment, or pixel alignment due to image format or rotator
 490 *             block size
 491 * @in_burst: horizontal input burst size in case of horizontal flip
 492 * @out_burst: horizontal output burst size or rotator block size
 493 * @downsize_coeff: downsizing section coefficient
 494 * @resize_coeff: main processing section resizing coefficient
 495 * @_in_seam: aligned input seam position return value
 496 * @_out_seam: aligned output seam position return value
 497 */
 498static void find_best_seam(struct ipu_image_convert_ctx *ctx,
 499			   unsigned int index,
 500			   unsigned int in_edge,
 501			   unsigned int out_edge,
 502			   unsigned int in_align,
 503			   unsigned int out_align,
 504			   unsigned int in_burst,
 505			   unsigned int out_burst,
 506			   unsigned int downsize_coeff,
 507			   unsigned int resize_coeff,
 508			   u32 *_in_seam,
 509			   u32 *_out_seam)
 510{
 511	struct device *dev = ctx->chan->priv->ipu->dev;
 512	unsigned int out_pos;
 513	/* Input / output seam position candidates */
 514	unsigned int out_seam = 0;
 515	unsigned int in_seam = 0;
 516	unsigned int min_diff = UINT_MAX;
 517	unsigned int out_start;
 518	unsigned int out_end;
 519	unsigned int in_start;
 520	unsigned int in_end;
 521
 522	/* Start within 1024 pixels of the right / bottom edge */
 523	out_start = max_t(int, index * out_align, out_edge - 1024);
 524	/* End before having to add more columns to the left / rows above */
 525	out_end = min_t(unsigned int, out_edge, index * 1024 + 1);
 526
 527	/*
 528	 * Limit input seam position to make sure that the downsized input tile
 529	 * to the right or bottom does not exceed 1024 pixels.
 530	 */
 531	in_start = max_t(int, index * in_align,
 532			 in_edge - (1024 << downsize_coeff));
 533	in_end = min_t(unsigned int, in_edge,
 534		       index * (1024 << downsize_coeff) + 1);
 535
 536	/*
 537	 * Output tiles must start at a multiple of 8 bytes horizontally and
 538	 * possibly at an even line horizontally depending on the pixel format.
 539	 * Only consider output aligned positions for the seam.
 540	 */
 541	out_start = round_up(out_start, out_align);
 542	for (out_pos = out_start; out_pos < out_end; out_pos += out_align) {
 543		unsigned int in_pos;
 544		unsigned int in_pos_aligned;
 545		unsigned int in_pos_rounded;
 546		unsigned int abs_diff;
 547
 548		/*
 549		 * Tiles in the right row / bottom column may not be allowed to
 550		 * overshoot horizontally / vertically. out_burst may be the
 551		 * actual DMA burst size, or the rotator block size.
 552		 */
 553		if ((out_burst > 1) && (out_edge - out_pos) % out_burst)
 554			continue;
 555
 556		/*
 557		 * Input sample position, corresponding to out_pos, 19.13 fixed
 558		 * point.
 559		 */
 560		in_pos = (out_pos * resize_coeff) << downsize_coeff;
 561		/*
 562		 * The closest input sample position that we could actually
 563		 * start the input tile at, 19.13 fixed point.
 564		 */
 565		in_pos_aligned = round_closest(in_pos, 8192U * in_align);
 566		/* Convert 19.13 fixed point to integer */
 567		in_pos_rounded = in_pos_aligned / 8192U;
 568
 569		if (in_pos_rounded < in_start)
 570			continue;
 571		if (in_pos_rounded >= in_end)
 572			break;
 573
 574		if ((in_burst > 1) &&
 575		    (in_edge - in_pos_rounded) % in_burst)
 576			continue;
 577
 578		if (in_pos < in_pos_aligned)
 579			abs_diff = in_pos_aligned - in_pos;
 580		else
 581			abs_diff = in_pos - in_pos_aligned;
 582
 583		if (abs_diff < min_diff) {
 584			in_seam = in_pos_rounded;
 585			out_seam = out_pos;
 586			min_diff = abs_diff;
 587		}
 588	}
 589
 590	*_out_seam = out_seam;
 591	*_in_seam = in_seam;
 592
 593	dev_dbg(dev, "%s: out_seam %u(%u) in [%u, %u], in_seam %u(%u) in [%u, %u] diff %u.%03u\n",
 594		__func__, out_seam, out_align, out_start, out_end,
 595		in_seam, in_align, in_start, in_end, min_diff / 8192,
 596		DIV_ROUND_CLOSEST(min_diff % 8192 * 1000, 8192));
 597}
 598
 599/*
 600 * Tile left edges are required to be aligned to multiples of 8 bytes
 601 * by the IDMAC.
 602 */
 603static inline u32 tile_left_align(const struct ipu_image_pixfmt *fmt)
 604{
 605	if (fmt->planar)
 606		return fmt->uv_packed ? 8 : 8 * fmt->uv_width_dec;
 607	else
 608		return fmt->bpp == 32 ? 2 : fmt->bpp == 16 ? 4 : 8;
 609}
 610
 611/*
 612 * Tile top edge alignment is only limited by chroma subsampling.
 613 */
 614static inline u32 tile_top_align(const struct ipu_image_pixfmt *fmt)
 615{
 616	return fmt->uv_height_dec > 1 ? 2 : 1;
 617}
 618
 619static inline u32 tile_width_align(enum ipu_image_convert_type type,
 620				   const struct ipu_image_pixfmt *fmt,
 621				   enum ipu_rotate_mode rot_mode)
 622{
 623	if (type == IMAGE_CONVERT_IN) {
 624		/*
 625		 * The IC burst reads 8 pixels at a time. Reading beyond the
 626		 * end of the line is usually acceptable. Those pixels are
 627		 * ignored, unless the IC has to write the scaled line in
 628		 * reverse.
 629		 */
 630		return (!ipu_rot_mode_is_irt(rot_mode) &&
 631			(rot_mode & IPU_ROT_BIT_HFLIP)) ? 8 : 2;
 632	}
 633
 634	/*
 635	 * Align to 16x16 pixel blocks for planar 4:2:0 chroma subsampled
 636	 * formats to guarantee 8-byte aligned line start addresses in the
 637	 * chroma planes when IRT is used. Align to 8x8 pixel IRT block size
 638	 * for all other formats.
 639	 */
 640	return (ipu_rot_mode_is_irt(rot_mode) &&
 641		fmt->planar && !fmt->uv_packed) ?
 642		8 * fmt->uv_width_dec : 8;
 643}
 644
 645static inline u32 tile_height_align(enum ipu_image_convert_type type,
 646				    const struct ipu_image_pixfmt *fmt,
 647				    enum ipu_rotate_mode rot_mode)
 648{
 649	if (type == IMAGE_CONVERT_IN || !ipu_rot_mode_is_irt(rot_mode))
 650		return 2;
 651
 652	/*
 653	 * Align to 16x16 pixel blocks for planar 4:2:0 chroma subsampled
 654	 * formats to guarantee 8-byte aligned line start addresses in the
 655	 * chroma planes when IRT is used. Align to 8x8 pixel IRT block size
 656	 * for all other formats.
 657	 */
 658	return (fmt->planar && !fmt->uv_packed) ? 8 * fmt->uv_width_dec : 8;
 659}
 660
 661/*
 662 * Fill in left position and width and for all tiles in an input column, and
 663 * for all corresponding output tiles. If the 90° rotator is used, the output
 664 * tiles are in a row, and output tile top position and height are set.
 665 */
 666static void fill_tile_column(struct ipu_image_convert_ctx *ctx,
 667			     unsigned int col,
 668			     struct ipu_image_convert_image *in,
 669			     unsigned int in_left, unsigned int in_width,
 670			     struct ipu_image_convert_image *out,
 671			     unsigned int out_left, unsigned int out_width)
 672{
 673	unsigned int row, tile_idx;
 674	struct ipu_image_tile *in_tile, *out_tile;
 675
 676	for (row = 0; row < in->num_rows; row++) {
 677		tile_idx = in->num_cols * row + col;
 678		in_tile = &in->tile[tile_idx];
 679		out_tile = &out->tile[ctx->out_tile_map[tile_idx]];
 680
 681		in_tile->left = in_left;
 682		in_tile->width = in_width;
 683
 684		if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
 685			out_tile->top = out_left;
 686			out_tile->height = out_width;
 687		} else {
 688			out_tile->left = out_left;
 689			out_tile->width = out_width;
 690		}
 691	}
 692}
 693
 694/*
 695 * Fill in top position and height and for all tiles in an input row, and
 696 * for all corresponding output tiles. If the 90° rotator is used, the output
 697 * tiles are in a column, and output tile left position and width are set.
 698 */
 699static void fill_tile_row(struct ipu_image_convert_ctx *ctx, unsigned int row,
 700			  struct ipu_image_convert_image *in,
 701			  unsigned int in_top, unsigned int in_height,
 702			  struct ipu_image_convert_image *out,
 703			  unsigned int out_top, unsigned int out_height)
 704{
 705	unsigned int col, tile_idx;
 706	struct ipu_image_tile *in_tile, *out_tile;
 707
 708	for (col = 0; col < in->num_cols; col++) {
 709		tile_idx = in->num_cols * row + col;
 710		in_tile = &in->tile[tile_idx];
 711		out_tile = &out->tile[ctx->out_tile_map[tile_idx]];
 712
 713		in_tile->top = in_top;
 714		in_tile->height = in_height;
 715
 716		if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
 717			out_tile->left = out_top;
 718			out_tile->width = out_height;
 719		} else {
 720			out_tile->top = out_top;
 721			out_tile->height = out_height;
 722		}
 723	}
 724}
 725
 726/*
 727 * Find the best horizontal and vertical seam positions to split into tiles.
 728 * Minimize the fractional part of the input sampling position for the
 729 * top / left pixels of each tile.
 730 */
 731static void find_seams(struct ipu_image_convert_ctx *ctx,
 732		       struct ipu_image_convert_image *in,
 733		       struct ipu_image_convert_image *out)
 734{
 735	struct device *dev = ctx->chan->priv->ipu->dev;
 736	unsigned int resized_width = out->base.rect.width;
 737	unsigned int resized_height = out->base.rect.height;
 738	unsigned int col;
 739	unsigned int row;
 740	unsigned int in_left_align = tile_left_align(in->fmt);
 741	unsigned int in_top_align = tile_top_align(in->fmt);
 742	unsigned int out_left_align = tile_left_align(out->fmt);
 743	unsigned int out_top_align = tile_top_align(out->fmt);
 744	unsigned int out_width_align = tile_width_align(out->type, out->fmt,
 745							ctx->rot_mode);
 746	unsigned int out_height_align = tile_height_align(out->type, out->fmt,
 747							  ctx->rot_mode);
 748	unsigned int in_right = in->base.rect.width;
 749	unsigned int in_bottom = in->base.rect.height;
 750	unsigned int out_right = out->base.rect.width;
 751	unsigned int out_bottom = out->base.rect.height;
 752	unsigned int flipped_out_left;
 753	unsigned int flipped_out_top;
 754
 755	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
 756		/* Switch width/height and align top left to IRT block size */
 757		resized_width = out->base.rect.height;
 758		resized_height = out->base.rect.width;
 759		out_left_align = out_height_align;
 760		out_top_align = out_width_align;
 761		out_width_align = out_left_align;
 762		out_height_align = out_top_align;
 763		out_right = out->base.rect.height;
 764		out_bottom = out->base.rect.width;
 765	}
 766
 767	for (col = in->num_cols - 1; col > 0; col--) {
 768		bool allow_in_overshoot = ipu_rot_mode_is_irt(ctx->rot_mode) ||
 769					  !(ctx->rot_mode & IPU_ROT_BIT_HFLIP);
 770		bool allow_out_overshoot = (col < in->num_cols - 1) &&
 771					   !(ctx->rot_mode & IPU_ROT_BIT_HFLIP);
 772		unsigned int in_left;
 773		unsigned int out_left;
 774
 775		/*
 776		 * Align input width to burst length if the scaling step flips
 777		 * horizontally.
 778		 */
 779
 780		find_best_seam(ctx, col,
 781			       in_right, out_right,
 782			       in_left_align, out_left_align,
 783			       allow_in_overshoot ? 1 : 8 /* burst length */,
 784			       allow_out_overshoot ? 1 : out_width_align,
 785			       ctx->downsize_coeff_h, ctx->image_resize_coeff_h,
 786			       &in_left, &out_left);
 787
 788		if (ctx->rot_mode & IPU_ROT_BIT_HFLIP)
 789			flipped_out_left = resized_width - out_right;
 790		else
 791			flipped_out_left = out_left;
 792
 793		fill_tile_column(ctx, col, in, in_left, in_right - in_left,
 794				 out, flipped_out_left, out_right - out_left);
 795
 796		dev_dbg(dev, "%s: col %u: %u, %u -> %u, %u\n", __func__, col,
 797			in_left, in_right - in_left,
 798			flipped_out_left, out_right - out_left);
 799
 800		in_right = in_left;
 801		out_right = out_left;
 802	}
 803
 804	flipped_out_left = (ctx->rot_mode & IPU_ROT_BIT_HFLIP) ?
 805			   resized_width - out_right : 0;
 806
 807	fill_tile_column(ctx, 0, in, 0, in_right,
 808			 out, flipped_out_left, out_right);
 809
 810	dev_dbg(dev, "%s: col 0: 0, %u -> %u, %u\n", __func__,
 811		in_right, flipped_out_left, out_right);
 812
 813	for (row = in->num_rows - 1; row > 0; row--) {
 814		bool allow_overshoot = row < in->num_rows - 1;
 815		unsigned int in_top;
 816		unsigned int out_top;
 817
 818		find_best_seam(ctx, row,
 819			       in_bottom, out_bottom,
 820			       in_top_align, out_top_align,
 821			       1, allow_overshoot ? 1 : out_height_align,
 822			       ctx->downsize_coeff_v, ctx->image_resize_coeff_v,
 823			       &in_top, &out_top);
 824
 825		if ((ctx->rot_mode & IPU_ROT_BIT_VFLIP) ^
 826		    ipu_rot_mode_is_irt(ctx->rot_mode))
 827			flipped_out_top = resized_height - out_bottom;
 828		else
 829			flipped_out_top = out_top;
 830
 831		fill_tile_row(ctx, row, in, in_top, in_bottom - in_top,
 832			      out, flipped_out_top, out_bottom - out_top);
 833
 834		dev_dbg(dev, "%s: row %u: %u, %u -> %u, %u\n", __func__, row,
 835			in_top, in_bottom - in_top,
 836			flipped_out_top, out_bottom - out_top);
 837
 838		in_bottom = in_top;
 839		out_bottom = out_top;
 840	}
 841
 842	if ((ctx->rot_mode & IPU_ROT_BIT_VFLIP) ^
 843	    ipu_rot_mode_is_irt(ctx->rot_mode))
 844		flipped_out_top = resized_height - out_bottom;
 845	else
 846		flipped_out_top = 0;
 847
 848	fill_tile_row(ctx, 0, in, 0, in_bottom,
 849		      out, flipped_out_top, out_bottom);
 850
 851	dev_dbg(dev, "%s: row 0: 0, %u -> %u, %u\n", __func__,
 852		in_bottom, flipped_out_top, out_bottom);
 853}
 854
 855static int calc_tile_dimensions(struct ipu_image_convert_ctx *ctx,
 856				struct ipu_image_convert_image *image)
 857{
 858	struct ipu_image_convert_chan *chan = ctx->chan;
 859	struct ipu_image_convert_priv *priv = chan->priv;
 860	unsigned int max_width = 1024;
 861	unsigned int max_height = 1024;
 862	unsigned int i;
 863
 864	if (image->type == IMAGE_CONVERT_IN) {
 865		/* Up to 4096x4096 input tile size */
 866		max_width <<= ctx->downsize_coeff_h;
 867		max_height <<= ctx->downsize_coeff_v;
 868	}
 869
 870	for (i = 0; i < ctx->num_tiles; i++) {
 871		struct ipu_image_tile *tile;
 872		const unsigned int row = i / image->num_cols;
 873		const unsigned int col = i % image->num_cols;
 874
 875		if (image->type == IMAGE_CONVERT_OUT)
 876			tile = &image->tile[ctx->out_tile_map[i]];
 877		else
 878			tile = &image->tile[i];
 879
 880		tile->size = ((tile->height * image->fmt->bpp) >> 3) *
 881			tile->width;
 882
 883		if (image->fmt->planar) {
 884			tile->stride = tile->width;
 885			tile->rot_stride = tile->height;
 886		} else {
 887			tile->stride =
 888				(image->fmt->bpp * tile->width) >> 3;
 889			tile->rot_stride =
 890				(image->fmt->bpp * tile->height) >> 3;
 891		}
 892
 893		dev_dbg(priv->ipu->dev,
 894			"task %u: ctx %p: %s@[%u,%u]: %ux%u@%u,%u\n",
 895			chan->ic_task, ctx,
 896			image->type == IMAGE_CONVERT_IN ? "Input" : "Output",
 897			row, col,
 898			tile->width, tile->height, tile->left, tile->top);
 899
 900		if (!tile->width || tile->width > max_width ||
 901		    !tile->height || tile->height > max_height) {
 902			dev_err(priv->ipu->dev, "invalid %s tile size: %ux%u\n",
 903				image->type == IMAGE_CONVERT_IN ? "input" :
 904				"output", tile->width, tile->height);
 905			return -EINVAL;
 906		}
 907	}
 908
 909	return 0;
 910}
 911
 912/*
 913 * Use the rotation transformation to find the tile coordinates
 914 * (row, col) of a tile in the destination frame that corresponds
 915 * to the given tile coordinates of a source frame. The destination
 916 * coordinate is then converted to a tile index.
 917 */
 918static int transform_tile_index(struct ipu_image_convert_ctx *ctx,
 919				int src_row, int src_col)
 920{
 921	struct ipu_image_convert_chan *chan = ctx->chan;
 922	struct ipu_image_convert_priv *priv = chan->priv;
 923	struct ipu_image_convert_image *s_image = &ctx->in;
 924	struct ipu_image_convert_image *d_image = &ctx->out;
 925	int dst_row, dst_col;
 926
 927	/* with no rotation it's a 1:1 mapping */
 928	if (ctx->rot_mode == IPU_ROTATE_NONE)
 929		return src_row * s_image->num_cols + src_col;
 930
 931	/*
 932	 * before doing the transform, first we have to translate
 933	 * source row,col for an origin in the center of s_image
 934	 */
 935	src_row = src_row * 2 - (s_image->num_rows - 1);
 936	src_col = src_col * 2 - (s_image->num_cols - 1);
 937
 938	/* do the rotation transform */
 939	if (ctx->rot_mode & IPU_ROT_BIT_90) {
 940		dst_col = -src_row;
 941		dst_row = src_col;
 942	} else {
 943		dst_col = src_col;
 944		dst_row = src_row;
 945	}
 946
 947	/* apply flip */
 948	if (ctx->rot_mode & IPU_ROT_BIT_HFLIP)
 949		dst_col = -dst_col;
 950	if (ctx->rot_mode & IPU_ROT_BIT_VFLIP)
 951		dst_row = -dst_row;
 952
 953	dev_dbg(priv->ipu->dev, "task %u: ctx %p: [%d,%d] --> [%d,%d]\n",
 954		chan->ic_task, ctx, src_col, src_row, dst_col, dst_row);
 955
 956	/*
 957	 * finally translate dest row,col using an origin in upper
 958	 * left of d_image
 959	 */
 960	dst_row += d_image->num_rows - 1;
 961	dst_col += d_image->num_cols - 1;
 962	dst_row /= 2;
 963	dst_col /= 2;
 964
 965	return dst_row * d_image->num_cols + dst_col;
 966}
 967
 968/*
 969 * Fill the out_tile_map[] with transformed destination tile indeces.
 970 */
 971static void calc_out_tile_map(struct ipu_image_convert_ctx *ctx)
 972{
 973	struct ipu_image_convert_image *s_image = &ctx->in;
 974	unsigned int row, col, tile = 0;
 975
 976	for (row = 0; row < s_image->num_rows; row++) {
 977		for (col = 0; col < s_image->num_cols; col++) {
 978			ctx->out_tile_map[tile] =
 979				transform_tile_index(ctx, row, col);
 980			tile++;
 981		}
 982	}
 983}
 984
 985static int calc_tile_offsets_planar(struct ipu_image_convert_ctx *ctx,
 986				    struct ipu_image_convert_image *image)
 987{
 988	struct ipu_image_convert_chan *chan = ctx->chan;
 989	struct ipu_image_convert_priv *priv = chan->priv;
 990	const struct ipu_image_pixfmt *fmt = image->fmt;
 991	unsigned int row, col, tile = 0;
 992	u32 H, top, y_stride, uv_stride;
 993	u32 uv_row_off, uv_col_off, uv_off, u_off, v_off;
 994	u32 y_row_off, y_col_off, y_off;
 995	u32 y_size, uv_size;
 996
 997	/* setup some convenience vars */
 998	H = image->base.pix.height;
 999
1000	y_stride = image->stride;
1001	uv_stride = y_stride / fmt->uv_width_dec;
1002	if (fmt->uv_packed)
1003		uv_stride *= 2;
1004
1005	y_size = H * y_stride;
1006	uv_size = y_size / (fmt->uv_width_dec * fmt->uv_height_dec);
1007
1008	for (row = 0; row < image->num_rows; row++) {
1009		top = image->tile[tile].top;
1010		y_row_off = top * y_stride;
1011		uv_row_off = (top * uv_stride) / fmt->uv_height_dec;
1012
1013		for (col = 0; col < image->num_cols; col++) {
1014			y_col_off = image->tile[tile].left;
1015			uv_col_off = y_col_off / fmt->uv_width_dec;
1016			if (fmt->uv_packed)
1017				uv_col_off *= 2;
1018
1019			y_off = y_row_off + y_col_off;
1020			uv_off = uv_row_off + uv_col_off;
1021
1022			u_off = y_size - y_off + uv_off;
1023			v_off = (fmt->uv_packed) ? 0 : u_off + uv_size;
1024			if (fmt->uv_swapped)
1025				swap(u_off, v_off);
1026
1027			image->tile[tile].offset = y_off;
1028			image->tile[tile].u_off = u_off;
1029			image->tile[tile++].v_off = v_off;
1030
1031			if ((y_off & 0x7) || (u_off & 0x7) || (v_off & 0x7)) {
1032				dev_err(priv->ipu->dev,
1033					"task %u: ctx %p: %s@[%d,%d]: "
1034					"y_off %08x, u_off %08x, v_off %08x\n",
1035					chan->ic_task, ctx,
1036					image->type == IMAGE_CONVERT_IN ?
1037					"Input" : "Output", row, col,
1038					y_off, u_off, v_off);
1039				return -EINVAL;
1040			}
1041		}
1042	}
1043
1044	return 0;
1045}
1046
1047static int calc_tile_offsets_packed(struct ipu_image_convert_ctx *ctx,
1048				    struct ipu_image_convert_image *image)
1049{
1050	struct ipu_image_convert_chan *chan = ctx->chan;
1051	struct ipu_image_convert_priv *priv = chan->priv;
1052	const struct ipu_image_pixfmt *fmt = image->fmt;
1053	unsigned int row, col, tile = 0;
1054	u32 bpp, stride, offset;
1055	u32 row_off, col_off;
1056
1057	/* setup some convenience vars */
1058	stride = image->stride;
1059	bpp = fmt->bpp;
1060
1061	for (row = 0; row < image->num_rows; row++) {
1062		row_off = image->tile[tile].top * stride;
1063
1064		for (col = 0; col < image->num_cols; col++) {
1065			col_off = (image->tile[tile].left * bpp) >> 3;
1066
1067			offset = row_off + col_off;
1068
1069			image->tile[tile].offset = offset;
1070			image->tile[tile].u_off = 0;
1071			image->tile[tile++].v_off = 0;
1072
1073			if (offset & 0x7) {
1074				dev_err(priv->ipu->dev,
1075					"task %u: ctx %p: %s@[%d,%d]: "
1076					"phys %08x\n",
1077					chan->ic_task, ctx,
1078					image->type == IMAGE_CONVERT_IN ?
1079					"Input" : "Output", row, col,
1080					row_off + col_off);
1081				return -EINVAL;
1082			}
1083		}
1084	}
1085
1086	return 0;
1087}
1088
1089static int calc_tile_offsets(struct ipu_image_convert_ctx *ctx,
1090			      struct ipu_image_convert_image *image)
1091{
1092	if (image->fmt->planar)
1093		return calc_tile_offsets_planar(ctx, image);
1094
1095	return calc_tile_offsets_packed(ctx, image);
1096}
1097
1098/*
1099 * Calculate the resizing ratio for the IC main processing section given input
1100 * size, fixed downsizing coefficient, and output size.
1101 * Either round to closest for the next tile's first pixel to minimize seams
1102 * and distortion (for all but right column / bottom row), or round down to
1103 * avoid sampling beyond the edges of the input image for this tile's last
1104 * pixel.
1105 * Returns the resizing coefficient, resizing ratio is 8192.0 / resize_coeff.
1106 */
1107static u32 calc_resize_coeff(u32 input_size, u32 downsize_coeff,
1108			     u32 output_size, bool allow_overshoot)
1109{
1110	u32 downsized = input_size >> downsize_coeff;
1111
1112	if (allow_overshoot)
1113		return DIV_ROUND_CLOSEST(8192 * downsized, output_size);
1114	else
1115		return 8192 * (downsized - 1) / (output_size - 1);
1116}
1117
1118/*
1119 * Slightly modify resize coefficients per tile to hide the bilinear
1120 * interpolator reset at tile borders, shifting the right / bottom edge
1121 * by up to a half input pixel. This removes noticeable seams between
1122 * tiles at higher upscaling factors.
1123 */
1124static void calc_tile_resize_coefficients(struct ipu_image_convert_ctx *ctx)
1125{
1126	struct ipu_image_convert_chan *chan = ctx->chan;
1127	struct ipu_image_convert_priv *priv = chan->priv;
1128	struct ipu_image_tile *in_tile, *out_tile;
1129	unsigned int col, row, tile_idx;
1130	unsigned int last_output;
1131
1132	for (col = 0; col < ctx->in.num_cols; col++) {
1133		bool closest = (col < ctx->in.num_cols - 1) &&
1134			       !(ctx->rot_mode & IPU_ROT_BIT_HFLIP);
1135		u32 resized_width;
1136		u32 resize_coeff_h;
1137		u32 in_width;
1138
1139		tile_idx = col;
1140		in_tile = &ctx->in.tile[tile_idx];
1141		out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]];
1142
1143		if (ipu_rot_mode_is_irt(ctx->rot_mode))
1144			resized_width = out_tile->height;
1145		else
1146			resized_width = out_tile->width;
1147
1148		resize_coeff_h = calc_resize_coeff(in_tile->width,
1149						   ctx->downsize_coeff_h,
1150						   resized_width, closest);
1151
1152		dev_dbg(priv->ipu->dev, "%s: column %u hscale: *8192/%u\n",
1153			__func__, col, resize_coeff_h);
1154
1155		/*
1156		 * With the horizontal scaling factor known, round up resized
1157		 * width (output width or height) to burst size.
1158		 */
1159		resized_width = round_up(resized_width, 8);
1160
1161		/*
1162		 * Calculate input width from the last accessed input pixel
1163		 * given resized width and scaling coefficients. Round up to
1164		 * burst size.
1165		 */
1166		last_output = resized_width - 1;
1167		if (closest && ((last_output * resize_coeff_h) % 8192))
1168			last_output++;
1169		in_width = round_up(
1170			(DIV_ROUND_UP(last_output * resize_coeff_h, 8192) + 1)
1171			<< ctx->downsize_coeff_h, 8);
1172
1173		for (row = 0; row < ctx->in.num_rows; row++) {
1174			tile_idx = row * ctx->in.num_cols + col;
1175			in_tile = &ctx->in.tile[tile_idx];
1176			out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]];
1177
1178			if (ipu_rot_mode_is_irt(ctx->rot_mode))
1179				out_tile->height = resized_width;
1180			else
1181				out_tile->width = resized_width;
1182
1183			in_tile->width = in_width;
1184		}
1185
1186		ctx->resize_coeffs_h[col] = resize_coeff_h;
1187	}
1188
1189	for (row = 0; row < ctx->in.num_rows; row++) {
1190		bool closest = (row < ctx->in.num_rows - 1) &&
1191			       !(ctx->rot_mode & IPU_ROT_BIT_VFLIP);
1192		u32 resized_height;
1193		u32 resize_coeff_v;
1194		u32 in_height;
1195
1196		tile_idx = row * ctx->in.num_cols;
1197		in_tile = &ctx->in.tile[tile_idx];
1198		out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]];
1199
1200		if (ipu_rot_mode_is_irt(ctx->rot_mode))
1201			resized_height = out_tile->width;
1202		else
1203			resized_height = out_tile->height;
1204
1205		resize_coeff_v = calc_resize_coeff(in_tile->height,
1206						   ctx->downsize_coeff_v,
1207						   resized_height, closest);
1208
1209		dev_dbg(priv->ipu->dev, "%s: row %u vscale: *8192/%u\n",
1210			__func__, row, resize_coeff_v);
1211
1212		/*
1213		 * With the vertical scaling factor known, round up resized
1214		 * height (output width or height) to IDMAC limitations.
1215		 */
1216		resized_height = round_up(resized_height, 2);
1217
1218		/*
1219		 * Calculate input width from the last accessed input pixel
1220		 * given resized height and scaling coefficients. Align to
1221		 * IDMAC restrictions.
1222		 */
1223		last_output = resized_height - 1;
1224		if (closest && ((last_output * resize_coeff_v) % 8192))
1225			last_output++;
1226		in_height = round_up(
1227			(DIV_ROUND_UP(last_output * resize_coeff_v, 8192) + 1)
1228			<< ctx->downsize_coeff_v, 2);
1229
1230		for (col = 0; col < ctx->in.num_cols; col++) {
1231			tile_idx = row * ctx->in.num_cols + col;
1232			in_tile = &ctx->in.tile[tile_idx];
1233			out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]];
1234
1235			if (ipu_rot_mode_is_irt(ctx->rot_mode))
1236				out_tile->width = resized_height;
1237			else
1238				out_tile->height = resized_height;
1239
1240			in_tile->height = in_height;
1241		}
1242
1243		ctx->resize_coeffs_v[row] = resize_coeff_v;
1244	}
1245}
1246
1247/*
1248 * return the number of runs in given queue (pending_q or done_q)
1249 * for this context. hold irqlock when calling.
1250 */
1251static int get_run_count(struct ipu_image_convert_ctx *ctx,
1252			 struct list_head *q)
1253{
1254	struct ipu_image_convert_run *run;
1255	int count = 0;
1256
1257	lockdep_assert_held(&ctx->chan->irqlock);
1258
1259	list_for_each_entry(run, q, list) {
1260		if (run->ctx == ctx)
1261			count++;
1262	}
1263
1264	return count;
1265}
1266
1267static void convert_stop(struct ipu_image_convert_run *run)
1268{
1269	struct ipu_image_convert_ctx *ctx = run->ctx;
1270	struct ipu_image_convert_chan *chan = ctx->chan;
1271	struct ipu_image_convert_priv *priv = chan->priv;
1272
1273	dev_dbg(priv->ipu->dev, "%s: task %u: stopping ctx %p run %p\n",
1274		__func__, chan->ic_task, ctx, run);
1275
1276	/* disable IC tasks and the channels */
1277	ipu_ic_task_disable(chan->ic);
1278	ipu_idmac_disable_channel(chan->in_chan);
1279	ipu_idmac_disable_channel(chan->out_chan);
1280
1281	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
1282		ipu_idmac_disable_channel(chan->rotation_in_chan);
1283		ipu_idmac_disable_channel(chan->rotation_out_chan);
1284		ipu_idmac_unlink(chan->out_chan, chan->rotation_in_chan);
1285	}
1286
1287	ipu_ic_disable(chan->ic);
1288}
1289
1290static void init_idmac_channel(struct ipu_image_convert_ctx *ctx,
1291			       struct ipuv3_channel *channel,
1292			       struct ipu_image_convert_image *image,
1293			       enum ipu_rotate_mode rot_mode,
1294			       bool rot_swap_width_height,
1295			       unsigned int tile)
1296{
1297	struct ipu_image_convert_chan *chan = ctx->chan;
1298	unsigned int burst_size;
1299	u32 width, height, stride;
1300	dma_addr_t addr0, addr1 = 0;
1301	struct ipu_image tile_image;
1302	unsigned int tile_idx[2];
1303
1304	if (image->type == IMAGE_CONVERT_OUT) {
1305		tile_idx[0] = ctx->out_tile_map[tile];
1306		tile_idx[1] = ctx->out_tile_map[1];
1307	} else {
1308		tile_idx[0] = tile;
1309		tile_idx[1] = 1;
1310	}
1311
1312	if (rot_swap_width_height) {
1313		width = image->tile[tile_idx[0]].height;
1314		height = image->tile[tile_idx[0]].width;
1315		stride = image->tile[tile_idx[0]].rot_stride;
1316		addr0 = ctx->rot_intermediate[0].phys;
1317		if (ctx->double_buffering)
1318			addr1 = ctx->rot_intermediate[1].phys;
1319	} else {
1320		width = image->tile[tile_idx[0]].width;
1321		height = image->tile[tile_idx[0]].height;
1322		stride = image->stride;
1323		addr0 = image->base.phys0 +
1324			image->tile[tile_idx[0]].offset;
1325		if (ctx->double_buffering)
1326			addr1 = image->base.phys0 +
1327				image->tile[tile_idx[1]].offset;
1328	}
1329
1330	ipu_cpmem_zero(channel);
1331
1332	memset(&tile_image, 0, sizeof(tile_image));
1333	tile_image.pix.width = tile_image.rect.width = width;
1334	tile_image.pix.height = tile_image.rect.height = height;
1335	tile_image.pix.bytesperline = stride;
1336	tile_image.pix.pixelformat =  image->fmt->fourcc;
1337	tile_image.phys0 = addr0;
1338	tile_image.phys1 = addr1;
1339	if (image->fmt->planar && !rot_swap_width_height) {
1340		tile_image.u_offset = image->tile[tile_idx[0]].u_off;
1341		tile_image.v_offset = image->tile[tile_idx[0]].v_off;
1342	}
1343
1344	ipu_cpmem_set_image(channel, &tile_image);
1345
1346	if (rot_mode)
1347		ipu_cpmem_set_rotation(channel, rot_mode);
1348
1349	/*
1350	 * Skip writing U and V components to odd rows in the output
1351	 * channels for planar 4:2:0.
1352	 */
1353	if ((channel == chan->out_chan ||
1354	     channel == chan->rotation_out_chan) &&
1355	    image->fmt->planar && image->fmt->uv_height_dec == 2)
1356		ipu_cpmem_skip_odd_chroma_rows(channel);
1357
1358	if (channel == chan->rotation_in_chan ||
1359	    channel == chan->rotation_out_chan) {
1360		burst_size = 8;
1361		ipu_cpmem_set_block_mode(channel);
1362	} else
1363		burst_size = (width % 16) ? 8 : 16;
1364
1365	ipu_cpmem_set_burstsize(channel, burst_size);
1366
1367	ipu_ic_task_idma_init(chan->ic, channel, width, height,
1368			      burst_size, rot_mode);
1369
1370	/*
1371	 * Setting a non-zero AXI ID collides with the PRG AXI snooping, so
1372	 * only do this when there is no PRG present.
1373	 */
1374	if (!channel->ipu->prg_priv)
1375		ipu_cpmem_set_axi_id(channel, 1);
1376
1377	ipu_idmac_set_double_buffer(channel, ctx->double_buffering);
1378}
1379
1380static int convert_start(struct ipu_image_convert_run *run, unsigned int tile)
1381{
1382	struct ipu_image_convert_ctx *ctx = run->ctx;
1383	struct ipu_image_convert_chan *chan = ctx->chan;
1384	struct ipu_image_convert_priv *priv = chan->priv;
1385	struct ipu_image_convert_image *s_image = &ctx->in;
1386	struct ipu_image_convert_image *d_image = &ctx->out;
1387	unsigned int dst_tile = ctx->out_tile_map[tile];
1388	unsigned int dest_width, dest_height;
1389	unsigned int col, row;
1390	u32 rsc;
1391	int ret;
1392
1393	dev_dbg(priv->ipu->dev, "%s: task %u: starting ctx %p run %p tile %u -> %u\n",
1394		__func__, chan->ic_task, ctx, run, tile, dst_tile);
1395
1396	/* clear EOF irq mask */
1397	ctx->eof_mask = 0;
1398
1399	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
1400		/* swap width/height for resizer */
1401		dest_width = d_image->tile[dst_tile].height;
1402		dest_height = d_image->tile[dst_tile].width;
1403	} else {
1404		dest_width = d_image->tile[dst_tile].width;
1405		dest_height = d_image->tile[dst_tile].height;
1406	}
1407
1408	row = tile / s_image->num_cols;
1409	col = tile % s_image->num_cols;
1410
1411	rsc =  (ctx->downsize_coeff_v << 30) |
1412	       (ctx->resize_coeffs_v[row] << 16) |
1413	       (ctx->downsize_coeff_h << 14) |
1414	       (ctx->resize_coeffs_h[col]);
1415
1416	dev_dbg(priv->ipu->dev, "%s: %ux%u -> %ux%u (rsc = 0x%x)\n",
1417		__func__, s_image->tile[tile].width,
1418		s_image->tile[tile].height, dest_width, dest_height, rsc);
1419
1420	/* setup the IC resizer and CSC */
1421	ret = ipu_ic_task_init_rsc(chan->ic, &ctx->csc,
1422				   s_image->tile[tile].width,
1423				   s_image->tile[tile].height,
1424				   dest_width,
1425				   dest_height,
1426				   rsc);
1427	if (ret) {
1428		dev_err(priv->ipu->dev, "ipu_ic_task_init failed, %d\n", ret);
1429		return ret;
1430	}
1431
1432	/* init the source MEM-->IC PP IDMAC channel */
1433	init_idmac_channel(ctx, chan->in_chan, s_image,
1434			   IPU_ROTATE_NONE, false, tile);
1435
1436	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
1437		/* init the IC PP-->MEM IDMAC channel */
1438		init_idmac_channel(ctx, chan->out_chan, d_image,
1439				   IPU_ROTATE_NONE, true, tile);
1440
1441		/* init the MEM-->IC PP ROT IDMAC channel */
1442		init_idmac_channel(ctx, chan->rotation_in_chan, d_image,
1443				   ctx->rot_mode, true, tile);
1444
1445		/* init the destination IC PP ROT-->MEM IDMAC channel */
1446		init_idmac_channel(ctx, chan->rotation_out_chan, d_image,
1447				   IPU_ROTATE_NONE, false, tile);
1448
1449		/* now link IC PP-->MEM to MEM-->IC PP ROT */
1450		ipu_idmac_link(chan->out_chan, chan->rotation_in_chan);
1451	} else {
1452		/* init the destination IC PP-->MEM IDMAC channel */
1453		init_idmac_channel(ctx, chan->out_chan, d_image,
1454				   ctx->rot_mode, false, tile);
1455	}
1456
1457	/* enable the IC */
1458	ipu_ic_enable(chan->ic);
1459
1460	/* set buffers ready */
1461	ipu_idmac_select_buffer(chan->in_chan, 0);
1462	ipu_idmac_select_buffer(chan->out_chan, 0);
1463	if (ipu_rot_mode_is_irt(ctx->rot_mode))
1464		ipu_idmac_select_buffer(chan->rotation_out_chan, 0);
1465	if (ctx->double_buffering) {
1466		ipu_idmac_select_buffer(chan->in_chan, 1);
1467		ipu_idmac_select_buffer(chan->out_chan, 1);
1468		if (ipu_rot_mode_is_irt(ctx->rot_mode))
1469			ipu_idmac_select_buffer(chan->rotation_out_chan, 1);
1470	}
1471
1472	/* enable the channels! */
1473	ipu_idmac_enable_channel(chan->in_chan);
1474	ipu_idmac_enable_channel(chan->out_chan);
1475	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
1476		ipu_idmac_enable_channel(chan->rotation_in_chan);
1477		ipu_idmac_enable_channel(chan->rotation_out_chan);
1478	}
1479
1480	ipu_ic_task_enable(chan->ic);
1481
1482	ipu_cpmem_dump(chan->in_chan);
1483	ipu_cpmem_dump(chan->out_chan);
1484	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
1485		ipu_cpmem_dump(chan->rotation_in_chan);
1486		ipu_cpmem_dump(chan->rotation_out_chan);
1487	}
1488
1489	ipu_dump(priv->ipu);
1490
1491	return 0;
1492}
1493
1494/* hold irqlock when calling */
1495static int do_run(struct ipu_image_convert_run *run)
1496{
1497	struct ipu_image_convert_ctx *ctx = run->ctx;
1498	struct ipu_image_convert_chan *chan = ctx->chan;
1499
1500	lockdep_assert_held(&chan->irqlock);
1501
1502	ctx->in.base.phys0 = run->in_phys;
1503	ctx->out.base.phys0 = run->out_phys;
1504
1505	ctx->cur_buf_num = 0;
1506	ctx->next_tile = 1;
1507
1508	/* remove run from pending_q and set as current */
1509	list_del(&run->list);
1510	chan->current_run = run;
1511
1512	return convert_start(run, 0);
1513}
1514
1515/* hold irqlock when calling */
1516static void run_next(struct ipu_image_convert_chan *chan)
1517{
1518	struct ipu_image_convert_priv *priv = chan->priv;
1519	struct ipu_image_convert_run *run, *tmp;
1520	int ret;
1521
1522	lockdep_assert_held(&chan->irqlock);
1523
1524	list_for_each_entry_safe(run, tmp, &chan->pending_q, list) {
1525		/* skip contexts that are aborting */
1526		if (run->ctx->aborting) {
1527			dev_dbg(priv->ipu->dev,
1528				"%s: task %u: skipping aborting ctx %p run %p\n",
1529				__func__, chan->ic_task, run->ctx, run);
1530			continue;
1531		}
1532
1533		ret = do_run(run);
1534		if (!ret)
1535			break;
1536
1537		/*
1538		 * something went wrong with start, add the run
1539		 * to done q and continue to the next run in the
1540		 * pending q.
1541		 */
1542		run->status = ret;
1543		list_add_tail(&run->list, &chan->done_q);
1544		chan->current_run = NULL;
1545	}
1546}
1547
1548static void empty_done_q(struct ipu_image_convert_chan *chan)
1549{
1550	struct ipu_image_convert_priv *priv = chan->priv;
1551	struct ipu_image_convert_run *run;
1552	unsigned long flags;
1553
1554	spin_lock_irqsave(&chan->irqlock, flags);
1555
1556	while (!list_empty(&chan->done_q)) {
1557		run = list_entry(chan->done_q.next,
1558				 struct ipu_image_convert_run,
1559				 list);
1560
1561		list_del(&run->list);
1562
1563		dev_dbg(priv->ipu->dev,
1564			"%s: task %u: completing ctx %p run %p with %d\n",
1565			__func__, chan->ic_task, run->ctx, run, run->status);
1566
1567		/* call the completion callback and free the run */
1568		spin_unlock_irqrestore(&chan->irqlock, flags);
1569		run->ctx->complete(run, run->ctx->complete_context);
1570		spin_lock_irqsave(&chan->irqlock, flags);
1571	}
1572
1573	spin_unlock_irqrestore(&chan->irqlock, flags);
1574}
1575
1576/*
1577 * the bottom half thread clears out the done_q, calling the
1578 * completion handler for each.
1579 */
1580static irqreturn_t do_bh(int irq, void *dev_id)
1581{
1582	struct ipu_image_convert_chan *chan = dev_id;
1583	struct ipu_image_convert_priv *priv = chan->priv;
1584	struct ipu_image_convert_ctx *ctx;
1585	unsigned long flags;
1586
1587	dev_dbg(priv->ipu->dev, "%s: task %u: enter\n", __func__,
1588		chan->ic_task);
1589
1590	empty_done_q(chan);
1591
1592	spin_lock_irqsave(&chan->irqlock, flags);
1593
1594	/*
1595	 * the done_q is cleared out, signal any contexts
1596	 * that are aborting that abort can complete.
1597	 */
1598	list_for_each_entry(ctx, &chan->ctx_list, list) {
1599		if (ctx->aborting) {
1600			dev_dbg(priv->ipu->dev,
1601				"%s: task %u: signaling abort for ctx %p\n",
1602				__func__, chan->ic_task, ctx);
1603			complete_all(&ctx->aborted);
1604		}
1605	}
1606
1607	spin_unlock_irqrestore(&chan->irqlock, flags);
1608
1609	dev_dbg(priv->ipu->dev, "%s: task %u: exit\n", __func__,
1610		chan->ic_task);
1611
1612	return IRQ_HANDLED;
1613}
1614
1615static bool ic_settings_changed(struct ipu_image_convert_ctx *ctx)
1616{
1617	unsigned int cur_tile = ctx->next_tile - 1;
1618	unsigned int next_tile = ctx->next_tile;
1619
1620	if (ctx->resize_coeffs_h[cur_tile % ctx->in.num_cols] !=
1621	    ctx->resize_coeffs_h[next_tile % ctx->in.num_cols] ||
1622	    ctx->resize_coeffs_v[cur_tile / ctx->in.num_cols] !=
1623	    ctx->resize_coeffs_v[next_tile / ctx->in.num_cols] ||
1624	    ctx->in.tile[cur_tile].width != ctx->in.tile[next_tile].width ||
1625	    ctx->in.tile[cur_tile].height != ctx->in.tile[next_tile].height ||
1626	    ctx->out.tile[cur_tile].width != ctx->out.tile[next_tile].width ||
1627	    ctx->out.tile[cur_tile].height != ctx->out.tile[next_tile].height)
1628		return true;
1629
1630	return false;
1631}
1632
1633/* hold irqlock when calling */
1634static irqreturn_t do_tile_complete(struct ipu_image_convert_run *run)
1635{
1636	struct ipu_image_convert_ctx *ctx = run->ctx;
1637	struct ipu_image_convert_chan *chan = ctx->chan;
1638	struct ipu_image_tile *src_tile, *dst_tile;
1639	struct ipu_image_convert_image *s_image = &ctx->in;
1640	struct ipu_image_convert_image *d_image = &ctx->out;
1641	struct ipuv3_channel *outch;
1642	unsigned int dst_idx;
1643
1644	lockdep_assert_held(&chan->irqlock);
1645
1646	outch = ipu_rot_mode_is_irt(ctx->rot_mode) ?
1647		chan->rotation_out_chan : chan->out_chan;
1648
1649	/*
1650	 * It is difficult to stop the channel DMA before the channels
1651	 * enter the paused state. Without double-buffering the channels
1652	 * are always in a paused state when the EOF irq occurs, so it
1653	 * is safe to stop the channels now. For double-buffering we
1654	 * just ignore the abort until the operation completes, when it
1655	 * is safe to shut down.
1656	 */
1657	if (ctx->aborting && !ctx->double_buffering) {
1658		convert_stop(run);
1659		run->status = -EIO;
1660		goto done;
1661	}
1662
1663	if (ctx->next_tile == ctx->num_tiles) {
1664		/*
1665		 * the conversion is complete
1666		 */
1667		convert_stop(run);
1668		run->status = 0;
1669		goto done;
1670	}
1671
1672	/*
1673	 * not done, place the next tile buffers.
1674	 */
1675	if (!ctx->double_buffering) {
1676		if (ic_settings_changed(ctx)) {
1677			convert_stop(run);
1678			convert_start(run, ctx->next_tile);
1679		} else {
1680			src_tile = &s_image->tile[ctx->next_tile];
1681			dst_idx = ctx->out_tile_map[ctx->next_tile];
1682			dst_tile = &d_image->tile[dst_idx];
1683
1684			ipu_cpmem_set_buffer(chan->in_chan, 0,
1685					     s_image->base.phys0 +
1686					     src_tile->offset);
1687			ipu_cpmem_set_buffer(outch, 0,
1688					     d_image->base.phys0 +
1689					     dst_tile->offset);
1690			if (s_image->fmt->planar)
1691				ipu_cpmem_set_uv_offset(chan->in_chan,
1692							src_tile->u_off,
1693							src_tile->v_off);
1694			if (d_image->fmt->planar)
1695				ipu_cpmem_set_uv_offset(outch,
1696							dst_tile->u_off,
1697							dst_tile->v_off);
1698
1699			ipu_idmac_select_buffer(chan->in_chan, 0);
1700			ipu_idmac_select_buffer(outch, 0);
1701		}
1702	} else if (ctx->next_tile < ctx->num_tiles - 1) {
1703
1704		src_tile = &s_image->tile[ctx->next_tile + 1];
1705		dst_idx = ctx->out_tile_map[ctx->next_tile + 1];
1706		dst_tile = &d_image->tile[dst_idx];
1707
1708		ipu_cpmem_set_buffer(chan->in_chan, ctx->cur_buf_num,
1709				     s_image->base.phys0 + src_tile->offset);
1710		ipu_cpmem_set_buffer(outch, ctx->cur_buf_num,
1711				     d_image->base.phys0 + dst_tile->offset);
1712
1713		ipu_idmac_select_buffer(chan->in_chan, ctx->cur_buf_num);
1714		ipu_idmac_select_buffer(outch, ctx->cur_buf_num);
1715
1716		ctx->cur_buf_num ^= 1;
1717	}
1718
1719	ctx->eof_mask = 0; /* clear EOF irq mask for next tile */
1720	ctx->next_tile++;
1721	return IRQ_HANDLED;
1722done:
1723	list_add_tail(&run->list, &chan->done_q);
1724	chan->current_run = NULL;
1725	run_next(chan);
1726	return IRQ_WAKE_THREAD;
1727}
1728
1729static irqreturn_t eof_irq(int irq, void *data)
1730{
1731	struct ipu_image_convert_chan *chan = data;
1732	struct ipu_image_convert_priv *priv = chan->priv;
1733	struct ipu_image_convert_ctx *ctx;
1734	struct ipu_image_convert_run *run;
1735	irqreturn_t ret = IRQ_HANDLED;
1736	bool tile_complete = false;
1737	unsigned long flags;
1738
1739	spin_lock_irqsave(&chan->irqlock, flags);
1740
1741	/* get current run and its context */
1742	run = chan->current_run;
1743	if (!run) {
1744		ret = IRQ_NONE;
1745		goto out;
1746	}
1747
1748	ctx = run->ctx;
1749
1750	if (irq == chan->in_eof_irq) {
1751		ctx->eof_mask |= EOF_IRQ_IN;
1752	} else if (irq == chan->out_eof_irq) {
1753		ctx->eof_mask |= EOF_IRQ_OUT;
1754	} else if (irq == chan->rot_in_eof_irq ||
1755		   irq == chan->rot_out_eof_irq) {
1756		if (!ipu_rot_mode_is_irt(ctx->rot_mode)) {
1757			/* this was NOT a rotation op, shouldn't happen */
1758			dev_err(priv->ipu->dev,
1759				"Unexpected rotation interrupt\n");
1760			goto out;
1761		}
1762		ctx->eof_mask |= (irq == chan->rot_in_eof_irq) ?
1763			EOF_IRQ_ROT_IN : EOF_IRQ_ROT_OUT;
1764	} else {
1765		dev_err(priv->ipu->dev, "Received unknown irq %d\n", irq);
1766		ret = IRQ_NONE;
1767		goto out;
1768	}
1769
1770	if (ipu_rot_mode_is_irt(ctx->rot_mode))
1771		tile_complete =	(ctx->eof_mask == EOF_IRQ_ROT_COMPLETE);
1772	else
1773		tile_complete = (ctx->eof_mask == EOF_IRQ_COMPLETE);
1774
1775	if (tile_complete)
1776		ret = do_tile_complete(run);
1777out:
1778	spin_unlock_irqrestore(&chan->irqlock, flags);
1779	return ret;
1780}
1781
1782/*
1783 * try to force the completion of runs for this ctx. Called when
1784 * abort wait times out in ipu_image_convert_abort().
1785 */
1786static void force_abort(struct ipu_image_convert_ctx *ctx)
1787{
1788	struct ipu_image_convert_chan *chan = ctx->chan;
1789	struct ipu_image_convert_run *run;
1790	unsigned long flags;
1791
1792	spin_lock_irqsave(&chan->irqlock, flags);
1793
1794	run = chan->current_run;
1795	if (run && run->ctx == ctx) {
1796		convert_stop(run);
1797		run->status = -EIO;
1798		list_add_tail(&run->list, &chan->done_q);
1799		chan->current_run = NULL;
1800		run_next(chan);
1801	}
1802
1803	spin_unlock_irqrestore(&chan->irqlock, flags);
1804
1805	empty_done_q(chan);
1806}
1807
1808static void release_ipu_resources(struct ipu_image_convert_chan *chan)
1809{
1810	if (chan->in_eof_irq >= 0)
1811		free_irq(chan->in_eof_irq, chan);
1812	if (chan->rot_in_eof_irq >= 0)
1813		free_irq(chan->rot_in_eof_irq, chan);
1814	if (chan->out_eof_irq >= 0)
1815		free_irq(chan->out_eof_irq, chan);
1816	if (chan->rot_out_eof_irq >= 0)
1817		free_irq(chan->rot_out_eof_irq, chan);
1818
1819	if (!IS_ERR_OR_NULL(chan->in_chan))
1820		ipu_idmac_put(chan->in_chan);
1821	if (!IS_ERR_OR_NULL(chan->out_chan))
1822		ipu_idmac_put(chan->out_chan);
1823	if (!IS_ERR_OR_NULL(chan->rotation_in_chan))
1824		ipu_idmac_put(chan->rotation_in_chan);
1825	if (!IS_ERR_OR_NULL(chan->rotation_out_chan))
1826		ipu_idmac_put(chan->rotation_out_chan);
1827	if (!IS_ERR_OR_NULL(chan->ic))
1828		ipu_ic_put(chan->ic);
1829
1830	chan->in_chan = chan->out_chan = chan->rotation_in_chan =
1831		chan->rotation_out_chan = NULL;
1832	chan->in_eof_irq = -1;
1833	chan->rot_in_eof_irq = -1;
1834	chan->out_eof_irq = -1;
1835	chan->rot_out_eof_irq = -1;
1836}
1837
1838static int get_eof_irq(struct ipu_image_convert_chan *chan,
1839		       struct ipuv3_channel *channel)
1840{
1841	struct ipu_image_convert_priv *priv = chan->priv;
1842	int ret, irq;
1843
1844	irq = ipu_idmac_channel_irq(priv->ipu, channel, IPU_IRQ_EOF);
1845
1846	ret = request_threaded_irq(irq, eof_irq, do_bh, 0, "ipu-ic", chan);
1847	if (ret < 0) {
1848		dev_err(priv->ipu->dev, "could not acquire irq %d\n", irq);
1849		return ret;
1850	}
1851
1852	return irq;
1853}
1854
1855static int get_ipu_resources(struct ipu_image_convert_chan *chan)
1856{
1857	const struct ipu_image_convert_dma_chan *dma = chan->dma_ch;
1858	struct ipu_image_convert_priv *priv = chan->priv;
1859	int ret;
1860
1861	/* get IC */
1862	chan->ic = ipu_ic_get(priv->ipu, chan->ic_task);
1863	if (IS_ERR(chan->ic)) {
1864		dev_err(priv->ipu->dev, "could not acquire IC\n");
1865		ret = PTR_ERR(chan->ic);
1866		goto err;
1867	}
1868
1869	/* get IDMAC channels */
1870	chan->in_chan = ipu_idmac_get(priv->ipu, dma->in);
1871	chan->out_chan = ipu_idmac_get(priv->ipu, dma->out);
1872	if (IS_ERR(chan->in_chan) || IS_ERR(chan->out_chan)) {
1873		dev_err(priv->ipu->dev, "could not acquire idmac channels\n");
1874		ret = -EBUSY;
1875		goto err;
1876	}
1877
1878	chan->rotation_in_chan = ipu_idmac_get(priv->ipu, dma->rot_in);
1879	chan->rotation_out_chan = ipu_idmac_get(priv->ipu, dma->rot_out);
1880	if (IS_ERR(chan->rotation_in_chan) || IS_ERR(chan->rotation_out_chan)) {
1881		dev_err(priv->ipu->dev,
1882			"could not acquire idmac rotation channels\n");
1883		ret = -EBUSY;
1884		goto err;
1885	}
1886
1887	/* acquire the EOF interrupts */
1888	ret = get_eof_irq(chan, chan->in_chan);
1889	if (ret < 0) {
1890		chan->in_eof_irq = -1;
1891		goto err;
1892	}
1893	chan->in_eof_irq = ret;
1894
1895	ret = get_eof_irq(chan, chan->rotation_in_chan);
1896	if (ret < 0) {
1897		chan->rot_in_eof_irq = -1;
1898		goto err;
1899	}
1900	chan->rot_in_eof_irq = ret;
1901
1902	ret = get_eof_irq(chan, chan->out_chan);
1903	if (ret < 0) {
1904		chan->out_eof_irq = -1;
1905		goto err;
1906	}
1907	chan->out_eof_irq = ret;
1908
1909	ret = get_eof_irq(chan, chan->rotation_out_chan);
1910	if (ret < 0) {
1911		chan->rot_out_eof_irq = -1;
1912		goto err;
1913	}
1914	chan->rot_out_eof_irq = ret;
1915
1916	return 0;
1917err:
1918	release_ipu_resources(chan);
1919	return ret;
1920}
1921
1922static int fill_image(struct ipu_image_convert_ctx *ctx,
1923		      struct ipu_image_convert_image *ic_image,
1924		      struct ipu_image *image,
1925		      enum ipu_image_convert_type type)
1926{
1927	struct ipu_image_convert_priv *priv = ctx->chan->priv;
1928
1929	ic_image->base = *image;
1930	ic_image->type = type;
1931
1932	ic_image->fmt = get_format(image->pix.pixelformat);
1933	if (!ic_image->fmt) {
1934		dev_err(priv->ipu->dev, "pixelformat not supported for %s\n",
1935			type == IMAGE_CONVERT_OUT ? "Output" : "Input");
1936		return -EINVAL;
1937	}
1938
1939	if (ic_image->fmt->planar)
1940		ic_image->stride = ic_image->base.pix.width;
1941	else
1942		ic_image->stride  = ic_image->base.pix.bytesperline;
1943
1944	return 0;
1945}
1946
1947/* borrowed from drivers/media/v4l2-core/v4l2-common.c */
1948static unsigned int clamp_align(unsigned int x, unsigned int min,
1949				unsigned int max, unsigned int align)
1950{
1951	/* Bits that must be zero to be aligned */
1952	unsigned int mask = ~((1 << align) - 1);
1953
1954	/* Clamp to aligned min and max */
1955	x = clamp(x, (min + ~mask) & mask, max & mask);
1956
1957	/* Round to nearest aligned value */
1958	if (align)
1959		x = (x + (1 << (align - 1))) & mask;
1960
1961	return x;
1962}
1963
1964/* Adjusts input/output images to IPU restrictions */
1965void ipu_image_convert_adjust(struct ipu_image *in, struct ipu_image *out,
1966			      enum ipu_rotate_mode rot_mode)
1967{
1968	const struct ipu_image_pixfmt *infmt, *outfmt;
1969	u32 w_align_out, h_align_out;
1970	u32 w_align_in, h_align_in;
1971
1972	infmt = get_format(in->pix.pixelformat);
1973	outfmt = get_format(out->pix.pixelformat);
1974
1975	/* set some default pixel formats if needed */
1976	if (!infmt) {
1977		in->pix.pixelformat = V4L2_PIX_FMT_RGB24;
1978		infmt = get_format(V4L2_PIX_FMT_RGB24);
1979	}
1980	if (!outfmt) {
1981		out->pix.pixelformat = V4L2_PIX_FMT_RGB24;
1982		outfmt = get_format(V4L2_PIX_FMT_RGB24);
1983	}
1984
1985	/* image converter does not handle fields */
1986	in->pix.field = out->pix.field = V4L2_FIELD_NONE;
1987
1988	/* resizer cannot downsize more than 4:1 */
1989	if (ipu_rot_mode_is_irt(rot_mode)) {
1990		out->pix.height = max_t(__u32, out->pix.height,
1991					in->pix.width / 4);
1992		out->pix.width = max_t(__u32, out->pix.width,
1993				       in->pix.height / 4);
1994	} else {
1995		out->pix.width = max_t(__u32, out->pix.width,
1996				       in->pix.width / 4);
1997		out->pix.height = max_t(__u32, out->pix.height,
1998					in->pix.height / 4);
1999	}
2000
2001	/* align input width/height */
2002	w_align_in = ilog2(tile_width_align(IMAGE_CONVERT_IN, infmt,
2003					    rot_mode));
2004	h_align_in = ilog2(tile_height_align(IMAGE_CONVERT_IN, infmt,
2005					     rot_mode));
2006	in->pix.width = clamp_align(in->pix.width, MIN_W, MAX_W,
2007				    w_align_in);
2008	in->pix.height = clamp_align(in->pix.height, MIN_H, MAX_H,
2009				     h_align_in);
2010
2011	/* align output width/height */
2012	w_align_out = ilog2(tile_width_align(IMAGE_CONVERT_OUT, outfmt,
2013					     rot_mode));
2014	h_align_out = ilog2(tile_height_align(IMAGE_CONVERT_OUT, outfmt,
2015					      rot_mode));
2016	out->pix.width = clamp_align(out->pix.width, MIN_W, MAX_W,
2017				     w_align_out);
2018	out->pix.height = clamp_align(out->pix.height, MIN_H, MAX_H,
2019				      h_align_out);
2020
2021	/* set input/output strides and image sizes */
2022	in->pix.bytesperline = infmt->planar ?
2023		clamp_align(in->pix.width, 2 << w_align_in, MAX_W,
2024			    w_align_in) :
2025		clamp_align((in->pix.width * infmt->bpp) >> 3,
2026			    ((2 << w_align_in) * infmt->bpp) >> 3,
2027			    (MAX_W * infmt->bpp) >> 3,
2028			    w_align_in);
2029	in->pix.sizeimage = infmt->planar ?
2030		(in->pix.height * in->pix.bytesperline * infmt->bpp) >> 3 :
2031		in->pix.height * in->pix.bytesperline;
2032	out->pix.bytesperline = outfmt->planar ? out->pix.width :
2033		(out->pix.width * outfmt->bpp) >> 3;
2034	out->pix.sizeimage = outfmt->planar ?
2035		(out->pix.height * out->pix.bytesperline * outfmt->bpp) >> 3 :
2036		out->pix.height * out->pix.bytesperline;
2037}
2038EXPORT_SYMBOL_GPL(ipu_image_convert_adjust);
2039
2040/*
2041 * this is used by ipu_image_convert_prepare() to verify set input and
2042 * output images are valid before starting the conversion. Clients can
2043 * also call it before calling ipu_image_convert_prepare().
2044 */
2045int ipu_image_convert_verify(struct ipu_image *in, struct ipu_image *out,
2046			     enum ipu_rotate_mode rot_mode)
2047{
2048	struct ipu_image testin, testout;
2049
2050	testin = *in;
2051	testout = *out;
2052
2053	ipu_image_convert_adjust(&testin, &testout, rot_mode);
2054
2055	if (testin.pix.width != in->pix.width ||
2056	    testin.pix.height != in->pix.height ||
2057	    testout.pix.width != out->pix.width ||
2058	    testout.pix.height != out->pix.height)
2059		return -EINVAL;
2060
2061	return 0;
2062}
2063EXPORT_SYMBOL_GPL(ipu_image_convert_verify);
2064
2065/*
2066 * Call ipu_image_convert_prepare() to prepare for the conversion of
2067 * given images and rotation mode. Returns a new conversion context.
2068 */
2069struct ipu_image_convert_ctx *
2070ipu_image_convert_prepare(struct ipu_soc *ipu, enum ipu_ic_task ic_task,
2071			  struct ipu_image *in, struct ipu_image *out,
2072			  enum ipu_rotate_mode rot_mode,
2073			  ipu_image_convert_cb_t complete,
2074			  void *complete_context)
2075{
2076	struct ipu_image_convert_priv *priv = ipu->image_convert_priv;
2077	struct ipu_image_convert_image *s_image, *d_image;
2078	struct ipu_image_convert_chan *chan;
2079	struct ipu_image_convert_ctx *ctx;
2080	unsigned long flags;
2081	unsigned int i;
2082	bool get_res;
2083	int ret;
2084
2085	if (!in || !out || !complete ||
2086	    (ic_task != IC_TASK_VIEWFINDER &&
2087	     ic_task != IC_TASK_POST_PROCESSOR))
2088		return ERR_PTR(-EINVAL);
2089
2090	/* verify the in/out images before continuing */
2091	ret = ipu_image_convert_verify(in, out, rot_mode);
2092	if (ret) {
2093		dev_err(priv->ipu->dev, "%s: in/out formats invalid\n",
2094			__func__);
2095		return ERR_PTR(ret);
2096	}
2097
2098	chan = &priv->chan[ic_task];
2099
2100	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
2101	if (!ctx)
2102		return ERR_PTR(-ENOMEM);
2103
2104	dev_dbg(priv->ipu->dev, "%s: task %u: ctx %p\n", __func__,
2105		chan->ic_task, ctx);
2106
2107	ctx->chan = chan;
2108	init_completion(&ctx->aborted);
2109
2110	ctx->rot_mode = rot_mode;
2111
2112	/* Sets ctx->in.num_rows/cols as well */
2113	ret = calc_image_resize_coefficients(ctx, in, out);
2114	if (ret)
2115		goto out_free;
2116
2117	s_image = &ctx->in;
2118	d_image = &ctx->out;
2119
2120	/* set tiling and rotation */
2121	if (ipu_rot_mode_is_irt(rot_mode)) {
2122		d_image->num_rows = s_image->num_cols;
2123		d_image->num_cols = s_image->num_rows;
2124	} else {
2125		d_image->num_rows = s_image->num_rows;
2126		d_image->num_cols = s_image->num_cols;
2127	}
2128
2129	ctx->num_tiles = d_image->num_cols * d_image->num_rows;
2130
2131	ret = fill_image(ctx, s_image, in, IMAGE_CONVERT_IN);
2132	if (ret)
2133		goto out_free;
2134	ret = fill_image(ctx, d_image, out, IMAGE_CONVERT_OUT);
2135	if (ret)
2136		goto out_free;
2137
2138	calc_out_tile_map(ctx);
2139
2140	find_seams(ctx, s_image, d_image);
2141
2142	ret = calc_tile_dimensions(ctx, s_image);
2143	if (ret)
2144		goto out_free;
2145
2146	ret = calc_tile_offsets(ctx, s_image);
2147	if (ret)
2148		goto out_free;
2149
2150	calc_tile_dimensions(ctx, d_image);
2151	ret = calc_tile_offsets(ctx, d_image);
2152	if (ret)
2153		goto out_free;
2154
2155	calc_tile_resize_coefficients(ctx);
2156
2157	ret = ipu_ic_calc_csc(&ctx->csc,
2158			s_image->base.pix.ycbcr_enc,
2159			s_image->base.pix.quantization,
2160			ipu_pixelformat_to_colorspace(s_image->fmt->fourcc),
2161			d_image->base.pix.ycbcr_enc,
2162			d_image->base.pix.quantization,
2163			ipu_pixelformat_to_colorspace(d_image->fmt->fourcc));
2164	if (ret)
2165		goto out_free;
2166
2167	dump_format(ctx, s_image);
2168	dump_format(ctx, d_image);
2169
2170	ctx->complete = complete;
2171	ctx->complete_context = complete_context;
2172
2173	/*
2174	 * Can we use double-buffering for this operation? If there is
2175	 * only one tile (the whole image can be converted in a single
2176	 * operation) there's no point in using double-buffering. Also,
2177	 * the IPU's IDMAC channels allow only a single U and V plane
2178	 * offset shared between both buffers, but these offsets change
2179	 * for every tile, and therefore would have to be updated for
2180	 * each buffer which is not possible. So double-buffering is
2181	 * impossible when either the source or destination images are
2182	 * a planar format (YUV420, YUV422P, etc.). Further, differently
2183	 * sized tiles or different resizing coefficients per tile
2184	 * prevent double-buffering as well.
2185	 */
2186	ctx->double_buffering = (ctx->num_tiles > 1 &&
2187				 !s_image->fmt->planar &&
2188				 !d_image->fmt->planar);
2189	for (i = 1; i < ctx->num_tiles; i++) {
2190		if (ctx->in.tile[i].width != ctx->in.tile[0].width ||
2191		    ctx->in.tile[i].height != ctx->in.tile[0].height ||
2192		    ctx->out.tile[i].width != ctx->out.tile[0].width ||
2193		    ctx->out.tile[i].height != ctx->out.tile[0].height) {
2194			ctx->double_buffering = false;
2195			break;
2196		}
2197	}
2198	for (i = 1; i < ctx->in.num_cols; i++) {
2199		if (ctx->resize_coeffs_h[i] != ctx->resize_coeffs_h[0]) {
2200			ctx->double_buffering = false;
2201			break;
2202		}
2203	}
2204	for (i = 1; i < ctx->in.num_rows; i++) {
2205		if (ctx->resize_coeffs_v[i] != ctx->resize_coeffs_v[0]) {
2206			ctx->double_buffering = false;
2207			break;
2208		}
2209	}
2210
2211	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
2212		unsigned long intermediate_size = d_image->tile[0].size;
2213
2214		for (i = 1; i < ctx->num_tiles; i++) {
2215			if (d_image->tile[i].size > intermediate_size)
2216				intermediate_size = d_image->tile[i].size;
2217		}
2218
2219		ret = alloc_dma_buf(priv, &ctx->rot_intermediate[0],
2220				    intermediate_size);
2221		if (ret)
2222			goto out_free;
2223		if (ctx->double_buffering) {
2224			ret = alloc_dma_buf(priv,
2225					    &ctx->rot_intermediate[1],
2226					    intermediate_size);
2227			if (ret)
2228				goto out_free_dmabuf0;
2229		}
2230	}
2231
2232	spin_lock_irqsave(&chan->irqlock, flags);
2233
2234	get_res = list_empty(&chan->ctx_list);
2235
2236	list_add_tail(&ctx->list, &chan->ctx_list);
2237
2238	spin_unlock_irqrestore(&chan->irqlock, flags);
2239
2240	if (get_res) {
2241		ret = get_ipu_resources(chan);
2242		if (ret)
2243			goto out_free_dmabuf1;
2244	}
2245
2246	return ctx;
2247
2248out_free_dmabuf1:
2249	free_dma_buf(priv, &ctx->rot_intermediate[1]);
2250	spin_lock_irqsave(&chan->irqlock, flags);
2251	list_del(&ctx->list);
2252	spin_unlock_irqrestore(&chan->irqlock, flags);
2253out_free_dmabuf0:
2254	free_dma_buf(priv, &ctx->rot_intermediate[0]);
2255out_free:
2256	kfree(ctx);
2257	return ERR_PTR(ret);
2258}
2259EXPORT_SYMBOL_GPL(ipu_image_convert_prepare);
2260
2261/*
2262 * Carry out a single image conversion run. Only the physaddr's of the input
2263 * and output image buffers are needed. The conversion context must have
2264 * been created previously with ipu_image_convert_prepare().
2265 */
2266int ipu_image_convert_queue(struct ipu_image_convert_run *run)
2267{
2268	struct ipu_image_convert_chan *chan;
2269	struct ipu_image_convert_priv *priv;
2270	struct ipu_image_convert_ctx *ctx;
2271	unsigned long flags;
2272	int ret = 0;
2273
2274	if (!run || !run->ctx || !run->in_phys || !run->out_phys)
2275		return -EINVAL;
2276
2277	ctx = run->ctx;
2278	chan = ctx->chan;
2279	priv = chan->priv;
2280
2281	dev_dbg(priv->ipu->dev, "%s: task %u: ctx %p run %p\n", __func__,
2282		chan->ic_task, ctx, run);
2283
2284	INIT_LIST_HEAD(&run->list);
2285
2286	spin_lock_irqsave(&chan->irqlock, flags);
2287
2288	if (ctx->aborting) {
2289		ret = -EIO;
2290		goto unlock;
2291	}
2292
2293	list_add_tail(&run->list, &chan->pending_q);
2294
2295	if (!chan->current_run) {
2296		ret = do_run(run);
2297		if (ret)
2298			chan->current_run = NULL;
2299	}
2300unlock:
2301	spin_unlock_irqrestore(&chan->irqlock, flags);
2302	return ret;
2303}
2304EXPORT_SYMBOL_GPL(ipu_image_convert_queue);
2305
2306/* Abort any active or pending conversions for this context */
2307static void __ipu_image_convert_abort(struct ipu_image_convert_ctx *ctx)
2308{
2309	struct ipu_image_convert_chan *chan = ctx->chan;
2310	struct ipu_image_convert_priv *priv = chan->priv;
2311	struct ipu_image_convert_run *run, *active_run, *tmp;
2312	unsigned long flags;
2313	int run_count, ret;
2314
2315	spin_lock_irqsave(&chan->irqlock, flags);
2316
2317	/* move all remaining pending runs in this context to done_q */
2318	list_for_each_entry_safe(run, tmp, &chan->pending_q, list) {
2319		if (run->ctx != ctx)
2320			continue;
2321		run->status = -EIO;
2322		list_move_tail(&run->list, &chan->done_q);
2323	}
2324
2325	run_count = get_run_count(ctx, &chan->done_q);
2326	active_run = (chan->current_run && chan->current_run->ctx == ctx) ?
2327		chan->current_run : NULL;
2328
2329	if (active_run)
2330		reinit_completion(&ctx->aborted);
2331
2332	ctx->aborting = true;
2333
2334	spin_unlock_irqrestore(&chan->irqlock, flags);
2335
2336	if (!run_count && !active_run) {
2337		dev_dbg(priv->ipu->dev,
2338			"%s: task %u: no abort needed for ctx %p\n",
2339			__func__, chan->ic_task, ctx);
2340		return;
2341	}
2342
2343	if (!active_run) {
2344		empty_done_q(chan);
2345		return;
2346	}
2347
2348	dev_dbg(priv->ipu->dev,
2349		"%s: task %u: wait for completion: %d runs\n",
2350		__func__, chan->ic_task, run_count);
2351
2352	ret = wait_for_completion_timeout(&ctx->aborted,
2353					  msecs_to_jiffies(10000));
2354	if (ret == 0) {
2355		dev_warn(priv->ipu->dev, "%s: timeout\n", __func__);
2356		force_abort(ctx);
2357	}
2358}
2359
2360void ipu_image_convert_abort(struct ipu_image_convert_ctx *ctx)
2361{
2362	__ipu_image_convert_abort(ctx);
2363	ctx->aborting = false;
2364}
2365EXPORT_SYMBOL_GPL(ipu_image_convert_abort);
2366
2367/* Unprepare image conversion context */
2368void ipu_image_convert_unprepare(struct ipu_image_convert_ctx *ctx)
2369{
2370	struct ipu_image_convert_chan *chan = ctx->chan;
2371	struct ipu_image_convert_priv *priv = chan->priv;
2372	unsigned long flags;
2373	bool put_res;
2374
2375	/* make sure no runs are hanging around */
2376	__ipu_image_convert_abort(ctx);
2377
2378	dev_dbg(priv->ipu->dev, "%s: task %u: removing ctx %p\n", __func__,
2379		chan->ic_task, ctx);
2380
2381	spin_lock_irqsave(&chan->irqlock, flags);
2382
2383	list_del(&ctx->list);
2384
2385	put_res = list_empty(&chan->ctx_list);
2386
2387	spin_unlock_irqrestore(&chan->irqlock, flags);
2388
2389	if (put_res)
2390		release_ipu_resources(chan);
2391
2392	free_dma_buf(priv, &ctx->rot_intermediate[1]);
2393	free_dma_buf(priv, &ctx->rot_intermediate[0]);
2394
2395	kfree(ctx);
2396}
2397EXPORT_SYMBOL_GPL(ipu_image_convert_unprepare);
2398
2399/*
2400 * "Canned" asynchronous single image conversion. Allocates and returns
2401 * a new conversion run.  On successful return the caller must free the
2402 * run and call ipu_image_convert_unprepare() after conversion completes.
2403 */
2404struct ipu_image_convert_run *
2405ipu_image_convert(struct ipu_soc *ipu, enum ipu_ic_task ic_task,
2406		  struct ipu_image *in, struct ipu_image *out,
2407		  enum ipu_rotate_mode rot_mode,
2408		  ipu_image_convert_cb_t complete,
2409		  void *complete_context)
2410{
2411	struct ipu_image_convert_ctx *ctx;
2412	struct ipu_image_convert_run *run;
2413	int ret;
2414
2415	ctx = ipu_image_convert_prepare(ipu, ic_task, in, out, rot_mode,
2416					complete, complete_context);
2417	if (IS_ERR(ctx))
2418		return ERR_CAST(ctx);
2419
2420	run = kzalloc(sizeof(*run), GFP_KERNEL);
2421	if (!run) {
2422		ipu_image_convert_unprepare(ctx);
2423		return ERR_PTR(-ENOMEM);
2424	}
2425
2426	run->ctx = ctx;
2427	run->in_phys = in->phys0;
2428	run->out_phys = out->phys0;
2429
2430	ret = ipu_image_convert_queue(run);
2431	if (ret) {
2432		ipu_image_convert_unprepare(ctx);
2433		kfree(run);
2434		return ERR_PTR(ret);
2435	}
2436
2437	return run;
2438}
2439EXPORT_SYMBOL_GPL(ipu_image_convert);
2440
2441/* "Canned" synchronous single image conversion */
2442static void image_convert_sync_complete(struct ipu_image_convert_run *run,
2443					void *data)
2444{
2445	struct completion *comp = data;
2446
2447	complete(comp);
2448}
2449
2450int ipu_image_convert_sync(struct ipu_soc *ipu, enum ipu_ic_task ic_task,
2451			   struct ipu_image *in, struct ipu_image *out,
2452			   enum ipu_rotate_mode rot_mode)
2453{
2454	struct ipu_image_convert_run *run;
2455	struct completion comp;
2456	int ret;
2457
2458	init_completion(&comp);
2459
2460	run = ipu_image_convert(ipu, ic_task, in, out, rot_mode,
2461				image_convert_sync_complete, &comp);
2462	if (IS_ERR(run))
2463		return PTR_ERR(run);
2464
2465	ret = wait_for_completion_timeout(&comp, msecs_to_jiffies(10000));
2466	ret = (ret == 0) ? -ETIMEDOUT : 0;
2467
2468	ipu_image_convert_unprepare(run->ctx);
2469	kfree(run);
2470
2471	return ret;
2472}
2473EXPORT_SYMBOL_GPL(ipu_image_convert_sync);
2474
2475int ipu_image_convert_init(struct ipu_soc *ipu, struct device *dev)
2476{
2477	struct ipu_image_convert_priv *priv;
2478	int i;
2479
2480	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
2481	if (!priv)
2482		return -ENOMEM;
2483
2484	ipu->image_convert_priv = priv;
2485	priv->ipu = ipu;
2486
2487	for (i = 0; i < IC_NUM_TASKS; i++) {
2488		struct ipu_image_convert_chan *chan = &priv->chan[i];
2489
2490		chan->ic_task = i;
2491		chan->priv = priv;
2492		chan->dma_ch = &image_convert_dma_chan[i];
2493		chan->in_eof_irq = -1;
2494		chan->rot_in_eof_irq = -1;
2495		chan->out_eof_irq = -1;
2496		chan->rot_out_eof_irq = -1;
2497
2498		spin_lock_init(&chan->irqlock);
2499		INIT_LIST_HEAD(&chan->ctx_list);
2500		INIT_LIST_HEAD(&chan->pending_q);
2501		INIT_LIST_HEAD(&chan->done_q);
2502	}
2503
2504	return 0;
2505}
2506
2507void ipu_image_convert_exit(struct ipu_soc *ipu)
2508{
2509}