ipu-image-convert.c - drivers/gpu/ipu-v3/ipu-image-convert.c - Linux diff v6.13.7

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Copyright (C) 2012-2016 Mentor Graphics Inc.
   4 *
   5 * Queued image conversion support, with tiling and rotation.
 
 
 
 
 
 
 
 
 
 
   6 */
   7
   8#include <linux/interrupt.h>
   9#include <linux/dma-mapping.h>
  10#include <linux/math.h>
  11
  12#include <video/imx-ipu-image-convert.h>
  13
  14#include "ipu-prv.h"
  15
  16/*
  17 * The IC Resizer has a restriction that the output frame from the
  18 * resizer must be 1024 or less in both width (pixels) and height
  19 * (lines).
  20 *
  21 * The image converter attempts to split up a conversion when
  22 * the desired output (converted) frame resolution exceeds the
  23 * IC resizer limit of 1024 in either dimension.
  24 *
  25 * If either dimension of the output frame exceeds the limit, the
  26 * dimension is split into 1, 2, or 4 equal stripes, for a maximum
  27 * of 4*4 or 16 tiles. A conversion is then carried out for each
  28 * tile (but taking care to pass the full frame stride length to
  29 * the DMA channel's parameter memory!). IDMA double-buffering is used
  30 * to convert each tile back-to-back when possible (see note below
  31 * when double_buffering boolean is set).
  32 *
  33 * Note that the input frame must be split up into the same number
  34 * of tiles as the output frame:
  35 *
  36 *                       +---------+-----+
  37 *   +-----+---+         |  A      | B   |
  38 *   | A   | B |         |         |     |
  39 *   +-----+---+   -->   +---------+-----+
  40 *   | C   | D |         |  C      | D   |
  41 *   +-----+---+         |         |     |
  42 *                       +---------+-----+
  43 *
  44 * Clockwise 90° rotations are handled by first rescaling into a
  45 * reusable temporary tile buffer and then rotating with the 8x8
  46 * block rotator, writing to the correct destination:
  47 *
  48 *                                         +-----+-----+
  49 *                                         |     |     |
  50 *   +-----+---+         +---------+       | C   | A   |
  51 *   | A   | B |         | A,B, |  |       |     |     |
  52 *   +-----+---+   -->   | C,D  |  |  -->  |     |     |
  53 *   | C   | D |         +---------+       +-----+-----+
  54 *   +-----+---+                           | D   | B   |
  55 *                                         |     |     |
  56 *                                         +-----+-----+
  57 *
  58 * If the 8x8 block rotator is used, horizontal or vertical flipping
  59 * is done during the rotation step, otherwise flipping is done
  60 * during the scaling step.
  61 * With rotation or flipping, tile order changes between input and
  62 * output image. Tiles are numbered row major from top left to bottom
  63 * right for both input and output image.
  64 */
  65
  66#define MAX_STRIPES_W    4
  67#define MAX_STRIPES_H    4
  68#define MAX_TILES (MAX_STRIPES_W * MAX_STRIPES_H)
  69
  70#define MIN_W     16
  71#define MIN_H     8
  72#define MAX_W     4096
  73#define MAX_H     4096
  74
  75enum ipu_image_convert_type {
  76	IMAGE_CONVERT_IN = 0,
  77	IMAGE_CONVERT_OUT,
  78};
  79
  80struct ipu_image_convert_dma_buf {
  81	void          *virt;
  82	dma_addr_t    phys;
  83	unsigned long len;
  84};
  85
  86struct ipu_image_convert_dma_chan {
  87	int in;
  88	int out;
  89	int rot_in;
  90	int rot_out;
  91	int vdi_in_p;
  92	int vdi_in;
  93	int vdi_in_n;
  94};
  95
  96/* dimensions of one tile */
  97struct ipu_image_tile {
  98	u32 width;
  99	u32 height;
 100	u32 left;
 101	u32 top;
 102	/* size and strides are in bytes */
 103	u32 size;
 104	u32 stride;
 105	u32 rot_stride;
 106	/* start Y or packed offset of this tile */
 107	u32 offset;
 108	/* offset from start to tile in U plane, for planar formats */
 109	u32 u_off;
 110	/* offset from start to tile in V plane, for planar formats */
 111	u32 v_off;
 112};
 113
 114struct ipu_image_convert_image {
 115	struct ipu_image base;
 116	enum ipu_image_convert_type type;
 117
 118	const struct ipu_image_pixfmt *fmt;
 119	unsigned int stride;
 120
 121	/* # of rows (horizontal stripes) if dest height is > 1024 */
 122	unsigned int num_rows;
 123	/* # of columns (vertical stripes) if dest width is > 1024 */
 124	unsigned int num_cols;
 125
 126	struct ipu_image_tile tile[MAX_TILES];
 127};
 128
 129struct ipu_image_pixfmt {
 130	u32	fourcc;        /* V4L2 fourcc */
 131	int     bpp;           /* total bpp */
 132	int     uv_width_dec;  /* decimation in width for U/V planes */
 133	int     uv_height_dec; /* decimation in height for U/V planes */
 134	bool    planar;        /* planar format */
 135	bool    uv_swapped;    /* U and V planes are swapped */
 136	bool    uv_packed;     /* partial planar (U and V in same plane) */
 137};
 138
 139struct ipu_image_convert_ctx;
 140struct ipu_image_convert_chan;
 141struct ipu_image_convert_priv;
 142
 143enum eof_irq_mask {
 144	EOF_IRQ_IN      = BIT(0),
 145	EOF_IRQ_ROT_IN  = BIT(1),
 146	EOF_IRQ_OUT     = BIT(2),
 147	EOF_IRQ_ROT_OUT = BIT(3),
 148};
 149
 150#define EOF_IRQ_COMPLETE (EOF_IRQ_IN | EOF_IRQ_OUT)
 151#define EOF_IRQ_ROT_COMPLETE (EOF_IRQ_IN | EOF_IRQ_OUT |	\
 152			      EOF_IRQ_ROT_IN | EOF_IRQ_ROT_OUT)
 153
 154struct ipu_image_convert_ctx {
 155	struct ipu_image_convert_chan *chan;
 156
 157	ipu_image_convert_cb_t complete;
 158	void *complete_context;
 159
 160	/* Source/destination image data and rotation mode */
 161	struct ipu_image_convert_image in;
 162	struct ipu_image_convert_image out;
 163	struct ipu_ic_csc csc;
 164	enum ipu_rotate_mode rot_mode;
 165	u32 downsize_coeff_h;
 166	u32 downsize_coeff_v;
 167	u32 image_resize_coeff_h;
 168	u32 image_resize_coeff_v;
 169	u32 resize_coeffs_h[MAX_STRIPES_W];
 170	u32 resize_coeffs_v[MAX_STRIPES_H];
 171
 172	/* intermediate buffer for rotation */
 173	struct ipu_image_convert_dma_buf rot_intermediate[2];
 174
 175	/* current buffer number for double buffering */
 176	int cur_buf_num;
 177
 178	bool aborting;
 179	struct completion aborted;
 180
 181	/* can we use double-buffering for this conversion operation? */
 182	bool double_buffering;
 183	/* num_rows * num_cols */
 184	unsigned int num_tiles;
 185	/* next tile to process */
 186	unsigned int next_tile;
 187	/* where to place converted tile in dest image */
 188	unsigned int out_tile_map[MAX_TILES];
 189
 190	/* mask of completed EOF irqs at every tile conversion */
 191	enum eof_irq_mask eof_mask;
 192
 193	struct list_head list;
 194};
 195
 196struct ipu_image_convert_chan {
 197	struct ipu_image_convert_priv *priv;
 198
 199	enum ipu_ic_task ic_task;
 200	const struct ipu_image_convert_dma_chan *dma_ch;
 201
 202	struct ipu_ic *ic;
 203	struct ipuv3_channel *in_chan;
 204	struct ipuv3_channel *out_chan;
 205	struct ipuv3_channel *rotation_in_chan;
 206	struct ipuv3_channel *rotation_out_chan;
 207
 208	/* the IPU end-of-frame irqs */
 209	int in_eof_irq;
 210	int rot_in_eof_irq;
 211	int out_eof_irq;
 212	int rot_out_eof_irq;
 213
 214	spinlock_t irqlock;
 215
 216	/* list of convert contexts */
 217	struct list_head ctx_list;
 218	/* queue of conversion runs */
 219	struct list_head pending_q;
 220	/* queue of completed runs */
 221	struct list_head done_q;
 222
 223	/* the current conversion run */
 224	struct ipu_image_convert_run *current_run;
 225};
 226
 227struct ipu_image_convert_priv {
 228	struct ipu_image_convert_chan chan[IC_NUM_TASKS];
 229	struct ipu_soc *ipu;
 230};
 231
 232static const struct ipu_image_convert_dma_chan
 233image_convert_dma_chan[IC_NUM_TASKS] = {
 234	[IC_TASK_VIEWFINDER] = {
 235		.in = IPUV3_CHANNEL_MEM_IC_PRP_VF,
 236		.out = IPUV3_CHANNEL_IC_PRP_VF_MEM,
 237		.rot_in = IPUV3_CHANNEL_MEM_ROT_VF,
 238		.rot_out = IPUV3_CHANNEL_ROT_VF_MEM,
 239		.vdi_in_p = IPUV3_CHANNEL_MEM_VDI_PREV,
 240		.vdi_in = IPUV3_CHANNEL_MEM_VDI_CUR,
 241		.vdi_in_n = IPUV3_CHANNEL_MEM_VDI_NEXT,
 242	},
 243	[IC_TASK_POST_PROCESSOR] = {
 244		.in = IPUV3_CHANNEL_MEM_IC_PP,
 245		.out = IPUV3_CHANNEL_IC_PP_MEM,
 246		.rot_in = IPUV3_CHANNEL_MEM_ROT_PP,
 247		.rot_out = IPUV3_CHANNEL_ROT_PP_MEM,
 248	},
 249};
 250
 251static const struct ipu_image_pixfmt image_convert_formats[] = {
 252	{
 253		.fourcc	= V4L2_PIX_FMT_RGB565,
 254		.bpp    = 16,
 255	}, {
 256		.fourcc	= V4L2_PIX_FMT_RGB24,
 257		.bpp    = 24,
 258	}, {
 259		.fourcc	= V4L2_PIX_FMT_BGR24,
 260		.bpp    = 24,
 261	}, {
 262		.fourcc	= V4L2_PIX_FMT_RGB32,
 263		.bpp    = 32,
 264	}, {
 265		.fourcc	= V4L2_PIX_FMT_BGR32,
 266		.bpp    = 32,
 267	}, {
 268		.fourcc	= V4L2_PIX_FMT_XRGB32,
 269		.bpp    = 32,
 270	}, {
 271		.fourcc	= V4L2_PIX_FMT_XBGR32,
 272		.bpp    = 32,
 273	}, {
 274		.fourcc	= V4L2_PIX_FMT_BGRX32,
 275		.bpp    = 32,
 276	}, {
 277		.fourcc	= V4L2_PIX_FMT_RGBX32,
 278		.bpp    = 32,
 279	}, {
 280		.fourcc	= V4L2_PIX_FMT_YUYV,
 281		.bpp    = 16,
 282		.uv_width_dec = 2,
 283		.uv_height_dec = 1,
 284	}, {
 285		.fourcc	= V4L2_PIX_FMT_UYVY,
 286		.bpp    = 16,
 287		.uv_width_dec = 2,
 288		.uv_height_dec = 1,
 289	}, {
 290		.fourcc	= V4L2_PIX_FMT_YUV420,
 291		.bpp    = 12,
 292		.planar = true,
 293		.uv_width_dec = 2,
 294		.uv_height_dec = 2,
 295	}, {
 296		.fourcc	= V4L2_PIX_FMT_YVU420,
 297		.bpp    = 12,
 298		.planar = true,
 299		.uv_width_dec = 2,
 300		.uv_height_dec = 2,
 301		.uv_swapped = true,
 302	}, {
 303		.fourcc = V4L2_PIX_FMT_NV12,
 304		.bpp    = 12,
 305		.planar = true,
 306		.uv_width_dec = 2,
 307		.uv_height_dec = 2,
 308		.uv_packed = true,
 309	}, {
 310		.fourcc = V4L2_PIX_FMT_YUV422P,
 311		.bpp    = 16,
 312		.planar = true,
 313		.uv_width_dec = 2,
 314		.uv_height_dec = 1,
 315	}, {
 316		.fourcc = V4L2_PIX_FMT_NV16,
 317		.bpp    = 16,
 318		.planar = true,
 319		.uv_width_dec = 2,
 320		.uv_height_dec = 1,
 321		.uv_packed = true,
 322	},
 323};
 324
 325static const struct ipu_image_pixfmt *get_format(u32 fourcc)
 326{
 327	const struct ipu_image_pixfmt *ret = NULL;
 328	unsigned int i;
 329
 330	for (i = 0; i < ARRAY_SIZE(image_convert_formats); i++) {
 331		if (image_convert_formats[i].fourcc == fourcc) {
 332			ret = &image_convert_formats[i];
 333			break;
 334		}
 335	}
 336
 337	return ret;
 338}
 339
 340static void dump_format(struct ipu_image_convert_ctx *ctx,
 341			struct ipu_image_convert_image *ic_image)
 342{
 343	struct ipu_image_convert_chan *chan = ctx->chan;
 344	struct ipu_image_convert_priv *priv = chan->priv;
 345
 346	dev_dbg(priv->ipu->dev,
 347		"task %u: ctx %p: %s format: %dx%d (%dx%d tiles), %c%c%c%c\n",
 348		chan->ic_task, ctx,
 349		ic_image->type == IMAGE_CONVERT_OUT ? "Output" : "Input",
 350		ic_image->base.pix.width, ic_image->base.pix.height,
 351		ic_image->num_cols, ic_image->num_rows,
 
 352		ic_image->fmt->fourcc & 0xff,
 353		(ic_image->fmt->fourcc >> 8) & 0xff,
 354		(ic_image->fmt->fourcc >> 16) & 0xff,
 355		(ic_image->fmt->fourcc >> 24) & 0xff);
 356}
 357
 358int ipu_image_convert_enum_format(int index, u32 *fourcc)
 359{
 360	const struct ipu_image_pixfmt *fmt;
 361
 362	if (index >= (int)ARRAY_SIZE(image_convert_formats))
 363		return -EINVAL;
 364
 365	/* Format found */
 366	fmt = &image_convert_formats[index];
 367	*fourcc = fmt->fourcc;
 368	return 0;
 369}
 370EXPORT_SYMBOL_GPL(ipu_image_convert_enum_format);
 371
 372static void free_dma_buf(struct ipu_image_convert_priv *priv,
 373			 struct ipu_image_convert_dma_buf *buf)
 374{
 375	if (buf->virt)
 376		dma_free_coherent(priv->ipu->dev,
 377				  buf->len, buf->virt, buf->phys);
 378	buf->virt = NULL;
 379	buf->phys = 0;
 380}
 381
 382static int alloc_dma_buf(struct ipu_image_convert_priv *priv,
 383			 struct ipu_image_convert_dma_buf *buf,
 384			 int size)
 385{
 386	buf->len = PAGE_ALIGN(size);
 387	buf->virt = dma_alloc_coherent(priv->ipu->dev, buf->len, &buf->phys,
 388				       GFP_DMA | GFP_KERNEL);
 389	if (!buf->virt) {
 390		dev_err(priv->ipu->dev, "failed to alloc dma buffer\n");
 391		return -ENOMEM;
 392	}
 393
 394	return 0;
 395}
 396
 397static inline int num_stripes(int dim)
 398{
 399	return (dim - 1) / 1024 + 1;
 400}
 401
 402/*
 403 * Calculate downsizing coefficients, which are the same for all tiles,
 404 * and initial bilinear resizing coefficients, which are used to find the
 405 * best seam positions.
 406 * Also determine the number of tiles necessary to guarantee that no tile
 407 * is larger than 1024 pixels in either dimension at the output and between
 408 * IC downsizing and main processing sections.
 409 */
 410static int calc_image_resize_coefficients(struct ipu_image_convert_ctx *ctx,
 411					  struct ipu_image *in,
 412					  struct ipu_image *out)
 413{
 414	u32 downsized_width = in->rect.width;
 415	u32 downsized_height = in->rect.height;
 416	u32 downsize_coeff_v = 0;
 417	u32 downsize_coeff_h = 0;
 418	u32 resized_width = out->rect.width;
 419	u32 resized_height = out->rect.height;
 420	u32 resize_coeff_h;
 421	u32 resize_coeff_v;
 422	u32 cols;
 423	u32 rows;
 424
 425	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
 426		resized_width = out->rect.height;
 427		resized_height = out->rect.width;
 428	}
 429
 430	/* Do not let invalid input lead to an endless loop below */
 431	if (WARN_ON(resized_width == 0 || resized_height == 0))
 432		return -EINVAL;
 433
 434	while (downsized_width >= resized_width * 2) {
 435		downsized_width >>= 1;
 436		downsize_coeff_h++;
 437	}
 438
 439	while (downsized_height >= resized_height * 2) {
 440		downsized_height >>= 1;
 441		downsize_coeff_v++;
 442	}
 443
 444	/*
 445	 * Calculate the bilinear resizing coefficients that could be used if
 446	 * we were converting with a single tile. The bottom right output pixel
 447	 * should sample as close as possible to the bottom right input pixel
 448	 * out of the decimator, but not overshoot it:
 449	 */
 450	resize_coeff_h = 8192 * (downsized_width - 1) / (resized_width - 1);
 451	resize_coeff_v = 8192 * (downsized_height - 1) / (resized_height - 1);
 452
 453	/*
 454	 * Both the output of the IC downsizing section before being passed to
 455	 * the IC main processing section and the final output of the IC main
 456	 * processing section must be <= 1024 pixels in both dimensions.
 457	 */
 458	cols = num_stripes(max_t(u32, downsized_width, resized_width));
 459	rows = num_stripes(max_t(u32, downsized_height, resized_height));
 460
 461	dev_dbg(ctx->chan->priv->ipu->dev,
 462		"%s: hscale: >>%u, *8192/%u vscale: >>%u, *8192/%u, %ux%u tiles\n",
 463		__func__, downsize_coeff_h, resize_coeff_h, downsize_coeff_v,
 464		resize_coeff_v, cols, rows);
 465
 466	if (downsize_coeff_h > 2 || downsize_coeff_v  > 2 ||
 467	    resize_coeff_h > 0x3fff || resize_coeff_v > 0x3fff)
 468		return -EINVAL;
 469
 470	ctx->downsize_coeff_h = downsize_coeff_h;
 471	ctx->downsize_coeff_v = downsize_coeff_v;
 472	ctx->image_resize_coeff_h = resize_coeff_h;
 473	ctx->image_resize_coeff_v = resize_coeff_v;
 474	ctx->in.num_cols = cols;
 475	ctx->in.num_rows = rows;
 476
 477	return 0;
 478}
 479
 480#define round_closest(x, y) round_down((x) + (y)/2, (y))
 481
 482/*
 483 * Find the best aligned seam position for the given column / row index.
 484 * Rotation and image offsets are out of scope.
 485 *
 486 * @index: column / row index, used to calculate valid interval
 487 * @in_edge: input right / bottom edge
 488 * @out_edge: output right / bottom edge
 489 * @in_align: input alignment, either horizontal 8-byte line start address
 490 *            alignment, or pixel alignment due to image format
 491 * @out_align: output alignment, either horizontal 8-byte line start address
 492 *             alignment, or pixel alignment due to image format or rotator
 493 *             block size
 494 * @in_burst: horizontal input burst size in case of horizontal flip
 495 * @out_burst: horizontal output burst size or rotator block size
 496 * @downsize_coeff: downsizing section coefficient
 497 * @resize_coeff: main processing section resizing coefficient
 498 * @_in_seam: aligned input seam position return value
 499 * @_out_seam: aligned output seam position return value
 500 */
 501static void find_best_seam(struct ipu_image_convert_ctx *ctx,
 502			   unsigned int index,
 503			   unsigned int in_edge,
 504			   unsigned int out_edge,
 505			   unsigned int in_align,
 506			   unsigned int out_align,
 507			   unsigned int in_burst,
 508			   unsigned int out_burst,
 509			   unsigned int downsize_coeff,
 510			   unsigned int resize_coeff,
 511			   u32 *_in_seam,
 512			   u32 *_out_seam)
 513{
 514	struct device *dev = ctx->chan->priv->ipu->dev;
 515	unsigned int out_pos;
 516	/* Input / output seam position candidates */
 517	unsigned int out_seam = 0;
 518	unsigned int in_seam = 0;
 519	unsigned int min_diff = UINT_MAX;
 520	unsigned int out_start;
 521	unsigned int out_end;
 522	unsigned int in_start;
 523	unsigned int in_end;
 524
 525	/* Start within 1024 pixels of the right / bottom edge */
 526	out_start = max_t(int, index * out_align, out_edge - 1024);
 527	/* End before having to add more columns to the left / rows above */
 528	out_end = min_t(unsigned int, out_edge, index * 1024 + 1);
 529
 530	/*
 531	 * Limit input seam position to make sure that the downsized input tile
 532	 * to the right or bottom does not exceed 1024 pixels.
 533	 */
 534	in_start = max_t(int, index * in_align,
 535			 in_edge - (1024 << downsize_coeff));
 536	in_end = min_t(unsigned int, in_edge,
 537		       index * (1024 << downsize_coeff) + 1);
 538
 539	/*
 540	 * Output tiles must start at a multiple of 8 bytes horizontally and
 541	 * possibly at an even line horizontally depending on the pixel format.
 542	 * Only consider output aligned positions for the seam.
 543	 */
 544	out_start = round_up(out_start, out_align);
 545	for (out_pos = out_start; out_pos < out_end; out_pos += out_align) {
 546		unsigned int in_pos;
 547		unsigned int in_pos_aligned;
 548		unsigned int in_pos_rounded;
 549		unsigned int diff;
 550
 551		/*
 552		 * Tiles in the right row / bottom column may not be allowed to
 553		 * overshoot horizontally / vertically. out_burst may be the
 554		 * actual DMA burst size, or the rotator block size.
 555		 */
 556		if ((out_burst > 1) && (out_edge - out_pos) % out_burst)
 557			continue;
 558
 559		/*
 560		 * Input sample position, corresponding to out_pos, 19.13 fixed
 561		 * point.
 562		 */
 563		in_pos = (out_pos * resize_coeff) << downsize_coeff;
 564		/*
 565		 * The closest input sample position that we could actually
 566		 * start the input tile at, 19.13 fixed point.
 567		 */
 568		in_pos_aligned = round_closest(in_pos, 8192U * in_align);
 569		/* Convert 19.13 fixed point to integer */
 570		in_pos_rounded = in_pos_aligned / 8192U;
 571
 572		if (in_pos_rounded < in_start)
 573			continue;
 574		if (in_pos_rounded >= in_end)
 575			break;
 576
 577		if ((in_burst > 1) &&
 578		    (in_edge - in_pos_rounded) % in_burst)
 579			continue;
 580
 581		diff = abs_diff(in_pos, in_pos_aligned);
 582		if (diff < min_diff) {
 583			in_seam = in_pos_rounded;
 584			out_seam = out_pos;
 585			min_diff = diff;
 586		}
 587	}
 588
 589	*_out_seam = out_seam;
 590	*_in_seam = in_seam;
 591
 592	dev_dbg(dev, "%s: out_seam %u(%u) in [%u, %u], in_seam %u(%u) in [%u, %u] diff %u.%03u\n",
 593		__func__, out_seam, out_align, out_start, out_end,
 594		in_seam, in_align, in_start, in_end, min_diff / 8192,
 595		DIV_ROUND_CLOSEST(min_diff % 8192 * 1000, 8192));
 596}
 597
 598/*
 599 * Tile left edges are required to be aligned to multiples of 8 bytes
 600 * by the IDMAC.
 601 */
 602static inline u32 tile_left_align(const struct ipu_image_pixfmt *fmt)
 603{
 604	if (fmt->planar)
 605		return fmt->uv_packed ? 8 : 8 * fmt->uv_width_dec;
 606	else
 607		return fmt->bpp == 32 ? 2 : fmt->bpp == 16 ? 4 : 8;
 608}
 609
 610/*
 611 * Tile top edge alignment is only limited by chroma subsampling.
 612 */
 613static inline u32 tile_top_align(const struct ipu_image_pixfmt *fmt)
 614{
 615	return fmt->uv_height_dec > 1 ? 2 : 1;
 616}
 617
 618static inline u32 tile_width_align(enum ipu_image_convert_type type,
 619				   const struct ipu_image_pixfmt *fmt,
 620				   enum ipu_rotate_mode rot_mode)
 621{
 622	if (type == IMAGE_CONVERT_IN) {
 623		/*
 624		 * The IC burst reads 8 pixels at a time. Reading beyond the
 625		 * end of the line is usually acceptable. Those pixels are
 626		 * ignored, unless the IC has to write the scaled line in
 627		 * reverse.
 628		 */
 629		return (!ipu_rot_mode_is_irt(rot_mode) &&
 630			(rot_mode & IPU_ROT_BIT_HFLIP)) ? 8 : 2;
 631	}
 632
 633	/*
 634	 * Align to 16x16 pixel blocks for planar 4:2:0 chroma subsampled
 635	 * formats to guarantee 8-byte aligned line start addresses in the
 636	 * chroma planes when IRT is used. Align to 8x8 pixel IRT block size
 637	 * for all other formats.
 638	 */
 639	return (ipu_rot_mode_is_irt(rot_mode) &&
 640		fmt->planar && !fmt->uv_packed) ?
 641		8 * fmt->uv_width_dec : 8;
 642}
 643
 644static inline u32 tile_height_align(enum ipu_image_convert_type type,
 645				    const struct ipu_image_pixfmt *fmt,
 646				    enum ipu_rotate_mode rot_mode)
 647{
 648	if (type == IMAGE_CONVERT_IN || !ipu_rot_mode_is_irt(rot_mode))
 649		return 2;
 650
 651	/*
 652	 * Align to 16x16 pixel blocks for planar 4:2:0 chroma subsampled
 653	 * formats to guarantee 8-byte aligned line start addresses in the
 654	 * chroma planes when IRT is used. Align to 8x8 pixel IRT block size
 655	 * for all other formats.
 656	 */
 657	return (fmt->planar && !fmt->uv_packed) ? 8 * fmt->uv_width_dec : 8;
 658}
 659
 660/*
 661 * Fill in left position and width and for all tiles in an input column, and
 662 * for all corresponding output tiles. If the 90° rotator is used, the output
 663 * tiles are in a row, and output tile top position and height are set.
 664 */
 665static void fill_tile_column(struct ipu_image_convert_ctx *ctx,
 666			     unsigned int col,
 667			     struct ipu_image_convert_image *in,
 668			     unsigned int in_left, unsigned int in_width,
 669			     struct ipu_image_convert_image *out,
 670			     unsigned int out_left, unsigned int out_width)
 671{
 672	unsigned int row, tile_idx;
 673	struct ipu_image_tile *in_tile, *out_tile;
 674
 675	for (row = 0; row < in->num_rows; row++) {
 676		tile_idx = in->num_cols * row + col;
 677		in_tile = &in->tile[tile_idx];
 678		out_tile = &out->tile[ctx->out_tile_map[tile_idx]];
 679
 680		in_tile->left = in_left;
 681		in_tile->width = in_width;
 682
 683		if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
 684			out_tile->top = out_left;
 685			out_tile->height = out_width;
 686		} else {
 687			out_tile->left = out_left;
 688			out_tile->width = out_width;
 689		}
 690	}
 691}
 692
 693/*
 694 * Fill in top position and height and for all tiles in an input row, and
 695 * for all corresponding output tiles. If the 90° rotator is used, the output
 696 * tiles are in a column, and output tile left position and width are set.
 697 */
 698static void fill_tile_row(struct ipu_image_convert_ctx *ctx, unsigned int row,
 699			  struct ipu_image_convert_image *in,
 700			  unsigned int in_top, unsigned int in_height,
 701			  struct ipu_image_convert_image *out,
 702			  unsigned int out_top, unsigned int out_height)
 703{
 704	unsigned int col, tile_idx;
 705	struct ipu_image_tile *in_tile, *out_tile;
 706
 707	for (col = 0; col < in->num_cols; col++) {
 708		tile_idx = in->num_cols * row + col;
 709		in_tile = &in->tile[tile_idx];
 710		out_tile = &out->tile[ctx->out_tile_map[tile_idx]];
 711
 712		in_tile->top = in_top;
 713		in_tile->height = in_height;
 714
 715		if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
 716			out_tile->left = out_top;
 717			out_tile->width = out_height;
 718		} else {
 719			out_tile->top = out_top;
 720			out_tile->height = out_height;
 721		}
 722	}
 723}
 724
 725/*
 726 * Find the best horizontal and vertical seam positions to split into tiles.
 727 * Minimize the fractional part of the input sampling position for the
 728 * top / left pixels of each tile.
 729 */
 730static void find_seams(struct ipu_image_convert_ctx *ctx,
 731		       struct ipu_image_convert_image *in,
 732		       struct ipu_image_convert_image *out)
 733{
 734	struct device *dev = ctx->chan->priv->ipu->dev;
 735	unsigned int resized_width = out->base.rect.width;
 736	unsigned int resized_height = out->base.rect.height;
 737	unsigned int col;
 738	unsigned int row;
 739	unsigned int in_left_align = tile_left_align(in->fmt);
 740	unsigned int in_top_align = tile_top_align(in->fmt);
 741	unsigned int out_left_align = tile_left_align(out->fmt);
 742	unsigned int out_top_align = tile_top_align(out->fmt);
 743	unsigned int out_width_align = tile_width_align(out->type, out->fmt,
 744							ctx->rot_mode);
 745	unsigned int out_height_align = tile_height_align(out->type, out->fmt,
 746							  ctx->rot_mode);
 747	unsigned int in_right = in->base.rect.width;
 748	unsigned int in_bottom = in->base.rect.height;
 749	unsigned int out_right = out->base.rect.width;
 750	unsigned int out_bottom = out->base.rect.height;
 751	unsigned int flipped_out_left;
 752	unsigned int flipped_out_top;
 753
 754	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
 755		/* Switch width/height and align top left to IRT block size */
 756		resized_width = out->base.rect.height;
 757		resized_height = out->base.rect.width;
 758		out_left_align = out_height_align;
 759		out_top_align = out_width_align;
 760		out_width_align = out_left_align;
 761		out_height_align = out_top_align;
 762		out_right = out->base.rect.height;
 763		out_bottom = out->base.rect.width;
 764	}
 765
 766	for (col = in->num_cols - 1; col > 0; col--) {
 767		bool allow_in_overshoot = ipu_rot_mode_is_irt(ctx->rot_mode) ||
 768					  !(ctx->rot_mode & IPU_ROT_BIT_HFLIP);
 769		bool allow_out_overshoot = (col < in->num_cols - 1) &&
 770					   !(ctx->rot_mode & IPU_ROT_BIT_HFLIP);
 771		unsigned int in_left;
 772		unsigned int out_left;
 773
 774		/*
 775		 * Align input width to burst length if the scaling step flips
 776		 * horizontally.
 777		 */
 778
 779		find_best_seam(ctx, col,
 780			       in_right, out_right,
 781			       in_left_align, out_left_align,
 782			       allow_in_overshoot ? 1 : 8 /* burst length */,
 783			       allow_out_overshoot ? 1 : out_width_align,
 784			       ctx->downsize_coeff_h, ctx->image_resize_coeff_h,
 785			       &in_left, &out_left);
 786
 787		if (ctx->rot_mode & IPU_ROT_BIT_HFLIP)
 788			flipped_out_left = resized_width - out_right;
 789		else
 790			flipped_out_left = out_left;
 791
 792		fill_tile_column(ctx, col, in, in_left, in_right - in_left,
 793				 out, flipped_out_left, out_right - out_left);
 794
 795		dev_dbg(dev, "%s: col %u: %u, %u -> %u, %u\n", __func__, col,
 796			in_left, in_right - in_left,
 797			flipped_out_left, out_right - out_left);
 798
 799		in_right = in_left;
 800		out_right = out_left;
 801	}
 802
 803	flipped_out_left = (ctx->rot_mode & IPU_ROT_BIT_HFLIP) ?
 804			   resized_width - out_right : 0;
 805
 806	fill_tile_column(ctx, 0, in, 0, in_right,
 807			 out, flipped_out_left, out_right);
 808
 809	dev_dbg(dev, "%s: col 0: 0, %u -> %u, %u\n", __func__,
 810		in_right, flipped_out_left, out_right);
 811
 812	for (row = in->num_rows - 1; row > 0; row--) {
 813		bool allow_overshoot = row < in->num_rows - 1;
 814		unsigned int in_top;
 815		unsigned int out_top;
 816
 817		find_best_seam(ctx, row,
 818			       in_bottom, out_bottom,
 819			       in_top_align, out_top_align,
 820			       1, allow_overshoot ? 1 : out_height_align,
 821			       ctx->downsize_coeff_v, ctx->image_resize_coeff_v,
 822			       &in_top, &out_top);
 823
 824		if ((ctx->rot_mode & IPU_ROT_BIT_VFLIP) ^
 825		    ipu_rot_mode_is_irt(ctx->rot_mode))
 826			flipped_out_top = resized_height - out_bottom;
 827		else
 828			flipped_out_top = out_top;
 829
 830		fill_tile_row(ctx, row, in, in_top, in_bottom - in_top,
 831			      out, flipped_out_top, out_bottom - out_top);
 832
 833		dev_dbg(dev, "%s: row %u: %u, %u -> %u, %u\n", __func__, row,
 834			in_top, in_bottom - in_top,
 835			flipped_out_top, out_bottom - out_top);
 836
 837		in_bottom = in_top;
 838		out_bottom = out_top;
 839	}
 840
 841	if ((ctx->rot_mode & IPU_ROT_BIT_VFLIP) ^
 842	    ipu_rot_mode_is_irt(ctx->rot_mode))
 843		flipped_out_top = resized_height - out_bottom;
 844	else
 845		flipped_out_top = 0;
 846
 847	fill_tile_row(ctx, 0, in, 0, in_bottom,
 848		      out, flipped_out_top, out_bottom);
 849
 850	dev_dbg(dev, "%s: row 0: 0, %u -> %u, %u\n", __func__,
 851		in_bottom, flipped_out_top, out_bottom);
 852}
 853
 854static int calc_tile_dimensions(struct ipu_image_convert_ctx *ctx,
 855				struct ipu_image_convert_image *image)
 856{
 857	struct ipu_image_convert_chan *chan = ctx->chan;
 858	struct ipu_image_convert_priv *priv = chan->priv;
 859	unsigned int max_width = 1024;
 860	unsigned int max_height = 1024;
 861	unsigned int i;
 862
 863	if (image->type == IMAGE_CONVERT_IN) {
 864		/* Up to 4096x4096 input tile size */
 865		max_width <<= ctx->downsize_coeff_h;
 866		max_height <<= ctx->downsize_coeff_v;
 867	}
 868
 869	for (i = 0; i < ctx->num_tiles; i++) {
 870		struct ipu_image_tile *tile;
 871		const unsigned int row = i / image->num_cols;
 872		const unsigned int col = i % image->num_cols;
 873
 874		if (image->type == IMAGE_CONVERT_OUT)
 875			tile = &image->tile[ctx->out_tile_map[i]];
 876		else
 877			tile = &image->tile[i];
 878
 
 
 879		tile->size = ((tile->height * image->fmt->bpp) >> 3) *
 880			tile->width;
 881
 882		if (image->fmt->planar) {
 883			tile->stride = tile->width;
 884			tile->rot_stride = tile->height;
 885		} else {
 886			tile->stride =
 887				(image->fmt->bpp * tile->width) >> 3;
 888			tile->rot_stride =
 889				(image->fmt->bpp * tile->height) >> 3;
 890		}
 891
 892		dev_dbg(priv->ipu->dev,
 893			"task %u: ctx %p: %s@[%u,%u]: %ux%u@%u,%u\n",
 894			chan->ic_task, ctx,
 895			image->type == IMAGE_CONVERT_IN ? "Input" : "Output",
 896			row, col,
 897			tile->width, tile->height, tile->left, tile->top);
 898
 899		if (!tile->width || tile->width > max_width ||
 900		    !tile->height || tile->height > max_height) {
 901			dev_err(priv->ipu->dev, "invalid %s tile size: %ux%u\n",
 902				image->type == IMAGE_CONVERT_IN ? "input" :
 903				"output", tile->width, tile->height);
 904			return -EINVAL;
 905		}
 906	}
 907
 908	return 0;
 909}
 910
 911/*
 912 * Use the rotation transformation to find the tile coordinates
 913 * (row, col) of a tile in the destination frame that corresponds
 914 * to the given tile coordinates of a source frame. The destination
 915 * coordinate is then converted to a tile index.
 916 */
 917static int transform_tile_index(struct ipu_image_convert_ctx *ctx,
 918				int src_row, int src_col)
 919{
 920	struct ipu_image_convert_chan *chan = ctx->chan;
 921	struct ipu_image_convert_priv *priv = chan->priv;
 922	struct ipu_image_convert_image *s_image = &ctx->in;
 923	struct ipu_image_convert_image *d_image = &ctx->out;
 924	int dst_row, dst_col;
 925
 926	/* with no rotation it's a 1:1 mapping */
 927	if (ctx->rot_mode == IPU_ROTATE_NONE)
 928		return src_row * s_image->num_cols + src_col;
 929
 930	/*
 931	 * before doing the transform, first we have to translate
 932	 * source row,col for an origin in the center of s_image
 933	 */
 934	src_row = src_row * 2 - (s_image->num_rows - 1);
 935	src_col = src_col * 2 - (s_image->num_cols - 1);
 936
 937	/* do the rotation transform */
 938	if (ctx->rot_mode & IPU_ROT_BIT_90) {
 939		dst_col = -src_row;
 940		dst_row = src_col;
 941	} else {
 942		dst_col = src_col;
 943		dst_row = src_row;
 944	}
 945
 946	/* apply flip */
 947	if (ctx->rot_mode & IPU_ROT_BIT_HFLIP)
 948		dst_col = -dst_col;
 949	if (ctx->rot_mode & IPU_ROT_BIT_VFLIP)
 950		dst_row = -dst_row;
 951
 952	dev_dbg(priv->ipu->dev, "task %u: ctx %p: [%d,%d] --> [%d,%d]\n",
 953		chan->ic_task, ctx, src_col, src_row, dst_col, dst_row);
 954
 955	/*
 956	 * finally translate dest row,col using an origin in upper
 957	 * left of d_image
 958	 */
 959	dst_row += d_image->num_rows - 1;
 960	dst_col += d_image->num_cols - 1;
 961	dst_row /= 2;
 962	dst_col /= 2;
 963
 964	return dst_row * d_image->num_cols + dst_col;
 965}
 966
 967/*
 968 * Fill the out_tile_map[] with transformed destination tile indeces.
 969 */
 970static void calc_out_tile_map(struct ipu_image_convert_ctx *ctx)
 971{
 972	struct ipu_image_convert_image *s_image = &ctx->in;
 973	unsigned int row, col, tile = 0;
 974
 975	for (row = 0; row < s_image->num_rows; row++) {
 976		for (col = 0; col < s_image->num_cols; col++) {
 977			ctx->out_tile_map[tile] =
 978				transform_tile_index(ctx, row, col);
 979			tile++;
 980		}
 981	}
 982}
 983
 984static int calc_tile_offsets_planar(struct ipu_image_convert_ctx *ctx,
 985				    struct ipu_image_convert_image *image)
 986{
 987	struct ipu_image_convert_chan *chan = ctx->chan;
 988	struct ipu_image_convert_priv *priv = chan->priv;
 989	const struct ipu_image_pixfmt *fmt = image->fmt;
 990	unsigned int row, col, tile = 0;
 991	u32 H, top, y_stride, uv_stride;
 992	u32 uv_row_off, uv_col_off, uv_off, u_off, v_off;
 993	u32 y_row_off, y_col_off, y_off;
 994	u32 y_size, uv_size;
 995
 996	/* setup some convenience vars */
 997	H = image->base.pix.height;
 998
 999	y_stride = image->stride;
1000	uv_stride = y_stride / fmt->uv_width_dec;
1001	if (fmt->uv_packed)
1002		uv_stride *= 2;
1003
1004	y_size = H * y_stride;
1005	uv_size = y_size / (fmt->uv_width_dec * fmt->uv_height_dec);
1006
1007	for (row = 0; row < image->num_rows; row++) {
1008		top = image->tile[tile].top;
1009		y_row_off = top * y_stride;
1010		uv_row_off = (top * uv_stride) / fmt->uv_height_dec;
 
1011
1012		for (col = 0; col < image->num_cols; col++) {
1013			y_col_off = image->tile[tile].left;
1014			uv_col_off = y_col_off / fmt->uv_width_dec;
1015			if (fmt->uv_packed)
1016				uv_col_off *= 2;
1017
1018			y_off = y_row_off + y_col_off;
1019			uv_off = uv_row_off + uv_col_off;
1020
1021			u_off = y_size - y_off + uv_off;
1022			v_off = (fmt->uv_packed) ? 0 : u_off + uv_size;
1023			if (fmt->uv_swapped)
1024				swap(u_off, v_off);
 
 
 
1025
1026			image->tile[tile].offset = y_off;
1027			image->tile[tile].u_off = u_off;
1028			image->tile[tile++].v_off = v_off;
1029
1030			if ((y_off & 0x7) || (u_off & 0x7) || (v_off & 0x7)) {
1031				dev_err(priv->ipu->dev,
1032					"task %u: ctx %p: %s@[%d,%d]: "
1033					"y_off %08x, u_off %08x, v_off %08x\n",
1034					chan->ic_task, ctx,
1035					image->type == IMAGE_CONVERT_IN ?
1036					"Input" : "Output", row, col,
1037					y_off, u_off, v_off);
1038				return -EINVAL;
1039			}
1040		}
1041	}
1042
1043	return 0;
1044}
1045
1046static int calc_tile_offsets_packed(struct ipu_image_convert_ctx *ctx,
1047				    struct ipu_image_convert_image *image)
1048{
1049	struct ipu_image_convert_chan *chan = ctx->chan;
1050	struct ipu_image_convert_priv *priv = chan->priv;
1051	const struct ipu_image_pixfmt *fmt = image->fmt;
1052	unsigned int row, col, tile = 0;
1053	u32 bpp, stride, offset;
1054	u32 row_off, col_off;
1055
1056	/* setup some convenience vars */
1057	stride = image->stride;
1058	bpp = fmt->bpp;
1059
1060	for (row = 0; row < image->num_rows; row++) {
1061		row_off = image->tile[tile].top * stride;
 
 
1062
1063		for (col = 0; col < image->num_cols; col++) {
1064			col_off = (image->tile[tile].left * bpp) >> 3;
1065
1066			offset = row_off + col_off;
1067
1068			image->tile[tile].offset = offset;
1069			image->tile[tile].u_off = 0;
1070			image->tile[tile++].v_off = 0;
1071
1072			if (offset & 0x7) {
1073				dev_err(priv->ipu->dev,
1074					"task %u: ctx %p: %s@[%d,%d]: "
1075					"phys %08x\n",
1076					chan->ic_task, ctx,
1077					image->type == IMAGE_CONVERT_IN ?
1078					"Input" : "Output", row, col,
1079					row_off + col_off);
1080				return -EINVAL;
1081			}
1082		}
1083	}
1084
1085	return 0;
1086}
1087
1088static int calc_tile_offsets(struct ipu_image_convert_ctx *ctx,
1089			      struct ipu_image_convert_image *image)
1090{
1091	if (image->fmt->planar)
1092		return calc_tile_offsets_planar(ctx, image);
1093
1094	return calc_tile_offsets_packed(ctx, image);
1095}
1096
1097/*
1098 * Calculate the resizing ratio for the IC main processing section given input
1099 * size, fixed downsizing coefficient, and output size.
1100 * Either round to closest for the next tile's first pixel to minimize seams
1101 * and distortion (for all but right column / bottom row), or round down to
1102 * avoid sampling beyond the edges of the input image for this tile's last
1103 * pixel.
1104 * Returns the resizing coefficient, resizing ratio is 8192.0 / resize_coeff.
1105 */
1106static u32 calc_resize_coeff(u32 input_size, u32 downsize_coeff,
1107			     u32 output_size, bool allow_overshoot)
1108{
1109	u32 downsized = input_size >> downsize_coeff;
1110
1111	if (allow_overshoot)
1112		return DIV_ROUND_CLOSEST(8192 * downsized, output_size);
1113	else
1114		return 8192 * (downsized - 1) / (output_size - 1);
1115}
1116
1117/*
1118 * Slightly modify resize coefficients per tile to hide the bilinear
1119 * interpolator reset at tile borders, shifting the right / bottom edge
1120 * by up to a half input pixel. This removes noticeable seams between
1121 * tiles at higher upscaling factors.
1122 */
1123static void calc_tile_resize_coefficients(struct ipu_image_convert_ctx *ctx)
1124{
1125	struct ipu_image_convert_chan *chan = ctx->chan;
1126	struct ipu_image_convert_priv *priv = chan->priv;
1127	struct ipu_image_tile *in_tile, *out_tile;
1128	unsigned int col, row, tile_idx;
1129	unsigned int last_output;
1130
1131	for (col = 0; col < ctx->in.num_cols; col++) {
1132		bool closest = (col < ctx->in.num_cols - 1) &&
1133			       !(ctx->rot_mode & IPU_ROT_BIT_HFLIP);
1134		u32 resized_width;
1135		u32 resize_coeff_h;
1136		u32 in_width;
1137
1138		tile_idx = col;
1139		in_tile = &ctx->in.tile[tile_idx];
1140		out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]];
1141
1142		if (ipu_rot_mode_is_irt(ctx->rot_mode))
1143			resized_width = out_tile->height;
1144		else
1145			resized_width = out_tile->width;
1146
1147		resize_coeff_h = calc_resize_coeff(in_tile->width,
1148						   ctx->downsize_coeff_h,
1149						   resized_width, closest);
1150
1151		dev_dbg(priv->ipu->dev, "%s: column %u hscale: *8192/%u\n",
1152			__func__, col, resize_coeff_h);
1153
1154		/*
1155		 * With the horizontal scaling factor known, round up resized
1156		 * width (output width or height) to burst size.
1157		 */
1158		resized_width = round_up(resized_width, 8);
1159
1160		/*
1161		 * Calculate input width from the last accessed input pixel
1162		 * given resized width and scaling coefficients. Round up to
1163		 * burst size.
1164		 */
1165		last_output = resized_width - 1;
1166		if (closest && ((last_output * resize_coeff_h) % 8192))
1167			last_output++;
1168		in_width = round_up(
1169			(DIV_ROUND_UP(last_output * resize_coeff_h, 8192) + 1)
1170			<< ctx->downsize_coeff_h, 8);
1171
1172		for (row = 0; row < ctx->in.num_rows; row++) {
1173			tile_idx = row * ctx->in.num_cols + col;
1174			in_tile = &ctx->in.tile[tile_idx];
1175			out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]];
1176
1177			if (ipu_rot_mode_is_irt(ctx->rot_mode))
1178				out_tile->height = resized_width;
1179			else
1180				out_tile->width = resized_width;
1181
1182			in_tile->width = in_width;
1183		}
1184
1185		ctx->resize_coeffs_h[col] = resize_coeff_h;
1186	}
1187
1188	for (row = 0; row < ctx->in.num_rows; row++) {
1189		bool closest = (row < ctx->in.num_rows - 1) &&
1190			       !(ctx->rot_mode & IPU_ROT_BIT_VFLIP);
1191		u32 resized_height;
1192		u32 resize_coeff_v;
1193		u32 in_height;
1194
1195		tile_idx = row * ctx->in.num_cols;
1196		in_tile = &ctx->in.tile[tile_idx];
1197		out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]];
1198
1199		if (ipu_rot_mode_is_irt(ctx->rot_mode))
1200			resized_height = out_tile->width;
1201		else
1202			resized_height = out_tile->height;
1203
1204		resize_coeff_v = calc_resize_coeff(in_tile->height,
1205						   ctx->downsize_coeff_v,
1206						   resized_height, closest);
1207
1208		dev_dbg(priv->ipu->dev, "%s: row %u vscale: *8192/%u\n",
1209			__func__, row, resize_coeff_v);
1210
1211		/*
1212		 * With the vertical scaling factor known, round up resized
1213		 * height (output width or height) to IDMAC limitations.
1214		 */
1215		resized_height = round_up(resized_height, 2);
1216
1217		/*
1218		 * Calculate input width from the last accessed input pixel
1219		 * given resized height and scaling coefficients. Align to
1220		 * IDMAC restrictions.
1221		 */
1222		last_output = resized_height - 1;
1223		if (closest && ((last_output * resize_coeff_v) % 8192))
1224			last_output++;
1225		in_height = round_up(
1226			(DIV_ROUND_UP(last_output * resize_coeff_v, 8192) + 1)
1227			<< ctx->downsize_coeff_v, 2);
1228
1229		for (col = 0; col < ctx->in.num_cols; col++) {
1230			tile_idx = row * ctx->in.num_cols + col;
1231			in_tile = &ctx->in.tile[tile_idx];
1232			out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]];
1233
1234			if (ipu_rot_mode_is_irt(ctx->rot_mode))
1235				out_tile->width = resized_height;
1236			else
1237				out_tile->height = resized_height;
1238
1239			in_tile->height = in_height;
1240		}
1241
1242		ctx->resize_coeffs_v[row] = resize_coeff_v;
1243	}
1244}
1245
1246/*
1247 * return the number of runs in given queue (pending_q or done_q)
1248 * for this context. hold irqlock when calling.
1249 */
1250static int get_run_count(struct ipu_image_convert_ctx *ctx,
1251			 struct list_head *q)
1252{
1253	struct ipu_image_convert_run *run;
1254	int count = 0;
1255
1256	lockdep_assert_held(&ctx->chan->irqlock);
1257
1258	list_for_each_entry(run, q, list) {
1259		if (run->ctx == ctx)
1260			count++;
1261	}
1262
1263	return count;
1264}
1265
1266static void convert_stop(struct ipu_image_convert_run *run)
1267{
1268	struct ipu_image_convert_ctx *ctx = run->ctx;
1269	struct ipu_image_convert_chan *chan = ctx->chan;
1270	struct ipu_image_convert_priv *priv = chan->priv;
1271
1272	dev_dbg(priv->ipu->dev, "%s: task %u: stopping ctx %p run %p\n",
1273		__func__, chan->ic_task, ctx, run);
1274
1275	/* disable IC tasks and the channels */
1276	ipu_ic_task_disable(chan->ic);
1277	ipu_idmac_disable_channel(chan->in_chan);
1278	ipu_idmac_disable_channel(chan->out_chan);
1279
1280	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
1281		ipu_idmac_disable_channel(chan->rotation_in_chan);
1282		ipu_idmac_disable_channel(chan->rotation_out_chan);
1283		ipu_idmac_unlink(chan->out_chan, chan->rotation_in_chan);
1284	}
1285
1286	ipu_ic_disable(chan->ic);
1287}
1288
1289static void init_idmac_channel(struct ipu_image_convert_ctx *ctx,
1290			       struct ipuv3_channel *channel,
1291			       struct ipu_image_convert_image *image,
1292			       enum ipu_rotate_mode rot_mode,
1293			       bool rot_swap_width_height,
1294			       unsigned int tile)
1295{
1296	struct ipu_image_convert_chan *chan = ctx->chan;
1297	unsigned int burst_size;
1298	u32 width, height, stride;
1299	dma_addr_t addr0, addr1 = 0;
1300	struct ipu_image tile_image;
1301	unsigned int tile_idx[2];
1302
1303	if (image->type == IMAGE_CONVERT_OUT) {
1304		tile_idx[0] = ctx->out_tile_map[tile];
1305		tile_idx[1] = ctx->out_tile_map[1];
1306	} else {
1307		tile_idx[0] = tile;
1308		tile_idx[1] = 1;
1309	}
1310
1311	if (rot_swap_width_height) {
1312		width = image->tile[tile_idx[0]].height;
1313		height = image->tile[tile_idx[0]].width;
1314		stride = image->tile[tile_idx[0]].rot_stride;
1315		addr0 = ctx->rot_intermediate[0].phys;
1316		if (ctx->double_buffering)
1317			addr1 = ctx->rot_intermediate[1].phys;
1318	} else {
1319		width = image->tile[tile_idx[0]].width;
1320		height = image->tile[tile_idx[0]].height;
1321		stride = image->stride;
1322		addr0 = image->base.phys0 +
1323			image->tile[tile_idx[0]].offset;
1324		if (ctx->double_buffering)
1325			addr1 = image->base.phys0 +
1326				image->tile[tile_idx[1]].offset;
1327	}
1328
1329	ipu_cpmem_zero(channel);
1330
1331	memset(&tile_image, 0, sizeof(tile_image));
1332	tile_image.pix.width = tile_image.rect.width = width;
1333	tile_image.pix.height = tile_image.rect.height = height;
1334	tile_image.pix.bytesperline = stride;
1335	tile_image.pix.pixelformat =  image->fmt->fourcc;
1336	tile_image.phys0 = addr0;
1337	tile_image.phys1 = addr1;
1338	if (image->fmt->planar && !rot_swap_width_height) {
1339		tile_image.u_offset = image->tile[tile_idx[0]].u_off;
1340		tile_image.v_offset = image->tile[tile_idx[0]].v_off;
1341	}
1342
1343	ipu_cpmem_set_image(channel, &tile_image);
1344
 
 
 
 
 
1345	if (rot_mode)
1346		ipu_cpmem_set_rotation(channel, rot_mode);
1347
1348	/*
1349	 * Skip writing U and V components to odd rows in the output
1350	 * channels for planar 4:2:0.
1351	 */
1352	if ((channel == chan->out_chan ||
1353	     channel == chan->rotation_out_chan) &&
1354	    image->fmt->planar && image->fmt->uv_height_dec == 2)
1355		ipu_cpmem_skip_odd_chroma_rows(channel);
1356
1357	if (channel == chan->rotation_in_chan ||
1358	    channel == chan->rotation_out_chan) {
1359		burst_size = 8;
1360		ipu_cpmem_set_block_mode(channel);
1361	} else
1362		burst_size = (width % 16) ? 8 : 16;
1363
1364	ipu_cpmem_set_burstsize(channel, burst_size);
1365
1366	ipu_ic_task_idma_init(chan->ic, channel, width, height,
1367			      burst_size, rot_mode);
1368
1369	/*
1370	 * Setting a non-zero AXI ID collides with the PRG AXI snooping, so
1371	 * only do this when there is no PRG present.
1372	 */
1373	if (!channel->ipu->prg_priv)
1374		ipu_cpmem_set_axi_id(channel, 1);
1375
1376	ipu_idmac_set_double_buffer(channel, ctx->double_buffering);
1377}
1378
1379static int convert_start(struct ipu_image_convert_run *run, unsigned int tile)
1380{
1381	struct ipu_image_convert_ctx *ctx = run->ctx;
1382	struct ipu_image_convert_chan *chan = ctx->chan;
1383	struct ipu_image_convert_priv *priv = chan->priv;
1384	struct ipu_image_convert_image *s_image = &ctx->in;
1385	struct ipu_image_convert_image *d_image = &ctx->out;
1386	unsigned int dst_tile = ctx->out_tile_map[tile];
1387	unsigned int dest_width, dest_height;
1388	unsigned int col, row;
1389	u32 rsc;
1390	int ret;
1391
1392	dev_dbg(priv->ipu->dev, "%s: task %u: starting ctx %p run %p tile %u -> %u\n",
1393		__func__, chan->ic_task, ctx, run, tile, dst_tile);
1394
1395	/* clear EOF irq mask */
1396	ctx->eof_mask = 0;
1397
1398	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
1399		/* swap width/height for resizer */
1400		dest_width = d_image->tile[dst_tile].height;
1401		dest_height = d_image->tile[dst_tile].width;
1402	} else {
1403		dest_width = d_image->tile[dst_tile].width;
1404		dest_height = d_image->tile[dst_tile].height;
1405	}
1406
1407	row = tile / s_image->num_cols;
1408	col = tile % s_image->num_cols;
1409
1410	rsc =  (ctx->downsize_coeff_v << 30) |
1411	       (ctx->resize_coeffs_v[row] << 16) |
1412	       (ctx->downsize_coeff_h << 14) |
1413	       (ctx->resize_coeffs_h[col]);
1414
1415	dev_dbg(priv->ipu->dev, "%s: %ux%u -> %ux%u (rsc = 0x%x)\n",
1416		__func__, s_image->tile[tile].width,
1417		s_image->tile[tile].height, dest_width, dest_height, rsc);
1418
1419	/* setup the IC resizer and CSC */
1420	ret = ipu_ic_task_init_rsc(chan->ic, &ctx->csc,
1421				   s_image->tile[tile].width,
1422				   s_image->tile[tile].height,
1423				   dest_width,
1424				   dest_height,
1425				   rsc);
1426	if (ret) {
1427		dev_err(priv->ipu->dev, "ipu_ic_task_init failed, %d\n", ret);
1428		return ret;
1429	}
1430
1431	/* init the source MEM-->IC PP IDMAC channel */
1432	init_idmac_channel(ctx, chan->in_chan, s_image,
1433			   IPU_ROTATE_NONE, false, tile);
1434
1435	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
1436		/* init the IC PP-->MEM IDMAC channel */
1437		init_idmac_channel(ctx, chan->out_chan, d_image,
1438				   IPU_ROTATE_NONE, true, tile);
1439
1440		/* init the MEM-->IC PP ROT IDMAC channel */
1441		init_idmac_channel(ctx, chan->rotation_in_chan, d_image,
1442				   ctx->rot_mode, true, tile);
1443
1444		/* init the destination IC PP ROT-->MEM IDMAC channel */
1445		init_idmac_channel(ctx, chan->rotation_out_chan, d_image,
1446				   IPU_ROTATE_NONE, false, tile);
1447
1448		/* now link IC PP-->MEM to MEM-->IC PP ROT */
1449		ipu_idmac_link(chan->out_chan, chan->rotation_in_chan);
1450	} else {
1451		/* init the destination IC PP-->MEM IDMAC channel */
1452		init_idmac_channel(ctx, chan->out_chan, d_image,
1453				   ctx->rot_mode, false, tile);
1454	}
1455
1456	/* enable the IC */
1457	ipu_ic_enable(chan->ic);
1458
1459	/* set buffers ready */
1460	ipu_idmac_select_buffer(chan->in_chan, 0);
1461	ipu_idmac_select_buffer(chan->out_chan, 0);
1462	if (ipu_rot_mode_is_irt(ctx->rot_mode))
1463		ipu_idmac_select_buffer(chan->rotation_out_chan, 0);
1464	if (ctx->double_buffering) {
1465		ipu_idmac_select_buffer(chan->in_chan, 1);
1466		ipu_idmac_select_buffer(chan->out_chan, 1);
1467		if (ipu_rot_mode_is_irt(ctx->rot_mode))
1468			ipu_idmac_select_buffer(chan->rotation_out_chan, 1);
1469	}
1470
1471	/* enable the channels! */
1472	ipu_idmac_enable_channel(chan->in_chan);
1473	ipu_idmac_enable_channel(chan->out_chan);
1474	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
1475		ipu_idmac_enable_channel(chan->rotation_in_chan);
1476		ipu_idmac_enable_channel(chan->rotation_out_chan);
1477	}
1478
1479	ipu_ic_task_enable(chan->ic);
1480
1481	ipu_cpmem_dump(chan->in_chan);
1482	ipu_cpmem_dump(chan->out_chan);
1483	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
1484		ipu_cpmem_dump(chan->rotation_in_chan);
1485		ipu_cpmem_dump(chan->rotation_out_chan);
1486	}
1487
1488	ipu_dump(priv->ipu);
1489
1490	return 0;
1491}
1492
1493/* hold irqlock when calling */
1494static int do_run(struct ipu_image_convert_run *run)
1495{
1496	struct ipu_image_convert_ctx *ctx = run->ctx;
1497	struct ipu_image_convert_chan *chan = ctx->chan;
1498
1499	lockdep_assert_held(&chan->irqlock);
1500
1501	ctx->in.base.phys0 = run->in_phys;
1502	ctx->out.base.phys0 = run->out_phys;
1503
1504	ctx->cur_buf_num = 0;
1505	ctx->next_tile = 1;
1506
1507	/* remove run from pending_q and set as current */
1508	list_del(&run->list);
1509	chan->current_run = run;
1510
1511	return convert_start(run, 0);
1512}
1513
1514/* hold irqlock when calling */
1515static void run_next(struct ipu_image_convert_chan *chan)
1516{
1517	struct ipu_image_convert_priv *priv = chan->priv;
1518	struct ipu_image_convert_run *run, *tmp;
1519	int ret;
1520
1521	lockdep_assert_held(&chan->irqlock);
1522
1523	list_for_each_entry_safe(run, tmp, &chan->pending_q, list) {
1524		/* skip contexts that are aborting */
1525		if (run->ctx->aborting) {
1526			dev_dbg(priv->ipu->dev,
1527				"%s: task %u: skipping aborting ctx %p run %p\n",
1528				__func__, chan->ic_task, run->ctx, run);
1529			continue;
1530		}
1531
1532		ret = do_run(run);
1533		if (!ret)
1534			break;
1535
1536		/*
1537		 * something went wrong with start, add the run
1538		 * to done q and continue to the next run in the
1539		 * pending q.
1540		 */
1541		run->status = ret;
1542		list_add_tail(&run->list, &chan->done_q);
1543		chan->current_run = NULL;
1544	}
1545}
1546
1547static void empty_done_q(struct ipu_image_convert_chan *chan)
1548{
1549	struct ipu_image_convert_priv *priv = chan->priv;
1550	struct ipu_image_convert_run *run;
1551	unsigned long flags;
1552
1553	spin_lock_irqsave(&chan->irqlock, flags);
1554
1555	while (!list_empty(&chan->done_q)) {
1556		run = list_entry(chan->done_q.next,
1557				 struct ipu_image_convert_run,
1558				 list);
1559
1560		list_del(&run->list);
1561
1562		dev_dbg(priv->ipu->dev,
1563			"%s: task %u: completing ctx %p run %p with %d\n",
1564			__func__, chan->ic_task, run->ctx, run, run->status);
1565
1566		/* call the completion callback and free the run */
1567		spin_unlock_irqrestore(&chan->irqlock, flags);
1568		run->ctx->complete(run, run->ctx->complete_context);
1569		spin_lock_irqsave(&chan->irqlock, flags);
1570	}
1571
1572	spin_unlock_irqrestore(&chan->irqlock, flags);
1573}
1574
1575/*
1576 * the bottom half thread clears out the done_q, calling the
1577 * completion handler for each.
1578 */
1579static irqreturn_t do_bh(int irq, void *dev_id)
1580{
1581	struct ipu_image_convert_chan *chan = dev_id;
1582	struct ipu_image_convert_priv *priv = chan->priv;
1583	struct ipu_image_convert_ctx *ctx;
1584	unsigned long flags;
1585
1586	dev_dbg(priv->ipu->dev, "%s: task %u: enter\n", __func__,
1587		chan->ic_task);
1588
1589	empty_done_q(chan);
1590
1591	spin_lock_irqsave(&chan->irqlock, flags);
1592
1593	/*
1594	 * the done_q is cleared out, signal any contexts
1595	 * that are aborting that abort can complete.
1596	 */
1597	list_for_each_entry(ctx, &chan->ctx_list, list) {
1598		if (ctx->aborting) {
1599			dev_dbg(priv->ipu->dev,
1600				"%s: task %u: signaling abort for ctx %p\n",
1601				__func__, chan->ic_task, ctx);
1602			complete_all(&ctx->aborted);
1603		}
1604	}
1605
1606	spin_unlock_irqrestore(&chan->irqlock, flags);
1607
1608	dev_dbg(priv->ipu->dev, "%s: task %u: exit\n", __func__,
1609		chan->ic_task);
1610
1611	return IRQ_HANDLED;
1612}
1613
1614static bool ic_settings_changed(struct ipu_image_convert_ctx *ctx)
1615{
1616	unsigned int cur_tile = ctx->next_tile - 1;
1617	unsigned int next_tile = ctx->next_tile;
1618
1619	if (ctx->resize_coeffs_h[cur_tile % ctx->in.num_cols] !=
1620	    ctx->resize_coeffs_h[next_tile % ctx->in.num_cols] ||
1621	    ctx->resize_coeffs_v[cur_tile / ctx->in.num_cols] !=
1622	    ctx->resize_coeffs_v[next_tile / ctx->in.num_cols] ||
1623	    ctx->in.tile[cur_tile].width != ctx->in.tile[next_tile].width ||
1624	    ctx->in.tile[cur_tile].height != ctx->in.tile[next_tile].height ||
1625	    ctx->out.tile[cur_tile].width != ctx->out.tile[next_tile].width ||
1626	    ctx->out.tile[cur_tile].height != ctx->out.tile[next_tile].height)
1627		return true;
1628
1629	return false;
1630}
1631
1632/* hold irqlock when calling */
1633static irqreturn_t do_tile_complete(struct ipu_image_convert_run *run)
1634{
1635	struct ipu_image_convert_ctx *ctx = run->ctx;
1636	struct ipu_image_convert_chan *chan = ctx->chan;
1637	struct ipu_image_tile *src_tile, *dst_tile;
1638	struct ipu_image_convert_image *s_image = &ctx->in;
1639	struct ipu_image_convert_image *d_image = &ctx->out;
1640	struct ipuv3_channel *outch;
1641	unsigned int dst_idx;
1642
1643	lockdep_assert_held(&chan->irqlock);
1644
1645	outch = ipu_rot_mode_is_irt(ctx->rot_mode) ?
1646		chan->rotation_out_chan : chan->out_chan;
1647
1648	/*
1649	 * It is difficult to stop the channel DMA before the channels
1650	 * enter the paused state. Without double-buffering the channels
1651	 * are always in a paused state when the EOF irq occurs, so it
1652	 * is safe to stop the channels now. For double-buffering we
1653	 * just ignore the abort until the operation completes, when it
1654	 * is safe to shut down.
1655	 */
1656	if (ctx->aborting && !ctx->double_buffering) {
1657		convert_stop(run);
1658		run->status = -EIO;
1659		goto done;
1660	}
1661
1662	if (ctx->next_tile == ctx->num_tiles) {
1663		/*
1664		 * the conversion is complete
1665		 */
1666		convert_stop(run);
1667		run->status = 0;
1668		goto done;
1669	}
1670
1671	/*
1672	 * not done, place the next tile buffers.
1673	 */
1674	if (!ctx->double_buffering) {
1675		if (ic_settings_changed(ctx)) {
1676			convert_stop(run);
1677			convert_start(run, ctx->next_tile);
1678		} else {
1679			src_tile = &s_image->tile[ctx->next_tile];
1680			dst_idx = ctx->out_tile_map[ctx->next_tile];
1681			dst_tile = &d_image->tile[dst_idx];
1682
1683			ipu_cpmem_set_buffer(chan->in_chan, 0,
1684					     s_image->base.phys0 +
1685					     src_tile->offset);
1686			ipu_cpmem_set_buffer(outch, 0,
1687					     d_image->base.phys0 +
1688					     dst_tile->offset);
1689			if (s_image->fmt->planar)
1690				ipu_cpmem_set_uv_offset(chan->in_chan,
1691							src_tile->u_off,
1692							src_tile->v_off);
1693			if (d_image->fmt->planar)
1694				ipu_cpmem_set_uv_offset(outch,
1695							dst_tile->u_off,
1696							dst_tile->v_off);
1697
1698			ipu_idmac_select_buffer(chan->in_chan, 0);
1699			ipu_idmac_select_buffer(outch, 0);
1700		}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1701	} else if (ctx->next_tile < ctx->num_tiles - 1) {
1702
1703		src_tile = &s_image->tile[ctx->next_tile + 1];
1704		dst_idx = ctx->out_tile_map[ctx->next_tile + 1];
1705		dst_tile = &d_image->tile[dst_idx];
1706
1707		ipu_cpmem_set_buffer(chan->in_chan, ctx->cur_buf_num,
1708				     s_image->base.phys0 + src_tile->offset);
1709		ipu_cpmem_set_buffer(outch, ctx->cur_buf_num,
1710				     d_image->base.phys0 + dst_tile->offset);
1711
1712		ipu_idmac_select_buffer(chan->in_chan, ctx->cur_buf_num);
1713		ipu_idmac_select_buffer(outch, ctx->cur_buf_num);
1714
1715		ctx->cur_buf_num ^= 1;
1716	}
1717
1718	ctx->eof_mask = 0; /* clear EOF irq mask for next tile */
1719	ctx->next_tile++;
1720	return IRQ_HANDLED;
1721done:
1722	list_add_tail(&run->list, &chan->done_q);
1723	chan->current_run = NULL;
1724	run_next(chan);
1725	return IRQ_WAKE_THREAD;
1726}
1727
1728static irqreturn_t eof_irq(int irq, void *data)
1729{
1730	struct ipu_image_convert_chan *chan = data;
1731	struct ipu_image_convert_priv *priv = chan->priv;
1732	struct ipu_image_convert_ctx *ctx;
1733	struct ipu_image_convert_run *run;
1734	irqreturn_t ret = IRQ_HANDLED;
1735	bool tile_complete = false;
1736	unsigned long flags;
 
1737
1738	spin_lock_irqsave(&chan->irqlock, flags);
1739
1740	/* get current run and its context */
1741	run = chan->current_run;
1742	if (!run) {
1743		ret = IRQ_NONE;
1744		goto out;
1745	}
1746
1747	ctx = run->ctx;
1748
1749	if (irq == chan->in_eof_irq) {
1750		ctx->eof_mask |= EOF_IRQ_IN;
1751	} else if (irq == chan->out_eof_irq) {
1752		ctx->eof_mask |= EOF_IRQ_OUT;
1753	} else if (irq == chan->rot_in_eof_irq ||
1754		   irq == chan->rot_out_eof_irq) {
1755		if (!ipu_rot_mode_is_irt(ctx->rot_mode)) {
1756			/* this was NOT a rotation op, shouldn't happen */
1757			dev_err(priv->ipu->dev,
1758				"Unexpected rotation interrupt\n");
1759			goto out;
1760		}
1761		ctx->eof_mask |= (irq == chan->rot_in_eof_irq) ?
1762			EOF_IRQ_ROT_IN : EOF_IRQ_ROT_OUT;
1763	} else {
1764		dev_err(priv->ipu->dev, "Received unknown irq %d\n", irq);
 
 
 
 
 
 
 
 
 
 
1765		ret = IRQ_NONE;
1766		goto out;
1767	}
1768
1769	if (ipu_rot_mode_is_irt(ctx->rot_mode))
1770		tile_complete =	(ctx->eof_mask == EOF_IRQ_ROT_COMPLETE);
1771	else
1772		tile_complete = (ctx->eof_mask == EOF_IRQ_COMPLETE);
1773
1774	if (tile_complete)
1775		ret = do_tile_complete(run);
 
 
 
 
 
 
1776out:
1777	spin_unlock_irqrestore(&chan->irqlock, flags);
1778	return ret;
1779}
1780
1781/*
1782 * try to force the completion of runs for this ctx. Called when
1783 * abort wait times out in ipu_image_convert_abort().
1784 */
1785static void force_abort(struct ipu_image_convert_ctx *ctx)
1786{
1787	struct ipu_image_convert_chan *chan = ctx->chan;
1788	struct ipu_image_convert_run *run;
1789	unsigned long flags;
1790
1791	spin_lock_irqsave(&chan->irqlock, flags);
1792
1793	run = chan->current_run;
1794	if (run && run->ctx == ctx) {
1795		convert_stop(run);
1796		run->status = -EIO;
1797		list_add_tail(&run->list, &chan->done_q);
1798		chan->current_run = NULL;
1799		run_next(chan);
1800	}
1801
1802	spin_unlock_irqrestore(&chan->irqlock, flags);
1803
1804	empty_done_q(chan);
1805}
1806
1807static void release_ipu_resources(struct ipu_image_convert_chan *chan)
1808{
1809	if (chan->in_eof_irq >= 0)
1810		free_irq(chan->in_eof_irq, chan);
1811	if (chan->rot_in_eof_irq >= 0)
1812		free_irq(chan->rot_in_eof_irq, chan);
1813	if (chan->out_eof_irq >= 0)
1814		free_irq(chan->out_eof_irq, chan);
1815	if (chan->rot_out_eof_irq >= 0)
1816		free_irq(chan->rot_out_eof_irq, chan);
1817
1818	if (!IS_ERR_OR_NULL(chan->in_chan))
1819		ipu_idmac_put(chan->in_chan);
1820	if (!IS_ERR_OR_NULL(chan->out_chan))
1821		ipu_idmac_put(chan->out_chan);
1822	if (!IS_ERR_OR_NULL(chan->rotation_in_chan))
1823		ipu_idmac_put(chan->rotation_in_chan);
1824	if (!IS_ERR_OR_NULL(chan->rotation_out_chan))
1825		ipu_idmac_put(chan->rotation_out_chan);
1826	if (!IS_ERR_OR_NULL(chan->ic))
1827		ipu_ic_put(chan->ic);
1828
1829	chan->in_chan = chan->out_chan = chan->rotation_in_chan =
1830		chan->rotation_out_chan = NULL;
1831	chan->in_eof_irq = -1;
1832	chan->rot_in_eof_irq = -1;
1833	chan->out_eof_irq = -1;
1834	chan->rot_out_eof_irq = -1;
1835}
1836
1837static int get_eof_irq(struct ipu_image_convert_chan *chan,
1838		       struct ipuv3_channel *channel)
1839{
1840	struct ipu_image_convert_priv *priv = chan->priv;
1841	int ret, irq;
1842
1843	irq = ipu_idmac_channel_irq(priv->ipu, channel, IPU_IRQ_EOF);
1844
1845	ret = request_threaded_irq(irq, eof_irq, do_bh, 0, "ipu-ic", chan);
1846	if (ret < 0) {
1847		dev_err(priv->ipu->dev, "could not acquire irq %d\n", irq);
1848		return ret;
1849	}
1850
1851	return irq;
1852}
1853
1854static int get_ipu_resources(struct ipu_image_convert_chan *chan)
1855{
1856	const struct ipu_image_convert_dma_chan *dma = chan->dma_ch;
1857	struct ipu_image_convert_priv *priv = chan->priv;
1858	int ret;
1859
1860	/* get IC */
1861	chan->ic = ipu_ic_get(priv->ipu, chan->ic_task);
1862	if (IS_ERR(chan->ic)) {
1863		dev_err(priv->ipu->dev, "could not acquire IC\n");
1864		ret = PTR_ERR(chan->ic);
1865		goto err;
1866	}
1867
1868	/* get IDMAC channels */
1869	chan->in_chan = ipu_idmac_get(priv->ipu, dma->in);
1870	chan->out_chan = ipu_idmac_get(priv->ipu, dma->out);
1871	if (IS_ERR(chan->in_chan) || IS_ERR(chan->out_chan)) {
1872		dev_err(priv->ipu->dev, "could not acquire idmac channels\n");
1873		ret = -EBUSY;
1874		goto err;
1875	}
1876
1877	chan->rotation_in_chan = ipu_idmac_get(priv->ipu, dma->rot_in);
1878	chan->rotation_out_chan = ipu_idmac_get(priv->ipu, dma->rot_out);
1879	if (IS_ERR(chan->rotation_in_chan) || IS_ERR(chan->rotation_out_chan)) {
1880		dev_err(priv->ipu->dev,
1881			"could not acquire idmac rotation channels\n");
1882		ret = -EBUSY;
1883		goto err;
1884	}
1885
1886	/* acquire the EOF interrupts */
1887	ret = get_eof_irq(chan, chan->in_chan);
1888	if (ret < 0) {
1889		chan->in_eof_irq = -1;
1890		goto err;
1891	}
1892	chan->in_eof_irq = ret;
1893
1894	ret = get_eof_irq(chan, chan->rotation_in_chan);
1895	if (ret < 0) {
1896		chan->rot_in_eof_irq = -1;
1897		goto err;
1898	}
1899	chan->rot_in_eof_irq = ret;
1900
1901	ret = get_eof_irq(chan, chan->out_chan);
 
1902	if (ret < 0) {
 
 
1903		chan->out_eof_irq = -1;
1904		goto err;
1905	}
1906	chan->out_eof_irq = ret;
1907
1908	ret = get_eof_irq(chan, chan->rotation_out_chan);
 
 
 
 
 
1909	if (ret < 0) {
 
 
1910		chan->rot_out_eof_irq = -1;
1911		goto err;
1912	}
1913	chan->rot_out_eof_irq = ret;
1914
1915	return 0;
1916err:
1917	release_ipu_resources(chan);
1918	return ret;
1919}
1920
1921static int fill_image(struct ipu_image_convert_ctx *ctx,
1922		      struct ipu_image_convert_image *ic_image,
1923		      struct ipu_image *image,
1924		      enum ipu_image_convert_type type)
1925{
1926	struct ipu_image_convert_priv *priv = ctx->chan->priv;
1927
1928	ic_image->base = *image;
1929	ic_image->type = type;
1930
1931	ic_image->fmt = get_format(image->pix.pixelformat);
1932	if (!ic_image->fmt) {
1933		dev_err(priv->ipu->dev, "pixelformat not supported for %s\n",
1934			type == IMAGE_CONVERT_OUT ? "Output" : "Input");
1935		return -EINVAL;
1936	}
1937
1938	if (ic_image->fmt->planar)
1939		ic_image->stride = ic_image->base.pix.width;
1940	else
1941		ic_image->stride  = ic_image->base.pix.bytesperline;
1942
 
 
 
1943	return 0;
1944}
1945
1946/* borrowed from drivers/media/v4l2-core/v4l2-common.c */
1947static unsigned int clamp_align(unsigned int x, unsigned int min,
1948				unsigned int max, unsigned int align)
1949{
1950	/* Bits that must be zero to be aligned */
1951	unsigned int mask = ~((1 << align) - 1);
1952
1953	/* Clamp to aligned min and max */
1954	x = clamp(x, (min + ~mask) & mask, max & mask);
1955
1956	/* Round to nearest aligned value */
1957	if (align)
1958		x = (x + (1 << (align - 1))) & mask;
1959
1960	return x;
1961}
1962
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1963/* Adjusts input/output images to IPU restrictions */
1964void ipu_image_convert_adjust(struct ipu_image *in, struct ipu_image *out,
1965			      enum ipu_rotate_mode rot_mode)
1966{
1967	const struct ipu_image_pixfmt *infmt, *outfmt;
1968	u32 w_align_out, h_align_out;
1969	u32 w_align_in, h_align_in;
 
1970
1971	infmt = get_format(in->pix.pixelformat);
1972	outfmt = get_format(out->pix.pixelformat);
1973
1974	/* set some default pixel formats if needed */
1975	if (!infmt) {
1976		in->pix.pixelformat = V4L2_PIX_FMT_RGB24;
1977		infmt = get_format(V4L2_PIX_FMT_RGB24);
1978	}
1979	if (!outfmt) {
1980		out->pix.pixelformat = V4L2_PIX_FMT_RGB24;
1981		outfmt = get_format(V4L2_PIX_FMT_RGB24);
1982	}
1983
1984	/* image converter does not handle fields */
1985	in->pix.field = out->pix.field = V4L2_FIELD_NONE;
1986
1987	/* resizer cannot downsize more than 4:1 */
1988	if (ipu_rot_mode_is_irt(rot_mode)) {
1989		out->pix.height = max_t(__u32, out->pix.height,
1990					in->pix.width / 4);
1991		out->pix.width = max_t(__u32, out->pix.width,
1992				       in->pix.height / 4);
1993	} else {
1994		out->pix.width = max_t(__u32, out->pix.width,
1995				       in->pix.width / 4);
1996		out->pix.height = max_t(__u32, out->pix.height,
1997					in->pix.height / 4);
1998	}
1999
 
 
 
 
 
 
 
 
 
 
 
2000	/* align input width/height */
2001	w_align_in = ilog2(tile_width_align(IMAGE_CONVERT_IN, infmt,
2002					    rot_mode));
2003	h_align_in = ilog2(tile_height_align(IMAGE_CONVERT_IN, infmt,
2004					     rot_mode));
2005	in->pix.width = clamp_align(in->pix.width, MIN_W, MAX_W,
2006				    w_align_in);
2007	in->pix.height = clamp_align(in->pix.height, MIN_H, MAX_H,
2008				     h_align_in);
2009
2010	/* align output width/height */
2011	w_align_out = ilog2(tile_width_align(IMAGE_CONVERT_OUT, outfmt,
2012					     rot_mode));
2013	h_align_out = ilog2(tile_height_align(IMAGE_CONVERT_OUT, outfmt,
2014					      rot_mode));
2015	out->pix.width = clamp_align(out->pix.width, MIN_W, MAX_W,
2016				     w_align_out);
2017	out->pix.height = clamp_align(out->pix.height, MIN_H, MAX_H,
2018				      h_align_out);
2019
2020	/* set input/output strides and image sizes */
2021	in->pix.bytesperline = infmt->planar ?
2022		clamp_align(in->pix.width, 2 << w_align_in, MAX_W,
2023			    w_align_in) :
2024		clamp_align((in->pix.width * infmt->bpp) >> 3,
2025			    ((2 << w_align_in) * infmt->bpp) >> 3,
2026			    (MAX_W * infmt->bpp) >> 3,
2027			    w_align_in);
2028	in->pix.sizeimage = infmt->planar ?
2029		(in->pix.height * in->pix.bytesperline * infmt->bpp) >> 3 :
2030		in->pix.height * in->pix.bytesperline;
2031	out->pix.bytesperline = outfmt->planar ? out->pix.width :
2032		(out->pix.width * outfmt->bpp) >> 3;
2033	out->pix.sizeimage = outfmt->planar ?
2034		(out->pix.height * out->pix.bytesperline * outfmt->bpp) >> 3 :
2035		out->pix.height * out->pix.bytesperline;
2036}
2037EXPORT_SYMBOL_GPL(ipu_image_convert_adjust);
2038
2039/*
2040 * this is used by ipu_image_convert_prepare() to verify set input and
2041 * output images are valid before starting the conversion. Clients can
2042 * also call it before calling ipu_image_convert_prepare().
2043 */
2044int ipu_image_convert_verify(struct ipu_image *in, struct ipu_image *out,
2045			     enum ipu_rotate_mode rot_mode)
2046{
2047	struct ipu_image testin, testout;
2048
2049	testin = *in;
2050	testout = *out;
2051
2052	ipu_image_convert_adjust(&testin, &testout, rot_mode);
2053
2054	if (testin.pix.width != in->pix.width ||
2055	    testin.pix.height != in->pix.height ||
2056	    testout.pix.width != out->pix.width ||
2057	    testout.pix.height != out->pix.height)
2058		return -EINVAL;
2059
2060	return 0;
2061}
2062EXPORT_SYMBOL_GPL(ipu_image_convert_verify);
2063
2064/*
2065 * Call ipu_image_convert_prepare() to prepare for the conversion of
2066 * given images and rotation mode. Returns a new conversion context.
2067 */
2068struct ipu_image_convert_ctx *
2069ipu_image_convert_prepare(struct ipu_soc *ipu, enum ipu_ic_task ic_task,
2070			  struct ipu_image *in, struct ipu_image *out,
2071			  enum ipu_rotate_mode rot_mode,
2072			  ipu_image_convert_cb_t complete,
2073			  void *complete_context)
2074{
2075	struct ipu_image_convert_priv *priv = ipu->image_convert_priv;
2076	struct ipu_image_convert_image *s_image, *d_image;
2077	struct ipu_image_convert_chan *chan;
2078	struct ipu_image_convert_ctx *ctx;
2079	unsigned long flags;
2080	unsigned int i;
2081	bool get_res;
2082	int ret;
2083
2084	if (!in || !out || !complete ||
2085	    (ic_task != IC_TASK_VIEWFINDER &&
2086	     ic_task != IC_TASK_POST_PROCESSOR))
2087		return ERR_PTR(-EINVAL);
2088
2089	/* verify the in/out images before continuing */
2090	ret = ipu_image_convert_verify(in, out, rot_mode);
2091	if (ret) {
2092		dev_err(priv->ipu->dev, "%s: in/out formats invalid\n",
2093			__func__);
2094		return ERR_PTR(ret);
2095	}
2096
2097	chan = &priv->chan[ic_task];
2098
2099	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
2100	if (!ctx)
2101		return ERR_PTR(-ENOMEM);
2102
2103	dev_dbg(priv->ipu->dev, "%s: task %u: ctx %p\n", __func__,
2104		chan->ic_task, ctx);
2105
2106	ctx->chan = chan;
2107	init_completion(&ctx->aborted);
2108
2109	ctx->rot_mode = rot_mode;
2110
2111	/* Sets ctx->in.num_rows/cols as well */
2112	ret = calc_image_resize_coefficients(ctx, in, out);
2113	if (ret)
2114		goto out_free;
2115
2116	s_image = &ctx->in;
2117	d_image = &ctx->out;
2118
2119	/* set tiling and rotation */
 
 
2120	if (ipu_rot_mode_is_irt(rot_mode)) {
2121		d_image->num_rows = s_image->num_cols;
2122		d_image->num_cols = s_image->num_rows;
2123	} else {
2124		d_image->num_rows = s_image->num_rows;
2125		d_image->num_cols = s_image->num_cols;
2126	}
2127
2128	ctx->num_tiles = d_image->num_cols * d_image->num_rows;
 
2129
2130	ret = fill_image(ctx, s_image, in, IMAGE_CONVERT_IN);
2131	if (ret)
2132		goto out_free;
2133	ret = fill_image(ctx, d_image, out, IMAGE_CONVERT_OUT);
2134	if (ret)
2135		goto out_free;
2136
2137	calc_out_tile_map(ctx);
2138
2139	find_seams(ctx, s_image, d_image);
2140
2141	ret = calc_tile_dimensions(ctx, s_image);
2142	if (ret)
2143		goto out_free;
2144
2145	ret = calc_tile_offsets(ctx, s_image);
2146	if (ret)
2147		goto out_free;
2148
2149	calc_tile_dimensions(ctx, d_image);
2150	ret = calc_tile_offsets(ctx, d_image);
2151	if (ret)
2152		goto out_free;
2153
2154	calc_tile_resize_coefficients(ctx);
2155
2156	ret = ipu_ic_calc_csc(&ctx->csc,
2157			s_image->base.pix.ycbcr_enc,
2158			s_image->base.pix.quantization,
2159			ipu_pixelformat_to_colorspace(s_image->fmt->fourcc),
2160			d_image->base.pix.ycbcr_enc,
2161			d_image->base.pix.quantization,
2162			ipu_pixelformat_to_colorspace(d_image->fmt->fourcc));
2163	if (ret)
2164		goto out_free;
2165
2166	dump_format(ctx, s_image);
2167	dump_format(ctx, d_image);
2168
2169	ctx->complete = complete;
2170	ctx->complete_context = complete_context;
2171
2172	/*
2173	 * Can we use double-buffering for this operation? If there is
2174	 * only one tile (the whole image can be converted in a single
2175	 * operation) there's no point in using double-buffering. Also,
2176	 * the IPU's IDMAC channels allow only a single U and V plane
2177	 * offset shared between both buffers, but these offsets change
2178	 * for every tile, and therefore would have to be updated for
2179	 * each buffer which is not possible. So double-buffering is
2180	 * impossible when either the source or destination images are
2181	 * a planar format (YUV420, YUV422P, etc.). Further, differently
2182	 * sized tiles or different resizing coefficients per tile
2183	 * prevent double-buffering as well.
2184	 */
2185	ctx->double_buffering = (ctx->num_tiles > 1 &&
2186				 !s_image->fmt->planar &&
2187				 !d_image->fmt->planar);
2188	for (i = 1; i < ctx->num_tiles; i++) {
2189		if (ctx->in.tile[i].width != ctx->in.tile[0].width ||
2190		    ctx->in.tile[i].height != ctx->in.tile[0].height ||
2191		    ctx->out.tile[i].width != ctx->out.tile[0].width ||
2192		    ctx->out.tile[i].height != ctx->out.tile[0].height) {
2193			ctx->double_buffering = false;
2194			break;
2195		}
2196	}
2197	for (i = 1; i < ctx->in.num_cols; i++) {
2198		if (ctx->resize_coeffs_h[i] != ctx->resize_coeffs_h[0]) {
2199			ctx->double_buffering = false;
2200			break;
2201		}
2202	}
2203	for (i = 1; i < ctx->in.num_rows; i++) {
2204		if (ctx->resize_coeffs_v[i] != ctx->resize_coeffs_v[0]) {
2205			ctx->double_buffering = false;
2206			break;
2207		}
2208	}
2209
2210	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
2211		unsigned long intermediate_size = d_image->tile[0].size;
2212
2213		for (i = 1; i < ctx->num_tiles; i++) {
2214			if (d_image->tile[i].size > intermediate_size)
2215				intermediate_size = d_image->tile[i].size;
2216		}
2217
2218		ret = alloc_dma_buf(priv, &ctx->rot_intermediate[0],
2219				    intermediate_size);
2220		if (ret)
2221			goto out_free;
2222		if (ctx->double_buffering) {
2223			ret = alloc_dma_buf(priv,
2224					    &ctx->rot_intermediate[1],
2225					    intermediate_size);
2226			if (ret)
2227				goto out_free_dmabuf0;
2228		}
2229	}
2230
2231	spin_lock_irqsave(&chan->irqlock, flags);
2232
2233	get_res = list_empty(&chan->ctx_list);
2234
2235	list_add_tail(&ctx->list, &chan->ctx_list);
2236
2237	spin_unlock_irqrestore(&chan->irqlock, flags);
2238
2239	if (get_res) {
2240		ret = get_ipu_resources(chan);
2241		if (ret)
2242			goto out_free_dmabuf1;
2243	}
2244
2245	return ctx;
2246
2247out_free_dmabuf1:
2248	free_dma_buf(priv, &ctx->rot_intermediate[1]);
2249	spin_lock_irqsave(&chan->irqlock, flags);
2250	list_del(&ctx->list);
2251	spin_unlock_irqrestore(&chan->irqlock, flags);
2252out_free_dmabuf0:
2253	free_dma_buf(priv, &ctx->rot_intermediate[0]);
2254out_free:
2255	kfree(ctx);
2256	return ERR_PTR(ret);
2257}
2258EXPORT_SYMBOL_GPL(ipu_image_convert_prepare);
2259
2260/*
2261 * Carry out a single image conversion run. Only the physaddr's of the input
2262 * and output image buffers are needed. The conversion context must have
2263 * been created previously with ipu_image_convert_prepare().
2264 */
2265int ipu_image_convert_queue(struct ipu_image_convert_run *run)
2266{
2267	struct ipu_image_convert_chan *chan;
2268	struct ipu_image_convert_priv *priv;
2269	struct ipu_image_convert_ctx *ctx;
2270	unsigned long flags;
2271	int ret = 0;
2272
2273	if (!run || !run->ctx || !run->in_phys || !run->out_phys)
2274		return -EINVAL;
2275
2276	ctx = run->ctx;
2277	chan = ctx->chan;
2278	priv = chan->priv;
2279
2280	dev_dbg(priv->ipu->dev, "%s: task %u: ctx %p run %p\n", __func__,
2281		chan->ic_task, ctx, run);
2282
2283	INIT_LIST_HEAD(&run->list);
2284
2285	spin_lock_irqsave(&chan->irqlock, flags);
2286
2287	if (ctx->aborting) {
2288		ret = -EIO;
2289		goto unlock;
2290	}
2291
2292	list_add_tail(&run->list, &chan->pending_q);
2293
2294	if (!chan->current_run) {
2295		ret = do_run(run);
2296		if (ret)
2297			chan->current_run = NULL;
2298	}
2299unlock:
2300	spin_unlock_irqrestore(&chan->irqlock, flags);
2301	return ret;
2302}
2303EXPORT_SYMBOL_GPL(ipu_image_convert_queue);
2304
2305/* Abort any active or pending conversions for this context */
2306static void __ipu_image_convert_abort(struct ipu_image_convert_ctx *ctx)
2307{
2308	struct ipu_image_convert_chan *chan = ctx->chan;
2309	struct ipu_image_convert_priv *priv = chan->priv;
2310	struct ipu_image_convert_run *run, *active_run, *tmp;
2311	unsigned long flags;
2312	int run_count, ret;
 
 
 
2313
2314	spin_lock_irqsave(&chan->irqlock, flags);
2315
2316	/* move all remaining pending runs in this context to done_q */
2317	list_for_each_entry_safe(run, tmp, &chan->pending_q, list) {
2318		if (run->ctx != ctx)
2319			continue;
2320		run->status = -EIO;
2321		list_move_tail(&run->list, &chan->done_q);
2322	}
2323
2324	run_count = get_run_count(ctx, &chan->done_q);
2325	active_run = (chan->current_run && chan->current_run->ctx == ctx) ?
2326		chan->current_run : NULL;
2327
2328	if (active_run)
2329		reinit_completion(&ctx->aborted);
2330
2331	ctx->aborting = true;
2332
2333	spin_unlock_irqrestore(&chan->irqlock, flags);
2334
2335	if (!run_count && !active_run) {
2336		dev_dbg(priv->ipu->dev,
2337			"%s: task %u: no abort needed for ctx %p\n",
2338			__func__, chan->ic_task, ctx);
2339		return;
2340	}
2341
2342	if (!active_run) {
2343		empty_done_q(chan);
2344		return;
2345	}
2346
2347	dev_dbg(priv->ipu->dev,
2348		"%s: task %u: wait for completion: %d runs\n",
2349		__func__, chan->ic_task, run_count);
2350
2351	ret = wait_for_completion_timeout(&ctx->aborted,
2352					  msecs_to_jiffies(10000));
2353	if (ret == 0) {
2354		dev_warn(priv->ipu->dev, "%s: timeout\n", __func__);
2355		force_abort(ctx);
2356	}
2357}
2358
2359void ipu_image_convert_abort(struct ipu_image_convert_ctx *ctx)
2360{
2361	__ipu_image_convert_abort(ctx);
2362	ctx->aborting = false;
2363}
2364EXPORT_SYMBOL_GPL(ipu_image_convert_abort);
2365
2366/* Unprepare image conversion context */
2367void ipu_image_convert_unprepare(struct ipu_image_convert_ctx *ctx)
2368{
2369	struct ipu_image_convert_chan *chan = ctx->chan;
2370	struct ipu_image_convert_priv *priv = chan->priv;
2371	unsigned long flags;
2372	bool put_res;
2373
2374	/* make sure no runs are hanging around */
2375	__ipu_image_convert_abort(ctx);
2376
2377	dev_dbg(priv->ipu->dev, "%s: task %u: removing ctx %p\n", __func__,
2378		chan->ic_task, ctx);
2379
2380	spin_lock_irqsave(&chan->irqlock, flags);
2381
2382	list_del(&ctx->list);
2383
2384	put_res = list_empty(&chan->ctx_list);
2385
2386	spin_unlock_irqrestore(&chan->irqlock, flags);
2387
2388	if (put_res)
2389		release_ipu_resources(chan);
2390
2391	free_dma_buf(priv, &ctx->rot_intermediate[1]);
2392	free_dma_buf(priv, &ctx->rot_intermediate[0]);
2393
2394	kfree(ctx);
2395}
2396EXPORT_SYMBOL_GPL(ipu_image_convert_unprepare);
2397
2398/*
2399 * "Canned" asynchronous single image conversion. Allocates and returns
2400 * a new conversion run.  On successful return the caller must free the
2401 * run and call ipu_image_convert_unprepare() after conversion completes.
2402 */
2403struct ipu_image_convert_run *
2404ipu_image_convert(struct ipu_soc *ipu, enum ipu_ic_task ic_task,
2405		  struct ipu_image *in, struct ipu_image *out,
2406		  enum ipu_rotate_mode rot_mode,
2407		  ipu_image_convert_cb_t complete,
2408		  void *complete_context)
2409{
2410	struct ipu_image_convert_ctx *ctx;
2411	struct ipu_image_convert_run *run;
2412	int ret;
2413
2414	ctx = ipu_image_convert_prepare(ipu, ic_task, in, out, rot_mode,
2415					complete, complete_context);
2416	if (IS_ERR(ctx))
2417		return ERR_CAST(ctx);
2418
2419	run = kzalloc(sizeof(*run), GFP_KERNEL);
2420	if (!run) {
2421		ipu_image_convert_unprepare(ctx);
2422		return ERR_PTR(-ENOMEM);
2423	}
2424
2425	run->ctx = ctx;
2426	run->in_phys = in->phys0;
2427	run->out_phys = out->phys0;
2428
2429	ret = ipu_image_convert_queue(run);
2430	if (ret) {
2431		ipu_image_convert_unprepare(ctx);
2432		kfree(run);
2433		return ERR_PTR(ret);
2434	}
2435
2436	return run;
2437}
2438EXPORT_SYMBOL_GPL(ipu_image_convert);
2439
2440/* "Canned" synchronous single image conversion */
2441static void image_convert_sync_complete(struct ipu_image_convert_run *run,
2442					void *data)
2443{
2444	struct completion *comp = data;
2445
2446	complete(comp);
2447}
2448
2449int ipu_image_convert_sync(struct ipu_soc *ipu, enum ipu_ic_task ic_task,
2450			   struct ipu_image *in, struct ipu_image *out,
2451			   enum ipu_rotate_mode rot_mode)
2452{
2453	struct ipu_image_convert_run *run;
2454	struct completion comp;
2455	int ret;
2456
2457	init_completion(&comp);
2458
2459	run = ipu_image_convert(ipu, ic_task, in, out, rot_mode,
2460				image_convert_sync_complete, &comp);
2461	if (IS_ERR(run))
2462		return PTR_ERR(run);
2463
2464	ret = wait_for_completion_timeout(&comp, msecs_to_jiffies(10000));
2465	ret = (ret == 0) ? -ETIMEDOUT : 0;
2466
2467	ipu_image_convert_unprepare(run->ctx);
2468	kfree(run);
2469
2470	return ret;
2471}
2472EXPORT_SYMBOL_GPL(ipu_image_convert_sync);
2473
2474int ipu_image_convert_init(struct ipu_soc *ipu, struct device *dev)
2475{
2476	struct ipu_image_convert_priv *priv;
2477	int i;
2478
2479	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
2480	if (!priv)
2481		return -ENOMEM;
2482
2483	ipu->image_convert_priv = priv;
2484	priv->ipu = ipu;
2485
2486	for (i = 0; i < IC_NUM_TASKS; i++) {
2487		struct ipu_image_convert_chan *chan = &priv->chan[i];
2488
2489		chan->ic_task = i;
2490		chan->priv = priv;
2491		chan->dma_ch = &image_convert_dma_chan[i];
2492		chan->in_eof_irq = -1;
2493		chan->rot_in_eof_irq = -1;
2494		chan->out_eof_irq = -1;
2495		chan->rot_out_eof_irq = -1;
2496
2497		spin_lock_init(&chan->irqlock);
2498		INIT_LIST_HEAD(&chan->ctx_list);
2499		INIT_LIST_HEAD(&chan->pending_q);
2500		INIT_LIST_HEAD(&chan->done_q);
2501	}
2502
2503	return 0;
2504}
2505
2506void ipu_image_convert_exit(struct ipu_soc *ipu)
2507{
2508}

 
   1/*
   2 * Copyright (C) 2012-2016 Mentor Graphics Inc.
   3 *
   4 * Queued image conversion support, with tiling and rotation.
   5 *
   6 * This program is free software; you can redistribute it and/or modify it
   7 * under the terms of the GNU General Public License as published by the
   8 * Free Software Foundation; either version 2 of the License, or (at your
   9 * option) any later version.
  10 *
  11 * This program is distributed in the hope that it will be useful, but
  12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 * for more details.
  15 */
  16
  17#include <linux/interrupt.h>
  18#include <linux/dma-mapping.h>
 
 
  19#include <video/imx-ipu-image-convert.h>
 
  20#include "ipu-prv.h"
  21
  22/*
  23 * The IC Resizer has a restriction that the output frame from the
  24 * resizer must be 1024 or less in both width (pixels) and height
  25 * (lines).
  26 *
  27 * The image converter attempts to split up a conversion when
  28 * the desired output (converted) frame resolution exceeds the
  29 * IC resizer limit of 1024 in either dimension.
  30 *
  31 * If either dimension of the output frame exceeds the limit, the
  32 * dimension is split into 1, 2, or 4 equal stripes, for a maximum
  33 * of 4*4 or 16 tiles. A conversion is then carried out for each
  34 * tile (but taking care to pass the full frame stride length to
  35 * the DMA channel's parameter memory!). IDMA double-buffering is used
  36 * to convert each tile back-to-back when possible (see note below
  37 * when double_buffering boolean is set).
  38 *
  39 * Note that the input frame must be split up into the same number
  40 * of tiles as the output frame.
 
 
 
 
 
 
 
 
 
 
 
 
  41 *
  42 * FIXME: at this point there is no attempt to deal with visible seams
  43 * at the tile boundaries when upscaling. The seams are caused by a reset
  44 * of the bilinear upscale interpolation when starting a new tile. The
  45 * seams are barely visible for small upscale factors, but become
  46 * increasingly visible as the upscale factor gets larger, since more
  47 * interpolated pixels get thrown out at the tile boundaries. A possilble
  48 * fix might be to overlap tiles of different sizes, but this must be done
  49 * while also maintaining the IDMAC dma buffer address alignment and 8x8 IRT
  50 * alignment restrictions of each tile.
 
 
 
 
 
 
 
  51 */
  52
  53#define MAX_STRIPES_W    4
  54#define MAX_STRIPES_H    4
  55#define MAX_TILES (MAX_STRIPES_W * MAX_STRIPES_H)
  56
  57#define MIN_W     16
  58#define MIN_H     8
  59#define MAX_W     4096
  60#define MAX_H     4096
  61
  62enum ipu_image_convert_type {
  63	IMAGE_CONVERT_IN = 0,
  64	IMAGE_CONVERT_OUT,
  65};
  66
  67struct ipu_image_convert_dma_buf {
  68	void          *virt;
  69	dma_addr_t    phys;
  70	unsigned long len;
  71};
  72
  73struct ipu_image_convert_dma_chan {
  74	int in;
  75	int out;
  76	int rot_in;
  77	int rot_out;
  78	int vdi_in_p;
  79	int vdi_in;
  80	int vdi_in_n;
  81};
  82
  83/* dimensions of one tile */
  84struct ipu_image_tile {
  85	u32 width;
  86	u32 height;
 
 
  87	/* size and strides are in bytes */
  88	u32 size;
  89	u32 stride;
  90	u32 rot_stride;
  91	/* start Y or packed offset of this tile */
  92	u32 offset;
  93	/* offset from start to tile in U plane, for planar formats */
  94	u32 u_off;
  95	/* offset from start to tile in V plane, for planar formats */
  96	u32 v_off;
  97};
  98
  99struct ipu_image_convert_image {
 100	struct ipu_image base;
 101	enum ipu_image_convert_type type;
 102
 103	const struct ipu_image_pixfmt *fmt;
 104	unsigned int stride;
 105
 106	/* # of rows (horizontal stripes) if dest height is > 1024 */
 107	unsigned int num_rows;
 108	/* # of columns (vertical stripes) if dest width is > 1024 */
 109	unsigned int num_cols;
 110
 111	struct ipu_image_tile tile[MAX_TILES];
 112};
 113
 114struct ipu_image_pixfmt {
 115	u32	fourcc;        /* V4L2 fourcc */
 116	int     bpp;           /* total bpp */
 117	int     uv_width_dec;  /* decimation in width for U/V planes */
 118	int     uv_height_dec; /* decimation in height for U/V planes */
 119	bool    planar;        /* planar format */
 120	bool    uv_swapped;    /* U and V planes are swapped */
 121	bool    uv_packed;     /* partial planar (U and V in same plane) */
 122};
 123
 124struct ipu_image_convert_ctx;
 125struct ipu_image_convert_chan;
 126struct ipu_image_convert_priv;
 127
 
 
 
 
 
 
 
 
 
 
 
 128struct ipu_image_convert_ctx {
 129	struct ipu_image_convert_chan *chan;
 130
 131	ipu_image_convert_cb_t complete;
 132	void *complete_context;
 133
 134	/* Source/destination image data and rotation mode */
 135	struct ipu_image_convert_image in;
 136	struct ipu_image_convert_image out;
 
 137	enum ipu_rotate_mode rot_mode;
 
 
 
 
 
 
 138
 139	/* intermediate buffer for rotation */
 140	struct ipu_image_convert_dma_buf rot_intermediate[2];
 141
 142	/* current buffer number for double buffering */
 143	int cur_buf_num;
 144
 145	bool aborting;
 146	struct completion aborted;
 147
 148	/* can we use double-buffering for this conversion operation? */
 149	bool double_buffering;
 150	/* num_rows * num_cols */
 151	unsigned int num_tiles;
 152	/* next tile to process */
 153	unsigned int next_tile;
 154	/* where to place converted tile in dest image */
 155	unsigned int out_tile_map[MAX_TILES];
 156
 
 
 
 157	struct list_head list;
 158};
 159
 160struct ipu_image_convert_chan {
 161	struct ipu_image_convert_priv *priv;
 162
 163	enum ipu_ic_task ic_task;
 164	const struct ipu_image_convert_dma_chan *dma_ch;
 165
 166	struct ipu_ic *ic;
 167	struct ipuv3_channel *in_chan;
 168	struct ipuv3_channel *out_chan;
 169	struct ipuv3_channel *rotation_in_chan;
 170	struct ipuv3_channel *rotation_out_chan;
 171
 172	/* the IPU end-of-frame irqs */
 
 
 173	int out_eof_irq;
 174	int rot_out_eof_irq;
 175
 176	spinlock_t irqlock;
 177
 178	/* list of convert contexts */
 179	struct list_head ctx_list;
 180	/* queue of conversion runs */
 181	struct list_head pending_q;
 182	/* queue of completed runs */
 183	struct list_head done_q;
 184
 185	/* the current conversion run */
 186	struct ipu_image_convert_run *current_run;
 187};
 188
 189struct ipu_image_convert_priv {
 190	struct ipu_image_convert_chan chan[IC_NUM_TASKS];
 191	struct ipu_soc *ipu;
 192};
 193
 194static const struct ipu_image_convert_dma_chan
 195image_convert_dma_chan[IC_NUM_TASKS] = {
 196	[IC_TASK_VIEWFINDER] = {
 197		.in = IPUV3_CHANNEL_MEM_IC_PRP_VF,
 198		.out = IPUV3_CHANNEL_IC_PRP_VF_MEM,
 199		.rot_in = IPUV3_CHANNEL_MEM_ROT_VF,
 200		.rot_out = IPUV3_CHANNEL_ROT_VF_MEM,
 201		.vdi_in_p = IPUV3_CHANNEL_MEM_VDI_PREV,
 202		.vdi_in = IPUV3_CHANNEL_MEM_VDI_CUR,
 203		.vdi_in_n = IPUV3_CHANNEL_MEM_VDI_NEXT,
 204	},
 205	[IC_TASK_POST_PROCESSOR] = {
 206		.in = IPUV3_CHANNEL_MEM_IC_PP,
 207		.out = IPUV3_CHANNEL_IC_PP_MEM,
 208		.rot_in = IPUV3_CHANNEL_MEM_ROT_PP,
 209		.rot_out = IPUV3_CHANNEL_ROT_PP_MEM,
 210	},
 211};
 212
 213static const struct ipu_image_pixfmt image_convert_formats[] = {
 214	{
 215		.fourcc	= V4L2_PIX_FMT_RGB565,
 216		.bpp    = 16,
 217	}, {
 218		.fourcc	= V4L2_PIX_FMT_RGB24,
 219		.bpp    = 24,
 220	}, {
 221		.fourcc	= V4L2_PIX_FMT_BGR24,
 222		.bpp    = 24,
 223	}, {
 224		.fourcc	= V4L2_PIX_FMT_RGB32,
 225		.bpp    = 32,
 226	}, {
 227		.fourcc	= V4L2_PIX_FMT_BGR32,
 228		.bpp    = 32,
 229	}, {
 
 
 
 
 
 
 
 
 
 
 
 
 230		.fourcc	= V4L2_PIX_FMT_YUYV,
 231		.bpp    = 16,
 232		.uv_width_dec = 2,
 233		.uv_height_dec = 1,
 234	}, {
 235		.fourcc	= V4L2_PIX_FMT_UYVY,
 236		.bpp    = 16,
 237		.uv_width_dec = 2,
 238		.uv_height_dec = 1,
 239	}, {
 240		.fourcc	= V4L2_PIX_FMT_YUV420,
 241		.bpp    = 12,
 242		.planar = true,
 243		.uv_width_dec = 2,
 244		.uv_height_dec = 2,
 245	}, {
 246		.fourcc	= V4L2_PIX_FMT_YVU420,
 247		.bpp    = 12,
 248		.planar = true,
 249		.uv_width_dec = 2,
 250		.uv_height_dec = 2,
 251		.uv_swapped = true,
 252	}, {
 253		.fourcc = V4L2_PIX_FMT_NV12,
 254		.bpp    = 12,
 255		.planar = true,
 256		.uv_width_dec = 2,
 257		.uv_height_dec = 2,
 258		.uv_packed = true,
 259	}, {
 260		.fourcc = V4L2_PIX_FMT_YUV422P,
 261		.bpp    = 16,
 262		.planar = true,
 263		.uv_width_dec = 2,
 264		.uv_height_dec = 1,
 265	}, {
 266		.fourcc = V4L2_PIX_FMT_NV16,
 267		.bpp    = 16,
 268		.planar = true,
 269		.uv_width_dec = 2,
 270		.uv_height_dec = 1,
 271		.uv_packed = true,
 272	},
 273};
 274
 275static const struct ipu_image_pixfmt *get_format(u32 fourcc)
 276{
 277	const struct ipu_image_pixfmt *ret = NULL;
 278	unsigned int i;
 279
 280	for (i = 0; i < ARRAY_SIZE(image_convert_formats); i++) {
 281		if (image_convert_formats[i].fourcc == fourcc) {
 282			ret = &image_convert_formats[i];
 283			break;
 284		}
 285	}
 286
 287	return ret;
 288}
 289
 290static void dump_format(struct ipu_image_convert_ctx *ctx,
 291			struct ipu_image_convert_image *ic_image)
 292{
 293	struct ipu_image_convert_chan *chan = ctx->chan;
 294	struct ipu_image_convert_priv *priv = chan->priv;
 295
 296	dev_dbg(priv->ipu->dev,
 297		"task %u: ctx %p: %s format: %dx%d (%dx%d tiles of size %dx%d), %c%c%c%c\n",
 298		chan->ic_task, ctx,
 299		ic_image->type == IMAGE_CONVERT_OUT ? "Output" : "Input",
 300		ic_image->base.pix.width, ic_image->base.pix.height,
 301		ic_image->num_cols, ic_image->num_rows,
 302		ic_image->tile[0].width, ic_image->tile[0].height,
 303		ic_image->fmt->fourcc & 0xff,
 304		(ic_image->fmt->fourcc >> 8) & 0xff,
 305		(ic_image->fmt->fourcc >> 16) & 0xff,
 306		(ic_image->fmt->fourcc >> 24) & 0xff);
 307}
 308
 309int ipu_image_convert_enum_format(int index, u32 *fourcc)
 310{
 311	const struct ipu_image_pixfmt *fmt;
 312
 313	if (index >= (int)ARRAY_SIZE(image_convert_formats))
 314		return -EINVAL;
 315
 316	/* Format found */
 317	fmt = &image_convert_formats[index];
 318	*fourcc = fmt->fourcc;
 319	return 0;
 320}
 321EXPORT_SYMBOL_GPL(ipu_image_convert_enum_format);
 322
 323static void free_dma_buf(struct ipu_image_convert_priv *priv,
 324			 struct ipu_image_convert_dma_buf *buf)
 325{
 326	if (buf->virt)
 327		dma_free_coherent(priv->ipu->dev,
 328				  buf->len, buf->virt, buf->phys);
 329	buf->virt = NULL;
 330	buf->phys = 0;
 331}
 332
 333static int alloc_dma_buf(struct ipu_image_convert_priv *priv,
 334			 struct ipu_image_convert_dma_buf *buf,
 335			 int size)
 336{
 337	buf->len = PAGE_ALIGN(size);
 338	buf->virt = dma_alloc_coherent(priv->ipu->dev, buf->len, &buf->phys,
 339				       GFP_DMA | GFP_KERNEL);
 340	if (!buf->virt) {
 341		dev_err(priv->ipu->dev, "failed to alloc dma buffer\n");
 342		return -ENOMEM;
 343	}
 344
 345	return 0;
 346}
 347
 348static inline int num_stripes(int dim)
 349{
 350	if (dim <= 1024)
 351		return 1;
 352	else if (dim <= 2048)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 353		return 2;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 354	else
 355		return 4;
 
 
 
 
 
 
 356}
 357
 358static void calc_tile_dimensions(struct ipu_image_convert_ctx *ctx,
 359				 struct ipu_image_convert_image *image)
 360{
 361	int i;
 
 
 
 
 
 
 
 
 
 
 362
 363	for (i = 0; i < ctx->num_tiles; i++) {
 364		struct ipu_image_tile *tile = &image->tile[i];
 
 
 
 
 
 
 
 365
 366		tile->height = image->base.pix.height / image->num_rows;
 367		tile->width = image->base.pix.width / image->num_cols;
 368		tile->size = ((tile->height * image->fmt->bpp) >> 3) *
 369			tile->width;
 370
 371		if (image->fmt->planar) {
 372			tile->stride = tile->width;
 373			tile->rot_stride = tile->height;
 374		} else {
 375			tile->stride =
 376				(image->fmt->bpp * tile->width) >> 3;
 377			tile->rot_stride =
 378				(image->fmt->bpp * tile->height) >> 3;
 379		}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 380	}
 
 
 381}
 382
 383/*
 384 * Use the rotation transformation to find the tile coordinates
 385 * (row, col) of a tile in the destination frame that corresponds
 386 * to the given tile coordinates of a source frame. The destination
 387 * coordinate is then converted to a tile index.
 388 */
 389static int transform_tile_index(struct ipu_image_convert_ctx *ctx,
 390				int src_row, int src_col)
 391{
 392	struct ipu_image_convert_chan *chan = ctx->chan;
 393	struct ipu_image_convert_priv *priv = chan->priv;
 394	struct ipu_image_convert_image *s_image = &ctx->in;
 395	struct ipu_image_convert_image *d_image = &ctx->out;
 396	int dst_row, dst_col;
 397
 398	/* with no rotation it's a 1:1 mapping */
 399	if (ctx->rot_mode == IPU_ROTATE_NONE)
 400		return src_row * s_image->num_cols + src_col;
 401
 402	/*
 403	 * before doing the transform, first we have to translate
 404	 * source row,col for an origin in the center of s_image
 405	 */
 406	src_row = src_row * 2 - (s_image->num_rows - 1);
 407	src_col = src_col * 2 - (s_image->num_cols - 1);
 408
 409	/* do the rotation transform */
 410	if (ctx->rot_mode & IPU_ROT_BIT_90) {
 411		dst_col = -src_row;
 412		dst_row = src_col;
 413	} else {
 414		dst_col = src_col;
 415		dst_row = src_row;
 416	}
 417
 418	/* apply flip */
 419	if (ctx->rot_mode & IPU_ROT_BIT_HFLIP)
 420		dst_col = -dst_col;
 421	if (ctx->rot_mode & IPU_ROT_BIT_VFLIP)
 422		dst_row = -dst_row;
 423
 424	dev_dbg(priv->ipu->dev, "task %u: ctx %p: [%d,%d] --> [%d,%d]\n",
 425		chan->ic_task, ctx, src_col, src_row, dst_col, dst_row);
 426
 427	/*
 428	 * finally translate dest row,col using an origin in upper
 429	 * left of d_image
 430	 */
 431	dst_row += d_image->num_rows - 1;
 432	dst_col += d_image->num_cols - 1;
 433	dst_row /= 2;
 434	dst_col /= 2;
 435
 436	return dst_row * d_image->num_cols + dst_col;
 437}
 438
 439/*
 440 * Fill the out_tile_map[] with transformed destination tile indeces.
 441 */
 442static void calc_out_tile_map(struct ipu_image_convert_ctx *ctx)
 443{
 444	struct ipu_image_convert_image *s_image = &ctx->in;
 445	unsigned int row, col, tile = 0;
 446
 447	for (row = 0; row < s_image->num_rows; row++) {
 448		for (col = 0; col < s_image->num_cols; col++) {
 449			ctx->out_tile_map[tile] =
 450				transform_tile_index(ctx, row, col);
 451			tile++;
 452		}
 453	}
 454}
 455
 456static void calc_tile_offsets_planar(struct ipu_image_convert_ctx *ctx,
 457				     struct ipu_image_convert_image *image)
 458{
 459	struct ipu_image_convert_chan *chan = ctx->chan;
 460	struct ipu_image_convert_priv *priv = chan->priv;
 461	const struct ipu_image_pixfmt *fmt = image->fmt;
 462	unsigned int row, col, tile = 0;
 463	u32 H, w, h, y_stride, uv_stride;
 464	u32 uv_row_off, uv_col_off, uv_off, u_off, v_off, tmp;
 465	u32 y_row_off, y_col_off, y_off;
 466	u32 y_size, uv_size;
 467
 468	/* setup some convenience vars */
 469	H = image->base.pix.height;
 470
 471	y_stride = image->stride;
 472	uv_stride = y_stride / fmt->uv_width_dec;
 473	if (fmt->uv_packed)
 474		uv_stride *= 2;
 475
 476	y_size = H * y_stride;
 477	uv_size = y_size / (fmt->uv_width_dec * fmt->uv_height_dec);
 478
 479	for (row = 0; row < image->num_rows; row++) {
 480		w = image->tile[tile].width;
 481		h = image->tile[tile].height;
 482		y_row_off = row * h * y_stride;
 483		uv_row_off = (row * h * uv_stride) / fmt->uv_height_dec;
 484
 485		for (col = 0; col < image->num_cols; col++) {
 486			y_col_off = col * w;
 487			uv_col_off = y_col_off / fmt->uv_width_dec;
 488			if (fmt->uv_packed)
 489				uv_col_off *= 2;
 490
 491			y_off = y_row_off + y_col_off;
 492			uv_off = uv_row_off + uv_col_off;
 493
 494			u_off = y_size - y_off + uv_off;
 495			v_off = (fmt->uv_packed) ? 0 : u_off + uv_size;
 496			if (fmt->uv_swapped) {
 497				tmp = u_off;
 498				u_off = v_off;
 499				v_off = tmp;
 500			}
 501
 502			image->tile[tile].offset = y_off;
 503			image->tile[tile].u_off = u_off;
 504			image->tile[tile++].v_off = v_off;
 505
 506			dev_dbg(priv->ipu->dev,
 507				"task %u: ctx %p: %s@[%d,%d]: y_off %08x, u_off %08x, v_off %08x\n",
 508				chan->ic_task, ctx,
 509				image->type == IMAGE_CONVERT_IN ?
 510				"Input" : "Output", row, col,
 511				y_off, u_off, v_off);
 
 
 
 
 512		}
 513	}
 
 
 514}
 515
 516static void calc_tile_offsets_packed(struct ipu_image_convert_ctx *ctx,
 517				     struct ipu_image_convert_image *image)
 518{
 519	struct ipu_image_convert_chan *chan = ctx->chan;
 520	struct ipu_image_convert_priv *priv = chan->priv;
 521	const struct ipu_image_pixfmt *fmt = image->fmt;
 522	unsigned int row, col, tile = 0;
 523	u32 w, h, bpp, stride;
 524	u32 row_off, col_off;
 525
 526	/* setup some convenience vars */
 527	stride = image->stride;
 528	bpp = fmt->bpp;
 529
 530	for (row = 0; row < image->num_rows; row++) {
 531		w = image->tile[tile].width;
 532		h = image->tile[tile].height;
 533		row_off = row * h * stride;
 534
 535		for (col = 0; col < image->num_cols; col++) {
 536			col_off = (col * w * bpp) >> 3;
 
 
 537
 538			image->tile[tile].offset = row_off + col_off;
 539			image->tile[tile].u_off = 0;
 540			image->tile[tile++].v_off = 0;
 541
 542			dev_dbg(priv->ipu->dev,
 543				"task %u: ctx %p: %s@[%d,%d]: phys %08x\n",
 544				chan->ic_task, ctx,
 545				image->type == IMAGE_CONVERT_IN ?
 546				"Input" : "Output", row, col,
 547				row_off + col_off);
 
 
 
 
 548		}
 549	}
 
 
 550}
 551
 552static void calc_tile_offsets(struct ipu_image_convert_ctx *ctx,
 553			      struct ipu_image_convert_image *image)
 554{
 555	if (image->fmt->planar)
 556		calc_tile_offsets_planar(ctx, image);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 557	else
 558		calc_tile_offsets_packed(ctx, image);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 559}
 560
 561/*
 562 * return the number of runs in given queue (pending_q or done_q)
 563 * for this context. hold irqlock when calling.
 564 */
 565static int get_run_count(struct ipu_image_convert_ctx *ctx,
 566			 struct list_head *q)
 567{
 568	struct ipu_image_convert_run *run;
 569	int count = 0;
 570
 571	lockdep_assert_held(&ctx->chan->irqlock);
 572
 573	list_for_each_entry(run, q, list) {
 574		if (run->ctx == ctx)
 575			count++;
 576	}
 577
 578	return count;
 579}
 580
 581static void convert_stop(struct ipu_image_convert_run *run)
 582{
 583	struct ipu_image_convert_ctx *ctx = run->ctx;
 584	struct ipu_image_convert_chan *chan = ctx->chan;
 585	struct ipu_image_convert_priv *priv = chan->priv;
 586
 587	dev_dbg(priv->ipu->dev, "%s: task %u: stopping ctx %p run %p\n",
 588		__func__, chan->ic_task, ctx, run);
 589
 590	/* disable IC tasks and the channels */
 591	ipu_ic_task_disable(chan->ic);
 592	ipu_idmac_disable_channel(chan->in_chan);
 593	ipu_idmac_disable_channel(chan->out_chan);
 594
 595	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
 596		ipu_idmac_disable_channel(chan->rotation_in_chan);
 597		ipu_idmac_disable_channel(chan->rotation_out_chan);
 598		ipu_idmac_unlink(chan->out_chan, chan->rotation_in_chan);
 599	}
 600
 601	ipu_ic_disable(chan->ic);
 602}
 603
 604static void init_idmac_channel(struct ipu_image_convert_ctx *ctx,
 605			       struct ipuv3_channel *channel,
 606			       struct ipu_image_convert_image *image,
 607			       enum ipu_rotate_mode rot_mode,
 608			       bool rot_swap_width_height)
 
 609{
 610	struct ipu_image_convert_chan *chan = ctx->chan;
 611	unsigned int burst_size;
 612	u32 width, height, stride;
 613	dma_addr_t addr0, addr1 = 0;
 614	struct ipu_image tile_image;
 615	unsigned int tile_idx[2];
 616
 617	if (image->type == IMAGE_CONVERT_OUT) {
 618		tile_idx[0] = ctx->out_tile_map[0];
 619		tile_idx[1] = ctx->out_tile_map[1];
 620	} else {
 621		tile_idx[0] = 0;
 622		tile_idx[1] = 1;
 623	}
 624
 625	if (rot_swap_width_height) {
 626		width = image->tile[0].height;
 627		height = image->tile[0].width;
 628		stride = image->tile[0].rot_stride;
 629		addr0 = ctx->rot_intermediate[0].phys;
 630		if (ctx->double_buffering)
 631			addr1 = ctx->rot_intermediate[1].phys;
 632	} else {
 633		width = image->tile[0].width;
 634		height = image->tile[0].height;
 635		stride = image->stride;
 636		addr0 = image->base.phys0 +
 637			image->tile[tile_idx[0]].offset;
 638		if (ctx->double_buffering)
 639			addr1 = image->base.phys0 +
 640				image->tile[tile_idx[1]].offset;
 641	}
 642
 643	ipu_cpmem_zero(channel);
 644
 645	memset(&tile_image, 0, sizeof(tile_image));
 646	tile_image.pix.width = tile_image.rect.width = width;
 647	tile_image.pix.height = tile_image.rect.height = height;
 648	tile_image.pix.bytesperline = stride;
 649	tile_image.pix.pixelformat =  image->fmt->fourcc;
 650	tile_image.phys0 = addr0;
 651	tile_image.phys1 = addr1;
 
 
 
 
 
 652	ipu_cpmem_set_image(channel, &tile_image);
 653
 654	if (image->fmt->planar && !rot_swap_width_height)
 655		ipu_cpmem_set_uv_offset(channel,
 656					image->tile[tile_idx[0]].u_off,
 657					image->tile[tile_idx[0]].v_off);
 658
 659	if (rot_mode)
 660		ipu_cpmem_set_rotation(channel, rot_mode);
 661
 
 
 
 
 
 
 
 
 
 662	if (channel == chan->rotation_in_chan ||
 663	    channel == chan->rotation_out_chan) {
 664		burst_size = 8;
 665		ipu_cpmem_set_block_mode(channel);
 666	} else
 667		burst_size = (width % 16) ? 8 : 16;
 668
 669	ipu_cpmem_set_burstsize(channel, burst_size);
 670
 671	ipu_ic_task_idma_init(chan->ic, channel, width, height,
 672			      burst_size, rot_mode);
 673
 674	ipu_cpmem_set_axi_id(channel, 1);
 
 
 
 
 
 675
 676	ipu_idmac_set_double_buffer(channel, ctx->double_buffering);
 677}
 678
 679static int convert_start(struct ipu_image_convert_run *run)
 680{
 681	struct ipu_image_convert_ctx *ctx = run->ctx;
 682	struct ipu_image_convert_chan *chan = ctx->chan;
 683	struct ipu_image_convert_priv *priv = chan->priv;
 684	struct ipu_image_convert_image *s_image = &ctx->in;
 685	struct ipu_image_convert_image *d_image = &ctx->out;
 686	enum ipu_color_space src_cs, dest_cs;
 687	unsigned int dest_width, dest_height;
 
 
 688	int ret;
 689
 690	dev_dbg(priv->ipu->dev, "%s: task %u: starting ctx %p run %p\n",
 691		__func__, chan->ic_task, ctx, run);
 692
 693	src_cs = ipu_pixelformat_to_colorspace(s_image->fmt->fourcc);
 694	dest_cs = ipu_pixelformat_to_colorspace(d_image->fmt->fourcc);
 695
 696	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
 697		/* swap width/height for resizer */
 698		dest_width = d_image->tile[0].height;
 699		dest_height = d_image->tile[0].width;
 700	} else {
 701		dest_width = d_image->tile[0].width;
 702		dest_height = d_image->tile[0].height;
 703	}
 704
 
 
 
 
 
 
 
 
 
 
 
 
 705	/* setup the IC resizer and CSC */
 706	ret = ipu_ic_task_init(chan->ic,
 707			       s_image->tile[0].width,
 708			       s_image->tile[0].height,
 709			       dest_width,
 710			       dest_height,
 711			       src_cs, dest_cs);
 712	if (ret) {
 713		dev_err(priv->ipu->dev, "ipu_ic_task_init failed, %d\n", ret);
 714		return ret;
 715	}
 716
 717	/* init the source MEM-->IC PP IDMAC channel */
 718	init_idmac_channel(ctx, chan->in_chan, s_image,
 719			   IPU_ROTATE_NONE, false);
 720
 721	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
 722		/* init the IC PP-->MEM IDMAC channel */
 723		init_idmac_channel(ctx, chan->out_chan, d_image,
 724				   IPU_ROTATE_NONE, true);
 725
 726		/* init the MEM-->IC PP ROT IDMAC channel */
 727		init_idmac_channel(ctx, chan->rotation_in_chan, d_image,
 728				   ctx->rot_mode, true);
 729
 730		/* init the destination IC PP ROT-->MEM IDMAC channel */
 731		init_idmac_channel(ctx, chan->rotation_out_chan, d_image,
 732				   IPU_ROTATE_NONE, false);
 733
 734		/* now link IC PP-->MEM to MEM-->IC PP ROT */
 735		ipu_idmac_link(chan->out_chan, chan->rotation_in_chan);
 736	} else {
 737		/* init the destination IC PP-->MEM IDMAC channel */
 738		init_idmac_channel(ctx, chan->out_chan, d_image,
 739				   ctx->rot_mode, false);
 740	}
 741
 742	/* enable the IC */
 743	ipu_ic_enable(chan->ic);
 744
 745	/* set buffers ready */
 746	ipu_idmac_select_buffer(chan->in_chan, 0);
 747	ipu_idmac_select_buffer(chan->out_chan, 0);
 748	if (ipu_rot_mode_is_irt(ctx->rot_mode))
 749		ipu_idmac_select_buffer(chan->rotation_out_chan, 0);
 750	if (ctx->double_buffering) {
 751		ipu_idmac_select_buffer(chan->in_chan, 1);
 752		ipu_idmac_select_buffer(chan->out_chan, 1);
 753		if (ipu_rot_mode_is_irt(ctx->rot_mode))
 754			ipu_idmac_select_buffer(chan->rotation_out_chan, 1);
 755	}
 756
 757	/* enable the channels! */
 758	ipu_idmac_enable_channel(chan->in_chan);
 759	ipu_idmac_enable_channel(chan->out_chan);
 760	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
 761		ipu_idmac_enable_channel(chan->rotation_in_chan);
 762		ipu_idmac_enable_channel(chan->rotation_out_chan);
 763	}
 764
 765	ipu_ic_task_enable(chan->ic);
 766
 767	ipu_cpmem_dump(chan->in_chan);
 768	ipu_cpmem_dump(chan->out_chan);
 769	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
 770		ipu_cpmem_dump(chan->rotation_in_chan);
 771		ipu_cpmem_dump(chan->rotation_out_chan);
 772	}
 773
 774	ipu_dump(priv->ipu);
 775
 776	return 0;
 777}
 778
 779/* hold irqlock when calling */
 780static int do_run(struct ipu_image_convert_run *run)
 781{
 782	struct ipu_image_convert_ctx *ctx = run->ctx;
 783	struct ipu_image_convert_chan *chan = ctx->chan;
 784
 785	lockdep_assert_held(&chan->irqlock);
 786
 787	ctx->in.base.phys0 = run->in_phys;
 788	ctx->out.base.phys0 = run->out_phys;
 789
 790	ctx->cur_buf_num = 0;
 791	ctx->next_tile = 1;
 792
 793	/* remove run from pending_q and set as current */
 794	list_del(&run->list);
 795	chan->current_run = run;
 796
 797	return convert_start(run);
 798}
 799
 800/* hold irqlock when calling */
 801static void run_next(struct ipu_image_convert_chan *chan)
 802{
 803	struct ipu_image_convert_priv *priv = chan->priv;
 804	struct ipu_image_convert_run *run, *tmp;
 805	int ret;
 806
 807	lockdep_assert_held(&chan->irqlock);
 808
 809	list_for_each_entry_safe(run, tmp, &chan->pending_q, list) {
 810		/* skip contexts that are aborting */
 811		if (run->ctx->aborting) {
 812			dev_dbg(priv->ipu->dev,
 813				"%s: task %u: skipping aborting ctx %p run %p\n",
 814				__func__, chan->ic_task, run->ctx, run);
 815			continue;
 816		}
 817
 818		ret = do_run(run);
 819		if (!ret)
 820			break;
 821
 822		/*
 823		 * something went wrong with start, add the run
 824		 * to done q and continue to the next run in the
 825		 * pending q.
 826		 */
 827		run->status = ret;
 828		list_add_tail(&run->list, &chan->done_q);
 829		chan->current_run = NULL;
 830	}
 831}
 832
 833static void empty_done_q(struct ipu_image_convert_chan *chan)
 834{
 835	struct ipu_image_convert_priv *priv = chan->priv;
 836	struct ipu_image_convert_run *run;
 837	unsigned long flags;
 838
 839	spin_lock_irqsave(&chan->irqlock, flags);
 840
 841	while (!list_empty(&chan->done_q)) {
 842		run = list_entry(chan->done_q.next,
 843				 struct ipu_image_convert_run,
 844				 list);
 845
 846		list_del(&run->list);
 847
 848		dev_dbg(priv->ipu->dev,
 849			"%s: task %u: completing ctx %p run %p with %d\n",
 850			__func__, chan->ic_task, run->ctx, run, run->status);
 851
 852		/* call the completion callback and free the run */
 853		spin_unlock_irqrestore(&chan->irqlock, flags);
 854		run->ctx->complete(run, run->ctx->complete_context);
 855		spin_lock_irqsave(&chan->irqlock, flags);
 856	}
 857
 858	spin_unlock_irqrestore(&chan->irqlock, flags);
 859}
 860
 861/*
 862 * the bottom half thread clears out the done_q, calling the
 863 * completion handler for each.
 864 */
 865static irqreturn_t do_bh(int irq, void *dev_id)
 866{
 867	struct ipu_image_convert_chan *chan = dev_id;
 868	struct ipu_image_convert_priv *priv = chan->priv;
 869	struct ipu_image_convert_ctx *ctx;
 870	unsigned long flags;
 871
 872	dev_dbg(priv->ipu->dev, "%s: task %u: enter\n", __func__,
 873		chan->ic_task);
 874
 875	empty_done_q(chan);
 876
 877	spin_lock_irqsave(&chan->irqlock, flags);
 878
 879	/*
 880	 * the done_q is cleared out, signal any contexts
 881	 * that are aborting that abort can complete.
 882	 */
 883	list_for_each_entry(ctx, &chan->ctx_list, list) {
 884		if (ctx->aborting) {
 885			dev_dbg(priv->ipu->dev,
 886				"%s: task %u: signaling abort for ctx %p\n",
 887				__func__, chan->ic_task, ctx);
 888			complete(&ctx->aborted);
 889		}
 890	}
 891
 892	spin_unlock_irqrestore(&chan->irqlock, flags);
 893
 894	dev_dbg(priv->ipu->dev, "%s: task %u: exit\n", __func__,
 895		chan->ic_task);
 896
 897	return IRQ_HANDLED;
 898}
 899
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 900/* hold irqlock when calling */
 901static irqreturn_t do_irq(struct ipu_image_convert_run *run)
 902{
 903	struct ipu_image_convert_ctx *ctx = run->ctx;
 904	struct ipu_image_convert_chan *chan = ctx->chan;
 905	struct ipu_image_tile *src_tile, *dst_tile;
 906	struct ipu_image_convert_image *s_image = &ctx->in;
 907	struct ipu_image_convert_image *d_image = &ctx->out;
 908	struct ipuv3_channel *outch;
 909	unsigned int dst_idx;
 910
 911	lockdep_assert_held(&chan->irqlock);
 912
 913	outch = ipu_rot_mode_is_irt(ctx->rot_mode) ?
 914		chan->rotation_out_chan : chan->out_chan;
 915
 916	/*
 917	 * It is difficult to stop the channel DMA before the channels
 918	 * enter the paused state. Without double-buffering the channels
 919	 * are always in a paused state when the EOF irq occurs, so it
 920	 * is safe to stop the channels now. For double-buffering we
 921	 * just ignore the abort until the operation completes, when it
 922	 * is safe to shut down.
 923	 */
 924	if (ctx->aborting && !ctx->double_buffering) {
 925		convert_stop(run);
 926		run->status = -EIO;
 927		goto done;
 928	}
 929
 930	if (ctx->next_tile == ctx->num_tiles) {
 931		/*
 932		 * the conversion is complete
 933		 */
 934		convert_stop(run);
 935		run->status = 0;
 936		goto done;
 937	}
 938
 939	/*
 940	 * not done, place the next tile buffers.
 941	 */
 942	if (!ctx->double_buffering) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 943
 944		src_tile = &s_image->tile[ctx->next_tile];
 945		dst_idx = ctx->out_tile_map[ctx->next_tile];
 946		dst_tile = &d_image->tile[dst_idx];
 947
 948		ipu_cpmem_set_buffer(chan->in_chan, 0,
 949				     s_image->base.phys0 + src_tile->offset);
 950		ipu_cpmem_set_buffer(outch, 0,
 951				     d_image->base.phys0 + dst_tile->offset);
 952		if (s_image->fmt->planar)
 953			ipu_cpmem_set_uv_offset(chan->in_chan,
 954						src_tile->u_off,
 955						src_tile->v_off);
 956		if (d_image->fmt->planar)
 957			ipu_cpmem_set_uv_offset(outch,
 958						dst_tile->u_off,
 959						dst_tile->v_off);
 960
 961		ipu_idmac_select_buffer(chan->in_chan, 0);
 962		ipu_idmac_select_buffer(outch, 0);
 963
 964	} else if (ctx->next_tile < ctx->num_tiles - 1) {
 965
 966		src_tile = &s_image->tile[ctx->next_tile + 1];
 967		dst_idx = ctx->out_tile_map[ctx->next_tile + 1];
 968		dst_tile = &d_image->tile[dst_idx];
 969
 970		ipu_cpmem_set_buffer(chan->in_chan, ctx->cur_buf_num,
 971				     s_image->base.phys0 + src_tile->offset);
 972		ipu_cpmem_set_buffer(outch, ctx->cur_buf_num,
 973				     d_image->base.phys0 + dst_tile->offset);
 974
 975		ipu_idmac_select_buffer(chan->in_chan, ctx->cur_buf_num);
 976		ipu_idmac_select_buffer(outch, ctx->cur_buf_num);
 977
 978		ctx->cur_buf_num ^= 1;
 979	}
 980
 
 981	ctx->next_tile++;
 982	return IRQ_HANDLED;
 983done:
 984	list_add_tail(&run->list, &chan->done_q);
 985	chan->current_run = NULL;
 986	run_next(chan);
 987	return IRQ_WAKE_THREAD;
 988}
 989
 990static irqreturn_t norotate_irq(int irq, void *data)
 991{
 992	struct ipu_image_convert_chan *chan = data;
 
 993	struct ipu_image_convert_ctx *ctx;
 994	struct ipu_image_convert_run *run;
 
 
 995	unsigned long flags;
 996	irqreturn_t ret;
 997
 998	spin_lock_irqsave(&chan->irqlock, flags);
 999
1000	/* get current run and its context */
1001	run = chan->current_run;
1002	if (!run) {
1003		ret = IRQ_NONE;
1004		goto out;
1005	}
1006
1007	ctx = run->ctx;
1008
1009	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
1010		/* this is a rotation operation, just ignore */
1011		spin_unlock_irqrestore(&chan->irqlock, flags);
1012		return IRQ_HANDLED;
1013	}
1014
1015	ret = do_irq(run);
1016out:
1017	spin_unlock_irqrestore(&chan->irqlock, flags);
1018	return ret;
1019}
1020
1021static irqreturn_t rotate_irq(int irq, void *data)
1022{
1023	struct ipu_image_convert_chan *chan = data;
1024	struct ipu_image_convert_priv *priv = chan->priv;
1025	struct ipu_image_convert_ctx *ctx;
1026	struct ipu_image_convert_run *run;
1027	unsigned long flags;
1028	irqreturn_t ret;
1029
1030	spin_lock_irqsave(&chan->irqlock, flags);
1031
1032	/* get current run and its context */
1033	run = chan->current_run;
1034	if (!run) {
1035		ret = IRQ_NONE;
1036		goto out;
1037	}
1038
1039	ctx = run->ctx;
 
 
 
1040
1041	if (!ipu_rot_mode_is_irt(ctx->rot_mode)) {
1042		/* this was NOT a rotation operation, shouldn't happen */
1043		dev_err(priv->ipu->dev, "Unexpected rotation interrupt\n");
1044		spin_unlock_irqrestore(&chan->irqlock, flags);
1045		return IRQ_HANDLED;
1046	}
1047
1048	ret = do_irq(run);
1049out:
1050	spin_unlock_irqrestore(&chan->irqlock, flags);
1051	return ret;
1052}
1053
1054/*
1055 * try to force the completion of runs for this ctx. Called when
1056 * abort wait times out in ipu_image_convert_abort().
1057 */
1058static void force_abort(struct ipu_image_convert_ctx *ctx)
1059{
1060	struct ipu_image_convert_chan *chan = ctx->chan;
1061	struct ipu_image_convert_run *run;
1062	unsigned long flags;
1063
1064	spin_lock_irqsave(&chan->irqlock, flags);
1065
1066	run = chan->current_run;
1067	if (run && run->ctx == ctx) {
1068		convert_stop(run);
1069		run->status = -EIO;
1070		list_add_tail(&run->list, &chan->done_q);
1071		chan->current_run = NULL;
1072		run_next(chan);
1073	}
1074
1075	spin_unlock_irqrestore(&chan->irqlock, flags);
1076
1077	empty_done_q(chan);
1078}
1079
1080static void release_ipu_resources(struct ipu_image_convert_chan *chan)
1081{
 
 
 
 
1082	if (chan->out_eof_irq >= 0)
1083		free_irq(chan->out_eof_irq, chan);
1084	if (chan->rot_out_eof_irq >= 0)
1085		free_irq(chan->rot_out_eof_irq, chan);
1086
1087	if (!IS_ERR_OR_NULL(chan->in_chan))
1088		ipu_idmac_put(chan->in_chan);
1089	if (!IS_ERR_OR_NULL(chan->out_chan))
1090		ipu_idmac_put(chan->out_chan);
1091	if (!IS_ERR_OR_NULL(chan->rotation_in_chan))
1092		ipu_idmac_put(chan->rotation_in_chan);
1093	if (!IS_ERR_OR_NULL(chan->rotation_out_chan))
1094		ipu_idmac_put(chan->rotation_out_chan);
1095	if (!IS_ERR_OR_NULL(chan->ic))
1096		ipu_ic_put(chan->ic);
1097
1098	chan->in_chan = chan->out_chan = chan->rotation_in_chan =
1099		chan->rotation_out_chan = NULL;
1100	chan->out_eof_irq = chan->rot_out_eof_irq = -1;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1101}
1102
1103static int get_ipu_resources(struct ipu_image_convert_chan *chan)
1104{
1105	const struct ipu_image_convert_dma_chan *dma = chan->dma_ch;
1106	struct ipu_image_convert_priv *priv = chan->priv;
1107	int ret;
1108
1109	/* get IC */
1110	chan->ic = ipu_ic_get(priv->ipu, chan->ic_task);
1111	if (IS_ERR(chan->ic)) {
1112		dev_err(priv->ipu->dev, "could not acquire IC\n");
1113		ret = PTR_ERR(chan->ic);
1114		goto err;
1115	}
1116
1117	/* get IDMAC channels */
1118	chan->in_chan = ipu_idmac_get(priv->ipu, dma->in);
1119	chan->out_chan = ipu_idmac_get(priv->ipu, dma->out);
1120	if (IS_ERR(chan->in_chan) || IS_ERR(chan->out_chan)) {
1121		dev_err(priv->ipu->dev, "could not acquire idmac channels\n");
1122		ret = -EBUSY;
1123		goto err;
1124	}
1125
1126	chan->rotation_in_chan = ipu_idmac_get(priv->ipu, dma->rot_in);
1127	chan->rotation_out_chan = ipu_idmac_get(priv->ipu, dma->rot_out);
1128	if (IS_ERR(chan->rotation_in_chan) || IS_ERR(chan->rotation_out_chan)) {
1129		dev_err(priv->ipu->dev,
1130			"could not acquire idmac rotation channels\n");
1131		ret = -EBUSY;
1132		goto err;
1133	}
1134
1135	/* acquire the EOF interrupts */
1136	chan->out_eof_irq = ipu_idmac_channel_irq(priv->ipu,
1137						  chan->out_chan,
1138						  IPU_IRQ_EOF);
 
 
 
 
 
 
 
 
 
 
1139
1140	ret = request_threaded_irq(chan->out_eof_irq, norotate_irq, do_bh,
1141				   0, "ipu-ic", chan);
1142	if (ret < 0) {
1143		dev_err(priv->ipu->dev, "could not acquire irq %d\n",
1144			 chan->out_eof_irq);
1145		chan->out_eof_irq = -1;
1146		goto err;
1147	}
 
1148
1149	chan->rot_out_eof_irq = ipu_idmac_channel_irq(priv->ipu,
1150						     chan->rotation_out_chan,
1151						     IPU_IRQ_EOF);
1152
1153	ret = request_threaded_irq(chan->rot_out_eof_irq, rotate_irq, do_bh,
1154				   0, "ipu-ic", chan);
1155	if (ret < 0) {
1156		dev_err(priv->ipu->dev, "could not acquire irq %d\n",
1157			chan->rot_out_eof_irq);
1158		chan->rot_out_eof_irq = -1;
1159		goto err;
1160	}
 
1161
1162	return 0;
1163err:
1164	release_ipu_resources(chan);
1165	return ret;
1166}
1167
1168static int fill_image(struct ipu_image_convert_ctx *ctx,
1169		      struct ipu_image_convert_image *ic_image,
1170		      struct ipu_image *image,
1171		      enum ipu_image_convert_type type)
1172{
1173	struct ipu_image_convert_priv *priv = ctx->chan->priv;
1174
1175	ic_image->base = *image;
1176	ic_image->type = type;
1177
1178	ic_image->fmt = get_format(image->pix.pixelformat);
1179	if (!ic_image->fmt) {
1180		dev_err(priv->ipu->dev, "pixelformat not supported for %s\n",
1181			type == IMAGE_CONVERT_OUT ? "Output" : "Input");
1182		return -EINVAL;
1183	}
1184
1185	if (ic_image->fmt->planar)
1186		ic_image->stride = ic_image->base.pix.width;
1187	else
1188		ic_image->stride  = ic_image->base.pix.bytesperline;
1189
1190	calc_tile_dimensions(ctx, ic_image);
1191	calc_tile_offsets(ctx, ic_image);
1192
1193	return 0;
1194}
1195
1196/* borrowed from drivers/media/v4l2-core/v4l2-common.c */
1197static unsigned int clamp_align(unsigned int x, unsigned int min,
1198				unsigned int max, unsigned int align)
1199{
1200	/* Bits that must be zero to be aligned */
1201	unsigned int mask = ~((1 << align) - 1);
1202
1203	/* Clamp to aligned min and max */
1204	x = clamp(x, (min + ~mask) & mask, max & mask);
1205
1206	/* Round to nearest aligned value */
1207	if (align)
1208		x = (x + (1 << (align - 1))) & mask;
1209
1210	return x;
1211}
1212
1213/*
1214 * We have to adjust the tile width such that the tile physaddrs and
1215 * U and V plane offsets are multiples of 8 bytes as required by
1216 * the IPU DMA Controller. For the planar formats, this corresponds
1217 * to a pixel alignment of 16 (but use a more formal equation since
1218 * the variables are available). For all the packed formats, 8 is
1219 * good enough.
1220 */
1221static inline u32 tile_width_align(const struct ipu_image_pixfmt *fmt)
1222{
1223	return fmt->planar ? 8 * fmt->uv_width_dec : 8;
1224}
1225
1226/*
1227 * For tile height alignment, we have to ensure that the output tile
1228 * heights are multiples of 8 lines if the IRT is required by the
1229 * given rotation mode (the IRT performs rotations on 8x8 blocks
1230 * at a time). If the IRT is not used, or for input image tiles,
1231 * 2 lines are good enough.
1232 */
1233static inline u32 tile_height_align(enum ipu_image_convert_type type,
1234				    enum ipu_rotate_mode rot_mode)
1235{
1236	return (type == IMAGE_CONVERT_OUT &&
1237		ipu_rot_mode_is_irt(rot_mode)) ? 8 : 2;
1238}
1239
1240/* Adjusts input/output images to IPU restrictions */
1241void ipu_image_convert_adjust(struct ipu_image *in, struct ipu_image *out,
1242			      enum ipu_rotate_mode rot_mode)
1243{
1244	const struct ipu_image_pixfmt *infmt, *outfmt;
1245	unsigned int num_in_rows, num_in_cols;
1246	unsigned int num_out_rows, num_out_cols;
1247	u32 w_align, h_align;
1248
1249	infmt = get_format(in->pix.pixelformat);
1250	outfmt = get_format(out->pix.pixelformat);
1251
1252	/* set some default pixel formats if needed */
1253	if (!infmt) {
1254		in->pix.pixelformat = V4L2_PIX_FMT_RGB24;
1255		infmt = get_format(V4L2_PIX_FMT_RGB24);
1256	}
1257	if (!outfmt) {
1258		out->pix.pixelformat = V4L2_PIX_FMT_RGB24;
1259		outfmt = get_format(V4L2_PIX_FMT_RGB24);
1260	}
1261
1262	/* image converter does not handle fields */
1263	in->pix.field = out->pix.field = V4L2_FIELD_NONE;
1264
1265	/* resizer cannot downsize more than 4:1 */
1266	if (ipu_rot_mode_is_irt(rot_mode)) {
1267		out->pix.height = max_t(__u32, out->pix.height,
1268					in->pix.width / 4);
1269		out->pix.width = max_t(__u32, out->pix.width,
1270				       in->pix.height / 4);
1271	} else {
1272		out->pix.width = max_t(__u32, out->pix.width,
1273				       in->pix.width / 4);
1274		out->pix.height = max_t(__u32, out->pix.height,
1275					in->pix.height / 4);
1276	}
1277
1278	/* get tiling rows/cols from output format */
1279	num_out_rows = num_stripes(out->pix.height);
1280	num_out_cols = num_stripes(out->pix.width);
1281	if (ipu_rot_mode_is_irt(rot_mode)) {
1282		num_in_rows = num_out_cols;
1283		num_in_cols = num_out_rows;
1284	} else {
1285		num_in_rows = num_out_rows;
1286		num_in_cols = num_out_cols;
1287	}
1288
1289	/* align input width/height */
1290	w_align = ilog2(tile_width_align(infmt) * num_in_cols);
1291	h_align = ilog2(tile_height_align(IMAGE_CONVERT_IN, rot_mode) *
1292			num_in_rows);
1293	in->pix.width = clamp_align(in->pix.width, MIN_W, MAX_W, w_align);
1294	in->pix.height = clamp_align(in->pix.height, MIN_H, MAX_H, h_align);
 
 
 
1295
1296	/* align output width/height */
1297	w_align = ilog2(tile_width_align(outfmt) * num_out_cols);
1298	h_align = ilog2(tile_height_align(IMAGE_CONVERT_OUT, rot_mode) *
1299			num_out_rows);
1300	out->pix.width = clamp_align(out->pix.width, MIN_W, MAX_W, w_align);
1301	out->pix.height = clamp_align(out->pix.height, MIN_H, MAX_H, h_align);
 
 
 
1302
1303	/* set input/output strides and image sizes */
1304	in->pix.bytesperline = (in->pix.width * infmt->bpp) >> 3;
1305	in->pix.sizeimage = in->pix.height * in->pix.bytesperline;
1306	out->pix.bytesperline = (out->pix.width * outfmt->bpp) >> 3;
1307	out->pix.sizeimage = out->pix.height * out->pix.bytesperline;
 
 
 
 
 
 
 
 
 
 
 
1308}
1309EXPORT_SYMBOL_GPL(ipu_image_convert_adjust);
1310
1311/*
1312 * this is used by ipu_image_convert_prepare() to verify set input and
1313 * output images are valid before starting the conversion. Clients can
1314 * also call it before calling ipu_image_convert_prepare().
1315 */
1316int ipu_image_convert_verify(struct ipu_image *in, struct ipu_image *out,
1317			     enum ipu_rotate_mode rot_mode)
1318{
1319	struct ipu_image testin, testout;
1320
1321	testin = *in;
1322	testout = *out;
1323
1324	ipu_image_convert_adjust(&testin, &testout, rot_mode);
1325
1326	if (testin.pix.width != in->pix.width ||
1327	    testin.pix.height != in->pix.height ||
1328	    testout.pix.width != out->pix.width ||
1329	    testout.pix.height != out->pix.height)
1330		return -EINVAL;
1331
1332	return 0;
1333}
1334EXPORT_SYMBOL_GPL(ipu_image_convert_verify);
1335
1336/*
1337 * Call ipu_image_convert_prepare() to prepare for the conversion of
1338 * given images and rotation mode. Returns a new conversion context.
1339 */
1340struct ipu_image_convert_ctx *
1341ipu_image_convert_prepare(struct ipu_soc *ipu, enum ipu_ic_task ic_task,
1342			  struct ipu_image *in, struct ipu_image *out,
1343			  enum ipu_rotate_mode rot_mode,
1344			  ipu_image_convert_cb_t complete,
1345			  void *complete_context)
1346{
1347	struct ipu_image_convert_priv *priv = ipu->image_convert_priv;
1348	struct ipu_image_convert_image *s_image, *d_image;
1349	struct ipu_image_convert_chan *chan;
1350	struct ipu_image_convert_ctx *ctx;
1351	unsigned long flags;
 
1352	bool get_res;
1353	int ret;
1354
1355	if (!in || !out || !complete ||
1356	    (ic_task != IC_TASK_VIEWFINDER &&
1357	     ic_task != IC_TASK_POST_PROCESSOR))
1358		return ERR_PTR(-EINVAL);
1359
1360	/* verify the in/out images before continuing */
1361	ret = ipu_image_convert_verify(in, out, rot_mode);
1362	if (ret) {
1363		dev_err(priv->ipu->dev, "%s: in/out formats invalid\n",
1364			__func__);
1365		return ERR_PTR(ret);
1366	}
1367
1368	chan = &priv->chan[ic_task];
1369
1370	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1371	if (!ctx)
1372		return ERR_PTR(-ENOMEM);
1373
1374	dev_dbg(priv->ipu->dev, "%s: task %u: ctx %p\n", __func__,
1375		chan->ic_task, ctx);
1376
1377	ctx->chan = chan;
1378	init_completion(&ctx->aborted);
1379
 
 
 
 
 
 
 
1380	s_image = &ctx->in;
1381	d_image = &ctx->out;
1382
1383	/* set tiling and rotation */
1384	d_image->num_rows = num_stripes(out->pix.height);
1385	d_image->num_cols = num_stripes(out->pix.width);
1386	if (ipu_rot_mode_is_irt(rot_mode)) {
1387		s_image->num_rows = d_image->num_cols;
1388		s_image->num_cols = d_image->num_rows;
1389	} else {
1390		s_image->num_rows = d_image->num_rows;
1391		s_image->num_cols = d_image->num_cols;
1392	}
1393
1394	ctx->num_tiles = d_image->num_cols * d_image->num_rows;
1395	ctx->rot_mode = rot_mode;
1396
1397	ret = fill_image(ctx, s_image, in, IMAGE_CONVERT_IN);
1398	if (ret)
1399		goto out_free;
1400	ret = fill_image(ctx, d_image, out, IMAGE_CONVERT_OUT);
1401	if (ret)
1402		goto out_free;
1403
1404	calc_out_tile_map(ctx);
1405
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1406	dump_format(ctx, s_image);
1407	dump_format(ctx, d_image);
1408
1409	ctx->complete = complete;
1410	ctx->complete_context = complete_context;
1411
1412	/*
1413	 * Can we use double-buffering for this operation? If there is
1414	 * only one tile (the whole image can be converted in a single
1415	 * operation) there's no point in using double-buffering. Also,
1416	 * the IPU's IDMAC channels allow only a single U and V plane
1417	 * offset shared between both buffers, but these offsets change
1418	 * for every tile, and therefore would have to be updated for
1419	 * each buffer which is not possible. So double-buffering is
1420	 * impossible when either the source or destination images are
1421	 * a planar format (YUV420, YUV422P, etc.).
 
 
1422	 */
1423	ctx->double_buffering = (ctx->num_tiles > 1 &&
1424				 !s_image->fmt->planar &&
1425				 !d_image->fmt->planar);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1426
1427	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
 
 
 
 
 
 
 
1428		ret = alloc_dma_buf(priv, &ctx->rot_intermediate[0],
1429				    d_image->tile[0].size);
1430		if (ret)
1431			goto out_free;
1432		if (ctx->double_buffering) {
1433			ret = alloc_dma_buf(priv,
1434					    &ctx->rot_intermediate[1],
1435					    d_image->tile[0].size);
1436			if (ret)
1437				goto out_free_dmabuf0;
1438		}
1439	}
1440
1441	spin_lock_irqsave(&chan->irqlock, flags);
1442
1443	get_res = list_empty(&chan->ctx_list);
1444
1445	list_add_tail(&ctx->list, &chan->ctx_list);
1446
1447	spin_unlock_irqrestore(&chan->irqlock, flags);
1448
1449	if (get_res) {
1450		ret = get_ipu_resources(chan);
1451		if (ret)
1452			goto out_free_dmabuf1;
1453	}
1454
1455	return ctx;
1456
1457out_free_dmabuf1:
1458	free_dma_buf(priv, &ctx->rot_intermediate[1]);
1459	spin_lock_irqsave(&chan->irqlock, flags);
1460	list_del(&ctx->list);
1461	spin_unlock_irqrestore(&chan->irqlock, flags);
1462out_free_dmabuf0:
1463	free_dma_buf(priv, &ctx->rot_intermediate[0]);
1464out_free:
1465	kfree(ctx);
1466	return ERR_PTR(ret);
1467}
1468EXPORT_SYMBOL_GPL(ipu_image_convert_prepare);
1469
1470/*
1471 * Carry out a single image conversion run. Only the physaddr's of the input
1472 * and output image buffers are needed. The conversion context must have
1473 * been created previously with ipu_image_convert_prepare().
1474 */
1475int ipu_image_convert_queue(struct ipu_image_convert_run *run)
1476{
1477	struct ipu_image_convert_chan *chan;
1478	struct ipu_image_convert_priv *priv;
1479	struct ipu_image_convert_ctx *ctx;
1480	unsigned long flags;
1481	int ret = 0;
1482
1483	if (!run || !run->ctx || !run->in_phys || !run->out_phys)
1484		return -EINVAL;
1485
1486	ctx = run->ctx;
1487	chan = ctx->chan;
1488	priv = chan->priv;
1489
1490	dev_dbg(priv->ipu->dev, "%s: task %u: ctx %p run %p\n", __func__,
1491		chan->ic_task, ctx, run);
1492
1493	INIT_LIST_HEAD(&run->list);
1494
1495	spin_lock_irqsave(&chan->irqlock, flags);
1496
1497	if (ctx->aborting) {
1498		ret = -EIO;
1499		goto unlock;
1500	}
1501
1502	list_add_tail(&run->list, &chan->pending_q);
1503
1504	if (!chan->current_run) {
1505		ret = do_run(run);
1506		if (ret)
1507			chan->current_run = NULL;
1508	}
1509unlock:
1510	spin_unlock_irqrestore(&chan->irqlock, flags);
1511	return ret;
1512}
1513EXPORT_SYMBOL_GPL(ipu_image_convert_queue);
1514
1515/* Abort any active or pending conversions for this context */
1516void ipu_image_convert_abort(struct ipu_image_convert_ctx *ctx)
1517{
1518	struct ipu_image_convert_chan *chan = ctx->chan;
1519	struct ipu_image_convert_priv *priv = chan->priv;
1520	struct ipu_image_convert_run *run, *active_run, *tmp;
1521	unsigned long flags;
1522	int run_count, ret;
1523	bool need_abort;
1524
1525	reinit_completion(&ctx->aborted);
1526
1527	spin_lock_irqsave(&chan->irqlock, flags);
1528
1529	/* move all remaining pending runs in this context to done_q */
1530	list_for_each_entry_safe(run, tmp, &chan->pending_q, list) {
1531		if (run->ctx != ctx)
1532			continue;
1533		run->status = -EIO;
1534		list_move_tail(&run->list, &chan->done_q);
1535	}
1536
1537	run_count = get_run_count(ctx, &chan->done_q);
1538	active_run = (chan->current_run && chan->current_run->ctx == ctx) ?
1539		chan->current_run : NULL;
1540
1541	need_abort = (run_count || active_run);
 
1542
1543	ctx->aborting = need_abort;
1544
1545	spin_unlock_irqrestore(&chan->irqlock, flags);
1546
1547	if (!need_abort) {
1548		dev_dbg(priv->ipu->dev,
1549			"%s: task %u: no abort needed for ctx %p\n",
1550			__func__, chan->ic_task, ctx);
1551		return;
1552	}
1553
 
 
 
 
 
1554	dev_dbg(priv->ipu->dev,
1555		"%s: task %u: wait for completion: %d runs, active run %p\n",
1556		__func__, chan->ic_task, run_count, active_run);
1557
1558	ret = wait_for_completion_timeout(&ctx->aborted,
1559					  msecs_to_jiffies(10000));
1560	if (ret == 0) {
1561		dev_warn(priv->ipu->dev, "%s: timeout\n", __func__);
1562		force_abort(ctx);
1563	}
 
1564
 
 
 
1565	ctx->aborting = false;
1566}
1567EXPORT_SYMBOL_GPL(ipu_image_convert_abort);
1568
1569/* Unprepare image conversion context */
1570void ipu_image_convert_unprepare(struct ipu_image_convert_ctx *ctx)
1571{
1572	struct ipu_image_convert_chan *chan = ctx->chan;
1573	struct ipu_image_convert_priv *priv = chan->priv;
1574	unsigned long flags;
1575	bool put_res;
1576
1577	/* make sure no runs are hanging around */
1578	ipu_image_convert_abort(ctx);
1579
1580	dev_dbg(priv->ipu->dev, "%s: task %u: removing ctx %p\n", __func__,
1581		chan->ic_task, ctx);
1582
1583	spin_lock_irqsave(&chan->irqlock, flags);
1584
1585	list_del(&ctx->list);
1586
1587	put_res = list_empty(&chan->ctx_list);
1588
1589	spin_unlock_irqrestore(&chan->irqlock, flags);
1590
1591	if (put_res)
1592		release_ipu_resources(chan);
1593
1594	free_dma_buf(priv, &ctx->rot_intermediate[1]);
1595	free_dma_buf(priv, &ctx->rot_intermediate[0]);
1596
1597	kfree(ctx);
1598}
1599EXPORT_SYMBOL_GPL(ipu_image_convert_unprepare);
1600
1601/*
1602 * "Canned" asynchronous single image conversion. Allocates and returns
1603 * a new conversion run.  On successful return the caller must free the
1604 * run and call ipu_image_convert_unprepare() after conversion completes.
1605 */
1606struct ipu_image_convert_run *
1607ipu_image_convert(struct ipu_soc *ipu, enum ipu_ic_task ic_task,
1608		  struct ipu_image *in, struct ipu_image *out,
1609		  enum ipu_rotate_mode rot_mode,
1610		  ipu_image_convert_cb_t complete,
1611		  void *complete_context)
1612{
1613	struct ipu_image_convert_ctx *ctx;
1614	struct ipu_image_convert_run *run;
1615	int ret;
1616
1617	ctx = ipu_image_convert_prepare(ipu, ic_task, in, out, rot_mode,
1618					complete, complete_context);
1619	if (IS_ERR(ctx))
1620		return ERR_CAST(ctx);
1621
1622	run = kzalloc(sizeof(*run), GFP_KERNEL);
1623	if (!run) {
1624		ipu_image_convert_unprepare(ctx);
1625		return ERR_PTR(-ENOMEM);
1626	}
1627
1628	run->ctx = ctx;
1629	run->in_phys = in->phys0;
1630	run->out_phys = out->phys0;
1631
1632	ret = ipu_image_convert_queue(run);
1633	if (ret) {
1634		ipu_image_convert_unprepare(ctx);
1635		kfree(run);
1636		return ERR_PTR(ret);
1637	}
1638
1639	return run;
1640}
1641EXPORT_SYMBOL_GPL(ipu_image_convert);
1642
1643/* "Canned" synchronous single image conversion */
1644static void image_convert_sync_complete(struct ipu_image_convert_run *run,
1645					void *data)
1646{
1647	struct completion *comp = data;
1648
1649	complete(comp);
1650}
1651
1652int ipu_image_convert_sync(struct ipu_soc *ipu, enum ipu_ic_task ic_task,
1653			   struct ipu_image *in, struct ipu_image *out,
1654			   enum ipu_rotate_mode rot_mode)
1655{
1656	struct ipu_image_convert_run *run;
1657	struct completion comp;
1658	int ret;
1659
1660	init_completion(&comp);
1661
1662	run = ipu_image_convert(ipu, ic_task, in, out, rot_mode,
1663				image_convert_sync_complete, &comp);
1664	if (IS_ERR(run))
1665		return PTR_ERR(run);
1666
1667	ret = wait_for_completion_timeout(&comp, msecs_to_jiffies(10000));
1668	ret = (ret == 0) ? -ETIMEDOUT : 0;
1669
1670	ipu_image_convert_unprepare(run->ctx);
1671	kfree(run);
1672
1673	return ret;
1674}
1675EXPORT_SYMBOL_GPL(ipu_image_convert_sync);
1676
1677int ipu_image_convert_init(struct ipu_soc *ipu, struct device *dev)
1678{
1679	struct ipu_image_convert_priv *priv;
1680	int i;
1681
1682	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
1683	if (!priv)
1684		return -ENOMEM;
1685
1686	ipu->image_convert_priv = priv;
1687	priv->ipu = ipu;
1688
1689	for (i = 0; i < IC_NUM_TASKS; i++) {
1690		struct ipu_image_convert_chan *chan = &priv->chan[i];
1691
1692		chan->ic_task = i;
1693		chan->priv = priv;
1694		chan->dma_ch = &image_convert_dma_chan[i];
 
 
1695		chan->out_eof_irq = -1;
1696		chan->rot_out_eof_irq = -1;
1697
1698		spin_lock_init(&chan->irqlock);
1699		INIT_LIST_HEAD(&chan->ctx_list);
1700		INIT_LIST_HEAD(&chan->pending_q);
1701		INIT_LIST_HEAD(&chan->done_q);
1702	}
1703
1704	return 0;
1705}
1706
1707void ipu_image_convert_exit(struct ipu_soc *ipu)
1708{
1709}