1// SPDX-License-Identifier: GPL-2.0+
   2// Copyright 2004-2007 Freescale Semiconductor, Inc. All Rights Reserved.
   3// Copyright (C) 2008 Juergen Beisert
   4
   5#include <linux/bits.h>
   6#include <linux/clk.h>
   7#include <linux/completion.h>
   8#include <linux/delay.h>
   9#include <linux/dmaengine.h>
  10#include <linux/dma-mapping.h>
  11#include <linux/err.h>
  12#include <linux/interrupt.h>
  13#include <linux/io.h>
  14#include <linux/irq.h>
  15#include <linux/kernel.h>
  16#include <linux/module.h>
  17#include <linux/pinctrl/consumer.h>
  18#include <linux/platform_device.h>
  19#include <linux/pm_runtime.h>
  20#include <linux/slab.h>
  21#include <linux/spi/spi.h>
  22#include <linux/types.h>
  23#include <linux/of.h>
 
  24#include <linux/property.h>
  25
  26#include <linux/dma/imx-dma.h>
  27
  28#define DRIVER_NAME "spi_imx"
  29
  30static bool use_dma = true;
  31module_param(use_dma, bool, 0644);
  32MODULE_PARM_DESC(use_dma, "Enable usage of DMA when available (default)");
  33
  34/* define polling limits */
  35static unsigned int polling_limit_us = 30;
  36module_param(polling_limit_us, uint, 0664);
  37MODULE_PARM_DESC(polling_limit_us,
  38		 "time in us to run a transfer in polling mode\n");
  39
  40#define MXC_RPM_TIMEOUT		2000 /* 2000ms */
  41
  42#define MXC_CSPIRXDATA		0x00
  43#define MXC_CSPITXDATA		0x04
  44#define MXC_CSPICTRL		0x08
  45#define MXC_CSPIINT		0x0c
  46#define MXC_RESET		0x1c
  47
  48/* generic defines to abstract from the different register layouts */
  49#define MXC_INT_RR	(1 << 0) /* Receive data ready interrupt */
  50#define MXC_INT_TE	(1 << 1) /* Transmit FIFO empty interrupt */
  51#define MXC_INT_RDR	BIT(4) /* Receive date threshold interrupt */
  52
  53/* The maximum bytes that a sdma BD can transfer. */
  54#define MAX_SDMA_BD_BYTES (1 << 15)
  55#define MX51_ECSPI_CTRL_MAX_BURST	512
  56/* The maximum bytes that IMX53_ECSPI can transfer in target mode.*/
  57#define MX53_MAX_TRANSFER_BYTES		512
  58
  59enum spi_imx_devtype {
  60	IMX1_CSPI,
  61	IMX21_CSPI,
  62	IMX27_CSPI,
  63	IMX31_CSPI,
  64	IMX35_CSPI,	/* CSPI on all i.mx except above */
  65	IMX51_ECSPI,	/* ECSPI on i.mx51 */
  66	IMX53_ECSPI,	/* ECSPI on i.mx53 and later */
  67};
  68
  69struct spi_imx_data;
  70
  71struct spi_imx_devtype_data {
  72	void (*intctrl)(struct spi_imx_data *spi_imx, int enable);
  73	int (*prepare_message)(struct spi_imx_data *spi_imx, struct spi_message *msg);
  74	int (*prepare_transfer)(struct spi_imx_data *spi_imx, struct spi_device *spi);
  75	void (*trigger)(struct spi_imx_data *spi_imx);
  76	int (*rx_available)(struct spi_imx_data *spi_imx);
  77	void (*reset)(struct spi_imx_data *spi_imx);
  78	void (*setup_wml)(struct spi_imx_data *spi_imx);
  79	void (*disable)(struct spi_imx_data *spi_imx);
  80	bool has_dmamode;
  81	bool has_targetmode;
  82	unsigned int fifo_size;
  83	bool dynamic_burst;
  84	/*
  85	 * ERR009165 fixed or not:
  86	 * https://www.nxp.com/docs/en/errata/IMX6DQCE.pdf
  87	 */
  88	bool tx_glitch_fixed;
  89	enum spi_imx_devtype devtype;
  90};
  91
  92struct spi_imx_data {
  93	struct spi_controller *controller;
  94	struct device *dev;
  95
  96	struct completion xfer_done;
  97	void __iomem *base;
  98	unsigned long base_phys;
  99
 100	struct clk *clk_per;
 101	struct clk *clk_ipg;
 102	unsigned long spi_clk;
 103	unsigned int spi_bus_clk;
 104
 105	unsigned int bits_per_word;
 106	unsigned int spi_drctl;
 107
 108	unsigned int count, remainder;
 109	void (*tx)(struct spi_imx_data *spi_imx);
 110	void (*rx)(struct spi_imx_data *spi_imx);
 111	void *rx_buf;
 112	const void *tx_buf;
 113	unsigned int txfifo; /* number of words pushed in tx FIFO */
 114	unsigned int dynamic_burst;
 115	bool rx_only;
 116
 117	/* Target mode */
 118	bool target_mode;
 119	bool target_aborted;
 120	unsigned int target_burst;
 121
 122	/* DMA */
 123	bool usedma;
 124	u32 wml;
 125	struct completion dma_rx_completion;
 126	struct completion dma_tx_completion;
 127
 128	const struct spi_imx_devtype_data *devtype_data;
 129};
 130
 131static inline int is_imx27_cspi(struct spi_imx_data *d)
 132{
 133	return d->devtype_data->devtype == IMX27_CSPI;
 134}
 135
 136static inline int is_imx35_cspi(struct spi_imx_data *d)
 137{
 138	return d->devtype_data->devtype == IMX35_CSPI;
 139}
 140
 141static inline int is_imx51_ecspi(struct spi_imx_data *d)
 142{
 143	return d->devtype_data->devtype == IMX51_ECSPI;
 144}
 145
 146static inline int is_imx53_ecspi(struct spi_imx_data *d)
 147{
 148	return d->devtype_data->devtype == IMX53_ECSPI;
 149}
 150
 151#define MXC_SPI_BUF_RX(type)						\
 152static void spi_imx_buf_rx_##type(struct spi_imx_data *spi_imx)		\
 153{									\
 154	unsigned int val = readl(spi_imx->base + MXC_CSPIRXDATA);	\
 155									\
 156	if (spi_imx->rx_buf) {						\
 157		*(type *)spi_imx->rx_buf = val;				\
 158		spi_imx->rx_buf += sizeof(type);			\
 159	}								\
 160									\
 161	spi_imx->remainder -= sizeof(type);				\
 162}
 163
 164#define MXC_SPI_BUF_TX(type)						\
 165static void spi_imx_buf_tx_##type(struct spi_imx_data *spi_imx)		\
 166{									\
 167	type val = 0;							\
 168									\
 169	if (spi_imx->tx_buf) {						\
 170		val = *(type *)spi_imx->tx_buf;				\
 171		spi_imx->tx_buf += sizeof(type);			\
 172	}								\
 173									\
 174	spi_imx->count -= sizeof(type);					\
 175									\
 176	writel(val, spi_imx->base + MXC_CSPITXDATA);			\
 177}
 178
 179MXC_SPI_BUF_RX(u8)
 180MXC_SPI_BUF_TX(u8)
 181MXC_SPI_BUF_RX(u16)
 182MXC_SPI_BUF_TX(u16)
 183MXC_SPI_BUF_RX(u32)
 184MXC_SPI_BUF_TX(u32)
 185
 186/* First entry is reserved, second entry is valid only if SDHC_SPIEN is set
 187 * (which is currently not the case in this driver)
 188 */
 189static int mxc_clkdivs[] = {0, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192,
 190	256, 384, 512, 768, 1024};
 191
 192/* MX21, MX27 */
 193static unsigned int spi_imx_clkdiv_1(unsigned int fin,
 194		unsigned int fspi, unsigned int max, unsigned int *fres)
 195{
 196	int i;
 197
 198	for (i = 2; i < max; i++)
 199		if (fspi * mxc_clkdivs[i] >= fin)
 200			break;
 201
 202	*fres = fin / mxc_clkdivs[i];
 203	return i;
 204}
 205
 206/* MX1, MX31, MX35, MX51 CSPI */
 207static unsigned int spi_imx_clkdiv_2(unsigned int fin,
 208		unsigned int fspi, unsigned int *fres)
 209{
 210	int i, div = 4;
 211
 212	for (i = 0; i < 7; i++) {
 213		if (fspi * div >= fin)
 214			goto out;
 215		div <<= 1;
 216	}
 217
 218out:
 219	*fres = fin / div;
 220	return i;
 221}
 222
 223static int spi_imx_bytes_per_word(const int bits_per_word)
 224{
 225	if (bits_per_word <= 8)
 226		return 1;
 227	else if (bits_per_word <= 16)
 228		return 2;
 229	else
 230		return 4;
 231}
 232
 233static bool spi_imx_can_dma(struct spi_controller *controller, struct spi_device *spi,
 234			 struct spi_transfer *transfer)
 235{
 236	struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
 237
 238	if (!use_dma || controller->fallback)
 239		return false;
 240
 241	if (!controller->dma_rx)
 242		return false;
 243
 244	if (spi_imx->target_mode)
 245		return false;
 246
 247	if (transfer->len < spi_imx->devtype_data->fifo_size)
 248		return false;
 249
 250	spi_imx->dynamic_burst = 0;
 251
 252	return true;
 253}
 254
 255/*
 256 * Note the number of natively supported chip selects for MX51 is 4. Some
 257 * devices may have less actual SS pins but the register map supports 4. When
 258 * using gpio chip selects the cs values passed into the macros below can go
 259 * outside the range 0 - 3. We therefore need to limit the cs value to avoid
 260 * corrupting bits outside the allocated locations.
 261 *
 262 * The simplest way to do this is to just mask the cs bits to 2 bits. This
 263 * still allows all 4 native chip selects to work as well as gpio chip selects
 264 * (which can use any of the 4 chip select configurations).
 265 */
 266
 267#define MX51_ECSPI_CTRL		0x08
 268#define MX51_ECSPI_CTRL_ENABLE		(1 <<  0)
 269#define MX51_ECSPI_CTRL_XCH		(1 <<  2)
 270#define MX51_ECSPI_CTRL_SMC		(1 << 3)
 271#define MX51_ECSPI_CTRL_MODE_MASK	(0xf << 4)
 272#define MX51_ECSPI_CTRL_DRCTL(drctl)	((drctl) << 16)
 273#define MX51_ECSPI_CTRL_POSTDIV_OFFSET	8
 274#define MX51_ECSPI_CTRL_PREDIV_OFFSET	12
 275#define MX51_ECSPI_CTRL_CS(cs)		((cs & 3) << 18)
 276#define MX51_ECSPI_CTRL_BL_OFFSET	20
 277#define MX51_ECSPI_CTRL_BL_MASK		(0xfff << 20)
 278
 279#define MX51_ECSPI_CONFIG	0x0c
 280#define MX51_ECSPI_CONFIG_SCLKPHA(cs)	(1 << ((cs & 3) +  0))
 281#define MX51_ECSPI_CONFIG_SCLKPOL(cs)	(1 << ((cs & 3) +  4))
 282#define MX51_ECSPI_CONFIG_SBBCTRL(cs)	(1 << ((cs & 3) +  8))
 283#define MX51_ECSPI_CONFIG_SSBPOL(cs)	(1 << ((cs & 3) + 12))
 284#define MX51_ECSPI_CONFIG_DATACTL(cs)	(1 << ((cs & 3) + 16))
 285#define MX51_ECSPI_CONFIG_SCLKCTL(cs)	(1 << ((cs & 3) + 20))
 286
 287#define MX51_ECSPI_INT		0x10
 288#define MX51_ECSPI_INT_TEEN		(1 <<  0)
 289#define MX51_ECSPI_INT_RREN		(1 <<  3)
 290#define MX51_ECSPI_INT_RDREN		(1 <<  4)
 291
 292#define MX51_ECSPI_DMA		0x14
 293#define MX51_ECSPI_DMA_TX_WML(wml)	((wml) & 0x3f)
 294#define MX51_ECSPI_DMA_RX_WML(wml)	(((wml) & 0x3f) << 16)
 295#define MX51_ECSPI_DMA_RXT_WML(wml)	(((wml) & 0x3f) << 24)
 296
 297#define MX51_ECSPI_DMA_TEDEN		(1 << 7)
 298#define MX51_ECSPI_DMA_RXDEN		(1 << 23)
 299#define MX51_ECSPI_DMA_RXTDEN		(1 << 31)
 300
 301#define MX51_ECSPI_STAT		0x18
 302#define MX51_ECSPI_STAT_RR		(1 <<  3)
 303
 304#define MX51_ECSPI_TESTREG	0x20
 305#define MX51_ECSPI_TESTREG_LBC	BIT(31)
 306
 307static void spi_imx_buf_rx_swap_u32(struct spi_imx_data *spi_imx)
 308{
 309	unsigned int val = readl(spi_imx->base + MXC_CSPIRXDATA);
 310
 311	if (spi_imx->rx_buf) {
 312#ifdef __LITTLE_ENDIAN
 313		unsigned int bytes_per_word;
 314
 315		bytes_per_word = spi_imx_bytes_per_word(spi_imx->bits_per_word);
 316		if (bytes_per_word == 1)
 317			swab32s(&val);
 318		else if (bytes_per_word == 2)
 319			swahw32s(&val);
 320#endif
 321		*(u32 *)spi_imx->rx_buf = val;
 322		spi_imx->rx_buf += sizeof(u32);
 323	}
 324
 325	spi_imx->remainder -= sizeof(u32);
 326}
 327
 328static void spi_imx_buf_rx_swap(struct spi_imx_data *spi_imx)
 329{
 330	int unaligned;
 331	u32 val;
 332
 333	unaligned = spi_imx->remainder % 4;
 334
 335	if (!unaligned) {
 336		spi_imx_buf_rx_swap_u32(spi_imx);
 337		return;
 338	}
 339
 340	if (spi_imx_bytes_per_word(spi_imx->bits_per_word) == 2) {
 341		spi_imx_buf_rx_u16(spi_imx);
 342		return;
 343	}
 344
 345	val = readl(spi_imx->base + MXC_CSPIRXDATA);
 346
 347	while (unaligned--) {
 348		if (spi_imx->rx_buf) {
 349			*(u8 *)spi_imx->rx_buf = (val >> (8 * unaligned)) & 0xff;
 350			spi_imx->rx_buf++;
 351		}
 352		spi_imx->remainder--;
 353	}
 354}
 355
 356static void spi_imx_buf_tx_swap_u32(struct spi_imx_data *spi_imx)
 357{
 358	u32 val = 0;
 359#ifdef __LITTLE_ENDIAN
 360	unsigned int bytes_per_word;
 361#endif
 362
 363	if (spi_imx->tx_buf) {
 364		val = *(u32 *)spi_imx->tx_buf;
 365		spi_imx->tx_buf += sizeof(u32);
 366	}
 367
 368	spi_imx->count -= sizeof(u32);
 369#ifdef __LITTLE_ENDIAN
 370	bytes_per_word = spi_imx_bytes_per_word(spi_imx->bits_per_word);
 371
 372	if (bytes_per_word == 1)
 373		swab32s(&val);
 374	else if (bytes_per_word == 2)
 375		swahw32s(&val);
 376#endif
 377	writel(val, spi_imx->base + MXC_CSPITXDATA);
 378}
 379
 380static void spi_imx_buf_tx_swap(struct spi_imx_data *spi_imx)
 381{
 382	int unaligned;
 383	u32 val = 0;
 384
 385	unaligned = spi_imx->count % 4;
 386
 387	if (!unaligned) {
 388		spi_imx_buf_tx_swap_u32(spi_imx);
 389		return;
 390	}
 391
 392	if (spi_imx_bytes_per_word(spi_imx->bits_per_word) == 2) {
 393		spi_imx_buf_tx_u16(spi_imx);
 394		return;
 395	}
 396
 397	while (unaligned--) {
 398		if (spi_imx->tx_buf) {
 399			val |= *(u8 *)spi_imx->tx_buf << (8 * unaligned);
 400			spi_imx->tx_buf++;
 401		}
 402		spi_imx->count--;
 403	}
 404
 405	writel(val, spi_imx->base + MXC_CSPITXDATA);
 406}
 407
 408static void mx53_ecspi_rx_target(struct spi_imx_data *spi_imx)
 409{
 410	u32 val = be32_to_cpu(readl(spi_imx->base + MXC_CSPIRXDATA));
 411
 412	if (spi_imx->rx_buf) {
 413		int n_bytes = spi_imx->target_burst % sizeof(val);
 414
 415		if (!n_bytes)
 416			n_bytes = sizeof(val);
 417
 418		memcpy(spi_imx->rx_buf,
 419		       ((u8 *)&val) + sizeof(val) - n_bytes, n_bytes);
 420
 421		spi_imx->rx_buf += n_bytes;
 422		spi_imx->target_burst -= n_bytes;
 423	}
 424
 425	spi_imx->remainder -= sizeof(u32);
 426}
 427
 428static void mx53_ecspi_tx_target(struct spi_imx_data *spi_imx)
 429{
 430	u32 val = 0;
 431	int n_bytes = spi_imx->count % sizeof(val);
 432
 433	if (!n_bytes)
 434		n_bytes = sizeof(val);
 435
 436	if (spi_imx->tx_buf) {
 437		memcpy(((u8 *)&val) + sizeof(val) - n_bytes,
 438		       spi_imx->tx_buf, n_bytes);
 439		val = cpu_to_be32(val);
 440		spi_imx->tx_buf += n_bytes;
 441	}
 442
 443	spi_imx->count -= n_bytes;
 444
 445	writel(val, spi_imx->base + MXC_CSPITXDATA);
 446}
 447
 448/* MX51 eCSPI */
 449static unsigned int mx51_ecspi_clkdiv(struct spi_imx_data *spi_imx,
 450				      unsigned int fspi, unsigned int *fres)
 451{
 452	/*
 453	 * there are two 4-bit dividers, the pre-divider divides by
 454	 * $pre, the post-divider by 2^$post
 455	 */
 456	unsigned int pre, post;
 457	unsigned int fin = spi_imx->spi_clk;
 458
 459	fspi = min(fspi, fin);
 460
 461	post = fls(fin) - fls(fspi);
 462	if (fin > fspi << post)
 463		post++;
 464
 465	/* now we have: (fin <= fspi << post) with post being minimal */
 466
 467	post = max(4U, post) - 4;
 468	if (unlikely(post > 0xf)) {
 469		dev_err(spi_imx->dev, "cannot set clock freq: %u (base freq: %u)\n",
 470				fspi, fin);
 471		return 0xff;
 472	}
 473
 474	pre = DIV_ROUND_UP(fin, fspi << post) - 1;
 475
 476	dev_dbg(spi_imx->dev, "%s: fin: %u, fspi: %u, post: %u, pre: %u\n",
 477			__func__, fin, fspi, post, pre);
 478
 479	/* Resulting frequency for the SCLK line. */
 480	*fres = (fin / (pre + 1)) >> post;
 481
 482	return (pre << MX51_ECSPI_CTRL_PREDIV_OFFSET) |
 483		(post << MX51_ECSPI_CTRL_POSTDIV_OFFSET);
 484}
 485
 486static void mx51_ecspi_intctrl(struct spi_imx_data *spi_imx, int enable)
 487{
 488	unsigned int val = 0;
 489
 490	if (enable & MXC_INT_TE)
 491		val |= MX51_ECSPI_INT_TEEN;
 492
 493	if (enable & MXC_INT_RR)
 494		val |= MX51_ECSPI_INT_RREN;
 495
 496	if (enable & MXC_INT_RDR)
 497		val |= MX51_ECSPI_INT_RDREN;
 498
 499	writel(val, spi_imx->base + MX51_ECSPI_INT);
 500}
 501
 502static void mx51_ecspi_trigger(struct spi_imx_data *spi_imx)
 503{
 504	u32 reg;
 505
 506	reg = readl(spi_imx->base + MX51_ECSPI_CTRL);
 507	reg |= MX51_ECSPI_CTRL_XCH;
 508	writel(reg, spi_imx->base + MX51_ECSPI_CTRL);
 509}
 510
 511static void mx51_ecspi_disable(struct spi_imx_data *spi_imx)
 512{
 513	u32 ctrl;
 514
 515	ctrl = readl(spi_imx->base + MX51_ECSPI_CTRL);
 516	ctrl &= ~MX51_ECSPI_CTRL_ENABLE;
 517	writel(ctrl, spi_imx->base + MX51_ECSPI_CTRL);
 518}
 519
 520static int mx51_ecspi_channel(const struct spi_device *spi)
 521{
 522	if (!spi_get_csgpiod(spi, 0))
 523		return spi_get_chipselect(spi, 0);
 524	return spi->controller->unused_native_cs;
 525}
 526
 527static int mx51_ecspi_prepare_message(struct spi_imx_data *spi_imx,
 528				      struct spi_message *msg)
 529{
 530	struct spi_device *spi = msg->spi;
 531	struct spi_transfer *xfer;
 532	u32 ctrl = MX51_ECSPI_CTRL_ENABLE;
 533	u32 min_speed_hz = ~0U;
 534	u32 testreg, delay;
 535	u32 cfg = readl(spi_imx->base + MX51_ECSPI_CONFIG);
 536	u32 current_cfg = cfg;
 537	int channel = mx51_ecspi_channel(spi);
 538
 539	/* set Host or Target mode */
 540	if (spi_imx->target_mode)
 541		ctrl &= ~MX51_ECSPI_CTRL_MODE_MASK;
 542	else
 543		ctrl |= MX51_ECSPI_CTRL_MODE_MASK;
 544
 545	/*
 546	 * Enable SPI_RDY handling (falling edge/level triggered).
 547	 */
 548	if (spi->mode & SPI_READY)
 549		ctrl |= MX51_ECSPI_CTRL_DRCTL(spi_imx->spi_drctl);
 550
 551	/* set chip select to use */
 552	ctrl |= MX51_ECSPI_CTRL_CS(channel);
 553
 554	/*
 555	 * The ctrl register must be written first, with the EN bit set other
 556	 * registers must not be written to.
 557	 */
 558	writel(ctrl, spi_imx->base + MX51_ECSPI_CTRL);
 559
 560	testreg = readl(spi_imx->base + MX51_ECSPI_TESTREG);
 561	if (spi->mode & SPI_LOOP)
 562		testreg |= MX51_ECSPI_TESTREG_LBC;
 563	else
 564		testreg &= ~MX51_ECSPI_TESTREG_LBC;
 565	writel(testreg, spi_imx->base + MX51_ECSPI_TESTREG);
 566
 567	/*
 568	 * eCSPI burst completion by Chip Select signal in Target mode
 569	 * is not functional for imx53 Soc, config SPI burst completed when
 570	 * BURST_LENGTH + 1 bits are received
 571	 */
 572	if (spi_imx->target_mode && is_imx53_ecspi(spi_imx))
 573		cfg &= ~MX51_ECSPI_CONFIG_SBBCTRL(channel);
 574	else
 575		cfg |= MX51_ECSPI_CONFIG_SBBCTRL(channel);
 576
 577	if (spi->mode & SPI_CPOL) {
 578		cfg |= MX51_ECSPI_CONFIG_SCLKPOL(channel);
 579		cfg |= MX51_ECSPI_CONFIG_SCLKCTL(channel);
 580	} else {
 581		cfg &= ~MX51_ECSPI_CONFIG_SCLKPOL(channel);
 582		cfg &= ~MX51_ECSPI_CONFIG_SCLKCTL(channel);
 583	}
 584
 585	if (spi->mode & SPI_MOSI_IDLE_LOW)
 586		cfg |= MX51_ECSPI_CONFIG_DATACTL(channel);
 587	else
 588		cfg &= ~MX51_ECSPI_CONFIG_DATACTL(channel);
 589
 590	if (spi->mode & SPI_CS_HIGH)
 591		cfg |= MX51_ECSPI_CONFIG_SSBPOL(channel);
 592	else
 593		cfg &= ~MX51_ECSPI_CONFIG_SSBPOL(channel);
 594
 595	if (cfg == current_cfg)
 596		return 0;
 597
 598	writel(cfg, spi_imx->base + MX51_ECSPI_CONFIG);
 599
 600	/*
 601	 * Wait until the changes in the configuration register CONFIGREG
 602	 * propagate into the hardware. It takes exactly one tick of the
 603	 * SCLK clock, but we will wait two SCLK clock just to be sure. The
 604	 * effect of the delay it takes for the hardware to apply changes
 605	 * is noticable if the SCLK clock run very slow. In such a case, if
 606	 * the polarity of SCLK should be inverted, the GPIO ChipSelect might
 607	 * be asserted before the SCLK polarity changes, which would disrupt
 608	 * the SPI communication as the device on the other end would consider
 609	 * the change of SCLK polarity as a clock tick already.
 610	 *
 611	 * Because spi_imx->spi_bus_clk is only set in prepare_message
 612	 * callback, iterate over all the transfers in spi_message, find the
 613	 * one with lowest bus frequency, and use that bus frequency for the
 614	 * delay calculation. In case all transfers have speed_hz == 0, then
 615	 * min_speed_hz is ~0 and the resulting delay is zero.
 616	 */
 617	list_for_each_entry(xfer, &msg->transfers, transfer_list) {
 618		if (!xfer->speed_hz)
 619			continue;
 620		min_speed_hz = min(xfer->speed_hz, min_speed_hz);
 621	}
 622
 623	delay = (2 * 1000000) / min_speed_hz;
 624	if (likely(delay < 10))	/* SCLK is faster than 200 kHz */
 625		udelay(delay);
 626	else			/* SCLK is _very_ slow */
 627		usleep_range(delay, delay + 10);
 628
 629	return 0;
 630}
 631
 632static void mx51_configure_cpha(struct spi_imx_data *spi_imx,
 633				struct spi_device *spi)
 634{
 635	bool cpha = (spi->mode & SPI_CPHA);
 636	bool flip_cpha = (spi->mode & SPI_RX_CPHA_FLIP) && spi_imx->rx_only;
 637	u32 cfg = readl(spi_imx->base + MX51_ECSPI_CONFIG);
 638	int channel = mx51_ecspi_channel(spi);
 639
 640	/* Flip cpha logical value iff flip_cpha */
 641	cpha ^= flip_cpha;
 642
 643	if (cpha)
 644		cfg |= MX51_ECSPI_CONFIG_SCLKPHA(channel);
 645	else
 646		cfg &= ~MX51_ECSPI_CONFIG_SCLKPHA(channel);
 647
 648	writel(cfg, spi_imx->base + MX51_ECSPI_CONFIG);
 649}
 650
 651static int mx51_ecspi_prepare_transfer(struct spi_imx_data *spi_imx,
 652				       struct spi_device *spi)
 653{
 654	u32 ctrl = readl(spi_imx->base + MX51_ECSPI_CTRL);
 655	u32 clk;
 656
 657	/* Clear BL field and set the right value */
 658	ctrl &= ~MX51_ECSPI_CTRL_BL_MASK;
 659	if (spi_imx->target_mode && is_imx53_ecspi(spi_imx))
 660		ctrl |= (spi_imx->target_burst * 8 - 1)
 
 
 
 661			<< MX51_ECSPI_CTRL_BL_OFFSET;
 662	else {
 663		if (spi_imx->usedma) {
 664			ctrl |= (spi_imx->bits_per_word - 1)
 665				<< MX51_ECSPI_CTRL_BL_OFFSET;
 666		} else {
 667			if (spi_imx->count >= MX51_ECSPI_CTRL_MAX_BURST)
 668				ctrl |= (MX51_ECSPI_CTRL_MAX_BURST * BITS_PER_BYTE - 1)
 669						<< MX51_ECSPI_CTRL_BL_OFFSET;
 670			else
 671				ctrl |= spi_imx->count / DIV_ROUND_UP(spi_imx->bits_per_word,
 672						BITS_PER_BYTE) * spi_imx->bits_per_word
 673						<< MX51_ECSPI_CTRL_BL_OFFSET;
 674		}
 675	}
 676
 677	/* set clock speed */
 678	ctrl &= ~(0xf << MX51_ECSPI_CTRL_POSTDIV_OFFSET |
 679		  0xf << MX51_ECSPI_CTRL_PREDIV_OFFSET);
 680	ctrl |= mx51_ecspi_clkdiv(spi_imx, spi_imx->spi_bus_clk, &clk);
 681	spi_imx->spi_bus_clk = clk;
 682
 683	mx51_configure_cpha(spi_imx, spi);
 684
 685	/*
 686	 * ERR009165: work in XHC mode instead of SMC as PIO on the chips
 687	 * before i.mx6ul.
 688	 */
 689	if (spi_imx->usedma && spi_imx->devtype_data->tx_glitch_fixed)
 690		ctrl |= MX51_ECSPI_CTRL_SMC;
 691	else
 692		ctrl &= ~MX51_ECSPI_CTRL_SMC;
 693
 694	writel(ctrl, spi_imx->base + MX51_ECSPI_CTRL);
 695
 696	return 0;
 697}
 698
 699static void mx51_setup_wml(struct spi_imx_data *spi_imx)
 700{
 701	u32 tx_wml = 0;
 702
 703	if (spi_imx->devtype_data->tx_glitch_fixed)
 704		tx_wml = spi_imx->wml;
 705	/*
 706	 * Configure the DMA register: setup the watermark
 707	 * and enable DMA request.
 708	 */
 709	writel(MX51_ECSPI_DMA_RX_WML(spi_imx->wml - 1) |
 710		MX51_ECSPI_DMA_TX_WML(tx_wml) |
 711		MX51_ECSPI_DMA_RXT_WML(spi_imx->wml) |
 712		MX51_ECSPI_DMA_TEDEN | MX51_ECSPI_DMA_RXDEN |
 713		MX51_ECSPI_DMA_RXTDEN, spi_imx->base + MX51_ECSPI_DMA);
 714}
 715
 716static int mx51_ecspi_rx_available(struct spi_imx_data *spi_imx)
 717{
 718	return readl(spi_imx->base + MX51_ECSPI_STAT) & MX51_ECSPI_STAT_RR;
 719}
 720
 721static void mx51_ecspi_reset(struct spi_imx_data *spi_imx)
 722{
 723	/* drain receive buffer */
 724	while (mx51_ecspi_rx_available(spi_imx))
 725		readl(spi_imx->base + MXC_CSPIRXDATA);
 726}
 727
 728#define MX31_INTREG_TEEN	(1 << 0)
 729#define MX31_INTREG_RREN	(1 << 3)
 730
 731#define MX31_CSPICTRL_ENABLE	(1 << 0)
 732#define MX31_CSPICTRL_HOST	(1 << 1)
 733#define MX31_CSPICTRL_XCH	(1 << 2)
 734#define MX31_CSPICTRL_SMC	(1 << 3)
 735#define MX31_CSPICTRL_POL	(1 << 4)
 736#define MX31_CSPICTRL_PHA	(1 << 5)
 737#define MX31_CSPICTRL_SSCTL	(1 << 6)
 738#define MX31_CSPICTRL_SSPOL	(1 << 7)
 739#define MX31_CSPICTRL_BC_SHIFT	8
 740#define MX35_CSPICTRL_BL_SHIFT	20
 741#define MX31_CSPICTRL_CS_SHIFT	24
 742#define MX35_CSPICTRL_CS_SHIFT	12
 743#define MX31_CSPICTRL_DR_SHIFT	16
 744
 745#define MX31_CSPI_DMAREG	0x10
 746#define MX31_DMAREG_RH_DEN	(1<<4)
 747#define MX31_DMAREG_TH_DEN	(1<<1)
 748
 749#define MX31_CSPISTATUS		0x14
 750#define MX31_STATUS_RR		(1 << 3)
 751
 752#define MX31_CSPI_TESTREG	0x1C
 753#define MX31_TEST_LBC		(1 << 14)
 754
 755/* These functions also work for the i.MX35, but be aware that
 756 * the i.MX35 has a slightly different register layout for bits
 757 * we do not use here.
 758 */
 759static void mx31_intctrl(struct spi_imx_data *spi_imx, int enable)
 760{
 761	unsigned int val = 0;
 762
 763	if (enable & MXC_INT_TE)
 764		val |= MX31_INTREG_TEEN;
 765	if (enable & MXC_INT_RR)
 766		val |= MX31_INTREG_RREN;
 767
 768	writel(val, spi_imx->base + MXC_CSPIINT);
 769}
 770
 771static void mx31_trigger(struct spi_imx_data *spi_imx)
 772{
 773	unsigned int reg;
 774
 775	reg = readl(spi_imx->base + MXC_CSPICTRL);
 776	reg |= MX31_CSPICTRL_XCH;
 777	writel(reg, spi_imx->base + MXC_CSPICTRL);
 778}
 779
 780static int mx31_prepare_message(struct spi_imx_data *spi_imx,
 781				struct spi_message *msg)
 782{
 783	return 0;
 784}
 785
 786static int mx31_prepare_transfer(struct spi_imx_data *spi_imx,
 787				 struct spi_device *spi)
 788{
 789	unsigned int reg = MX31_CSPICTRL_ENABLE | MX31_CSPICTRL_HOST;
 790	unsigned int clk;
 791
 792	reg |= spi_imx_clkdiv_2(spi_imx->spi_clk, spi_imx->spi_bus_clk, &clk) <<
 793		MX31_CSPICTRL_DR_SHIFT;
 794	spi_imx->spi_bus_clk = clk;
 795
 796	if (is_imx35_cspi(spi_imx)) {
 797		reg |= (spi_imx->bits_per_word - 1) << MX35_CSPICTRL_BL_SHIFT;
 798		reg |= MX31_CSPICTRL_SSCTL;
 799	} else {
 800		reg |= (spi_imx->bits_per_word - 1) << MX31_CSPICTRL_BC_SHIFT;
 801	}
 802
 803	if (spi->mode & SPI_CPHA)
 804		reg |= MX31_CSPICTRL_PHA;
 805	if (spi->mode & SPI_CPOL)
 806		reg |= MX31_CSPICTRL_POL;
 807	if (spi->mode & SPI_CS_HIGH)
 808		reg |= MX31_CSPICTRL_SSPOL;
 809	if (!spi_get_csgpiod(spi, 0))
 810		reg |= (spi_get_chipselect(spi, 0)) <<
 811			(is_imx35_cspi(spi_imx) ? MX35_CSPICTRL_CS_SHIFT :
 812						  MX31_CSPICTRL_CS_SHIFT);
 813
 814	if (spi_imx->usedma)
 815		reg |= MX31_CSPICTRL_SMC;
 816
 817	writel(reg, spi_imx->base + MXC_CSPICTRL);
 818
 819	reg = readl(spi_imx->base + MX31_CSPI_TESTREG);
 820	if (spi->mode & SPI_LOOP)
 821		reg |= MX31_TEST_LBC;
 822	else
 823		reg &= ~MX31_TEST_LBC;
 824	writel(reg, spi_imx->base + MX31_CSPI_TESTREG);
 825
 826	if (spi_imx->usedma) {
 827		/*
 828		 * configure DMA requests when RXFIFO is half full and
 829		 * when TXFIFO is half empty
 830		 */
 831		writel(MX31_DMAREG_RH_DEN | MX31_DMAREG_TH_DEN,
 832			spi_imx->base + MX31_CSPI_DMAREG);
 833	}
 834
 835	return 0;
 836}
 837
 838static int mx31_rx_available(struct spi_imx_data *spi_imx)
 839{
 840	return readl(spi_imx->base + MX31_CSPISTATUS) & MX31_STATUS_RR;
 841}
 842
 843static void mx31_reset(struct spi_imx_data *spi_imx)
 844{
 845	/* drain receive buffer */
 846	while (readl(spi_imx->base + MX31_CSPISTATUS) & MX31_STATUS_RR)
 847		readl(spi_imx->base + MXC_CSPIRXDATA);
 848}
 849
 850#define MX21_INTREG_RR		(1 << 4)
 851#define MX21_INTREG_TEEN	(1 << 9)
 852#define MX21_INTREG_RREN	(1 << 13)
 853
 854#define MX21_CSPICTRL_POL	(1 << 5)
 855#define MX21_CSPICTRL_PHA	(1 << 6)
 856#define MX21_CSPICTRL_SSPOL	(1 << 8)
 857#define MX21_CSPICTRL_XCH	(1 << 9)
 858#define MX21_CSPICTRL_ENABLE	(1 << 10)
 859#define MX21_CSPICTRL_HOST	(1 << 11)
 860#define MX21_CSPICTRL_DR_SHIFT	14
 861#define MX21_CSPICTRL_CS_SHIFT	19
 862
 863static void mx21_intctrl(struct spi_imx_data *spi_imx, int enable)
 864{
 865	unsigned int val = 0;
 866
 867	if (enable & MXC_INT_TE)
 868		val |= MX21_INTREG_TEEN;
 869	if (enable & MXC_INT_RR)
 870		val |= MX21_INTREG_RREN;
 871
 872	writel(val, spi_imx->base + MXC_CSPIINT);
 873}
 874
 875static void mx21_trigger(struct spi_imx_data *spi_imx)
 876{
 877	unsigned int reg;
 878
 879	reg = readl(spi_imx->base + MXC_CSPICTRL);
 880	reg |= MX21_CSPICTRL_XCH;
 881	writel(reg, spi_imx->base + MXC_CSPICTRL);
 882}
 883
 884static int mx21_prepare_message(struct spi_imx_data *spi_imx,
 885				struct spi_message *msg)
 886{
 887	return 0;
 888}
 889
 890static int mx21_prepare_transfer(struct spi_imx_data *spi_imx,
 891				 struct spi_device *spi)
 892{
 893	unsigned int reg = MX21_CSPICTRL_ENABLE | MX21_CSPICTRL_HOST;
 894	unsigned int max = is_imx27_cspi(spi_imx) ? 16 : 18;
 895	unsigned int clk;
 896
 897	reg |= spi_imx_clkdiv_1(spi_imx->spi_clk, spi_imx->spi_bus_clk, max, &clk)
 898		<< MX21_CSPICTRL_DR_SHIFT;
 899	spi_imx->spi_bus_clk = clk;
 900
 901	reg |= spi_imx->bits_per_word - 1;
 902
 903	if (spi->mode & SPI_CPHA)
 904		reg |= MX21_CSPICTRL_PHA;
 905	if (spi->mode & SPI_CPOL)
 906		reg |= MX21_CSPICTRL_POL;
 907	if (spi->mode & SPI_CS_HIGH)
 908		reg |= MX21_CSPICTRL_SSPOL;
 909	if (!spi_get_csgpiod(spi, 0))
 910		reg |= spi_get_chipselect(spi, 0) << MX21_CSPICTRL_CS_SHIFT;
 911
 912	writel(reg, spi_imx->base + MXC_CSPICTRL);
 913
 914	return 0;
 915}
 916
 917static int mx21_rx_available(struct spi_imx_data *spi_imx)
 918{
 919	return readl(spi_imx->base + MXC_CSPIINT) & MX21_INTREG_RR;
 920}
 921
 922static void mx21_reset(struct spi_imx_data *spi_imx)
 923{
 924	writel(1, spi_imx->base + MXC_RESET);
 925}
 926
 927#define MX1_INTREG_RR		(1 << 3)
 928#define MX1_INTREG_TEEN		(1 << 8)
 929#define MX1_INTREG_RREN		(1 << 11)
 930
 931#define MX1_CSPICTRL_POL	(1 << 4)
 932#define MX1_CSPICTRL_PHA	(1 << 5)
 933#define MX1_CSPICTRL_XCH	(1 << 8)
 934#define MX1_CSPICTRL_ENABLE	(1 << 9)
 935#define MX1_CSPICTRL_HOST	(1 << 10)
 936#define MX1_CSPICTRL_DR_SHIFT	13
 937
 938static void mx1_intctrl(struct spi_imx_data *spi_imx, int enable)
 939{
 940	unsigned int val = 0;
 941
 942	if (enable & MXC_INT_TE)
 943		val |= MX1_INTREG_TEEN;
 944	if (enable & MXC_INT_RR)
 945		val |= MX1_INTREG_RREN;
 946
 947	writel(val, spi_imx->base + MXC_CSPIINT);
 948}
 949
 950static void mx1_trigger(struct spi_imx_data *spi_imx)
 951{
 952	unsigned int reg;
 953
 954	reg = readl(spi_imx->base + MXC_CSPICTRL);
 955	reg |= MX1_CSPICTRL_XCH;
 956	writel(reg, spi_imx->base + MXC_CSPICTRL);
 957}
 958
 959static int mx1_prepare_message(struct spi_imx_data *spi_imx,
 960			       struct spi_message *msg)
 961{
 962	return 0;
 963}
 964
 965static int mx1_prepare_transfer(struct spi_imx_data *spi_imx,
 966				struct spi_device *spi)
 967{
 968	unsigned int reg = MX1_CSPICTRL_ENABLE | MX1_CSPICTRL_HOST;
 969	unsigned int clk;
 970
 971	reg |= spi_imx_clkdiv_2(spi_imx->spi_clk, spi_imx->spi_bus_clk, &clk) <<
 972		MX1_CSPICTRL_DR_SHIFT;
 973	spi_imx->spi_bus_clk = clk;
 974
 975	reg |= spi_imx->bits_per_word - 1;
 976
 977	if (spi->mode & SPI_CPHA)
 978		reg |= MX1_CSPICTRL_PHA;
 979	if (spi->mode & SPI_CPOL)
 980		reg |= MX1_CSPICTRL_POL;
 981
 982	writel(reg, spi_imx->base + MXC_CSPICTRL);
 983
 984	return 0;
 985}
 986
 987static int mx1_rx_available(struct spi_imx_data *spi_imx)
 988{
 989	return readl(spi_imx->base + MXC_CSPIINT) & MX1_INTREG_RR;
 990}
 991
 992static void mx1_reset(struct spi_imx_data *spi_imx)
 993{
 994	writel(1, spi_imx->base + MXC_RESET);
 995}
 996
 997static struct spi_imx_devtype_data imx1_cspi_devtype_data = {
 998	.intctrl = mx1_intctrl,
 999	.prepare_message = mx1_prepare_message,
1000	.prepare_transfer = mx1_prepare_transfer,
1001	.trigger = mx1_trigger,
1002	.rx_available = mx1_rx_available,
1003	.reset = mx1_reset,
1004	.fifo_size = 8,
1005	.has_dmamode = false,
1006	.dynamic_burst = false,
1007	.has_targetmode = false,
1008	.devtype = IMX1_CSPI,
1009};
1010
1011static struct spi_imx_devtype_data imx21_cspi_devtype_data = {
1012	.intctrl = mx21_intctrl,
1013	.prepare_message = mx21_prepare_message,
1014	.prepare_transfer = mx21_prepare_transfer,
1015	.trigger = mx21_trigger,
1016	.rx_available = mx21_rx_available,
1017	.reset = mx21_reset,
1018	.fifo_size = 8,
1019	.has_dmamode = false,
1020	.dynamic_burst = false,
1021	.has_targetmode = false,
1022	.devtype = IMX21_CSPI,
1023};
1024
1025static struct spi_imx_devtype_data imx27_cspi_devtype_data = {
1026	/* i.mx27 cspi shares the functions with i.mx21 one */
1027	.intctrl = mx21_intctrl,
1028	.prepare_message = mx21_prepare_message,
1029	.prepare_transfer = mx21_prepare_transfer,
1030	.trigger = mx21_trigger,
1031	.rx_available = mx21_rx_available,
1032	.reset = mx21_reset,
1033	.fifo_size = 8,
1034	.has_dmamode = false,
1035	.dynamic_burst = false,
1036	.has_targetmode = false,
1037	.devtype = IMX27_CSPI,
1038};
1039
1040static struct spi_imx_devtype_data imx31_cspi_devtype_data = {
1041	.intctrl = mx31_intctrl,
1042	.prepare_message = mx31_prepare_message,
1043	.prepare_transfer = mx31_prepare_transfer,
1044	.trigger = mx31_trigger,
1045	.rx_available = mx31_rx_available,
1046	.reset = mx31_reset,
1047	.fifo_size = 8,
1048	.has_dmamode = false,
1049	.dynamic_burst = false,
1050	.has_targetmode = false,
1051	.devtype = IMX31_CSPI,
1052};
1053
1054static struct spi_imx_devtype_data imx35_cspi_devtype_data = {
1055	/* i.mx35 and later cspi shares the functions with i.mx31 one */
1056	.intctrl = mx31_intctrl,
1057	.prepare_message = mx31_prepare_message,
1058	.prepare_transfer = mx31_prepare_transfer,
1059	.trigger = mx31_trigger,
1060	.rx_available = mx31_rx_available,
1061	.reset = mx31_reset,
1062	.fifo_size = 8,
1063	.has_dmamode = true,
1064	.dynamic_burst = false,
1065	.has_targetmode = false,
1066	.devtype = IMX35_CSPI,
1067};
1068
1069static struct spi_imx_devtype_data imx51_ecspi_devtype_data = {
1070	.intctrl = mx51_ecspi_intctrl,
1071	.prepare_message = mx51_ecspi_prepare_message,
1072	.prepare_transfer = mx51_ecspi_prepare_transfer,
1073	.trigger = mx51_ecspi_trigger,
1074	.rx_available = mx51_ecspi_rx_available,
1075	.reset = mx51_ecspi_reset,
1076	.setup_wml = mx51_setup_wml,
1077	.fifo_size = 64,
1078	.has_dmamode = true,
1079	.dynamic_burst = true,
1080	.has_targetmode = true,
1081	.disable = mx51_ecspi_disable,
1082	.devtype = IMX51_ECSPI,
1083};
1084
1085static struct spi_imx_devtype_data imx53_ecspi_devtype_data = {
1086	.intctrl = mx51_ecspi_intctrl,
1087	.prepare_message = mx51_ecspi_prepare_message,
1088	.prepare_transfer = mx51_ecspi_prepare_transfer,
1089	.trigger = mx51_ecspi_trigger,
1090	.rx_available = mx51_ecspi_rx_available,
1091	.reset = mx51_ecspi_reset,
1092	.fifo_size = 64,
1093	.has_dmamode = true,
1094	.has_targetmode = true,
1095	.disable = mx51_ecspi_disable,
1096	.devtype = IMX53_ECSPI,
1097};
1098
1099static struct spi_imx_devtype_data imx6ul_ecspi_devtype_data = {
1100	.intctrl = mx51_ecspi_intctrl,
1101	.prepare_message = mx51_ecspi_prepare_message,
1102	.prepare_transfer = mx51_ecspi_prepare_transfer,
1103	.trigger = mx51_ecspi_trigger,
1104	.rx_available = mx51_ecspi_rx_available,
1105	.reset = mx51_ecspi_reset,
1106	.setup_wml = mx51_setup_wml,
1107	.fifo_size = 64,
1108	.has_dmamode = true,
1109	.dynamic_burst = true,
1110	.has_targetmode = true,
1111	.tx_glitch_fixed = true,
1112	.disable = mx51_ecspi_disable,
1113	.devtype = IMX51_ECSPI,
1114};
1115
1116static const struct of_device_id spi_imx_dt_ids[] = {
1117	{ .compatible = "fsl,imx1-cspi", .data = &imx1_cspi_devtype_data, },
1118	{ .compatible = "fsl,imx21-cspi", .data = &imx21_cspi_devtype_data, },
1119	{ .compatible = "fsl,imx27-cspi", .data = &imx27_cspi_devtype_data, },
1120	{ .compatible = "fsl,imx31-cspi", .data = &imx31_cspi_devtype_data, },
1121	{ .compatible = "fsl,imx35-cspi", .data = &imx35_cspi_devtype_data, },
1122	{ .compatible = "fsl,imx51-ecspi", .data = &imx51_ecspi_devtype_data, },
1123	{ .compatible = "fsl,imx53-ecspi", .data = &imx53_ecspi_devtype_data, },
1124	{ .compatible = "fsl,imx6ul-ecspi", .data = &imx6ul_ecspi_devtype_data, },
1125	{ /* sentinel */ }
1126};
1127MODULE_DEVICE_TABLE(of, spi_imx_dt_ids);
1128
1129static void spi_imx_set_burst_len(struct spi_imx_data *spi_imx, int n_bits)
1130{
1131	u32 ctrl;
1132
1133	ctrl = readl(spi_imx->base + MX51_ECSPI_CTRL);
1134	ctrl &= ~MX51_ECSPI_CTRL_BL_MASK;
1135	ctrl |= ((n_bits - 1) << MX51_ECSPI_CTRL_BL_OFFSET);
1136	writel(ctrl, spi_imx->base + MX51_ECSPI_CTRL);
1137}
1138
1139static void spi_imx_push(struct spi_imx_data *spi_imx)
1140{
1141	unsigned int burst_len;
1142
1143	/*
1144	 * Reload the FIFO when the remaining bytes to be transferred in the
1145	 * current burst is 0. This only applies when bits_per_word is a
1146	 * multiple of 8.
1147	 */
1148	if (!spi_imx->remainder) {
1149		if (spi_imx->dynamic_burst) {
1150
1151			/* We need to deal unaligned data first */
1152			burst_len = spi_imx->count % MX51_ECSPI_CTRL_MAX_BURST;
1153
1154			if (!burst_len)
1155				burst_len = MX51_ECSPI_CTRL_MAX_BURST;
1156
1157			spi_imx_set_burst_len(spi_imx, burst_len * 8);
1158
1159			spi_imx->remainder = burst_len;
1160		} else {
1161			spi_imx->remainder = spi_imx_bytes_per_word(spi_imx->bits_per_word);
1162		}
1163	}
1164
1165	while (spi_imx->txfifo < spi_imx->devtype_data->fifo_size) {
1166		if (!spi_imx->count)
1167			break;
1168		if (spi_imx->dynamic_burst &&
1169		    spi_imx->txfifo >= DIV_ROUND_UP(spi_imx->remainder, 4))
1170			break;
1171		spi_imx->tx(spi_imx);
1172		spi_imx->txfifo++;
1173	}
1174
1175	if (!spi_imx->target_mode)
1176		spi_imx->devtype_data->trigger(spi_imx);
1177}
1178
1179static irqreturn_t spi_imx_isr(int irq, void *dev_id)
1180{
1181	struct spi_imx_data *spi_imx = dev_id;
1182
1183	while (spi_imx->txfifo &&
1184	       spi_imx->devtype_data->rx_available(spi_imx)) {
1185		spi_imx->rx(spi_imx);
1186		spi_imx->txfifo--;
1187	}
1188
1189	if (spi_imx->count) {
1190		spi_imx_push(spi_imx);
1191		return IRQ_HANDLED;
1192	}
1193
1194	if (spi_imx->txfifo) {
1195		/* No data left to push, but still waiting for rx data,
1196		 * enable receive data available interrupt.
1197		 */
1198		spi_imx->devtype_data->intctrl(
1199				spi_imx, MXC_INT_RR);
1200		return IRQ_HANDLED;
1201	}
1202
1203	spi_imx->devtype_data->intctrl(spi_imx, 0);
1204	complete(&spi_imx->xfer_done);
1205
1206	return IRQ_HANDLED;
1207}
1208
1209static int spi_imx_dma_configure(struct spi_controller *controller)
1210{
1211	int ret;
1212	enum dma_slave_buswidth buswidth;
1213	struct dma_slave_config rx = {}, tx = {};
1214	struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
1215
1216	switch (spi_imx_bytes_per_word(spi_imx->bits_per_word)) {
1217	case 4:
1218		buswidth = DMA_SLAVE_BUSWIDTH_4_BYTES;
1219		break;
1220	case 2:
1221		buswidth = DMA_SLAVE_BUSWIDTH_2_BYTES;
1222		break;
1223	case 1:
1224		buswidth = DMA_SLAVE_BUSWIDTH_1_BYTE;
1225		break;
1226	default:
1227		return -EINVAL;
1228	}
1229
1230	tx.direction = DMA_MEM_TO_DEV;
1231	tx.dst_addr = spi_imx->base_phys + MXC_CSPITXDATA;
1232	tx.dst_addr_width = buswidth;
1233	tx.dst_maxburst = spi_imx->wml;
1234	ret = dmaengine_slave_config(controller->dma_tx, &tx);
1235	if (ret) {
1236		dev_err(spi_imx->dev, "TX dma configuration failed with %d\n", ret);
1237		return ret;
1238	}
1239
1240	rx.direction = DMA_DEV_TO_MEM;
1241	rx.src_addr = spi_imx->base_phys + MXC_CSPIRXDATA;
1242	rx.src_addr_width = buswidth;
1243	rx.src_maxburst = spi_imx->wml;
1244	ret = dmaengine_slave_config(controller->dma_rx, &rx);
1245	if (ret) {
1246		dev_err(spi_imx->dev, "RX dma configuration failed with %d\n", ret);
1247		return ret;
1248	}
1249
1250	return 0;
1251}
1252
1253static int spi_imx_setupxfer(struct spi_device *spi,
1254				 struct spi_transfer *t)
1255{
1256	struct spi_imx_data *spi_imx = spi_controller_get_devdata(spi->controller);
1257
1258	if (!t)
1259		return 0;
1260
1261	if (!t->speed_hz) {
1262		if (!spi->max_speed_hz) {
1263			dev_err(&spi->dev, "no speed_hz provided!\n");
1264			return -EINVAL;
1265		}
1266		dev_dbg(&spi->dev, "using spi->max_speed_hz!\n");
1267		spi_imx->spi_bus_clk = spi->max_speed_hz;
1268	} else
1269		spi_imx->spi_bus_clk = t->speed_hz;
1270
1271	spi_imx->bits_per_word = t->bits_per_word;
1272	spi_imx->count = t->len;
1273
1274	/*
1275	 * Initialize the functions for transfer. To transfer non byte-aligned
1276	 * words, we have to use multiple word-size bursts, we can't use
1277	 * dynamic_burst in that case.
1278	 */
1279	if (spi_imx->devtype_data->dynamic_burst && !spi_imx->target_mode &&
1280	    !(spi->mode & SPI_CS_WORD) &&
1281	    (spi_imx->bits_per_word == 8 ||
1282	    spi_imx->bits_per_word == 16 ||
1283	    spi_imx->bits_per_word == 32)) {
1284
1285		spi_imx->rx = spi_imx_buf_rx_swap;
1286		spi_imx->tx = spi_imx_buf_tx_swap;
1287		spi_imx->dynamic_burst = 1;
1288
1289	} else {
1290		if (spi_imx->bits_per_word <= 8) {
1291			spi_imx->rx = spi_imx_buf_rx_u8;
1292			spi_imx->tx = spi_imx_buf_tx_u8;
1293		} else if (spi_imx->bits_per_word <= 16) {
1294			spi_imx->rx = spi_imx_buf_rx_u16;
1295			spi_imx->tx = spi_imx_buf_tx_u16;
1296		} else {
1297			spi_imx->rx = spi_imx_buf_rx_u32;
1298			spi_imx->tx = spi_imx_buf_tx_u32;
1299		}
1300		spi_imx->dynamic_burst = 0;
1301	}
1302
1303	if (spi_imx_can_dma(spi_imx->controller, spi, t))
1304		spi_imx->usedma = true;
1305	else
1306		spi_imx->usedma = false;
1307
1308	spi_imx->rx_only = ((t->tx_buf == NULL)
1309			|| (t->tx_buf == spi->controller->dummy_tx));
1310
1311	if (is_imx53_ecspi(spi_imx) && spi_imx->target_mode) {
1312		spi_imx->rx = mx53_ecspi_rx_target;
1313		spi_imx->tx = mx53_ecspi_tx_target;
1314		spi_imx->target_burst = t->len;
1315	}
1316
1317	spi_imx->devtype_data->prepare_transfer(spi_imx, spi);
1318
1319	return 0;
1320}
1321
1322static void spi_imx_sdma_exit(struct spi_imx_data *spi_imx)
1323{
1324	struct spi_controller *controller = spi_imx->controller;
1325
1326	if (controller->dma_rx) {
1327		dma_release_channel(controller->dma_rx);
1328		controller->dma_rx = NULL;
1329	}
1330
1331	if (controller->dma_tx) {
1332		dma_release_channel(controller->dma_tx);
1333		controller->dma_tx = NULL;
1334	}
1335}
1336
1337static int spi_imx_sdma_init(struct device *dev, struct spi_imx_data *spi_imx,
1338			     struct spi_controller *controller)
1339{
1340	int ret;
1341
1342	spi_imx->wml = spi_imx->devtype_data->fifo_size / 2;
1343
1344	/* Prepare for TX DMA: */
1345	controller->dma_tx = dma_request_chan(dev, "tx");
1346	if (IS_ERR(controller->dma_tx)) {
1347		ret = PTR_ERR(controller->dma_tx);
1348		dev_err_probe(dev, ret, "can't get the TX DMA channel!\n");
1349		controller->dma_tx = NULL;
1350		goto err;
1351	}
1352
1353	/* Prepare for RX : */
1354	controller->dma_rx = dma_request_chan(dev, "rx");
1355	if (IS_ERR(controller->dma_rx)) {
1356		ret = PTR_ERR(controller->dma_rx);
1357		dev_err_probe(dev, ret, "can't get the RX DMA channel!\n");
1358		controller->dma_rx = NULL;
1359		goto err;
1360	}
1361
1362	init_completion(&spi_imx->dma_rx_completion);
1363	init_completion(&spi_imx->dma_tx_completion);
1364	controller->can_dma = spi_imx_can_dma;
1365	controller->max_dma_len = MAX_SDMA_BD_BYTES;
1366	spi_imx->controller->flags = SPI_CONTROLLER_MUST_RX |
1367					 SPI_CONTROLLER_MUST_TX;
1368
1369	return 0;
1370err:
1371	spi_imx_sdma_exit(spi_imx);
1372	return ret;
1373}
1374
1375static void spi_imx_dma_rx_callback(void *cookie)
1376{
1377	struct spi_imx_data *spi_imx = (struct spi_imx_data *)cookie;
1378
1379	complete(&spi_imx->dma_rx_completion);
1380}
1381
1382static void spi_imx_dma_tx_callback(void *cookie)
1383{
1384	struct spi_imx_data *spi_imx = (struct spi_imx_data *)cookie;
1385
1386	complete(&spi_imx->dma_tx_completion);
1387}
1388
1389static int spi_imx_calculate_timeout(struct spi_imx_data *spi_imx, int size)
1390{
1391	unsigned long timeout = 0;
1392
1393	/* Time with actual data transfer and CS change delay related to HW */
1394	timeout = (8 + 4) * size / spi_imx->spi_bus_clk;
1395
1396	/* Add extra second for scheduler related activities */
1397	timeout += 1;
1398
1399	/* Double calculated timeout */
1400	return msecs_to_jiffies(2 * timeout * MSEC_PER_SEC);
1401}
1402
1403static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx,
1404				struct spi_transfer *transfer)
1405{
1406	struct dma_async_tx_descriptor *desc_tx, *desc_rx;
1407	unsigned long transfer_timeout;
1408	unsigned long timeout;
1409	struct spi_controller *controller = spi_imx->controller;
1410	struct sg_table *tx = &transfer->tx_sg, *rx = &transfer->rx_sg;
1411	struct scatterlist *last_sg = sg_last(rx->sgl, rx->nents);
1412	unsigned int bytes_per_word, i;
1413	int ret;
1414
1415	/* Get the right burst length from the last sg to ensure no tail data */
1416	bytes_per_word = spi_imx_bytes_per_word(transfer->bits_per_word);
1417	for (i = spi_imx->devtype_data->fifo_size / 2; i > 0; i--) {
1418		if (!(sg_dma_len(last_sg) % (i * bytes_per_word)))
1419			break;
1420	}
1421	/* Use 1 as wml in case no available burst length got */
1422	if (i == 0)
1423		i = 1;
1424
1425	spi_imx->wml =  i;
1426
1427	ret = spi_imx_dma_configure(controller);
1428	if (ret)
1429		goto dma_failure_no_start;
1430
1431	if (!spi_imx->devtype_data->setup_wml) {
1432		dev_err(spi_imx->dev, "No setup_wml()?\n");
1433		ret = -EINVAL;
1434		goto dma_failure_no_start;
1435	}
1436	spi_imx->devtype_data->setup_wml(spi_imx);
1437
1438	/*
1439	 * The TX DMA setup starts the transfer, so make sure RX is configured
1440	 * before TX.
1441	 */
1442	desc_rx = dmaengine_prep_slave_sg(controller->dma_rx,
1443				rx->sgl, rx->nents, DMA_DEV_TO_MEM,
1444				DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1445	if (!desc_rx) {
1446		ret = -EINVAL;
1447		goto dma_failure_no_start;
1448	}
1449
1450	desc_rx->callback = spi_imx_dma_rx_callback;
1451	desc_rx->callback_param = (void *)spi_imx;
1452	dmaengine_submit(desc_rx);
1453	reinit_completion(&spi_imx->dma_rx_completion);
1454	dma_async_issue_pending(controller->dma_rx);
1455
1456	desc_tx = dmaengine_prep_slave_sg(controller->dma_tx,
1457				tx->sgl, tx->nents, DMA_MEM_TO_DEV,
1458				DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1459	if (!desc_tx) {
1460		dmaengine_terminate_all(controller->dma_tx);
1461		dmaengine_terminate_all(controller->dma_rx);
1462		return -EINVAL;
1463	}
1464
1465	desc_tx->callback = spi_imx_dma_tx_callback;
1466	desc_tx->callback_param = (void *)spi_imx;
1467	dmaengine_submit(desc_tx);
1468	reinit_completion(&spi_imx->dma_tx_completion);
1469	dma_async_issue_pending(controller->dma_tx);
1470
1471	transfer_timeout = spi_imx_calculate_timeout(spi_imx, transfer->len);
1472
1473	/* Wait SDMA to finish the data transfer.*/
1474	timeout = wait_for_completion_timeout(&spi_imx->dma_tx_completion,
1475						transfer_timeout);
1476	if (!timeout) {
1477		dev_err(spi_imx->dev, "I/O Error in DMA TX\n");
1478		dmaengine_terminate_all(controller->dma_tx);
1479		dmaengine_terminate_all(controller->dma_rx);
1480		return -ETIMEDOUT;
1481	}
1482
1483	timeout = wait_for_completion_timeout(&spi_imx->dma_rx_completion,
1484					      transfer_timeout);
1485	if (!timeout) {
1486		dev_err(&controller->dev, "I/O Error in DMA RX\n");
1487		spi_imx->devtype_data->reset(spi_imx);
1488		dmaengine_terminate_all(controller->dma_rx);
1489		return -ETIMEDOUT;
1490	}
1491
1492	return 0;
1493/* fallback to pio */
1494dma_failure_no_start:
1495	transfer->error |= SPI_TRANS_FAIL_NO_START;
1496	return ret;
1497}
1498
1499static int spi_imx_pio_transfer(struct spi_device *spi,
1500				struct spi_transfer *transfer)
1501{
1502	struct spi_imx_data *spi_imx = spi_controller_get_devdata(spi->controller);
1503	unsigned long transfer_timeout;
1504	unsigned long timeout;
1505
1506	spi_imx->tx_buf = transfer->tx_buf;
1507	spi_imx->rx_buf = transfer->rx_buf;
1508	spi_imx->count = transfer->len;
1509	spi_imx->txfifo = 0;
1510	spi_imx->remainder = 0;
1511
1512	reinit_completion(&spi_imx->xfer_done);
1513
1514	spi_imx_push(spi_imx);
1515
1516	spi_imx->devtype_data->intctrl(spi_imx, MXC_INT_TE);
1517
1518	transfer_timeout = spi_imx_calculate_timeout(spi_imx, transfer->len);
1519
1520	timeout = wait_for_completion_timeout(&spi_imx->xfer_done,
1521					      transfer_timeout);
1522	if (!timeout) {
1523		dev_err(&spi->dev, "I/O Error in PIO\n");
1524		spi_imx->devtype_data->reset(spi_imx);
1525		return -ETIMEDOUT;
1526	}
1527
1528	return 0;
1529}
1530
1531static int spi_imx_poll_transfer(struct spi_device *spi,
1532				 struct spi_transfer *transfer)
1533{
1534	struct spi_imx_data *spi_imx = spi_controller_get_devdata(spi->controller);
1535	unsigned long timeout;
1536
1537	spi_imx->tx_buf = transfer->tx_buf;
1538	spi_imx->rx_buf = transfer->rx_buf;
1539	spi_imx->count = transfer->len;
1540	spi_imx->txfifo = 0;
1541	spi_imx->remainder = 0;
1542
1543	/* fill in the fifo before timeout calculations if we are
1544	 * interrupted here, then the data is getting transferred by
1545	 * the HW while we are interrupted
1546	 */
1547	spi_imx_push(spi_imx);
1548
1549	timeout = spi_imx_calculate_timeout(spi_imx, transfer->len) + jiffies;
1550	while (spi_imx->txfifo) {
1551		/* RX */
1552		while (spi_imx->txfifo &&
1553		       spi_imx->devtype_data->rx_available(spi_imx)) {
1554			spi_imx->rx(spi_imx);
1555			spi_imx->txfifo--;
1556		}
1557
1558		/* TX */
1559		if (spi_imx->count) {
1560			spi_imx_push(spi_imx);
1561			continue;
1562		}
1563
1564		if (spi_imx->txfifo &&
1565		    time_after(jiffies, timeout)) {
1566
1567			dev_err_ratelimited(&spi->dev,
1568					    "timeout period reached: jiffies: %lu- falling back to interrupt mode\n",
1569					    jiffies - timeout);
1570
1571			/* fall back to interrupt mode */
1572			return spi_imx_pio_transfer(spi, transfer);
1573		}
1574	}
1575
1576	return 0;
1577}
1578
1579static int spi_imx_pio_transfer_target(struct spi_device *spi,
1580				       struct spi_transfer *transfer)
1581{
1582	struct spi_imx_data *spi_imx = spi_controller_get_devdata(spi->controller);
1583	int ret = 0;
1584
1585	if (is_imx53_ecspi(spi_imx) &&
1586	    transfer->len > MX53_MAX_TRANSFER_BYTES) {
1587		dev_err(&spi->dev, "Transaction too big, max size is %d bytes\n",
1588			MX53_MAX_TRANSFER_BYTES);
1589		return -EMSGSIZE;
1590	}
1591
1592	spi_imx->tx_buf = transfer->tx_buf;
1593	spi_imx->rx_buf = transfer->rx_buf;
1594	spi_imx->count = transfer->len;
1595	spi_imx->txfifo = 0;
1596	spi_imx->remainder = 0;
1597
1598	reinit_completion(&spi_imx->xfer_done);
1599	spi_imx->target_aborted = false;
1600
1601	spi_imx_push(spi_imx);
1602
1603	spi_imx->devtype_data->intctrl(spi_imx, MXC_INT_TE | MXC_INT_RDR);
1604
1605	if (wait_for_completion_interruptible(&spi_imx->xfer_done) ||
1606	    spi_imx->target_aborted) {
1607		dev_dbg(&spi->dev, "interrupted\n");
1608		ret = -EINTR;
1609	}
1610
1611	/* ecspi has a HW issue when works in Target mode,
1612	 * after 64 words writtern to TXFIFO, even TXFIFO becomes empty,
1613	 * ECSPI_TXDATA keeps shift out the last word data,
1614	 * so we have to disable ECSPI when in target mode after the
1615	 * transfer completes
1616	 */
1617	if (spi_imx->devtype_data->disable)
1618		spi_imx->devtype_data->disable(spi_imx);
1619
1620	return ret;
1621}
1622
1623static int spi_imx_transfer_one(struct spi_controller *controller,
1624				struct spi_device *spi,
1625				struct spi_transfer *transfer)
1626{
1627	struct spi_imx_data *spi_imx = spi_controller_get_devdata(spi->controller);
1628	unsigned long hz_per_byte, byte_limit;
1629
1630	spi_imx_setupxfer(spi, transfer);
1631	transfer->effective_speed_hz = spi_imx->spi_bus_clk;
1632
1633	/* flush rxfifo before transfer */
1634	while (spi_imx->devtype_data->rx_available(spi_imx))
1635		readl(spi_imx->base + MXC_CSPIRXDATA);
1636
1637	if (spi_imx->target_mode)
1638		return spi_imx_pio_transfer_target(spi, transfer);
1639
1640	/*
1641	 * If we decided in spi_imx_can_dma() that we want to do a DMA
1642	 * transfer, the SPI transfer has already been mapped, so we
1643	 * have to do the DMA transfer here.
1644	 */
1645	if (spi_imx->usedma)
1646		return spi_imx_dma_transfer(spi_imx, transfer);
1647	/*
1648	 * Calculate the estimated time in us the transfer runs. Find
1649	 * the number of Hz per byte per polling limit.
1650	 */
1651	hz_per_byte = polling_limit_us ? ((8 + 4) * USEC_PER_SEC) / polling_limit_us : 0;
1652	byte_limit = hz_per_byte ? transfer->effective_speed_hz / hz_per_byte : 1;
1653
1654	/* run in polling mode for short transfers */
1655	if (transfer->len < byte_limit)
1656		return spi_imx_poll_transfer(spi, transfer);
1657
1658	return spi_imx_pio_transfer(spi, transfer);
1659}
1660
1661static int spi_imx_setup(struct spi_device *spi)
1662{
1663	dev_dbg(&spi->dev, "%s: mode %d, %u bpw, %d hz\n", __func__,
1664		 spi->mode, spi->bits_per_word, spi->max_speed_hz);
1665
1666	return 0;
1667}
1668
1669static void spi_imx_cleanup(struct spi_device *spi)
1670{
1671}
1672
1673static int
1674spi_imx_prepare_message(struct spi_controller *controller, struct spi_message *msg)
1675{
1676	struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
1677	int ret;
1678
1679	ret = pm_runtime_resume_and_get(spi_imx->dev);
1680	if (ret < 0) {
1681		dev_err(spi_imx->dev, "failed to enable clock\n");
1682		return ret;
1683	}
1684
1685	ret = spi_imx->devtype_data->prepare_message(spi_imx, msg);
1686	if (ret) {
1687		pm_runtime_mark_last_busy(spi_imx->dev);
1688		pm_runtime_put_autosuspend(spi_imx->dev);
1689	}
1690
1691	return ret;
1692}
1693
1694static int
1695spi_imx_unprepare_message(struct spi_controller *controller, struct spi_message *msg)
1696{
1697	struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
1698
1699	pm_runtime_mark_last_busy(spi_imx->dev);
1700	pm_runtime_put_autosuspend(spi_imx->dev);
1701	return 0;
1702}
1703
1704static int spi_imx_target_abort(struct spi_controller *controller)
1705{
1706	struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
1707
1708	spi_imx->target_aborted = true;
1709	complete(&spi_imx->xfer_done);
1710
1711	return 0;
1712}
1713
1714static int spi_imx_probe(struct platform_device *pdev)
1715{
1716	struct device_node *np = pdev->dev.of_node;
1717	struct spi_controller *controller;
1718	struct spi_imx_data *spi_imx;
1719	struct resource *res;
1720	int ret, irq, spi_drctl;
1721	const struct spi_imx_devtype_data *devtype_data =
1722			of_device_get_match_data(&pdev->dev);
1723	bool target_mode;
1724	u32 val;
1725
1726	target_mode = devtype_data->has_targetmode &&
1727		      of_property_read_bool(np, "spi-slave");
1728	if (target_mode)
1729		controller = spi_alloc_target(&pdev->dev,
1730					      sizeof(struct spi_imx_data));
1731	else
1732		controller = spi_alloc_host(&pdev->dev,
1733					    sizeof(struct spi_imx_data));
1734	if (!controller)
1735		return -ENOMEM;
1736
1737	ret = of_property_read_u32(np, "fsl,spi-rdy-drctl", &spi_drctl);
1738	if ((ret < 0) || (spi_drctl >= 0x3)) {
1739		/* '11' is reserved */
1740		spi_drctl = 0;
1741	}
1742
1743	platform_set_drvdata(pdev, controller);
1744
1745	controller->bits_per_word_mask = SPI_BPW_RANGE_MASK(1, 32);
1746	controller->bus_num = np ? -1 : pdev->id;
1747	controller->use_gpio_descriptors = true;
1748
1749	spi_imx = spi_controller_get_devdata(controller);
1750	spi_imx->controller = controller;
1751	spi_imx->dev = &pdev->dev;
1752	spi_imx->target_mode = target_mode;
1753
1754	spi_imx->devtype_data = devtype_data;
1755
1756	/*
1757	 * Get number of chip selects from device properties. This can be
1758	 * coming from device tree or boardfiles, if it is not defined,
1759	 * a default value of 3 chip selects will be used, as all the legacy
1760	 * board files have <= 3 chip selects.
1761	 */
1762	if (!device_property_read_u32(&pdev->dev, "num-cs", &val))
1763		controller->num_chipselect = val;
1764	else
1765		controller->num_chipselect = 3;
1766
1767	controller->transfer_one = spi_imx_transfer_one;
1768	controller->setup = spi_imx_setup;
1769	controller->cleanup = spi_imx_cleanup;
1770	controller->prepare_message = spi_imx_prepare_message;
1771	controller->unprepare_message = spi_imx_unprepare_message;
1772	controller->target_abort = spi_imx_target_abort;
1773	controller->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH | SPI_NO_CS |
1774				SPI_MOSI_IDLE_LOW;
1775
1776	if (is_imx35_cspi(spi_imx) || is_imx51_ecspi(spi_imx) ||
1777	    is_imx53_ecspi(spi_imx))
1778		controller->mode_bits |= SPI_LOOP | SPI_READY;
1779
1780	if (is_imx51_ecspi(spi_imx) || is_imx53_ecspi(spi_imx))
1781		controller->mode_bits |= SPI_RX_CPHA_FLIP;
1782
1783	if (is_imx51_ecspi(spi_imx) &&
1784	    device_property_read_u32(&pdev->dev, "cs-gpios", NULL))
1785		/*
1786		 * When using HW-CS implementing SPI_CS_WORD can be done by just
1787		 * setting the burst length to the word size. This is
1788		 * considerably faster than manually controlling the CS.
1789		 */
1790		controller->mode_bits |= SPI_CS_WORD;
1791
1792	if (is_imx51_ecspi(spi_imx) || is_imx53_ecspi(spi_imx)) {
1793		controller->max_native_cs = 4;
1794		controller->flags |= SPI_CONTROLLER_GPIO_SS;
1795	}
1796
1797	spi_imx->spi_drctl = spi_drctl;
1798
1799	init_completion(&spi_imx->xfer_done);
1800
1801	spi_imx->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 
1802	if (IS_ERR(spi_imx->base)) {
1803		ret = PTR_ERR(spi_imx->base);
1804		goto out_controller_put;
1805	}
1806	spi_imx->base_phys = res->start;
1807
1808	irq = platform_get_irq(pdev, 0);
1809	if (irq < 0) {
1810		ret = irq;
1811		goto out_controller_put;
1812	}
1813
1814	ret = devm_request_irq(&pdev->dev, irq, spi_imx_isr, 0,
1815			       dev_name(&pdev->dev), spi_imx);
1816	if (ret) {
1817		dev_err(&pdev->dev, "can't get irq%d: %d\n", irq, ret);
1818		goto out_controller_put;
1819	}
1820
1821	spi_imx->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
1822	if (IS_ERR(spi_imx->clk_ipg)) {
1823		ret = PTR_ERR(spi_imx->clk_ipg);
1824		goto out_controller_put;
1825	}
1826
1827	spi_imx->clk_per = devm_clk_get(&pdev->dev, "per");
1828	if (IS_ERR(spi_imx->clk_per)) {
1829		ret = PTR_ERR(spi_imx->clk_per);
1830		goto out_controller_put;
1831	}
1832
1833	ret = clk_prepare_enable(spi_imx->clk_per);
1834	if (ret)
1835		goto out_controller_put;
1836
1837	ret = clk_prepare_enable(spi_imx->clk_ipg);
1838	if (ret)
1839		goto out_put_per;
1840
1841	pm_runtime_set_autosuspend_delay(spi_imx->dev, MXC_RPM_TIMEOUT);
1842	pm_runtime_use_autosuspend(spi_imx->dev);
1843	pm_runtime_get_noresume(spi_imx->dev);
1844	pm_runtime_set_active(spi_imx->dev);
1845	pm_runtime_enable(spi_imx->dev);
1846
1847	spi_imx->spi_clk = clk_get_rate(spi_imx->clk_per);
1848	/*
1849	 * Only validated on i.mx35 and i.mx6 now, can remove the constraint
1850	 * if validated on other chips.
1851	 */
1852	if (spi_imx->devtype_data->has_dmamode) {
1853		ret = spi_imx_sdma_init(&pdev->dev, spi_imx, controller);
1854		if (ret == -EPROBE_DEFER)
1855			goto out_runtime_pm_put;
1856
1857		if (ret < 0)
1858			dev_dbg(&pdev->dev, "dma setup error %d, use pio\n",
1859				ret);
1860	}
1861
1862	spi_imx->devtype_data->reset(spi_imx);
1863
1864	spi_imx->devtype_data->intctrl(spi_imx, 0);
1865
1866	controller->dev.of_node = pdev->dev.of_node;
1867	ret = spi_register_controller(controller);
1868	if (ret) {
1869		dev_err_probe(&pdev->dev, ret, "register controller failed\n");
1870		goto out_register_controller;
1871	}
1872
1873	pm_runtime_mark_last_busy(spi_imx->dev);
1874	pm_runtime_put_autosuspend(spi_imx->dev);
1875
1876	return ret;
1877
1878out_register_controller:
1879	if (spi_imx->devtype_data->has_dmamode)
1880		spi_imx_sdma_exit(spi_imx);
1881out_runtime_pm_put:
1882	pm_runtime_dont_use_autosuspend(spi_imx->dev);
1883	pm_runtime_set_suspended(&pdev->dev);
1884	pm_runtime_disable(spi_imx->dev);
1885
1886	clk_disable_unprepare(spi_imx->clk_ipg);
1887out_put_per:
1888	clk_disable_unprepare(spi_imx->clk_per);
1889out_controller_put:
1890	spi_controller_put(controller);
1891
1892	return ret;
1893}
1894
1895static void spi_imx_remove(struct platform_device *pdev)
1896{
1897	struct spi_controller *controller = platform_get_drvdata(pdev);
1898	struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
1899	int ret;
1900
1901	spi_unregister_controller(controller);
1902
1903	ret = pm_runtime_get_sync(spi_imx->dev);
1904	if (ret >= 0)
1905		writel(0, spi_imx->base + MXC_CSPICTRL);
1906	else
1907		dev_warn(spi_imx->dev, "failed to enable clock, skip hw disable\n");
 
 
1908
1909	pm_runtime_dont_use_autosuspend(spi_imx->dev);
1910	pm_runtime_put_sync(spi_imx->dev);
1911	pm_runtime_disable(spi_imx->dev);
1912
1913	spi_imx_sdma_exit(spi_imx);
 
 
1914}
1915
1916static int __maybe_unused spi_imx_runtime_resume(struct device *dev)
1917{
1918	struct spi_controller *controller = dev_get_drvdata(dev);
1919	struct spi_imx_data *spi_imx;
1920	int ret;
1921
1922	spi_imx = spi_controller_get_devdata(controller);
1923
1924	ret = clk_prepare_enable(spi_imx->clk_per);
1925	if (ret)
1926		return ret;
1927
1928	ret = clk_prepare_enable(spi_imx->clk_ipg);
1929	if (ret) {
1930		clk_disable_unprepare(spi_imx->clk_per);
1931		return ret;
1932	}
1933
1934	return 0;
1935}
1936
1937static int __maybe_unused spi_imx_runtime_suspend(struct device *dev)
1938{
1939	struct spi_controller *controller = dev_get_drvdata(dev);
1940	struct spi_imx_data *spi_imx;
1941
1942	spi_imx = spi_controller_get_devdata(controller);
1943
1944	clk_disable_unprepare(spi_imx->clk_per);
1945	clk_disable_unprepare(spi_imx->clk_ipg);
1946
1947	return 0;
1948}
1949
1950static int __maybe_unused spi_imx_suspend(struct device *dev)
1951{
1952	pinctrl_pm_select_sleep_state(dev);
1953	return 0;
1954}
1955
1956static int __maybe_unused spi_imx_resume(struct device *dev)
1957{
1958	pinctrl_pm_select_default_state(dev);
1959	return 0;
1960}
1961
1962static const struct dev_pm_ops imx_spi_pm = {
1963	SET_RUNTIME_PM_OPS(spi_imx_runtime_suspend,
1964				spi_imx_runtime_resume, NULL)
1965	SET_SYSTEM_SLEEP_PM_OPS(spi_imx_suspend, spi_imx_resume)
1966};
1967
1968static struct platform_driver spi_imx_driver = {
1969	.driver = {
1970		   .name = DRIVER_NAME,
1971		   .of_match_table = spi_imx_dt_ids,
1972		   .pm = &imx_spi_pm,
1973	},
1974	.probe = spi_imx_probe,
1975	.remove_new = spi_imx_remove,
1976};
1977module_platform_driver(spi_imx_driver);
1978
1979MODULE_DESCRIPTION("i.MX SPI Controller driver");
1980MODULE_AUTHOR("Sascha Hauer, Pengutronix");
1981MODULE_LICENSE("GPL");
1982MODULE_ALIAS("platform:" DRIVER_NAME);