1/*
   2 * OMAP2 McSPI controller driver
   3 *
   4 * Copyright (C) 2005, 2006 Nokia Corporation
   5 * Author:	Samuel Ortiz <samuel.ortiz@nokia.com> and
   6 *		Juha Yrj�l� <juha.yrjola@nokia.com>
   7 *
   8 * This program is free software; you can redistribute it and/or modify
   9 * it under the terms of the GNU General Public License as published by
  10 * the Free Software Foundation; either version 2 of the License, or
  11 * (at your option) any later version.
  12 *
  13 * This program is distributed in the hope that it will be useful,
  14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16 * GNU General Public License for more details.
  17 */
  18
  19#include <linux/kernel.h>
  20#include <linux/interrupt.h>
  21#include <linux/module.h>
  22#include <linux/device.h>
  23#include <linux/delay.h>
  24#include <linux/dma-mapping.h>
  25#include <linux/dmaengine.h>
  26#include <linux/pinctrl/consumer.h>
  27#include <linux/platform_device.h>
  28#include <linux/err.h>
  29#include <linux/clk.h>
  30#include <linux/io.h>
  31#include <linux/slab.h>
  32#include <linux/pm_runtime.h>
  33#include <linux/of.h>
  34#include <linux/of_device.h>
  35#include <linux/gcd.h>
  36
  37#include <linux/spi/spi.h>
  38#include <linux/gpio.h>
  39
  40#include <linux/platform_data/spi-omap2-mcspi.h>
  41
  42#define OMAP2_MCSPI_MAX_FREQ		48000000
  43#define OMAP2_MCSPI_MAX_DIVIDER		4096
  44#define OMAP2_MCSPI_MAX_FIFODEPTH	64
  45#define OMAP2_MCSPI_MAX_FIFOWCNT	0xFFFF
  46#define SPI_AUTOSUSPEND_TIMEOUT		2000
  47
  48#define OMAP2_MCSPI_REVISION		0x00
  49#define OMAP2_MCSPI_SYSSTATUS		0x14
  50#define OMAP2_MCSPI_IRQSTATUS		0x18
  51#define OMAP2_MCSPI_IRQENABLE		0x1c
  52#define OMAP2_MCSPI_WAKEUPENABLE	0x20
  53#define OMAP2_MCSPI_SYST		0x24
  54#define OMAP2_MCSPI_MODULCTRL		0x28
  55#define OMAP2_MCSPI_XFERLEVEL		0x7c
  56
  57/* per-channel banks, 0x14 bytes each, first is: */
  58#define OMAP2_MCSPI_CHCONF0		0x2c
  59#define OMAP2_MCSPI_CHSTAT0		0x30
  60#define OMAP2_MCSPI_CHCTRL0		0x34
  61#define OMAP2_MCSPI_TX0			0x38
  62#define OMAP2_MCSPI_RX0			0x3c
  63
  64/* per-register bitmasks: */
  65#define OMAP2_MCSPI_IRQSTATUS_EOW	BIT(17)
  66
  67#define OMAP2_MCSPI_MODULCTRL_SINGLE	BIT(0)
  68#define OMAP2_MCSPI_MODULCTRL_MS	BIT(2)
  69#define OMAP2_MCSPI_MODULCTRL_STEST	BIT(3)
  70
  71#define OMAP2_MCSPI_CHCONF_PHA		BIT(0)
  72#define OMAP2_MCSPI_CHCONF_POL		BIT(1)
  73#define OMAP2_MCSPI_CHCONF_CLKD_MASK	(0x0f << 2)
  74#define OMAP2_MCSPI_CHCONF_EPOL		BIT(6)
  75#define OMAP2_MCSPI_CHCONF_WL_MASK	(0x1f << 7)
  76#define OMAP2_MCSPI_CHCONF_TRM_RX_ONLY	BIT(12)
  77#define OMAP2_MCSPI_CHCONF_TRM_TX_ONLY	BIT(13)
  78#define OMAP2_MCSPI_CHCONF_TRM_MASK	(0x03 << 12)
  79#define OMAP2_MCSPI_CHCONF_DMAW		BIT(14)
  80#define OMAP2_MCSPI_CHCONF_DMAR		BIT(15)
  81#define OMAP2_MCSPI_CHCONF_DPE0		BIT(16)
  82#define OMAP2_MCSPI_CHCONF_DPE1		BIT(17)
  83#define OMAP2_MCSPI_CHCONF_IS		BIT(18)
  84#define OMAP2_MCSPI_CHCONF_TURBO	BIT(19)
  85#define OMAP2_MCSPI_CHCONF_FORCE	BIT(20)
  86#define OMAP2_MCSPI_CHCONF_FFET		BIT(27)
  87#define OMAP2_MCSPI_CHCONF_FFER		BIT(28)
  88#define OMAP2_MCSPI_CHCONF_CLKG		BIT(29)
  89
  90#define OMAP2_MCSPI_CHSTAT_RXS		BIT(0)
  91#define OMAP2_MCSPI_CHSTAT_TXS		BIT(1)
  92#define OMAP2_MCSPI_CHSTAT_EOT		BIT(2)
  93#define OMAP2_MCSPI_CHSTAT_TXFFE	BIT(3)
  94
  95#define OMAP2_MCSPI_CHCTRL_EN		BIT(0)
  96#define OMAP2_MCSPI_CHCTRL_EXTCLK_MASK	(0xff << 8)
  97
  98#define OMAP2_MCSPI_WAKEUPENABLE_WKEN	BIT(0)
  99
 100/* We have 2 DMA channels per CS, one for RX and one for TX */
 101struct omap2_mcspi_dma {
 102	struct dma_chan *dma_tx;
 103	struct dma_chan *dma_rx;
 104
 105	struct completion dma_tx_completion;
 106	struct completion dma_rx_completion;
 107
 108	char dma_rx_ch_name[14];
 109	char dma_tx_ch_name[14];
 110};
 111
 112/* use PIO for small transfers, avoiding DMA setup/teardown overhead and
 113 * cache operations; better heuristics consider wordsize and bitrate.
 114 */
 115#define DMA_MIN_BYTES			160
 116
 117
 118/*
 119 * Used for context save and restore, structure members to be updated whenever
 120 * corresponding registers are modified.
 121 */
 122struct omap2_mcspi_regs {
 123	u32 modulctrl;
 124	u32 wakeupenable;
 125	struct list_head cs;
 126};
 127
 128struct omap2_mcspi {
 129	struct spi_master	*master;
 130	/* Virtual base address of the controller */
 131	void __iomem		*base;
 132	unsigned long		phys;
 133	/* SPI1 has 4 channels, while SPI2 has 2 */
 134	struct omap2_mcspi_dma	*dma_channels;
 135	struct device		*dev;
 136	struct omap2_mcspi_regs ctx;
 137	int			fifo_depth;
 138	unsigned int		pin_dir:1;
 139};
 140
 141struct omap2_mcspi_cs {
 142	void __iomem		*base;
 143	unsigned long		phys;
 144	int			word_len;
 145	u16			mode;
 146	struct list_head	node;
 147	/* Context save and restore shadow register */
 148	u32			chconf0, chctrl0;
 149};
 150
 151static inline void mcspi_write_reg(struct spi_master *master,
 152		int idx, u32 val)
 153{
 154	struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
 155
 156	writel_relaxed(val, mcspi->base + idx);
 157}
 158
 159static inline u32 mcspi_read_reg(struct spi_master *master, int idx)
 160{
 161	struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
 162
 163	return readl_relaxed(mcspi->base + idx);
 164}
 165
 166static inline void mcspi_write_cs_reg(const struct spi_device *spi,
 167		int idx, u32 val)
 168{
 169	struct omap2_mcspi_cs	*cs = spi->controller_state;
 170
 171	writel_relaxed(val, cs->base +  idx);
 172}
 173
 174static inline u32 mcspi_read_cs_reg(const struct spi_device *spi, int idx)
 175{
 176	struct omap2_mcspi_cs	*cs = spi->controller_state;
 177
 178	return readl_relaxed(cs->base + idx);
 179}
 180
 181static inline u32 mcspi_cached_chconf0(const struct spi_device *spi)
 182{
 183	struct omap2_mcspi_cs *cs = spi->controller_state;
 184
 185	return cs->chconf0;
 186}
 187
 188static inline void mcspi_write_chconf0(const struct spi_device *spi, u32 val)
 189{
 190	struct omap2_mcspi_cs *cs = spi->controller_state;
 191
 192	cs->chconf0 = val;
 193	mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCONF0, val);
 194	mcspi_read_cs_reg(spi, OMAP2_MCSPI_CHCONF0);
 195}
 196
 197static inline int mcspi_bytes_per_word(int word_len)
 198{
 199	if (word_len <= 8)
 200		return 1;
 201	else if (word_len <= 16)
 202		return 2;
 203	else /* word_len <= 32 */
 204		return 4;
 205}
 206
 207static void omap2_mcspi_set_dma_req(const struct spi_device *spi,
 208		int is_read, int enable)
 209{
 210	u32 l, rw;
 211
 212	l = mcspi_cached_chconf0(spi);
 213
 214	if (is_read) /* 1 is read, 0 write */
 215		rw = OMAP2_MCSPI_CHCONF_DMAR;
 216	else
 217		rw = OMAP2_MCSPI_CHCONF_DMAW;
 218
 219	if (enable)
 220		l |= rw;
 221	else
 222		l &= ~rw;
 223
 224	mcspi_write_chconf0(spi, l);
 225}
 226
 227static void omap2_mcspi_set_enable(const struct spi_device *spi, int enable)
 228{
 229	struct omap2_mcspi_cs *cs = spi->controller_state;
 230	u32 l;
 231
 232	l = cs->chctrl0;
 233	if (enable)
 234		l |= OMAP2_MCSPI_CHCTRL_EN;
 235	else
 236		l &= ~OMAP2_MCSPI_CHCTRL_EN;
 237	cs->chctrl0 = l;
 238	mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCTRL0, cs->chctrl0);
 239	/* Flash post-writes */
 240	mcspi_read_cs_reg(spi, OMAP2_MCSPI_CHCTRL0);
 241}
 242
 243static void omap2_mcspi_set_cs(struct spi_device *spi, bool enable)
 244{
 245	struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
 246	u32 l;
 247
 248	/* The controller handles the inverted chip selects
 249	 * using the OMAP2_MCSPI_CHCONF_EPOL bit so revert
 250	 * the inversion from the core spi_set_cs function.
 251	 */
 252	if (spi->mode & SPI_CS_HIGH)
 253		enable = !enable;
 254
 255	if (spi->controller_state) {
 256		int err = pm_runtime_get_sync(mcspi->dev);
 257		if (err < 0) {
 258			dev_err(mcspi->dev, "failed to get sync: %d\n", err);
 259			return;
 260		}
 261
 262		l = mcspi_cached_chconf0(spi);
 263
 264		if (enable)
 265			l &= ~OMAP2_MCSPI_CHCONF_FORCE;
 266		else
 267			l |= OMAP2_MCSPI_CHCONF_FORCE;
 268
 269		mcspi_write_chconf0(spi, l);
 270
 271		pm_runtime_mark_last_busy(mcspi->dev);
 272		pm_runtime_put_autosuspend(mcspi->dev);
 273	}
 274}
 275
 276static void omap2_mcspi_set_master_mode(struct spi_master *master)
 277{
 278	struct omap2_mcspi	*mcspi = spi_master_get_devdata(master);
 279	struct omap2_mcspi_regs	*ctx = &mcspi->ctx;
 280	u32 l;
 281
 282	/*
 283	 * Setup when switching from (reset default) slave mode
 284	 * to single-channel master mode
 285	 */
 286	l = mcspi_read_reg(master, OMAP2_MCSPI_MODULCTRL);
 287	l &= ~(OMAP2_MCSPI_MODULCTRL_STEST | OMAP2_MCSPI_MODULCTRL_MS);
 288	l |= OMAP2_MCSPI_MODULCTRL_SINGLE;
 289	mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, l);
 290
 291	ctx->modulctrl = l;
 292}
 293
 294static void omap2_mcspi_set_fifo(const struct spi_device *spi,
 295				struct spi_transfer *t, int enable)
 296{
 297	struct spi_master *master = spi->master;
 298	struct omap2_mcspi_cs *cs = spi->controller_state;
 299	struct omap2_mcspi *mcspi;
 300	unsigned int wcnt;
 301	int max_fifo_depth, fifo_depth, bytes_per_word;
 302	u32 chconf, xferlevel;
 303
 304	mcspi = spi_master_get_devdata(master);
 305
 306	chconf = mcspi_cached_chconf0(spi);
 307	if (enable) {
 308		bytes_per_word = mcspi_bytes_per_word(cs->word_len);
 309		if (t->len % bytes_per_word != 0)
 310			goto disable_fifo;
 311
 312		if (t->rx_buf != NULL && t->tx_buf != NULL)
 313			max_fifo_depth = OMAP2_MCSPI_MAX_FIFODEPTH / 2;
 314		else
 315			max_fifo_depth = OMAP2_MCSPI_MAX_FIFODEPTH;
 316
 317		fifo_depth = gcd(t->len, max_fifo_depth);
 318		if (fifo_depth < 2 || fifo_depth % bytes_per_word != 0)
 319			goto disable_fifo;
 320
 321		wcnt = t->len / bytes_per_word;
 322		if (wcnt > OMAP2_MCSPI_MAX_FIFOWCNT)
 323			goto disable_fifo;
 324
 325		xferlevel = wcnt << 16;
 326		if (t->rx_buf != NULL) {
 327			chconf |= OMAP2_MCSPI_CHCONF_FFER;
 328			xferlevel |= (fifo_depth - 1) << 8;
 329		}
 330		if (t->tx_buf != NULL) {
 331			chconf |= OMAP2_MCSPI_CHCONF_FFET;
 332			xferlevel |= fifo_depth - 1;
 333		}
 334
 335		mcspi_write_reg(master, OMAP2_MCSPI_XFERLEVEL, xferlevel);
 336		mcspi_write_chconf0(spi, chconf);
 337		mcspi->fifo_depth = fifo_depth;
 338
 339		return;
 340	}
 341
 342disable_fifo:
 343	if (t->rx_buf != NULL)
 344		chconf &= ~OMAP2_MCSPI_CHCONF_FFER;
 345
 346	if (t->tx_buf != NULL)
 347		chconf &= ~OMAP2_MCSPI_CHCONF_FFET;
 348
 349	mcspi_write_chconf0(spi, chconf);
 350	mcspi->fifo_depth = 0;
 351}
 352
 353static void omap2_mcspi_restore_ctx(struct omap2_mcspi *mcspi)
 354{
 355	struct spi_master	*spi_cntrl = mcspi->master;
 356	struct omap2_mcspi_regs	*ctx = &mcspi->ctx;
 357	struct omap2_mcspi_cs	*cs;
 358
 359	/* McSPI: context restore */
 360	mcspi_write_reg(spi_cntrl, OMAP2_MCSPI_MODULCTRL, ctx->modulctrl);
 361	mcspi_write_reg(spi_cntrl, OMAP2_MCSPI_WAKEUPENABLE, ctx->wakeupenable);
 362
 363	list_for_each_entry(cs, &ctx->cs, node)
 364		writel_relaxed(cs->chconf0, cs->base + OMAP2_MCSPI_CHCONF0);
 365}
 366
 367static int mcspi_wait_for_reg_bit(void __iomem *reg, unsigned long bit)
 368{
 369	unsigned long timeout;
 370
 371	timeout = jiffies + msecs_to_jiffies(1000);
 372	while (!(readl_relaxed(reg) & bit)) {
 373		if (time_after(jiffies, timeout)) {
 374			if (!(readl_relaxed(reg) & bit))
 375				return -ETIMEDOUT;
 376			else
 377				return 0;
 378		}
 379		cpu_relax();
 380	}
 381	return 0;
 382}
 383
 384static void omap2_mcspi_rx_callback(void *data)
 385{
 386	struct spi_device *spi = data;
 387	struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
 388	struct omap2_mcspi_dma *mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 389
 390	/* We must disable the DMA RX request */
 391	omap2_mcspi_set_dma_req(spi, 1, 0);
 392
 393	complete(&mcspi_dma->dma_rx_completion);
 394}
 395
 396static void omap2_mcspi_tx_callback(void *data)
 397{
 398	struct spi_device *spi = data;
 399	struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
 400	struct omap2_mcspi_dma *mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 401
 402	/* We must disable the DMA TX request */
 403	omap2_mcspi_set_dma_req(spi, 0, 0);
 404
 405	complete(&mcspi_dma->dma_tx_completion);
 406}
 407
 408static void omap2_mcspi_tx_dma(struct spi_device *spi,
 409				struct spi_transfer *xfer,
 410				struct dma_slave_config cfg)
 411{
 412	struct omap2_mcspi	*mcspi;
 413	struct omap2_mcspi_dma  *mcspi_dma;
 414	unsigned int		count;
 415
 416	mcspi = spi_master_get_devdata(spi->master);
 417	mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 418	count = xfer->len;
 419
 420	if (mcspi_dma->dma_tx) {
 421		struct dma_async_tx_descriptor *tx;
 422
 423		dmaengine_slave_config(mcspi_dma->dma_tx, &cfg);
 424
 425		tx = dmaengine_prep_slave_sg(mcspi_dma->dma_tx, xfer->tx_sg.sgl,
 426					     xfer->tx_sg.nents,
 427					     DMA_MEM_TO_DEV,
 428					     DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 429		if (tx) {
 430			tx->callback = omap2_mcspi_tx_callback;
 431			tx->callback_param = spi;
 432			dmaengine_submit(tx);
 433		} else {
 434			/* FIXME: fall back to PIO? */
 435		}
 436	}
 437	dma_async_issue_pending(mcspi_dma->dma_tx);
 438	omap2_mcspi_set_dma_req(spi, 0, 1);
 439
 440}
 441
 442static unsigned
 443omap2_mcspi_rx_dma(struct spi_device *spi, struct spi_transfer *xfer,
 444				struct dma_slave_config cfg,
 445				unsigned es)
 446{
 447	struct omap2_mcspi	*mcspi;
 448	struct omap2_mcspi_dma  *mcspi_dma;
 449	unsigned int		count, transfer_reduction = 0;
 450	struct scatterlist	*sg_out[2];
 451	int			nb_sizes = 0, out_mapped_nents[2], ret, x;
 452	size_t			sizes[2];
 453	u32			l;
 454	int			elements = 0;
 455	int			word_len, element_count;
 456	struct omap2_mcspi_cs	*cs = spi->controller_state;
 
 
 457	mcspi = spi_master_get_devdata(spi->master);
 458	mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 459	count = xfer->len;
 460
 461	/*
 462	 *  In the "End-of-Transfer Procedure" section for DMA RX in OMAP35x TRM
 463	 *  it mentions reducing DMA transfer length by one element in master
 464	 *  normal mode.
 465	 */
 466	if (mcspi->fifo_depth == 0)
 467		transfer_reduction = es;
 468
 469	word_len = cs->word_len;
 470	l = mcspi_cached_chconf0(spi);
 471
 472	if (word_len <= 8)
 473		element_count = count;
 474	else if (word_len <= 16)
 475		element_count = count >> 1;
 476	else /* word_len <= 32 */
 477		element_count = count >> 2;
 478
 479	if (mcspi_dma->dma_rx) {
 480		struct dma_async_tx_descriptor *tx;
 481
 482		dmaengine_slave_config(mcspi_dma->dma_rx, &cfg);
 483
 484		/*
 485		 *  Reduce DMA transfer length by one more if McSPI is
 486		 *  configured in turbo mode.
 487		 */
 488		if ((l & OMAP2_MCSPI_CHCONF_TURBO) && mcspi->fifo_depth == 0)
 489			transfer_reduction += es;
 490
 491		if (transfer_reduction) {
 492			/* Split sgl into two. The second sgl won't be used. */
 493			sizes[0] = count - transfer_reduction;
 494			sizes[1] = transfer_reduction;
 495			nb_sizes = 2;
 496		} else {
 497			/*
 498			 * Don't bother splitting the sgl. This essentially
 499			 * clones the original sgl.
 500			 */
 501			sizes[0] = count;
 502			nb_sizes = 1;
 503		}
 504
 505		ret = sg_split(xfer->rx_sg.sgl, xfer->rx_sg.nents,
 506			       0, nb_sizes,
 507			       sizes,
 508			       sg_out, out_mapped_nents,
 509			       GFP_KERNEL);
 510
 511		if (ret < 0) {
 512			dev_err(&spi->dev, "sg_split failed\n");
 513			return 0;
 514		}
 515
 516		tx = dmaengine_prep_slave_sg(mcspi_dma->dma_rx,
 517					     sg_out[0],
 518					     out_mapped_nents[0],
 519					     DMA_DEV_TO_MEM,
 520					     DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 521		if (tx) {
 522			tx->callback = omap2_mcspi_rx_callback;
 523			tx->callback_param = spi;
 524			dmaengine_submit(tx);
 525		} else {
 526				/* FIXME: fall back to PIO? */
 527		}
 528	}
 529
 530	dma_async_issue_pending(mcspi_dma->dma_rx);
 531	omap2_mcspi_set_dma_req(spi, 1, 1);
 532
 533	wait_for_completion(&mcspi_dma->dma_rx_completion);
 534
 535	for (x = 0; x < nb_sizes; x++)
 536		kfree(sg_out[x]);
 537
 538	if (mcspi->fifo_depth > 0)
 539		return count;
 540
 541	/*
 542	 *  Due to the DMA transfer length reduction the missing bytes must
 543	 *  be read manually to receive all of the expected data.
 544	 */
 545	omap2_mcspi_set_enable(spi, 0);
 546
 547	elements = element_count - 1;
 548
 549	if (l & OMAP2_MCSPI_CHCONF_TURBO) {
 550		elements--;
 551
 552		if (likely(mcspi_read_cs_reg(spi, OMAP2_MCSPI_CHSTAT0)
 553				   & OMAP2_MCSPI_CHSTAT_RXS)) {
 554			u32 w;
 555
 556			w = mcspi_read_cs_reg(spi, OMAP2_MCSPI_RX0);
 557			if (word_len <= 8)
 558				((u8 *)xfer->rx_buf)[elements++] = w;
 559			else if (word_len <= 16)
 560				((u16 *)xfer->rx_buf)[elements++] = w;
 561			else /* word_len <= 32 */
 562				((u32 *)xfer->rx_buf)[elements++] = w;
 563		} else {
 564			int bytes_per_word = mcspi_bytes_per_word(word_len);
 565			dev_err(&spi->dev, "DMA RX penultimate word empty\n");
 566			count -= (bytes_per_word << 1);
 567			omap2_mcspi_set_enable(spi, 1);
 568			return count;
 569		}
 570	}
 571	if (likely(mcspi_read_cs_reg(spi, OMAP2_MCSPI_CHSTAT0)
 572				& OMAP2_MCSPI_CHSTAT_RXS)) {
 573		u32 w;
 574
 575		w = mcspi_read_cs_reg(spi, OMAP2_MCSPI_RX0);
 576		if (word_len <= 8)
 577			((u8 *)xfer->rx_buf)[elements] = w;
 578		else if (word_len <= 16)
 579			((u16 *)xfer->rx_buf)[elements] = w;
 580		else /* word_len <= 32 */
 581			((u32 *)xfer->rx_buf)[elements] = w;
 582	} else {
 583		dev_err(&spi->dev, "DMA RX last word empty\n");
 584		count -= mcspi_bytes_per_word(word_len);
 585	}
 586	omap2_mcspi_set_enable(spi, 1);
 587	return count;
 588}
 589
 590static unsigned
 591omap2_mcspi_txrx_dma(struct spi_device *spi, struct spi_transfer *xfer)
 592{
 593	struct omap2_mcspi	*mcspi;
 594	struct omap2_mcspi_cs	*cs = spi->controller_state;
 595	struct omap2_mcspi_dma  *mcspi_dma;
 596	unsigned int		count;
 597	u32			l;
 598	u8			*rx;
 599	const u8		*tx;
 600	struct dma_slave_config	cfg;
 601	enum dma_slave_buswidth width;
 602	unsigned es;
 603	u32			burst;
 604	void __iomem		*chstat_reg;
 605	void __iomem            *irqstat_reg;
 606	int			wait_res;
 607
 608	mcspi = spi_master_get_devdata(spi->master);
 609	mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 610	l = mcspi_cached_chconf0(spi);
 611
 612
 613	if (cs->word_len <= 8) {
 614		width = DMA_SLAVE_BUSWIDTH_1_BYTE;
 615		es = 1;
 616	} else if (cs->word_len <= 16) {
 617		width = DMA_SLAVE_BUSWIDTH_2_BYTES;
 618		es = 2;
 619	} else {
 620		width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 621		es = 4;
 622	}
 623
 624	count = xfer->len;
 625	burst = 1;
 626
 627	if (mcspi->fifo_depth > 0) {
 628		if (count > mcspi->fifo_depth)
 629			burst = mcspi->fifo_depth / es;
 630		else
 631			burst = count / es;
 632	}
 633
 634	memset(&cfg, 0, sizeof(cfg));
 635	cfg.src_addr = cs->phys + OMAP2_MCSPI_RX0;
 636	cfg.dst_addr = cs->phys + OMAP2_MCSPI_TX0;
 637	cfg.src_addr_width = width;
 638	cfg.dst_addr_width = width;
 639	cfg.src_maxburst = burst;
 640	cfg.dst_maxburst = burst;
 641
 642	rx = xfer->rx_buf;
 643	tx = xfer->tx_buf;
 644
 645	if (tx != NULL)
 646		omap2_mcspi_tx_dma(spi, xfer, cfg);
 647
 648	if (rx != NULL)
 649		count = omap2_mcspi_rx_dma(spi, xfer, cfg, es);
 650
 651	if (tx != NULL) {
 652		wait_for_completion(&mcspi_dma->dma_tx_completion);
 653
 654		if (mcspi->fifo_depth > 0) {
 655			irqstat_reg = mcspi->base + OMAP2_MCSPI_IRQSTATUS;
 656
 657			if (mcspi_wait_for_reg_bit(irqstat_reg,
 658						OMAP2_MCSPI_IRQSTATUS_EOW) < 0)
 659				dev_err(&spi->dev, "EOW timed out\n");
 660
 661			mcspi_write_reg(mcspi->master, OMAP2_MCSPI_IRQSTATUS,
 662					OMAP2_MCSPI_IRQSTATUS_EOW);
 663		}
 664
 665		/* for TX_ONLY mode, be sure all words have shifted out */
 666		if (rx == NULL) {
 667			chstat_reg = cs->base + OMAP2_MCSPI_CHSTAT0;
 668			if (mcspi->fifo_depth > 0) {
 669				wait_res = mcspi_wait_for_reg_bit(chstat_reg,
 670						OMAP2_MCSPI_CHSTAT_TXFFE);
 671				if (wait_res < 0)
 672					dev_err(&spi->dev, "TXFFE timed out\n");
 673			} else {
 674				wait_res = mcspi_wait_for_reg_bit(chstat_reg,
 675						OMAP2_MCSPI_CHSTAT_TXS);
 676				if (wait_res < 0)
 677					dev_err(&spi->dev, "TXS timed out\n");
 678			}
 679			if (wait_res >= 0 &&
 680				(mcspi_wait_for_reg_bit(chstat_reg,
 681					OMAP2_MCSPI_CHSTAT_EOT) < 0))
 682				dev_err(&spi->dev, "EOT timed out\n");
 683		}
 684	}
 685	return count;
 686}
 687
 688static unsigned
 689omap2_mcspi_txrx_pio(struct spi_device *spi, struct spi_transfer *xfer)
 690{
 691	struct omap2_mcspi	*mcspi;
 692	struct omap2_mcspi_cs	*cs = spi->controller_state;
 693	unsigned int		count, c;
 694	u32			l;
 695	void __iomem		*base = cs->base;
 696	void __iomem		*tx_reg;
 697	void __iomem		*rx_reg;
 698	void __iomem		*chstat_reg;
 699	int			word_len;
 700
 701	mcspi = spi_master_get_devdata(spi->master);
 702	count = xfer->len;
 703	c = count;
 704	word_len = cs->word_len;
 705
 706	l = mcspi_cached_chconf0(spi);
 707
 708	/* We store the pre-calculated register addresses on stack to speed
 709	 * up the transfer loop. */
 710	tx_reg		= base + OMAP2_MCSPI_TX0;
 711	rx_reg		= base + OMAP2_MCSPI_RX0;
 712	chstat_reg	= base + OMAP2_MCSPI_CHSTAT0;
 713
 714	if (c < (word_len>>3))
 715		return 0;
 716
 717	if (word_len <= 8) {
 718		u8		*rx;
 719		const u8	*tx;
 720
 721		rx = xfer->rx_buf;
 722		tx = xfer->tx_buf;
 723
 724		do {
 725			c -= 1;
 726			if (tx != NULL) {
 727				if (mcspi_wait_for_reg_bit(chstat_reg,
 728						OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 729					dev_err(&spi->dev, "TXS timed out\n");
 730					goto out;
 731				}
 732				dev_vdbg(&spi->dev, "write-%d %02x\n",
 733						word_len, *tx);
 734				writel_relaxed(*tx++, tx_reg);
 735			}
 736			if (rx != NULL) {
 737				if (mcspi_wait_for_reg_bit(chstat_reg,
 738						OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 739					dev_err(&spi->dev, "RXS timed out\n");
 740					goto out;
 741				}
 742
 743				if (c == 1 && tx == NULL &&
 744				    (l & OMAP2_MCSPI_CHCONF_TURBO)) {
 745					omap2_mcspi_set_enable(spi, 0);
 746					*rx++ = readl_relaxed(rx_reg);
 747					dev_vdbg(&spi->dev, "read-%d %02x\n",
 748						    word_len, *(rx - 1));
 749					if (mcspi_wait_for_reg_bit(chstat_reg,
 750						OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 751						dev_err(&spi->dev,
 752							"RXS timed out\n");
 753						goto out;
 754					}
 755					c = 0;
 756				} else if (c == 0 && tx == NULL) {
 757					omap2_mcspi_set_enable(spi, 0);
 758				}
 759
 760				*rx++ = readl_relaxed(rx_reg);
 761				dev_vdbg(&spi->dev, "read-%d %02x\n",
 762						word_len, *(rx - 1));
 763			}
 764		} while (c);
 765	} else if (word_len <= 16) {
 766		u16		*rx;
 767		const u16	*tx;
 768
 769		rx = xfer->rx_buf;
 770		tx = xfer->tx_buf;
 771		do {
 772			c -= 2;
 773			if (tx != NULL) {
 774				if (mcspi_wait_for_reg_bit(chstat_reg,
 775						OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 776					dev_err(&spi->dev, "TXS timed out\n");
 777					goto out;
 778				}
 779				dev_vdbg(&spi->dev, "write-%d %04x\n",
 780						word_len, *tx);
 781				writel_relaxed(*tx++, tx_reg);
 782			}
 783			if (rx != NULL) {
 784				if (mcspi_wait_for_reg_bit(chstat_reg,
 785						OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 786					dev_err(&spi->dev, "RXS timed out\n");
 787					goto out;
 788				}
 789
 790				if (c == 2 && tx == NULL &&
 791				    (l & OMAP2_MCSPI_CHCONF_TURBO)) {
 792					omap2_mcspi_set_enable(spi, 0);
 793					*rx++ = readl_relaxed(rx_reg);
 794					dev_vdbg(&spi->dev, "read-%d %04x\n",
 795						    word_len, *(rx - 1));
 796					if (mcspi_wait_for_reg_bit(chstat_reg,
 797						OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 798						dev_err(&spi->dev,
 799							"RXS timed out\n");
 800						goto out;
 801					}
 802					c = 0;
 803				} else if (c == 0 && tx == NULL) {
 804					omap2_mcspi_set_enable(spi, 0);
 805				}
 806
 807				*rx++ = readl_relaxed(rx_reg);
 808				dev_vdbg(&spi->dev, "read-%d %04x\n",
 809						word_len, *(rx - 1));
 810			}
 811		} while (c >= 2);
 812	} else if (word_len <= 32) {
 813		u32		*rx;
 814		const u32	*tx;
 815
 816		rx = xfer->rx_buf;
 817		tx = xfer->tx_buf;
 818		do {
 819			c -= 4;
 820			if (tx != NULL) {
 821				if (mcspi_wait_for_reg_bit(chstat_reg,
 822						OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 823					dev_err(&spi->dev, "TXS timed out\n");
 824					goto out;
 825				}
 826				dev_vdbg(&spi->dev, "write-%d %08x\n",
 827						word_len, *tx);
 828				writel_relaxed(*tx++, tx_reg);
 829			}
 830			if (rx != NULL) {
 831				if (mcspi_wait_for_reg_bit(chstat_reg,
 832						OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 833					dev_err(&spi->dev, "RXS timed out\n");
 834					goto out;
 835				}
 836
 837				if (c == 4 && tx == NULL &&
 838				    (l & OMAP2_MCSPI_CHCONF_TURBO)) {
 839					omap2_mcspi_set_enable(spi, 0);
 840					*rx++ = readl_relaxed(rx_reg);
 841					dev_vdbg(&spi->dev, "read-%d %08x\n",
 842						    word_len, *(rx - 1));
 843					if (mcspi_wait_for_reg_bit(chstat_reg,
 844						OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 845						dev_err(&spi->dev,
 846							"RXS timed out\n");
 847						goto out;
 848					}
 849					c = 0;
 850				} else if (c == 0 && tx == NULL) {
 851					omap2_mcspi_set_enable(spi, 0);
 852				}
 853
 854				*rx++ = readl_relaxed(rx_reg);
 855				dev_vdbg(&spi->dev, "read-%d %08x\n",
 856						word_len, *(rx - 1));
 857			}
 858		} while (c >= 4);
 859	}
 860
 861	/* for TX_ONLY mode, be sure all words have shifted out */
 862	if (xfer->rx_buf == NULL) {
 863		if (mcspi_wait_for_reg_bit(chstat_reg,
 864				OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 865			dev_err(&spi->dev, "TXS timed out\n");
 866		} else if (mcspi_wait_for_reg_bit(chstat_reg,
 867				OMAP2_MCSPI_CHSTAT_EOT) < 0)
 868			dev_err(&spi->dev, "EOT timed out\n");
 869
 870		/* disable chan to purge rx datas received in TX_ONLY transfer,
 871		 * otherwise these rx datas will affect the direct following
 872		 * RX_ONLY transfer.
 873		 */
 874		omap2_mcspi_set_enable(spi, 0);
 875	}
 876out:
 877	omap2_mcspi_set_enable(spi, 1);
 878	return count - c;
 879}
 880
 881static u32 omap2_mcspi_calc_divisor(u32 speed_hz)
 882{
 883	u32 div;
 884
 885	for (div = 0; div < 15; div++)
 886		if (speed_hz >= (OMAP2_MCSPI_MAX_FREQ >> div))
 887			return div;
 888
 889	return 15;
 890}
 891
 892/* called only when no transfer is active to this device */
 893static int omap2_mcspi_setup_transfer(struct spi_device *spi,
 894		struct spi_transfer *t)
 895{
 896	struct omap2_mcspi_cs *cs = spi->controller_state;
 897	struct omap2_mcspi *mcspi;
 898	struct spi_master *spi_cntrl;
 899	u32 l = 0, clkd = 0, div, extclk = 0, clkg = 0;
 900	u8 word_len = spi->bits_per_word;
 901	u32 speed_hz = spi->max_speed_hz;
 902
 903	mcspi = spi_master_get_devdata(spi->master);
 904	spi_cntrl = mcspi->master;
 905
 906	if (t != NULL && t->bits_per_word)
 907		word_len = t->bits_per_word;
 908
 909	cs->word_len = word_len;
 910
 911	if (t && t->speed_hz)
 912		speed_hz = t->speed_hz;
 913
 914	speed_hz = min_t(u32, speed_hz, OMAP2_MCSPI_MAX_FREQ);
 915	if (speed_hz < (OMAP2_MCSPI_MAX_FREQ / OMAP2_MCSPI_MAX_DIVIDER)) {
 916		clkd = omap2_mcspi_calc_divisor(speed_hz);
 917		speed_hz = OMAP2_MCSPI_MAX_FREQ >> clkd;
 918		clkg = 0;
 919	} else {
 920		div = (OMAP2_MCSPI_MAX_FREQ + speed_hz - 1) / speed_hz;
 921		speed_hz = OMAP2_MCSPI_MAX_FREQ / div;
 922		clkd = (div - 1) & 0xf;
 923		extclk = (div - 1) >> 4;
 924		clkg = OMAP2_MCSPI_CHCONF_CLKG;
 925	}
 926
 927	l = mcspi_cached_chconf0(spi);
 928
 929	/* standard 4-wire master mode:  SCK, MOSI/out, MISO/in, nCS
 930	 * REVISIT: this controller could support SPI_3WIRE mode.
 931	 */
 932	if (mcspi->pin_dir == MCSPI_PINDIR_D0_IN_D1_OUT) {
 933		l &= ~OMAP2_MCSPI_CHCONF_IS;
 934		l &= ~OMAP2_MCSPI_CHCONF_DPE1;
 935		l |= OMAP2_MCSPI_CHCONF_DPE0;
 936	} else {
 937		l |= OMAP2_MCSPI_CHCONF_IS;
 938		l |= OMAP2_MCSPI_CHCONF_DPE1;
 939		l &= ~OMAP2_MCSPI_CHCONF_DPE0;
 940	}
 941
 942	/* wordlength */
 943	l &= ~OMAP2_MCSPI_CHCONF_WL_MASK;
 944	l |= (word_len - 1) << 7;
 945
 946	/* set chipselect polarity; manage with FORCE */
 947	if (!(spi->mode & SPI_CS_HIGH))
 948		l |= OMAP2_MCSPI_CHCONF_EPOL;	/* active-low; normal */
 949	else
 950		l &= ~OMAP2_MCSPI_CHCONF_EPOL;
 951
 952	/* set clock divisor */
 953	l &= ~OMAP2_MCSPI_CHCONF_CLKD_MASK;
 954	l |= clkd << 2;
 955
 956	/* set clock granularity */
 957	l &= ~OMAP2_MCSPI_CHCONF_CLKG;
 958	l |= clkg;
 959	if (clkg) {
 960		cs->chctrl0 &= ~OMAP2_MCSPI_CHCTRL_EXTCLK_MASK;
 961		cs->chctrl0 |= extclk << 8;
 962		mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCTRL0, cs->chctrl0);
 963	}
 964
 965	/* set SPI mode 0..3 */
 966	if (spi->mode & SPI_CPOL)
 967		l |= OMAP2_MCSPI_CHCONF_POL;
 968	else
 969		l &= ~OMAP2_MCSPI_CHCONF_POL;
 970	if (spi->mode & SPI_CPHA)
 971		l |= OMAP2_MCSPI_CHCONF_PHA;
 972	else
 973		l &= ~OMAP2_MCSPI_CHCONF_PHA;
 974
 975	mcspi_write_chconf0(spi, l);
 976
 977	cs->mode = spi->mode;
 978
 979	dev_dbg(&spi->dev, "setup: speed %d, sample %s edge, clk %s\n",
 980			speed_hz,
 981			(spi->mode & SPI_CPHA) ? "trailing" : "leading",
 982			(spi->mode & SPI_CPOL) ? "inverted" : "normal");
 983
 984	return 0;
 985}
 986
 987/*
 988 * Note that we currently allow DMA only if we get a channel
 989 * for both rx and tx. Otherwise we'll do PIO for both rx and tx.
 990 */
 991static int omap2_mcspi_request_dma(struct spi_device *spi)
 992{
 993	struct spi_master	*master = spi->master;
 994	struct omap2_mcspi	*mcspi;
 995	struct omap2_mcspi_dma	*mcspi_dma;
 996	int ret = 0;
 997
 998	mcspi = spi_master_get_devdata(master);
 999	mcspi_dma = mcspi->dma_channels + spi->chip_select;
1000
1001	init_completion(&mcspi_dma->dma_rx_completion);
1002	init_completion(&mcspi_dma->dma_tx_completion);
1003
1004	mcspi_dma->dma_rx = dma_request_chan(&master->dev,
1005					     mcspi_dma->dma_rx_ch_name);
1006	if (IS_ERR(mcspi_dma->dma_rx)) {
1007		ret = PTR_ERR(mcspi_dma->dma_rx);
1008		mcspi_dma->dma_rx = NULL;
1009		goto no_dma;
1010	}
1011
1012	mcspi_dma->dma_tx = dma_request_chan(&master->dev,
1013					     mcspi_dma->dma_tx_ch_name);
1014	if (IS_ERR(mcspi_dma->dma_tx)) {
1015		ret = PTR_ERR(mcspi_dma->dma_tx);
1016		mcspi_dma->dma_tx = NULL;
1017		dma_release_channel(mcspi_dma->dma_rx);
1018		mcspi_dma->dma_rx = NULL;
1019	}
1020
1021no_dma:
1022	return ret;
1023}
1024
1025static int omap2_mcspi_setup(struct spi_device *spi)
1026{
1027	int			ret;
1028	struct omap2_mcspi	*mcspi = spi_master_get_devdata(spi->master);
1029	struct omap2_mcspi_regs	*ctx = &mcspi->ctx;
1030	struct omap2_mcspi_dma	*mcspi_dma;
1031	struct omap2_mcspi_cs	*cs = spi->controller_state;
1032
1033	mcspi_dma = &mcspi->dma_channels[spi->chip_select];
1034
1035	if (!cs) {
1036		cs = kzalloc(sizeof *cs, GFP_KERNEL);
1037		if (!cs)
1038			return -ENOMEM;
1039		cs->base = mcspi->base + spi->chip_select * 0x14;
1040		cs->phys = mcspi->phys + spi->chip_select * 0x14;
1041		cs->mode = 0;
1042		cs->chconf0 = 0;
1043		cs->chctrl0 = 0;
1044		spi->controller_state = cs;
1045		/* Link this to context save list */
1046		list_add_tail(&cs->node, &ctx->cs);
1047
1048		if (gpio_is_valid(spi->cs_gpio)) {
1049			ret = gpio_request(spi->cs_gpio, dev_name(&spi->dev));
1050			if (ret) {
1051				dev_err(&spi->dev, "failed to request gpio\n");
1052				return ret;
1053			}
1054			gpio_direction_output(spi->cs_gpio,
1055					 !(spi->mode & SPI_CS_HIGH));
1056		}
1057	}
1058
1059	if (!mcspi_dma->dma_rx || !mcspi_dma->dma_tx) {
1060		ret = omap2_mcspi_request_dma(spi);
1061		if (ret)
1062			dev_warn(&spi->dev, "not using DMA for McSPI (%d)\n",
1063				 ret);
1064	}
1065
1066	ret = pm_runtime_get_sync(mcspi->dev);
1067	if (ret < 0)
1068		return ret;
1069
1070	ret = omap2_mcspi_setup_transfer(spi, NULL);
1071	pm_runtime_mark_last_busy(mcspi->dev);
1072	pm_runtime_put_autosuspend(mcspi->dev);
1073
1074	return ret;
1075}
1076
1077static void omap2_mcspi_cleanup(struct spi_device *spi)
1078{
1079	struct omap2_mcspi	*mcspi;
1080	struct omap2_mcspi_dma	*mcspi_dma;
1081	struct omap2_mcspi_cs	*cs;
1082
1083	mcspi = spi_master_get_devdata(spi->master);
1084
1085	if (spi->controller_state) {
1086		/* Unlink controller state from context save list */
1087		cs = spi->controller_state;
1088		list_del(&cs->node);
1089
1090		kfree(cs);
1091	}
1092
1093	if (spi->chip_select < spi->master->num_chipselect) {
1094		mcspi_dma = &mcspi->dma_channels[spi->chip_select];
1095
1096		if (mcspi_dma->dma_rx) {
1097			dma_release_channel(mcspi_dma->dma_rx);
1098			mcspi_dma->dma_rx = NULL;
1099		}
1100		if (mcspi_dma->dma_tx) {
1101			dma_release_channel(mcspi_dma->dma_tx);
1102			mcspi_dma->dma_tx = NULL;
1103		}
1104	}
1105
1106	if (gpio_is_valid(spi->cs_gpio))
1107		gpio_free(spi->cs_gpio);
1108}
1109
1110static int omap2_mcspi_transfer_one(struct spi_master *master,
1111				    struct spi_device *spi,
1112				    struct spi_transfer *t)
1113{
1114
1115	/* We only enable one channel at a time -- the one whose message is
1116	 * -- although this controller would gladly
1117	 * arbitrate among multiple channels.  This corresponds to "single
1118	 * channel" master mode.  As a side effect, we need to manage the
1119	 * chipselect with the FORCE bit ... CS != channel enable.
1120	 */
1121
1122	struct omap2_mcspi		*mcspi;
1123	struct omap2_mcspi_dma		*mcspi_dma;
1124	struct omap2_mcspi_cs		*cs;
1125	struct omap2_mcspi_device_config *cd;
1126	int				par_override = 0;
1127	int				status = 0;
1128	u32				chconf;
1129
1130	mcspi = spi_master_get_devdata(master);
1131	mcspi_dma = mcspi->dma_channels + spi->chip_select;
1132	cs = spi->controller_state;
1133	cd = spi->controller_data;
1134
1135	/*
1136	 * The slave driver could have changed spi->mode in which case
1137	 * it will be different from cs->mode (the current hardware setup).
1138	 * If so, set par_override (even though its not a parity issue) so
1139	 * omap2_mcspi_setup_transfer will be called to configure the hardware
1140	 * with the correct mode on the first iteration of the loop below.
1141	 */
1142	if (spi->mode != cs->mode)
1143		par_override = 1;
1144
1145	omap2_mcspi_set_enable(spi, 0);
1146
1147	if (gpio_is_valid(spi->cs_gpio))
1148		omap2_mcspi_set_cs(spi, spi->mode & SPI_CS_HIGH);
1149
1150	if (par_override ||
1151	    (t->speed_hz != spi->max_speed_hz) ||
1152	    (t->bits_per_word != spi->bits_per_word)) {
1153		par_override = 1;
1154		status = omap2_mcspi_setup_transfer(spi, t);
1155		if (status < 0)
1156			goto out;
1157		if (t->speed_hz == spi->max_speed_hz &&
1158		    t->bits_per_word == spi->bits_per_word)
1159			par_override = 0;
1160	}
1161	if (cd && cd->cs_per_word) {
1162		chconf = mcspi->ctx.modulctrl;
1163		chconf &= ~OMAP2_MCSPI_MODULCTRL_SINGLE;
1164		mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, chconf);
1165		mcspi->ctx.modulctrl =
1166			mcspi_read_cs_reg(spi, OMAP2_MCSPI_MODULCTRL);
1167	}
1168
1169	chconf = mcspi_cached_chconf0(spi);
1170	chconf &= ~OMAP2_MCSPI_CHCONF_TRM_MASK;
1171	chconf &= ~OMAP2_MCSPI_CHCONF_TURBO;
1172
1173	if (t->tx_buf == NULL)
1174		chconf |= OMAP2_MCSPI_CHCONF_TRM_RX_ONLY;
1175	else if (t->rx_buf == NULL)
1176		chconf |= OMAP2_MCSPI_CHCONF_TRM_TX_ONLY;
1177
1178	if (cd && cd->turbo_mode && t->tx_buf == NULL) {
1179		/* Turbo mode is for more than one word */
1180		if (t->len > ((cs->word_len + 7) >> 3))
1181			chconf |= OMAP2_MCSPI_CHCONF_TURBO;
1182	}
1183
1184	mcspi_write_chconf0(spi, chconf);
1185
1186	if (t->len) {
1187		unsigned	count;
1188
1189		if ((mcspi_dma->dma_rx && mcspi_dma->dma_tx) &&
1190		    master->cur_msg_mapped &&
1191		    master->can_dma(master, spi, t))
1192			omap2_mcspi_set_fifo(spi, t, 1);
1193
1194		omap2_mcspi_set_enable(spi, 1);
1195
1196		/* RX_ONLY mode needs dummy data in TX reg */
1197		if (t->tx_buf == NULL)
1198			writel_relaxed(0, cs->base
1199					+ OMAP2_MCSPI_TX0);
1200
1201		if ((mcspi_dma->dma_rx && mcspi_dma->dma_tx) &&
1202		    master->cur_msg_mapped &&
1203		    master->can_dma(master, spi, t))
1204			count = omap2_mcspi_txrx_dma(spi, t);
1205		else
1206			count = omap2_mcspi_txrx_pio(spi, t);
1207
1208		if (count != t->len) {
1209			status = -EIO;
1210			goto out;
1211		}
1212	}
1213
1214	omap2_mcspi_set_enable(spi, 0);
1215
1216	if (mcspi->fifo_depth > 0)
1217		omap2_mcspi_set_fifo(spi, t, 0);
1218
1219out:
1220	/* Restore defaults if they were overriden */
1221	if (par_override) {
1222		par_override = 0;
1223		status = omap2_mcspi_setup_transfer(spi, NULL);
1224	}
1225
1226	if (cd && cd->cs_per_word) {
1227		chconf = mcspi->ctx.modulctrl;
1228		chconf |= OMAP2_MCSPI_MODULCTRL_SINGLE;
1229		mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, chconf);
1230		mcspi->ctx.modulctrl =
1231			mcspi_read_cs_reg(spi, OMAP2_MCSPI_MODULCTRL);
1232	}
1233
1234	omap2_mcspi_set_enable(spi, 0);
1235
1236	if (gpio_is_valid(spi->cs_gpio))
1237		omap2_mcspi_set_cs(spi, !(spi->mode & SPI_CS_HIGH));
1238
1239	if (mcspi->fifo_depth > 0 && t)
1240		omap2_mcspi_set_fifo(spi, t, 0);
1241
1242	return status;
1243}
1244
1245static int omap2_mcspi_prepare_message(struct spi_master *master,
1246				       struct spi_message *msg)
1247{
1248	struct omap2_mcspi	*mcspi = spi_master_get_devdata(master);
1249	struct omap2_mcspi_regs	*ctx = &mcspi->ctx;
1250	struct omap2_mcspi_cs	*cs;
1251
1252	/* Only a single channel can have the FORCE bit enabled
1253	 * in its chconf0 register.
1254	 * Scan all channels and disable them except the current one.
1255	 * A FORCE can remain from a last transfer having cs_change enabled
1256	 */
1257	list_for_each_entry(cs, &ctx->cs, node) {
1258		if (msg->spi->controller_state == cs)
1259			continue;
1260
1261		if ((cs->chconf0 & OMAP2_MCSPI_CHCONF_FORCE)) {
1262			cs->chconf0 &= ~OMAP2_MCSPI_CHCONF_FORCE;
1263			writel_relaxed(cs->chconf0,
1264					cs->base + OMAP2_MCSPI_CHCONF0);
1265			readl_relaxed(cs->base + OMAP2_MCSPI_CHCONF0);
1266		}
1267	}
1268
1269	return 0;
1270}
1271
1272static bool omap2_mcspi_can_dma(struct spi_master *master,
1273				struct spi_device *spi,
1274				struct spi_transfer *xfer)
1275{
1276	return (xfer->len >= DMA_MIN_BYTES);
1277}
1278
1279static int omap2_mcspi_master_setup(struct omap2_mcspi *mcspi)
1280{
1281	struct spi_master	*master = mcspi->master;
1282	struct omap2_mcspi_regs	*ctx = &mcspi->ctx;
1283	int			ret = 0;
1284
1285	ret = pm_runtime_get_sync(mcspi->dev);
1286	if (ret < 0)
1287		return ret;
1288
1289	mcspi_write_reg(master, OMAP2_MCSPI_WAKEUPENABLE,
1290			OMAP2_MCSPI_WAKEUPENABLE_WKEN);
1291	ctx->wakeupenable = OMAP2_MCSPI_WAKEUPENABLE_WKEN;
1292
1293	omap2_mcspi_set_master_mode(master);
1294	pm_runtime_mark_last_busy(mcspi->dev);
1295	pm_runtime_put_autosuspend(mcspi->dev);
1296	return 0;
1297}
1298
1299static int omap_mcspi_runtime_resume(struct device *dev)
1300{
1301	struct omap2_mcspi	*mcspi;
1302	struct spi_master	*master;
1303
1304	master = dev_get_drvdata(dev);
1305	mcspi = spi_master_get_devdata(master);
1306	omap2_mcspi_restore_ctx(mcspi);
1307
1308	return 0;
1309}
1310
1311static struct omap2_mcspi_platform_config omap2_pdata = {
1312	.regs_offset = 0,
1313};
1314
1315static struct omap2_mcspi_platform_config omap4_pdata = {
1316	.regs_offset = OMAP4_MCSPI_REG_OFFSET,
1317};
1318
1319static const struct of_device_id omap_mcspi_of_match[] = {
1320	{
1321		.compatible = "ti,omap2-mcspi",
1322		.data = &omap2_pdata,
1323	},
1324	{
1325		.compatible = "ti,omap4-mcspi",
1326		.data = &omap4_pdata,
1327	},
1328	{ },
1329};
1330MODULE_DEVICE_TABLE(of, omap_mcspi_of_match);
1331
1332static int omap2_mcspi_probe(struct platform_device *pdev)
1333{
1334	struct spi_master	*master;
1335	const struct omap2_mcspi_platform_config *pdata;
1336	struct omap2_mcspi	*mcspi;
1337	struct resource		*r;
1338	int			status = 0, i;
1339	u32			regs_offset = 0;
1340	static int		bus_num = 1;
1341	struct device_node	*node = pdev->dev.of_node;
1342	const struct of_device_id *match;
1343
1344	master = spi_alloc_master(&pdev->dev, sizeof *mcspi);
1345	if (master == NULL) {
1346		dev_dbg(&pdev->dev, "master allocation failed\n");
1347		return -ENOMEM;
1348	}
1349
1350	/* the spi->mode bits understood by this driver: */
1351	master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;
1352	master->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 32);
1353	master->setup = omap2_mcspi_setup;
1354	master->auto_runtime_pm = true;
1355	master->prepare_message = omap2_mcspi_prepare_message;
1356	master->can_dma = omap2_mcspi_can_dma;
1357	master->transfer_one = omap2_mcspi_transfer_one;
1358	master->set_cs = omap2_mcspi_set_cs;
1359	master->cleanup = omap2_mcspi_cleanup;
1360	master->dev.of_node = node;
1361	master->max_speed_hz = OMAP2_MCSPI_MAX_FREQ;
1362	master->min_speed_hz = OMAP2_MCSPI_MAX_FREQ >> 15;
1363
1364	platform_set_drvdata(pdev, master);
1365
1366	mcspi = spi_master_get_devdata(master);
1367	mcspi->master = master;
1368
1369	match = of_match_device(omap_mcspi_of_match, &pdev->dev);
1370	if (match) {
1371		u32 num_cs = 1; /* default number of chipselect */
1372		pdata = match->data;
1373
1374		of_property_read_u32(node, "ti,spi-num-cs", &num_cs);
1375		master->num_chipselect = num_cs;
1376		master->bus_num = bus_num++;
1377		if (of_get_property(node, "ti,pindir-d0-out-d1-in", NULL))
1378			mcspi->pin_dir = MCSPI_PINDIR_D0_OUT_D1_IN;
1379	} else {
1380		pdata = dev_get_platdata(&pdev->dev);
1381		master->num_chipselect = pdata->num_cs;
1382		if (pdev->id != -1)
1383			master->bus_num = pdev->id;
1384		mcspi->pin_dir = pdata->pin_dir;
1385	}
1386	regs_offset = pdata->regs_offset;
1387
1388	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1389	mcspi->base = devm_ioremap_resource(&pdev->dev, r);
1390	if (IS_ERR(mcspi->base)) {
1391		status = PTR_ERR(mcspi->base);
1392		goto free_master;
1393	}
1394	mcspi->phys = r->start + regs_offset;
1395	mcspi->base += regs_offset;
1396
1397	mcspi->dev = &pdev->dev;
1398
1399	INIT_LIST_HEAD(&mcspi->ctx.cs);
1400
1401	mcspi->dma_channels = devm_kcalloc(&pdev->dev, master->num_chipselect,
1402					   sizeof(struct omap2_mcspi_dma),
1403					   GFP_KERNEL);
1404	if (mcspi->dma_channels == NULL) {
1405		status = -ENOMEM;
1406		goto free_master;
1407	}
1408
1409	for (i = 0; i < master->num_chipselect; i++) {
1410		sprintf(mcspi->dma_channels[i].dma_rx_ch_name, "rx%d", i);
1411		sprintf(mcspi->dma_channels[i].dma_tx_ch_name, "tx%d", i);
1412	}
1413
1414	if (status < 0)
1415		goto free_master;
1416
1417	pm_runtime_use_autosuspend(&pdev->dev);
1418	pm_runtime_set_autosuspend_delay(&pdev->dev, SPI_AUTOSUSPEND_TIMEOUT);
1419	pm_runtime_enable(&pdev->dev);
1420
1421	status = omap2_mcspi_master_setup(mcspi);
1422	if (status < 0)
1423		goto disable_pm;
1424
1425	status = devm_spi_register_master(&pdev->dev, master);
1426	if (status < 0)
1427		goto disable_pm;
1428
1429	return status;
1430
1431disable_pm:
1432	pm_runtime_dont_use_autosuspend(&pdev->dev);
1433	pm_runtime_put_sync(&pdev->dev);
1434	pm_runtime_disable(&pdev->dev);
1435free_master:
1436	spi_master_put(master);
1437	return status;
1438}
1439
1440static int omap2_mcspi_remove(struct platform_device *pdev)
1441{
1442	struct spi_master *master = platform_get_drvdata(pdev);
1443	struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1444
1445	pm_runtime_dont_use_autosuspend(mcspi->dev);
1446	pm_runtime_put_sync(mcspi->dev);
1447	pm_runtime_disable(&pdev->dev);
1448
1449	return 0;
1450}
1451
1452/* work with hotplug and coldplug */
1453MODULE_ALIAS("platform:omap2_mcspi");
1454
1455#ifdef	CONFIG_SUSPEND
1456/*
1457 * When SPI wake up from off-mode, CS is in activate state. If it was in
1458 * unactive state when driver was suspend, then force it to unactive state at
1459 * wake up.
1460 */
1461static int omap2_mcspi_resume(struct device *dev)
1462{
1463	struct spi_master	*master = dev_get_drvdata(dev);
1464	struct omap2_mcspi	*mcspi = spi_master_get_devdata(master);
1465	struct omap2_mcspi_regs	*ctx = &mcspi->ctx;
1466	struct omap2_mcspi_cs	*cs;
1467
1468	pm_runtime_get_sync(mcspi->dev);
1469	list_for_each_entry(cs, &ctx->cs, node) {
1470		if ((cs->chconf0 & OMAP2_MCSPI_CHCONF_FORCE) == 0) {
1471			/*
1472			 * We need to toggle CS state for OMAP take this
1473			 * change in account.
1474			 */
1475			cs->chconf0 |= OMAP2_MCSPI_CHCONF_FORCE;
1476			writel_relaxed(cs->chconf0, cs->base + OMAP2_MCSPI_CHCONF0);
1477			cs->chconf0 &= ~OMAP2_MCSPI_CHCONF_FORCE;
1478			writel_relaxed(cs->chconf0, cs->base + OMAP2_MCSPI_CHCONF0);
1479		}
1480	}
1481	pm_runtime_mark_last_busy(mcspi->dev);
1482	pm_runtime_put_autosuspend(mcspi->dev);
1483
1484	return pinctrl_pm_select_default_state(dev);
1485}
1486
1487static int omap2_mcspi_suspend(struct device *dev)
1488{
1489	return pinctrl_pm_select_sleep_state(dev);
1490}
1491
1492#else
1493#define omap2_mcspi_suspend	NULL
1494#define	omap2_mcspi_resume	NULL
1495#endif
1496
1497static const struct dev_pm_ops omap2_mcspi_pm_ops = {
1498	.resume = omap2_mcspi_resume,
1499	.suspend = omap2_mcspi_suspend,
1500	.runtime_resume	= omap_mcspi_runtime_resume,
1501};
1502
1503static struct platform_driver omap2_mcspi_driver = {
1504	.driver = {
1505		.name =		"omap2_mcspi",
1506		.pm =		&omap2_mcspi_pm_ops,
1507		.of_match_table = omap_mcspi_of_match,
1508	},
1509	.probe =	omap2_mcspi_probe,
1510	.remove =	omap2_mcspi_remove,
1511};
1512
1513module_platform_driver(omap2_mcspi_driver);
1514MODULE_LICENSE("GPL");