1/*
   2 * Driver for Atmel AT32 and AT91 SPI Controllers
   3 *
   4 * Copyright (C) 2006 Atmel Corporation
   5 *
   6 * This program is free software; you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License version 2 as
   8 * published by the Free Software Foundation.
   9 */
  10
  11#include <linux/kernel.h>
  12#include <linux/clk.h>
  13#include <linux/module.h>
  14#include <linux/platform_device.h>
  15#include <linux/delay.h>
  16#include <linux/dma-mapping.h>
  17#include <linux/dmaengine.h>
  18#include <linux/err.h>
  19#include <linux/interrupt.h>
  20#include <linux/spi/spi.h>
  21#include <linux/slab.h>
  22#include <linux/platform_data/dma-atmel.h>
  23#include <linux/of.h>
  24
  25#include <linux/io.h>
  26#include <linux/gpio.h>
 
  27#include <linux/pinctrl/consumer.h>
  28#include <linux/pm_runtime.h>
  29
  30/* SPI register offsets */
  31#define SPI_CR					0x0000
  32#define SPI_MR					0x0004
  33#define SPI_RDR					0x0008
  34#define SPI_TDR					0x000c
  35#define SPI_SR					0x0010
  36#define SPI_IER					0x0014
  37#define SPI_IDR					0x0018
  38#define SPI_IMR					0x001c
  39#define SPI_CSR0				0x0030
  40#define SPI_CSR1				0x0034
  41#define SPI_CSR2				0x0038
  42#define SPI_CSR3				0x003c
  43#define SPI_FMR					0x0040
  44#define SPI_FLR					0x0044
  45#define SPI_VERSION				0x00fc
  46#define SPI_RPR					0x0100
  47#define SPI_RCR					0x0104
  48#define SPI_TPR					0x0108
  49#define SPI_TCR					0x010c
  50#define SPI_RNPR				0x0110
  51#define SPI_RNCR				0x0114
  52#define SPI_TNPR				0x0118
  53#define SPI_TNCR				0x011c
  54#define SPI_PTCR				0x0120
  55#define SPI_PTSR				0x0124
  56
  57/* Bitfields in CR */
  58#define SPI_SPIEN_OFFSET			0
  59#define SPI_SPIEN_SIZE				1
  60#define SPI_SPIDIS_OFFSET			1
  61#define SPI_SPIDIS_SIZE				1
  62#define SPI_SWRST_OFFSET			7
  63#define SPI_SWRST_SIZE				1
  64#define SPI_LASTXFER_OFFSET			24
  65#define SPI_LASTXFER_SIZE			1
  66#define SPI_TXFCLR_OFFSET			16
  67#define SPI_TXFCLR_SIZE				1
  68#define SPI_RXFCLR_OFFSET			17
  69#define SPI_RXFCLR_SIZE				1
  70#define SPI_FIFOEN_OFFSET			30
  71#define SPI_FIFOEN_SIZE				1
  72#define SPI_FIFODIS_OFFSET			31
  73#define SPI_FIFODIS_SIZE			1
  74
  75/* Bitfields in MR */
  76#define SPI_MSTR_OFFSET				0
  77#define SPI_MSTR_SIZE				1
  78#define SPI_PS_OFFSET				1
  79#define SPI_PS_SIZE				1
  80#define SPI_PCSDEC_OFFSET			2
  81#define SPI_PCSDEC_SIZE				1
  82#define SPI_FDIV_OFFSET				3
  83#define SPI_FDIV_SIZE				1
  84#define SPI_MODFDIS_OFFSET			4
  85#define SPI_MODFDIS_SIZE			1
  86#define SPI_WDRBT_OFFSET			5
  87#define SPI_WDRBT_SIZE				1
  88#define SPI_LLB_OFFSET				7
  89#define SPI_LLB_SIZE				1
  90#define SPI_PCS_OFFSET				16
  91#define SPI_PCS_SIZE				4
  92#define SPI_DLYBCS_OFFSET			24
  93#define SPI_DLYBCS_SIZE				8
  94
  95/* Bitfields in RDR */
  96#define SPI_RD_OFFSET				0
  97#define SPI_RD_SIZE				16
  98
  99/* Bitfields in TDR */
 100#define SPI_TD_OFFSET				0
 101#define SPI_TD_SIZE				16
 102
 103/* Bitfields in SR */
 104#define SPI_RDRF_OFFSET				0
 105#define SPI_RDRF_SIZE				1
 106#define SPI_TDRE_OFFSET				1
 107#define SPI_TDRE_SIZE				1
 108#define SPI_MODF_OFFSET				2
 109#define SPI_MODF_SIZE				1
 110#define SPI_OVRES_OFFSET			3
 111#define SPI_OVRES_SIZE				1
 112#define SPI_ENDRX_OFFSET			4
 113#define SPI_ENDRX_SIZE				1
 114#define SPI_ENDTX_OFFSET			5
 115#define SPI_ENDTX_SIZE				1
 116#define SPI_RXBUFF_OFFSET			6
 117#define SPI_RXBUFF_SIZE				1
 118#define SPI_TXBUFE_OFFSET			7
 119#define SPI_TXBUFE_SIZE				1
 120#define SPI_NSSR_OFFSET				8
 121#define SPI_NSSR_SIZE				1
 122#define SPI_TXEMPTY_OFFSET			9
 123#define SPI_TXEMPTY_SIZE			1
 124#define SPI_SPIENS_OFFSET			16
 125#define SPI_SPIENS_SIZE				1
 126#define SPI_TXFEF_OFFSET			24
 127#define SPI_TXFEF_SIZE				1
 128#define SPI_TXFFF_OFFSET			25
 129#define SPI_TXFFF_SIZE				1
 130#define SPI_TXFTHF_OFFSET			26
 131#define SPI_TXFTHF_SIZE				1
 132#define SPI_RXFEF_OFFSET			27
 133#define SPI_RXFEF_SIZE				1
 134#define SPI_RXFFF_OFFSET			28
 135#define SPI_RXFFF_SIZE				1
 136#define SPI_RXFTHF_OFFSET			29
 137#define SPI_RXFTHF_SIZE				1
 138#define SPI_TXFPTEF_OFFSET			30
 139#define SPI_TXFPTEF_SIZE			1
 140#define SPI_RXFPTEF_OFFSET			31
 141#define SPI_RXFPTEF_SIZE			1
 142
 143/* Bitfields in CSR0 */
 144#define SPI_CPOL_OFFSET				0
 145#define SPI_CPOL_SIZE				1
 146#define SPI_NCPHA_OFFSET			1
 147#define SPI_NCPHA_SIZE				1
 148#define SPI_CSAAT_OFFSET			3
 149#define SPI_CSAAT_SIZE				1
 150#define SPI_BITS_OFFSET				4
 151#define SPI_BITS_SIZE				4
 152#define SPI_SCBR_OFFSET				8
 153#define SPI_SCBR_SIZE				8
 154#define SPI_DLYBS_OFFSET			16
 155#define SPI_DLYBS_SIZE				8
 156#define SPI_DLYBCT_OFFSET			24
 157#define SPI_DLYBCT_SIZE				8
 158
 159/* Bitfields in RCR */
 160#define SPI_RXCTR_OFFSET			0
 161#define SPI_RXCTR_SIZE				16
 162
 163/* Bitfields in TCR */
 164#define SPI_TXCTR_OFFSET			0
 165#define SPI_TXCTR_SIZE				16
 166
 167/* Bitfields in RNCR */
 168#define SPI_RXNCR_OFFSET			0
 169#define SPI_RXNCR_SIZE				16
 170
 171/* Bitfields in TNCR */
 172#define SPI_TXNCR_OFFSET			0
 173#define SPI_TXNCR_SIZE				16
 174
 175/* Bitfields in PTCR */
 176#define SPI_RXTEN_OFFSET			0
 177#define SPI_RXTEN_SIZE				1
 178#define SPI_RXTDIS_OFFSET			1
 179#define SPI_RXTDIS_SIZE				1
 180#define SPI_TXTEN_OFFSET			8
 181#define SPI_TXTEN_SIZE				1
 182#define SPI_TXTDIS_OFFSET			9
 183#define SPI_TXTDIS_SIZE				1
 184
 185/* Bitfields in FMR */
 186#define SPI_TXRDYM_OFFSET			0
 187#define SPI_TXRDYM_SIZE				2
 188#define SPI_RXRDYM_OFFSET			4
 189#define SPI_RXRDYM_SIZE				2
 190#define SPI_TXFTHRES_OFFSET			16
 191#define SPI_TXFTHRES_SIZE			6
 192#define SPI_RXFTHRES_OFFSET			24
 193#define SPI_RXFTHRES_SIZE			6
 194
 195/* Bitfields in FLR */
 196#define SPI_TXFL_OFFSET				0
 197#define SPI_TXFL_SIZE				6
 198#define SPI_RXFL_OFFSET				16
 199#define SPI_RXFL_SIZE				6
 200
 201/* Constants for BITS */
 202#define SPI_BITS_8_BPT				0
 203#define SPI_BITS_9_BPT				1
 204#define SPI_BITS_10_BPT				2
 205#define SPI_BITS_11_BPT				3
 206#define SPI_BITS_12_BPT				4
 207#define SPI_BITS_13_BPT				5
 208#define SPI_BITS_14_BPT				6
 209#define SPI_BITS_15_BPT				7
 210#define SPI_BITS_16_BPT				8
 211#define SPI_ONE_DATA				0
 212#define SPI_TWO_DATA				1
 213#define SPI_FOUR_DATA				2
 214
 215/* Bit manipulation macros */
 216#define SPI_BIT(name) \
 217	(1 << SPI_##name##_OFFSET)
 218#define SPI_BF(name, value) \
 219	(((value) & ((1 << SPI_##name##_SIZE) - 1)) << SPI_##name##_OFFSET)
 220#define SPI_BFEXT(name, value) \
 221	(((value) >> SPI_##name##_OFFSET) & ((1 << SPI_##name##_SIZE) - 1))
 222#define SPI_BFINS(name, value, old) \
 223	(((old) & ~(((1 << SPI_##name##_SIZE) - 1) << SPI_##name##_OFFSET)) \
 224	  | SPI_BF(name, value))
 225
 226/* Register access macros */
 227#ifdef CONFIG_AVR32
 228#define spi_readl(port, reg) \
 229	__raw_readl((port)->regs + SPI_##reg)
 230#define spi_writel(port, reg, value) \
 231	__raw_writel((value), (port)->regs + SPI_##reg)
 232
 233#define spi_readw(port, reg) \
 234	__raw_readw((port)->regs + SPI_##reg)
 235#define spi_writew(port, reg, value) \
 236	__raw_writew((value), (port)->regs + SPI_##reg)
 237
 238#define spi_readb(port, reg) \
 239	__raw_readb((port)->regs + SPI_##reg)
 240#define spi_writeb(port, reg, value) \
 241	__raw_writeb((value), (port)->regs + SPI_##reg)
 242#else
 243#define spi_readl(port, reg) \
 244	readl_relaxed((port)->regs + SPI_##reg)
 245#define spi_writel(port, reg, value) \
 246	writel_relaxed((value), (port)->regs + SPI_##reg)
 247
 248#define spi_readw(port, reg) \
 249	readw_relaxed((port)->regs + SPI_##reg)
 250#define spi_writew(port, reg, value) \
 251	writew_relaxed((value), (port)->regs + SPI_##reg)
 252
 253#define spi_readb(port, reg) \
 254	readb_relaxed((port)->regs + SPI_##reg)
 255#define spi_writeb(port, reg, value) \
 256	writeb_relaxed((value), (port)->regs + SPI_##reg)
 257#endif
 258/* use PIO for small transfers, avoiding DMA setup/teardown overhead and
 259 * cache operations; better heuristics consider wordsize and bitrate.
 260 */
 261#define DMA_MIN_BYTES	16
 262
 263#define SPI_DMA_TIMEOUT		(msecs_to_jiffies(1000))
 264
 265#define AUTOSUSPEND_TIMEOUT	2000
 266
 267struct atmel_spi_dma {
 268	struct dma_chan			*chan_rx;
 269	struct dma_chan			*chan_tx;
 270	struct scatterlist		sgrx;
 271	struct scatterlist		sgtx;
 272	struct dma_async_tx_descriptor	*data_desc_rx;
 273	struct dma_async_tx_descriptor	*data_desc_tx;
 274
 275	struct at_dma_slave	dma_slave;
 276};
 277
 278struct atmel_spi_caps {
 279	bool	is_spi2;
 280	bool	has_wdrbt;
 281	bool	has_dma_support;
 282};
 283
 284/*
 285 * The core SPI transfer engine just talks to a register bank to set up
 286 * DMA transfers; transfer queue progress is driven by IRQs.  The clock
 287 * framework provides the base clock, subdivided for each spi_device.
 288 */
 289struct atmel_spi {
 290	spinlock_t		lock;
 291	unsigned long		flags;
 292
 293	phys_addr_t		phybase;
 294	void __iomem		*regs;
 295	int			irq;
 296	struct clk		*clk;
 297	struct platform_device	*pdev;
 
 298
 299	struct spi_transfer	*current_transfer;
 300	int			current_remaining_bytes;
 301	int			done_status;
 302
 303	struct completion	xfer_completion;
 304
 305	/* scratch buffer */
 306	void			*buffer;
 307	dma_addr_t		buffer_dma;
 308
 309	struct atmel_spi_caps	caps;
 310
 311	bool			use_dma;
 312	bool			use_pdc;
 313	bool			use_cs_gpios;
 314	/* dmaengine data */
 315	struct atmel_spi_dma	dma;
 316
 317	bool			keep_cs;
 318	bool			cs_active;
 319
 320	u32			fifo_size;
 321};
 322
 323/* Controller-specific per-slave state */
 324struct atmel_spi_device {
 325	unsigned int		npcs_pin;
 326	u32			csr;
 327};
 328
 329#define BUFFER_SIZE		PAGE_SIZE
 330#define INVALID_DMA_ADDRESS	0xffffffff
 331
 332/*
 333 * Version 2 of the SPI controller has
 334 *  - CR.LASTXFER
 335 *  - SPI_MR.DIV32 may become FDIV or must-be-zero (here: always zero)
 336 *  - SPI_SR.TXEMPTY, SPI_SR.NSSR (and corresponding irqs)
 337 *  - SPI_CSRx.CSAAT
 338 *  - SPI_CSRx.SBCR allows faster clocking
 339 */
 340static bool atmel_spi_is_v2(struct atmel_spi *as)
 341{
 342	return as->caps.is_spi2;
 343}
 344
 345/*
 346 * Earlier SPI controllers (e.g. on at91rm9200) have a design bug whereby
 347 * they assume that spi slave device state will not change on deselect, so
 348 * that automagic deselection is OK.  ("NPCSx rises if no data is to be
 349 * transmitted")  Not so!  Workaround uses nCSx pins as GPIOs; or newer
 350 * controllers have CSAAT and friends.
 351 *
 352 * Since the CSAAT functionality is a bit weird on newer controllers as
 353 * well, we use GPIO to control nCSx pins on all controllers, updating
 354 * MR.PCS to avoid confusing the controller.  Using GPIOs also lets us
 355 * support active-high chipselects despite the controller's belief that
 356 * only active-low devices/systems exists.
 357 *
 358 * However, at91rm9200 has a second erratum whereby nCS0 doesn't work
 359 * right when driven with GPIO.  ("Mode Fault does not allow more than one
 360 * Master on Chip Select 0.")  No workaround exists for that ... so for
 361 * nCS0 on that chip, we (a) don't use the GPIO, (b) can't support CS_HIGH,
 362 * and (c) will trigger that first erratum in some cases.
 363 */
 364
 365static void cs_activate(struct atmel_spi *as, struct spi_device *spi)
 366{
 367	struct atmel_spi_device *asd = spi->controller_state;
 368	unsigned active = spi->mode & SPI_CS_HIGH;
 369	u32 mr;
 370
 371	if (atmel_spi_is_v2(as)) {
 372		spi_writel(as, CSR0 + 4 * spi->chip_select, asd->csr);
 373		/* For the low SPI version, there is a issue that PDC transfer
 374		 * on CS1,2,3 needs SPI_CSR0.BITS config as SPI_CSR1,2,3.BITS
 375		 */
 376		spi_writel(as, CSR0, asd->csr);
 377		if (as->caps.has_wdrbt) {
 378			spi_writel(as, MR,
 379					SPI_BF(PCS, ~(0x01 << spi->chip_select))
 380					| SPI_BIT(WDRBT)
 381					| SPI_BIT(MODFDIS)
 382					| SPI_BIT(MSTR));
 383		} else {
 384			spi_writel(as, MR,
 385					SPI_BF(PCS, ~(0x01 << spi->chip_select))
 386					| SPI_BIT(MODFDIS)
 387					| SPI_BIT(MSTR));
 388		}
 389
 390		mr = spi_readl(as, MR);
 391		if (as->use_cs_gpios)
 392			gpio_set_value(asd->npcs_pin, active);
 393	} else {
 394		u32 cpol = (spi->mode & SPI_CPOL) ? SPI_BIT(CPOL) : 0;
 395		int i;
 396		u32 csr;
 397
 398		/* Make sure clock polarity is correct */
 399		for (i = 0; i < spi->master->num_chipselect; i++) {
 400			csr = spi_readl(as, CSR0 + 4 * i);
 401			if ((csr ^ cpol) & SPI_BIT(CPOL))
 402				spi_writel(as, CSR0 + 4 * i,
 403						csr ^ SPI_BIT(CPOL));
 404		}
 405
 406		mr = spi_readl(as, MR);
 407		mr = SPI_BFINS(PCS, ~(1 << spi->chip_select), mr);
 408		if (as->use_cs_gpios && spi->chip_select != 0)
 409			gpio_set_value(asd->npcs_pin, active);
 410		spi_writel(as, MR, mr);
 411	}
 412
 413	dev_dbg(&spi->dev, "activate %u%s, mr %08x\n",
 414			asd->npcs_pin, active ? " (high)" : "",
 415			mr);
 416}
 417
 418static void cs_deactivate(struct atmel_spi *as, struct spi_device *spi)
 419{
 420	struct atmel_spi_device *asd = spi->controller_state;
 421	unsigned active = spi->mode & SPI_CS_HIGH;
 422	u32 mr;
 423
 424	/* only deactivate *this* device; sometimes transfers to
 425	 * another device may be active when this routine is called.
 426	 */
 427	mr = spi_readl(as, MR);
 428	if (~SPI_BFEXT(PCS, mr) & (1 << spi->chip_select)) {
 429		mr = SPI_BFINS(PCS, 0xf, mr);
 430		spi_writel(as, MR, mr);
 431	}
 432
 433	dev_dbg(&spi->dev, "DEactivate %u%s, mr %08x\n",
 434			asd->npcs_pin, active ? " (low)" : "",
 435			mr);
 436
 437	if (!as->use_cs_gpios)
 438		spi_writel(as, CR, SPI_BIT(LASTXFER));
 439	else if (atmel_spi_is_v2(as) || spi->chip_select != 0)
 440		gpio_set_value(asd->npcs_pin, !active);
 441}
 442
 443static void atmel_spi_lock(struct atmel_spi *as) __acquires(&as->lock)
 444{
 445	spin_lock_irqsave(&as->lock, as->flags);
 446}
 447
 448static void atmel_spi_unlock(struct atmel_spi *as) __releases(&as->lock)
 449{
 450	spin_unlock_irqrestore(&as->lock, as->flags);
 451}
 452
 453static inline bool atmel_spi_use_dma(struct atmel_spi *as,
 454				struct spi_transfer *xfer)
 455{
 456	return as->use_dma && xfer->len >= DMA_MIN_BYTES;
 457}
 458
 
 
 
 
 
 
 
 
 
 459static int atmel_spi_dma_slave_config(struct atmel_spi *as,
 460				struct dma_slave_config *slave_config,
 461				u8 bits_per_word)
 462{
 
 463	int err = 0;
 464
 465	if (bits_per_word > 8) {
 466		slave_config->dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
 467		slave_config->src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
 468	} else {
 469		slave_config->dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
 470		slave_config->src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
 471	}
 472
 473	slave_config->dst_addr = (dma_addr_t)as->phybase + SPI_TDR;
 474	slave_config->src_addr = (dma_addr_t)as->phybase + SPI_RDR;
 475	slave_config->src_maxburst = 1;
 476	slave_config->dst_maxburst = 1;
 477	slave_config->device_fc = false;
 478
 479	/*
 480	 * This driver uses fixed peripheral select mode (PS bit set to '0' in
 481	 * the Mode Register).
 482	 * So according to the datasheet, when FIFOs are available (and
 483	 * enabled), the Transmit FIFO operates in Multiple Data Mode.
 484	 * In this mode, up to 2 data, not 4, can be written into the Transmit
 485	 * Data Register in a single access.
 486	 * However, the first data has to be written into the lowest 16 bits and
 487	 * the second data into the highest 16 bits of the Transmit
 488	 * Data Register. For 8bit data (the most frequent case), it would
 489	 * require to rework tx_buf so each data would actualy fit 16 bits.
 490	 * So we'd rather write only one data at the time. Hence the transmit
 491	 * path works the same whether FIFOs are available (and enabled) or not.
 492	 */
 493	slave_config->direction = DMA_MEM_TO_DEV;
 494	if (dmaengine_slave_config(as->dma.chan_tx, slave_config)) {
 495		dev_err(&as->pdev->dev,
 496			"failed to configure tx dma channel\n");
 497		err = -EINVAL;
 498	}
 499
 500	/*
 501	 * This driver configures the spi controller for master mode (MSTR bit
 502	 * set to '1' in the Mode Register).
 503	 * So according to the datasheet, when FIFOs are available (and
 504	 * enabled), the Receive FIFO operates in Single Data Mode.
 505	 * So the receive path works the same whether FIFOs are available (and
 506	 * enabled) or not.
 507	 */
 508	slave_config->direction = DMA_DEV_TO_MEM;
 509	if (dmaengine_slave_config(as->dma.chan_rx, slave_config)) {
 510		dev_err(&as->pdev->dev,
 511			"failed to configure rx dma channel\n");
 512		err = -EINVAL;
 513	}
 514
 515	return err;
 516}
 517
 518static int atmel_spi_configure_dma(struct atmel_spi *as)
 
 519{
 520	struct dma_slave_config	slave_config;
 521	struct device *dev = &as->pdev->dev;
 522	int err;
 523
 524	dma_cap_mask_t mask;
 525	dma_cap_zero(mask);
 526	dma_cap_set(DMA_SLAVE, mask);
 527
 528	as->dma.chan_tx = dma_request_slave_channel_reason(dev, "tx");
 529	if (IS_ERR(as->dma.chan_tx)) {
 530		err = PTR_ERR(as->dma.chan_tx);
 531		if (err == -EPROBE_DEFER) {
 532			dev_warn(dev, "no DMA channel available at the moment\n");
 533			return err;
 534		}
 535		dev_err(dev,
 536			"DMA TX channel not available, SPI unable to use DMA\n");
 537		err = -EBUSY;
 538		goto error;
 539	}
 540
 541	/*
 542	 * No reason to check EPROBE_DEFER here since we have already requested
 543	 * tx channel. If it fails here, it's for another reason.
 544	 */
 545	as->dma.chan_rx = dma_request_slave_channel(dev, "rx");
 546
 547	if (!as->dma.chan_rx) {
 548		dev_err(dev,
 549			"DMA RX channel not available, SPI unable to use DMA\n");
 550		err = -EBUSY;
 551		goto error;
 552	}
 553
 554	err = atmel_spi_dma_slave_config(as, &slave_config, 8);
 555	if (err)
 556		goto error;
 557
 558	dev_info(&as->pdev->dev,
 559			"Using %s (tx) and %s (rx) for DMA transfers\n",
 560			dma_chan_name(as->dma.chan_tx),
 561			dma_chan_name(as->dma.chan_rx));
 
 562	return 0;
 563error:
 564	if (as->dma.chan_rx)
 565		dma_release_channel(as->dma.chan_rx);
 566	if (!IS_ERR(as->dma.chan_tx))
 567		dma_release_channel(as->dma.chan_tx);
 
 
 568	return err;
 569}
 570
 571static void atmel_spi_stop_dma(struct atmel_spi *as)
 572{
 573	if (as->dma.chan_rx)
 574		dmaengine_terminate_all(as->dma.chan_rx);
 575	if (as->dma.chan_tx)
 576		dmaengine_terminate_all(as->dma.chan_tx);
 577}
 578
 579static void atmel_spi_release_dma(struct atmel_spi *as)
 580{
 581	if (as->dma.chan_rx)
 582		dma_release_channel(as->dma.chan_rx);
 583	if (as->dma.chan_tx)
 584		dma_release_channel(as->dma.chan_tx);
 
 
 
 
 585}
 586
 587/* This function is called by the DMA driver from tasklet context */
 588static void dma_callback(void *data)
 589{
 590	struct spi_master	*master = data;
 591	struct atmel_spi	*as = spi_master_get_devdata(master);
 592
 593	complete(&as->xfer_completion);
 594}
 595
 596/*
 597 * Next transfer using PIO without FIFO.
 598 */
 599static void atmel_spi_next_xfer_single(struct spi_master *master,
 600				       struct spi_transfer *xfer)
 601{
 602	struct atmel_spi	*as = spi_master_get_devdata(master);
 603	unsigned long xfer_pos = xfer->len - as->current_remaining_bytes;
 604
 605	dev_vdbg(master->dev.parent, "atmel_spi_next_xfer_pio\n");
 606
 607	/* Make sure data is not remaining in RDR */
 608	spi_readl(as, RDR);
 609	while (spi_readl(as, SR) & SPI_BIT(RDRF)) {
 610		spi_readl(as, RDR);
 611		cpu_relax();
 612	}
 613
 614	if (xfer->tx_buf) {
 615		if (xfer->bits_per_word > 8)
 616			spi_writel(as, TDR, *(u16 *)(xfer->tx_buf + xfer_pos));
 617		else
 618			spi_writel(as, TDR, *(u8 *)(xfer->tx_buf + xfer_pos));
 619	} else {
 620		spi_writel(as, TDR, 0);
 621	}
 622
 623	dev_dbg(master->dev.parent,
 624		"  start pio xfer %p: len %u tx %p rx %p bitpw %d\n",
 625		xfer, xfer->len, xfer->tx_buf, xfer->rx_buf,
 626		xfer->bits_per_word);
 627
 628	/* Enable relevant interrupts */
 629	spi_writel(as, IER, SPI_BIT(RDRF) | SPI_BIT(OVRES));
 630}
 631
 632/*
 633 * Next transfer using PIO with FIFO.
 634 */
 635static void atmel_spi_next_xfer_fifo(struct spi_master *master,
 636				     struct spi_transfer *xfer)
 637{
 638	struct atmel_spi *as = spi_master_get_devdata(master);
 639	u32 current_remaining_data, num_data;
 640	u32 offset = xfer->len - as->current_remaining_bytes;
 641	const u16 *words = (const u16 *)((u8 *)xfer->tx_buf + offset);
 642	const u8  *bytes = (const u8  *)((u8 *)xfer->tx_buf + offset);
 643	u16 td0, td1;
 644	u32 fifomr;
 645
 646	dev_vdbg(master->dev.parent, "atmel_spi_next_xfer_fifo\n");
 647
 648	/* Compute the number of data to transfer in the current iteration */
 649	current_remaining_data = ((xfer->bits_per_word > 8) ?
 650				  ((u32)as->current_remaining_bytes >> 1) :
 651				  (u32)as->current_remaining_bytes);
 652	num_data = min(current_remaining_data, as->fifo_size);
 653
 654	/* Flush RX and TX FIFOs */
 655	spi_writel(as, CR, SPI_BIT(RXFCLR) | SPI_BIT(TXFCLR));
 656	while (spi_readl(as, FLR))
 657		cpu_relax();
 658
 659	/* Set RX FIFO Threshold to the number of data to transfer */
 660	fifomr = spi_readl(as, FMR);
 661	spi_writel(as, FMR, SPI_BFINS(RXFTHRES, num_data, fifomr));
 662
 663	/* Clear FIFO flags in the Status Register, especially RXFTHF */
 664	(void)spi_readl(as, SR);
 665
 666	/* Fill TX FIFO */
 667	while (num_data >= 2) {
 668		if (xfer->tx_buf) {
 669			if (xfer->bits_per_word > 8) {
 670				td0 = *words++;
 671				td1 = *words++;
 672			} else {
 673				td0 = *bytes++;
 674				td1 = *bytes++;
 675			}
 676		} else {
 677			td0 = 0;
 678			td1 = 0;
 679		}
 680
 681		spi_writel(as, TDR, (td1 << 16) | td0);
 682		num_data -= 2;
 683	}
 684
 685	if (num_data) {
 686		if (xfer->tx_buf) {
 687			if (xfer->bits_per_word > 8)
 688				td0 = *words++;
 689			else
 690				td0 = *bytes++;
 691		} else {
 692			td0 = 0;
 693		}
 694
 695		spi_writew(as, TDR, td0);
 696		num_data--;
 697	}
 698
 699	dev_dbg(master->dev.parent,
 700		"  start fifo xfer %p: len %u tx %p rx %p bitpw %d\n",
 701		xfer, xfer->len, xfer->tx_buf, xfer->rx_buf,
 702		xfer->bits_per_word);
 703
 704	/*
 705	 * Enable RX FIFO Threshold Flag interrupt to be notified about
 706	 * transfer completion.
 707	 */
 708	spi_writel(as, IER, SPI_BIT(RXFTHF) | SPI_BIT(OVRES));
 709}
 710
 711/*
 712 * Next transfer using PIO.
 713 */
 714static void atmel_spi_next_xfer_pio(struct spi_master *master,
 715				    struct spi_transfer *xfer)
 716{
 717	struct atmel_spi *as = spi_master_get_devdata(master);
 718
 719	if (as->fifo_size)
 720		atmel_spi_next_xfer_fifo(master, xfer);
 721	else
 722		atmel_spi_next_xfer_single(master, xfer);
 723}
 724
 725/*
 726 * Submit next transfer for DMA.
 727 */
 728static int atmel_spi_next_xfer_dma_submit(struct spi_master *master,
 729				struct spi_transfer *xfer,
 730				u32 *plen)
 731{
 732	struct atmel_spi	*as = spi_master_get_devdata(master);
 733	struct dma_chan		*rxchan = as->dma.chan_rx;
 734	struct dma_chan		*txchan = as->dma.chan_tx;
 735	struct dma_async_tx_descriptor *rxdesc;
 736	struct dma_async_tx_descriptor *txdesc;
 737	struct dma_slave_config	slave_config;
 738	dma_cookie_t		cookie;
 739	u32	len = *plen;
 740
 741	dev_vdbg(master->dev.parent, "atmel_spi_next_xfer_dma_submit\n");
 742
 743	/* Check that the channels are available */
 744	if (!rxchan || !txchan)
 745		return -ENODEV;
 746
 747	/* release lock for DMA operations */
 748	atmel_spi_unlock(as);
 749
 750	/* prepare the RX dma transfer */
 751	sg_init_table(&as->dma.sgrx, 1);
 752	if (xfer->rx_buf) {
 753		as->dma.sgrx.dma_address = xfer->rx_dma + xfer->len - *plen;
 754	} else {
 755		as->dma.sgrx.dma_address = as->buffer_dma;
 756		if (len > BUFFER_SIZE)
 757			len = BUFFER_SIZE;
 758	}
 759
 760	/* prepare the TX dma transfer */
 761	sg_init_table(&as->dma.sgtx, 1);
 762	if (xfer->tx_buf) {
 763		as->dma.sgtx.dma_address = xfer->tx_dma + xfer->len - *plen;
 764	} else {
 765		as->dma.sgtx.dma_address = as->buffer_dma;
 766		if (len > BUFFER_SIZE)
 767			len = BUFFER_SIZE;
 768		memset(as->buffer, 0, len);
 769	}
 770
 771	sg_dma_len(&as->dma.sgtx) = len;
 772	sg_dma_len(&as->dma.sgrx) = len;
 773
 774	*plen = len;
 775
 776	if (atmel_spi_dma_slave_config(as, &slave_config,
 777				       xfer->bits_per_word))
 778		goto err_exit;
 779
 780	/* Send both scatterlists */
 781	rxdesc = dmaengine_prep_slave_sg(rxchan, &as->dma.sgrx, 1,
 
 782					 DMA_FROM_DEVICE,
 783					 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 784	if (!rxdesc)
 785		goto err_dma;
 786
 787	txdesc = dmaengine_prep_slave_sg(txchan, &as->dma.sgtx, 1,
 
 788					 DMA_TO_DEVICE,
 789					 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 790	if (!txdesc)
 791		goto err_dma;
 792
 793	dev_dbg(master->dev.parent,
 794		"  start dma xfer %p: len %u tx %p/%08llx rx %p/%08llx\n",
 795		xfer, xfer->len, xfer->tx_buf, (unsigned long long)xfer->tx_dma,
 796		xfer->rx_buf, (unsigned long long)xfer->rx_dma);
 797
 798	/* Enable relevant interrupts */
 799	spi_writel(as, IER, SPI_BIT(OVRES));
 800
 801	/* Put the callback on the RX transfer only, that should finish last */
 802	rxdesc->callback = dma_callback;
 803	rxdesc->callback_param = master;
 804
 805	/* Submit and fire RX and TX with TX last so we're ready to read! */
 806	cookie = rxdesc->tx_submit(rxdesc);
 807	if (dma_submit_error(cookie))
 808		goto err_dma;
 809	cookie = txdesc->tx_submit(txdesc);
 810	if (dma_submit_error(cookie))
 811		goto err_dma;
 812	rxchan->device->device_issue_pending(rxchan);
 813	txchan->device->device_issue_pending(txchan);
 814
 815	/* take back lock */
 816	atmel_spi_lock(as);
 817	return 0;
 818
 819err_dma:
 820	spi_writel(as, IDR, SPI_BIT(OVRES));
 821	atmel_spi_stop_dma(as);
 822err_exit:
 823	atmel_spi_lock(as);
 824	return -ENOMEM;
 825}
 826
 827static void atmel_spi_next_xfer_data(struct spi_master *master,
 828				struct spi_transfer *xfer,
 829				dma_addr_t *tx_dma,
 830				dma_addr_t *rx_dma,
 831				u32 *plen)
 832{
 833	struct atmel_spi	*as = spi_master_get_devdata(master);
 834	u32			len = *plen;
 835
 836	/* use scratch buffer only when rx or tx data is unspecified */
 837	if (xfer->rx_buf)
 838		*rx_dma = xfer->rx_dma + xfer->len - *plen;
 839	else {
 840		*rx_dma = as->buffer_dma;
 841		if (len > BUFFER_SIZE)
 842			len = BUFFER_SIZE;
 843	}
 844
 845	if (xfer->tx_buf)
 846		*tx_dma = xfer->tx_dma + xfer->len - *plen;
 847	else {
 848		*tx_dma = as->buffer_dma;
 849		if (len > BUFFER_SIZE)
 850			len = BUFFER_SIZE;
 851		memset(as->buffer, 0, len);
 852		dma_sync_single_for_device(&as->pdev->dev,
 853				as->buffer_dma, len, DMA_TO_DEVICE);
 854	}
 855
 856	*plen = len;
 857}
 858
 859static int atmel_spi_set_xfer_speed(struct atmel_spi *as,
 860				    struct spi_device *spi,
 861				    struct spi_transfer *xfer)
 862{
 863	u32			scbr, csr;
 864	unsigned long		bus_hz;
 865
 866	/* v1 chips start out at half the peripheral bus speed. */
 867	bus_hz = clk_get_rate(as->clk);
 868	if (!atmel_spi_is_v2(as))
 869		bus_hz /= 2;
 870
 871	/*
 872	 * Calculate the lowest divider that satisfies the
 873	 * constraint, assuming div32/fdiv/mbz == 0.
 874	 */
 875	scbr = DIV_ROUND_UP(bus_hz, xfer->speed_hz);
 876
 877	/*
 878	 * If the resulting divider doesn't fit into the
 879	 * register bitfield, we can't satisfy the constraint.
 880	 */
 881	if (scbr >= (1 << SPI_SCBR_SIZE)) {
 882		dev_err(&spi->dev,
 883			"setup: %d Hz too slow, scbr %u; min %ld Hz\n",
 884			xfer->speed_hz, scbr, bus_hz/255);
 885		return -EINVAL;
 886	}
 887	if (scbr == 0) {
 888		dev_err(&spi->dev,
 889			"setup: %d Hz too high, scbr %u; max %ld Hz\n",
 890			xfer->speed_hz, scbr, bus_hz);
 891		return -EINVAL;
 892	}
 893	csr = spi_readl(as, CSR0 + 4 * spi->chip_select);
 894	csr = SPI_BFINS(SCBR, scbr, csr);
 895	spi_writel(as, CSR0 + 4 * spi->chip_select, csr);
 896
 897	return 0;
 898}
 899
 900/*
 901 * Submit next transfer for PDC.
 902 * lock is held, spi irq is blocked
 903 */
 904static void atmel_spi_pdc_next_xfer(struct spi_master *master,
 905					struct spi_message *msg,
 906					struct spi_transfer *xfer)
 907{
 908	struct atmel_spi	*as = spi_master_get_devdata(master);
 909	u32			len;
 910	dma_addr_t		tx_dma, rx_dma;
 911
 912	spi_writel(as, PTCR, SPI_BIT(RXTDIS) | SPI_BIT(TXTDIS));
 913
 914	len = as->current_remaining_bytes;
 915	atmel_spi_next_xfer_data(master, xfer, &tx_dma, &rx_dma, &len);
 916	as->current_remaining_bytes -= len;
 917
 918	spi_writel(as, RPR, rx_dma);
 919	spi_writel(as, TPR, tx_dma);
 920
 921	if (msg->spi->bits_per_word > 8)
 922		len >>= 1;
 923	spi_writel(as, RCR, len);
 924	spi_writel(as, TCR, len);
 925
 926	dev_dbg(&msg->spi->dev,
 927		"  start xfer %p: len %u tx %p/%08llx rx %p/%08llx\n",
 928		xfer, xfer->len, xfer->tx_buf,
 929		(unsigned long long)xfer->tx_dma, xfer->rx_buf,
 930		(unsigned long long)xfer->rx_dma);
 931
 932	if (as->current_remaining_bytes) {
 933		len = as->current_remaining_bytes;
 934		atmel_spi_next_xfer_data(master, xfer, &tx_dma, &rx_dma, &len);
 935		as->current_remaining_bytes -= len;
 936
 937		spi_writel(as, RNPR, rx_dma);
 938		spi_writel(as, TNPR, tx_dma);
 939
 940		if (msg->spi->bits_per_word > 8)
 941			len >>= 1;
 942		spi_writel(as, RNCR, len);
 943		spi_writel(as, TNCR, len);
 944
 945		dev_dbg(&msg->spi->dev,
 946			"  next xfer %p: len %u tx %p/%08llx rx %p/%08llx\n",
 947			xfer, xfer->len, xfer->tx_buf,
 948			(unsigned long long)xfer->tx_dma, xfer->rx_buf,
 949			(unsigned long long)xfer->rx_dma);
 950	}
 951
 952	/* REVISIT: We're waiting for RXBUFF before we start the next
 953	 * transfer because we need to handle some difficult timing
 954	 * issues otherwise. If we wait for TXBUFE in one transfer and
 955	 * then starts waiting for RXBUFF in the next, it's difficult
 956	 * to tell the difference between the RXBUFF interrupt we're
 957	 * actually waiting for and the RXBUFF interrupt of the
 958	 * previous transfer.
 959	 *
 960	 * It should be doable, though. Just not now...
 961	 */
 962	spi_writel(as, IER, SPI_BIT(RXBUFF) | SPI_BIT(OVRES));
 963	spi_writel(as, PTCR, SPI_BIT(TXTEN) | SPI_BIT(RXTEN));
 964}
 965
 966/*
 967 * For DMA, tx_buf/tx_dma have the same relationship as rx_buf/rx_dma:
 968 *  - The buffer is either valid for CPU access, else NULL
 969 *  - If the buffer is valid, so is its DMA address
 970 *
 971 * This driver manages the dma address unless message->is_dma_mapped.
 972 */
 973static int
 974atmel_spi_dma_map_xfer(struct atmel_spi *as, struct spi_transfer *xfer)
 975{
 976	struct device	*dev = &as->pdev->dev;
 977
 978	xfer->tx_dma = xfer->rx_dma = INVALID_DMA_ADDRESS;
 979	if (xfer->tx_buf) {
 980		/* tx_buf is a const void* where we need a void * for the dma
 981		 * mapping */
 982		void *nonconst_tx = (void *)xfer->tx_buf;
 983
 984		xfer->tx_dma = dma_map_single(dev,
 985				nonconst_tx, xfer->len,
 986				DMA_TO_DEVICE);
 987		if (dma_mapping_error(dev, xfer->tx_dma))
 988			return -ENOMEM;
 989	}
 990	if (xfer->rx_buf) {
 991		xfer->rx_dma = dma_map_single(dev,
 992				xfer->rx_buf, xfer->len,
 993				DMA_FROM_DEVICE);
 994		if (dma_mapping_error(dev, xfer->rx_dma)) {
 995			if (xfer->tx_buf)
 996				dma_unmap_single(dev,
 997						xfer->tx_dma, xfer->len,
 998						DMA_TO_DEVICE);
 999			return -ENOMEM;
1000		}
1001	}
1002	return 0;
1003}
1004
1005static void atmel_spi_dma_unmap_xfer(struct spi_master *master,
1006				     struct spi_transfer *xfer)
1007{
1008	if (xfer->tx_dma != INVALID_DMA_ADDRESS)
1009		dma_unmap_single(master->dev.parent, xfer->tx_dma,
1010				 xfer->len, DMA_TO_DEVICE);
1011	if (xfer->rx_dma != INVALID_DMA_ADDRESS)
1012		dma_unmap_single(master->dev.parent, xfer->rx_dma,
1013				 xfer->len, DMA_FROM_DEVICE);
1014}
1015
1016static void atmel_spi_disable_pdc_transfer(struct atmel_spi *as)
1017{
1018	spi_writel(as, PTCR, SPI_BIT(RXTDIS) | SPI_BIT(TXTDIS));
1019}
1020
1021static void
1022atmel_spi_pump_single_data(struct atmel_spi *as, struct spi_transfer *xfer)
1023{
1024	u8		*rxp;
1025	u16		*rxp16;
1026	unsigned long	xfer_pos = xfer->len - as->current_remaining_bytes;
1027
1028	if (xfer->rx_buf) {
1029		if (xfer->bits_per_word > 8) {
1030			rxp16 = (u16 *)(((u8 *)xfer->rx_buf) + xfer_pos);
1031			*rxp16 = spi_readl(as, RDR);
1032		} else {
1033			rxp = ((u8 *)xfer->rx_buf) + xfer_pos;
1034			*rxp = spi_readl(as, RDR);
1035		}
1036	} else {
1037		spi_readl(as, RDR);
 
1038	}
1039	if (xfer->bits_per_word > 8) {
1040		if (as->current_remaining_bytes > 2)
1041			as->current_remaining_bytes -= 2;
1042		else
1043			as->current_remaining_bytes = 0;
1044	} else {
1045		as->current_remaining_bytes--;
1046	}
1047}
1048
1049static void
1050atmel_spi_pump_fifo_data(struct atmel_spi *as, struct spi_transfer *xfer)
1051{
1052	u32 fifolr = spi_readl(as, FLR);
1053	u32 num_bytes, num_data = SPI_BFEXT(RXFL, fifolr);
1054	u32 offset = xfer->len - as->current_remaining_bytes;
1055	u16 *words = (u16 *)((u8 *)xfer->rx_buf + offset);
1056	u8  *bytes = (u8  *)((u8 *)xfer->rx_buf + offset);
1057	u16 rd; /* RD field is the lowest 16 bits of RDR */
1058
1059	/* Update the number of remaining bytes to transfer */
1060	num_bytes = ((xfer->bits_per_word > 8) ?
1061		     (num_data << 1) :
1062		     num_data);
1063
1064	if (as->current_remaining_bytes > num_bytes)
1065		as->current_remaining_bytes -= num_bytes;
1066	else
1067		as->current_remaining_bytes = 0;
1068
1069	/* Handle odd number of bytes when data are more than 8bit width */
1070	if (xfer->bits_per_word > 8)
1071		as->current_remaining_bytes &= ~0x1;
1072
1073	/* Read data */
1074	while (num_data) {
1075		rd = spi_readl(as, RDR);
1076		if (xfer->rx_buf) {
1077			if (xfer->bits_per_word > 8)
1078				*words++ = rd;
1079			else
1080				*bytes++ = rd;
1081		}
1082		num_data--;
1083	}
1084}
1085
1086/* Called from IRQ
1087 *
1088 * Must update "current_remaining_bytes" to keep track of data
1089 * to transfer.
1090 */
1091static void
1092atmel_spi_pump_pio_data(struct atmel_spi *as, struct spi_transfer *xfer)
1093{
1094	if (as->fifo_size)
1095		atmel_spi_pump_fifo_data(as, xfer);
1096	else
1097		atmel_spi_pump_single_data(as, xfer);
1098}
1099
1100/* Interrupt
1101 *
1102 * No need for locking in this Interrupt handler: done_status is the
1103 * only information modified.
1104 */
1105static irqreturn_t
1106atmel_spi_pio_interrupt(int irq, void *dev_id)
1107{
1108	struct spi_master	*master = dev_id;
1109	struct atmel_spi	*as = spi_master_get_devdata(master);
1110	u32			status, pending, imr;
1111	struct spi_transfer	*xfer;
1112	int			ret = IRQ_NONE;
1113
1114	imr = spi_readl(as, IMR);
1115	status = spi_readl(as, SR);
1116	pending = status & imr;
1117
1118	if (pending & SPI_BIT(OVRES)) {
1119		ret = IRQ_HANDLED;
1120		spi_writel(as, IDR, SPI_BIT(OVRES));
1121		dev_warn(master->dev.parent, "overrun\n");
1122
1123		/*
1124		 * When we get an overrun, we disregard the current
1125		 * transfer. Data will not be copied back from any
1126		 * bounce buffer and msg->actual_len will not be
1127		 * updated with the last xfer.
1128		 *
1129		 * We will also not process any remaning transfers in
1130		 * the message.
1131		 */
1132		as->done_status = -EIO;
1133		smp_wmb();
1134
1135		/* Clear any overrun happening while cleaning up */
1136		spi_readl(as, SR);
1137
1138		complete(&as->xfer_completion);
1139
1140	} else if (pending & (SPI_BIT(RDRF) | SPI_BIT(RXFTHF))) {
1141		atmel_spi_lock(as);
1142
1143		if (as->current_remaining_bytes) {
1144			ret = IRQ_HANDLED;
1145			xfer = as->current_transfer;
1146			atmel_spi_pump_pio_data(as, xfer);
1147			if (!as->current_remaining_bytes)
1148				spi_writel(as, IDR, pending);
1149
1150			complete(&as->xfer_completion);
1151		}
1152
1153		atmel_spi_unlock(as);
1154	} else {
1155		WARN_ONCE(pending, "IRQ not handled, pending = %x\n", pending);
1156		ret = IRQ_HANDLED;
1157		spi_writel(as, IDR, pending);
1158	}
1159
1160	return ret;
1161}
1162
1163static irqreturn_t
1164atmel_spi_pdc_interrupt(int irq, void *dev_id)
1165{
1166	struct spi_master	*master = dev_id;
1167	struct atmel_spi	*as = spi_master_get_devdata(master);
1168	u32			status, pending, imr;
1169	int			ret = IRQ_NONE;
1170
1171	imr = spi_readl(as, IMR);
1172	status = spi_readl(as, SR);
1173	pending = status & imr;
1174
1175	if (pending & SPI_BIT(OVRES)) {
1176
1177		ret = IRQ_HANDLED;
1178
1179		spi_writel(as, IDR, (SPI_BIT(RXBUFF) | SPI_BIT(ENDRX)
1180				     | SPI_BIT(OVRES)));
1181
1182		/* Clear any overrun happening while cleaning up */
1183		spi_readl(as, SR);
1184
1185		as->done_status = -EIO;
1186
1187		complete(&as->xfer_completion);
1188
1189	} else if (pending & (SPI_BIT(RXBUFF) | SPI_BIT(ENDRX))) {
1190		ret = IRQ_HANDLED;
1191
1192		spi_writel(as, IDR, pending);
1193
1194		complete(&as->xfer_completion);
1195	}
1196
1197	return ret;
1198}
1199
1200static int atmel_spi_setup(struct spi_device *spi)
1201{
1202	struct atmel_spi	*as;
1203	struct atmel_spi_device	*asd;
1204	u32			csr;
1205	unsigned int		bits = spi->bits_per_word;
1206	unsigned int		npcs_pin;
1207	int			ret;
1208
1209	as = spi_master_get_devdata(spi->master);
1210
1211	/* see notes above re chipselect */
1212	if (!atmel_spi_is_v2(as)
1213			&& spi->chip_select == 0
1214			&& (spi->mode & SPI_CS_HIGH)) {
1215		dev_dbg(&spi->dev, "setup: can't be active-high\n");
1216		return -EINVAL;
1217	}
1218
1219	csr = SPI_BF(BITS, bits - 8);
1220	if (spi->mode & SPI_CPOL)
1221		csr |= SPI_BIT(CPOL);
1222	if (!(spi->mode & SPI_CPHA))
1223		csr |= SPI_BIT(NCPHA);
1224	if (!as->use_cs_gpios)
1225		csr |= SPI_BIT(CSAAT);
1226
1227	/* DLYBS is mostly irrelevant since we manage chipselect using GPIOs.
1228	 *
1229	 * DLYBCT would add delays between words, slowing down transfers.
1230	 * It could potentially be useful to cope with DMA bottlenecks, but
1231	 * in those cases it's probably best to just use a lower bitrate.
1232	 */
1233	csr |= SPI_BF(DLYBS, 0);
1234	csr |= SPI_BF(DLYBCT, 0);
1235
1236	/* chipselect must have been muxed as GPIO (e.g. in board setup) */
1237	npcs_pin = (unsigned long)spi->controller_data;
1238
1239	if (!as->use_cs_gpios)
1240		npcs_pin = spi->chip_select;
1241	else if (gpio_is_valid(spi->cs_gpio))
1242		npcs_pin = spi->cs_gpio;
1243
1244	asd = spi->controller_state;
1245	if (!asd) {
1246		asd = kzalloc(sizeof(struct atmel_spi_device), GFP_KERNEL);
1247		if (!asd)
1248			return -ENOMEM;
1249
1250		if (as->use_cs_gpios) {
1251			ret = gpio_request(npcs_pin, dev_name(&spi->dev));
1252			if (ret) {
1253				kfree(asd);
1254				return ret;
1255			}
1256
1257			gpio_direction_output(npcs_pin,
1258					      !(spi->mode & SPI_CS_HIGH));
1259		}
1260
1261		asd->npcs_pin = npcs_pin;
1262		spi->controller_state = asd;
1263	}
1264
1265	asd->csr = csr;
1266
1267	dev_dbg(&spi->dev,
1268		"setup: bpw %u mode 0x%x -> csr%d %08x\n",
1269		bits, spi->mode, spi->chip_select, csr);
1270
1271	if (!atmel_spi_is_v2(as))
1272		spi_writel(as, CSR0 + 4 * spi->chip_select, csr);
1273
1274	return 0;
1275}
1276
1277static int atmel_spi_one_transfer(struct spi_master *master,
1278					struct spi_message *msg,
1279					struct spi_transfer *xfer)
1280{
1281	struct atmel_spi	*as;
1282	struct spi_device	*spi = msg->spi;
1283	u8			bits;
1284	u32			len;
1285	struct atmel_spi_device	*asd;
1286	int			timeout;
1287	int			ret;
1288	unsigned long		dma_timeout;
1289
1290	as = spi_master_get_devdata(master);
1291
1292	if (!(xfer->tx_buf || xfer->rx_buf) && xfer->len) {
1293		dev_dbg(&spi->dev, "missing rx or tx buf\n");
1294		return -EINVAL;
1295	}
1296
1297	asd = spi->controller_state;
1298	bits = (asd->csr >> 4) & 0xf;
1299	if (bits != xfer->bits_per_word - 8) {
1300		dev_dbg(&spi->dev,
1301			"you can't yet change bits_per_word in transfers\n");
1302		return -ENOPROTOOPT;
1303	}
1304
1305	/*
1306	 * DMA map early, for performance (empties dcache ASAP) and
1307	 * better fault reporting.
1308	 */
1309	if ((!msg->is_dma_mapped)
1310		&& (atmel_spi_use_dma(as, xfer)	|| as->use_pdc)) {
1311		if (atmel_spi_dma_map_xfer(as, xfer) < 0)
1312			return -ENOMEM;
1313	}
1314
1315	atmel_spi_set_xfer_speed(as, msg->spi, xfer);
1316
1317	as->done_status = 0;
1318	as->current_transfer = xfer;
1319	as->current_remaining_bytes = xfer->len;
1320	while (as->current_remaining_bytes) {
1321		reinit_completion(&as->xfer_completion);
1322
1323		if (as->use_pdc) {
1324			atmel_spi_pdc_next_xfer(master, msg, xfer);
1325		} else if (atmel_spi_use_dma(as, xfer)) {
1326			len = as->current_remaining_bytes;
1327			ret = atmel_spi_next_xfer_dma_submit(master,
1328								xfer, &len);
1329			if (ret) {
1330				dev_err(&spi->dev,
1331					"unable to use DMA, fallback to PIO\n");
1332				atmel_spi_next_xfer_pio(master, xfer);
1333			} else {
1334				as->current_remaining_bytes -= len;
1335				if (as->current_remaining_bytes < 0)
1336					as->current_remaining_bytes = 0;
1337			}
1338		} else {
1339			atmel_spi_next_xfer_pio(master, xfer);
1340		}
1341
1342		/* interrupts are disabled, so free the lock for schedule */
1343		atmel_spi_unlock(as);
1344		dma_timeout = wait_for_completion_timeout(&as->xfer_completion,
1345							  SPI_DMA_TIMEOUT);
1346		atmel_spi_lock(as);
1347		if (WARN_ON(dma_timeout == 0)) {
1348			dev_err(&spi->dev, "spi transfer timeout\n");
1349			as->done_status = -EIO;
1350		}
1351
1352		if (as->done_status)
1353			break;
1354	}
1355
1356	if (as->done_status) {
1357		if (as->use_pdc) {
1358			dev_warn(master->dev.parent,
1359				"overrun (%u/%u remaining)\n",
1360				spi_readl(as, TCR), spi_readl(as, RCR));
1361
1362			/*
1363			 * Clean up DMA registers and make sure the data
1364			 * registers are empty.
1365			 */
1366			spi_writel(as, RNCR, 0);
1367			spi_writel(as, TNCR, 0);
1368			spi_writel(as, RCR, 0);
1369			spi_writel(as, TCR, 0);
1370			for (timeout = 1000; timeout; timeout--)
1371				if (spi_readl(as, SR) & SPI_BIT(TXEMPTY))
1372					break;
1373			if (!timeout)
1374				dev_warn(master->dev.parent,
1375					 "timeout waiting for TXEMPTY");
1376			while (spi_readl(as, SR) & SPI_BIT(RDRF))
1377				spi_readl(as, RDR);
1378
1379			/* Clear any overrun happening while cleaning up */
1380			spi_readl(as, SR);
1381
1382		} else if (atmel_spi_use_dma(as, xfer)) {
1383			atmel_spi_stop_dma(as);
1384		}
1385
1386		if (!msg->is_dma_mapped
1387			&& (atmel_spi_use_dma(as, xfer) || as->use_pdc))
1388			atmel_spi_dma_unmap_xfer(master, xfer);
1389
1390		return 0;
1391
1392	} else {
1393		/* only update length if no error */
1394		msg->actual_length += xfer->len;
1395	}
1396
1397	if (!msg->is_dma_mapped
1398		&& (atmel_spi_use_dma(as, xfer) || as->use_pdc))
1399		atmel_spi_dma_unmap_xfer(master, xfer);
1400
1401	if (xfer->delay_usecs)
1402		udelay(xfer->delay_usecs);
1403
1404	if (xfer->cs_change) {
1405		if (list_is_last(&xfer->transfer_list,
1406				 &msg->transfers)) {
1407			as->keep_cs = true;
1408		} else {
1409			as->cs_active = !as->cs_active;
1410			if (as->cs_active)
1411				cs_activate(as, msg->spi);
1412			else
1413				cs_deactivate(as, msg->spi);
1414		}
1415	}
1416
1417	return 0;
1418}
1419
1420static int atmel_spi_transfer_one_message(struct spi_master *master,
1421						struct spi_message *msg)
1422{
1423	struct atmel_spi *as;
1424	struct spi_transfer *xfer;
1425	struct spi_device *spi = msg->spi;
1426	int ret = 0;
1427
1428	as = spi_master_get_devdata(master);
1429
1430	dev_dbg(&spi->dev, "new message %p submitted for %s\n",
1431					msg, dev_name(&spi->dev));
1432
1433	atmel_spi_lock(as);
1434	cs_activate(as, spi);
1435
1436	as->cs_active = true;
1437	as->keep_cs = false;
1438
1439	msg->status = 0;
1440	msg->actual_length = 0;
1441
1442	list_for_each_entry(xfer, &msg->transfers, transfer_list) {
1443		ret = atmel_spi_one_transfer(master, msg, xfer);
1444		if (ret)
1445			goto msg_done;
1446	}
1447
1448	if (as->use_pdc)
1449		atmel_spi_disable_pdc_transfer(as);
1450
1451	list_for_each_entry(xfer, &msg->transfers, transfer_list) {
1452		dev_dbg(&spi->dev,
1453			"  xfer %p: len %u tx %p/%pad rx %p/%pad\n",
1454			xfer, xfer->len,
1455			xfer->tx_buf, &xfer->tx_dma,
1456			xfer->rx_buf, &xfer->rx_dma);
1457	}
1458
1459msg_done:
1460	if (!as->keep_cs)
1461		cs_deactivate(as, msg->spi);
1462
1463	atmel_spi_unlock(as);
1464
1465	msg->status = as->done_status;
1466	spi_finalize_current_message(spi->master);
1467
1468	return ret;
1469}
1470
1471static void atmel_spi_cleanup(struct spi_device *spi)
1472{
1473	struct atmel_spi_device	*asd = spi->controller_state;
1474	unsigned		gpio = (unsigned long) spi->controller_data;
1475
1476	if (!asd)
1477		return;
1478
1479	spi->controller_state = NULL;
1480	gpio_free(gpio);
1481	kfree(asd);
1482}
1483
1484static inline unsigned int atmel_get_version(struct atmel_spi *as)
1485{
1486	return spi_readl(as, VERSION) & 0x00000fff;
1487}
1488
1489static void atmel_get_caps(struct atmel_spi *as)
1490{
1491	unsigned int version;
1492
1493	version = atmel_get_version(as);
1494	dev_info(&as->pdev->dev, "version: 0x%x\n", version);
1495
1496	as->caps.is_spi2 = version > 0x121;
1497	as->caps.has_wdrbt = version >= 0x210;
1498	as->caps.has_dma_support = version >= 0x212;
1499}
1500
1501/*-------------------------------------------------------------------------*/
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1502
1503static int atmel_spi_probe(struct platform_device *pdev)
1504{
1505	struct resource		*regs;
1506	int			irq;
1507	struct clk		*clk;
1508	int			ret;
1509	struct spi_master	*master;
1510	struct atmel_spi	*as;
1511
1512	/* Select default pin state */
1513	pinctrl_pm_select_default_state(&pdev->dev);
1514
1515	regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1516	if (!regs)
1517		return -ENXIO;
1518
1519	irq = platform_get_irq(pdev, 0);
1520	if (irq < 0)
1521		return irq;
1522
1523	clk = devm_clk_get(&pdev->dev, "spi_clk");
1524	if (IS_ERR(clk))
1525		return PTR_ERR(clk);
1526
1527	/* setup spi core then atmel-specific driver state */
1528	ret = -ENOMEM;
1529	master = spi_alloc_master(&pdev->dev, sizeof(*as));
1530	if (!master)
1531		goto out_free;
1532
1533	/* the spi->mode bits understood by this driver: */
1534	master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;
1535	master->bits_per_word_mask = SPI_BPW_RANGE_MASK(8, 16);
1536	master->dev.of_node = pdev->dev.of_node;
1537	master->bus_num = pdev->id;
1538	master->num_chipselect = master->dev.of_node ? 0 : 4;
1539	master->setup = atmel_spi_setup;
 
1540	master->transfer_one_message = atmel_spi_transfer_one_message;
1541	master->cleanup = atmel_spi_cleanup;
1542	master->auto_runtime_pm = true;
 
 
1543	platform_set_drvdata(pdev, master);
1544
1545	as = spi_master_get_devdata(master);
1546
1547	/*
1548	 * Scratch buffer is used for throwaway rx and tx data.
1549	 * It's coherent to minimize dcache pollution.
1550	 */
1551	as->buffer = dma_alloc_coherent(&pdev->dev, BUFFER_SIZE,
1552					&as->buffer_dma, GFP_KERNEL);
1553	if (!as->buffer)
1554		goto out_free;
1555
1556	spin_lock_init(&as->lock);
1557
1558	as->pdev = pdev;
1559	as->regs = devm_ioremap_resource(&pdev->dev, regs);
1560	if (IS_ERR(as->regs)) {
1561		ret = PTR_ERR(as->regs);
1562		goto out_free_buffer;
1563	}
1564	as->phybase = regs->start;
1565	as->irq = irq;
1566	as->clk = clk;
1567
1568	init_completion(&as->xfer_completion);
1569
1570	atmel_get_caps(as);
1571
1572	as->use_cs_gpios = true;
1573	if (atmel_spi_is_v2(as) &&
1574	    pdev->dev.of_node &&
1575	    !of_get_property(pdev->dev.of_node, "cs-gpios", NULL)) {
1576		as->use_cs_gpios = false;
1577		master->num_chipselect = 4;
1578	}
1579
 
 
 
 
1580	as->use_dma = false;
1581	as->use_pdc = false;
1582	if (as->caps.has_dma_support) {
1583		ret = atmel_spi_configure_dma(as);
1584		if (ret == 0)
1585			as->use_dma = true;
1586		else if (ret == -EPROBE_DEFER)
1587			return ret;
 
1588	} else {
1589		as->use_pdc = true;
1590	}
1591
1592	if (as->caps.has_dma_support && !as->use_dma)
1593		dev_info(&pdev->dev, "Atmel SPI Controller using PIO only\n");
1594
1595	if (as->use_pdc) {
1596		ret = devm_request_irq(&pdev->dev, irq, atmel_spi_pdc_interrupt,
1597					0, dev_name(&pdev->dev), master);
1598	} else {
1599		ret = devm_request_irq(&pdev->dev, irq, atmel_spi_pio_interrupt,
1600					0, dev_name(&pdev->dev), master);
1601	}
1602	if (ret)
1603		goto out_unmap_regs;
1604
1605	/* Initialize the hardware */
1606	ret = clk_prepare_enable(clk);
1607	if (ret)
1608		goto out_free_irq;
 
 
 
1609	spi_writel(as, CR, SPI_BIT(SWRST));
1610	spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */
1611	if (as->caps.has_wdrbt) {
1612		spi_writel(as, MR, SPI_BIT(WDRBT) | SPI_BIT(MODFDIS)
1613				| SPI_BIT(MSTR));
1614	} else {
1615		spi_writel(as, MR, SPI_BIT(MSTR) | SPI_BIT(MODFDIS));
1616	}
1617
1618	if (as->use_pdc)
1619		spi_writel(as, PTCR, SPI_BIT(RXTDIS) | SPI_BIT(TXTDIS));
1620	spi_writel(as, CR, SPI_BIT(SPIEN));
1621
1622	as->fifo_size = 0;
1623	if (!of_property_read_u32(pdev->dev.of_node, "atmel,fifo-size",
1624				  &as->fifo_size)) {
1625		dev_info(&pdev->dev, "Using FIFO (%u data)\n", as->fifo_size);
1626		spi_writel(as, CR, SPI_BIT(FIFOEN));
1627	}
1628
1629	/* go! */
1630	dev_info(&pdev->dev, "Atmel SPI Controller at 0x%08lx (irq %d)\n",
1631			(unsigned long)regs->start, irq);
1632
1633	pm_runtime_set_autosuspend_delay(&pdev->dev, AUTOSUSPEND_TIMEOUT);
1634	pm_runtime_use_autosuspend(&pdev->dev);
1635	pm_runtime_set_active(&pdev->dev);
1636	pm_runtime_enable(&pdev->dev);
1637
1638	ret = devm_spi_register_master(&pdev->dev, master);
1639	if (ret)
1640		goto out_free_dma;
1641
 
 
 
 
1642	return 0;
1643
1644out_free_dma:
1645	pm_runtime_disable(&pdev->dev);
1646	pm_runtime_set_suspended(&pdev->dev);
1647
1648	if (as->use_dma)
1649		atmel_spi_release_dma(as);
1650
1651	spi_writel(as, CR, SPI_BIT(SWRST));
1652	spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */
1653	clk_disable_unprepare(clk);
1654out_free_irq:
1655out_unmap_regs:
1656out_free_buffer:
1657	dma_free_coherent(&pdev->dev, BUFFER_SIZE, as->buffer,
1658			as->buffer_dma);
1659out_free:
1660	spi_master_put(master);
1661	return ret;
1662}
1663
1664static int atmel_spi_remove(struct platform_device *pdev)
1665{
1666	struct spi_master	*master = platform_get_drvdata(pdev);
1667	struct atmel_spi	*as = spi_master_get_devdata(master);
1668
1669	pm_runtime_get_sync(&pdev->dev);
1670
1671	/* reset the hardware and block queue progress */
1672	spin_lock_irq(&as->lock);
1673	if (as->use_dma) {
1674		atmel_spi_stop_dma(as);
1675		atmel_spi_release_dma(as);
1676	}
1677
1678	spi_writel(as, CR, SPI_BIT(SWRST));
1679	spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */
1680	spi_readl(as, SR);
1681	spin_unlock_irq(&as->lock);
1682
1683	dma_free_coherent(&pdev->dev, BUFFER_SIZE, as->buffer,
1684			as->buffer_dma);
1685
1686	clk_disable_unprepare(as->clk);
1687
1688	pm_runtime_put_noidle(&pdev->dev);
1689	pm_runtime_disable(&pdev->dev);
1690
1691	return 0;
1692}
1693
1694#ifdef CONFIG_PM
1695static int atmel_spi_runtime_suspend(struct device *dev)
1696{
1697	struct spi_master *master = dev_get_drvdata(dev);
1698	struct atmel_spi *as = spi_master_get_devdata(master);
1699
1700	clk_disable_unprepare(as->clk);
1701	pinctrl_pm_select_sleep_state(dev);
1702
1703	return 0;
1704}
1705
1706static int atmel_spi_runtime_resume(struct device *dev)
1707{
1708	struct spi_master *master = dev_get_drvdata(dev);
1709	struct atmel_spi *as = spi_master_get_devdata(master);
1710
1711	pinctrl_pm_select_default_state(dev);
1712
1713	return clk_prepare_enable(as->clk);
1714}
1715
1716#ifdef CONFIG_PM_SLEEP
1717static int atmel_spi_suspend(struct device *dev)
1718{
1719	struct spi_master *master = dev_get_drvdata(dev);
1720	int ret;
1721
1722	/* Stop the queue running */
1723	ret = spi_master_suspend(master);
1724	if (ret) {
1725		dev_warn(dev, "cannot suspend master\n");
1726		return ret;
1727	}
1728
1729	if (!pm_runtime_suspended(dev))
1730		atmel_spi_runtime_suspend(dev);
1731
1732	return 0;
1733}
1734
1735static int atmel_spi_resume(struct device *dev)
1736{
1737	struct spi_master *master = dev_get_drvdata(dev);
1738	int ret;
1739
1740	if (!pm_runtime_suspended(dev)) {
1741		ret = atmel_spi_runtime_resume(dev);
1742		if (ret)
1743			return ret;
1744	}
1745
1746	/* Start the queue running */
1747	ret = spi_master_resume(master);
1748	if (ret)
1749		dev_err(dev, "problem starting queue (%d)\n", ret);
1750
1751	return ret;
1752}
1753#endif
1754
1755static const struct dev_pm_ops atmel_spi_pm_ops = {
1756	SET_SYSTEM_SLEEP_PM_OPS(atmel_spi_suspend, atmel_spi_resume)
1757	SET_RUNTIME_PM_OPS(atmel_spi_runtime_suspend,
1758			   atmel_spi_runtime_resume, NULL)
1759};
1760#define ATMEL_SPI_PM_OPS	(&atmel_spi_pm_ops)
1761#else
1762#define ATMEL_SPI_PM_OPS	NULL
1763#endif
1764
1765#if defined(CONFIG_OF)
1766static const struct of_device_id atmel_spi_dt_ids[] = {
1767	{ .compatible = "atmel,at91rm9200-spi" },
1768	{ /* sentinel */ }
1769};
1770
1771MODULE_DEVICE_TABLE(of, atmel_spi_dt_ids);
1772#endif
1773
1774static struct platform_driver atmel_spi_driver = {
1775	.driver		= {
1776		.name	= "atmel_spi",
1777		.pm	= ATMEL_SPI_PM_OPS,
1778		.of_match_table	= of_match_ptr(atmel_spi_dt_ids),
1779	},
1780	.probe		= atmel_spi_probe,
1781	.remove		= atmel_spi_remove,
1782};
1783module_platform_driver(atmel_spi_driver);
1784
1785MODULE_DESCRIPTION("Atmel AT32/AT91 SPI Controller driver");
1786MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1787MODULE_LICENSE("GPL");
1788MODULE_ALIAS("platform:atmel_spi");