1/*
   2 * Driver for Atmel AT32 and AT91 SPI Controllers
   3 *
   4 * Copyright (C) 2006 Atmel Corporation
   5 *
   6 * This program is free software; you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License version 2 as
   8 * published by the Free Software Foundation.
   9 */
  10
  11#include <linux/kernel.h>
  12#include <linux/clk.h>
  13#include <linux/module.h>
  14#include <linux/platform_device.h>
  15#include <linux/delay.h>
  16#include <linux/dma-mapping.h>
  17#include <linux/dmaengine.h>
  18#include <linux/err.h>
  19#include <linux/interrupt.h>
  20#include <linux/spi/spi.h>
  21#include <linux/slab.h>
  22#include <linux/platform_data/dma-atmel.h>
  23#include <linux/of.h>
  24
  25#include <linux/io.h>
  26#include <linux/gpio.h>
  27#include <linux/of_gpio.h>
  28#include <linux/pinctrl/consumer.h>
  29#include <linux/pm_runtime.h>
  30
  31/* SPI register offsets */
  32#define SPI_CR					0x0000
  33#define SPI_MR					0x0004
  34#define SPI_RDR					0x0008
  35#define SPI_TDR					0x000c
  36#define SPI_SR					0x0010
  37#define SPI_IER					0x0014
  38#define SPI_IDR					0x0018
  39#define SPI_IMR					0x001c
  40#define SPI_CSR0				0x0030
  41#define SPI_CSR1				0x0034
  42#define SPI_CSR2				0x0038
  43#define SPI_CSR3				0x003c
  44#define SPI_FMR					0x0040
  45#define SPI_FLR					0x0044
  46#define SPI_VERSION				0x00fc
  47#define SPI_RPR					0x0100
  48#define SPI_RCR					0x0104
  49#define SPI_TPR					0x0108
  50#define SPI_TCR					0x010c
  51#define SPI_RNPR				0x0110
  52#define SPI_RNCR				0x0114
  53#define SPI_TNPR				0x0118
  54#define SPI_TNCR				0x011c
  55#define SPI_PTCR				0x0120
  56#define SPI_PTSR				0x0124
  57
  58/* Bitfields in CR */
  59#define SPI_SPIEN_OFFSET			0
  60#define SPI_SPIEN_SIZE				1
  61#define SPI_SPIDIS_OFFSET			1
  62#define SPI_SPIDIS_SIZE				1
  63#define SPI_SWRST_OFFSET			7
  64#define SPI_SWRST_SIZE				1
  65#define SPI_LASTXFER_OFFSET			24
  66#define SPI_LASTXFER_SIZE			1
  67#define SPI_TXFCLR_OFFSET			16
  68#define SPI_TXFCLR_SIZE				1
  69#define SPI_RXFCLR_OFFSET			17
  70#define SPI_RXFCLR_SIZE				1
  71#define SPI_FIFOEN_OFFSET			30
  72#define SPI_FIFOEN_SIZE				1
  73#define SPI_FIFODIS_OFFSET			31
  74#define SPI_FIFODIS_SIZE			1
  75
  76/* Bitfields in MR */
  77#define SPI_MSTR_OFFSET				0
  78#define SPI_MSTR_SIZE				1
  79#define SPI_PS_OFFSET				1
  80#define SPI_PS_SIZE				1
  81#define SPI_PCSDEC_OFFSET			2
  82#define SPI_PCSDEC_SIZE				1
  83#define SPI_FDIV_OFFSET				3
  84#define SPI_FDIV_SIZE				1
  85#define SPI_MODFDIS_OFFSET			4
  86#define SPI_MODFDIS_SIZE			1
  87#define SPI_WDRBT_OFFSET			5
  88#define SPI_WDRBT_SIZE				1
  89#define SPI_LLB_OFFSET				7
  90#define SPI_LLB_SIZE				1
  91#define SPI_PCS_OFFSET				16
  92#define SPI_PCS_SIZE				4
  93#define SPI_DLYBCS_OFFSET			24
  94#define SPI_DLYBCS_SIZE				8
  95
  96/* Bitfields in RDR */
  97#define SPI_RD_OFFSET				0
  98#define SPI_RD_SIZE				16
  99
 100/* Bitfields in TDR */
 101#define SPI_TD_OFFSET				0
 102#define SPI_TD_SIZE				16
 103
 104/* Bitfields in SR */
 105#define SPI_RDRF_OFFSET				0
 106#define SPI_RDRF_SIZE				1
 107#define SPI_TDRE_OFFSET				1
 108#define SPI_TDRE_SIZE				1
 109#define SPI_MODF_OFFSET				2
 110#define SPI_MODF_SIZE				1
 111#define SPI_OVRES_OFFSET			3
 112#define SPI_OVRES_SIZE				1
 113#define SPI_ENDRX_OFFSET			4
 114#define SPI_ENDRX_SIZE				1
 115#define SPI_ENDTX_OFFSET			5
 116#define SPI_ENDTX_SIZE				1
 117#define SPI_RXBUFF_OFFSET			6
 118#define SPI_RXBUFF_SIZE				1
 119#define SPI_TXBUFE_OFFSET			7
 120#define SPI_TXBUFE_SIZE				1
 121#define SPI_NSSR_OFFSET				8
 122#define SPI_NSSR_SIZE				1
 123#define SPI_TXEMPTY_OFFSET			9
 124#define SPI_TXEMPTY_SIZE			1
 125#define SPI_SPIENS_OFFSET			16
 126#define SPI_SPIENS_SIZE				1
 127#define SPI_TXFEF_OFFSET			24
 128#define SPI_TXFEF_SIZE				1
 129#define SPI_TXFFF_OFFSET			25
 130#define SPI_TXFFF_SIZE				1
 131#define SPI_TXFTHF_OFFSET			26
 132#define SPI_TXFTHF_SIZE				1
 133#define SPI_RXFEF_OFFSET			27
 134#define SPI_RXFEF_SIZE				1
 135#define SPI_RXFFF_OFFSET			28
 136#define SPI_RXFFF_SIZE				1
 137#define SPI_RXFTHF_OFFSET			29
 138#define SPI_RXFTHF_SIZE				1
 139#define SPI_TXFPTEF_OFFSET			30
 140#define SPI_TXFPTEF_SIZE			1
 141#define SPI_RXFPTEF_OFFSET			31
 142#define SPI_RXFPTEF_SIZE			1
 143
 144/* Bitfields in CSR0 */
 145#define SPI_CPOL_OFFSET				0
 146#define SPI_CPOL_SIZE				1
 147#define SPI_NCPHA_OFFSET			1
 148#define SPI_NCPHA_SIZE				1
 149#define SPI_CSAAT_OFFSET			3
 150#define SPI_CSAAT_SIZE				1
 151#define SPI_BITS_OFFSET				4
 152#define SPI_BITS_SIZE				4
 153#define SPI_SCBR_OFFSET				8
 154#define SPI_SCBR_SIZE				8
 155#define SPI_DLYBS_OFFSET			16
 156#define SPI_DLYBS_SIZE				8
 157#define SPI_DLYBCT_OFFSET			24
 158#define SPI_DLYBCT_SIZE				8
 159
 160/* Bitfields in RCR */
 161#define SPI_RXCTR_OFFSET			0
 162#define SPI_RXCTR_SIZE				16
 163
 164/* Bitfields in TCR */
 165#define SPI_TXCTR_OFFSET			0
 166#define SPI_TXCTR_SIZE				16
 167
 168/* Bitfields in RNCR */
 169#define SPI_RXNCR_OFFSET			0
 170#define SPI_RXNCR_SIZE				16
 171
 172/* Bitfields in TNCR */
 173#define SPI_TXNCR_OFFSET			0
 174#define SPI_TXNCR_SIZE				16
 175
 176/* Bitfields in PTCR */
 177#define SPI_RXTEN_OFFSET			0
 178#define SPI_RXTEN_SIZE				1
 179#define SPI_RXTDIS_OFFSET			1
 180#define SPI_RXTDIS_SIZE				1
 181#define SPI_TXTEN_OFFSET			8
 182#define SPI_TXTEN_SIZE				1
 183#define SPI_TXTDIS_OFFSET			9
 184#define SPI_TXTDIS_SIZE				1
 185
 186/* Bitfields in FMR */
 187#define SPI_TXRDYM_OFFSET			0
 188#define SPI_TXRDYM_SIZE				2
 189#define SPI_RXRDYM_OFFSET			4
 190#define SPI_RXRDYM_SIZE				2
 191#define SPI_TXFTHRES_OFFSET			16
 192#define SPI_TXFTHRES_SIZE			6
 193#define SPI_RXFTHRES_OFFSET			24
 194#define SPI_RXFTHRES_SIZE			6
 195
 196/* Bitfields in FLR */
 197#define SPI_TXFL_OFFSET				0
 198#define SPI_TXFL_SIZE				6
 199#define SPI_RXFL_OFFSET				16
 200#define SPI_RXFL_SIZE				6
 201
 202/* Constants for BITS */
 203#define SPI_BITS_8_BPT				0
 204#define SPI_BITS_9_BPT				1
 205#define SPI_BITS_10_BPT				2
 206#define SPI_BITS_11_BPT				3
 207#define SPI_BITS_12_BPT				4
 208#define SPI_BITS_13_BPT				5
 209#define SPI_BITS_14_BPT				6
 210#define SPI_BITS_15_BPT				7
 211#define SPI_BITS_16_BPT				8
 212#define SPI_ONE_DATA				0
 213#define SPI_TWO_DATA				1
 214#define SPI_FOUR_DATA				2
 215
 216/* Bit manipulation macros */
 217#define SPI_BIT(name) \
 218	(1 << SPI_##name##_OFFSET)
 219#define SPI_BF(name, value) \
 220	(((value) & ((1 << SPI_##name##_SIZE) - 1)) << SPI_##name##_OFFSET)
 221#define SPI_BFEXT(name, value) \
 222	(((value) >> SPI_##name##_OFFSET) & ((1 << SPI_##name##_SIZE) - 1))
 223#define SPI_BFINS(name, value, old) \
 224	(((old) & ~(((1 << SPI_##name##_SIZE) - 1) << SPI_##name##_OFFSET)) \
 225	  | SPI_BF(name, value))
 226
 227/* Register access macros */
 228#ifdef CONFIG_AVR32
 229#define spi_readl(port, reg) \
 230	__raw_readl((port)->regs + SPI_##reg)
 231#define spi_writel(port, reg, value) \
 232	__raw_writel((value), (port)->regs + SPI_##reg)
 233
 234#define spi_readw(port, reg) \
 235	__raw_readw((port)->regs + SPI_##reg)
 236#define spi_writew(port, reg, value) \
 237	__raw_writew((value), (port)->regs + SPI_##reg)
 238
 239#define spi_readb(port, reg) \
 240	__raw_readb((port)->regs + SPI_##reg)
 241#define spi_writeb(port, reg, value) \
 242	__raw_writeb((value), (port)->regs + SPI_##reg)
 243#else
 244#define spi_readl(port, reg) \
 245	readl_relaxed((port)->regs + SPI_##reg)
 246#define spi_writel(port, reg, value) \
 247	writel_relaxed((value), (port)->regs + SPI_##reg)
 248
 249#define spi_readw(port, reg) \
 250	readw_relaxed((port)->regs + SPI_##reg)
 251#define spi_writew(port, reg, value) \
 252	writew_relaxed((value), (port)->regs + SPI_##reg)
 253
 254#define spi_readb(port, reg) \
 255	readb_relaxed((port)->regs + SPI_##reg)
 256#define spi_writeb(port, reg, value) \
 257	writeb_relaxed((value), (port)->regs + SPI_##reg)
 258#endif
 259/* use PIO for small transfers, avoiding DMA setup/teardown overhead and
 260 * cache operations; better heuristics consider wordsize and bitrate.
 261 */
 262#define DMA_MIN_BYTES	16
 263
 264#define SPI_DMA_TIMEOUT		(msecs_to_jiffies(1000))
 265
 266#define AUTOSUSPEND_TIMEOUT	2000
 267
 268struct atmel_spi_caps {
 269	bool	is_spi2;
 270	bool	has_wdrbt;
 271	bool	has_dma_support;
 272	bool	has_pdc_support;
 273};
 274
 275/*
 276 * The core SPI transfer engine just talks to a register bank to set up
 277 * DMA transfers; transfer queue progress is driven by IRQs.  The clock
 278 * framework provides the base clock, subdivided for each spi_device.
 279 */
 280struct atmel_spi {
 281	spinlock_t		lock;
 282	unsigned long		flags;
 283
 284	phys_addr_t		phybase;
 285	void __iomem		*regs;
 286	int			irq;
 287	struct clk		*clk;
 288	struct platform_device	*pdev;
 289	unsigned long		spi_clk;
 290
 291	struct spi_transfer	*current_transfer;
 292	int			current_remaining_bytes;
 293	int			done_status;
 294	dma_addr_t		dma_addr_rx_bbuf;
 295	dma_addr_t		dma_addr_tx_bbuf;
 296	void			*addr_rx_bbuf;
 297	void			*addr_tx_bbuf;
 298
 299	struct completion	xfer_completion;
 300
 301	struct atmel_spi_caps	caps;
 302
 303	bool			use_dma;
 304	bool			use_pdc;
 305	bool			use_cs_gpios;
 306
 307	bool			keep_cs;
 308	bool			cs_active;
 309
 310	u32			fifo_size;
 311};
 312
 313/* Controller-specific per-slave state */
 314struct atmel_spi_device {
 315	unsigned int		npcs_pin;
 316	u32			csr;
 317};
 318
 319#define SPI_MAX_DMA_XFER	65535 /* true for both PDC and DMA */
 320#define INVALID_DMA_ADDRESS	0xffffffff
 321
 322/*
 323 * Version 2 of the SPI controller has
 324 *  - CR.LASTXFER
 325 *  - SPI_MR.DIV32 may become FDIV or must-be-zero (here: always zero)
 326 *  - SPI_SR.TXEMPTY, SPI_SR.NSSR (and corresponding irqs)
 327 *  - SPI_CSRx.CSAAT
 328 *  - SPI_CSRx.SBCR allows faster clocking
 329 */
 330static bool atmel_spi_is_v2(struct atmel_spi *as)
 331{
 332	return as->caps.is_spi2;
 333}
 334
 335/*
 336 * Earlier SPI controllers (e.g. on at91rm9200) have a design bug whereby
 337 * they assume that spi slave device state will not change on deselect, so
 338 * that automagic deselection is OK.  ("NPCSx rises if no data is to be
 339 * transmitted")  Not so!  Workaround uses nCSx pins as GPIOs; or newer
 340 * controllers have CSAAT and friends.
 341 *
 342 * Since the CSAAT functionality is a bit weird on newer controllers as
 343 * well, we use GPIO to control nCSx pins on all controllers, updating
 344 * MR.PCS to avoid confusing the controller.  Using GPIOs also lets us
 345 * support active-high chipselects despite the controller's belief that
 346 * only active-low devices/systems exists.
 347 *
 348 * However, at91rm9200 has a second erratum whereby nCS0 doesn't work
 349 * right when driven with GPIO.  ("Mode Fault does not allow more than one
 350 * Master on Chip Select 0.")  No workaround exists for that ... so for
 351 * nCS0 on that chip, we (a) don't use the GPIO, (b) can't support CS_HIGH,
 352 * and (c) will trigger that first erratum in some cases.
 353 */
 354
 355static void cs_activate(struct atmel_spi *as, struct spi_device *spi)
 356{
 357	struct atmel_spi_device *asd = spi->controller_state;
 358	unsigned active = spi->mode & SPI_CS_HIGH;
 359	u32 mr;
 360
 361	if (atmel_spi_is_v2(as)) {
 362		spi_writel(as, CSR0 + 4 * spi->chip_select, asd->csr);
 363		/* For the low SPI version, there is a issue that PDC transfer
 364		 * on CS1,2,3 needs SPI_CSR0.BITS config as SPI_CSR1,2,3.BITS
 365		 */
 366		spi_writel(as, CSR0, asd->csr);
 367		if (as->caps.has_wdrbt) {
 368			spi_writel(as, MR,
 369					SPI_BF(PCS, ~(0x01 << spi->chip_select))
 370					| SPI_BIT(WDRBT)
 371					| SPI_BIT(MODFDIS)
 372					| SPI_BIT(MSTR));
 373		} else {
 374			spi_writel(as, MR,
 375					SPI_BF(PCS, ~(0x01 << spi->chip_select))
 376					| SPI_BIT(MODFDIS)
 377					| SPI_BIT(MSTR));
 378		}
 379
 380		mr = spi_readl(as, MR);
 381		if (as->use_cs_gpios)
 382			gpio_set_value(asd->npcs_pin, active);
 383	} else {
 384		u32 cpol = (spi->mode & SPI_CPOL) ? SPI_BIT(CPOL) : 0;
 385		int i;
 386		u32 csr;
 387
 388		/* Make sure clock polarity is correct */
 389		for (i = 0; i < spi->master->num_chipselect; i++) {
 390			csr = spi_readl(as, CSR0 + 4 * i);
 391			if ((csr ^ cpol) & SPI_BIT(CPOL))
 392				spi_writel(as, CSR0 + 4 * i,
 393						csr ^ SPI_BIT(CPOL));
 394		}
 395
 396		mr = spi_readl(as, MR);
 397		mr = SPI_BFINS(PCS, ~(1 << spi->chip_select), mr);
 398		if (as->use_cs_gpios && spi->chip_select != 0)
 399			gpio_set_value(asd->npcs_pin, active);
 400		spi_writel(as, MR, mr);
 401	}
 402
 403	dev_dbg(&spi->dev, "activate %u%s, mr %08x\n",
 404			asd->npcs_pin, active ? " (high)" : "",
 405			mr);
 406}
 407
 408static void cs_deactivate(struct atmel_spi *as, struct spi_device *spi)
 409{
 410	struct atmel_spi_device *asd = spi->controller_state;
 411	unsigned active = spi->mode & SPI_CS_HIGH;
 412	u32 mr;
 413
 414	/* only deactivate *this* device; sometimes transfers to
 415	 * another device may be active when this routine is called.
 416	 */
 417	mr = spi_readl(as, MR);
 418	if (~SPI_BFEXT(PCS, mr) & (1 << spi->chip_select)) {
 419		mr = SPI_BFINS(PCS, 0xf, mr);
 420		spi_writel(as, MR, mr);
 421	}
 422
 423	dev_dbg(&spi->dev, "DEactivate %u%s, mr %08x\n",
 424			asd->npcs_pin, active ? " (low)" : "",
 425			mr);
 426
 427	if (!as->use_cs_gpios)
 428		spi_writel(as, CR, SPI_BIT(LASTXFER));
 429	else if (atmel_spi_is_v2(as) || spi->chip_select != 0)
 430		gpio_set_value(asd->npcs_pin, !active);
 431}
 432
 433static void atmel_spi_lock(struct atmel_spi *as) __acquires(&as->lock)
 434{
 435	spin_lock_irqsave(&as->lock, as->flags);
 436}
 437
 438static void atmel_spi_unlock(struct atmel_spi *as) __releases(&as->lock)
 439{
 440	spin_unlock_irqrestore(&as->lock, as->flags);
 441}
 442
 443static inline bool atmel_spi_is_vmalloc_xfer(struct spi_transfer *xfer)
 444{
 445	return is_vmalloc_addr(xfer->tx_buf) || is_vmalloc_addr(xfer->rx_buf);
 446}
 447
 448static inline bool atmel_spi_use_dma(struct atmel_spi *as,
 449				struct spi_transfer *xfer)
 450{
 451	return as->use_dma && xfer->len >= DMA_MIN_BYTES;
 452}
 453
 454static bool atmel_spi_can_dma(struct spi_master *master,
 455			      struct spi_device *spi,
 456			      struct spi_transfer *xfer)
 457{
 458	struct atmel_spi *as = spi_master_get_devdata(master);
 459
 460	if (IS_ENABLED(CONFIG_SOC_SAM_V4_V5))
 461		return atmel_spi_use_dma(as, xfer) &&
 462			!atmel_spi_is_vmalloc_xfer(xfer);
 463	else
 464		return atmel_spi_use_dma(as, xfer);
 465
 466}
 467
 468static int atmel_spi_dma_slave_config(struct atmel_spi *as,
 469				struct dma_slave_config *slave_config,
 470				u8 bits_per_word)
 471{
 472	struct spi_master *master = platform_get_drvdata(as->pdev);
 473	int err = 0;
 474
 475	if (bits_per_word > 8) {
 476		slave_config->dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
 477		slave_config->src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
 478	} else {
 479		slave_config->dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
 480		slave_config->src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
 481	}
 482
 483	slave_config->dst_addr = (dma_addr_t)as->phybase + SPI_TDR;
 484	slave_config->src_addr = (dma_addr_t)as->phybase + SPI_RDR;
 485	slave_config->src_maxburst = 1;
 486	slave_config->dst_maxburst = 1;
 487	slave_config->device_fc = false;
 488
 489	/*
 490	 * This driver uses fixed peripheral select mode (PS bit set to '0' in
 491	 * the Mode Register).
 492	 * So according to the datasheet, when FIFOs are available (and
 493	 * enabled), the Transmit FIFO operates in Multiple Data Mode.
 494	 * In this mode, up to 2 data, not 4, can be written into the Transmit
 495	 * Data Register in a single access.
 496	 * However, the first data has to be written into the lowest 16 bits and
 497	 * the second data into the highest 16 bits of the Transmit
 498	 * Data Register. For 8bit data (the most frequent case), it would
 499	 * require to rework tx_buf so each data would actualy fit 16 bits.
 500	 * So we'd rather write only one data at the time. Hence the transmit
 501	 * path works the same whether FIFOs are available (and enabled) or not.
 502	 */
 503	slave_config->direction = DMA_MEM_TO_DEV;
 504	if (dmaengine_slave_config(master->dma_tx, slave_config)) {
 505		dev_err(&as->pdev->dev,
 506			"failed to configure tx dma channel\n");
 507		err = -EINVAL;
 508	}
 509
 510	/*
 511	 * This driver configures the spi controller for master mode (MSTR bit
 512	 * set to '1' in the Mode Register).
 513	 * So according to the datasheet, when FIFOs are available (and
 514	 * enabled), the Receive FIFO operates in Single Data Mode.
 515	 * So the receive path works the same whether FIFOs are available (and
 516	 * enabled) or not.
 517	 */
 518	slave_config->direction = DMA_DEV_TO_MEM;
 519	if (dmaengine_slave_config(master->dma_rx, slave_config)) {
 520		dev_err(&as->pdev->dev,
 521			"failed to configure rx dma channel\n");
 522		err = -EINVAL;
 523	}
 524
 525	return err;
 526}
 527
 528static int atmel_spi_configure_dma(struct spi_master *master,
 529				   struct atmel_spi *as)
 530{
 531	struct dma_slave_config	slave_config;
 532	struct device *dev = &as->pdev->dev;
 533	int err;
 534
 535	dma_cap_mask_t mask;
 536	dma_cap_zero(mask);
 537	dma_cap_set(DMA_SLAVE, mask);
 538
 539	master->dma_tx = dma_request_slave_channel_reason(dev, "tx");
 540	if (IS_ERR(master->dma_tx)) {
 541		err = PTR_ERR(master->dma_tx);
 542		if (err == -EPROBE_DEFER) {
 543			dev_warn(dev, "no DMA channel available at the moment\n");
 544			goto error_clear;
 545		}
 546		dev_err(dev,
 547			"DMA TX channel not available, SPI unable to use DMA\n");
 548		err = -EBUSY;
 549		goto error_clear;
 550	}
 551
 552	/*
 553	 * No reason to check EPROBE_DEFER here since we have already requested
 554	 * tx channel. If it fails here, it's for another reason.
 555	 */
 556	master->dma_rx = dma_request_slave_channel(dev, "rx");
 557
 558	if (!master->dma_rx) {
 559		dev_err(dev,
 560			"DMA RX channel not available, SPI unable to use DMA\n");
 561		err = -EBUSY;
 562		goto error;
 563	}
 564
 565	err = atmel_spi_dma_slave_config(as, &slave_config, 8);
 566	if (err)
 567		goto error;
 568
 569	dev_info(&as->pdev->dev,
 570			"Using %s (tx) and %s (rx) for DMA transfers\n",
 571			dma_chan_name(master->dma_tx),
 572			dma_chan_name(master->dma_rx));
 573
 574	return 0;
 575error:
 576	if (master->dma_rx)
 577		dma_release_channel(master->dma_rx);
 578	if (!IS_ERR(master->dma_tx))
 579		dma_release_channel(master->dma_tx);
 580error_clear:
 581	master->dma_tx = master->dma_rx = NULL;
 582	return err;
 583}
 584
 585static void atmel_spi_stop_dma(struct spi_master *master)
 586{
 587	if (master->dma_rx)
 588		dmaengine_terminate_all(master->dma_rx);
 589	if (master->dma_tx)
 590		dmaengine_terminate_all(master->dma_tx);
 591}
 592
 593static void atmel_spi_release_dma(struct spi_master *master)
 594{
 595	if (master->dma_rx) {
 596		dma_release_channel(master->dma_rx);
 597		master->dma_rx = NULL;
 598	}
 599	if (master->dma_tx) {
 600		dma_release_channel(master->dma_tx);
 601		master->dma_tx = NULL;
 602	}
 603}
 604
 605/* This function is called by the DMA driver from tasklet context */
 606static void dma_callback(void *data)
 607{
 608	struct spi_master	*master = data;
 609	struct atmel_spi	*as = spi_master_get_devdata(master);
 610
 611	if (is_vmalloc_addr(as->current_transfer->rx_buf) &&
 612	    IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) {
 613		memcpy(as->current_transfer->rx_buf, as->addr_rx_bbuf,
 614		       as->current_transfer->len);
 615	}
 616	complete(&as->xfer_completion);
 617}
 618
 619/*
 620 * Next transfer using PIO without FIFO.
 621 */
 622static void atmel_spi_next_xfer_single(struct spi_master *master,
 623				       struct spi_transfer *xfer)
 624{
 625	struct atmel_spi	*as = spi_master_get_devdata(master);
 626	unsigned long xfer_pos = xfer->len - as->current_remaining_bytes;
 627
 628	dev_vdbg(master->dev.parent, "atmel_spi_next_xfer_pio\n");
 629
 630	/* Make sure data is not remaining in RDR */
 631	spi_readl(as, RDR);
 632	while (spi_readl(as, SR) & SPI_BIT(RDRF)) {
 633		spi_readl(as, RDR);
 634		cpu_relax();
 635	}
 636
 637	if (xfer->bits_per_word > 8)
 638		spi_writel(as, TDR, *(u16 *)(xfer->tx_buf + xfer_pos));
 639	else
 640		spi_writel(as, TDR, *(u8 *)(xfer->tx_buf + xfer_pos));
 641
 642	dev_dbg(master->dev.parent,
 643		"  start pio xfer %p: len %u tx %p rx %p bitpw %d\n",
 644		xfer, xfer->len, xfer->tx_buf, xfer->rx_buf,
 645		xfer->bits_per_word);
 646
 647	/* Enable relevant interrupts */
 648	spi_writel(as, IER, SPI_BIT(RDRF) | SPI_BIT(OVRES));
 649}
 650
 651/*
 652 * Next transfer using PIO with FIFO.
 653 */
 654static void atmel_spi_next_xfer_fifo(struct spi_master *master,
 655				     struct spi_transfer *xfer)
 656{
 657	struct atmel_spi *as = spi_master_get_devdata(master);
 658	u32 current_remaining_data, num_data;
 659	u32 offset = xfer->len - as->current_remaining_bytes;
 660	const u16 *words = (const u16 *)((u8 *)xfer->tx_buf + offset);
 661	const u8  *bytes = (const u8  *)((u8 *)xfer->tx_buf + offset);
 662	u16 td0, td1;
 663	u32 fifomr;
 664
 665	dev_vdbg(master->dev.parent, "atmel_spi_next_xfer_fifo\n");
 666
 667	/* Compute the number of data to transfer in the current iteration */
 668	current_remaining_data = ((xfer->bits_per_word > 8) ?
 669				  ((u32)as->current_remaining_bytes >> 1) :
 670				  (u32)as->current_remaining_bytes);
 671	num_data = min(current_remaining_data, as->fifo_size);
 672
 673	/* Flush RX and TX FIFOs */
 674	spi_writel(as, CR, SPI_BIT(RXFCLR) | SPI_BIT(TXFCLR));
 675	while (spi_readl(as, FLR))
 676		cpu_relax();
 677
 678	/* Set RX FIFO Threshold to the number of data to transfer */
 679	fifomr = spi_readl(as, FMR);
 680	spi_writel(as, FMR, SPI_BFINS(RXFTHRES, num_data, fifomr));
 681
 682	/* Clear FIFO flags in the Status Register, especially RXFTHF */
 683	(void)spi_readl(as, SR);
 684
 685	/* Fill TX FIFO */
 686	while (num_data >= 2) {
 687		if (xfer->bits_per_word > 8) {
 688			td0 = *words++;
 689			td1 = *words++;
 690		} else {
 691			td0 = *bytes++;
 692			td1 = *bytes++;
 693		}
 694
 695		spi_writel(as, TDR, (td1 << 16) | td0);
 696		num_data -= 2;
 697	}
 698
 699	if (num_data) {
 700		if (xfer->bits_per_word > 8)
 701			td0 = *words++;
 702		else
 703			td0 = *bytes++;
 704
 705		spi_writew(as, TDR, td0);
 706		num_data--;
 707	}
 708
 709	dev_dbg(master->dev.parent,
 710		"  start fifo xfer %p: len %u tx %p rx %p bitpw %d\n",
 711		xfer, xfer->len, xfer->tx_buf, xfer->rx_buf,
 712		xfer->bits_per_word);
 713
 714	/*
 715	 * Enable RX FIFO Threshold Flag interrupt to be notified about
 716	 * transfer completion.
 717	 */
 718	spi_writel(as, IER, SPI_BIT(RXFTHF) | SPI_BIT(OVRES));
 719}
 720
 721/*
 722 * Next transfer using PIO.
 723 */
 724static void atmel_spi_next_xfer_pio(struct spi_master *master,
 725				    struct spi_transfer *xfer)
 726{
 727	struct atmel_spi *as = spi_master_get_devdata(master);
 728
 729	if (as->fifo_size)
 730		atmel_spi_next_xfer_fifo(master, xfer);
 731	else
 732		atmel_spi_next_xfer_single(master, xfer);
 733}
 734
 735/*
 736 * Submit next transfer for DMA.
 737 */
 738static int atmel_spi_next_xfer_dma_submit(struct spi_master *master,
 739				struct spi_transfer *xfer,
 740				u32 *plen)
 741{
 742	struct atmel_spi	*as = spi_master_get_devdata(master);
 743	struct dma_chan		*rxchan = master->dma_rx;
 744	struct dma_chan		*txchan = master->dma_tx;
 745	struct dma_async_tx_descriptor *rxdesc;
 746	struct dma_async_tx_descriptor *txdesc;
 747	struct dma_slave_config	slave_config;
 748	dma_cookie_t		cookie;
 749
 750	dev_vdbg(master->dev.parent, "atmel_spi_next_xfer_dma_submit\n");
 751
 752	/* Check that the channels are available */
 753	if (!rxchan || !txchan)
 754		return -ENODEV;
 755
 756	/* release lock for DMA operations */
 757	atmel_spi_unlock(as);
 758
 759	*plen = xfer->len;
 760
 761	if (atmel_spi_dma_slave_config(as, &slave_config,
 762				       xfer->bits_per_word))
 763		goto err_exit;
 764
 765	/* Send both scatterlists */
 766	if (atmel_spi_is_vmalloc_xfer(xfer) &&
 767	    IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) {
 768		rxdesc = dmaengine_prep_slave_single(rxchan,
 769						     as->dma_addr_rx_bbuf,
 770						     xfer->len,
 771						     DMA_DEV_TO_MEM,
 772						     DMA_PREP_INTERRUPT |
 773						     DMA_CTRL_ACK);
 774	} else {
 775		rxdesc = dmaengine_prep_slave_sg(rxchan,
 776						 xfer->rx_sg.sgl,
 777						 xfer->rx_sg.nents,
 778						 DMA_DEV_TO_MEM,
 779						 DMA_PREP_INTERRUPT |
 780						 DMA_CTRL_ACK);
 781	}
 782	if (!rxdesc)
 783		goto err_dma;
 784
 785	if (atmel_spi_is_vmalloc_xfer(xfer) &&
 786	    IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) {
 787		memcpy(as->addr_tx_bbuf, xfer->tx_buf, xfer->len);
 788		txdesc = dmaengine_prep_slave_single(txchan,
 789						     as->dma_addr_tx_bbuf,
 790						     xfer->len, DMA_MEM_TO_DEV,
 791						     DMA_PREP_INTERRUPT |
 792						     DMA_CTRL_ACK);
 793	} else {
 794		txdesc = dmaengine_prep_slave_sg(txchan,
 795						 xfer->tx_sg.sgl,
 796						 xfer->tx_sg.nents,
 797						 DMA_MEM_TO_DEV,
 798						 DMA_PREP_INTERRUPT |
 799						 DMA_CTRL_ACK);
 800	}
 801	if (!txdesc)
 802		goto err_dma;
 803
 804	dev_dbg(master->dev.parent,
 805		"  start dma xfer %p: len %u tx %p/%08llx rx %p/%08llx\n",
 806		xfer, xfer->len, xfer->tx_buf, (unsigned long long)xfer->tx_dma,
 807		xfer->rx_buf, (unsigned long long)xfer->rx_dma);
 808
 809	/* Enable relevant interrupts */
 810	spi_writel(as, IER, SPI_BIT(OVRES));
 811
 812	/* Put the callback on the RX transfer only, that should finish last */
 813	rxdesc->callback = dma_callback;
 814	rxdesc->callback_param = master;
 815
 816	/* Submit and fire RX and TX with TX last so we're ready to read! */
 817	cookie = rxdesc->tx_submit(rxdesc);
 818	if (dma_submit_error(cookie))
 819		goto err_dma;
 820	cookie = txdesc->tx_submit(txdesc);
 821	if (dma_submit_error(cookie))
 822		goto err_dma;
 823	rxchan->device->device_issue_pending(rxchan);
 824	txchan->device->device_issue_pending(txchan);
 825
 826	/* take back lock */
 827	atmel_spi_lock(as);
 828	return 0;
 829
 830err_dma:
 831	spi_writel(as, IDR, SPI_BIT(OVRES));
 832	atmel_spi_stop_dma(master);
 833err_exit:
 834	atmel_spi_lock(as);
 835	return -ENOMEM;
 836}
 837
 838static void atmel_spi_next_xfer_data(struct spi_master *master,
 839				struct spi_transfer *xfer,
 840				dma_addr_t *tx_dma,
 841				dma_addr_t *rx_dma,
 842				u32 *plen)
 843{
 844	*rx_dma = xfer->rx_dma + xfer->len - *plen;
 845	*tx_dma = xfer->tx_dma + xfer->len - *plen;
 846	if (*plen > master->max_dma_len)
 847		*plen = master->max_dma_len;
 848}
 849
 850static int atmel_spi_set_xfer_speed(struct atmel_spi *as,
 851				    struct spi_device *spi,
 852				    struct spi_transfer *xfer)
 853{
 854	u32			scbr, csr;
 855	unsigned long		bus_hz;
 856
 857	/* v1 chips start out at half the peripheral bus speed. */
 858	bus_hz = as->spi_clk;
 859	if (!atmel_spi_is_v2(as))
 860		bus_hz /= 2;
 861
 862	/*
 863	 * Calculate the lowest divider that satisfies the
 864	 * constraint, assuming div32/fdiv/mbz == 0.
 865	 */
 866	scbr = DIV_ROUND_UP(bus_hz, xfer->speed_hz);
 867
 868	/*
 869	 * If the resulting divider doesn't fit into the
 870	 * register bitfield, we can't satisfy the constraint.
 871	 */
 872	if (scbr >= (1 << SPI_SCBR_SIZE)) {
 873		dev_err(&spi->dev,
 874			"setup: %d Hz too slow, scbr %u; min %ld Hz\n",
 875			xfer->speed_hz, scbr, bus_hz/255);
 876		return -EINVAL;
 877	}
 878	if (scbr == 0) {
 879		dev_err(&spi->dev,
 880			"setup: %d Hz too high, scbr %u; max %ld Hz\n",
 881			xfer->speed_hz, scbr, bus_hz);
 882		return -EINVAL;
 883	}
 884	csr = spi_readl(as, CSR0 + 4 * spi->chip_select);
 885	csr = SPI_BFINS(SCBR, scbr, csr);
 886	spi_writel(as, CSR0 + 4 * spi->chip_select, csr);
 887
 888	return 0;
 889}
 890
 891/*
 892 * Submit next transfer for PDC.
 893 * lock is held, spi irq is blocked
 894 */
 895static void atmel_spi_pdc_next_xfer(struct spi_master *master,
 896					struct spi_message *msg,
 897					struct spi_transfer *xfer)
 898{
 899	struct atmel_spi	*as = spi_master_get_devdata(master);
 900	u32			len;
 901	dma_addr_t		tx_dma, rx_dma;
 902
 903	spi_writel(as, PTCR, SPI_BIT(RXTDIS) | SPI_BIT(TXTDIS));
 904
 905	len = as->current_remaining_bytes;
 906	atmel_spi_next_xfer_data(master, xfer, &tx_dma, &rx_dma, &len);
 907	as->current_remaining_bytes -= len;
 908
 909	spi_writel(as, RPR, rx_dma);
 910	spi_writel(as, TPR, tx_dma);
 911
 912	if (msg->spi->bits_per_word > 8)
 913		len >>= 1;
 914	spi_writel(as, RCR, len);
 915	spi_writel(as, TCR, len);
 916
 917	dev_dbg(&msg->spi->dev,
 918		"  start xfer %p: len %u tx %p/%08llx rx %p/%08llx\n",
 919		xfer, xfer->len, xfer->tx_buf,
 920		(unsigned long long)xfer->tx_dma, xfer->rx_buf,
 921		(unsigned long long)xfer->rx_dma);
 922
 923	if (as->current_remaining_bytes) {
 924		len = as->current_remaining_bytes;
 925		atmel_spi_next_xfer_data(master, xfer, &tx_dma, &rx_dma, &len);
 926		as->current_remaining_bytes -= len;
 927
 928		spi_writel(as, RNPR, rx_dma);
 929		spi_writel(as, TNPR, tx_dma);
 930
 931		if (msg->spi->bits_per_word > 8)
 932			len >>= 1;
 933		spi_writel(as, RNCR, len);
 934		spi_writel(as, TNCR, len);
 935
 936		dev_dbg(&msg->spi->dev,
 937			"  next xfer %p: len %u tx %p/%08llx rx %p/%08llx\n",
 938			xfer, xfer->len, xfer->tx_buf,
 939			(unsigned long long)xfer->tx_dma, xfer->rx_buf,
 940			(unsigned long long)xfer->rx_dma);
 941	}
 942
 943	/* REVISIT: We're waiting for RXBUFF before we start the next
 944	 * transfer because we need to handle some difficult timing
 945	 * issues otherwise. If we wait for TXBUFE in one transfer and
 946	 * then starts waiting for RXBUFF in the next, it's difficult
 947	 * to tell the difference between the RXBUFF interrupt we're
 948	 * actually waiting for and the RXBUFF interrupt of the
 949	 * previous transfer.
 950	 *
 951	 * It should be doable, though. Just not now...
 952	 */
 953	spi_writel(as, IER, SPI_BIT(RXBUFF) | SPI_BIT(OVRES));
 954	spi_writel(as, PTCR, SPI_BIT(TXTEN) | SPI_BIT(RXTEN));
 955}
 956
 957/*
 958 * For DMA, tx_buf/tx_dma have the same relationship as rx_buf/rx_dma:
 959 *  - The buffer is either valid for CPU access, else NULL
 960 *  - If the buffer is valid, so is its DMA address
 961 *
 962 * This driver manages the dma address unless message->is_dma_mapped.
 963 */
 964static int
 965atmel_spi_dma_map_xfer(struct atmel_spi *as, struct spi_transfer *xfer)
 966{
 967	struct device	*dev = &as->pdev->dev;
 968
 969	xfer->tx_dma = xfer->rx_dma = INVALID_DMA_ADDRESS;
 970	if (xfer->tx_buf) {
 971		/* tx_buf is a const void* where we need a void * for the dma
 972		 * mapping */
 973		void *nonconst_tx = (void *)xfer->tx_buf;
 974
 975		xfer->tx_dma = dma_map_single(dev,
 976				nonconst_tx, xfer->len,
 977				DMA_TO_DEVICE);
 978		if (dma_mapping_error(dev, xfer->tx_dma))
 979			return -ENOMEM;
 980	}
 981	if (xfer->rx_buf) {
 982		xfer->rx_dma = dma_map_single(dev,
 983				xfer->rx_buf, xfer->len,
 984				DMA_FROM_DEVICE);
 985		if (dma_mapping_error(dev, xfer->rx_dma)) {
 986			if (xfer->tx_buf)
 987				dma_unmap_single(dev,
 988						xfer->tx_dma, xfer->len,
 989						DMA_TO_DEVICE);
 990			return -ENOMEM;
 991		}
 992	}
 993	return 0;
 994}
 995
 996static void atmel_spi_dma_unmap_xfer(struct spi_master *master,
 997				     struct spi_transfer *xfer)
 998{
 999	if (xfer->tx_dma != INVALID_DMA_ADDRESS)
1000		dma_unmap_single(master->dev.parent, xfer->tx_dma,
1001				 xfer->len, DMA_TO_DEVICE);
1002	if (xfer->rx_dma != INVALID_DMA_ADDRESS)
1003		dma_unmap_single(master->dev.parent, xfer->rx_dma,
1004				 xfer->len, DMA_FROM_DEVICE);
1005}
1006
1007static void atmel_spi_disable_pdc_transfer(struct atmel_spi *as)
1008{
1009	spi_writel(as, PTCR, SPI_BIT(RXTDIS) | SPI_BIT(TXTDIS));
1010}
1011
1012static void
1013atmel_spi_pump_single_data(struct atmel_spi *as, struct spi_transfer *xfer)
1014{
1015	u8		*rxp;
1016	u16		*rxp16;
1017	unsigned long	xfer_pos = xfer->len - as->current_remaining_bytes;
1018
1019	if (xfer->bits_per_word > 8) {
1020		rxp16 = (u16 *)(((u8 *)xfer->rx_buf) + xfer_pos);
1021		*rxp16 = spi_readl(as, RDR);
1022	} else {
1023		rxp = ((u8 *)xfer->rx_buf) + xfer_pos;
1024		*rxp = spi_readl(as, RDR);
1025	}
1026	if (xfer->bits_per_word > 8) {
1027		if (as->current_remaining_bytes > 2)
1028			as->current_remaining_bytes -= 2;
1029		else
1030			as->current_remaining_bytes = 0;
1031	} else {
1032		as->current_remaining_bytes--;
1033	}
1034}
1035
1036static void
1037atmel_spi_pump_fifo_data(struct atmel_spi *as, struct spi_transfer *xfer)
1038{
1039	u32 fifolr = spi_readl(as, FLR);
1040	u32 num_bytes, num_data = SPI_BFEXT(RXFL, fifolr);
1041	u32 offset = xfer->len - as->current_remaining_bytes;
1042	u16 *words = (u16 *)((u8 *)xfer->rx_buf + offset);
1043	u8  *bytes = (u8  *)((u8 *)xfer->rx_buf + offset);
1044	u16 rd; /* RD field is the lowest 16 bits of RDR */
1045
1046	/* Update the number of remaining bytes to transfer */
1047	num_bytes = ((xfer->bits_per_word > 8) ?
1048		     (num_data << 1) :
1049		     num_data);
1050
1051	if (as->current_remaining_bytes > num_bytes)
1052		as->current_remaining_bytes -= num_bytes;
1053	else
1054		as->current_remaining_bytes = 0;
1055
1056	/* Handle odd number of bytes when data are more than 8bit width */
1057	if (xfer->bits_per_word > 8)
1058		as->current_remaining_bytes &= ~0x1;
1059
1060	/* Read data */
1061	while (num_data) {
1062		rd = spi_readl(as, RDR);
1063		if (xfer->bits_per_word > 8)
1064			*words++ = rd;
1065		else
1066			*bytes++ = rd;
1067		num_data--;
1068	}
1069}
1070
1071/* Called from IRQ
1072 *
1073 * Must update "current_remaining_bytes" to keep track of data
1074 * to transfer.
1075 */
1076static void
1077atmel_spi_pump_pio_data(struct atmel_spi *as, struct spi_transfer *xfer)
1078{
1079	if (as->fifo_size)
1080		atmel_spi_pump_fifo_data(as, xfer);
1081	else
1082		atmel_spi_pump_single_data(as, xfer);
1083}
1084
1085/* Interrupt
1086 *
1087 * No need for locking in this Interrupt handler: done_status is the
1088 * only information modified.
1089 */
1090static irqreturn_t
1091atmel_spi_pio_interrupt(int irq, void *dev_id)
1092{
1093	struct spi_master	*master = dev_id;
1094	struct atmel_spi	*as = spi_master_get_devdata(master);
1095	u32			status, pending, imr;
1096	struct spi_transfer	*xfer;
1097	int			ret = IRQ_NONE;
1098
1099	imr = spi_readl(as, IMR);
1100	status = spi_readl(as, SR);
1101	pending = status & imr;
1102
1103	if (pending & SPI_BIT(OVRES)) {
1104		ret = IRQ_HANDLED;
1105		spi_writel(as, IDR, SPI_BIT(OVRES));
1106		dev_warn(master->dev.parent, "overrun\n");
1107
1108		/*
1109		 * When we get an overrun, we disregard the current
1110		 * transfer. Data will not be copied back from any
1111		 * bounce buffer and msg->actual_len will not be
1112		 * updated with the last xfer.
1113		 *
1114		 * We will also not process any remaning transfers in
1115		 * the message.
1116		 */
1117		as->done_status = -EIO;
1118		smp_wmb();
1119
1120		/* Clear any overrun happening while cleaning up */
1121		spi_readl(as, SR);
1122
1123		complete(&as->xfer_completion);
1124
1125	} else if (pending & (SPI_BIT(RDRF) | SPI_BIT(RXFTHF))) {
1126		atmel_spi_lock(as);
1127
1128		if (as->current_remaining_bytes) {
1129			ret = IRQ_HANDLED;
1130			xfer = as->current_transfer;
1131			atmel_spi_pump_pio_data(as, xfer);
1132			if (!as->current_remaining_bytes)
1133				spi_writel(as, IDR, pending);
1134
1135			complete(&as->xfer_completion);
1136		}
1137
1138		atmel_spi_unlock(as);
1139	} else {
1140		WARN_ONCE(pending, "IRQ not handled, pending = %x\n", pending);
1141		ret = IRQ_HANDLED;
1142		spi_writel(as, IDR, pending);
1143	}
1144
1145	return ret;
1146}
1147
1148static irqreturn_t
1149atmel_spi_pdc_interrupt(int irq, void *dev_id)
1150{
1151	struct spi_master	*master = dev_id;
1152	struct atmel_spi	*as = spi_master_get_devdata(master);
1153	u32			status, pending, imr;
1154	int			ret = IRQ_NONE;
1155
1156	imr = spi_readl(as, IMR);
1157	status = spi_readl(as, SR);
1158	pending = status & imr;
1159
1160	if (pending & SPI_BIT(OVRES)) {
1161
1162		ret = IRQ_HANDLED;
1163
1164		spi_writel(as, IDR, (SPI_BIT(RXBUFF) | SPI_BIT(ENDRX)
1165				     | SPI_BIT(OVRES)));
1166
1167		/* Clear any overrun happening while cleaning up */
1168		spi_readl(as, SR);
1169
1170		as->done_status = -EIO;
1171
1172		complete(&as->xfer_completion);
1173
1174	} else if (pending & (SPI_BIT(RXBUFF) | SPI_BIT(ENDRX))) {
1175		ret = IRQ_HANDLED;
1176
1177		spi_writel(as, IDR, pending);
1178
1179		complete(&as->xfer_completion);
1180	}
1181
1182	return ret;
1183}
1184
1185static int atmel_spi_setup(struct spi_device *spi)
1186{
1187	struct atmel_spi	*as;
1188	struct atmel_spi_device	*asd;
1189	u32			csr;
1190	unsigned int		bits = spi->bits_per_word;
1191	unsigned int		npcs_pin;
1192
1193	as = spi_master_get_devdata(spi->master);
1194
1195	/* see notes above re chipselect */
1196	if (!atmel_spi_is_v2(as)
1197			&& spi->chip_select == 0
1198			&& (spi->mode & SPI_CS_HIGH)) {
1199		dev_dbg(&spi->dev, "setup: can't be active-high\n");
1200		return -EINVAL;
1201	}
1202
1203	csr = SPI_BF(BITS, bits - 8);
1204	if (spi->mode & SPI_CPOL)
1205		csr |= SPI_BIT(CPOL);
1206	if (!(spi->mode & SPI_CPHA))
1207		csr |= SPI_BIT(NCPHA);
1208	if (!as->use_cs_gpios)
1209		csr |= SPI_BIT(CSAAT);
1210
1211	/* DLYBS is mostly irrelevant since we manage chipselect using GPIOs.
1212	 *
1213	 * DLYBCT would add delays between words, slowing down transfers.
1214	 * It could potentially be useful to cope with DMA bottlenecks, but
1215	 * in those cases it's probably best to just use a lower bitrate.
1216	 */
1217	csr |= SPI_BF(DLYBS, 0);
1218	csr |= SPI_BF(DLYBCT, 0);
1219
1220	/* chipselect must have been muxed as GPIO (e.g. in board setup) */
1221	npcs_pin = (unsigned long)spi->controller_data;
1222
1223	if (!as->use_cs_gpios)
1224		npcs_pin = spi->chip_select;
1225	else if (gpio_is_valid(spi->cs_gpio))
1226		npcs_pin = spi->cs_gpio;
1227
1228	asd = spi->controller_state;
1229	if (!asd) {
1230		asd = kzalloc(sizeof(struct atmel_spi_device), GFP_KERNEL);
1231		if (!asd)
1232			return -ENOMEM;
1233
1234		if (as->use_cs_gpios)
1235			gpio_direction_output(npcs_pin,
1236					      !(spi->mode & SPI_CS_HIGH));
1237
1238		asd->npcs_pin = npcs_pin;
1239		spi->controller_state = asd;
1240	}
1241
1242	asd->csr = csr;
1243
1244	dev_dbg(&spi->dev,
1245		"setup: bpw %u mode 0x%x -> csr%d %08x\n",
1246		bits, spi->mode, spi->chip_select, csr);
1247
1248	if (!atmel_spi_is_v2(as))
1249		spi_writel(as, CSR0 + 4 * spi->chip_select, csr);
1250
1251	return 0;
1252}
1253
1254static int atmel_spi_one_transfer(struct spi_master *master,
1255					struct spi_message *msg,
1256					struct spi_transfer *xfer)
1257{
1258	struct atmel_spi	*as;
1259	struct spi_device	*spi = msg->spi;
1260	u8			bits;
1261	u32			len;
1262	struct atmel_spi_device	*asd;
1263	int			timeout;
1264	int			ret;
1265	unsigned long		dma_timeout;
1266
1267	as = spi_master_get_devdata(master);
1268
1269	if (!(xfer->tx_buf || xfer->rx_buf) && xfer->len) {
1270		dev_dbg(&spi->dev, "missing rx or tx buf\n");
1271		return -EINVAL;
1272	}
1273
1274	asd = spi->controller_state;
1275	bits = (asd->csr >> 4) & 0xf;
1276	if (bits != xfer->bits_per_word - 8) {
1277		dev_dbg(&spi->dev,
1278			"you can't yet change bits_per_word in transfers\n");
1279		return -ENOPROTOOPT;
1280	}
1281
1282	/*
1283	 * DMA map early, for performance (empties dcache ASAP) and
1284	 * better fault reporting.
1285	 */
1286	if ((!msg->is_dma_mapped)
1287		&& as->use_pdc) {
1288		if (atmel_spi_dma_map_xfer(as, xfer) < 0)
1289			return -ENOMEM;
1290	}
1291
1292	atmel_spi_set_xfer_speed(as, msg->spi, xfer);
1293
1294	as->done_status = 0;
1295	as->current_transfer = xfer;
1296	as->current_remaining_bytes = xfer->len;
1297	while (as->current_remaining_bytes) {
1298		reinit_completion(&as->xfer_completion);
1299
1300		if (as->use_pdc) {
1301			atmel_spi_pdc_next_xfer(master, msg, xfer);
1302		} else if (atmel_spi_use_dma(as, xfer)) {
1303			len = as->current_remaining_bytes;
1304			ret = atmel_spi_next_xfer_dma_submit(master,
1305								xfer, &len);
1306			if (ret) {
1307				dev_err(&spi->dev,
1308					"unable to use DMA, fallback to PIO\n");
1309				atmel_spi_next_xfer_pio(master, xfer);
1310			} else {
1311				as->current_remaining_bytes -= len;
1312				if (as->current_remaining_bytes < 0)
1313					as->current_remaining_bytes = 0;
1314			}
1315		} else {
1316			atmel_spi_next_xfer_pio(master, xfer);
1317		}
1318
1319		/* interrupts are disabled, so free the lock for schedule */
1320		atmel_spi_unlock(as);
1321		dma_timeout = wait_for_completion_timeout(&as->xfer_completion,
1322							  SPI_DMA_TIMEOUT);
1323		atmel_spi_lock(as);
1324		if (WARN_ON(dma_timeout == 0)) {
1325			dev_err(&spi->dev, "spi transfer timeout\n");
1326			as->done_status = -EIO;
1327		}
1328
1329		if (as->done_status)
1330			break;
1331	}
1332
1333	if (as->done_status) {
1334		if (as->use_pdc) {
1335			dev_warn(master->dev.parent,
1336				"overrun (%u/%u remaining)\n",
1337				spi_readl(as, TCR), spi_readl(as, RCR));
1338
1339			/*
1340			 * Clean up DMA registers and make sure the data
1341			 * registers are empty.
1342			 */
1343			spi_writel(as, RNCR, 0);
1344			spi_writel(as, TNCR, 0);
1345			spi_writel(as, RCR, 0);
1346			spi_writel(as, TCR, 0);
1347			for (timeout = 1000; timeout; timeout--)
1348				if (spi_readl(as, SR) & SPI_BIT(TXEMPTY))
1349					break;
1350			if (!timeout)
1351				dev_warn(master->dev.parent,
1352					 "timeout waiting for TXEMPTY");
1353			while (spi_readl(as, SR) & SPI_BIT(RDRF))
1354				spi_readl(as, RDR);
1355
1356			/* Clear any overrun happening while cleaning up */
1357			spi_readl(as, SR);
1358
1359		} else if (atmel_spi_use_dma(as, xfer)) {
1360			atmel_spi_stop_dma(master);
1361		}
1362
1363		if (!msg->is_dma_mapped
1364			&& as->use_pdc)
1365			atmel_spi_dma_unmap_xfer(master, xfer);
1366
1367		return 0;
1368
1369	} else {
1370		/* only update length if no error */
1371		msg->actual_length += xfer->len;
1372	}
1373
1374	if (!msg->is_dma_mapped
1375		&& as->use_pdc)
1376		atmel_spi_dma_unmap_xfer(master, xfer);
1377
1378	if (xfer->delay_usecs)
1379		udelay(xfer->delay_usecs);
1380
1381	if (xfer->cs_change) {
1382		if (list_is_last(&xfer->transfer_list,
1383				 &msg->transfers)) {
1384			as->keep_cs = true;
1385		} else {
1386			as->cs_active = !as->cs_active;
1387			if (as->cs_active)
1388				cs_activate(as, msg->spi);
1389			else
1390				cs_deactivate(as, msg->spi);
1391		}
1392	}
1393
1394	return 0;
1395}
1396
1397static int atmel_spi_transfer_one_message(struct spi_master *master,
1398						struct spi_message *msg)
1399{
1400	struct atmel_spi *as;
1401	struct spi_transfer *xfer;
1402	struct spi_device *spi = msg->spi;
1403	int ret = 0;
1404
1405	as = spi_master_get_devdata(master);
1406
1407	dev_dbg(&spi->dev, "new message %p submitted for %s\n",
1408					msg, dev_name(&spi->dev));
1409
1410	atmel_spi_lock(as);
1411	cs_activate(as, spi);
1412
1413	as->cs_active = true;
1414	as->keep_cs = false;
1415
1416	msg->status = 0;
1417	msg->actual_length = 0;
1418
1419	list_for_each_entry(xfer, &msg->transfers, transfer_list) {
1420		ret = atmel_spi_one_transfer(master, msg, xfer);
1421		if (ret)
1422			goto msg_done;
1423	}
1424
1425	if (as->use_pdc)
1426		atmel_spi_disable_pdc_transfer(as);
1427
1428	list_for_each_entry(xfer, &msg->transfers, transfer_list) {
1429		dev_dbg(&spi->dev,
1430			"  xfer %p: len %u tx %p/%pad rx %p/%pad\n",
1431			xfer, xfer->len,
1432			xfer->tx_buf, &xfer->tx_dma,
1433			xfer->rx_buf, &xfer->rx_dma);
1434	}
1435
1436msg_done:
1437	if (!as->keep_cs)
1438		cs_deactivate(as, msg->spi);
1439
1440	atmel_spi_unlock(as);
1441
1442	msg->status = as->done_status;
1443	spi_finalize_current_message(spi->master);
1444
1445	return ret;
1446}
1447
1448static void atmel_spi_cleanup(struct spi_device *spi)
1449{
1450	struct atmel_spi_device	*asd = spi->controller_state;
1451
1452	if (!asd)
1453		return;
1454
1455	spi->controller_state = NULL;
1456	kfree(asd);
1457}
1458
1459static inline unsigned int atmel_get_version(struct atmel_spi *as)
1460{
1461	return spi_readl(as, VERSION) & 0x00000fff;
1462}
1463
1464static void atmel_get_caps(struct atmel_spi *as)
1465{
1466	unsigned int version;
1467
1468	version = atmel_get_version(as);
 
1469
1470	as->caps.is_spi2 = version > 0x121;
1471	as->caps.has_wdrbt = version >= 0x210;
1472	as->caps.has_dma_support = version >= 0x212;
1473	as->caps.has_pdc_support = version < 0x212;
1474}
1475
1476/*-------------------------------------------------------------------------*/
1477static int atmel_spi_gpio_cs(struct platform_device *pdev)
1478{
1479	struct spi_master	*master = platform_get_drvdata(pdev);
1480	struct atmel_spi	*as = spi_master_get_devdata(master);
1481	struct device_node	*np = master->dev.of_node;
1482	int			i;
1483	int			ret = 0;
1484	int			nb = 0;
1485
1486	if (!as->use_cs_gpios)
1487		return 0;
1488
1489	if (!np)
1490		return 0;
1491
1492	nb = of_gpio_named_count(np, "cs-gpios");
1493	for (i = 0; i < nb; i++) {
1494		int cs_gpio = of_get_named_gpio(pdev->dev.of_node,
1495						"cs-gpios", i);
1496
1497		if (cs_gpio == -EPROBE_DEFER)
1498			return cs_gpio;
1499
1500		if (gpio_is_valid(cs_gpio)) {
1501			ret = devm_gpio_request(&pdev->dev, cs_gpio,
1502						dev_name(&pdev->dev));
1503			if (ret)
1504				return ret;
1505		}
1506	}
1507
1508	return 0;
1509}
1510
1511static void atmel_spi_init(struct atmel_spi *as)
1512{
1513	spi_writel(as, CR, SPI_BIT(SWRST));
1514	spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */
1515
1516	/* It is recommended to enable FIFOs first thing after reset */
1517	if (as->fifo_size)
1518		spi_writel(as, CR, SPI_BIT(FIFOEN));
1519
1520	if (as->caps.has_wdrbt) {
1521		spi_writel(as, MR, SPI_BIT(WDRBT) | SPI_BIT(MODFDIS)
1522				| SPI_BIT(MSTR));
1523	} else {
1524		spi_writel(as, MR, SPI_BIT(MSTR) | SPI_BIT(MODFDIS));
1525	}
1526
1527	if (as->use_pdc)
1528		spi_writel(as, PTCR, SPI_BIT(RXTDIS) | SPI_BIT(TXTDIS));
1529	spi_writel(as, CR, SPI_BIT(SPIEN));
1530}
1531
1532static int atmel_spi_probe(struct platform_device *pdev)
1533{
1534	struct resource		*regs;
1535	int			irq;
1536	struct clk		*clk;
1537	int			ret;
1538	struct spi_master	*master;
1539	struct atmel_spi	*as;
1540
1541	/* Select default pin state */
1542	pinctrl_pm_select_default_state(&pdev->dev);
1543
1544	regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1545	if (!regs)
1546		return -ENXIO;
1547
1548	irq = platform_get_irq(pdev, 0);
1549	if (irq < 0)
1550		return irq;
1551
1552	clk = devm_clk_get(&pdev->dev, "spi_clk");
1553	if (IS_ERR(clk))
1554		return PTR_ERR(clk);
1555
1556	/* setup spi core then atmel-specific driver state */
1557	ret = -ENOMEM;
1558	master = spi_alloc_master(&pdev->dev, sizeof(*as));
1559	if (!master)
1560		goto out_free;
1561
1562	/* the spi->mode bits understood by this driver: */
1563	master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;
1564	master->bits_per_word_mask = SPI_BPW_RANGE_MASK(8, 16);
1565	master->dev.of_node = pdev->dev.of_node;
1566	master->bus_num = pdev->id;
1567	master->num_chipselect = master->dev.of_node ? 0 : 4;
1568	master->setup = atmel_spi_setup;
1569	master->flags = (SPI_MASTER_MUST_RX | SPI_MASTER_MUST_TX);
1570	master->transfer_one_message = atmel_spi_transfer_one_message;
1571	master->cleanup = atmel_spi_cleanup;
1572	master->auto_runtime_pm = true;
1573	master->max_dma_len = SPI_MAX_DMA_XFER;
1574	master->can_dma = atmel_spi_can_dma;
1575	platform_set_drvdata(pdev, master);
1576
1577	as = spi_master_get_devdata(master);
1578
1579	spin_lock_init(&as->lock);
1580
1581	as->pdev = pdev;
1582	as->regs = devm_ioremap_resource(&pdev->dev, regs);
1583	if (IS_ERR(as->regs)) {
1584		ret = PTR_ERR(as->regs);
1585		goto out_unmap_regs;
1586	}
1587	as->phybase = regs->start;
1588	as->irq = irq;
1589	as->clk = clk;
1590
1591	init_completion(&as->xfer_completion);
1592
1593	atmel_get_caps(as);
1594
1595	as->use_cs_gpios = true;
1596	if (atmel_spi_is_v2(as) &&
1597	    pdev->dev.of_node &&
1598	    !of_get_property(pdev->dev.of_node, "cs-gpios", NULL)) {
1599		as->use_cs_gpios = false;
1600		master->num_chipselect = 4;
1601	}
1602
1603	ret = atmel_spi_gpio_cs(pdev);
1604	if (ret)
1605		goto out_unmap_regs;
1606
1607	as->use_dma = false;
1608	as->use_pdc = false;
1609	if (as->caps.has_dma_support) {
1610		ret = atmel_spi_configure_dma(master, as);
1611		if (ret == 0) {
1612			as->use_dma = true;
1613		} else if (ret == -EPROBE_DEFER) {
1614			return ret;
1615		}
1616	} else if (as->caps.has_pdc_support) {
1617		as->use_pdc = true;
1618	}
1619
1620	if (IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) {
1621		as->addr_rx_bbuf = dma_alloc_coherent(&pdev->dev,
1622						      SPI_MAX_DMA_XFER,
1623						      &as->dma_addr_rx_bbuf,
1624						      GFP_KERNEL | GFP_DMA);
1625		if (!as->addr_rx_bbuf) {
1626			as->use_dma = false;
1627		} else {
1628			as->addr_tx_bbuf = dma_alloc_coherent(&pdev->dev,
1629					SPI_MAX_DMA_XFER,
1630					&as->dma_addr_tx_bbuf,
1631					GFP_KERNEL | GFP_DMA);
1632			if (!as->addr_tx_bbuf) {
1633				as->use_dma = false;
1634				dma_free_coherent(&pdev->dev, SPI_MAX_DMA_XFER,
1635						  as->addr_rx_bbuf,
1636						  as->dma_addr_rx_bbuf);
1637			}
1638		}
1639		if (!as->use_dma)
1640			dev_info(master->dev.parent,
1641				 "  can not allocate dma coherent memory\n");
1642	}
1643
1644	if (as->caps.has_dma_support && !as->use_dma)
1645		dev_info(&pdev->dev, "Atmel SPI Controller using PIO only\n");
1646
1647	if (as->use_pdc) {
1648		ret = devm_request_irq(&pdev->dev, irq, atmel_spi_pdc_interrupt,
1649					0, dev_name(&pdev->dev), master);
1650	} else {
1651		ret = devm_request_irq(&pdev->dev, irq, atmel_spi_pio_interrupt,
1652					0, dev_name(&pdev->dev), master);
1653	}
1654	if (ret)
1655		goto out_unmap_regs;
1656
1657	/* Initialize the hardware */
1658	ret = clk_prepare_enable(clk);
1659	if (ret)
1660		goto out_free_irq;
1661
1662	as->spi_clk = clk_get_rate(clk);
1663
 
 
 
 
 
 
 
 
 
 
 
 
 
1664	as->fifo_size = 0;
1665	if (!of_property_read_u32(pdev->dev.of_node, "atmel,fifo-size",
1666				  &as->fifo_size)) {
1667		dev_info(&pdev->dev, "Using FIFO (%u data)\n", as->fifo_size);
 
1668	}
1669
1670	atmel_spi_init(as);
1671
1672	pm_runtime_set_autosuspend_delay(&pdev->dev, AUTOSUSPEND_TIMEOUT);
1673	pm_runtime_use_autosuspend(&pdev->dev);
1674	pm_runtime_set_active(&pdev->dev);
1675	pm_runtime_enable(&pdev->dev);
1676
1677	ret = devm_spi_register_master(&pdev->dev, master);
1678	if (ret)
1679		goto out_free_dma;
1680
1681	/* go! */
1682	dev_info(&pdev->dev, "Atmel SPI Controller version 0x%x at 0x%08lx (irq %d)\n",
1683			atmel_get_version(as), (unsigned long)regs->start,
1684			irq);
1685
1686	return 0;
1687
1688out_free_dma:
1689	pm_runtime_disable(&pdev->dev);
1690	pm_runtime_set_suspended(&pdev->dev);
1691
1692	if (as->use_dma)
1693		atmel_spi_release_dma(master);
1694
1695	spi_writel(as, CR, SPI_BIT(SWRST));
1696	spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */
1697	clk_disable_unprepare(clk);
1698out_free_irq:
1699out_unmap_regs:
1700out_free:
1701	spi_master_put(master);
1702	return ret;
1703}
1704
1705static int atmel_spi_remove(struct platform_device *pdev)
1706{
1707	struct spi_master	*master = platform_get_drvdata(pdev);
1708	struct atmel_spi	*as = spi_master_get_devdata(master);
1709
1710	pm_runtime_get_sync(&pdev->dev);
1711
1712	/* reset the hardware and block queue progress */
 
1713	if (as->use_dma) {
1714		atmel_spi_stop_dma(master);
1715		atmel_spi_release_dma(master);
1716		if (IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) {
1717			dma_free_coherent(&pdev->dev, SPI_MAX_DMA_XFER,
1718					  as->addr_tx_bbuf,
1719					  as->dma_addr_tx_bbuf);
1720			dma_free_coherent(&pdev->dev, SPI_MAX_DMA_XFER,
1721					  as->addr_rx_bbuf,
1722					  as->dma_addr_rx_bbuf);
1723		}
1724	}
1725
1726	spin_lock_irq(&as->lock);
1727	spi_writel(as, CR, SPI_BIT(SWRST));
1728	spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */
1729	spi_readl(as, SR);
1730	spin_unlock_irq(&as->lock);
1731
1732	clk_disable_unprepare(as->clk);
1733
1734	pm_runtime_put_noidle(&pdev->dev);
1735	pm_runtime_disable(&pdev->dev);
1736
1737	return 0;
1738}
1739
1740#ifdef CONFIG_PM
1741static int atmel_spi_runtime_suspend(struct device *dev)
1742{
1743	struct spi_master *master = dev_get_drvdata(dev);
1744	struct atmel_spi *as = spi_master_get_devdata(master);
1745
1746	clk_disable_unprepare(as->clk);
1747	pinctrl_pm_select_sleep_state(dev);
1748
1749	return 0;
1750}
1751
1752static int atmel_spi_runtime_resume(struct device *dev)
1753{
1754	struct spi_master *master = dev_get_drvdata(dev);
1755	struct atmel_spi *as = spi_master_get_devdata(master);
1756
1757	pinctrl_pm_select_default_state(dev);
1758
1759	return clk_prepare_enable(as->clk);
1760}
1761
1762#ifdef CONFIG_PM_SLEEP
1763static int atmel_spi_suspend(struct device *dev)
1764{
1765	struct spi_master *master = dev_get_drvdata(dev);
1766	int ret;
1767
1768	/* Stop the queue running */
1769	ret = spi_master_suspend(master);
1770	if (ret) {
1771		dev_warn(dev, "cannot suspend master\n");
1772		return ret;
1773	}
1774
1775	if (!pm_runtime_suspended(dev))
1776		atmel_spi_runtime_suspend(dev);
1777
1778	return 0;
1779}
1780
1781static int atmel_spi_resume(struct device *dev)
1782{
1783	struct spi_master *master = dev_get_drvdata(dev);
1784	struct atmel_spi *as = spi_master_get_devdata(master);
1785	int ret;
1786
1787	ret = clk_prepare_enable(as->clk);
1788	if (ret)
1789		return ret;
1790
1791	atmel_spi_init(as);
1792
1793	clk_disable_unprepare(as->clk);
1794
1795	if (!pm_runtime_suspended(dev)) {
1796		ret = atmel_spi_runtime_resume(dev);
1797		if (ret)
1798			return ret;
1799	}
1800
1801	/* Start the queue running */
1802	ret = spi_master_resume(master);
1803	if (ret)
1804		dev_err(dev, "problem starting queue (%d)\n", ret);
1805
1806	return ret;
1807}
1808#endif
1809
1810static const struct dev_pm_ops atmel_spi_pm_ops = {
1811	SET_SYSTEM_SLEEP_PM_OPS(atmel_spi_suspend, atmel_spi_resume)
1812	SET_RUNTIME_PM_OPS(atmel_spi_runtime_suspend,
1813			   atmel_spi_runtime_resume, NULL)
1814};
1815#define ATMEL_SPI_PM_OPS	(&atmel_spi_pm_ops)
1816#else
1817#define ATMEL_SPI_PM_OPS	NULL
1818#endif
1819
1820#if defined(CONFIG_OF)
1821static const struct of_device_id atmel_spi_dt_ids[] = {
1822	{ .compatible = "atmel,at91rm9200-spi" },
1823	{ /* sentinel */ }
1824};
1825
1826MODULE_DEVICE_TABLE(of, atmel_spi_dt_ids);
1827#endif
1828
1829static struct platform_driver atmel_spi_driver = {
1830	.driver		= {
1831		.name	= "atmel_spi",
1832		.pm	= ATMEL_SPI_PM_OPS,
1833		.of_match_table	= of_match_ptr(atmel_spi_dt_ids),
1834	},
1835	.probe		= atmel_spi_probe,
1836	.remove		= atmel_spi_remove,
1837};
1838module_platform_driver(atmel_spi_driver);
1839
1840MODULE_DESCRIPTION("Atmel AT32/AT91 SPI Controller driver");
1841MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1842MODULE_LICENSE("GPL");
1843MODULE_ALIAS("platform:atmel_spi");