1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Driver for Atmel AT32 and AT91 SPI Controllers
   4 *
   5 * Copyright (C) 2006 Atmel Corporation
   6 */
   7
   8#include <linux/kernel.h>
   9#include <linux/clk.h>
  10#include <linux/module.h>
  11#include <linux/platform_device.h>
  12#include <linux/delay.h>
  13#include <linux/dma-mapping.h>
  14#include <linux/dmaengine.h>
  15#include <linux/err.h>
  16#include <linux/interrupt.h>
  17#include <linux/spi/spi.h>
  18#include <linux/slab.h>
  19#include <linux/of.h>
  20
  21#include <linux/io.h>
  22#include <linux/gpio/consumer.h>
  23#include <linux/pinctrl/consumer.h>
  24#include <linux/pm_runtime.h>
  25#include <linux/iopoll.h>
  26#include <trace/events/spi.h>
  27
  28/* SPI register offsets */
  29#define SPI_CR					0x0000
  30#define SPI_MR					0x0004
  31#define SPI_RDR					0x0008
  32#define SPI_TDR					0x000c
  33#define SPI_SR					0x0010
  34#define SPI_IER					0x0014
  35#define SPI_IDR					0x0018
  36#define SPI_IMR					0x001c
  37#define SPI_CSR0				0x0030
  38#define SPI_CSR1				0x0034
  39#define SPI_CSR2				0x0038
  40#define SPI_CSR3				0x003c
  41#define SPI_FMR					0x0040
  42#define SPI_FLR					0x0044
  43#define SPI_VERSION				0x00fc
  44#define SPI_RPR					0x0100
  45#define SPI_RCR					0x0104
  46#define SPI_TPR					0x0108
  47#define SPI_TCR					0x010c
  48#define SPI_RNPR				0x0110
  49#define SPI_RNCR				0x0114
  50#define SPI_TNPR				0x0118
  51#define SPI_TNCR				0x011c
  52#define SPI_PTCR				0x0120
  53#define SPI_PTSR				0x0124
  54
  55/* Bitfields in CR */
  56#define SPI_SPIEN_OFFSET			0
  57#define SPI_SPIEN_SIZE				1
  58#define SPI_SPIDIS_OFFSET			1
  59#define SPI_SPIDIS_SIZE				1
  60#define SPI_SWRST_OFFSET			7
  61#define SPI_SWRST_SIZE				1
  62#define SPI_LASTXFER_OFFSET			24
  63#define SPI_LASTXFER_SIZE			1
  64#define SPI_TXFCLR_OFFSET			16
  65#define SPI_TXFCLR_SIZE				1
  66#define SPI_RXFCLR_OFFSET			17
  67#define SPI_RXFCLR_SIZE				1
  68#define SPI_FIFOEN_OFFSET			30
  69#define SPI_FIFOEN_SIZE				1
  70#define SPI_FIFODIS_OFFSET			31
  71#define SPI_FIFODIS_SIZE			1
  72
  73/* Bitfields in MR */
  74#define SPI_MSTR_OFFSET				0
  75#define SPI_MSTR_SIZE				1
  76#define SPI_PS_OFFSET				1
  77#define SPI_PS_SIZE				1
  78#define SPI_PCSDEC_OFFSET			2
  79#define SPI_PCSDEC_SIZE				1
  80#define SPI_FDIV_OFFSET				3
  81#define SPI_FDIV_SIZE				1
  82#define SPI_MODFDIS_OFFSET			4
  83#define SPI_MODFDIS_SIZE			1
  84#define SPI_WDRBT_OFFSET			5
  85#define SPI_WDRBT_SIZE				1
  86#define SPI_LLB_OFFSET				7
  87#define SPI_LLB_SIZE				1
  88#define SPI_PCS_OFFSET				16
  89#define SPI_PCS_SIZE				4
  90#define SPI_DLYBCS_OFFSET			24
  91#define SPI_DLYBCS_SIZE				8
  92
  93/* Bitfields in RDR */
  94#define SPI_RD_OFFSET				0
  95#define SPI_RD_SIZE				16
  96
  97/* Bitfields in TDR */
  98#define SPI_TD_OFFSET				0
  99#define SPI_TD_SIZE				16
 100
 101/* Bitfields in SR */
 102#define SPI_RDRF_OFFSET				0
 103#define SPI_RDRF_SIZE				1
 104#define SPI_TDRE_OFFSET				1
 105#define SPI_TDRE_SIZE				1
 106#define SPI_MODF_OFFSET				2
 107#define SPI_MODF_SIZE				1
 108#define SPI_OVRES_OFFSET			3
 109#define SPI_OVRES_SIZE				1
 110#define SPI_ENDRX_OFFSET			4
 111#define SPI_ENDRX_SIZE				1
 112#define SPI_ENDTX_OFFSET			5
 113#define SPI_ENDTX_SIZE				1
 114#define SPI_RXBUFF_OFFSET			6
 115#define SPI_RXBUFF_SIZE				1
 116#define SPI_TXBUFE_OFFSET			7
 117#define SPI_TXBUFE_SIZE				1
 118#define SPI_NSSR_OFFSET				8
 119#define SPI_NSSR_SIZE				1
 120#define SPI_TXEMPTY_OFFSET			9
 121#define SPI_TXEMPTY_SIZE			1
 122#define SPI_SPIENS_OFFSET			16
 123#define SPI_SPIENS_SIZE				1
 124#define SPI_TXFEF_OFFSET			24
 125#define SPI_TXFEF_SIZE				1
 126#define SPI_TXFFF_OFFSET			25
 127#define SPI_TXFFF_SIZE				1
 128#define SPI_TXFTHF_OFFSET			26
 129#define SPI_TXFTHF_SIZE				1
 130#define SPI_RXFEF_OFFSET			27
 131#define SPI_RXFEF_SIZE				1
 132#define SPI_RXFFF_OFFSET			28
 133#define SPI_RXFFF_SIZE				1
 134#define SPI_RXFTHF_OFFSET			29
 135#define SPI_RXFTHF_SIZE				1
 136#define SPI_TXFPTEF_OFFSET			30
 137#define SPI_TXFPTEF_SIZE			1
 138#define SPI_RXFPTEF_OFFSET			31
 139#define SPI_RXFPTEF_SIZE			1
 140
 141/* Bitfields in CSR0 */
 142#define SPI_CPOL_OFFSET				0
 143#define SPI_CPOL_SIZE				1
 144#define SPI_NCPHA_OFFSET			1
 145#define SPI_NCPHA_SIZE				1
 146#define SPI_CSAAT_OFFSET			3
 147#define SPI_CSAAT_SIZE				1
 148#define SPI_BITS_OFFSET				4
 149#define SPI_BITS_SIZE				4
 150#define SPI_SCBR_OFFSET				8
 151#define SPI_SCBR_SIZE				8
 152#define SPI_DLYBS_OFFSET			16
 153#define SPI_DLYBS_SIZE				8
 154#define SPI_DLYBCT_OFFSET			24
 155#define SPI_DLYBCT_SIZE				8
 156
 157/* Bitfields in RCR */
 158#define SPI_RXCTR_OFFSET			0
 159#define SPI_RXCTR_SIZE				16
 160
 161/* Bitfields in TCR */
 162#define SPI_TXCTR_OFFSET			0
 163#define SPI_TXCTR_SIZE				16
 164
 165/* Bitfields in RNCR */
 166#define SPI_RXNCR_OFFSET			0
 167#define SPI_RXNCR_SIZE				16
 168
 169/* Bitfields in TNCR */
 170#define SPI_TXNCR_OFFSET			0
 171#define SPI_TXNCR_SIZE				16
 172
 173/* Bitfields in PTCR */
 174#define SPI_RXTEN_OFFSET			0
 175#define SPI_RXTEN_SIZE				1
 176#define SPI_RXTDIS_OFFSET			1
 177#define SPI_RXTDIS_SIZE				1
 178#define SPI_TXTEN_OFFSET			8
 179#define SPI_TXTEN_SIZE				1
 180#define SPI_TXTDIS_OFFSET			9
 181#define SPI_TXTDIS_SIZE				1
 182
 183/* Bitfields in FMR */
 184#define SPI_TXRDYM_OFFSET			0
 185#define SPI_TXRDYM_SIZE				2
 186#define SPI_RXRDYM_OFFSET			4
 187#define SPI_RXRDYM_SIZE				2
 188#define SPI_TXFTHRES_OFFSET			16
 189#define SPI_TXFTHRES_SIZE			6
 190#define SPI_RXFTHRES_OFFSET			24
 191#define SPI_RXFTHRES_SIZE			6
 192
 193/* Bitfields in FLR */
 194#define SPI_TXFL_OFFSET				0
 195#define SPI_TXFL_SIZE				6
 196#define SPI_RXFL_OFFSET				16
 197#define SPI_RXFL_SIZE				6
 198
 199/* Constants for BITS */
 200#define SPI_BITS_8_BPT				0
 201#define SPI_BITS_9_BPT				1
 202#define SPI_BITS_10_BPT				2
 203#define SPI_BITS_11_BPT				3
 204#define SPI_BITS_12_BPT				4
 205#define SPI_BITS_13_BPT				5
 206#define SPI_BITS_14_BPT				6
 207#define SPI_BITS_15_BPT				7
 208#define SPI_BITS_16_BPT				8
 209#define SPI_ONE_DATA				0
 210#define SPI_TWO_DATA				1
 211#define SPI_FOUR_DATA				2
 212
 213/* Bit manipulation macros */
 214#define SPI_BIT(name) \
 215	(1 << SPI_##name##_OFFSET)
 216#define SPI_BF(name, value) \
 217	(((value) & ((1 << SPI_##name##_SIZE) - 1)) << SPI_##name##_OFFSET)
 218#define SPI_BFEXT(name, value) \
 219	(((value) >> SPI_##name##_OFFSET) & ((1 << SPI_##name##_SIZE) - 1))
 220#define SPI_BFINS(name, value, old) \
 221	(((old) & ~(((1 << SPI_##name##_SIZE) - 1) << SPI_##name##_OFFSET)) \
 222	  | SPI_BF(name, value))
 223
 224/* Register access macros */
 225#define spi_readl(port, reg) \
 226	readl_relaxed((port)->regs + SPI_##reg)
 227#define spi_writel(port, reg, value) \
 228	writel_relaxed((value), (port)->regs + SPI_##reg)
 229#define spi_writew(port, reg, value) \
 230	writew_relaxed((value), (port)->regs + SPI_##reg)
 231
 232/* use PIO for small transfers, avoiding DMA setup/teardown overhead and
 233 * cache operations; better heuristics consider wordsize and bitrate.
 234 */
 235#define DMA_MIN_BYTES	16
 236
 237#define AUTOSUSPEND_TIMEOUT	2000
 238
 239struct atmel_spi_caps {
 240	bool	is_spi2;
 241	bool	has_wdrbt;
 242	bool	has_dma_support;
 243	bool	has_pdc_support;
 244};
 245
 246/*
 247 * The core SPI transfer engine just talks to a register bank to set up
 248 * DMA transfers; transfer queue progress is driven by IRQs.  The clock
 249 * framework provides the base clock, subdivided for each spi_device.
 250 */
 251struct atmel_spi {
 252	spinlock_t		lock;
 253	unsigned long		flags;
 254
 255	phys_addr_t		phybase;
 256	void __iomem		*regs;
 257	int			irq;
 258	struct clk		*clk;
 259	struct platform_device	*pdev;
 260	unsigned long		spi_clk;
 261
 262	struct spi_transfer	*current_transfer;
 263	int			current_remaining_bytes;
 264	int			done_status;
 265	dma_addr_t		dma_addr_rx_bbuf;
 266	dma_addr_t		dma_addr_tx_bbuf;
 267	void			*addr_rx_bbuf;
 268	void			*addr_tx_bbuf;
 269
 270	struct completion	xfer_completion;
 271
 272	struct atmel_spi_caps	caps;
 273
 274	bool			use_dma;
 275	bool			use_pdc;
 276
 277	bool			keep_cs;
 278
 279	u32			fifo_size;
 280	bool			last_polarity;
 281	u8			native_cs_free;
 282	u8			native_cs_for_gpio;
 283};
 284
 285/* Controller-specific per-slave state */
 286struct atmel_spi_device {
 287	u32			csr;
 288};
 289
 290#define SPI_MAX_DMA_XFER	65535 /* true for both PDC and DMA */
 291#define INVALID_DMA_ADDRESS	0xffffffff
 292
 293/*
 294 * This frequency can be anything supported by the controller, but to avoid
 295 * unnecessary delay, the highest possible frequency is chosen.
 296 *
 297 * This frequency is the highest possible which is not interfering with other
 298 * chip select registers (see Note for Serial Clock Bit Rate configuration in
 299 * Atmel-11121F-ATARM-SAMA5D3-Series-Datasheet_02-Feb-16, page 1283)
 300 */
 301#define DUMMY_MSG_FREQUENCY	0x02
 302/*
 303 * 8 bits is the minimum data the controller is capable of sending.
 304 *
 305 * This message can be anything as it should not be treated by any SPI device.
 306 */
 307#define DUMMY_MSG		0xAA
 308
 309/*
 310 * Version 2 of the SPI controller has
 311 *  - CR.LASTXFER
 312 *  - SPI_MR.DIV32 may become FDIV or must-be-zero (here: always zero)
 313 *  - SPI_SR.TXEMPTY, SPI_SR.NSSR (and corresponding irqs)
 314 *  - SPI_CSRx.CSAAT
 315 *  - SPI_CSRx.SBCR allows faster clocking
 316 */
 317static bool atmel_spi_is_v2(struct atmel_spi *as)
 318{
 319	return as->caps.is_spi2;
 320}
 321
 322/*
 323 * Send a dummy message.
 324 *
 325 * This is sometimes needed when using a CS GPIO to force clock transition when
 326 * switching between devices with different polarities.
 327 */
 328static void atmel_spi_send_dummy(struct atmel_spi *as, struct spi_device *spi, int chip_select)
 329{
 330	u32 status;
 331	u32 csr;
 332
 333	/*
 334	 * Set a clock frequency to allow sending message on SPI bus.
 335	 * The frequency here can be anything, but is needed for
 336	 * the controller to send the data.
 337	 */
 338	csr = spi_readl(as, CSR0 + 4 * chip_select);
 339	csr = SPI_BFINS(SCBR, DUMMY_MSG_FREQUENCY, csr);
 340	spi_writel(as, CSR0 + 4 * chip_select, csr);
 341
 342	/*
 343	 * Read all data coming from SPI bus, needed to be able to send
 344	 * the message.
 345	 */
 346	spi_readl(as, RDR);
 347	while (spi_readl(as, SR) & SPI_BIT(RDRF)) {
 348		spi_readl(as, RDR);
 349		cpu_relax();
 350	}
 351
 352	spi_writel(as, TDR, DUMMY_MSG);
 353
 354	readl_poll_timeout_atomic(as->regs + SPI_SR, status,
 355				  (status & SPI_BIT(TXEMPTY)), 1, 1000);
 356}
 357
 358
 359/*
 360 * Earlier SPI controllers (e.g. on at91rm9200) have a design bug whereby
 361 * they assume that spi slave device state will not change on deselect, so
 362 * that automagic deselection is OK.  ("NPCSx rises if no data is to be
 363 * transmitted")  Not so!  Workaround uses nCSx pins as GPIOs; or newer
 364 * controllers have CSAAT and friends.
 365 *
 366 * Even controller newer than ar91rm9200, using GPIOs can make sens as
 367 * it lets us support active-high chipselects despite the controller's
 368 * belief that only active-low devices/systems exists.
 369 *
 370 * However, at91rm9200 has a second erratum whereby nCS0 doesn't work
 371 * right when driven with GPIO.  ("Mode Fault does not allow more than one
 372 * Master on Chip Select 0.")  No workaround exists for that ... so for
 373 * nCS0 on that chip, we (a) don't use the GPIO, (b) can't support CS_HIGH,
 374 * and (c) will trigger that first erratum in some cases.
 375 *
 376 * When changing the clock polarity, the SPI controller waits for the next
 377 * transmission to enforce the default clock state. This may be an issue when
 378 * using a GPIO as Chip Select: the clock level is applied only when the first
 379 * packet is sent, once the CS has already been asserted. The workaround is to
 380 * avoid this by sending a first (dummy) message before toggling the CS state.
 381 */
 382static void cs_activate(struct atmel_spi *as, struct spi_device *spi)
 383{
 384	struct atmel_spi_device *asd = spi->controller_state;
 385	bool new_polarity;
 386	int chip_select;
 387	u32 mr;
 388
 389	if (spi_get_csgpiod(spi, 0))
 390		chip_select = as->native_cs_for_gpio;
 391	else
 392		chip_select = spi_get_chipselect(spi, 0);
 393
 394	if (atmel_spi_is_v2(as)) {
 395		spi_writel(as, CSR0 + 4 * chip_select, asd->csr);
 396		/* For the low SPI version, there is a issue that PDC transfer
 397		 * on CS1,2,3 needs SPI_CSR0.BITS config as SPI_CSR1,2,3.BITS
 398		 */
 399		spi_writel(as, CSR0, asd->csr);
 400		if (as->caps.has_wdrbt) {
 401			spi_writel(as, MR,
 402					SPI_BF(PCS, ~(0x01 << chip_select))
 403					| SPI_BIT(WDRBT)
 404					| SPI_BIT(MODFDIS)
 405					| SPI_BIT(MSTR));
 406		} else {
 407			spi_writel(as, MR,
 408					SPI_BF(PCS, ~(0x01 << chip_select))
 409					| SPI_BIT(MODFDIS)
 410					| SPI_BIT(MSTR));
 411		}
 412
 413		mr = spi_readl(as, MR);
 414
 415		/*
 416		 * Ensures the clock polarity is valid before we actually
 417		 * assert the CS to avoid spurious clock edges to be
 418		 * processed by the spi devices.
 419		 */
 420		if (spi_get_csgpiod(spi, 0)) {
 421			new_polarity = (asd->csr & SPI_BIT(CPOL)) != 0;
 422			if (new_polarity != as->last_polarity) {
 423				/*
 424				 * Need to disable the GPIO before sending the dummy
 425				 * message because it is already set by the spi core.
 426				 */
 427				gpiod_set_value_cansleep(spi_get_csgpiod(spi, 0), 0);
 428				atmel_spi_send_dummy(as, spi, chip_select);
 429				as->last_polarity = new_polarity;
 430				gpiod_set_value_cansleep(spi_get_csgpiod(spi, 0), 1);
 431			}
 432		}
 433	} else {
 434		u32 cpol = (spi->mode & SPI_CPOL) ? SPI_BIT(CPOL) : 0;
 435		int i;
 436		u32 csr;
 437
 438		/* Make sure clock polarity is correct */
 439		for (i = 0; i < spi->controller->num_chipselect; i++) {
 440			csr = spi_readl(as, CSR0 + 4 * i);
 441			if ((csr ^ cpol) & SPI_BIT(CPOL))
 442				spi_writel(as, CSR0 + 4 * i,
 443						csr ^ SPI_BIT(CPOL));
 444		}
 445
 446		mr = spi_readl(as, MR);
 447		mr = SPI_BFINS(PCS, ~(1 << chip_select), mr);
 448		spi_writel(as, MR, mr);
 449	}
 450
 451	dev_dbg(&spi->dev, "activate NPCS, mr %08x\n", mr);
 452}
 453
 454static void cs_deactivate(struct atmel_spi *as, struct spi_device *spi)
 455{
 456	int chip_select;
 457	u32 mr;
 458
 459	if (spi_get_csgpiod(spi, 0))
 460		chip_select = as->native_cs_for_gpio;
 461	else
 462		chip_select = spi_get_chipselect(spi, 0);
 463
 464	/* only deactivate *this* device; sometimes transfers to
 465	 * another device may be active when this routine is called.
 466	 */
 467	mr = spi_readl(as, MR);
 468	if (~SPI_BFEXT(PCS, mr) & (1 << chip_select)) {
 469		mr = SPI_BFINS(PCS, 0xf, mr);
 470		spi_writel(as, MR, mr);
 471	}
 472
 473	dev_dbg(&spi->dev, "DEactivate NPCS, mr %08x\n", mr);
 474
 475	if (!spi_get_csgpiod(spi, 0))
 476		spi_writel(as, CR, SPI_BIT(LASTXFER));
 477}
 478
 479static void atmel_spi_lock(struct atmel_spi *as) __acquires(&as->lock)
 480{
 481	spin_lock_irqsave(&as->lock, as->flags);
 482}
 483
 484static void atmel_spi_unlock(struct atmel_spi *as) __releases(&as->lock)
 485{
 486	spin_unlock_irqrestore(&as->lock, as->flags);
 487}
 488
 489static inline bool atmel_spi_is_vmalloc_xfer(struct spi_transfer *xfer)
 490{
 491	return is_vmalloc_addr(xfer->tx_buf) || is_vmalloc_addr(xfer->rx_buf);
 492}
 493
 494static inline bool atmel_spi_use_dma(struct atmel_spi *as,
 495				struct spi_transfer *xfer)
 496{
 497	return as->use_dma && xfer->len >= DMA_MIN_BYTES;
 498}
 499
 500static bool atmel_spi_can_dma(struct spi_controller *host,
 501			      struct spi_device *spi,
 502			      struct spi_transfer *xfer)
 503{
 504	struct atmel_spi *as = spi_controller_get_devdata(host);
 505
 506	if (IS_ENABLED(CONFIG_SOC_SAM_V4_V5))
 507		return atmel_spi_use_dma(as, xfer) &&
 508			!atmel_spi_is_vmalloc_xfer(xfer);
 509	else
 510		return atmel_spi_use_dma(as, xfer);
 511
 512}
 513
 514static int atmel_spi_dma_slave_config(struct atmel_spi *as, u8 bits_per_word)
 515{
 516	struct spi_controller *host = platform_get_drvdata(as->pdev);
 517	struct dma_slave_config	slave_config;
 518	int err = 0;
 519
 520	if (bits_per_word > 8) {
 521		slave_config.dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
 522		slave_config.src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
 523	} else {
 524		slave_config.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
 525		slave_config.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
 526	}
 527
 528	slave_config.dst_addr = (dma_addr_t)as->phybase + SPI_TDR;
 529	slave_config.src_addr = (dma_addr_t)as->phybase + SPI_RDR;
 530	slave_config.src_maxburst = 1;
 531	slave_config.dst_maxburst = 1;
 532	slave_config.device_fc = false;
 533
 534	/*
 535	 * This driver uses fixed peripheral select mode (PS bit set to '0' in
 536	 * the Mode Register).
 537	 * So according to the datasheet, when FIFOs are available (and
 538	 * enabled), the Transmit FIFO operates in Multiple Data Mode.
 539	 * In this mode, up to 2 data, not 4, can be written into the Transmit
 540	 * Data Register in a single access.
 541	 * However, the first data has to be written into the lowest 16 bits and
 542	 * the second data into the highest 16 bits of the Transmit
 543	 * Data Register. For 8bit data (the most frequent case), it would
 544	 * require to rework tx_buf so each data would actually fit 16 bits.
 545	 * So we'd rather write only one data at the time. Hence the transmit
 546	 * path works the same whether FIFOs are available (and enabled) or not.
 547	 */
 548	if (dmaengine_slave_config(host->dma_tx, &slave_config)) {
 549		dev_err(&as->pdev->dev,
 550			"failed to configure tx dma channel\n");
 551		err = -EINVAL;
 552	}
 553
 554	/*
 555	 * This driver configures the spi controller for host mode (MSTR bit
 556	 * set to '1' in the Mode Register).
 557	 * So according to the datasheet, when FIFOs are available (and
 558	 * enabled), the Receive FIFO operates in Single Data Mode.
 559	 * So the receive path works the same whether FIFOs are available (and
 560	 * enabled) or not.
 561	 */
 562	if (dmaengine_slave_config(host->dma_rx, &slave_config)) {
 563		dev_err(&as->pdev->dev,
 564			"failed to configure rx dma channel\n");
 565		err = -EINVAL;
 566	}
 567
 568	return err;
 569}
 570
 571static int atmel_spi_configure_dma(struct spi_controller *host,
 572				   struct atmel_spi *as)
 573{
 574	struct device *dev = &as->pdev->dev;
 575	int err;
 576
 577	host->dma_tx = dma_request_chan(dev, "tx");
 578	if (IS_ERR(host->dma_tx)) {
 579		err = PTR_ERR(host->dma_tx);
 580		dev_dbg(dev, "No TX DMA channel, DMA is disabled\n");
 581		goto error_clear;
 582	}
 583
 584	host->dma_rx = dma_request_chan(dev, "rx");
 585	if (IS_ERR(host->dma_rx)) {
 586		err = PTR_ERR(host->dma_rx);
 587		/*
 588		 * No reason to check EPROBE_DEFER here since we have already
 589		 * requested tx channel.
 590		 */
 591		dev_dbg(dev, "No RX DMA channel, DMA is disabled\n");
 592		goto error;
 593	}
 594
 595	err = atmel_spi_dma_slave_config(as, 8);
 596	if (err)
 597		goto error;
 598
 599	dev_info(&as->pdev->dev,
 600			"Using %s (tx) and %s (rx) for DMA transfers\n",
 601			dma_chan_name(host->dma_tx),
 602			dma_chan_name(host->dma_rx));
 603
 604	return 0;
 605error:
 606	if (!IS_ERR(host->dma_rx))
 607		dma_release_channel(host->dma_rx);
 608	if (!IS_ERR(host->dma_tx))
 609		dma_release_channel(host->dma_tx);
 610error_clear:
 611	host->dma_tx = host->dma_rx = NULL;
 612	return err;
 613}
 614
 615static void atmel_spi_stop_dma(struct spi_controller *host)
 616{
 617	if (host->dma_rx)
 618		dmaengine_terminate_all(host->dma_rx);
 619	if (host->dma_tx)
 620		dmaengine_terminate_all(host->dma_tx);
 621}
 622
 623static void atmel_spi_release_dma(struct spi_controller *host)
 624{
 625	if (host->dma_rx) {
 626		dma_release_channel(host->dma_rx);
 627		host->dma_rx = NULL;
 628	}
 629	if (host->dma_tx) {
 630		dma_release_channel(host->dma_tx);
 631		host->dma_tx = NULL;
 632	}
 633}
 634
 635/* This function is called by the DMA driver from tasklet context */
 636static void dma_callback(void *data)
 637{
 638	struct spi_controller	*host = data;
 639	struct atmel_spi	*as = spi_controller_get_devdata(host);
 640
 641	if (is_vmalloc_addr(as->current_transfer->rx_buf) &&
 642	    IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) {
 643		memcpy(as->current_transfer->rx_buf, as->addr_rx_bbuf,
 644		       as->current_transfer->len);
 645	}
 646	complete(&as->xfer_completion);
 647}
 648
 649/*
 650 * Next transfer using PIO without FIFO.
 651 */
 652static void atmel_spi_next_xfer_single(struct spi_controller *host,
 653				       struct spi_transfer *xfer)
 654{
 655	struct atmel_spi	*as = spi_controller_get_devdata(host);
 656	unsigned long xfer_pos = xfer->len - as->current_remaining_bytes;
 657
 658	dev_vdbg(host->dev.parent, "atmel_spi_next_xfer_pio\n");
 659
 660	/* Make sure data is not remaining in RDR */
 661	spi_readl(as, RDR);
 662	while (spi_readl(as, SR) & SPI_BIT(RDRF)) {
 663		spi_readl(as, RDR);
 664		cpu_relax();
 665	}
 666
 667	if (xfer->bits_per_word > 8)
 668		spi_writel(as, TDR, *(u16 *)(xfer->tx_buf + xfer_pos));
 669	else
 670		spi_writel(as, TDR, *(u8 *)(xfer->tx_buf + xfer_pos));
 671
 672	dev_dbg(host->dev.parent,
 673		"  start pio xfer %p: len %u tx %p rx %p bitpw %d\n",
 674		xfer, xfer->len, xfer->tx_buf, xfer->rx_buf,
 675		xfer->bits_per_word);
 676
 677	/* Enable relevant interrupts */
 678	spi_writel(as, IER, SPI_BIT(RDRF) | SPI_BIT(OVRES));
 679}
 680
 681/*
 682 * Next transfer using PIO with FIFO.
 683 */
 684static void atmel_spi_next_xfer_fifo(struct spi_controller *host,
 685				     struct spi_transfer *xfer)
 686{
 687	struct atmel_spi *as = spi_controller_get_devdata(host);
 688	u32 current_remaining_data, num_data;
 689	u32 offset = xfer->len - as->current_remaining_bytes;
 690	const u16 *words = (const u16 *)((u8 *)xfer->tx_buf + offset);
 691	const u8  *bytes = (const u8  *)((u8 *)xfer->tx_buf + offset);
 692	u16 td0, td1;
 693	u32 fifomr;
 694
 695	dev_vdbg(host->dev.parent, "atmel_spi_next_xfer_fifo\n");
 696
 697	/* Compute the number of data to transfer in the current iteration */
 698	current_remaining_data = ((xfer->bits_per_word > 8) ?
 699				  ((u32)as->current_remaining_bytes >> 1) :
 700				  (u32)as->current_remaining_bytes);
 701	num_data = min(current_remaining_data, as->fifo_size);
 702
 703	/* Flush RX and TX FIFOs */
 704	spi_writel(as, CR, SPI_BIT(RXFCLR) | SPI_BIT(TXFCLR));
 705	while (spi_readl(as, FLR))
 706		cpu_relax();
 707
 708	/* Set RX FIFO Threshold to the number of data to transfer */
 709	fifomr = spi_readl(as, FMR);
 710	spi_writel(as, FMR, SPI_BFINS(RXFTHRES, num_data, fifomr));
 711
 712	/* Clear FIFO flags in the Status Register, especially RXFTHF */
 713	(void)spi_readl(as, SR);
 714
 715	/* Fill TX FIFO */
 716	while (num_data >= 2) {
 717		if (xfer->bits_per_word > 8) {
 718			td0 = *words++;
 719			td1 = *words++;
 720		} else {
 721			td0 = *bytes++;
 722			td1 = *bytes++;
 723		}
 724
 725		spi_writel(as, TDR, (td1 << 16) | td0);
 726		num_data -= 2;
 727	}
 728
 729	if (num_data) {
 730		if (xfer->bits_per_word > 8)
 731			td0 = *words++;
 732		else
 733			td0 = *bytes++;
 734
 735		spi_writew(as, TDR, td0);
 736		num_data--;
 737	}
 738
 739	dev_dbg(host->dev.parent,
 740		"  start fifo xfer %p: len %u tx %p rx %p bitpw %d\n",
 741		xfer, xfer->len, xfer->tx_buf, xfer->rx_buf,
 742		xfer->bits_per_word);
 743
 744	/*
 745	 * Enable RX FIFO Threshold Flag interrupt to be notified about
 746	 * transfer completion.
 747	 */
 748	spi_writel(as, IER, SPI_BIT(RXFTHF) | SPI_BIT(OVRES));
 749}
 750
 751/*
 752 * Next transfer using PIO.
 753 */
 754static void atmel_spi_next_xfer_pio(struct spi_controller *host,
 755				    struct spi_transfer *xfer)
 756{
 757	struct atmel_spi *as = spi_controller_get_devdata(host);
 758
 759	if (as->fifo_size)
 760		atmel_spi_next_xfer_fifo(host, xfer);
 761	else
 762		atmel_spi_next_xfer_single(host, xfer);
 763}
 764
 765/*
 766 * Submit next transfer for DMA.
 767 */
 768static int atmel_spi_next_xfer_dma_submit(struct spi_controller *host,
 769				struct spi_transfer *xfer,
 770				u32 *plen)
 771{
 772	struct atmel_spi	*as = spi_controller_get_devdata(host);
 773	struct dma_chan		*rxchan = host->dma_rx;
 774	struct dma_chan		*txchan = host->dma_tx;
 775	struct dma_async_tx_descriptor *rxdesc;
 776	struct dma_async_tx_descriptor *txdesc;
 777	dma_cookie_t		cookie;
 778
 779	dev_vdbg(host->dev.parent, "atmel_spi_next_xfer_dma_submit\n");
 780
 781	/* Check that the channels are available */
 782	if (!rxchan || !txchan)
 783		return -ENODEV;
 784
 785
 786	*plen = xfer->len;
 787
 788	if (atmel_spi_dma_slave_config(as, xfer->bits_per_word))
 789		goto err_exit;
 790
 791	/* Send both scatterlists */
 792	if (atmel_spi_is_vmalloc_xfer(xfer) &&
 793	    IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) {
 794		rxdesc = dmaengine_prep_slave_single(rxchan,
 795						     as->dma_addr_rx_bbuf,
 796						     xfer->len,
 797						     DMA_DEV_TO_MEM,
 798						     DMA_PREP_INTERRUPT |
 799						     DMA_CTRL_ACK);
 800	} else {
 801		rxdesc = dmaengine_prep_slave_sg(rxchan,
 802						 xfer->rx_sg.sgl,
 803						 xfer->rx_sg.nents,
 804						 DMA_DEV_TO_MEM,
 805						 DMA_PREP_INTERRUPT |
 806						 DMA_CTRL_ACK);
 807	}
 808	if (!rxdesc)
 809		goto err_dma;
 810
 811	if (atmel_spi_is_vmalloc_xfer(xfer) &&
 812	    IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) {
 813		memcpy(as->addr_tx_bbuf, xfer->tx_buf, xfer->len);
 814		txdesc = dmaengine_prep_slave_single(txchan,
 815						     as->dma_addr_tx_bbuf,
 816						     xfer->len, DMA_MEM_TO_DEV,
 817						     DMA_PREP_INTERRUPT |
 818						     DMA_CTRL_ACK);
 819	} else {
 820		txdesc = dmaengine_prep_slave_sg(txchan,
 821						 xfer->tx_sg.sgl,
 822						 xfer->tx_sg.nents,
 823						 DMA_MEM_TO_DEV,
 824						 DMA_PREP_INTERRUPT |
 825						 DMA_CTRL_ACK);
 826	}
 827	if (!txdesc)
 828		goto err_dma;
 829
 830	dev_dbg(host->dev.parent,
 831		"  start dma xfer %p: len %u tx %p/%08llx rx %p/%08llx\n",
 832		xfer, xfer->len, xfer->tx_buf, (unsigned long long)xfer->tx_dma,
 833		xfer->rx_buf, (unsigned long long)xfer->rx_dma);
 834
 835	/* Enable relevant interrupts */
 836	spi_writel(as, IER, SPI_BIT(OVRES));
 837
 838	/* Put the callback on the RX transfer only, that should finish last */
 839	rxdesc->callback = dma_callback;
 840	rxdesc->callback_param = host;
 841
 842	/* Submit and fire RX and TX with TX last so we're ready to read! */
 843	cookie = rxdesc->tx_submit(rxdesc);
 844	if (dma_submit_error(cookie))
 845		goto err_dma;
 846	cookie = txdesc->tx_submit(txdesc);
 847	if (dma_submit_error(cookie))
 848		goto err_dma;
 849	rxchan->device->device_issue_pending(rxchan);
 850	txchan->device->device_issue_pending(txchan);
 851
 852	return 0;
 853
 854err_dma:
 855	spi_writel(as, IDR, SPI_BIT(OVRES));
 856	atmel_spi_stop_dma(host);
 857err_exit:
 858	return -ENOMEM;
 859}
 860
 861static void atmel_spi_next_xfer_data(struct spi_controller *host,
 862				struct spi_transfer *xfer,
 863				dma_addr_t *tx_dma,
 864				dma_addr_t *rx_dma,
 865				u32 *plen)
 866{
 867	*rx_dma = xfer->rx_dma + xfer->len - *plen;
 868	*tx_dma = xfer->tx_dma + xfer->len - *plen;
 869	if (*plen > host->max_dma_len)
 870		*plen = host->max_dma_len;
 871}
 872
 873static int atmel_spi_set_xfer_speed(struct atmel_spi *as,
 874				    struct spi_device *spi,
 875				    struct spi_transfer *xfer)
 876{
 877	u32			scbr, csr;
 878	unsigned long		bus_hz;
 879	int chip_select;
 880
 881	if (spi_get_csgpiod(spi, 0))
 882		chip_select = as->native_cs_for_gpio;
 883	else
 884		chip_select = spi_get_chipselect(spi, 0);
 885
 886	/* v1 chips start out at half the peripheral bus speed. */
 887	bus_hz = as->spi_clk;
 888	if (!atmel_spi_is_v2(as))
 889		bus_hz /= 2;
 890
 891	/*
 892	 * Calculate the lowest divider that satisfies the
 893	 * constraint, assuming div32/fdiv/mbz == 0.
 894	 */
 895	scbr = DIV_ROUND_UP(bus_hz, xfer->speed_hz);
 896
 897	/*
 898	 * If the resulting divider doesn't fit into the
 899	 * register bitfield, we can't satisfy the constraint.
 900	 */
 901	if (scbr >= (1 << SPI_SCBR_SIZE)) {
 902		dev_err(&spi->dev,
 903			"setup: %d Hz too slow, scbr %u; min %ld Hz\n",
 904			xfer->speed_hz, scbr, bus_hz/255);
 905		return -EINVAL;
 906	}
 907	if (scbr == 0) {
 908		dev_err(&spi->dev,
 909			"setup: %d Hz too high, scbr %u; max %ld Hz\n",
 910			xfer->speed_hz, scbr, bus_hz);
 911		return -EINVAL;
 912	}
 913	csr = spi_readl(as, CSR0 + 4 * chip_select);
 914	csr = SPI_BFINS(SCBR, scbr, csr);
 915	spi_writel(as, CSR0 + 4 * chip_select, csr);
 916	xfer->effective_speed_hz = bus_hz / scbr;
 917
 918	return 0;
 919}
 920
 921/*
 922 * Submit next transfer for PDC.
 923 * lock is held, spi irq is blocked
 924 */
 925static void atmel_spi_pdc_next_xfer(struct spi_controller *host,
 926					struct spi_transfer *xfer)
 927{
 928	struct atmel_spi	*as = spi_controller_get_devdata(host);
 929	u32			len;
 930	dma_addr_t		tx_dma, rx_dma;
 931
 932	spi_writel(as, PTCR, SPI_BIT(RXTDIS) | SPI_BIT(TXTDIS));
 933
 934	len = as->current_remaining_bytes;
 935	atmel_spi_next_xfer_data(host, xfer, &tx_dma, &rx_dma, &len);
 936	as->current_remaining_bytes -= len;
 937
 938	spi_writel(as, RPR, rx_dma);
 939	spi_writel(as, TPR, tx_dma);
 940
 941	if (xfer->bits_per_word > 8)
 942		len >>= 1;
 943	spi_writel(as, RCR, len);
 944	spi_writel(as, TCR, len);
 945
 946	dev_dbg(&host->dev,
 947		"  start xfer %p: len %u tx %p/%08llx rx %p/%08llx\n",
 948		xfer, xfer->len, xfer->tx_buf,
 949		(unsigned long long)xfer->tx_dma, xfer->rx_buf,
 950		(unsigned long long)xfer->rx_dma);
 951
 952	if (as->current_remaining_bytes) {
 953		len = as->current_remaining_bytes;
 954		atmel_spi_next_xfer_data(host, xfer, &tx_dma, &rx_dma, &len);
 955		as->current_remaining_bytes -= len;
 956
 957		spi_writel(as, RNPR, rx_dma);
 958		spi_writel(as, TNPR, tx_dma);
 959
 960		if (xfer->bits_per_word > 8)
 961			len >>= 1;
 962		spi_writel(as, RNCR, len);
 963		spi_writel(as, TNCR, len);
 964
 965		dev_dbg(&host->dev,
 966			"  next xfer %p: len %u tx %p/%08llx rx %p/%08llx\n",
 967			xfer, xfer->len, xfer->tx_buf,
 968			(unsigned long long)xfer->tx_dma, xfer->rx_buf,
 969			(unsigned long long)xfer->rx_dma);
 970	}
 971
 972	/* REVISIT: We're waiting for RXBUFF before we start the next
 973	 * transfer because we need to handle some difficult timing
 974	 * issues otherwise. If we wait for TXBUFE in one transfer and
 975	 * then starts waiting for RXBUFF in the next, it's difficult
 976	 * to tell the difference between the RXBUFF interrupt we're
 977	 * actually waiting for and the RXBUFF interrupt of the
 978	 * previous transfer.
 979	 *
 980	 * It should be doable, though. Just not now...
 981	 */
 982	spi_writel(as, IER, SPI_BIT(RXBUFF) | SPI_BIT(OVRES));
 983	spi_writel(as, PTCR, SPI_BIT(TXTEN) | SPI_BIT(RXTEN));
 984}
 985
 986/*
 987 * For DMA, tx_buf/tx_dma have the same relationship as rx_buf/rx_dma:
 988 *  - The buffer is either valid for CPU access, else NULL
 989 *  - If the buffer is valid, so is its DMA address
 
 
 990 */
 991static int
 992atmel_spi_dma_map_xfer(struct atmel_spi *as, struct spi_transfer *xfer)
 993{
 994	struct device	*dev = &as->pdev->dev;
 995
 996	xfer->tx_dma = xfer->rx_dma = INVALID_DMA_ADDRESS;
 997	if (xfer->tx_buf) {
 998		/* tx_buf is a const void* where we need a void * for the dma
 999		 * mapping */
1000		void *nonconst_tx = (void *)xfer->tx_buf;
1001
1002		xfer->tx_dma = dma_map_single(dev,
1003				nonconst_tx, xfer->len,
1004				DMA_TO_DEVICE);
1005		if (dma_mapping_error(dev, xfer->tx_dma))
1006			return -ENOMEM;
1007	}
1008	if (xfer->rx_buf) {
1009		xfer->rx_dma = dma_map_single(dev,
1010				xfer->rx_buf, xfer->len,
1011				DMA_FROM_DEVICE);
1012		if (dma_mapping_error(dev, xfer->rx_dma)) {
1013			if (xfer->tx_buf)
1014				dma_unmap_single(dev,
1015						xfer->tx_dma, xfer->len,
1016						DMA_TO_DEVICE);
1017			return -ENOMEM;
1018		}
1019	}
1020	return 0;
1021}
1022
1023static void atmel_spi_dma_unmap_xfer(struct spi_controller *host,
1024				     struct spi_transfer *xfer)
1025{
1026	if (xfer->tx_dma != INVALID_DMA_ADDRESS)
1027		dma_unmap_single(host->dev.parent, xfer->tx_dma,
1028				 xfer->len, DMA_TO_DEVICE);
1029	if (xfer->rx_dma != INVALID_DMA_ADDRESS)
1030		dma_unmap_single(host->dev.parent, xfer->rx_dma,
1031				 xfer->len, DMA_FROM_DEVICE);
1032}
1033
1034static void atmel_spi_disable_pdc_transfer(struct atmel_spi *as)
1035{
1036	spi_writel(as, PTCR, SPI_BIT(RXTDIS) | SPI_BIT(TXTDIS));
1037}
1038
1039static void
1040atmel_spi_pump_single_data(struct atmel_spi *as, struct spi_transfer *xfer)
1041{
1042	u8		*rxp;
1043	u16		*rxp16;
1044	unsigned long	xfer_pos = xfer->len - as->current_remaining_bytes;
1045
1046	if (xfer->bits_per_word > 8) {
1047		rxp16 = (u16 *)(((u8 *)xfer->rx_buf) + xfer_pos);
1048		*rxp16 = spi_readl(as, RDR);
1049	} else {
1050		rxp = ((u8 *)xfer->rx_buf) + xfer_pos;
1051		*rxp = spi_readl(as, RDR);
1052	}
1053	if (xfer->bits_per_word > 8) {
1054		if (as->current_remaining_bytes > 2)
1055			as->current_remaining_bytes -= 2;
1056		else
1057			as->current_remaining_bytes = 0;
1058	} else {
1059		as->current_remaining_bytes--;
1060	}
1061}
1062
1063static void
1064atmel_spi_pump_fifo_data(struct atmel_spi *as, struct spi_transfer *xfer)
1065{
1066	u32 fifolr = spi_readl(as, FLR);
1067	u32 num_bytes, num_data = SPI_BFEXT(RXFL, fifolr);
1068	u32 offset = xfer->len - as->current_remaining_bytes;
1069	u16 *words = (u16 *)((u8 *)xfer->rx_buf + offset);
1070	u8  *bytes = (u8  *)((u8 *)xfer->rx_buf + offset);
1071	u16 rd; /* RD field is the lowest 16 bits of RDR */
1072
1073	/* Update the number of remaining bytes to transfer */
1074	num_bytes = ((xfer->bits_per_word > 8) ?
1075		     (num_data << 1) :
1076		     num_data);
1077
1078	if (as->current_remaining_bytes > num_bytes)
1079		as->current_remaining_bytes -= num_bytes;
1080	else
1081		as->current_remaining_bytes = 0;
1082
1083	/* Handle odd number of bytes when data are more than 8bit width */
1084	if (xfer->bits_per_word > 8)
1085		as->current_remaining_bytes &= ~0x1;
1086
1087	/* Read data */
1088	while (num_data) {
1089		rd = spi_readl(as, RDR);
1090		if (xfer->bits_per_word > 8)
1091			*words++ = rd;
1092		else
1093			*bytes++ = rd;
1094		num_data--;
1095	}
1096}
1097
1098/* Called from IRQ
1099 *
1100 * Must update "current_remaining_bytes" to keep track of data
1101 * to transfer.
1102 */
1103static void
1104atmel_spi_pump_pio_data(struct atmel_spi *as, struct spi_transfer *xfer)
1105{
1106	if (as->fifo_size)
1107		atmel_spi_pump_fifo_data(as, xfer);
1108	else
1109		atmel_spi_pump_single_data(as, xfer);
1110}
1111
1112/* Interrupt
1113 *
1114 */
1115static irqreturn_t
1116atmel_spi_pio_interrupt(int irq, void *dev_id)
1117{
1118	struct spi_controller	*host = dev_id;
1119	struct atmel_spi	*as = spi_controller_get_devdata(host);
1120	u32			status, pending, imr;
1121	struct spi_transfer	*xfer;
1122	int			ret = IRQ_NONE;
1123
1124	imr = spi_readl(as, IMR);
1125	status = spi_readl(as, SR);
1126	pending = status & imr;
1127
1128	if (pending & SPI_BIT(OVRES)) {
1129		ret = IRQ_HANDLED;
1130		spi_writel(as, IDR, SPI_BIT(OVRES));
1131		dev_warn(host->dev.parent, "overrun\n");
1132
1133		/*
1134		 * When we get an overrun, we disregard the current
1135		 * transfer. Data will not be copied back from any
1136		 * bounce buffer and msg->actual_len will not be
1137		 * updated with the last xfer.
1138		 *
1139		 * We will also not process any remaning transfers in
1140		 * the message.
1141		 */
1142		as->done_status = -EIO;
1143		smp_wmb();
1144
1145		/* Clear any overrun happening while cleaning up */
1146		spi_readl(as, SR);
1147
1148		complete(&as->xfer_completion);
1149
1150	} else if (pending & (SPI_BIT(RDRF) | SPI_BIT(RXFTHF))) {
1151		atmel_spi_lock(as);
1152
1153		if (as->current_remaining_bytes) {
1154			ret = IRQ_HANDLED;
1155			xfer = as->current_transfer;
1156			atmel_spi_pump_pio_data(as, xfer);
1157			if (!as->current_remaining_bytes)
1158				spi_writel(as, IDR, pending);
1159
1160			complete(&as->xfer_completion);
1161		}
1162
1163		atmel_spi_unlock(as);
1164	} else {
1165		WARN_ONCE(pending, "IRQ not handled, pending = %x\n", pending);
1166		ret = IRQ_HANDLED;
1167		spi_writel(as, IDR, pending);
1168	}
1169
1170	return ret;
1171}
1172
1173static irqreturn_t
1174atmel_spi_pdc_interrupt(int irq, void *dev_id)
1175{
1176	struct spi_controller	*host = dev_id;
1177	struct atmel_spi	*as = spi_controller_get_devdata(host);
1178	u32			status, pending, imr;
1179	int			ret = IRQ_NONE;
1180
1181	imr = spi_readl(as, IMR);
1182	status = spi_readl(as, SR);
1183	pending = status & imr;
1184
1185	if (pending & SPI_BIT(OVRES)) {
1186
1187		ret = IRQ_HANDLED;
1188
1189		spi_writel(as, IDR, (SPI_BIT(RXBUFF) | SPI_BIT(ENDRX)
1190				     | SPI_BIT(OVRES)));
1191
1192		/* Clear any overrun happening while cleaning up */
1193		spi_readl(as, SR);
1194
1195		as->done_status = -EIO;
1196
1197		complete(&as->xfer_completion);
1198
1199	} else if (pending & (SPI_BIT(RXBUFF) | SPI_BIT(ENDRX))) {
1200		ret = IRQ_HANDLED;
1201
1202		spi_writel(as, IDR, pending);
1203
1204		complete(&as->xfer_completion);
1205	}
1206
1207	return ret;
1208}
1209
1210static int atmel_word_delay_csr(struct spi_device *spi, struct atmel_spi *as)
1211{
1212	struct spi_delay *delay = &spi->word_delay;
1213	u32 value = delay->value;
1214
1215	switch (delay->unit) {
1216	case SPI_DELAY_UNIT_NSECS:
1217		value /= 1000;
1218		break;
1219	case SPI_DELAY_UNIT_USECS:
1220		break;
1221	default:
1222		return -EINVAL;
1223	}
1224
1225	return (as->spi_clk / 1000000 * value) >> 5;
1226}
1227
1228static void initialize_native_cs_for_gpio(struct atmel_spi *as)
1229{
1230	int i;
1231	struct spi_controller *host = platform_get_drvdata(as->pdev);
1232
1233	if (!as->native_cs_free)
1234		return; /* already initialized */
1235
1236	if (!host->cs_gpiods)
1237		return; /* No CS GPIO */
1238
1239	/*
1240	 * On the first version of the controller (AT91RM9200), CS0
1241	 * can't be used associated with GPIO
1242	 */
1243	if (atmel_spi_is_v2(as))
1244		i = 0;
1245	else
1246		i = 1;
1247
1248	for (; i < 4; i++)
1249		if (host->cs_gpiods[i])
1250			as->native_cs_free |= BIT(i);
1251
1252	if (as->native_cs_free)
1253		as->native_cs_for_gpio = ffs(as->native_cs_free);
1254}
1255
1256static int atmel_spi_setup(struct spi_device *spi)
1257{
1258	struct atmel_spi	*as;
1259	struct atmel_spi_device	*asd;
1260	u32			csr;
1261	unsigned int		bits = spi->bits_per_word;
1262	int chip_select;
1263	int			word_delay_csr;
1264
1265	as = spi_controller_get_devdata(spi->controller);
1266
1267	/* see notes above re chipselect */
1268	if (!spi_get_csgpiod(spi, 0) && (spi->mode & SPI_CS_HIGH)) {
1269		dev_warn(&spi->dev, "setup: non GPIO CS can't be active-high\n");
1270		return -EINVAL;
1271	}
1272
1273	/* Setup() is called during spi_register_controller(aka
1274	 * spi_register_master) but after all membmers of the cs_gpiod
1275	 * array have been filled, so we can looked for which native
1276	 * CS will be free for using with GPIO
1277	 */
1278	initialize_native_cs_for_gpio(as);
1279
1280	if (spi_get_csgpiod(spi, 0) && as->native_cs_free) {
1281		dev_err(&spi->dev,
1282			"No native CS available to support this GPIO CS\n");
1283		return -EBUSY;
1284	}
1285
1286	if (spi_get_csgpiod(spi, 0))
1287		chip_select = as->native_cs_for_gpio;
1288	else
1289		chip_select = spi_get_chipselect(spi, 0);
1290
1291	csr = SPI_BF(BITS, bits - 8);
1292	if (spi->mode & SPI_CPOL)
1293		csr |= SPI_BIT(CPOL);
1294	if (!(spi->mode & SPI_CPHA))
1295		csr |= SPI_BIT(NCPHA);
1296
1297	if (!spi_get_csgpiod(spi, 0))
1298		csr |= SPI_BIT(CSAAT);
1299	csr |= SPI_BF(DLYBS, 0);
1300
1301	word_delay_csr = atmel_word_delay_csr(spi, as);
1302	if (word_delay_csr < 0)
1303		return word_delay_csr;
1304
1305	/* DLYBCT adds delays between words.  This is useful for slow devices
1306	 * that need a bit of time to setup the next transfer.
1307	 */
1308	csr |= SPI_BF(DLYBCT, word_delay_csr);
1309
1310	asd = spi->controller_state;
1311	if (!asd) {
1312		asd = kzalloc(sizeof(struct atmel_spi_device), GFP_KERNEL);
1313		if (!asd)
1314			return -ENOMEM;
1315
1316		spi->controller_state = asd;
1317	}
1318
1319	asd->csr = csr;
1320
1321	dev_dbg(&spi->dev,
1322		"setup: bpw %u mode 0x%x -> csr%d %08x\n",
1323		bits, spi->mode, spi_get_chipselect(spi, 0), csr);
1324
1325	if (!atmel_spi_is_v2(as))
1326		spi_writel(as, CSR0 + 4 * chip_select, csr);
1327
1328	return 0;
1329}
1330
1331static void atmel_spi_set_cs(struct spi_device *spi, bool enable)
1332{
1333	struct atmel_spi *as = spi_controller_get_devdata(spi->controller);
1334	/* the core doesn't really pass us enable/disable, but CS HIGH vs CS LOW
1335	 * since we already have routines for activate/deactivate translate
1336	 * high/low to active/inactive
1337	 */
1338	enable = (!!(spi->mode & SPI_CS_HIGH) == enable);
1339
1340	if (enable) {
1341		cs_activate(as, spi);
1342	} else {
1343		cs_deactivate(as, spi);
1344	}
1345
1346}
1347
1348static int atmel_spi_one_transfer(struct spi_controller *host,
1349					struct spi_device *spi,
1350					struct spi_transfer *xfer)
1351{
1352	struct atmel_spi	*as;
1353	u8			bits;
1354	u32			len;
1355	struct atmel_spi_device	*asd;
1356	int			timeout;
1357	int			ret;
1358	unsigned int		dma_timeout;
1359	long			ret_timeout;
1360
1361	as = spi_controller_get_devdata(host);
1362
1363	asd = spi->controller_state;
1364	bits = (asd->csr >> 4) & 0xf;
1365	if (bits != xfer->bits_per_word - 8) {
1366		dev_dbg(&spi->dev,
1367			"you can't yet change bits_per_word in transfers\n");
1368		return -ENOPROTOOPT;
1369	}
1370
1371	/*
1372	 * DMA map early, for performance (empties dcache ASAP) and
1373	 * better fault reporting.
1374	 */
1375	if (as->use_pdc) {
 
1376		if (atmel_spi_dma_map_xfer(as, xfer) < 0)
1377			return -ENOMEM;
1378	}
1379
1380	atmel_spi_set_xfer_speed(as, spi, xfer);
1381
1382	as->done_status = 0;
1383	as->current_transfer = xfer;
1384	as->current_remaining_bytes = xfer->len;
1385	while (as->current_remaining_bytes) {
1386		reinit_completion(&as->xfer_completion);
1387
1388		if (as->use_pdc) {
1389			atmel_spi_lock(as);
1390			atmel_spi_pdc_next_xfer(host, xfer);
1391			atmel_spi_unlock(as);
1392		} else if (atmel_spi_use_dma(as, xfer)) {
1393			len = as->current_remaining_bytes;
1394			ret = atmel_spi_next_xfer_dma_submit(host,
1395								xfer, &len);
1396			if (ret) {
1397				dev_err(&spi->dev,
1398					"unable to use DMA, fallback to PIO\n");
1399				as->done_status = ret;
1400				break;
1401			} else {
1402				as->current_remaining_bytes -= len;
1403				if (as->current_remaining_bytes < 0)
1404					as->current_remaining_bytes = 0;
1405			}
1406		} else {
1407			atmel_spi_lock(as);
1408			atmel_spi_next_xfer_pio(host, xfer);
1409			atmel_spi_unlock(as);
1410		}
1411
1412		dma_timeout = msecs_to_jiffies(spi_controller_xfer_timeout(host, xfer));
1413		ret_timeout = wait_for_completion_timeout(&as->xfer_completion, dma_timeout);
1414		if (!ret_timeout) {
1415			dev_err(&spi->dev, "spi transfer timeout\n");
1416			as->done_status = -EIO;
1417		}
1418
1419		if (as->done_status)
1420			break;
1421	}
1422
1423	if (as->done_status) {
1424		if (as->use_pdc) {
1425			dev_warn(host->dev.parent,
1426				"overrun (%u/%u remaining)\n",
1427				spi_readl(as, TCR), spi_readl(as, RCR));
1428
1429			/*
1430			 * Clean up DMA registers and make sure the data
1431			 * registers are empty.
1432			 */
1433			spi_writel(as, RNCR, 0);
1434			spi_writel(as, TNCR, 0);
1435			spi_writel(as, RCR, 0);
1436			spi_writel(as, TCR, 0);
1437			for (timeout = 1000; timeout; timeout--)
1438				if (spi_readl(as, SR) & SPI_BIT(TXEMPTY))
1439					break;
1440			if (!timeout)
1441				dev_warn(host->dev.parent,
1442					 "timeout waiting for TXEMPTY");
1443			while (spi_readl(as, SR) & SPI_BIT(RDRF))
1444				spi_readl(as, RDR);
1445
1446			/* Clear any overrun happening while cleaning up */
1447			spi_readl(as, SR);
1448
1449		} else if (atmel_spi_use_dma(as, xfer)) {
1450			atmel_spi_stop_dma(host);
1451		}
1452	}
1453
1454	if (as->use_pdc)
 
1455		atmel_spi_dma_unmap_xfer(host, xfer);
1456
1457	if (as->use_pdc)
1458		atmel_spi_disable_pdc_transfer(as);
1459
1460	return as->done_status;
1461}
1462
1463static void atmel_spi_cleanup(struct spi_device *spi)
1464{
1465	struct atmel_spi_device	*asd = spi->controller_state;
1466
1467	if (!asd)
1468		return;
1469
1470	spi->controller_state = NULL;
1471	kfree(asd);
1472}
1473
1474static inline unsigned int atmel_get_version(struct atmel_spi *as)
1475{
1476	return spi_readl(as, VERSION) & 0x00000fff;
1477}
1478
1479static void atmel_get_caps(struct atmel_spi *as)
1480{
1481	unsigned int version;
1482
1483	version = atmel_get_version(as);
1484
1485	as->caps.is_spi2 = version > 0x121;
1486	as->caps.has_wdrbt = version >= 0x210;
1487	as->caps.has_dma_support = version >= 0x212;
1488	as->caps.has_pdc_support = version < 0x212;
1489}
1490
1491static void atmel_spi_init(struct atmel_spi *as)
1492{
1493	spi_writel(as, CR, SPI_BIT(SWRST));
1494	spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */
1495
1496	/* It is recommended to enable FIFOs first thing after reset */
1497	if (as->fifo_size)
1498		spi_writel(as, CR, SPI_BIT(FIFOEN));
1499
1500	if (as->caps.has_wdrbt) {
1501		spi_writel(as, MR, SPI_BIT(WDRBT) | SPI_BIT(MODFDIS)
1502				| SPI_BIT(MSTR));
1503	} else {
1504		spi_writel(as, MR, SPI_BIT(MSTR) | SPI_BIT(MODFDIS));
1505	}
1506
1507	if (as->use_pdc)
1508		spi_writel(as, PTCR, SPI_BIT(RXTDIS) | SPI_BIT(TXTDIS));
1509	spi_writel(as, CR, SPI_BIT(SPIEN));
1510}
1511
1512static int atmel_spi_probe(struct platform_device *pdev)
1513{
1514	struct resource		*regs;
1515	int			irq;
1516	struct clk		*clk;
1517	int			ret;
1518	struct spi_controller	*host;
1519	struct atmel_spi	*as;
1520
1521	/* Select default pin state */
1522	pinctrl_pm_select_default_state(&pdev->dev);
1523
1524	irq = platform_get_irq(pdev, 0);
1525	if (irq < 0)
1526		return irq;
1527
1528	clk = devm_clk_get(&pdev->dev, "spi_clk");
1529	if (IS_ERR(clk))
1530		return PTR_ERR(clk);
1531
1532	/* setup spi core then atmel-specific driver state */
1533	host = spi_alloc_host(&pdev->dev, sizeof(*as));
1534	if (!host)
1535		return -ENOMEM;
1536
1537	/* the spi->mode bits understood by this driver: */
1538	host->use_gpio_descriptors = true;
1539	host->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;
1540	host->bits_per_word_mask = SPI_BPW_RANGE_MASK(8, 16);
1541	host->dev.of_node = pdev->dev.of_node;
1542	host->bus_num = pdev->id;
1543	host->num_chipselect = 4;
1544	host->setup = atmel_spi_setup;
1545	host->flags = (SPI_CONTROLLER_MUST_RX | SPI_CONTROLLER_MUST_TX |
1546			SPI_CONTROLLER_GPIO_SS);
1547	host->transfer_one = atmel_spi_one_transfer;
1548	host->set_cs = atmel_spi_set_cs;
1549	host->cleanup = atmel_spi_cleanup;
1550	host->auto_runtime_pm = true;
1551	host->max_dma_len = SPI_MAX_DMA_XFER;
1552	host->can_dma = atmel_spi_can_dma;
1553	platform_set_drvdata(pdev, host);
1554
1555	as = spi_controller_get_devdata(host);
1556
1557	spin_lock_init(&as->lock);
1558
1559	as->pdev = pdev;
1560	as->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &regs);
1561	if (IS_ERR(as->regs)) {
1562		ret = PTR_ERR(as->regs);
1563		goto out_unmap_regs;
1564	}
1565	as->phybase = regs->start;
1566	as->irq = irq;
1567	as->clk = clk;
1568
1569	init_completion(&as->xfer_completion);
1570
1571	atmel_get_caps(as);
1572
1573	as->use_dma = false;
1574	as->use_pdc = false;
1575	if (as->caps.has_dma_support) {
1576		ret = atmel_spi_configure_dma(host, as);
1577		if (ret == 0) {
1578			as->use_dma = true;
1579		} else if (ret == -EPROBE_DEFER) {
1580			goto out_unmap_regs;
1581		}
1582	} else if (as->caps.has_pdc_support) {
1583		as->use_pdc = true;
1584	}
1585
1586	if (IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) {
1587		as->addr_rx_bbuf = dma_alloc_coherent(&pdev->dev,
1588						      SPI_MAX_DMA_XFER,
1589						      &as->dma_addr_rx_bbuf,
1590						      GFP_KERNEL | GFP_DMA);
1591		if (!as->addr_rx_bbuf) {
1592			as->use_dma = false;
1593		} else {
1594			as->addr_tx_bbuf = dma_alloc_coherent(&pdev->dev,
1595					SPI_MAX_DMA_XFER,
1596					&as->dma_addr_tx_bbuf,
1597					GFP_KERNEL | GFP_DMA);
1598			if (!as->addr_tx_bbuf) {
1599				as->use_dma = false;
1600				dma_free_coherent(&pdev->dev, SPI_MAX_DMA_XFER,
1601						  as->addr_rx_bbuf,
1602						  as->dma_addr_rx_bbuf);
1603			}
1604		}
1605		if (!as->use_dma)
1606			dev_info(host->dev.parent,
1607				 "  can not allocate dma coherent memory\n");
1608	}
1609
1610	if (as->caps.has_dma_support && !as->use_dma)
1611		dev_info(&pdev->dev, "Atmel SPI Controller using PIO only\n");
1612
1613	if (as->use_pdc) {
1614		ret = devm_request_irq(&pdev->dev, irq, atmel_spi_pdc_interrupt,
1615					0, dev_name(&pdev->dev), host);
1616	} else {
1617		ret = devm_request_irq(&pdev->dev, irq, atmel_spi_pio_interrupt,
1618					0, dev_name(&pdev->dev), host);
1619	}
1620	if (ret)
1621		goto out_unmap_regs;
1622
1623	/* Initialize the hardware */
1624	ret = clk_prepare_enable(clk);
1625	if (ret)
1626		goto out_free_irq;
1627
1628	as->spi_clk = clk_get_rate(clk);
1629
1630	as->fifo_size = 0;
1631	if (!of_property_read_u32(pdev->dev.of_node, "atmel,fifo-size",
1632				  &as->fifo_size)) {
1633		dev_info(&pdev->dev, "Using FIFO (%u data)\n", as->fifo_size);
1634	}
1635
1636	atmel_spi_init(as);
1637
1638	pm_runtime_set_autosuspend_delay(&pdev->dev, AUTOSUSPEND_TIMEOUT);
1639	pm_runtime_use_autosuspend(&pdev->dev);
1640	pm_runtime_set_active(&pdev->dev);
1641	pm_runtime_enable(&pdev->dev);
1642
1643	ret = devm_spi_register_controller(&pdev->dev, host);
1644	if (ret)
1645		goto out_free_dma;
1646
1647	/* go! */
1648	dev_info(&pdev->dev, "Atmel SPI Controller version 0x%x at 0x%08lx (irq %d)\n",
1649			atmel_get_version(as), (unsigned long)regs->start,
1650			irq);
1651
1652	return 0;
1653
1654out_free_dma:
1655	pm_runtime_disable(&pdev->dev);
1656	pm_runtime_set_suspended(&pdev->dev);
1657
1658	if (as->use_dma)
1659		atmel_spi_release_dma(host);
1660
1661	spi_writel(as, CR, SPI_BIT(SWRST));
1662	spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */
1663	clk_disable_unprepare(clk);
1664out_free_irq:
1665out_unmap_regs:
1666	spi_controller_put(host);
1667	return ret;
1668}
1669
1670static void atmel_spi_remove(struct platform_device *pdev)
1671{
1672	struct spi_controller	*host = platform_get_drvdata(pdev);
1673	struct atmel_spi	*as = spi_controller_get_devdata(host);
1674
1675	pm_runtime_get_sync(&pdev->dev);
1676
1677	/* reset the hardware and block queue progress */
1678	if (as->use_dma) {
1679		atmel_spi_stop_dma(host);
1680		atmel_spi_release_dma(host);
1681		if (IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) {
1682			dma_free_coherent(&pdev->dev, SPI_MAX_DMA_XFER,
1683					  as->addr_tx_bbuf,
1684					  as->dma_addr_tx_bbuf);
1685			dma_free_coherent(&pdev->dev, SPI_MAX_DMA_XFER,
1686					  as->addr_rx_bbuf,
1687					  as->dma_addr_rx_bbuf);
1688		}
1689	}
1690
1691	spin_lock_irq(&as->lock);
1692	spi_writel(as, CR, SPI_BIT(SWRST));
1693	spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */
1694	spi_readl(as, SR);
1695	spin_unlock_irq(&as->lock);
1696
1697	clk_disable_unprepare(as->clk);
1698
1699	pm_runtime_put_noidle(&pdev->dev);
1700	pm_runtime_disable(&pdev->dev);
1701}
1702
1703static int atmel_spi_runtime_suspend(struct device *dev)
1704{
1705	struct spi_controller *host = dev_get_drvdata(dev);
1706	struct atmel_spi *as = spi_controller_get_devdata(host);
1707
1708	clk_disable_unprepare(as->clk);
1709	pinctrl_pm_select_sleep_state(dev);
1710
1711	return 0;
1712}
1713
1714static int atmel_spi_runtime_resume(struct device *dev)
1715{
1716	struct spi_controller *host = dev_get_drvdata(dev);
1717	struct atmel_spi *as = spi_controller_get_devdata(host);
1718
1719	pinctrl_pm_select_default_state(dev);
1720
1721	return clk_prepare_enable(as->clk);
1722}
1723
1724static int atmel_spi_suspend(struct device *dev)
1725{
1726	struct spi_controller *host = dev_get_drvdata(dev);
1727	int ret;
1728
1729	/* Stop the queue running */
1730	ret = spi_controller_suspend(host);
1731	if (ret)
1732		return ret;
1733
1734	if (!pm_runtime_suspended(dev))
1735		atmel_spi_runtime_suspend(dev);
1736
1737	return 0;
1738}
1739
1740static int atmel_spi_resume(struct device *dev)
1741{
1742	struct spi_controller *host = dev_get_drvdata(dev);
1743	struct atmel_spi *as = spi_controller_get_devdata(host);
1744	int ret;
1745
1746	ret = clk_prepare_enable(as->clk);
1747	if (ret)
1748		return ret;
1749
1750	atmel_spi_init(as);
1751
1752	clk_disable_unprepare(as->clk);
1753
1754	if (!pm_runtime_suspended(dev)) {
1755		ret = atmel_spi_runtime_resume(dev);
1756		if (ret)
1757			return ret;
1758	}
1759
1760	/* Start the queue running */
1761	return spi_controller_resume(host);
1762}
1763
1764static const struct dev_pm_ops atmel_spi_pm_ops = {
1765	SYSTEM_SLEEP_PM_OPS(atmel_spi_suspend, atmel_spi_resume)
1766	RUNTIME_PM_OPS(atmel_spi_runtime_suspend,
1767		       atmel_spi_runtime_resume, NULL)
1768};
1769
1770static const struct of_device_id atmel_spi_dt_ids[] = {
1771	{ .compatible = "atmel,at91rm9200-spi" },
1772	{ /* sentinel */ }
1773};
1774
1775MODULE_DEVICE_TABLE(of, atmel_spi_dt_ids);
1776
1777static struct platform_driver atmel_spi_driver = {
1778	.driver		= {
1779		.name	= "atmel_spi",
1780		.pm	= pm_ptr(&atmel_spi_pm_ops),
1781		.of_match_table	= atmel_spi_dt_ids,
1782	},
1783	.probe		= atmel_spi_probe,
1784	.remove		= atmel_spi_remove,
1785};
1786module_platform_driver(atmel_spi_driver);
1787
1788MODULE_DESCRIPTION("Atmel AT32/AT91 SPI Controller driver");
1789MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1790MODULE_LICENSE("GPL");
1791MODULE_ALIAS("platform:atmel_spi");