Linux Audio

Check our new training course

Open-source upstreaming

Need help get the support for your hardware in upstream Linux?
Loading...
   1// SPDX-License-Identifier: GPL-2.0-only
   2/* Copyright (c) 2020, Broadcom */
   3/*
   4 * 8250-core based driver for Broadcom ns16550a UARTs
   5 *
   6 * This driver uses the standard 8250 driver core but adds additional
   7 * optional features including the ability to use a baud rate clock
   8 * mux for more accurate high speed baud rate selection and also
   9 * an optional DMA engine.
  10 *
  11 */
  12
  13#include <linux/module.h>
  14#include <linux/types.h>
  15#include <linux/tty.h>
  16#include <linux/errno.h>
  17#include <linux/device.h>
  18#include <linux/io.h>
  19#include <linux/of.h>
  20#include <linux/dma-mapping.h>
  21#include <linux/tty_flip.h>
  22#include <linux/delay.h>
  23#include <linux/clk.h>
  24#include <linux/debugfs.h>
  25#include <linux/units.h>
  26
  27#include "8250.h"
  28
  29/* Register definitions for UART DMA block. Version 1.1 or later. */
  30#define UDMA_ARB_RX		0x00
  31#define UDMA_ARB_TX		0x04
  32#define		UDMA_ARB_REQ				0x00000001
  33#define		UDMA_ARB_GRANT				0x00000002
  34
  35#define UDMA_RX_REVISION	0x00
  36#define UDMA_RX_REVISION_REQUIRED			0x00000101
  37#define UDMA_RX_CTRL		0x04
  38#define		UDMA_RX_CTRL_BUF_CLOSE_MODE		0x00010000
  39#define		UDMA_RX_CTRL_MASK_WR_DONE		0x00008000
  40#define		UDMA_RX_CTRL_ENDIAN_OVERRIDE		0x00004000
  41#define		UDMA_RX_CTRL_ENDIAN			0x00002000
  42#define		UDMA_RX_CTRL_OE_IS_ERR			0x00001000
  43#define		UDMA_RX_CTRL_PE_IS_ERR			0x00000800
  44#define		UDMA_RX_CTRL_FE_IS_ERR			0x00000400
  45#define		UDMA_RX_CTRL_NUM_BUF_USED_MASK		0x000003c0
  46#define		UDMA_RX_CTRL_NUM_BUF_USED_SHIFT	6
  47#define		UDMA_RX_CTRL_BUF_CLOSE_CLK_SEL_SYS	0x00000020
  48#define		UDMA_RX_CTRL_BUF_CLOSE_ENA		0x00000010
  49#define		UDMA_RX_CTRL_TIMEOUT_CLK_SEL_SYS	0x00000008
  50#define		UDMA_RX_CTRL_TIMEOUT_ENA		0x00000004
  51#define		UDMA_RX_CTRL_ABORT			0x00000002
  52#define		UDMA_RX_CTRL_ENA			0x00000001
  53#define UDMA_RX_STATUS		0x08
  54#define		UDMA_RX_STATUS_ACTIVE_BUF_MASK		0x0000000f
  55#define UDMA_RX_TRANSFER_LEN	0x0c
  56#define UDMA_RX_TRANSFER_TOTAL	0x10
  57#define UDMA_RX_BUFFER_SIZE	0x14
  58#define UDMA_RX_SRC_ADDR	0x18
  59#define UDMA_RX_TIMEOUT		0x1c
  60#define UDMA_RX_BUFFER_CLOSE	0x20
  61#define UDMA_RX_BLOCKOUT_COUNTER 0x24
  62#define UDMA_RX_BUF0_PTR_LO	0x28
  63#define UDMA_RX_BUF0_PTR_HI	0x2c
  64#define UDMA_RX_BUF0_STATUS	0x30
  65#define		UDMA_RX_BUFX_STATUS_OVERRUN_ERR		0x00000010
  66#define		UDMA_RX_BUFX_STATUS_FRAME_ERR		0x00000008
  67#define		UDMA_RX_BUFX_STATUS_PARITY_ERR		0x00000004
  68#define		UDMA_RX_BUFX_STATUS_CLOSE_EXPIRED	0x00000002
  69#define		UDMA_RX_BUFX_STATUS_DATA_RDY		0x00000001
  70#define UDMA_RX_BUF0_DATA_LEN	0x34
  71#define UDMA_RX_BUF1_PTR_LO	0x38
  72#define UDMA_RX_BUF1_PTR_HI	0x3c
  73#define UDMA_RX_BUF1_STATUS	0x40
  74#define UDMA_RX_BUF1_DATA_LEN	0x44
  75
  76#define UDMA_TX_REVISION	0x00
  77#define UDMA_TX_REVISION_REQUIRED			0x00000101
  78#define UDMA_TX_CTRL		0x04
  79#define		UDMA_TX_CTRL_ENDIAN_OVERRIDE		0x00000080
  80#define		UDMA_TX_CTRL_ENDIAN			0x00000040
  81#define		UDMA_TX_CTRL_NUM_BUF_USED_MASK		0x00000030
  82#define		UDMA_TX_CTRL_NUM_BUF_USED_1		0x00000010
  83#define		UDMA_TX_CTRL_ABORT			0x00000002
  84#define		UDMA_TX_CTRL_ENA			0x00000001
  85#define UDMA_TX_DST_ADDR	0x08
  86#define UDMA_TX_BLOCKOUT_COUNTER 0x10
  87#define UDMA_TX_TRANSFER_LEN	0x14
  88#define UDMA_TX_TRANSFER_TOTAL	0x18
  89#define UDMA_TX_STATUS		0x20
  90#define UDMA_TX_BUF0_PTR_LO	0x24
  91#define UDMA_TX_BUF0_PTR_HI	0x28
  92#define UDMA_TX_BUF0_STATUS	0x2c
  93#define		UDMA_TX_BUFX_LAST			0x00000002
  94#define		UDMA_TX_BUFX_EMPTY			0x00000001
  95#define UDMA_TX_BUF0_DATA_LEN	0x30
  96#define UDMA_TX_BUF0_DATA_SENT	0x34
  97#define UDMA_TX_BUF1_PTR_LO	0x38
  98
  99#define UDMA_INTR_STATUS	0x00
 100#define		UDMA_INTR_ARB_TX_GRANT			0x00040000
 101#define		UDMA_INTR_ARB_RX_GRANT			0x00020000
 102#define		UDMA_INTR_TX_ALL_EMPTY			0x00010000
 103#define		UDMA_INTR_TX_EMPTY_BUF1			0x00008000
 104#define		UDMA_INTR_TX_EMPTY_BUF0			0x00004000
 105#define		UDMA_INTR_TX_ABORT			0x00002000
 106#define		UDMA_INTR_TX_DONE			0x00001000
 107#define		UDMA_INTR_RX_ERROR			0x00000800
 108#define		UDMA_INTR_RX_TIMEOUT			0x00000400
 109#define		UDMA_INTR_RX_READY_BUF7			0x00000200
 110#define		UDMA_INTR_RX_READY_BUF6			0x00000100
 111#define		UDMA_INTR_RX_READY_BUF5			0x00000080
 112#define		UDMA_INTR_RX_READY_BUF4			0x00000040
 113#define		UDMA_INTR_RX_READY_BUF3			0x00000020
 114#define		UDMA_INTR_RX_READY_BUF2			0x00000010
 115#define		UDMA_INTR_RX_READY_BUF1			0x00000008
 116#define		UDMA_INTR_RX_READY_BUF0			0x00000004
 117#define		UDMA_INTR_RX_READY_MASK			0x000003fc
 118#define		UDMA_INTR_RX_READY_SHIFT		2
 119#define		UDMA_INTR_RX_ABORT			0x00000002
 120#define		UDMA_INTR_RX_DONE			0x00000001
 121#define UDMA_INTR_SET		0x04
 122#define UDMA_INTR_CLEAR		0x08
 123#define UDMA_INTR_MASK_STATUS	0x0c
 124#define UDMA_INTR_MASK_SET	0x10
 125#define UDMA_INTR_MASK_CLEAR	0x14
 126
 127
 128#define UDMA_RX_INTERRUPTS ( \
 129	UDMA_INTR_RX_ERROR | \
 130	UDMA_INTR_RX_TIMEOUT | \
 131	UDMA_INTR_RX_READY_BUF0 | \
 132	UDMA_INTR_RX_READY_BUF1 | \
 133	UDMA_INTR_RX_READY_BUF2 | \
 134	UDMA_INTR_RX_READY_BUF3 | \
 135	UDMA_INTR_RX_READY_BUF4 | \
 136	UDMA_INTR_RX_READY_BUF5 | \
 137	UDMA_INTR_RX_READY_BUF6 | \
 138	UDMA_INTR_RX_READY_BUF7 | \
 139	UDMA_INTR_RX_ABORT | \
 140	UDMA_INTR_RX_DONE)
 141
 142#define UDMA_RX_ERR_INTERRUPTS ( \
 143	UDMA_INTR_RX_ERROR | \
 144	UDMA_INTR_RX_TIMEOUT | \
 145	UDMA_INTR_RX_ABORT | \
 146	UDMA_INTR_RX_DONE)
 147
 148#define UDMA_TX_INTERRUPTS ( \
 149	UDMA_INTR_TX_ABORT | \
 150	UDMA_INTR_TX_DONE)
 151
 152#define UDMA_IS_RX_INTERRUPT(status) ((status) & UDMA_RX_INTERRUPTS)
 153#define UDMA_IS_TX_INTERRUPT(status) ((status) & UDMA_TX_INTERRUPTS)
 154
 155
 156/* Current devices have 8 sets of RX buffer registers */
 157#define UDMA_RX_BUFS_COUNT	8
 158#define UDMA_RX_BUFS_REG_OFFSET (UDMA_RX_BUF1_PTR_LO - UDMA_RX_BUF0_PTR_LO)
 159#define UDMA_RX_BUFx_PTR_LO(x)	(UDMA_RX_BUF0_PTR_LO + \
 160				 ((x) * UDMA_RX_BUFS_REG_OFFSET))
 161#define UDMA_RX_BUFx_PTR_HI(x)	(UDMA_RX_BUF0_PTR_HI + \
 162				 ((x) * UDMA_RX_BUFS_REG_OFFSET))
 163#define UDMA_RX_BUFx_STATUS(x)	(UDMA_RX_BUF0_STATUS + \
 164				 ((x) * UDMA_RX_BUFS_REG_OFFSET))
 165#define UDMA_RX_BUFx_DATA_LEN(x) (UDMA_RX_BUF0_DATA_LEN + \
 166				  ((x) * UDMA_RX_BUFS_REG_OFFSET))
 167
 168/* Current devices have 2 sets of TX buffer registers */
 169#define UDMA_TX_BUFS_COUNT	2
 170#define UDMA_TX_BUFS_REG_OFFSET (UDMA_TX_BUF1_PTR_LO - UDMA_TX_BUF0_PTR_LO)
 171#define UDMA_TX_BUFx_PTR_LO(x)	(UDMA_TX_BUF0_PTR_LO + \
 172				 ((x) * UDMA_TX_BUFS_REG_OFFSET))
 173#define UDMA_TX_BUFx_PTR_HI(x)	(UDMA_TX_BUF0_PTR_HI + \
 174				 ((x) * UDMA_TX_BUFS_REG_OFFSET))
 175#define UDMA_TX_BUFx_STATUS(x)	(UDMA_TX_BUF0_STATUS + \
 176				 ((x) * UDMA_TX_BUFS_REG_OFFSET))
 177#define UDMA_TX_BUFx_DATA_LEN(x) (UDMA_TX_BUF0_DATA_LEN + \
 178				  ((x) * UDMA_TX_BUFS_REG_OFFSET))
 179#define UDMA_TX_BUFx_DATA_SENT(x) (UDMA_TX_BUF0_DATA_SENT + \
 180				   ((x) * UDMA_TX_BUFS_REG_OFFSET))
 181#define REGS_8250 0
 182#define REGS_DMA_RX 1
 183#define REGS_DMA_TX 2
 184#define REGS_DMA_ISR 3
 185#define REGS_DMA_ARB 4
 186#define REGS_MAX 5
 187
 188#define TX_BUF_SIZE 4096
 189#define RX_BUF_SIZE 4096
 190#define RX_BUFS_COUNT 2
 191
 192static const u32 brcmstb_rate_table[] = {
 193	81 * HZ_PER_MHZ,
 194	108 * HZ_PER_MHZ,
 195	64 * HZ_PER_MHZ,		/* Actually 64285715 for some chips */
 196	48 * HZ_PER_MHZ,
 197};
 198
 199static const u32 brcmstb_rate_table_7278[] = {
 200	81 * HZ_PER_MHZ,
 201	108 * HZ_PER_MHZ,
 202	0,
 203	48 * HZ_PER_MHZ,
 204};
 205
 206struct brcmuart_priv {
 207	int		line;
 208	struct clk	*baud_mux_clk;
 209	unsigned long	default_mux_rate;
 210	u32		real_rates[ARRAY_SIZE(brcmstb_rate_table)];
 211	const u32	*rate_table;
 212	ktime_t		char_wait;
 213	struct uart_port *up;
 214	struct hrtimer	hrt;
 215	bool		shutdown;
 216	bool		dma_enabled;
 217	struct uart_8250_dma dma;
 218	void __iomem	*regs[REGS_MAX];
 219	dma_addr_t	rx_addr;
 220	void		*rx_bufs;
 221	size_t		rx_size;
 222	int		rx_next_buf;
 223	dma_addr_t	tx_addr;
 224	void		*tx_buf;
 225	size_t		tx_size;
 226	bool		tx_running;
 227	bool		rx_running;
 228	struct dentry	*debugfs_dir;
 229
 230	/* stats exposed through debugfs */
 231	u64		dma_rx_partial_buf;
 232	u64		dma_rx_full_buf;
 233	u32		rx_bad_timeout_late_char;
 234	u32		rx_bad_timeout_no_char;
 235	u32		rx_missing_close_timeout;
 236	u32		rx_err;
 237	u32		rx_timeout;
 238	u32		rx_abort;
 239	u32		saved_mctrl;
 240};
 241
 242static struct dentry *brcmuart_debugfs_root;
 243
 244/*
 245 * Register access routines
 246 */
 247static u32 udma_readl(struct brcmuart_priv *priv,
 248		int reg_type, int offset)
 249{
 250	return readl(priv->regs[reg_type] + offset);
 251}
 252
 253static void udma_writel(struct brcmuart_priv *priv,
 254			int reg_type, int offset, u32 value)
 255{
 256	writel(value, priv->regs[reg_type] + offset);
 257}
 258
 259static void udma_set(struct brcmuart_priv *priv,
 260		int reg_type, int offset, u32 bits)
 261{
 262	void __iomem *reg = priv->regs[reg_type] + offset;
 263	u32 value;
 264
 265	value = readl(reg);
 266	value |= bits;
 267	writel(value, reg);
 268}
 269
 270static void udma_unset(struct brcmuart_priv *priv,
 271		int reg_type, int offset, u32 bits)
 272{
 273	void __iomem *reg = priv->regs[reg_type] + offset;
 274	u32 value;
 275
 276	value = readl(reg);
 277	value &= ~bits;
 278	writel(value, reg);
 279}
 280
 281/*
 282 * The UART DMA engine hardware can be used by multiple UARTS, but
 283 * only one at a time. Sharing is not currently supported so
 284 * the first UART to request the DMA engine will get it and any
 285 * subsequent requests by other UARTS will fail.
 286 */
 287static int brcmuart_arbitration(struct brcmuart_priv *priv, bool acquire)
 288{
 289	u32 rx_grant;
 290	u32 tx_grant;
 291	int waits;
 292	int ret = 0;
 293
 294	if (acquire) {
 295		udma_set(priv, REGS_DMA_ARB, UDMA_ARB_RX, UDMA_ARB_REQ);
 296		udma_set(priv, REGS_DMA_ARB, UDMA_ARB_TX, UDMA_ARB_REQ);
 297
 298		waits = 1;
 299		while (1) {
 300			rx_grant = udma_readl(priv, REGS_DMA_ARB, UDMA_ARB_RX);
 301			tx_grant = udma_readl(priv, REGS_DMA_ARB, UDMA_ARB_TX);
 302			if (rx_grant & tx_grant & UDMA_ARB_GRANT)
 303				return 0;
 304			if (waits-- == 0)
 305				break;
 306			msleep(1);
 307		}
 308		ret = 1;
 309	}
 310
 311	udma_unset(priv, REGS_DMA_ARB, UDMA_ARB_RX, UDMA_ARB_REQ);
 312	udma_unset(priv, REGS_DMA_ARB, UDMA_ARB_TX, UDMA_ARB_REQ);
 313	return ret;
 314}
 315
 316static void brcmuart_init_dma_hardware(struct brcmuart_priv *priv)
 317{
 318	u32 daddr;
 319	u32 value;
 320	int x;
 321
 322	/* Start with all interrupts disabled */
 323	udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_SET, 0xffffffff);
 324
 325	udma_writel(priv, REGS_DMA_RX, UDMA_RX_BUFFER_SIZE, RX_BUF_SIZE);
 326
 327	/*
 328	 * Setup buffer close to happen when 32 character times have
 329	 * elapsed since the last character was received.
 330	 */
 331	udma_writel(priv, REGS_DMA_RX, UDMA_RX_BUFFER_CLOSE, 16*10*32);
 332	value = (RX_BUFS_COUNT << UDMA_RX_CTRL_NUM_BUF_USED_SHIFT)
 333		| UDMA_RX_CTRL_BUF_CLOSE_MODE
 334		| UDMA_RX_CTRL_BUF_CLOSE_ENA;
 335	udma_writel(priv, REGS_DMA_RX, UDMA_RX_CTRL, value);
 336
 337	udma_writel(priv, REGS_DMA_RX, UDMA_RX_BLOCKOUT_COUNTER, 0);
 338	daddr = priv->rx_addr;
 339	for (x = 0; x < RX_BUFS_COUNT; x++) {
 340
 341		/* Set RX transfer length to 0 for unknown */
 342		udma_writel(priv, REGS_DMA_RX, UDMA_RX_TRANSFER_LEN, 0);
 343
 344		udma_writel(priv, REGS_DMA_RX, UDMA_RX_BUFx_PTR_LO(x),
 345			    lower_32_bits(daddr));
 346		udma_writel(priv, REGS_DMA_RX, UDMA_RX_BUFx_PTR_HI(x),
 347			    upper_32_bits(daddr));
 348		daddr += RX_BUF_SIZE;
 349	}
 350
 351	daddr = priv->tx_addr;
 352	udma_writel(priv, REGS_DMA_TX, UDMA_TX_BUFx_PTR_LO(0),
 353		    lower_32_bits(daddr));
 354	udma_writel(priv, REGS_DMA_TX, UDMA_TX_BUFx_PTR_HI(0),
 355		    upper_32_bits(daddr));
 356	udma_writel(priv, REGS_DMA_TX, UDMA_TX_CTRL,
 357		    UDMA_TX_CTRL_NUM_BUF_USED_1);
 358
 359	/* clear all interrupts then enable them */
 360	udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_CLEAR, 0xffffffff);
 361	udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_CLEAR,
 362		UDMA_RX_INTERRUPTS | UDMA_TX_INTERRUPTS);
 363
 364}
 365
 366static void start_rx_dma(struct uart_8250_port *p)
 367{
 368	struct brcmuart_priv *priv = p->port.private_data;
 369	int x;
 370
 371	udma_unset(priv, REGS_DMA_RX, UDMA_RX_CTRL, UDMA_RX_CTRL_ENA);
 372
 373	/* Clear the RX ready bit for all buffers */
 374	for (x = 0; x < RX_BUFS_COUNT; x++)
 375		udma_unset(priv, REGS_DMA_RX, UDMA_RX_BUFx_STATUS(x),
 376			UDMA_RX_BUFX_STATUS_DATA_RDY);
 377
 378	/* always start with buffer 0 */
 379	udma_unset(priv, REGS_DMA_RX, UDMA_RX_STATUS,
 380		   UDMA_RX_STATUS_ACTIVE_BUF_MASK);
 381	priv->rx_next_buf = 0;
 382
 383	udma_set(priv, REGS_DMA_RX, UDMA_RX_CTRL, UDMA_RX_CTRL_ENA);
 384	priv->rx_running = true;
 385}
 386
 387static void stop_rx_dma(struct uart_8250_port *p)
 388{
 389	struct brcmuart_priv *priv = p->port.private_data;
 390
 391	/* If RX is running, set the RX ABORT */
 392	if (priv->rx_running)
 393		udma_set(priv, REGS_DMA_RX, UDMA_RX_CTRL, UDMA_RX_CTRL_ABORT);
 394}
 395
 396static int stop_tx_dma(struct uart_8250_port *p)
 397{
 398	struct brcmuart_priv *priv = p->port.private_data;
 399	u32 value;
 400
 401	/* If TX is running, set the TX ABORT */
 402	value = udma_readl(priv, REGS_DMA_TX, UDMA_TX_CTRL);
 403	if (value & UDMA_TX_CTRL_ENA)
 404		udma_set(priv, REGS_DMA_TX, UDMA_TX_CTRL, UDMA_TX_CTRL_ABORT);
 405	priv->tx_running = false;
 406	return 0;
 407}
 408
 409/*
 410 * NOTE: printk's in this routine will hang the system if this is
 411 * the console tty
 412 */
 413static int brcmuart_tx_dma(struct uart_8250_port *p)
 414{
 415	struct brcmuart_priv *priv = p->port.private_data;
 416	struct tty_port *tport = &p->port.state->port;
 417	u32 tx_size;
 418
 419	if (uart_tx_stopped(&p->port) || priv->tx_running ||
 420		kfifo_is_empty(&tport->xmit_fifo)) {
 421		return 0;
 422	}
 423
 424	priv->dma.tx_err = 0;
 425	tx_size = uart_fifo_out(&p->port, priv->tx_buf, UART_XMIT_SIZE);
 426
 427	if (kfifo_len(&tport->xmit_fifo) < WAKEUP_CHARS)
 428		uart_write_wakeup(&p->port);
 429
 430	udma_writel(priv, REGS_DMA_TX, UDMA_TX_TRANSFER_LEN, tx_size);
 431	udma_writel(priv, REGS_DMA_TX, UDMA_TX_BUF0_DATA_LEN, tx_size);
 432	udma_unset(priv, REGS_DMA_TX, UDMA_TX_BUF0_STATUS, UDMA_TX_BUFX_EMPTY);
 433	udma_set(priv, REGS_DMA_TX, UDMA_TX_CTRL, UDMA_TX_CTRL_ENA);
 434	priv->tx_running = true;
 435
 436	return 0;
 437}
 438
 439static void brcmuart_rx_buf_done_isr(struct uart_port *up, int index)
 440{
 441	struct brcmuart_priv *priv = up->private_data;
 442	struct tty_port *tty_port = &up->state->port;
 443	u32 status;
 444	u32 length;
 445	u32 copied;
 446
 447	/* Make sure we're still in sync with the hardware */
 448	status = udma_readl(priv, REGS_DMA_RX, UDMA_RX_BUFx_STATUS(index));
 449	length = udma_readl(priv, REGS_DMA_RX, UDMA_RX_BUFx_DATA_LEN(index));
 450
 451	if ((status & UDMA_RX_BUFX_STATUS_DATA_RDY) == 0) {
 452		dev_err(up->dev, "RX done interrupt but DATA_RDY not found\n");
 453		return;
 454	}
 455	if (status & (UDMA_RX_BUFX_STATUS_OVERRUN_ERR |
 456		      UDMA_RX_BUFX_STATUS_FRAME_ERR |
 457		      UDMA_RX_BUFX_STATUS_PARITY_ERR)) {
 458		if (status & UDMA_RX_BUFX_STATUS_OVERRUN_ERR) {
 459			up->icount.overrun++;
 460			dev_warn(up->dev, "RX OVERRUN Error\n");
 461		}
 462		if (status & UDMA_RX_BUFX_STATUS_FRAME_ERR) {
 463			up->icount.frame++;
 464			dev_warn(up->dev, "RX FRAMING Error\n");
 465		}
 466		if (status & UDMA_RX_BUFX_STATUS_PARITY_ERR) {
 467			up->icount.parity++;
 468			dev_warn(up->dev, "RX PARITY Error\n");
 469		}
 470	}
 471	copied = (u32)tty_insert_flip_string(
 472		tty_port,
 473		priv->rx_bufs + (index * RX_BUF_SIZE),
 474		length);
 475	if (copied != length) {
 476		dev_warn(up->dev, "Flip buffer overrun of %d bytes\n",
 477			 length - copied);
 478		up->icount.overrun += length - copied;
 479	}
 480	up->icount.rx += length;
 481	if (status & UDMA_RX_BUFX_STATUS_CLOSE_EXPIRED)
 482		priv->dma_rx_partial_buf++;
 483	else if (length != RX_BUF_SIZE)
 484		/*
 485		 * This is a bug in the controller that doesn't cause
 486		 * any problems but will be fixed in the future.
 487		 */
 488		priv->rx_missing_close_timeout++;
 489	else
 490		priv->dma_rx_full_buf++;
 491
 492	tty_flip_buffer_push(tty_port);
 493}
 494
 495static void brcmuart_rx_isr(struct uart_port *up, u32 rx_isr)
 496{
 497	struct brcmuart_priv *priv = up->private_data;
 498	struct device *dev = up->dev;
 499	u32 rx_done_isr;
 500	u32 check_isr;
 501
 502	rx_done_isr = (rx_isr & UDMA_INTR_RX_READY_MASK);
 503	while (rx_done_isr) {
 504		check_isr = UDMA_INTR_RX_READY_BUF0 << priv->rx_next_buf;
 505		if (check_isr & rx_done_isr) {
 506			brcmuart_rx_buf_done_isr(up, priv->rx_next_buf);
 507		} else {
 508			dev_err(dev,
 509				"RX buffer ready out of sequence, restarting RX DMA\n");
 510			start_rx_dma(up_to_u8250p(up));
 511			break;
 512		}
 513		if (rx_isr & UDMA_RX_ERR_INTERRUPTS) {
 514			if (rx_isr & UDMA_INTR_RX_ERROR)
 515				priv->rx_err++;
 516			if (rx_isr & UDMA_INTR_RX_TIMEOUT) {
 517				priv->rx_timeout++;
 518				dev_err(dev, "RX TIMEOUT Error\n");
 519			}
 520			if (rx_isr & UDMA_INTR_RX_ABORT)
 521				priv->rx_abort++;
 522			priv->rx_running = false;
 523		}
 524		/* If not ABORT, re-enable RX buffer */
 525		if (!(rx_isr & UDMA_INTR_RX_ABORT))
 526			udma_unset(priv, REGS_DMA_RX,
 527				   UDMA_RX_BUFx_STATUS(priv->rx_next_buf),
 528				   UDMA_RX_BUFX_STATUS_DATA_RDY);
 529		rx_done_isr &= ~check_isr;
 530		priv->rx_next_buf++;
 531		if (priv->rx_next_buf == RX_BUFS_COUNT)
 532			priv->rx_next_buf = 0;
 533	}
 534}
 535
 536static void brcmuart_tx_isr(struct uart_port *up, u32 isr)
 537{
 538	struct brcmuart_priv *priv = up->private_data;
 539	struct device *dev = up->dev;
 540	struct uart_8250_port *port_8250 = up_to_u8250p(up);
 541	struct tty_port *tport = &port_8250->port.state->port;
 542
 543	if (isr & UDMA_INTR_TX_ABORT) {
 544		if (priv->tx_running)
 545			dev_err(dev, "Unexpected TX_ABORT interrupt\n");
 546		return;
 547	}
 548	priv->tx_running = false;
 549	if (!kfifo_is_empty(&tport->xmit_fifo) && !uart_tx_stopped(up))
 550		brcmuart_tx_dma(port_8250);
 551}
 552
 553static irqreturn_t brcmuart_isr(int irq, void *dev_id)
 554{
 555	struct uart_port *up = dev_id;
 556	struct device *dev = up->dev;
 557	struct brcmuart_priv *priv = up->private_data;
 558	unsigned long flags;
 559	u32 interrupts;
 560	u32 rval;
 561	u32 tval;
 562
 563	interrupts = udma_readl(priv, REGS_DMA_ISR, UDMA_INTR_STATUS);
 564	if (interrupts == 0)
 565		return IRQ_NONE;
 566
 567	uart_port_lock_irqsave(up, &flags);
 568
 569	/* Clear all interrupts */
 570	udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_CLEAR, interrupts);
 571
 572	rval = UDMA_IS_RX_INTERRUPT(interrupts);
 573	if (rval)
 574		brcmuart_rx_isr(up, rval);
 575	tval = UDMA_IS_TX_INTERRUPT(interrupts);
 576	if (tval)
 577		brcmuart_tx_isr(up, tval);
 578	if ((rval | tval) == 0)
 579		dev_warn(dev, "Spurious interrupt: 0x%x\n", interrupts);
 580
 581	uart_port_unlock_irqrestore(up, flags);
 582	return IRQ_HANDLED;
 583}
 584
 585static int brcmuart_startup(struct uart_port *port)
 586{
 587	int res;
 588	struct uart_8250_port *up = up_to_u8250p(port);
 589	struct brcmuart_priv *priv = up->port.private_data;
 590
 591	priv->shutdown = false;
 592
 593	/*
 594	 * prevent serial8250_do_startup() from allocating non-existent
 595	 * DMA resources
 596	 */
 597	up->dma = NULL;
 598
 599	res = serial8250_do_startup(port);
 600	if (!priv->dma_enabled)
 601		return res;
 602	/*
 603	 * Disable the Receive Data Interrupt because the DMA engine
 604	 * will handle this.
 605	 *
 606	 * Synchronize UART_IER access against the console.
 607	 */
 608	uart_port_lock_irq(port);
 609	up->ier &= ~UART_IER_RDI;
 610	serial_port_out(port, UART_IER, up->ier);
 611	uart_port_unlock_irq(port);
 612
 613	priv->tx_running = false;
 614	priv->dma.rx_dma = NULL;
 615	priv->dma.tx_dma = brcmuart_tx_dma;
 616	up->dma = &priv->dma;
 617
 618	brcmuart_init_dma_hardware(priv);
 619	start_rx_dma(up);
 620	return res;
 621}
 622
 623static void brcmuart_shutdown(struct uart_port *port)
 624{
 625	struct uart_8250_port *up = up_to_u8250p(port);
 626	struct brcmuart_priv *priv = up->port.private_data;
 627	unsigned long flags;
 628
 629	uart_port_lock_irqsave(port, &flags);
 630	priv->shutdown = true;
 631	if (priv->dma_enabled) {
 632		stop_rx_dma(up);
 633		stop_tx_dma(up);
 634		/* disable all interrupts */
 635		udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_SET,
 636			UDMA_RX_INTERRUPTS | UDMA_TX_INTERRUPTS);
 637	}
 638
 639	/*
 640	 * prevent serial8250_do_shutdown() from trying to free
 641	 * DMA resources that we never alloc'd for this driver.
 642	 */
 643	up->dma = NULL;
 644
 645	uart_port_unlock_irqrestore(port, flags);
 646	serial8250_do_shutdown(port);
 647}
 648
 649/*
 650 * Not all clocks run at the exact specified rate, so set each requested
 651 * rate and then get the actual rate.
 652 */
 653static void init_real_clk_rates(struct device *dev, struct brcmuart_priv *priv)
 654{
 655	int x;
 656	int rc;
 657
 658	priv->default_mux_rate = clk_get_rate(priv->baud_mux_clk);
 659	for (x = 0; x < ARRAY_SIZE(priv->real_rates); x++) {
 660		if (priv->rate_table[x] == 0) {
 661			priv->real_rates[x] = 0;
 662			continue;
 663		}
 664		rc = clk_set_rate(priv->baud_mux_clk, priv->rate_table[x]);
 665		if (rc) {
 666			dev_err(dev, "Error selecting BAUD MUX clock for %u\n",
 667				priv->rate_table[x]);
 668			priv->real_rates[x] = priv->rate_table[x];
 669		} else {
 670			priv->real_rates[x] = clk_get_rate(priv->baud_mux_clk);
 671		}
 672	}
 673	clk_set_rate(priv->baud_mux_clk, priv->default_mux_rate);
 674}
 675
 676static u32 find_quot(struct device *dev, u32 freq, u32 baud, u32 *percent)
 677{
 678	u32 quot;
 679	u32 rate;
 680	u64 hires_rate;
 681	u64 hires_baud;
 682	u64 hires_err;
 683
 684	rate = freq / 16;
 685	quot = DIV_ROUND_CLOSEST(rate, baud);
 686	if (!quot)
 687		return 0;
 688
 689	/* increase resolution to get xx.xx percent */
 690	hires_rate = div_u64((u64)rate * 10000, (u64)quot);
 691	hires_baud = (u64)baud * 10000;
 692
 693	/* get the delta */
 694	if (hires_rate > hires_baud)
 695		hires_err = (hires_rate - hires_baud);
 696	else
 697		hires_err = (hires_baud - hires_rate);
 698
 699	*percent = (unsigned long)DIV_ROUND_CLOSEST_ULL(hires_err, baud);
 700
 701	dev_dbg(dev, "Baud rate: %u, MUX Clk: %u, Error: %u.%u%%\n",
 702		baud, freq, *percent / 100, *percent % 100);
 703
 704	return quot;
 705}
 706
 707static void set_clock_mux(struct uart_port *up, struct brcmuart_priv *priv,
 708			u32 baud)
 709{
 710	u32 percent;
 711	u32 best_percent = UINT_MAX;
 712	u32 quot;
 713	u32 freq;
 714	u32 best_quot = 1;
 715	u32 best_freq = 0;
 716	int rc;
 717	int i;
 718	int real_baud;
 719
 720	/* If the Baud Mux Clock was not specified, just return */
 721	if (priv->baud_mux_clk == NULL)
 722		return;
 723
 724	/* Try default_mux_rate first */
 725	quot = find_quot(up->dev, priv->default_mux_rate, baud, &percent);
 726	if (quot) {
 727		best_percent = percent;
 728		best_freq = priv->default_mux_rate;
 729		best_quot = quot;
 730	}
 731	/* If more than 1% error, find the closest match for specified baud */
 732	if (best_percent > 100) {
 733		for (i = 0; i < ARRAY_SIZE(priv->real_rates); i++) {
 734			freq = priv->real_rates[i];
 735			if (freq == 0 || freq == priv->default_mux_rate)
 736				continue;
 737			quot = find_quot(up->dev, freq, baud, &percent);
 738			if (!quot)
 739				continue;
 740
 741			if (percent < best_percent) {
 742				best_percent = percent;
 743				best_freq = freq;
 744				best_quot = quot;
 745			}
 746		}
 747	}
 748	if (!best_freq) {
 749		dev_err(up->dev, "Error, %d BAUD rate is too fast.\n", baud);
 750		return;
 751	}
 752	rc = clk_set_rate(priv->baud_mux_clk, best_freq);
 753	if (rc)
 754		dev_err(up->dev, "Error selecting BAUD MUX clock\n");
 755
 756	/* Error over 3 percent will cause data errors */
 757	if (best_percent > 300)
 758		dev_err(up->dev, "Error, baud: %d has %u.%u%% error\n",
 759			baud, percent / 100, percent % 100);
 760
 761	real_baud = best_freq / 16 / best_quot;
 762	dev_dbg(up->dev, "Selecting BAUD MUX rate: %u\n", best_freq);
 763	dev_dbg(up->dev, "Requested baud: %u, Actual baud: %u\n",
 764		baud, real_baud);
 765
 766	/* calc nanoseconds for 1.5 characters time at the given baud rate */
 767	i = NSEC_PER_SEC / real_baud / 10;
 768	i += (i / 2);
 769	priv->char_wait = ns_to_ktime(i);
 770
 771	up->uartclk = best_freq;
 772}
 773
 774static void brcmstb_set_termios(struct uart_port *up,
 775				struct ktermios *termios,
 776				const struct ktermios *old)
 777{
 778	struct uart_8250_port *p8250 = up_to_u8250p(up);
 779	struct brcmuart_priv *priv = up->private_data;
 780
 781	if (priv->dma_enabled)
 782		stop_rx_dma(p8250);
 783	set_clock_mux(up, priv, tty_termios_baud_rate(termios));
 784	serial8250_do_set_termios(up, termios, old);
 785	if (p8250->mcr & UART_MCR_AFE)
 786		p8250->port.status |= UPSTAT_AUTOCTS;
 787	if (priv->dma_enabled)
 788		start_rx_dma(p8250);
 789}
 790
 791static int brcmuart_handle_irq(struct uart_port *p)
 792{
 793	unsigned int iir = serial_port_in(p, UART_IIR);
 794	struct brcmuart_priv *priv = p->private_data;
 795	struct uart_8250_port *up = up_to_u8250p(p);
 796	unsigned int status;
 797	unsigned long flags;
 798	unsigned int ier;
 799	unsigned int mcr;
 800	int handled = 0;
 801
 802	/*
 803	 * There's a bug in some 8250 cores where we get a timeout
 804	 * interrupt but there is no data ready.
 805	 */
 806	if (((iir & UART_IIR_ID) == UART_IIR_RX_TIMEOUT) && !(priv->shutdown)) {
 807		uart_port_lock_irqsave(p, &flags);
 808		status = serial_port_in(p, UART_LSR);
 809		if ((status & UART_LSR_DR) == 0) {
 810
 811			ier = serial_port_in(p, UART_IER);
 812			/*
 813			 * if Receive Data Interrupt is enabled and
 814			 * we're uing hardware flow control, deassert
 815			 * RTS and wait for any chars in the pipeline to
 816			 * arrive and then check for DR again.
 817			 */
 818			if ((ier & UART_IER_RDI) && (up->mcr & UART_MCR_AFE)) {
 819				ier &= ~(UART_IER_RLSI | UART_IER_RDI);
 820				serial_port_out(p, UART_IER, ier);
 821				mcr = serial_port_in(p, UART_MCR);
 822				mcr &= ~UART_MCR_RTS;
 823				serial_port_out(p, UART_MCR, mcr);
 824				hrtimer_start(&priv->hrt, priv->char_wait,
 825					      HRTIMER_MODE_REL);
 826			} else {
 827				serial_port_in(p, UART_RX);
 828			}
 829
 830			handled = 1;
 831		}
 832		uart_port_unlock_irqrestore(p, flags);
 833		if (handled)
 834			return 1;
 835	}
 836	return serial8250_handle_irq(p, iir);
 837}
 838
 839static enum hrtimer_restart brcmuart_hrtimer_func(struct hrtimer *t)
 840{
 841	struct brcmuart_priv *priv = container_of(t, struct brcmuart_priv, hrt);
 842	struct uart_port *p = priv->up;
 843	struct uart_8250_port *up = up_to_u8250p(p);
 844	unsigned int status;
 845	unsigned long flags;
 846
 847	if (priv->shutdown)
 848		return HRTIMER_NORESTART;
 849
 850	uart_port_lock_irqsave(p, &flags);
 851	status = serial_port_in(p, UART_LSR);
 852
 853	/*
 854	 * If a character did not arrive after the timeout, clear the false
 855	 * receive timeout.
 856	 */
 857	if ((status & UART_LSR_DR) == 0) {
 858		serial_port_in(p, UART_RX);
 859		priv->rx_bad_timeout_no_char++;
 860	} else {
 861		priv->rx_bad_timeout_late_char++;
 862	}
 863
 864	/* re-enable receive unless upper layer has disabled it */
 865	if ((up->ier & (UART_IER_RLSI | UART_IER_RDI)) ==
 866	    (UART_IER_RLSI | UART_IER_RDI)) {
 867		status = serial_port_in(p, UART_IER);
 868		status |= (UART_IER_RLSI | UART_IER_RDI);
 869		serial_port_out(p, UART_IER, status);
 870		status = serial_port_in(p, UART_MCR);
 871		status |= UART_MCR_RTS;
 872		serial_port_out(p, UART_MCR, status);
 873	}
 874	uart_port_unlock_irqrestore(p, flags);
 875	return HRTIMER_NORESTART;
 876}
 877
 878static const struct of_device_id brcmuart_dt_ids[] = {
 879	{
 880		.compatible = "brcm,bcm7278-uart",
 881		.data = brcmstb_rate_table_7278,
 882	},
 883	{
 884		.compatible = "brcm,bcm7271-uart",
 885		.data = brcmstb_rate_table,
 886	},
 887	{},
 888};
 889
 890MODULE_DEVICE_TABLE(of, brcmuart_dt_ids);
 891
 892static void brcmuart_free_bufs(struct device *dev, struct brcmuart_priv *priv)
 893{
 894	if (priv->rx_bufs)
 895		dma_free_coherent(dev, priv->rx_size, priv->rx_bufs,
 896				  priv->rx_addr);
 897	if (priv->tx_buf)
 898		dma_free_coherent(dev, priv->tx_size, priv->tx_buf,
 899				  priv->tx_addr);
 900}
 901
 902static void brcmuart_throttle(struct uart_port *port)
 903{
 904	struct brcmuart_priv *priv = port->private_data;
 905
 906	udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_SET, UDMA_RX_INTERRUPTS);
 907}
 908
 909static void brcmuart_unthrottle(struct uart_port *port)
 910{
 911	struct brcmuart_priv *priv = port->private_data;
 912
 913	udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_CLEAR,
 914		    UDMA_RX_INTERRUPTS);
 915}
 916
 917static int debugfs_stats_show(struct seq_file *s, void *unused)
 918{
 919	struct brcmuart_priv *priv = s->private;
 920
 921	seq_printf(s, "rx_err:\t\t\t\t%u\n",
 922		   priv->rx_err);
 923	seq_printf(s, "rx_timeout:\t\t\t%u\n",
 924		   priv->rx_timeout);
 925	seq_printf(s, "rx_abort:\t\t\t%u\n",
 926		   priv->rx_abort);
 927	seq_printf(s, "rx_bad_timeout_late_char:\t%u\n",
 928		   priv->rx_bad_timeout_late_char);
 929	seq_printf(s, "rx_bad_timeout_no_char:\t\t%u\n",
 930		   priv->rx_bad_timeout_no_char);
 931	seq_printf(s, "rx_missing_close_timeout:\t%u\n",
 932		   priv->rx_missing_close_timeout);
 933	if (priv->dma_enabled) {
 934		seq_printf(s, "dma_rx_partial_buf:\t\t%llu\n",
 935			   priv->dma_rx_partial_buf);
 936		seq_printf(s, "dma_rx_full_buf:\t\t%llu\n",
 937			   priv->dma_rx_full_buf);
 938	}
 939	return 0;
 940}
 941DEFINE_SHOW_ATTRIBUTE(debugfs_stats);
 942
 943static void brcmuart_init_debugfs(struct brcmuart_priv *priv,
 944				  const char *device)
 945{
 946	priv->debugfs_dir = debugfs_create_dir(device, brcmuart_debugfs_root);
 947	debugfs_create_file("stats", 0444, priv->debugfs_dir, priv,
 948			    &debugfs_stats_fops);
 949}
 950
 951
 952static int brcmuart_probe(struct platform_device *pdev)
 953{
 954	struct resource *regs;
 955	const struct of_device_id *of_id = NULL;
 956	struct uart_8250_port *new_port;
 957	struct device *dev = &pdev->dev;
 958	struct brcmuart_priv *priv;
 959	struct clk *baud_mux_clk;
 960	struct uart_8250_port up;
 961	void __iomem *membase = NULL;
 962	resource_size_t mapbase = 0;
 963	int ret;
 964	int x;
 965	int dma_irq;
 966	static const char * const reg_names[REGS_MAX] = {
 967		"uart", "dma_rx", "dma_tx", "dma_intr2", "dma_arb"
 968	};
 969
 970	priv = devm_kzalloc(dev, sizeof(struct brcmuart_priv),
 971			GFP_KERNEL);
 972	if (!priv)
 973		return -ENOMEM;
 974
 975	of_id = of_match_node(brcmuart_dt_ids, dev->of_node);
 976	if (!of_id || !of_id->data)
 977		priv->rate_table = brcmstb_rate_table;
 978	else
 979		priv->rate_table = of_id->data;
 980
 981	for (x = 0; x < REGS_MAX; x++) {
 982		regs = platform_get_resource_byname(pdev, IORESOURCE_MEM,
 983						reg_names[x]);
 984		if (!regs)
 985			break;
 986		priv->regs[x] =	devm_ioremap(dev, regs->start,
 987					     resource_size(regs));
 988		if (!priv->regs[x])
 989			return -ENOMEM;
 990		if (x == REGS_8250) {
 991			mapbase = regs->start;
 992			membase = priv->regs[x];
 993		}
 994	}
 995
 996	/* We should have just the uart base registers or all the registers */
 997	if (x != 1 && x != REGS_MAX)
 998		return dev_err_probe(dev, -EINVAL, "%s registers not specified\n",
 999				     reg_names[x]);
1000
1001	/* if the DMA registers were specified, try to enable DMA */
1002	if (x > REGS_DMA_RX) {
1003		if (brcmuart_arbitration(priv, 1) == 0) {
1004			u32 txrev = 0;
1005			u32 rxrev = 0;
1006
1007			txrev = udma_readl(priv, REGS_DMA_RX, UDMA_RX_REVISION);
1008			rxrev = udma_readl(priv, REGS_DMA_TX, UDMA_TX_REVISION);
1009			if ((txrev >= UDMA_TX_REVISION_REQUIRED) &&
1010				(rxrev >= UDMA_RX_REVISION_REQUIRED)) {
1011
1012				/* Enable the use of the DMA hardware */
1013				priv->dma_enabled = true;
1014			} else {
1015				brcmuart_arbitration(priv, 0);
1016				dev_err(dev,
1017					"Unsupported DMA Hardware Revision\n");
1018			}
1019		} else {
1020			dev_err(dev,
1021				"Timeout arbitrating for UART DMA hardware\n");
1022		}
1023	}
1024
1025	dev_dbg(dev, "DMA is %senabled\n", priv->dma_enabled ? "" : "not ");
1026
1027	memset(&up, 0, sizeof(up));
1028	up.port.type = PORT_BCM7271;
1029	up.port.dev = dev;
1030	up.port.mapbase = mapbase;
1031	up.port.membase = membase;
1032	up.port.handle_irq = brcmuart_handle_irq;
1033	up.port.flags = UPF_BOOT_AUTOCONF | UPF_FIXED_PORT | UPF_FIXED_TYPE;
1034	up.port.private_data = priv;
1035
1036	ret = uart_read_port_properties(&up.port);
1037	if (ret)
1038		goto release_dma;
1039
1040	up.port.regshift = 2;
1041	up.port.iotype = device_is_big_endian(dev) ? UPIO_MEM32BE : UPIO_MEM32;
1042
1043	/* See if a Baud clock has been specified */
1044	baud_mux_clk = devm_clk_get_optional_enabled(dev, "sw_baud");
1045	ret = PTR_ERR_OR_ZERO(baud_mux_clk);
1046	if (ret)
1047		goto release_dma;
1048	if (baud_mux_clk) {
1049		dev_dbg(dev, "BAUD MUX clock found\n");
1050
1051		priv->baud_mux_clk = baud_mux_clk;
1052		init_real_clk_rates(dev, priv);
1053		up.port.uartclk = priv->default_mux_rate;
1054	} else {
1055		dev_dbg(dev, "BAUD MUX clock not specified\n");
1056	}
1057
1058	/* setup HR timer */
1059	hrtimer_init(&priv->hrt, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1060	priv->hrt.function = brcmuart_hrtimer_func;
1061
1062	up.port.shutdown = brcmuart_shutdown;
1063	up.port.startup = brcmuart_startup;
1064	up.port.throttle = brcmuart_throttle;
1065	up.port.unthrottle = brcmuart_unthrottle;
1066	up.port.set_termios = brcmstb_set_termios;
1067
1068	if (priv->dma_enabled) {
1069		priv->rx_size = RX_BUF_SIZE * RX_BUFS_COUNT;
1070		priv->rx_bufs = dma_alloc_coherent(dev,
1071						   priv->rx_size,
1072						   &priv->rx_addr, GFP_KERNEL);
1073		if (!priv->rx_bufs) {
1074			ret = -ENOMEM;
1075			goto err;
1076		}
1077		priv->tx_size = UART_XMIT_SIZE;
1078		priv->tx_buf = dma_alloc_coherent(dev,
1079						  priv->tx_size,
1080						  &priv->tx_addr, GFP_KERNEL);
1081		if (!priv->tx_buf) {
1082			ret = -ENOMEM;
1083			goto err;
1084		}
1085	}
1086
1087	ret = serial8250_register_8250_port(&up);
1088	if (ret < 0) {
1089		dev_err_probe(dev, ret, "unable to register 8250 port\n");
1090		goto err;
1091	}
1092	priv->line = ret;
1093	new_port = serial8250_get_port(ret);
1094	priv->up = &new_port->port;
1095	if (priv->dma_enabled) {
1096		dma_irq = platform_get_irq_byname(pdev,  "dma");
1097		if (dma_irq < 0) {
1098			ret = dev_err_probe(dev, dma_irq, "no IRQ resource info\n");
1099			goto err1;
1100		}
1101		ret = devm_request_irq(dev, dma_irq, brcmuart_isr,
1102				IRQF_SHARED, "uart DMA irq", &new_port->port);
1103		if (ret) {
1104			dev_err_probe(dev, ret, "unable to register IRQ handler\n");
1105			goto err1;
1106		}
1107	}
1108	platform_set_drvdata(pdev, priv);
1109	brcmuart_init_debugfs(priv, dev_name(&pdev->dev));
1110	return 0;
1111
1112err1:
1113	serial8250_unregister_port(priv->line);
1114err:
1115	brcmuart_free_bufs(dev, priv);
1116release_dma:
1117	if (priv->dma_enabled)
1118		brcmuart_arbitration(priv, 0);
1119	return ret;
1120}
1121
1122static void brcmuart_remove(struct platform_device *pdev)
1123{
1124	struct brcmuart_priv *priv = platform_get_drvdata(pdev);
1125
1126	debugfs_remove_recursive(priv->debugfs_dir);
1127	hrtimer_cancel(&priv->hrt);
1128	serial8250_unregister_port(priv->line);
1129	brcmuart_free_bufs(&pdev->dev, priv);
1130	if (priv->dma_enabled)
1131		brcmuart_arbitration(priv, 0);
1132}
1133
1134static int __maybe_unused brcmuart_suspend(struct device *dev)
1135{
1136	struct brcmuart_priv *priv = dev_get_drvdata(dev);
1137	struct uart_8250_port *up = serial8250_get_port(priv->line);
1138	struct uart_port *port = &up->port;
1139	unsigned long flags;
1140
1141	/*
1142	 * This will prevent resume from enabling RTS before the
1143	 *  baud rate has been restored.
1144	 */
1145	uart_port_lock_irqsave(port, &flags);
1146	priv->saved_mctrl = port->mctrl;
1147	port->mctrl &= ~TIOCM_RTS;
1148	uart_port_unlock_irqrestore(port, flags);
1149
1150	serial8250_suspend_port(priv->line);
1151	clk_disable_unprepare(priv->baud_mux_clk);
1152
1153	return 0;
1154}
1155
1156static int __maybe_unused brcmuart_resume(struct device *dev)
1157{
1158	struct brcmuart_priv *priv = dev_get_drvdata(dev);
1159	struct uart_8250_port *up = serial8250_get_port(priv->line);
1160	struct uart_port *port = &up->port;
1161	unsigned long flags;
1162	int ret;
1163
1164	ret = clk_prepare_enable(priv->baud_mux_clk);
1165	if (ret)
1166		dev_err(dev, "Error enabling BAUD MUX clock\n");
1167
1168	/*
1169	 * The hardware goes back to it's default after suspend
1170	 * so get the "clk" back in sync.
1171	 */
1172	ret = clk_set_rate(priv->baud_mux_clk, priv->default_mux_rate);
1173	if (ret)
1174		dev_err(dev, "Error restoring default BAUD MUX clock\n");
1175	if (priv->dma_enabled) {
1176		if (brcmuart_arbitration(priv, 1)) {
1177			dev_err(dev, "Timeout arbitrating for DMA hardware on resume\n");
1178			return(-EBUSY);
1179		}
1180		brcmuart_init_dma_hardware(priv);
1181		start_rx_dma(serial8250_get_port(priv->line));
1182	}
1183	serial8250_resume_port(priv->line);
1184
1185	if (priv->saved_mctrl & TIOCM_RTS) {
1186		/* Restore RTS */
1187		uart_port_lock_irqsave(port, &flags);
1188		port->mctrl |= TIOCM_RTS;
1189		port->ops->set_mctrl(port, port->mctrl);
1190		uart_port_unlock_irqrestore(port, flags);
1191	}
1192
1193	return 0;
1194}
1195
1196static const struct dev_pm_ops brcmuart_dev_pm_ops = {
1197	SET_SYSTEM_SLEEP_PM_OPS(brcmuart_suspend, brcmuart_resume)
1198};
1199
1200static struct platform_driver brcmuart_platform_driver = {
1201	.driver = {
1202		.name	= "bcm7271-uart",
1203		.pm		= &brcmuart_dev_pm_ops,
1204		.of_match_table = brcmuart_dt_ids,
1205	},
1206	.probe		= brcmuart_probe,
1207	.remove		= brcmuart_remove,
1208};
1209
1210static int __init brcmuart_init(void)
1211{
1212	int ret;
1213
1214	brcmuart_debugfs_root = debugfs_create_dir(
1215		brcmuart_platform_driver.driver.name, NULL);
1216	ret = platform_driver_register(&brcmuart_platform_driver);
1217	if (ret) {
1218		debugfs_remove_recursive(brcmuart_debugfs_root);
1219		return ret;
1220	}
1221
1222	return 0;
1223}
1224module_init(brcmuart_init);
1225
1226static void __exit brcmuart_deinit(void)
1227{
1228	platform_driver_unregister(&brcmuart_platform_driver);
1229	debugfs_remove_recursive(brcmuart_debugfs_root);
1230}
1231module_exit(brcmuart_deinit);
1232
1233MODULE_AUTHOR("Al Cooper");
1234MODULE_DESCRIPTION("Broadcom NS16550A compatible serial port driver");
1235MODULE_LICENSE("GPL v2");