1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Driver for msm7k serial device and console
   4 *
   5 * Copyright (C) 2007 Google, Inc.
   6 * Author: Robert Love <rlove@google.com>
   7 * Copyright (c) 2011, Code Aurora Forum. All rights reserved.
   8 */
   9
  10#include <linux/kernel.h>
  11#include <linux/atomic.h>
  12#include <linux/dma-mapping.h>
  13#include <linux/dmaengine.h>
  14#include <linux/module.h>
  15#include <linux/io.h>
  16#include <linux/ioport.h>
  17#include <linux/interrupt.h>
  18#include <linux/init.h>
  19#include <linux/console.h>
  20#include <linux/tty.h>
  21#include <linux/tty_flip.h>
  22#include <linux/serial_core.h>
  23#include <linux/slab.h>
  24#include <linux/clk.h>
  25#include <linux/platform_device.h>
  26#include <linux/delay.h>
  27#include <linux/of.h>
  28#include <linux/of_device.h>
  29#include <linux/wait.h>
  30
  31#define UART_MR1			0x0000
  32
  33#define UART_MR1_AUTO_RFR_LEVEL0	0x3F
  34#define UART_MR1_AUTO_RFR_LEVEL1	0x3FF00
  35#define UART_DM_MR1_AUTO_RFR_LEVEL1	0xFFFFFF00
  36#define UART_MR1_RX_RDY_CTL		BIT(7)
  37#define UART_MR1_CTS_CTL		BIT(6)
  38
  39#define UART_MR2			0x0004
  40#define UART_MR2_ERROR_MODE		BIT(6)
  41#define UART_MR2_BITS_PER_CHAR		0x30
  42#define UART_MR2_BITS_PER_CHAR_5	(0x0 << 4)
  43#define UART_MR2_BITS_PER_CHAR_6	(0x1 << 4)
  44#define UART_MR2_BITS_PER_CHAR_7	(0x2 << 4)
  45#define UART_MR2_BITS_PER_CHAR_8	(0x3 << 4)
  46#define UART_MR2_STOP_BIT_LEN_ONE	(0x1 << 2)
  47#define UART_MR2_STOP_BIT_LEN_TWO	(0x3 << 2)
  48#define UART_MR2_PARITY_MODE_NONE	0x0
  49#define UART_MR2_PARITY_MODE_ODD	0x1
  50#define UART_MR2_PARITY_MODE_EVEN	0x2
  51#define UART_MR2_PARITY_MODE_SPACE	0x3
  52#define UART_MR2_PARITY_MODE		0x3
  53
  54#define UART_CSR			0x0008
  55
  56#define UART_TF				0x000C
  57#define UARTDM_TF			0x0070
  58
  59#define UART_CR				0x0010
  60#define UART_CR_CMD_NULL		(0 << 4)
  61#define UART_CR_CMD_RESET_RX		(1 << 4)
  62#define UART_CR_CMD_RESET_TX		(2 << 4)
  63#define UART_CR_CMD_RESET_ERR		(3 << 4)
  64#define UART_CR_CMD_RESET_BREAK_INT	(4 << 4)
  65#define UART_CR_CMD_START_BREAK		(5 << 4)
  66#define UART_CR_CMD_STOP_BREAK		(6 << 4)
  67#define UART_CR_CMD_RESET_CTS		(7 << 4)
  68#define UART_CR_CMD_RESET_STALE_INT	(8 << 4)
  69#define UART_CR_CMD_PACKET_MODE		(9 << 4)
  70#define UART_CR_CMD_MODE_RESET		(12 << 4)
  71#define UART_CR_CMD_SET_RFR		(13 << 4)
  72#define UART_CR_CMD_RESET_RFR		(14 << 4)
  73#define UART_CR_CMD_PROTECTION_EN	(16 << 4)
  74#define UART_CR_CMD_STALE_EVENT_DISABLE	(6 << 8)
  75#define UART_CR_CMD_STALE_EVENT_ENABLE	(80 << 4)
  76#define UART_CR_CMD_FORCE_STALE		(4 << 8)
  77#define UART_CR_CMD_RESET_TX_READY	(3 << 8)
  78#define UART_CR_TX_DISABLE		BIT(3)
  79#define UART_CR_TX_ENABLE		BIT(2)
  80#define UART_CR_RX_DISABLE		BIT(1)
  81#define UART_CR_RX_ENABLE		BIT(0)
  82#define UART_CR_CMD_RESET_RXBREAK_START	((1 << 11) | (2 << 4))
  83
  84#define UART_IMR			0x0014
  85#define UART_IMR_TXLEV			BIT(0)
  86#define UART_IMR_RXSTALE		BIT(3)
  87#define UART_IMR_RXLEV			BIT(4)
  88#define UART_IMR_DELTA_CTS		BIT(5)
  89#define UART_IMR_CURRENT_CTS		BIT(6)
  90#define UART_IMR_RXBREAK_START		BIT(10)
  91
  92#define UART_IPR_RXSTALE_LAST		0x20
  93#define UART_IPR_STALE_LSB		0x1F
  94#define UART_IPR_STALE_TIMEOUT_MSB	0x3FF80
  95#define UART_DM_IPR_STALE_TIMEOUT_MSB	0xFFFFFF80
  96
  97#define UART_IPR			0x0018
  98#define UART_TFWR			0x001C
  99#define UART_RFWR			0x0020
 100#define UART_HCR			0x0024
 101
 102#define UART_MREG			0x0028
 103#define UART_NREG			0x002C
 104#define UART_DREG			0x0030
 105#define UART_MNDREG			0x0034
 106#define UART_IRDA			0x0038
 107#define UART_MISR_MODE			0x0040
 108#define UART_MISR_RESET			0x0044
 109#define UART_MISR_EXPORT		0x0048
 110#define UART_MISR_VAL			0x004C
 111#define UART_TEST_CTRL			0x0050
 112
 113#define UART_SR				0x0008
 114#define UART_SR_HUNT_CHAR		BIT(7)
 115#define UART_SR_RX_BREAK		BIT(6)
 116#define UART_SR_PAR_FRAME_ERR		BIT(5)
 117#define UART_SR_OVERRUN			BIT(4)
 118#define UART_SR_TX_EMPTY		BIT(3)
 119#define UART_SR_TX_READY		BIT(2)
 120#define UART_SR_RX_FULL			BIT(1)
 121#define UART_SR_RX_READY		BIT(0)
 122
 123#define UART_RF				0x000C
 124#define UARTDM_RF			0x0070
 125#define UART_MISR			0x0010
 126#define UART_ISR			0x0014
 127#define UART_ISR_TX_READY		BIT(7)
 128
 129#define UARTDM_RXFS			0x50
 130#define UARTDM_RXFS_BUF_SHIFT		0x7
 131#define UARTDM_RXFS_BUF_MASK		0x7
 132
 133#define UARTDM_DMEN			0x3C
 134#define UARTDM_DMEN_RX_SC_ENABLE	BIT(5)
 135#define UARTDM_DMEN_TX_SC_ENABLE	BIT(4)
 136
 137#define UARTDM_DMEN_TX_BAM_ENABLE	BIT(2)	/* UARTDM_1P4 */
 138#define UARTDM_DMEN_TX_DM_ENABLE	BIT(0)	/* < UARTDM_1P4 */
 139
 140#define UARTDM_DMEN_RX_BAM_ENABLE	BIT(3)	/* UARTDM_1P4 */
 141#define UARTDM_DMEN_RX_DM_ENABLE	BIT(1)	/* < UARTDM_1P4 */
 142
 143#define UARTDM_DMRX			0x34
 144#define UARTDM_NCF_TX			0x40
 145#define UARTDM_RX_TOTAL_SNAP		0x38
 146
 147#define UARTDM_BURST_SIZE		16   /* in bytes */
 148#define UARTDM_TX_AIGN(x)		((x) & ~0x3) /* valid for > 1p3 */
 149#define UARTDM_TX_MAX			256   /* in bytes, valid for <= 1p3 */
 150#define UARTDM_RX_SIZE			(UART_XMIT_SIZE / 4)
 151
 152enum {
 153	UARTDM_1P1 = 1,
 154	UARTDM_1P2,
 155	UARTDM_1P3,
 156	UARTDM_1P4,
 157};
 158
 159struct msm_dma {
 160	struct dma_chan		*chan;
 161	enum dma_data_direction dir;
 162	dma_addr_t		phys;
 163	unsigned char		*virt;
 164	dma_cookie_t		cookie;
 165	u32			enable_bit;
 166	unsigned int		count;
 167	struct dma_async_tx_descriptor	*desc;
 168};
 169
 170struct msm_port {
 171	struct uart_port	uart;
 172	char			name[16];
 173	struct clk		*clk;
 174	struct clk		*pclk;
 175	unsigned int		imr;
 176	int			is_uartdm;
 177	unsigned int		old_snap_state;
 178	bool			break_detected;
 179	struct msm_dma		tx_dma;
 180	struct msm_dma		rx_dma;
 181};
 182
 183#define UART_TO_MSM(uart_port)	container_of(uart_port, struct msm_port, uart)
 184
 185static
 186void msm_write(struct uart_port *port, unsigned int val, unsigned int off)
 187{
 188	writel_relaxed(val, port->membase + off);
 189}
 190
 191static
 192unsigned int msm_read(struct uart_port *port, unsigned int off)
 193{
 194	return readl_relaxed(port->membase + off);
 195}
 196
 197/*
 198 * Setup the MND registers to use the TCXO clock.
 199 */
 200static void msm_serial_set_mnd_regs_tcxo(struct uart_port *port)
 201{
 202	msm_write(port, 0x06, UART_MREG);
 203	msm_write(port, 0xF1, UART_NREG);
 204	msm_write(port, 0x0F, UART_DREG);
 205	msm_write(port, 0x1A, UART_MNDREG);
 206	port->uartclk = 1843200;
 207}
 208
 209/*
 210 * Setup the MND registers to use the TCXO clock divided by 4.
 211 */
 212static void msm_serial_set_mnd_regs_tcxoby4(struct uart_port *port)
 213{
 214	msm_write(port, 0x18, UART_MREG);
 215	msm_write(port, 0xF6, UART_NREG);
 216	msm_write(port, 0x0F, UART_DREG);
 217	msm_write(port, 0x0A, UART_MNDREG);
 218	port->uartclk = 1843200;
 219}
 220
 221static void msm_serial_set_mnd_regs(struct uart_port *port)
 222{
 223	struct msm_port *msm_port = UART_TO_MSM(port);
 224
 225	/*
 226	 * These registers don't exist so we change the clk input rate
 227	 * on uartdm hardware instead
 228	 */
 229	if (msm_port->is_uartdm)
 230		return;
 231
 232	if (port->uartclk == 19200000)
 233		msm_serial_set_mnd_regs_tcxo(port);
 234	else if (port->uartclk == 4800000)
 235		msm_serial_set_mnd_regs_tcxoby4(port);
 236}
 237
 238static void msm_handle_tx(struct uart_port *port);
 239static void msm_start_rx_dma(struct msm_port *msm_port);
 240
 241static void msm_stop_dma(struct uart_port *port, struct msm_dma *dma)
 242{
 243	struct device *dev = port->dev;
 244	unsigned int mapped;
 245	u32 val;
 246
 247	mapped = dma->count;
 248	dma->count = 0;
 249
 250	dmaengine_terminate_all(dma->chan);
 251
 252	/*
 253	 * DMA Stall happens if enqueue and flush command happens concurrently.
 254	 * For example before changing the baud rate/protocol configuration and
 255	 * sending flush command to ADM, disable the channel of UARTDM.
 256	 * Note: should not reset the receiver here immediately as it is not
 257	 * suggested to do disable/reset or reset/disable at the same time.
 258	 */
 259	val = msm_read(port, UARTDM_DMEN);
 260	val &= ~dma->enable_bit;
 261	msm_write(port, val, UARTDM_DMEN);
 262
 263	if (mapped)
 264		dma_unmap_single(dev, dma->phys, mapped, dma->dir);
 265}
 266
 267static void msm_release_dma(struct msm_port *msm_port)
 268{
 269	struct msm_dma *dma;
 270
 271	dma = &msm_port->tx_dma;
 272	if (dma->chan) {
 273		msm_stop_dma(&msm_port->uart, dma);
 274		dma_release_channel(dma->chan);
 275	}
 276
 277	memset(dma, 0, sizeof(*dma));
 278
 279	dma = &msm_port->rx_dma;
 280	if (dma->chan) {
 281		msm_stop_dma(&msm_port->uart, dma);
 282		dma_release_channel(dma->chan);
 283		kfree(dma->virt);
 284	}
 285
 286	memset(dma, 0, sizeof(*dma));
 287}
 288
 289static void msm_request_tx_dma(struct msm_port *msm_port, resource_size_t base)
 290{
 291	struct device *dev = msm_port->uart.dev;
 292	struct dma_slave_config conf;
 293	struct msm_dma *dma;
 294	u32 crci = 0;
 295	int ret;
 296
 297	dma = &msm_port->tx_dma;
 298
 299	/* allocate DMA resources, if available */
 300	dma->chan = dma_request_chan(dev, "tx");
 301	if (IS_ERR(dma->chan))
 302		goto no_tx;
 303
 304	of_property_read_u32(dev->of_node, "qcom,tx-crci", &crci);
 305
 306	memset(&conf, 0, sizeof(conf));
 307	conf.direction = DMA_MEM_TO_DEV;
 308	conf.device_fc = true;
 309	conf.dst_addr = base + UARTDM_TF;
 310	conf.dst_maxburst = UARTDM_BURST_SIZE;
 311	conf.slave_id = crci;
 312
 313	ret = dmaengine_slave_config(dma->chan, &conf);
 314	if (ret)
 315		goto rel_tx;
 316
 317	dma->dir = DMA_TO_DEVICE;
 318
 319	if (msm_port->is_uartdm < UARTDM_1P4)
 320		dma->enable_bit = UARTDM_DMEN_TX_DM_ENABLE;
 321	else
 322		dma->enable_bit = UARTDM_DMEN_TX_BAM_ENABLE;
 323
 324	return;
 325
 326rel_tx:
 327	dma_release_channel(dma->chan);
 328no_tx:
 329	memset(dma, 0, sizeof(*dma));
 330}
 331
 332static void msm_request_rx_dma(struct msm_port *msm_port, resource_size_t base)
 333{
 334	struct device *dev = msm_port->uart.dev;
 335	struct dma_slave_config conf;
 336	struct msm_dma *dma;
 337	u32 crci = 0;
 338	int ret;
 339
 340	dma = &msm_port->rx_dma;
 341
 342	/* allocate DMA resources, if available */
 343	dma->chan = dma_request_chan(dev, "rx");
 344	if (IS_ERR(dma->chan))
 345		goto no_rx;
 346
 347	of_property_read_u32(dev->of_node, "qcom,rx-crci", &crci);
 348
 349	dma->virt = kzalloc(UARTDM_RX_SIZE, GFP_KERNEL);
 350	if (!dma->virt)
 351		goto rel_rx;
 352
 353	memset(&conf, 0, sizeof(conf));
 354	conf.direction = DMA_DEV_TO_MEM;
 355	conf.device_fc = true;
 356	conf.src_addr = base + UARTDM_RF;
 357	conf.src_maxburst = UARTDM_BURST_SIZE;
 358	conf.slave_id = crci;
 359
 360	ret = dmaengine_slave_config(dma->chan, &conf);
 361	if (ret)
 362		goto err;
 363
 364	dma->dir = DMA_FROM_DEVICE;
 365
 366	if (msm_port->is_uartdm < UARTDM_1P4)
 367		dma->enable_bit = UARTDM_DMEN_RX_DM_ENABLE;
 368	else
 369		dma->enable_bit = UARTDM_DMEN_RX_BAM_ENABLE;
 370
 371	return;
 372err:
 373	kfree(dma->virt);
 374rel_rx:
 375	dma_release_channel(dma->chan);
 376no_rx:
 377	memset(dma, 0, sizeof(*dma));
 378}
 379
 380static inline void msm_wait_for_xmitr(struct uart_port *port)
 381{
 382	unsigned int timeout = 500000;
 383
 384	while (!(msm_read(port, UART_SR) & UART_SR_TX_EMPTY)) {
 385		if (msm_read(port, UART_ISR) & UART_ISR_TX_READY)
 386			break;
 387		udelay(1);
 388		if (!timeout--)
 389			break;
 390	}
 391	msm_write(port, UART_CR_CMD_RESET_TX_READY, UART_CR);
 392}
 393
 394static void msm_stop_tx(struct uart_port *port)
 395{
 396	struct msm_port *msm_port = UART_TO_MSM(port);
 397
 398	msm_port->imr &= ~UART_IMR_TXLEV;
 399	msm_write(port, msm_port->imr, UART_IMR);
 400}
 401
 402static void msm_start_tx(struct uart_port *port)
 403{
 404	struct msm_port *msm_port = UART_TO_MSM(port);
 405	struct msm_dma *dma = &msm_port->tx_dma;
 406
 407	/* Already started in DMA mode */
 408	if (dma->count)
 409		return;
 410
 411	msm_port->imr |= UART_IMR_TXLEV;
 412	msm_write(port, msm_port->imr, UART_IMR);
 413}
 414
 415static void msm_reset_dm_count(struct uart_port *port, int count)
 416{
 417	msm_wait_for_xmitr(port);
 418	msm_write(port, count, UARTDM_NCF_TX);
 419	msm_read(port, UARTDM_NCF_TX);
 420}
 421
 422static void msm_complete_tx_dma(void *args)
 423{
 424	struct msm_port *msm_port = args;
 425	struct uart_port *port = &msm_port->uart;
 426	struct circ_buf *xmit = &port->state->xmit;
 427	struct msm_dma *dma = &msm_port->tx_dma;
 428	struct dma_tx_state state;
 429	enum dma_status status;
 430	unsigned long flags;
 431	unsigned int count;
 432	u32 val;
 433
 434	spin_lock_irqsave(&port->lock, flags);
 435
 436	/* Already stopped */
 437	if (!dma->count)
 438		goto done;
 439
 440	status = dmaengine_tx_status(dma->chan, dma->cookie, &state);
 441
 442	dma_unmap_single(port->dev, dma->phys, dma->count, dma->dir);
 443
 444	val = msm_read(port, UARTDM_DMEN);
 445	val &= ~dma->enable_bit;
 446	msm_write(port, val, UARTDM_DMEN);
 447
 448	if (msm_port->is_uartdm > UARTDM_1P3) {
 449		msm_write(port, UART_CR_CMD_RESET_TX, UART_CR);
 450		msm_write(port, UART_CR_TX_ENABLE, UART_CR);
 451	}
 452
 453	count = dma->count - state.residue;
 454	port->icount.tx += count;
 455	dma->count = 0;
 456
 457	xmit->tail += count;
 458	xmit->tail &= UART_XMIT_SIZE - 1;
 459
 460	/* Restore "Tx FIFO below watermark" interrupt */
 461	msm_port->imr |= UART_IMR_TXLEV;
 462	msm_write(port, msm_port->imr, UART_IMR);
 463
 464	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 465		uart_write_wakeup(port);
 466
 467	msm_handle_tx(port);
 468done:
 469	spin_unlock_irqrestore(&port->lock, flags);
 470}
 471
 472static int msm_handle_tx_dma(struct msm_port *msm_port, unsigned int count)
 473{
 474	struct circ_buf *xmit = &msm_port->uart.state->xmit;
 475	struct uart_port *port = &msm_port->uart;
 476	struct msm_dma *dma = &msm_port->tx_dma;
 477	void *cpu_addr;
 478	int ret;
 479	u32 val;
 480
 481	cpu_addr = &xmit->buf[xmit->tail];
 482
 483	dma->phys = dma_map_single(port->dev, cpu_addr, count, dma->dir);
 484	ret = dma_mapping_error(port->dev, dma->phys);
 485	if (ret)
 486		return ret;
 487
 488	dma->desc = dmaengine_prep_slave_single(dma->chan, dma->phys,
 489						count, DMA_MEM_TO_DEV,
 490						DMA_PREP_INTERRUPT |
 491						DMA_PREP_FENCE);
 492	if (!dma->desc) {
 493		ret = -EIO;
 494		goto unmap;
 495	}
 496
 497	dma->desc->callback = msm_complete_tx_dma;
 498	dma->desc->callback_param = msm_port;
 499
 500	dma->cookie = dmaengine_submit(dma->desc);
 501	ret = dma_submit_error(dma->cookie);
 502	if (ret)
 503		goto unmap;
 504
 505	/*
 506	 * Using DMA complete for Tx FIFO reload, no need for
 507	 * "Tx FIFO below watermark" one, disable it
 508	 */
 509	msm_port->imr &= ~UART_IMR_TXLEV;
 510	msm_write(port, msm_port->imr, UART_IMR);
 511
 512	dma->count = count;
 513
 514	val = msm_read(port, UARTDM_DMEN);
 515	val |= dma->enable_bit;
 516
 517	if (msm_port->is_uartdm < UARTDM_1P4)
 518		msm_write(port, val, UARTDM_DMEN);
 519
 520	msm_reset_dm_count(port, count);
 521
 522	if (msm_port->is_uartdm > UARTDM_1P3)
 523		msm_write(port, val, UARTDM_DMEN);
 524
 525	dma_async_issue_pending(dma->chan);
 526	return 0;
 527unmap:
 528	dma_unmap_single(port->dev, dma->phys, count, dma->dir);
 529	return ret;
 530}
 531
 532static void msm_complete_rx_dma(void *args)
 533{
 534	struct msm_port *msm_port = args;
 535	struct uart_port *port = &msm_port->uart;
 536	struct tty_port *tport = &port->state->port;
 537	struct msm_dma *dma = &msm_port->rx_dma;
 538	int count = 0, i, sysrq;
 539	unsigned long flags;
 540	u32 val;
 541
 542	spin_lock_irqsave(&port->lock, flags);
 543
 544	/* Already stopped */
 545	if (!dma->count)
 546		goto done;
 547
 548	val = msm_read(port, UARTDM_DMEN);
 549	val &= ~dma->enable_bit;
 550	msm_write(port, val, UARTDM_DMEN);
 551
 552	if (msm_read(port, UART_SR) & UART_SR_OVERRUN) {
 553		port->icount.overrun++;
 554		tty_insert_flip_char(tport, 0, TTY_OVERRUN);
 555		msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
 556	}
 557
 558	count = msm_read(port, UARTDM_RX_TOTAL_SNAP);
 559
 560	port->icount.rx += count;
 561
 562	dma->count = 0;
 563
 564	dma_unmap_single(port->dev, dma->phys, UARTDM_RX_SIZE, dma->dir);
 565
 566	for (i = 0; i < count; i++) {
 567		char flag = TTY_NORMAL;
 568
 569		if (msm_port->break_detected && dma->virt[i] == 0) {
 570			port->icount.brk++;
 571			flag = TTY_BREAK;
 572			msm_port->break_detected = false;
 573			if (uart_handle_break(port))
 574				continue;
 575		}
 576
 577		if (!(port->read_status_mask & UART_SR_RX_BREAK))
 578			flag = TTY_NORMAL;
 579
 580		spin_unlock_irqrestore(&port->lock, flags);
 581		sysrq = uart_handle_sysrq_char(port, dma->virt[i]);
 582		spin_lock_irqsave(&port->lock, flags);
 583		if (!sysrq)
 584			tty_insert_flip_char(tport, dma->virt[i], flag);
 585	}
 586
 587	msm_start_rx_dma(msm_port);
 588done:
 589	spin_unlock_irqrestore(&port->lock, flags);
 590
 591	if (count)
 592		tty_flip_buffer_push(tport);
 593}
 594
 595static void msm_start_rx_dma(struct msm_port *msm_port)
 596{
 597	struct msm_dma *dma = &msm_port->rx_dma;
 598	struct uart_port *uart = &msm_port->uart;
 599	u32 val;
 600	int ret;
 601
 602	if (!dma->chan)
 603		return;
 604
 605	dma->phys = dma_map_single(uart->dev, dma->virt,
 606				   UARTDM_RX_SIZE, dma->dir);
 607	ret = dma_mapping_error(uart->dev, dma->phys);
 608	if (ret)
 609		goto sw_mode;
 610
 611	dma->desc = dmaengine_prep_slave_single(dma->chan, dma->phys,
 612						UARTDM_RX_SIZE, DMA_DEV_TO_MEM,
 613						DMA_PREP_INTERRUPT);
 614	if (!dma->desc)
 615		goto unmap;
 616
 617	dma->desc->callback = msm_complete_rx_dma;
 618	dma->desc->callback_param = msm_port;
 619
 620	dma->cookie = dmaengine_submit(dma->desc);
 621	ret = dma_submit_error(dma->cookie);
 622	if (ret)
 623		goto unmap;
 624	/*
 625	 * Using DMA for FIFO off-load, no need for "Rx FIFO over
 626	 * watermark" or "stale" interrupts, disable them
 627	 */
 628	msm_port->imr &= ~(UART_IMR_RXLEV | UART_IMR_RXSTALE);
 629
 630	/*
 631	 * Well, when DMA is ADM3 engine(implied by <= UARTDM v1.3),
 632	 * we need RXSTALE to flush input DMA fifo to memory
 633	 */
 634	if (msm_port->is_uartdm < UARTDM_1P4)
 635		msm_port->imr |= UART_IMR_RXSTALE;
 636
 637	msm_write(uart, msm_port->imr, UART_IMR);
 638
 639	dma->count = UARTDM_RX_SIZE;
 640
 641	dma_async_issue_pending(dma->chan);
 642
 643	msm_write(uart, UART_CR_CMD_RESET_STALE_INT, UART_CR);
 644	msm_write(uart, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
 645
 646	val = msm_read(uart, UARTDM_DMEN);
 647	val |= dma->enable_bit;
 648
 649	if (msm_port->is_uartdm < UARTDM_1P4)
 650		msm_write(uart, val, UARTDM_DMEN);
 651
 652	msm_write(uart, UARTDM_RX_SIZE, UARTDM_DMRX);
 653
 654	if (msm_port->is_uartdm > UARTDM_1P3)
 655		msm_write(uart, val, UARTDM_DMEN);
 656
 657	return;
 658unmap:
 659	dma_unmap_single(uart->dev, dma->phys, UARTDM_RX_SIZE, dma->dir);
 660
 661sw_mode:
 662	/*
 663	 * Switch from DMA to SW/FIFO mode. After clearing Rx BAM (UARTDM_DMEN),
 664	 * receiver must be reset.
 665	 */
 666	msm_write(uart, UART_CR_CMD_RESET_RX, UART_CR);
 667	msm_write(uart, UART_CR_RX_ENABLE, UART_CR);
 668
 669	msm_write(uart, UART_CR_CMD_RESET_STALE_INT, UART_CR);
 670	msm_write(uart, 0xFFFFFF, UARTDM_DMRX);
 671	msm_write(uart, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
 672
 673	/* Re-enable RX interrupts */
 674	msm_port->imr |= (UART_IMR_RXLEV | UART_IMR_RXSTALE);
 675	msm_write(uart, msm_port->imr, UART_IMR);
 676}
 677
 678static void msm_stop_rx(struct uart_port *port)
 679{
 680	struct msm_port *msm_port = UART_TO_MSM(port);
 681	struct msm_dma *dma = &msm_port->rx_dma;
 682
 683	msm_port->imr &= ~(UART_IMR_RXLEV | UART_IMR_RXSTALE);
 684	msm_write(port, msm_port->imr, UART_IMR);
 685
 686	if (dma->chan)
 687		msm_stop_dma(port, dma);
 688}
 689
 690static void msm_enable_ms(struct uart_port *port)
 691{
 692	struct msm_port *msm_port = UART_TO_MSM(port);
 693
 694	msm_port->imr |= UART_IMR_DELTA_CTS;
 695	msm_write(port, msm_port->imr, UART_IMR);
 696}
 697
 698static void msm_handle_rx_dm(struct uart_port *port, unsigned int misr)
 699	__must_hold(&port->lock)
 700{
 701	struct tty_port *tport = &port->state->port;
 702	unsigned int sr;
 703	int count = 0;
 704	struct msm_port *msm_port = UART_TO_MSM(port);
 705
 706	if ((msm_read(port, UART_SR) & UART_SR_OVERRUN)) {
 707		port->icount.overrun++;
 708		tty_insert_flip_char(tport, 0, TTY_OVERRUN);
 709		msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
 710	}
 711
 712	if (misr & UART_IMR_RXSTALE) {
 713		count = msm_read(port, UARTDM_RX_TOTAL_SNAP) -
 714			msm_port->old_snap_state;
 715		msm_port->old_snap_state = 0;
 716	} else {
 717		count = 4 * (msm_read(port, UART_RFWR));
 718		msm_port->old_snap_state += count;
 719	}
 720
 721	/* TODO: Precise error reporting */
 722
 723	port->icount.rx += count;
 724
 725	while (count > 0) {
 726		unsigned char buf[4];
 727		int sysrq, r_count, i;
 728
 729		sr = msm_read(port, UART_SR);
 730		if ((sr & UART_SR_RX_READY) == 0) {
 731			msm_port->old_snap_state -= count;
 732			break;
 733		}
 734
 735		ioread32_rep(port->membase + UARTDM_RF, buf, 1);
 736		r_count = min_t(int, count, sizeof(buf));
 737
 738		for (i = 0; i < r_count; i++) {
 739			char flag = TTY_NORMAL;
 740
 741			if (msm_port->break_detected && buf[i] == 0) {
 742				port->icount.brk++;
 743				flag = TTY_BREAK;
 744				msm_port->break_detected = false;
 745				if (uart_handle_break(port))
 746					continue;
 747			}
 748
 749			if (!(port->read_status_mask & UART_SR_RX_BREAK))
 750				flag = TTY_NORMAL;
 751
 752			spin_unlock(&port->lock);
 753			sysrq = uart_handle_sysrq_char(port, buf[i]);
 754			spin_lock(&port->lock);
 755			if (!sysrq)
 756				tty_insert_flip_char(tport, buf[i], flag);
 757		}
 758		count -= r_count;
 759	}
 760
 761	spin_unlock(&port->lock);
 762	tty_flip_buffer_push(tport);
 763	spin_lock(&port->lock);
 764
 765	if (misr & (UART_IMR_RXSTALE))
 766		msm_write(port, UART_CR_CMD_RESET_STALE_INT, UART_CR);
 767	msm_write(port, 0xFFFFFF, UARTDM_DMRX);
 768	msm_write(port, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
 769
 770	/* Try to use DMA */
 771	msm_start_rx_dma(msm_port);
 772}
 773
 774static void msm_handle_rx(struct uart_port *port)
 775	__must_hold(&port->lock)
 776{
 777	struct tty_port *tport = &port->state->port;
 778	unsigned int sr;
 779
 780	/*
 781	 * Handle overrun. My understanding of the hardware is that overrun
 782	 * is not tied to the RX buffer, so we handle the case out of band.
 783	 */
 784	if ((msm_read(port, UART_SR) & UART_SR_OVERRUN)) {
 785		port->icount.overrun++;
 786		tty_insert_flip_char(tport, 0, TTY_OVERRUN);
 787		msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
 788	}
 789
 790	/* and now the main RX loop */
 791	while ((sr = msm_read(port, UART_SR)) & UART_SR_RX_READY) {
 792		unsigned int c;
 793		char flag = TTY_NORMAL;
 794		int sysrq;
 795
 796		c = msm_read(port, UART_RF);
 797
 798		if (sr & UART_SR_RX_BREAK) {
 799			port->icount.brk++;
 800			if (uart_handle_break(port))
 801				continue;
 802		} else if (sr & UART_SR_PAR_FRAME_ERR) {
 803			port->icount.frame++;
 804		} else {
 805			port->icount.rx++;
 806		}
 807
 808		/* Mask conditions we're ignorning. */
 809		sr &= port->read_status_mask;
 810
 811		if (sr & UART_SR_RX_BREAK)
 812			flag = TTY_BREAK;
 813		else if (sr & UART_SR_PAR_FRAME_ERR)
 814			flag = TTY_FRAME;
 815
 816		spin_unlock(&port->lock);
 817		sysrq = uart_handle_sysrq_char(port, c);
 818		spin_lock(&port->lock);
 819		if (!sysrq)
 820			tty_insert_flip_char(tport, c, flag);
 821	}
 822
 823	spin_unlock(&port->lock);
 824	tty_flip_buffer_push(tport);
 825	spin_lock(&port->lock);
 826}
 827
 828static void msm_handle_tx_pio(struct uart_port *port, unsigned int tx_count)
 829{
 830	struct circ_buf *xmit = &port->state->xmit;
 831	struct msm_port *msm_port = UART_TO_MSM(port);
 832	unsigned int num_chars;
 833	unsigned int tf_pointer = 0;
 834	void __iomem *tf;
 835
 836	if (msm_port->is_uartdm)
 837		tf = port->membase + UARTDM_TF;
 838	else
 839		tf = port->membase + UART_TF;
 840
 841	if (tx_count && msm_port->is_uartdm)
 842		msm_reset_dm_count(port, tx_count);
 843
 844	while (tf_pointer < tx_count) {
 845		int i;
 846		char buf[4] = { 0 };
 847
 848		if (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
 849			break;
 850
 851		if (msm_port->is_uartdm)
 852			num_chars = min(tx_count - tf_pointer,
 853					(unsigned int)sizeof(buf));
 854		else
 855			num_chars = 1;
 856
 857		for (i = 0; i < num_chars; i++) {
 858			buf[i] = xmit->buf[xmit->tail + i];
 859			port->icount.tx++;
 860		}
 861
 862		iowrite32_rep(tf, buf, 1);
 863		xmit->tail = (xmit->tail + num_chars) & (UART_XMIT_SIZE - 1);
 864		tf_pointer += num_chars;
 865	}
 866
 867	/* disable tx interrupts if nothing more to send */
 868	if (uart_circ_empty(xmit))
 869		msm_stop_tx(port);
 870
 871	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 872		uart_write_wakeup(port);
 873}
 874
 875static void msm_handle_tx(struct uart_port *port)
 876{
 877	struct msm_port *msm_port = UART_TO_MSM(port);
 878	struct circ_buf *xmit = &msm_port->uart.state->xmit;
 879	struct msm_dma *dma = &msm_port->tx_dma;
 880	unsigned int pio_count, dma_count, dma_min;
 881	char buf[4] = { 0 };
 882	void __iomem *tf;
 883	int err = 0;
 884
 885	if (port->x_char) {
 886		if (msm_port->is_uartdm)
 887			tf = port->membase + UARTDM_TF;
 888		else
 889			tf = port->membase + UART_TF;
 890
 891		buf[0] = port->x_char;
 892
 893		if (msm_port->is_uartdm)
 894			msm_reset_dm_count(port, 1);
 895
 896		iowrite32_rep(tf, buf, 1);
 897		port->icount.tx++;
 898		port->x_char = 0;
 899		return;
 900	}
 901
 902	if (uart_circ_empty(xmit) || uart_tx_stopped(port)) {
 903		msm_stop_tx(port);
 904		return;
 905	}
 906
 907	pio_count = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
 908	dma_count = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
 909
 910	dma_min = 1;	/* Always DMA */
 911	if (msm_port->is_uartdm > UARTDM_1P3) {
 912		dma_count = UARTDM_TX_AIGN(dma_count);
 913		dma_min = UARTDM_BURST_SIZE;
 914	} else {
 915		if (dma_count > UARTDM_TX_MAX)
 916			dma_count = UARTDM_TX_MAX;
 917	}
 918
 919	if (pio_count > port->fifosize)
 920		pio_count = port->fifosize;
 921
 922	if (!dma->chan || dma_count < dma_min)
 923		msm_handle_tx_pio(port, pio_count);
 924	else
 925		err = msm_handle_tx_dma(msm_port, dma_count);
 926
 927	if (err)	/* fall back to PIO mode */
 928		msm_handle_tx_pio(port, pio_count);
 929}
 930
 931static void msm_handle_delta_cts(struct uart_port *port)
 932{
 933	msm_write(port, UART_CR_CMD_RESET_CTS, UART_CR);
 934	port->icount.cts++;
 935	wake_up_interruptible(&port->state->port.delta_msr_wait);
 936}
 937
 938static irqreturn_t msm_uart_irq(int irq, void *dev_id)
 939{
 940	struct uart_port *port = dev_id;
 941	struct msm_port *msm_port = UART_TO_MSM(port);
 942	struct msm_dma *dma = &msm_port->rx_dma;
 943	unsigned long flags;
 944	unsigned int misr;
 945	u32 val;
 946
 947	spin_lock_irqsave(&port->lock, flags);
 948	misr = msm_read(port, UART_MISR);
 949	msm_write(port, 0, UART_IMR); /* disable interrupt */
 950
 951	if (misr & UART_IMR_RXBREAK_START) {
 952		msm_port->break_detected = true;
 953		msm_write(port, UART_CR_CMD_RESET_RXBREAK_START, UART_CR);
 954	}
 955
 956	if (misr & (UART_IMR_RXLEV | UART_IMR_RXSTALE)) {
 957		if (dma->count) {
 958			val = UART_CR_CMD_STALE_EVENT_DISABLE;
 959			msm_write(port, val, UART_CR);
 960			val = UART_CR_CMD_RESET_STALE_INT;
 961			msm_write(port, val, UART_CR);
 962			/*
 963			 * Flush DMA input fifo to memory, this will also
 964			 * trigger DMA RX completion
 965			 */
 966			dmaengine_terminate_all(dma->chan);
 967		} else if (msm_port->is_uartdm) {
 968			msm_handle_rx_dm(port, misr);
 969		} else {
 970			msm_handle_rx(port);
 971		}
 972	}
 973	if (misr & UART_IMR_TXLEV)
 974		msm_handle_tx(port);
 975	if (misr & UART_IMR_DELTA_CTS)
 976		msm_handle_delta_cts(port);
 977
 978	msm_write(port, msm_port->imr, UART_IMR); /* restore interrupt */
 979	spin_unlock_irqrestore(&port->lock, flags);
 980
 981	return IRQ_HANDLED;
 982}
 983
 984static unsigned int msm_tx_empty(struct uart_port *port)
 985{
 986	return (msm_read(port, UART_SR) & UART_SR_TX_EMPTY) ? TIOCSER_TEMT : 0;
 987}
 988
 989static unsigned int msm_get_mctrl(struct uart_port *port)
 990{
 991	return TIOCM_CAR | TIOCM_CTS | TIOCM_DSR | TIOCM_RTS;
 992}
 993
 994static void msm_reset(struct uart_port *port)
 995{
 996	struct msm_port *msm_port = UART_TO_MSM(port);
 997	unsigned int mr;
 998
 999	/* reset everything */
1000	msm_write(port, UART_CR_CMD_RESET_RX, UART_CR);
1001	msm_write(port, UART_CR_CMD_RESET_TX, UART_CR);
1002	msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
1003	msm_write(port, UART_CR_CMD_RESET_BREAK_INT, UART_CR);
1004	msm_write(port, UART_CR_CMD_RESET_CTS, UART_CR);
1005	msm_write(port, UART_CR_CMD_RESET_RFR, UART_CR);
1006	mr = msm_read(port, UART_MR1);
1007	mr &= ~UART_MR1_RX_RDY_CTL;
1008	msm_write(port, mr, UART_MR1);
1009
1010	/* Disable DM modes */
1011	if (msm_port->is_uartdm)
1012		msm_write(port, 0, UARTDM_DMEN);
1013}
1014
1015static void msm_set_mctrl(struct uart_port *port, unsigned int mctrl)
1016{
1017	unsigned int mr;
1018
1019	mr = msm_read(port, UART_MR1);
1020
1021	if (!(mctrl & TIOCM_RTS)) {
1022		mr &= ~UART_MR1_RX_RDY_CTL;
1023		msm_write(port, mr, UART_MR1);
1024		msm_write(port, UART_CR_CMD_RESET_RFR, UART_CR);
1025	} else {
1026		mr |= UART_MR1_RX_RDY_CTL;
1027		msm_write(port, mr, UART_MR1);
1028	}
1029}
1030
1031static void msm_break_ctl(struct uart_port *port, int break_ctl)
1032{
1033	if (break_ctl)
1034		msm_write(port, UART_CR_CMD_START_BREAK, UART_CR);
1035	else
1036		msm_write(port, UART_CR_CMD_STOP_BREAK, UART_CR);
1037}
1038
1039struct msm_baud_map {
1040	u16	divisor;
1041	u8	code;
1042	u8	rxstale;
1043};
1044
1045static const struct msm_baud_map *
1046msm_find_best_baud(struct uart_port *port, unsigned int baud,
1047		   unsigned long *rate)
1048{
1049	struct msm_port *msm_port = UART_TO_MSM(port);
1050	unsigned int divisor, result;
1051	unsigned long target, old, best_rate = 0, diff, best_diff = ULONG_MAX;
1052	const struct msm_baud_map *entry, *end, *best;
1053	static const struct msm_baud_map table[] = {
1054		{    1, 0xff, 31 },
1055		{    2, 0xee, 16 },
1056		{    3, 0xdd,  8 },
1057		{    4, 0xcc,  6 },
1058		{    6, 0xbb,  6 },
1059		{    8, 0xaa,  6 },
1060		{   12, 0x99,  6 },
1061		{   16, 0x88,  1 },
1062		{   24, 0x77,  1 },
1063		{   32, 0x66,  1 },
1064		{   48, 0x55,  1 },
1065		{   96, 0x44,  1 },
1066		{  192, 0x33,  1 },
1067		{  384, 0x22,  1 },
1068		{  768, 0x11,  1 },
1069		{ 1536, 0x00,  1 },
1070	};
1071
1072	best = table; /* Default to smallest divider */
1073	target = clk_round_rate(msm_port->clk, 16 * baud);
1074	divisor = DIV_ROUND_CLOSEST(target, 16 * baud);
1075
1076	end = table + ARRAY_SIZE(table);
1077	entry = table;
1078	while (entry < end) {
1079		if (entry->divisor <= divisor) {
1080			result = target / entry->divisor / 16;
1081			diff = abs(result - baud);
1082
1083			/* Keep track of best entry */
1084			if (diff < best_diff) {
1085				best_diff = diff;
1086				best = entry;
1087				best_rate = target;
1088			}
1089
1090			if (result == baud)
1091				break;
1092		} else if (entry->divisor > divisor) {
1093			old = target;
1094			target = clk_round_rate(msm_port->clk, old + 1);
1095			/*
1096			 * The rate didn't get any faster so we can't do
1097			 * better at dividing it down
1098			 */
1099			if (target == old)
1100				break;
1101
1102			/* Start the divisor search over at this new rate */
1103			entry = table;
1104			divisor = DIV_ROUND_CLOSEST(target, 16 * baud);
1105			continue;
1106		}
1107		entry++;
1108	}
1109
1110	*rate = best_rate;
1111	return best;
1112}
1113
1114static int msm_set_baud_rate(struct uart_port *port, unsigned int baud,
1115			     unsigned long *saved_flags)
1116{
1117	unsigned int rxstale, watermark, mask;
1118	struct msm_port *msm_port = UART_TO_MSM(port);
1119	const struct msm_baud_map *entry;
1120	unsigned long flags, rate;
1121
1122	flags = *saved_flags;
1123	spin_unlock_irqrestore(&port->lock, flags);
1124
1125	entry = msm_find_best_baud(port, baud, &rate);
1126	clk_set_rate(msm_port->clk, rate);
1127	baud = rate / 16 / entry->divisor;
1128
1129	spin_lock_irqsave(&port->lock, flags);
1130	*saved_flags = flags;
1131	port->uartclk = rate;
1132
1133	msm_write(port, entry->code, UART_CSR);
1134
1135	/* RX stale watermark */
1136	rxstale = entry->rxstale;
1137	watermark = UART_IPR_STALE_LSB & rxstale;
1138	if (msm_port->is_uartdm) {
1139		mask = UART_DM_IPR_STALE_TIMEOUT_MSB;
1140	} else {
1141		watermark |= UART_IPR_RXSTALE_LAST;
1142		mask = UART_IPR_STALE_TIMEOUT_MSB;
1143	}
1144
1145	watermark |= mask & (rxstale << 2);
1146
1147	msm_write(port, watermark, UART_IPR);
1148
1149	/* set RX watermark */
1150	watermark = (port->fifosize * 3) / 4;
1151	msm_write(port, watermark, UART_RFWR);
1152
1153	/* set TX watermark */
1154	msm_write(port, 10, UART_TFWR);
1155
1156	msm_write(port, UART_CR_CMD_PROTECTION_EN, UART_CR);
1157	msm_reset(port);
1158
1159	/* Enable RX and TX */
1160	msm_write(port, UART_CR_TX_ENABLE | UART_CR_RX_ENABLE, UART_CR);
1161
1162	/* turn on RX and CTS interrupts */
1163	msm_port->imr = UART_IMR_RXLEV | UART_IMR_RXSTALE |
1164			UART_IMR_CURRENT_CTS | UART_IMR_RXBREAK_START;
1165
1166	msm_write(port, msm_port->imr, UART_IMR);
1167
1168	if (msm_port->is_uartdm) {
1169		msm_write(port, UART_CR_CMD_RESET_STALE_INT, UART_CR);
1170		msm_write(port, 0xFFFFFF, UARTDM_DMRX);
1171		msm_write(port, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
1172	}
1173
1174	return baud;
1175}
1176
1177static void msm_init_clock(struct uart_port *port)
1178{
1179	struct msm_port *msm_port = UART_TO_MSM(port);
1180
1181	clk_prepare_enable(msm_port->clk);
1182	clk_prepare_enable(msm_port->pclk);
1183	msm_serial_set_mnd_regs(port);
1184}
1185
1186static int msm_startup(struct uart_port *port)
1187{
1188	struct msm_port *msm_port = UART_TO_MSM(port);
1189	unsigned int data, rfr_level, mask;
1190	int ret;
1191
1192	snprintf(msm_port->name, sizeof(msm_port->name),
1193		 "msm_serial%d", port->line);
1194
1195	msm_init_clock(port);
1196
1197	if (likely(port->fifosize > 12))
1198		rfr_level = port->fifosize - 12;
1199	else
1200		rfr_level = port->fifosize;
1201
1202	/* set automatic RFR level */
1203	data = msm_read(port, UART_MR1);
1204
1205	if (msm_port->is_uartdm)
1206		mask = UART_DM_MR1_AUTO_RFR_LEVEL1;
1207	else
1208		mask = UART_MR1_AUTO_RFR_LEVEL1;
1209
1210	data &= ~mask;
1211	data &= ~UART_MR1_AUTO_RFR_LEVEL0;
1212	data |= mask & (rfr_level << 2);
1213	data |= UART_MR1_AUTO_RFR_LEVEL0 & rfr_level;
1214	msm_write(port, data, UART_MR1);
1215
1216	if (msm_port->is_uartdm) {
1217		msm_request_tx_dma(msm_port, msm_port->uart.mapbase);
1218		msm_request_rx_dma(msm_port, msm_port->uart.mapbase);
1219	}
1220
1221	ret = request_irq(port->irq, msm_uart_irq, IRQF_TRIGGER_HIGH,
1222			  msm_port->name, port);
1223	if (unlikely(ret))
1224		goto err_irq;
1225
1226	return 0;
1227
1228err_irq:
1229	if (msm_port->is_uartdm)
1230		msm_release_dma(msm_port);
1231
1232	clk_disable_unprepare(msm_port->pclk);
1233	clk_disable_unprepare(msm_port->clk);
1234
1235	return ret;
1236}
1237
1238static void msm_shutdown(struct uart_port *port)
1239{
1240	struct msm_port *msm_port = UART_TO_MSM(port);
1241
1242	msm_port->imr = 0;
1243	msm_write(port, 0, UART_IMR); /* disable interrupts */
1244
1245	if (msm_port->is_uartdm)
1246		msm_release_dma(msm_port);
1247
1248	clk_disable_unprepare(msm_port->clk);
1249
1250	free_irq(port->irq, port);
1251}
1252
1253static void msm_set_termios(struct uart_port *port, struct ktermios *termios,
1254			    struct ktermios *old)
1255{
1256	struct msm_port *msm_port = UART_TO_MSM(port);
1257	struct msm_dma *dma = &msm_port->rx_dma;
1258	unsigned long flags;
1259	unsigned int baud, mr;
1260
1261	spin_lock_irqsave(&port->lock, flags);
1262
1263	if (dma->chan) /* Terminate if any */
1264		msm_stop_dma(port, dma);
1265
1266	/* calculate and set baud rate */
1267	baud = uart_get_baud_rate(port, termios, old, 300, 4000000);
1268	baud = msm_set_baud_rate(port, baud, &flags);
1269	if (tty_termios_baud_rate(termios))
1270		tty_termios_encode_baud_rate(termios, baud, baud);
1271
1272	/* calculate parity */
1273	mr = msm_read(port, UART_MR2);
1274	mr &= ~UART_MR2_PARITY_MODE;
1275	if (termios->c_cflag & PARENB) {
1276		if (termios->c_cflag & PARODD)
1277			mr |= UART_MR2_PARITY_MODE_ODD;
1278		else if (termios->c_cflag & CMSPAR)
1279			mr |= UART_MR2_PARITY_MODE_SPACE;
1280		else
1281			mr |= UART_MR2_PARITY_MODE_EVEN;
1282	}
1283
1284	/* calculate bits per char */
1285	mr &= ~UART_MR2_BITS_PER_CHAR;
1286	switch (termios->c_cflag & CSIZE) {
1287	case CS5:
1288		mr |= UART_MR2_BITS_PER_CHAR_5;
1289		break;
1290	case CS6:
1291		mr |= UART_MR2_BITS_PER_CHAR_6;
1292		break;
1293	case CS7:
1294		mr |= UART_MR2_BITS_PER_CHAR_7;
1295		break;
1296	case CS8:
1297	default:
1298		mr |= UART_MR2_BITS_PER_CHAR_8;
1299		break;
1300	}
1301
1302	/* calculate stop bits */
1303	mr &= ~(UART_MR2_STOP_BIT_LEN_ONE | UART_MR2_STOP_BIT_LEN_TWO);
1304	if (termios->c_cflag & CSTOPB)
1305		mr |= UART_MR2_STOP_BIT_LEN_TWO;
1306	else
1307		mr |= UART_MR2_STOP_BIT_LEN_ONE;
1308
1309	/* set parity, bits per char, and stop bit */
1310	msm_write(port, mr, UART_MR2);
1311
1312	/* calculate and set hardware flow control */
1313	mr = msm_read(port, UART_MR1);
1314	mr &= ~(UART_MR1_CTS_CTL | UART_MR1_RX_RDY_CTL);
1315	if (termios->c_cflag & CRTSCTS) {
1316		mr |= UART_MR1_CTS_CTL;
1317		mr |= UART_MR1_RX_RDY_CTL;
1318	}
1319	msm_write(port, mr, UART_MR1);
1320
1321	/* Configure status bits to ignore based on termio flags. */
1322	port->read_status_mask = 0;
1323	if (termios->c_iflag & INPCK)
1324		port->read_status_mask |= UART_SR_PAR_FRAME_ERR;
1325	if (termios->c_iflag & (IGNBRK | BRKINT | PARMRK))
1326		port->read_status_mask |= UART_SR_RX_BREAK;
1327
1328	uart_update_timeout(port, termios->c_cflag, baud);
1329
1330	/* Try to use DMA */
1331	msm_start_rx_dma(msm_port);
1332
1333	spin_unlock_irqrestore(&port->lock, flags);
1334}
1335
1336static const char *msm_type(struct uart_port *port)
1337{
1338	return "MSM";
1339}
1340
1341static void msm_release_port(struct uart_port *port)
1342{
1343	struct platform_device *pdev = to_platform_device(port->dev);
1344	struct resource *uart_resource;
1345	resource_size_t size;
1346
1347	uart_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1348	if (unlikely(!uart_resource))
1349		return;
1350	size = resource_size(uart_resource);
1351
1352	release_mem_region(port->mapbase, size);
1353	iounmap(port->membase);
1354	port->membase = NULL;
1355}
1356
1357static int msm_request_port(struct uart_port *port)
1358{
1359	struct platform_device *pdev = to_platform_device(port->dev);
1360	struct resource *uart_resource;
1361	resource_size_t size;
1362	int ret;
1363
1364	uart_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1365	if (unlikely(!uart_resource))
1366		return -ENXIO;
1367
1368	size = resource_size(uart_resource);
1369
1370	if (!request_mem_region(port->mapbase, size, "msm_serial"))
1371		return -EBUSY;
1372
1373	port->membase = ioremap(port->mapbase, size);
1374	if (!port->membase) {
1375		ret = -EBUSY;
1376		goto fail_release_port;
1377	}
1378
1379	return 0;
1380
1381fail_release_port:
1382	release_mem_region(port->mapbase, size);
1383	return ret;
1384}
1385
1386static void msm_config_port(struct uart_port *port, int flags)
1387{
1388	int ret;
1389
1390	if (flags & UART_CONFIG_TYPE) {
1391		port->type = PORT_MSM;
1392		ret = msm_request_port(port);
1393		if (ret)
1394			return;
1395	}
1396}
1397
1398static int msm_verify_port(struct uart_port *port, struct serial_struct *ser)
1399{
1400	if (unlikely(ser->type != PORT_UNKNOWN && ser->type != PORT_MSM))
1401		return -EINVAL;
1402	if (unlikely(port->irq != ser->irq))
1403		return -EINVAL;
1404	return 0;
1405}
1406
1407static void msm_power(struct uart_port *port, unsigned int state,
1408		      unsigned int oldstate)
1409{
1410	struct msm_port *msm_port = UART_TO_MSM(port);
1411
1412	switch (state) {
1413	case 0:
1414		clk_prepare_enable(msm_port->clk);
1415		clk_prepare_enable(msm_port->pclk);
1416		break;
1417	case 3:
1418		clk_disable_unprepare(msm_port->clk);
1419		clk_disable_unprepare(msm_port->pclk);
1420		break;
1421	default:
1422		pr_err("msm_serial: Unknown PM state %d\n", state);
1423	}
1424}
1425
1426#ifdef CONFIG_CONSOLE_POLL
1427static int msm_poll_get_char_single(struct uart_port *port)
1428{
1429	struct msm_port *msm_port = UART_TO_MSM(port);
1430	unsigned int rf_reg = msm_port->is_uartdm ? UARTDM_RF : UART_RF;
1431
1432	if (!(msm_read(port, UART_SR) & UART_SR_RX_READY))
1433		return NO_POLL_CHAR;
1434
1435	return msm_read(port, rf_reg) & 0xff;
1436}
1437
1438static int msm_poll_get_char_dm(struct uart_port *port)
1439{
1440	int c;
1441	static u32 slop;
1442	static int count;
1443	unsigned char *sp = (unsigned char *)&slop;
1444
1445	/* Check if a previous read had more than one char */
1446	if (count) {
1447		c = sp[sizeof(slop) - count];
1448		count--;
1449	/* Or if FIFO is empty */
1450	} else if (!(msm_read(port, UART_SR) & UART_SR_RX_READY)) {
1451		/*
1452		 * If RX packing buffer has less than a word, force stale to
1453		 * push contents into RX FIFO
1454		 */
1455		count = msm_read(port, UARTDM_RXFS);
1456		count = (count >> UARTDM_RXFS_BUF_SHIFT) & UARTDM_RXFS_BUF_MASK;
1457		if (count) {
1458			msm_write(port, UART_CR_CMD_FORCE_STALE, UART_CR);
1459			slop = msm_read(port, UARTDM_RF);
1460			c = sp[0];
1461			count--;
1462			msm_write(port, UART_CR_CMD_RESET_STALE_INT, UART_CR);
1463			msm_write(port, 0xFFFFFF, UARTDM_DMRX);
1464			msm_write(port, UART_CR_CMD_STALE_EVENT_ENABLE,
1465				  UART_CR);
1466		} else {
1467			c = NO_POLL_CHAR;
1468		}
1469	/* FIFO has a word */
1470	} else {
1471		slop = msm_read(port, UARTDM_RF);
1472		c = sp[0];
1473		count = sizeof(slop) - 1;
1474	}
1475
1476	return c;
1477}
1478
1479static int msm_poll_get_char(struct uart_port *port)
1480{
1481	u32 imr;
1482	int c;
1483	struct msm_port *msm_port = UART_TO_MSM(port);
1484
1485	/* Disable all interrupts */
1486	imr = msm_read(port, UART_IMR);
1487	msm_write(port, 0, UART_IMR);
1488
1489	if (msm_port->is_uartdm)
1490		c = msm_poll_get_char_dm(port);
1491	else
1492		c = msm_poll_get_char_single(port);
1493
1494	/* Enable interrupts */
1495	msm_write(port, imr, UART_IMR);
1496
1497	return c;
1498}
1499
1500static void msm_poll_put_char(struct uart_port *port, unsigned char c)
1501{
1502	u32 imr;
1503	struct msm_port *msm_port = UART_TO_MSM(port);
1504
1505	/* Disable all interrupts */
1506	imr = msm_read(port, UART_IMR);
1507	msm_write(port, 0, UART_IMR);
1508
1509	if (msm_port->is_uartdm)
1510		msm_reset_dm_count(port, 1);
1511
1512	/* Wait until FIFO is empty */
1513	while (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
1514		cpu_relax();
1515
1516	/* Write a character */
1517	msm_write(port, c, msm_port->is_uartdm ? UARTDM_TF : UART_TF);
1518
1519	/* Wait until FIFO is empty */
1520	while (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
1521		cpu_relax();
1522
1523	/* Enable interrupts */
1524	msm_write(port, imr, UART_IMR);
1525}
1526#endif
1527
1528static struct uart_ops msm_uart_pops = {
1529	.tx_empty = msm_tx_empty,
1530	.set_mctrl = msm_set_mctrl,
1531	.get_mctrl = msm_get_mctrl,
1532	.stop_tx = msm_stop_tx,
1533	.start_tx = msm_start_tx,
1534	.stop_rx = msm_stop_rx,
1535	.enable_ms = msm_enable_ms,
1536	.break_ctl = msm_break_ctl,
1537	.startup = msm_startup,
1538	.shutdown = msm_shutdown,
1539	.set_termios = msm_set_termios,
1540	.type = msm_type,
1541	.release_port = msm_release_port,
1542	.request_port = msm_request_port,
1543	.config_port = msm_config_port,
1544	.verify_port = msm_verify_port,
1545	.pm = msm_power,
1546#ifdef CONFIG_CONSOLE_POLL
1547	.poll_get_char	= msm_poll_get_char,
1548	.poll_put_char	= msm_poll_put_char,
1549#endif
1550};
1551
1552static struct msm_port msm_uart_ports[] = {
1553	{
1554		.uart = {
1555			.iotype = UPIO_MEM,
1556			.ops = &msm_uart_pops,
1557			.flags = UPF_BOOT_AUTOCONF,
1558			.fifosize = 64,
1559			.line = 0,
1560		},
1561	},
1562	{
1563		.uart = {
1564			.iotype = UPIO_MEM,
1565			.ops = &msm_uart_pops,
1566			.flags = UPF_BOOT_AUTOCONF,
1567			.fifosize = 64,
1568			.line = 1,
1569		},
1570	},
1571	{
1572		.uart = {
1573			.iotype = UPIO_MEM,
1574			.ops = &msm_uart_pops,
1575			.flags = UPF_BOOT_AUTOCONF,
1576			.fifosize = 64,
1577			.line = 2,
1578		},
1579	},
1580};
1581
1582#define UART_NR	ARRAY_SIZE(msm_uart_ports)
1583
1584static inline struct uart_port *msm_get_port_from_line(unsigned int line)
1585{
1586	return &msm_uart_ports[line].uart;
1587}
1588
1589#ifdef CONFIG_SERIAL_MSM_CONSOLE
1590static void __msm_console_write(struct uart_port *port, const char *s,
1591				unsigned int count, bool is_uartdm)
1592{
1593	int i;
1594	int num_newlines = 0;
1595	bool replaced = false;
1596	void __iomem *tf;
1597	int locked = 1;
1598
1599	if (is_uartdm)
1600		tf = port->membase + UARTDM_TF;
1601	else
1602		tf = port->membase + UART_TF;
1603
1604	/* Account for newlines that will get a carriage return added */
1605	for (i = 0; i < count; i++)
1606		if (s[i] == '\n')
1607			num_newlines++;
1608	count += num_newlines;
1609
1610	if (port->sysrq)
1611		locked = 0;
1612	else if (oops_in_progress)
1613		locked = spin_trylock(&port->lock);
1614	else
1615		spin_lock(&port->lock);
1616
1617	if (is_uartdm)
1618		msm_reset_dm_count(port, count);
1619
1620	i = 0;
1621	while (i < count) {
1622		int j;
1623		unsigned int num_chars;
1624		char buf[4] = { 0 };
1625
1626		if (is_uartdm)
1627			num_chars = min(count - i, (unsigned int)sizeof(buf));
1628		else
1629			num_chars = 1;
1630
1631		for (j = 0; j < num_chars; j++) {
1632			char c = *s;
1633
1634			if (c == '\n' && !replaced) {
1635				buf[j] = '\r';
1636				j++;
1637				replaced = true;
1638			}
1639			if (j < num_chars) {
1640				buf[j] = c;
1641				s++;
1642				replaced = false;
1643			}
1644		}
1645
1646		while (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
1647			cpu_relax();
1648
1649		iowrite32_rep(tf, buf, 1);
1650		i += num_chars;
1651	}
1652
1653	if (locked)
1654		spin_unlock(&port->lock);
1655}
1656
1657static void msm_console_write(struct console *co, const char *s,
1658			      unsigned int count)
1659{
1660	struct uart_port *port;
1661	struct msm_port *msm_port;
1662
1663	BUG_ON(co->index < 0 || co->index >= UART_NR);
1664
1665	port = msm_get_port_from_line(co->index);
1666	msm_port = UART_TO_MSM(port);
1667
1668	__msm_console_write(port, s, count, msm_port->is_uartdm);
1669}
1670
1671static int msm_console_setup(struct console *co, char *options)
1672{
1673	struct uart_port *port;
1674	int baud = 115200;
1675	int bits = 8;
1676	int parity = 'n';
1677	int flow = 'n';
1678
1679	if (unlikely(co->index >= UART_NR || co->index < 0))
1680		return -ENXIO;
1681
1682	port = msm_get_port_from_line(co->index);
1683
1684	if (unlikely(!port->membase))
1685		return -ENXIO;
1686
1687	msm_init_clock(port);
1688
1689	if (options)
1690		uart_parse_options(options, &baud, &parity, &bits, &flow);
1691
1692	pr_info("msm_serial: console setup on port #%d\n", port->line);
1693
1694	return uart_set_options(port, co, baud, parity, bits, flow);
1695}
1696
1697static void
1698msm_serial_early_write(struct console *con, const char *s, unsigned n)
1699{
1700	struct earlycon_device *dev = con->data;
1701
1702	__msm_console_write(&dev->port, s, n, false);
1703}
1704
1705static int __init
1706msm_serial_early_console_setup(struct earlycon_device *device, const char *opt)
1707{
1708	if (!device->port.membase)
1709		return -ENODEV;
1710
1711	device->con->write = msm_serial_early_write;
1712	return 0;
1713}
1714OF_EARLYCON_DECLARE(msm_serial, "qcom,msm-uart",
1715		    msm_serial_early_console_setup);
1716
1717static void
1718msm_serial_early_write_dm(struct console *con, const char *s, unsigned n)
1719{
1720	struct earlycon_device *dev = con->data;
1721
1722	__msm_console_write(&dev->port, s, n, true);
1723}
1724
1725static int __init
1726msm_serial_early_console_setup_dm(struct earlycon_device *device,
1727				  const char *opt)
1728{
1729	if (!device->port.membase)
1730		return -ENODEV;
1731
1732	device->con->write = msm_serial_early_write_dm;
1733	return 0;
1734}
1735OF_EARLYCON_DECLARE(msm_serial_dm, "qcom,msm-uartdm",
1736		    msm_serial_early_console_setup_dm);
1737
1738static struct uart_driver msm_uart_driver;
1739
1740static struct console msm_console = {
1741	.name = "ttyMSM",
1742	.write = msm_console_write,
1743	.device = uart_console_device,
1744	.setup = msm_console_setup,
1745	.flags = CON_PRINTBUFFER,
1746	.index = -1,
1747	.data = &msm_uart_driver,
1748};
1749
1750#define MSM_CONSOLE	(&msm_console)
1751
1752#else
1753#define MSM_CONSOLE	NULL
1754#endif
1755
1756static struct uart_driver msm_uart_driver = {
1757	.owner = THIS_MODULE,
1758	.driver_name = "msm_serial",
1759	.dev_name = "ttyMSM",
1760	.nr = UART_NR,
1761	.cons = MSM_CONSOLE,
1762};
1763
1764static atomic_t msm_uart_next_id = ATOMIC_INIT(0);
1765
1766static const struct of_device_id msm_uartdm_table[] = {
1767	{ .compatible = "qcom,msm-uartdm-v1.1", .data = (void *)UARTDM_1P1 },
1768	{ .compatible = "qcom,msm-uartdm-v1.2", .data = (void *)UARTDM_1P2 },
1769	{ .compatible = "qcom,msm-uartdm-v1.3", .data = (void *)UARTDM_1P3 },
1770	{ .compatible = "qcom,msm-uartdm-v1.4", .data = (void *)UARTDM_1P4 },
1771	{ }
1772};
1773
1774static int msm_serial_probe(struct platform_device *pdev)
1775{
1776	struct msm_port *msm_port;
1777	struct resource *resource;
1778	struct uart_port *port;
1779	const struct of_device_id *id;
1780	int irq, line;
1781
1782	if (pdev->dev.of_node)
1783		line = of_alias_get_id(pdev->dev.of_node, "serial");
1784	else
1785		line = pdev->id;
1786
1787	if (line < 0)
1788		line = atomic_inc_return(&msm_uart_next_id) - 1;
1789
1790	if (unlikely(line < 0 || line >= UART_NR))
1791		return -ENXIO;
1792
1793	dev_info(&pdev->dev, "msm_serial: detected port #%d\n", line);
1794
1795	port = msm_get_port_from_line(line);
1796	port->dev = &pdev->dev;
1797	msm_port = UART_TO_MSM(port);
1798
1799	id = of_match_device(msm_uartdm_table, &pdev->dev);
1800	if (id)
1801		msm_port->is_uartdm = (unsigned long)id->data;
1802	else
1803		msm_port->is_uartdm = 0;
1804
1805	msm_port->clk = devm_clk_get(&pdev->dev, "core");
1806	if (IS_ERR(msm_port->clk))
1807		return PTR_ERR(msm_port->clk);
1808
1809	if (msm_port->is_uartdm) {
1810		msm_port->pclk = devm_clk_get(&pdev->dev, "iface");
1811		if (IS_ERR(msm_port->pclk))
1812			return PTR_ERR(msm_port->pclk);
1813	}
1814
1815	port->uartclk = clk_get_rate(msm_port->clk);
1816	dev_info(&pdev->dev, "uartclk = %d\n", port->uartclk);
1817
1818	resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1819	if (unlikely(!resource))
1820		return -ENXIO;
1821	port->mapbase = resource->start;
1822
1823	irq = platform_get_irq(pdev, 0);
1824	if (unlikely(irq < 0))
1825		return -ENXIO;
1826	port->irq = irq;
1827	port->has_sysrq = IS_ENABLED(CONFIG_SERIAL_MSM_CONSOLE);
1828
1829	platform_set_drvdata(pdev, port);
1830
1831	return uart_add_one_port(&msm_uart_driver, port);
1832}
1833
1834static int msm_serial_remove(struct platform_device *pdev)
1835{
1836	struct uart_port *port = platform_get_drvdata(pdev);
1837
1838	uart_remove_one_port(&msm_uart_driver, port);
1839
1840	return 0;
1841}
1842
1843static const struct of_device_id msm_match_table[] = {
1844	{ .compatible = "qcom,msm-uart" },
1845	{ .compatible = "qcom,msm-uartdm" },
1846	{}
1847};
1848MODULE_DEVICE_TABLE(of, msm_match_table);
1849
1850static int __maybe_unused msm_serial_suspend(struct device *dev)
1851{
1852	struct msm_port *port = dev_get_drvdata(dev);
1853
1854	uart_suspend_port(&msm_uart_driver, &port->uart);
1855
1856	return 0;
1857}
1858
1859static int __maybe_unused msm_serial_resume(struct device *dev)
1860{
1861	struct msm_port *port = dev_get_drvdata(dev);
1862
1863	uart_resume_port(&msm_uart_driver, &port->uart);
1864
1865	return 0;
1866}
1867
1868static const struct dev_pm_ops msm_serial_dev_pm_ops = {
1869	SET_SYSTEM_SLEEP_PM_OPS(msm_serial_suspend, msm_serial_resume)
1870};
1871
1872static struct platform_driver msm_platform_driver = {
1873	.remove = msm_serial_remove,
1874	.probe = msm_serial_probe,
1875	.driver = {
1876		.name = "msm_serial",
1877		.pm = &msm_serial_dev_pm_ops,
1878		.of_match_table = msm_match_table,
1879	},
1880};
1881
1882static int __init msm_serial_init(void)
1883{
1884	int ret;
1885
1886	ret = uart_register_driver(&msm_uart_driver);
1887	if (unlikely(ret))
1888		return ret;
1889
1890	ret = platform_driver_register(&msm_platform_driver);
1891	if (unlikely(ret))
1892		uart_unregister_driver(&msm_uart_driver);
1893
1894	pr_info("msm_serial: driver initialized\n");
1895
1896	return ret;
1897}
1898
1899static void __exit msm_serial_exit(void)
1900{
1901	platform_driver_unregister(&msm_platform_driver);
1902	uart_unregister_driver(&msm_uart_driver);
1903}
1904
1905module_init(msm_serial_init);
1906module_exit(msm_serial_exit);
1907
1908MODULE_AUTHOR("Robert Love <rlove@google.com>");
1909MODULE_DESCRIPTION("Driver for msm7x serial device");
1910MODULE_LICENSE("GPL");