1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Driver for msm7k serial device and console
   4 *
   5 * Copyright (C) 2007 Google, Inc.
   6 * Author: Robert Love <rlove@google.com>
   7 * Copyright (c) 2011, Code Aurora Forum. All rights reserved.
   8 */
   9
  10#include <linux/kernel.h>
  11#include <linux/atomic.h>
  12#include <linux/dma/qcom_adm.h>
  13#include <linux/dma-mapping.h>
  14#include <linux/dmaengine.h>
  15#include <linux/module.h>
  16#include <linux/io.h>
  17#include <linux/ioport.h>
  18#include <linux/interrupt.h>
  19#include <linux/init.h>
  20#include <linux/console.h>
  21#include <linux/tty.h>
  22#include <linux/tty_flip.h>
  23#include <linux/serial_core.h>
  24#include <linux/slab.h>
  25#include <linux/clk.h>
  26#include <linux/platform_device.h>
  27#include <linux/pm_opp.h>
  28#include <linux/delay.h>
  29#include <linux/of.h>
  30#include <linux/of_device.h>
  31#include <linux/wait.h>
  32
  33#define MSM_UART_MR1			0x0000
  34
  35#define MSM_UART_MR1_AUTO_RFR_LEVEL0	0x3F
  36#define MSM_UART_MR1_AUTO_RFR_LEVEL1	0x3FF00
  37#define MSM_UART_DM_MR1_AUTO_RFR_LEVEL1	0xFFFFFF00
  38#define MSM_UART_MR1_RX_RDY_CTL		BIT(7)
  39#define MSM_UART_MR1_CTS_CTL		BIT(6)
  40
  41#define MSM_UART_MR2			0x0004
  42#define MSM_UART_MR2_ERROR_MODE		BIT(6)
  43#define MSM_UART_MR2_BITS_PER_CHAR	0x30
  44#define MSM_UART_MR2_BITS_PER_CHAR_5	(0x0 << 4)
  45#define MSM_UART_MR2_BITS_PER_CHAR_6	(0x1 << 4)
  46#define MSM_UART_MR2_BITS_PER_CHAR_7	(0x2 << 4)
  47#define MSM_UART_MR2_BITS_PER_CHAR_8	(0x3 << 4)
  48#define MSM_UART_MR2_STOP_BIT_LEN_ONE	(0x1 << 2)
  49#define MSM_UART_MR2_STOP_BIT_LEN_TWO	(0x3 << 2)
  50#define MSM_UART_MR2_PARITY_MODE_NONE	0x0
  51#define MSM_UART_MR2_PARITY_MODE_ODD	0x1
  52#define MSM_UART_MR2_PARITY_MODE_EVEN	0x2
  53#define MSM_UART_MR2_PARITY_MODE_SPACE	0x3
  54#define MSM_UART_MR2_PARITY_MODE	0x3
  55
  56#define MSM_UART_CSR			0x0008
  57
  58#define MSM_UART_TF			0x000C
  59#define UARTDM_TF			0x0070
  60
  61#define MSM_UART_CR				0x0010
  62#define MSM_UART_CR_CMD_NULL			(0 << 4)
  63#define MSM_UART_CR_CMD_RESET_RX		(1 << 4)
  64#define MSM_UART_CR_CMD_RESET_TX		(2 << 4)
  65#define MSM_UART_CR_CMD_RESET_ERR		(3 << 4)
  66#define MSM_UART_CR_CMD_RESET_BREAK_INT		(4 << 4)
  67#define MSM_UART_CR_CMD_START_BREAK		(5 << 4)
  68#define MSM_UART_CR_CMD_STOP_BREAK		(6 << 4)
  69#define MSM_UART_CR_CMD_RESET_CTS		(7 << 4)
  70#define MSM_UART_CR_CMD_RESET_STALE_INT		(8 << 4)
  71#define MSM_UART_CR_CMD_PACKET_MODE		(9 << 4)
  72#define MSM_UART_CR_CMD_MODE_RESET		(12 << 4)
  73#define MSM_UART_CR_CMD_SET_RFR			(13 << 4)
  74#define MSM_UART_CR_CMD_RESET_RFR		(14 << 4)
  75#define MSM_UART_CR_CMD_PROTECTION_EN		(16 << 4)
  76#define MSM_UART_CR_CMD_STALE_EVENT_DISABLE	(6 << 8)
  77#define MSM_UART_CR_CMD_STALE_EVENT_ENABLE	(80 << 4)
  78#define MSM_UART_CR_CMD_FORCE_STALE		(4 << 8)
  79#define MSM_UART_CR_CMD_RESET_TX_READY		(3 << 8)
  80#define MSM_UART_CR_TX_DISABLE			BIT(3)
  81#define MSM_UART_CR_TX_ENABLE			BIT(2)
  82#define MSM_UART_CR_RX_DISABLE			BIT(1)
  83#define MSM_UART_CR_RX_ENABLE			BIT(0)
  84#define MSM_UART_CR_CMD_RESET_RXBREAK_START	((1 << 11) | (2 << 4))
  85
  86#define MSM_UART_IMR			0x0014
  87#define MSM_UART_IMR_TXLEV		BIT(0)
  88#define MSM_UART_IMR_RXSTALE		BIT(3)
  89#define MSM_UART_IMR_RXLEV		BIT(4)
  90#define MSM_UART_IMR_DELTA_CTS		BIT(5)
  91#define MSM_UART_IMR_CURRENT_CTS	BIT(6)
  92#define MSM_UART_IMR_RXBREAK_START	BIT(10)
  93
  94#define MSM_UART_IPR_RXSTALE_LAST		0x20
  95#define MSM_UART_IPR_STALE_LSB			0x1F
  96#define MSM_UART_IPR_STALE_TIMEOUT_MSB		0x3FF80
  97#define MSM_UART_DM_IPR_STALE_TIMEOUT_MSB	0xFFFFFF80
  98
  99#define MSM_UART_IPR			0x0018
 100#define MSM_UART_TFWR			0x001C
 101#define MSM_UART_RFWR			0x0020
 102#define MSM_UART_HCR			0x0024
 103
 104#define MSM_UART_MREG			0x0028
 105#define MSM_UART_NREG			0x002C
 106#define MSM_UART_DREG			0x0030
 107#define MSM_UART_MNDREG			0x0034
 108#define MSM_UART_IRDA			0x0038
 109#define MSM_UART_MISR_MODE		0x0040
 110#define MSM_UART_MISR_RESET		0x0044
 111#define MSM_UART_MISR_EXPORT		0x0048
 112#define MSM_UART_MISR_VAL		0x004C
 113#define MSM_UART_TEST_CTRL		0x0050
 114
 115#define MSM_UART_SR			0x0008
 116#define MSM_UART_SR_HUNT_CHAR		BIT(7)
 117#define MSM_UART_SR_RX_BREAK		BIT(6)
 118#define MSM_UART_SR_PAR_FRAME_ERR	BIT(5)
 119#define MSM_UART_SR_OVERRUN		BIT(4)
 120#define MSM_UART_SR_TX_EMPTY		BIT(3)
 121#define MSM_UART_SR_TX_READY		BIT(2)
 122#define MSM_UART_SR_RX_FULL		BIT(1)
 123#define MSM_UART_SR_RX_READY		BIT(0)
 124
 125#define MSM_UART_RF			0x000C
 126#define UARTDM_RF			0x0070
 127#define MSM_UART_MISR			0x0010
 128#define MSM_UART_ISR			0x0014
 129#define MSM_UART_ISR_TX_READY		BIT(7)
 130
 131#define UARTDM_RXFS			0x50
 132#define UARTDM_RXFS_BUF_SHIFT		0x7
 133#define UARTDM_RXFS_BUF_MASK		0x7
 134
 135#define UARTDM_DMEN			0x3C
 136#define UARTDM_DMEN_RX_SC_ENABLE	BIT(5)
 137#define UARTDM_DMEN_TX_SC_ENABLE	BIT(4)
 138
 139#define UARTDM_DMEN_TX_BAM_ENABLE	BIT(2)	/* UARTDM_1P4 */
 140#define UARTDM_DMEN_TX_DM_ENABLE	BIT(0)	/* < UARTDM_1P4 */
 141
 142#define UARTDM_DMEN_RX_BAM_ENABLE	BIT(3)	/* UARTDM_1P4 */
 143#define UARTDM_DMEN_RX_DM_ENABLE	BIT(1)	/* < UARTDM_1P4 */
 144
 145#define UARTDM_DMRX			0x34
 146#define UARTDM_NCF_TX			0x40
 147#define UARTDM_RX_TOTAL_SNAP		0x38
 148
 149#define UARTDM_BURST_SIZE		16   /* in bytes */
 150#define UARTDM_TX_AIGN(x)		((x) & ~0x3) /* valid for > 1p3 */
 151#define UARTDM_TX_MAX			256   /* in bytes, valid for <= 1p3 */
 152#define UARTDM_RX_SIZE			(UART_XMIT_SIZE / 4)
 153
 154enum {
 155	UARTDM_1P1 = 1,
 156	UARTDM_1P2,
 157	UARTDM_1P3,
 158	UARTDM_1P4,
 159};
 160
 161struct msm_dma {
 162	struct dma_chan		*chan;
 163	enum dma_data_direction dir;
 164	dma_addr_t		phys;
 165	unsigned char		*virt;
 166	dma_cookie_t		cookie;
 167	u32			enable_bit;
 168	unsigned int		count;
 169	struct dma_async_tx_descriptor	*desc;
 170};
 171
 172struct msm_port {
 173	struct uart_port	uart;
 174	char			name[16];
 175	struct clk		*clk;
 176	struct clk		*pclk;
 177	unsigned int		imr;
 178	int			is_uartdm;
 179	unsigned int		old_snap_state;
 180	bool			break_detected;
 181	struct msm_dma		tx_dma;
 182	struct msm_dma		rx_dma;
 183};
 184
 185static inline struct msm_port *to_msm_port(struct uart_port *up)
 186{
 187	return container_of(up, struct msm_port, uart);
 188}
 189
 190static
 191void msm_write(struct uart_port *port, unsigned int val, unsigned int off)
 192{
 193	writel_relaxed(val, port->membase + off);
 194}
 195
 196static
 197unsigned int msm_read(struct uart_port *port, unsigned int off)
 198{
 199	return readl_relaxed(port->membase + off);
 200}
 201
 202/*
 203 * Setup the MND registers to use the TCXO clock.
 204 */
 205static void msm_serial_set_mnd_regs_tcxo(struct uart_port *port)
 206{
 207	msm_write(port, 0x06, MSM_UART_MREG);
 208	msm_write(port, 0xF1, MSM_UART_NREG);
 209	msm_write(port, 0x0F, MSM_UART_DREG);
 210	msm_write(port, 0x1A, MSM_UART_MNDREG);
 211	port->uartclk = 1843200;
 212}
 213
 214/*
 215 * Setup the MND registers to use the TCXO clock divided by 4.
 216 */
 217static void msm_serial_set_mnd_regs_tcxoby4(struct uart_port *port)
 218{
 219	msm_write(port, 0x18, MSM_UART_MREG);
 220	msm_write(port, 0xF6, MSM_UART_NREG);
 221	msm_write(port, 0x0F, MSM_UART_DREG);
 222	msm_write(port, 0x0A, MSM_UART_MNDREG);
 223	port->uartclk = 1843200;
 224}
 225
 226static void msm_serial_set_mnd_regs(struct uart_port *port)
 227{
 228	struct msm_port *msm_port = to_msm_port(port);
 229
 230	/*
 231	 * These registers don't exist so we change the clk input rate
 232	 * on uartdm hardware instead
 233	 */
 234	if (msm_port->is_uartdm)
 235		return;
 236
 237	if (port->uartclk == 19200000)
 238		msm_serial_set_mnd_regs_tcxo(port);
 239	else if (port->uartclk == 4800000)
 240		msm_serial_set_mnd_regs_tcxoby4(port);
 241}
 242
 243static void msm_handle_tx(struct uart_port *port);
 244static void msm_start_rx_dma(struct msm_port *msm_port);
 245
 246static void msm_stop_dma(struct uart_port *port, struct msm_dma *dma)
 247{
 248	struct device *dev = port->dev;
 249	unsigned int mapped;
 250	u32 val;
 251
 252	mapped = dma->count;
 253	dma->count = 0;
 254
 255	dmaengine_terminate_all(dma->chan);
 256
 257	/*
 258	 * DMA Stall happens if enqueue and flush command happens concurrently.
 259	 * For example before changing the baud rate/protocol configuration and
 260	 * sending flush command to ADM, disable the channel of UARTDM.
 261	 * Note: should not reset the receiver here immediately as it is not
 262	 * suggested to do disable/reset or reset/disable at the same time.
 263	 */
 264	val = msm_read(port, UARTDM_DMEN);
 265	val &= ~dma->enable_bit;
 266	msm_write(port, val, UARTDM_DMEN);
 267
 268	if (mapped)
 269		dma_unmap_single(dev, dma->phys, mapped, dma->dir);
 270}
 271
 272static void msm_release_dma(struct msm_port *msm_port)
 273{
 274	struct msm_dma *dma;
 275
 276	dma = &msm_port->tx_dma;
 277	if (dma->chan) {
 278		msm_stop_dma(&msm_port->uart, dma);
 279		dma_release_channel(dma->chan);
 280	}
 281
 282	memset(dma, 0, sizeof(*dma));
 283
 284	dma = &msm_port->rx_dma;
 285	if (dma->chan) {
 286		msm_stop_dma(&msm_port->uart, dma);
 287		dma_release_channel(dma->chan);
 288		kfree(dma->virt);
 289	}
 290
 291	memset(dma, 0, sizeof(*dma));
 292}
 293
 294static void msm_request_tx_dma(struct msm_port *msm_port, resource_size_t base)
 295{
 296	struct device *dev = msm_port->uart.dev;
 297	struct dma_slave_config conf;
 298	struct qcom_adm_peripheral_config periph_conf = {};
 299	struct msm_dma *dma;
 300	u32 crci = 0;
 301	int ret;
 302
 303	dma = &msm_port->tx_dma;
 304
 305	/* allocate DMA resources, if available */
 306	dma->chan = dma_request_chan(dev, "tx");
 307	if (IS_ERR(dma->chan))
 308		goto no_tx;
 309
 310	of_property_read_u32(dev->of_node, "qcom,tx-crci", &crci);
 311
 312	memset(&conf, 0, sizeof(conf));
 313	conf.direction = DMA_MEM_TO_DEV;
 314	conf.device_fc = true;
 315	conf.dst_addr = base + UARTDM_TF;
 316	conf.dst_maxburst = UARTDM_BURST_SIZE;
 317	if (crci) {
 318		conf.peripheral_config = &periph_conf;
 319		conf.peripheral_size = sizeof(periph_conf);
 320		periph_conf.crci = crci;
 321	}
 322
 323	ret = dmaengine_slave_config(dma->chan, &conf);
 324	if (ret)
 325		goto rel_tx;
 326
 327	dma->dir = DMA_TO_DEVICE;
 328
 329	if (msm_port->is_uartdm < UARTDM_1P4)
 330		dma->enable_bit = UARTDM_DMEN_TX_DM_ENABLE;
 331	else
 332		dma->enable_bit = UARTDM_DMEN_TX_BAM_ENABLE;
 333
 334	return;
 335
 336rel_tx:
 337	dma_release_channel(dma->chan);
 338no_tx:
 339	memset(dma, 0, sizeof(*dma));
 340}
 341
 342static void msm_request_rx_dma(struct msm_port *msm_port, resource_size_t base)
 343{
 344	struct device *dev = msm_port->uart.dev;
 345	struct dma_slave_config conf;
 346	struct qcom_adm_peripheral_config periph_conf = {};
 347	struct msm_dma *dma;
 348	u32 crci = 0;
 349	int ret;
 350
 351	dma = &msm_port->rx_dma;
 352
 353	/* allocate DMA resources, if available */
 354	dma->chan = dma_request_chan(dev, "rx");
 355	if (IS_ERR(dma->chan))
 356		goto no_rx;
 357
 358	of_property_read_u32(dev->of_node, "qcom,rx-crci", &crci);
 359
 360	dma->virt = kzalloc(UARTDM_RX_SIZE, GFP_KERNEL);
 361	if (!dma->virt)
 362		goto rel_rx;
 363
 364	memset(&conf, 0, sizeof(conf));
 365	conf.direction = DMA_DEV_TO_MEM;
 366	conf.device_fc = true;
 367	conf.src_addr = base + UARTDM_RF;
 368	conf.src_maxburst = UARTDM_BURST_SIZE;
 369	if (crci) {
 370		conf.peripheral_config = &periph_conf;
 371		conf.peripheral_size = sizeof(periph_conf);
 372		periph_conf.crci = crci;
 373	}
 374
 375	ret = dmaengine_slave_config(dma->chan, &conf);
 376	if (ret)
 377		goto err;
 378
 379	dma->dir = DMA_FROM_DEVICE;
 380
 381	if (msm_port->is_uartdm < UARTDM_1P4)
 382		dma->enable_bit = UARTDM_DMEN_RX_DM_ENABLE;
 383	else
 384		dma->enable_bit = UARTDM_DMEN_RX_BAM_ENABLE;
 385
 386	return;
 387err:
 388	kfree(dma->virt);
 389rel_rx:
 390	dma_release_channel(dma->chan);
 391no_rx:
 392	memset(dma, 0, sizeof(*dma));
 393}
 394
 395static inline void msm_wait_for_xmitr(struct uart_port *port)
 396{
 397	unsigned int timeout = 500000;
 398
 399	while (!(msm_read(port, MSM_UART_SR) & MSM_UART_SR_TX_EMPTY)) {
 400		if (msm_read(port, MSM_UART_ISR) & MSM_UART_ISR_TX_READY)
 401			break;
 402		udelay(1);
 403		if (!timeout--)
 404			break;
 405	}
 406	msm_write(port, MSM_UART_CR_CMD_RESET_TX_READY, MSM_UART_CR);
 407}
 408
 409static void msm_stop_tx(struct uart_port *port)
 410{
 411	struct msm_port *msm_port = to_msm_port(port);
 412
 413	msm_port->imr &= ~MSM_UART_IMR_TXLEV;
 414	msm_write(port, msm_port->imr, MSM_UART_IMR);
 415}
 416
 417static void msm_start_tx(struct uart_port *port)
 418{
 419	struct msm_port *msm_port = to_msm_port(port);
 420	struct msm_dma *dma = &msm_port->tx_dma;
 421
 422	/* Already started in DMA mode */
 423	if (dma->count)
 424		return;
 425
 426	msm_port->imr |= MSM_UART_IMR_TXLEV;
 427	msm_write(port, msm_port->imr, MSM_UART_IMR);
 428}
 429
 430static void msm_reset_dm_count(struct uart_port *port, int count)
 431{
 432	msm_wait_for_xmitr(port);
 433	msm_write(port, count, UARTDM_NCF_TX);
 434	msm_read(port, UARTDM_NCF_TX);
 435}
 436
 437static void msm_complete_tx_dma(void *args)
 438{
 439	struct msm_port *msm_port = args;
 440	struct uart_port *port = &msm_port->uart;
 441	struct circ_buf *xmit = &port->state->xmit;
 442	struct msm_dma *dma = &msm_port->tx_dma;
 443	struct dma_tx_state state;
 444	unsigned long flags;
 445	unsigned int count;
 446	u32 val;
 447
 448	uart_port_lock_irqsave(port, &flags);
 449
 450	/* Already stopped */
 451	if (!dma->count)
 452		goto done;
 453
 454	dmaengine_tx_status(dma->chan, dma->cookie, &state);
 455
 456	dma_unmap_single(port->dev, dma->phys, dma->count, dma->dir);
 457
 458	val = msm_read(port, UARTDM_DMEN);
 459	val &= ~dma->enable_bit;
 460	msm_write(port, val, UARTDM_DMEN);
 461
 462	if (msm_port->is_uartdm > UARTDM_1P3) {
 463		msm_write(port, MSM_UART_CR_CMD_RESET_TX, MSM_UART_CR);
 464		msm_write(port, MSM_UART_CR_TX_ENABLE, MSM_UART_CR);
 465	}
 466
 467	count = dma->count - state.residue;
 468	uart_xmit_advance(port, count);
 469	dma->count = 0;
 470
 471	/* Restore "Tx FIFO below watermark" interrupt */
 472	msm_port->imr |= MSM_UART_IMR_TXLEV;
 473	msm_write(port, msm_port->imr, MSM_UART_IMR);
 474
 475	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 476		uart_write_wakeup(port);
 477
 478	msm_handle_tx(port);
 479done:
 480	uart_port_unlock_irqrestore(port, flags);
 481}
 482
 483static int msm_handle_tx_dma(struct msm_port *msm_port, unsigned int count)
 484{
 485	struct circ_buf *xmit = &msm_port->uart.state->xmit;
 486	struct uart_port *port = &msm_port->uart;
 487	struct msm_dma *dma = &msm_port->tx_dma;
 488	void *cpu_addr;
 489	int ret;
 490	u32 val;
 491
 492	cpu_addr = &xmit->buf[xmit->tail];
 493
 494	dma->phys = dma_map_single(port->dev, cpu_addr, count, dma->dir);
 495	ret = dma_mapping_error(port->dev, dma->phys);
 496	if (ret)
 497		return ret;
 498
 499	dma->desc = dmaengine_prep_slave_single(dma->chan, dma->phys,
 500						count, DMA_MEM_TO_DEV,
 501						DMA_PREP_INTERRUPT |
 502						DMA_PREP_FENCE);
 503	if (!dma->desc) {
 504		ret = -EIO;
 505		goto unmap;
 506	}
 507
 508	dma->desc->callback = msm_complete_tx_dma;
 509	dma->desc->callback_param = msm_port;
 510
 511	dma->cookie = dmaengine_submit(dma->desc);
 512	ret = dma_submit_error(dma->cookie);
 513	if (ret)
 514		goto unmap;
 515
 516	/*
 517	 * Using DMA complete for Tx FIFO reload, no need for
 518	 * "Tx FIFO below watermark" one, disable it
 519	 */
 520	msm_port->imr &= ~MSM_UART_IMR_TXLEV;
 521	msm_write(port, msm_port->imr, MSM_UART_IMR);
 522
 523	dma->count = count;
 524
 525	val = msm_read(port, UARTDM_DMEN);
 526	val |= dma->enable_bit;
 527
 528	if (msm_port->is_uartdm < UARTDM_1P4)
 529		msm_write(port, val, UARTDM_DMEN);
 530
 531	msm_reset_dm_count(port, count);
 532
 533	if (msm_port->is_uartdm > UARTDM_1P3)
 534		msm_write(port, val, UARTDM_DMEN);
 535
 536	dma_async_issue_pending(dma->chan);
 537	return 0;
 538unmap:
 539	dma_unmap_single(port->dev, dma->phys, count, dma->dir);
 540	return ret;
 541}
 542
 543static void msm_complete_rx_dma(void *args)
 544{
 545	struct msm_port *msm_port = args;
 546	struct uart_port *port = &msm_port->uart;
 547	struct tty_port *tport = &port->state->port;
 548	struct msm_dma *dma = &msm_port->rx_dma;
 549	int count = 0, i, sysrq;
 550	unsigned long flags;
 551	u32 val;
 552
 553	uart_port_lock_irqsave(port, &flags);
 554
 555	/* Already stopped */
 556	if (!dma->count)
 557		goto done;
 558
 559	val = msm_read(port, UARTDM_DMEN);
 560	val &= ~dma->enable_bit;
 561	msm_write(port, val, UARTDM_DMEN);
 562
 563	if (msm_read(port, MSM_UART_SR) & MSM_UART_SR_OVERRUN) {
 564		port->icount.overrun++;
 565		tty_insert_flip_char(tport, 0, TTY_OVERRUN);
 566		msm_write(port, MSM_UART_CR_CMD_RESET_ERR, MSM_UART_CR);
 567	}
 568
 569	count = msm_read(port, UARTDM_RX_TOTAL_SNAP);
 570
 571	port->icount.rx += count;
 572
 573	dma->count = 0;
 574
 575	dma_unmap_single(port->dev, dma->phys, UARTDM_RX_SIZE, dma->dir);
 576
 577	for (i = 0; i < count; i++) {
 578		char flag = TTY_NORMAL;
 579
 580		if (msm_port->break_detected && dma->virt[i] == 0) {
 581			port->icount.brk++;
 582			flag = TTY_BREAK;
 583			msm_port->break_detected = false;
 584			if (uart_handle_break(port))
 585				continue;
 586		}
 587
 588		if (!(port->read_status_mask & MSM_UART_SR_RX_BREAK))
 589			flag = TTY_NORMAL;
 590
 591		uart_port_unlock_irqrestore(port, flags);
 592		sysrq = uart_handle_sysrq_char(port, dma->virt[i]);
 593		uart_port_lock_irqsave(port, &flags);
 594		if (!sysrq)
 595			tty_insert_flip_char(tport, dma->virt[i], flag);
 596	}
 597
 598	msm_start_rx_dma(msm_port);
 599done:
 600	uart_port_unlock_irqrestore(port, flags);
 601
 602	if (count)
 603		tty_flip_buffer_push(tport);
 604}
 605
 606static void msm_start_rx_dma(struct msm_port *msm_port)
 607{
 608	struct msm_dma *dma = &msm_port->rx_dma;
 609	struct uart_port *uart = &msm_port->uart;
 610	u32 val;
 611	int ret;
 612
 613	if (IS_ENABLED(CONFIG_CONSOLE_POLL))
 614		return;
 615
 616	if (!dma->chan)
 617		return;
 618
 619	dma->phys = dma_map_single(uart->dev, dma->virt,
 620				   UARTDM_RX_SIZE, dma->dir);
 621	ret = dma_mapping_error(uart->dev, dma->phys);
 622	if (ret)
 623		goto sw_mode;
 624
 625	dma->desc = dmaengine_prep_slave_single(dma->chan, dma->phys,
 626						UARTDM_RX_SIZE, DMA_DEV_TO_MEM,
 627						DMA_PREP_INTERRUPT);
 628	if (!dma->desc)
 629		goto unmap;
 630
 631	dma->desc->callback = msm_complete_rx_dma;
 632	dma->desc->callback_param = msm_port;
 633
 634	dma->cookie = dmaengine_submit(dma->desc);
 635	ret = dma_submit_error(dma->cookie);
 636	if (ret)
 637		goto unmap;
 638	/*
 639	 * Using DMA for FIFO off-load, no need for "Rx FIFO over
 640	 * watermark" or "stale" interrupts, disable them
 641	 */
 642	msm_port->imr &= ~(MSM_UART_IMR_RXLEV | MSM_UART_IMR_RXSTALE);
 643
 644	/*
 645	 * Well, when DMA is ADM3 engine(implied by <= UARTDM v1.3),
 646	 * we need RXSTALE to flush input DMA fifo to memory
 647	 */
 648	if (msm_port->is_uartdm < UARTDM_1P4)
 649		msm_port->imr |= MSM_UART_IMR_RXSTALE;
 650
 651	msm_write(uart, msm_port->imr, MSM_UART_IMR);
 652
 653	dma->count = UARTDM_RX_SIZE;
 654
 655	dma_async_issue_pending(dma->chan);
 656
 657	msm_write(uart, MSM_UART_CR_CMD_RESET_STALE_INT, MSM_UART_CR);
 658	msm_write(uart, MSM_UART_CR_CMD_STALE_EVENT_ENABLE, MSM_UART_CR);
 659
 660	val = msm_read(uart, UARTDM_DMEN);
 661	val |= dma->enable_bit;
 662
 663	if (msm_port->is_uartdm < UARTDM_1P4)
 664		msm_write(uart, val, UARTDM_DMEN);
 665
 666	msm_write(uart, UARTDM_RX_SIZE, UARTDM_DMRX);
 667
 668	if (msm_port->is_uartdm > UARTDM_1P3)
 669		msm_write(uart, val, UARTDM_DMEN);
 670
 671	return;
 672unmap:
 673	dma_unmap_single(uart->dev, dma->phys, UARTDM_RX_SIZE, dma->dir);
 674
 675sw_mode:
 676	/*
 677	 * Switch from DMA to SW/FIFO mode. After clearing Rx BAM (UARTDM_DMEN),
 678	 * receiver must be reset.
 679	 */
 680	msm_write(uart, MSM_UART_CR_CMD_RESET_RX, MSM_UART_CR);
 681	msm_write(uart, MSM_UART_CR_RX_ENABLE, MSM_UART_CR);
 682
 683	msm_write(uart, MSM_UART_CR_CMD_RESET_STALE_INT, MSM_UART_CR);
 684	msm_write(uart, 0xFFFFFF, UARTDM_DMRX);
 685	msm_write(uart, MSM_UART_CR_CMD_STALE_EVENT_ENABLE, MSM_UART_CR);
 686
 687	/* Re-enable RX interrupts */
 688	msm_port->imr |= MSM_UART_IMR_RXLEV | MSM_UART_IMR_RXSTALE;
 689	msm_write(uart, msm_port->imr, MSM_UART_IMR);
 690}
 691
 692static void msm_stop_rx(struct uart_port *port)
 693{
 694	struct msm_port *msm_port = to_msm_port(port);
 695	struct msm_dma *dma = &msm_port->rx_dma;
 696
 697	msm_port->imr &= ~(MSM_UART_IMR_RXLEV | MSM_UART_IMR_RXSTALE);
 698	msm_write(port, msm_port->imr, MSM_UART_IMR);
 699
 700	if (dma->chan)
 701		msm_stop_dma(port, dma);
 702}
 703
 704static void msm_enable_ms(struct uart_port *port)
 705{
 706	struct msm_port *msm_port = to_msm_port(port);
 707
 708	msm_port->imr |= MSM_UART_IMR_DELTA_CTS;
 709	msm_write(port, msm_port->imr, MSM_UART_IMR);
 710}
 711
 712static void msm_handle_rx_dm(struct uart_port *port, unsigned int misr)
 713	__must_hold(&port->lock)
 714{
 715	struct tty_port *tport = &port->state->port;
 716	unsigned int sr;
 717	int count = 0;
 718	struct msm_port *msm_port = to_msm_port(port);
 719
 720	if ((msm_read(port, MSM_UART_SR) & MSM_UART_SR_OVERRUN)) {
 721		port->icount.overrun++;
 722		tty_insert_flip_char(tport, 0, TTY_OVERRUN);
 723		msm_write(port, MSM_UART_CR_CMD_RESET_ERR, MSM_UART_CR);
 724	}
 725
 726	if (misr & MSM_UART_IMR_RXSTALE) {
 727		count = msm_read(port, UARTDM_RX_TOTAL_SNAP) -
 728			msm_port->old_snap_state;
 729		msm_port->old_snap_state = 0;
 730	} else {
 731		count = 4 * (msm_read(port, MSM_UART_RFWR));
 732		msm_port->old_snap_state += count;
 733	}
 734
 735	/* TODO: Precise error reporting */
 736
 737	port->icount.rx += count;
 738
 739	while (count > 0) {
 740		unsigned char buf[4];
 741		int sysrq, r_count, i;
 742
 743		sr = msm_read(port, MSM_UART_SR);
 744		if ((sr & MSM_UART_SR_RX_READY) == 0) {
 745			msm_port->old_snap_state -= count;
 746			break;
 747		}
 748
 749		ioread32_rep(port->membase + UARTDM_RF, buf, 1);
 750		r_count = min_t(int, count, sizeof(buf));
 751
 752		for (i = 0; i < r_count; i++) {
 753			char flag = TTY_NORMAL;
 754
 755			if (msm_port->break_detected && buf[i] == 0) {
 756				port->icount.brk++;
 757				flag = TTY_BREAK;
 758				msm_port->break_detected = false;
 759				if (uart_handle_break(port))
 760					continue;
 761			}
 762
 763			if (!(port->read_status_mask & MSM_UART_SR_RX_BREAK))
 764				flag = TTY_NORMAL;
 765
 766			uart_port_unlock(port);
 767			sysrq = uart_handle_sysrq_char(port, buf[i]);
 768			uart_port_lock(port);
 769			if (!sysrq)
 770				tty_insert_flip_char(tport, buf[i], flag);
 771		}
 772		count -= r_count;
 773	}
 774
 775	tty_flip_buffer_push(tport);
 776
 777	if (misr & (MSM_UART_IMR_RXSTALE))
 778		msm_write(port, MSM_UART_CR_CMD_RESET_STALE_INT, MSM_UART_CR);
 779	msm_write(port, 0xFFFFFF, UARTDM_DMRX);
 780	msm_write(port, MSM_UART_CR_CMD_STALE_EVENT_ENABLE, MSM_UART_CR);
 781
 782	/* Try to use DMA */
 783	msm_start_rx_dma(msm_port);
 784}
 785
 786static void msm_handle_rx(struct uart_port *port)
 787	__must_hold(&port->lock)
 788{
 789	struct tty_port *tport = &port->state->port;
 790	unsigned int sr;
 791
 792	/*
 793	 * Handle overrun. My understanding of the hardware is that overrun
 794	 * is not tied to the RX buffer, so we handle the case out of band.
 795	 */
 796	if ((msm_read(port, MSM_UART_SR) & MSM_UART_SR_OVERRUN)) {
 797		port->icount.overrun++;
 798		tty_insert_flip_char(tport, 0, TTY_OVERRUN);
 799		msm_write(port, MSM_UART_CR_CMD_RESET_ERR, MSM_UART_CR);
 800	}
 801
 802	/* and now the main RX loop */
 803	while ((sr = msm_read(port, MSM_UART_SR)) & MSM_UART_SR_RX_READY) {
 804		unsigned int c;
 805		char flag = TTY_NORMAL;
 806		int sysrq;
 807
 808		c = msm_read(port, MSM_UART_RF);
 809
 810		if (sr & MSM_UART_SR_RX_BREAK) {
 811			port->icount.brk++;
 812			if (uart_handle_break(port))
 813				continue;
 814		} else if (sr & MSM_UART_SR_PAR_FRAME_ERR) {
 815			port->icount.frame++;
 816		} else {
 817			port->icount.rx++;
 818		}
 819
 820		/* Mask conditions we're ignoring. */
 821		sr &= port->read_status_mask;
 822
 823		if (sr & MSM_UART_SR_RX_BREAK)
 824			flag = TTY_BREAK;
 825		else if (sr & MSM_UART_SR_PAR_FRAME_ERR)
 826			flag = TTY_FRAME;
 827
 828		uart_port_unlock(port);
 829		sysrq = uart_handle_sysrq_char(port, c);
 830		uart_port_lock(port);
 831		if (!sysrq)
 832			tty_insert_flip_char(tport, c, flag);
 833	}
 834
 835	tty_flip_buffer_push(tport);
 836}
 837
 838static void msm_handle_tx_pio(struct uart_port *port, unsigned int tx_count)
 839{
 840	struct circ_buf *xmit = &port->state->xmit;
 841	struct msm_port *msm_port = to_msm_port(port);
 842	unsigned int num_chars;
 843	unsigned int tf_pointer = 0;
 844	void __iomem *tf;
 845
 846	if (msm_port->is_uartdm)
 847		tf = port->membase + UARTDM_TF;
 848	else
 849		tf = port->membase + MSM_UART_TF;
 850
 851	if (tx_count && msm_port->is_uartdm)
 852		msm_reset_dm_count(port, tx_count);
 853
 854	while (tf_pointer < tx_count) {
 855		int i;
 856		char buf[4] = { 0 };
 857
 858		if (!(msm_read(port, MSM_UART_SR) & MSM_UART_SR_TX_READY))
 859			break;
 860
 861		if (msm_port->is_uartdm)
 862			num_chars = min(tx_count - tf_pointer,
 863					(unsigned int)sizeof(buf));
 864		else
 865			num_chars = 1;
 866
 867		for (i = 0; i < num_chars; i++)
 868			buf[i] = xmit->buf[xmit->tail + i];
 869
 870		iowrite32_rep(tf, buf, 1);
 871		uart_xmit_advance(port, num_chars);
 872		tf_pointer += num_chars;
 873	}
 874
 875	/* disable tx interrupts if nothing more to send */
 876	if (uart_circ_empty(xmit))
 877		msm_stop_tx(port);
 878
 879	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 880		uart_write_wakeup(port);
 881}
 882
 883static void msm_handle_tx(struct uart_port *port)
 884{
 885	struct msm_port *msm_port = to_msm_port(port);
 886	struct circ_buf *xmit = &msm_port->uart.state->xmit;
 887	struct msm_dma *dma = &msm_port->tx_dma;
 888	unsigned int pio_count, dma_count, dma_min;
 889	char buf[4] = { 0 };
 890	void __iomem *tf;
 891	int err = 0;
 892
 893	if (port->x_char) {
 894		if (msm_port->is_uartdm)
 895			tf = port->membase + UARTDM_TF;
 896		else
 897			tf = port->membase + MSM_UART_TF;
 898
 899		buf[0] = port->x_char;
 900
 901		if (msm_port->is_uartdm)
 902			msm_reset_dm_count(port, 1);
 903
 904		iowrite32_rep(tf, buf, 1);
 905		port->icount.tx++;
 906		port->x_char = 0;
 907		return;
 908	}
 909
 910	if (uart_circ_empty(xmit) || uart_tx_stopped(port)) {
 911		msm_stop_tx(port);
 912		return;
 913	}
 914
 915	pio_count = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
 916	dma_count = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
 917
 918	dma_min = 1;	/* Always DMA */
 919	if (msm_port->is_uartdm > UARTDM_1P3) {
 920		dma_count = UARTDM_TX_AIGN(dma_count);
 921		dma_min = UARTDM_BURST_SIZE;
 922	} else {
 923		if (dma_count > UARTDM_TX_MAX)
 924			dma_count = UARTDM_TX_MAX;
 925	}
 926
 927	if (pio_count > port->fifosize)
 928		pio_count = port->fifosize;
 929
 930	if (!dma->chan || dma_count < dma_min)
 931		msm_handle_tx_pio(port, pio_count);
 932	else
 933		err = msm_handle_tx_dma(msm_port, dma_count);
 934
 935	if (err)	/* fall back to PIO mode */
 936		msm_handle_tx_pio(port, pio_count);
 937}
 938
 939static void msm_handle_delta_cts(struct uart_port *port)
 940{
 941	msm_write(port, MSM_UART_CR_CMD_RESET_CTS, MSM_UART_CR);
 942	port->icount.cts++;
 943	wake_up_interruptible(&port->state->port.delta_msr_wait);
 944}
 945
 946static irqreturn_t msm_uart_irq(int irq, void *dev_id)
 947{
 948	struct uart_port *port = dev_id;
 949	struct msm_port *msm_port = to_msm_port(port);
 950	struct msm_dma *dma = &msm_port->rx_dma;
 951	unsigned long flags;
 952	unsigned int misr;
 953	u32 val;
 954
 955	uart_port_lock_irqsave(port, &flags);
 956	misr = msm_read(port, MSM_UART_MISR);
 957	msm_write(port, 0, MSM_UART_IMR); /* disable interrupt */
 958
 959	if (misr & MSM_UART_IMR_RXBREAK_START) {
 960		msm_port->break_detected = true;
 961		msm_write(port, MSM_UART_CR_CMD_RESET_RXBREAK_START, MSM_UART_CR);
 962	}
 963
 964	if (misr & (MSM_UART_IMR_RXLEV | MSM_UART_IMR_RXSTALE)) {
 965		if (dma->count) {
 966			val = MSM_UART_CR_CMD_STALE_EVENT_DISABLE;
 967			msm_write(port, val, MSM_UART_CR);
 968			val = MSM_UART_CR_CMD_RESET_STALE_INT;
 969			msm_write(port, val, MSM_UART_CR);
 970			/*
 971			 * Flush DMA input fifo to memory, this will also
 972			 * trigger DMA RX completion
 973			 */
 974			dmaengine_terminate_all(dma->chan);
 975		} else if (msm_port->is_uartdm) {
 976			msm_handle_rx_dm(port, misr);
 977		} else {
 978			msm_handle_rx(port);
 979		}
 980	}
 981	if (misr & MSM_UART_IMR_TXLEV)
 982		msm_handle_tx(port);
 983	if (misr & MSM_UART_IMR_DELTA_CTS)
 984		msm_handle_delta_cts(port);
 985
 986	msm_write(port, msm_port->imr, MSM_UART_IMR); /* restore interrupt */
 987	uart_port_unlock_irqrestore(port, flags);
 988
 989	return IRQ_HANDLED;
 990}
 991
 992static unsigned int msm_tx_empty(struct uart_port *port)
 993{
 994	return (msm_read(port, MSM_UART_SR) & MSM_UART_SR_TX_EMPTY) ? TIOCSER_TEMT : 0;
 995}
 996
 997static unsigned int msm_get_mctrl(struct uart_port *port)
 998{
 999	return TIOCM_CAR | TIOCM_CTS | TIOCM_DSR | TIOCM_RTS;
1000}
1001
1002static void msm_reset(struct uart_port *port)
1003{
1004	struct msm_port *msm_port = to_msm_port(port);
1005	unsigned int mr;
1006
1007	/* reset everything */
1008	msm_write(port, MSM_UART_CR_CMD_RESET_RX, MSM_UART_CR);
1009	msm_write(port, MSM_UART_CR_CMD_RESET_TX, MSM_UART_CR);
1010	msm_write(port, MSM_UART_CR_CMD_RESET_ERR, MSM_UART_CR);
1011	msm_write(port, MSM_UART_CR_CMD_RESET_BREAK_INT, MSM_UART_CR);
1012	msm_write(port, MSM_UART_CR_CMD_RESET_CTS, MSM_UART_CR);
1013	msm_write(port, MSM_UART_CR_CMD_RESET_RFR, MSM_UART_CR);
1014	mr = msm_read(port, MSM_UART_MR1);
1015	mr &= ~MSM_UART_MR1_RX_RDY_CTL;
1016	msm_write(port, mr, MSM_UART_MR1);
1017
1018	/* Disable DM modes */
1019	if (msm_port->is_uartdm)
1020		msm_write(port, 0, UARTDM_DMEN);
1021}
1022
1023static void msm_set_mctrl(struct uart_port *port, unsigned int mctrl)
1024{
1025	unsigned int mr;
1026
1027	mr = msm_read(port, MSM_UART_MR1);
1028
1029	if (!(mctrl & TIOCM_RTS)) {
1030		mr &= ~MSM_UART_MR1_RX_RDY_CTL;
1031		msm_write(port, mr, MSM_UART_MR1);
1032		msm_write(port, MSM_UART_CR_CMD_RESET_RFR, MSM_UART_CR);
1033	} else {
1034		mr |= MSM_UART_MR1_RX_RDY_CTL;
1035		msm_write(port, mr, MSM_UART_MR1);
1036	}
1037}
1038
1039static void msm_break_ctl(struct uart_port *port, int break_ctl)
1040{
1041	if (break_ctl)
1042		msm_write(port, MSM_UART_CR_CMD_START_BREAK, MSM_UART_CR);
1043	else
1044		msm_write(port, MSM_UART_CR_CMD_STOP_BREAK, MSM_UART_CR);
1045}
1046
1047struct msm_baud_map {
1048	u16	divisor;
1049	u8	code;
1050	u8	rxstale;
1051};
1052
1053static const struct msm_baud_map *
1054msm_find_best_baud(struct uart_port *port, unsigned int baud,
1055		   unsigned long *rate)
1056{
1057	struct msm_port *msm_port = to_msm_port(port);
1058	unsigned int divisor, result;
1059	unsigned long target, old, best_rate = 0, diff, best_diff = ULONG_MAX;
1060	const struct msm_baud_map *entry, *end, *best;
1061	static const struct msm_baud_map table[] = {
1062		{    1, 0xff, 31 },
1063		{    2, 0xee, 16 },
1064		{    3, 0xdd,  8 },
1065		{    4, 0xcc,  6 },
1066		{    6, 0xbb,  6 },
1067		{    8, 0xaa,  6 },
1068		{   12, 0x99,  6 },
1069		{   16, 0x88,  1 },
1070		{   24, 0x77,  1 },
1071		{   32, 0x66,  1 },
1072		{   48, 0x55,  1 },
1073		{   96, 0x44,  1 },
1074		{  192, 0x33,  1 },
1075		{  384, 0x22,  1 },
1076		{  768, 0x11,  1 },
1077		{ 1536, 0x00,  1 },
1078	};
1079
1080	best = table; /* Default to smallest divider */
1081	target = clk_round_rate(msm_port->clk, 16 * baud);
1082	divisor = DIV_ROUND_CLOSEST(target, 16 * baud);
1083
1084	end = table + ARRAY_SIZE(table);
1085	entry = table;
1086	while (entry < end) {
1087		if (entry->divisor <= divisor) {
1088			result = target / entry->divisor / 16;
1089			diff = abs(result - baud);
1090
1091			/* Keep track of best entry */
1092			if (diff < best_diff) {
1093				best_diff = diff;
1094				best = entry;
1095				best_rate = target;
1096			}
1097
1098			if (result == baud)
1099				break;
1100		} else if (entry->divisor > divisor) {
1101			old = target;
1102			target = clk_round_rate(msm_port->clk, old + 1);
1103			/*
1104			 * The rate didn't get any faster so we can't do
1105			 * better at dividing it down
1106			 */
1107			if (target == old)
1108				break;
1109
1110			/* Start the divisor search over at this new rate */
1111			entry = table;
1112			divisor = DIV_ROUND_CLOSEST(target, 16 * baud);
1113			continue;
1114		}
1115		entry++;
1116	}
1117
1118	*rate = best_rate;
1119	return best;
1120}
1121
1122static int msm_set_baud_rate(struct uart_port *port, unsigned int baud,
1123			     unsigned long *saved_flags)
1124	__must_hold(&port->lock)
1125{
1126	unsigned int rxstale, watermark, mask;
1127	struct msm_port *msm_port = to_msm_port(port);
1128	const struct msm_baud_map *entry;
1129	unsigned long flags, rate;
1130
1131	flags = *saved_flags;
1132	uart_port_unlock_irqrestore(port, flags);
1133
1134	entry = msm_find_best_baud(port, baud, &rate);
1135	dev_pm_opp_set_rate(port->dev, rate);
1136	baud = rate / 16 / entry->divisor;
1137
1138	uart_port_lock_irqsave(port, &flags);
1139	*saved_flags = flags;
1140	port->uartclk = rate;
1141
1142	msm_write(port, entry->code, MSM_UART_CSR);
1143
1144	/* RX stale watermark */
1145	rxstale = entry->rxstale;
1146	watermark = MSM_UART_IPR_STALE_LSB & rxstale;
1147	if (msm_port->is_uartdm) {
1148		mask = MSM_UART_DM_IPR_STALE_TIMEOUT_MSB;
1149	} else {
1150		watermark |= MSM_UART_IPR_RXSTALE_LAST;
1151		mask = MSM_UART_IPR_STALE_TIMEOUT_MSB;
1152	}
1153
1154	watermark |= mask & (rxstale << 2);
1155
1156	msm_write(port, watermark, MSM_UART_IPR);
1157
1158	/* set RX watermark */
1159	watermark = (port->fifosize * 3) / 4;
1160	msm_write(port, watermark, MSM_UART_RFWR);
1161
1162	/* set TX watermark */
1163	msm_write(port, 10, MSM_UART_TFWR);
1164
1165	msm_write(port, MSM_UART_CR_CMD_PROTECTION_EN, MSM_UART_CR);
1166	msm_reset(port);
1167
1168	/* Enable RX and TX */
1169	msm_write(port, MSM_UART_CR_TX_ENABLE | MSM_UART_CR_RX_ENABLE, MSM_UART_CR);
1170
1171	/* turn on RX and CTS interrupts */
1172	msm_port->imr = MSM_UART_IMR_RXLEV | MSM_UART_IMR_RXSTALE |
1173			MSM_UART_IMR_CURRENT_CTS | MSM_UART_IMR_RXBREAK_START;
1174
1175	msm_write(port, msm_port->imr, MSM_UART_IMR);
1176
1177	if (msm_port->is_uartdm) {
1178		msm_write(port, MSM_UART_CR_CMD_RESET_STALE_INT, MSM_UART_CR);
1179		msm_write(port, 0xFFFFFF, UARTDM_DMRX);
1180		msm_write(port, MSM_UART_CR_CMD_STALE_EVENT_ENABLE, MSM_UART_CR);
1181	}
1182
1183	return baud;
1184}
1185
1186static void msm_init_clock(struct uart_port *port)
1187{
1188	struct msm_port *msm_port = to_msm_port(port);
1189
1190	dev_pm_opp_set_rate(port->dev, port->uartclk);
1191	clk_prepare_enable(msm_port->clk);
1192	clk_prepare_enable(msm_port->pclk);
1193	msm_serial_set_mnd_regs(port);
1194}
1195
1196static int msm_startup(struct uart_port *port)
1197{
1198	struct msm_port *msm_port = to_msm_port(port);
1199	unsigned int data, rfr_level, mask;
1200	int ret;
1201
1202	snprintf(msm_port->name, sizeof(msm_port->name),
1203		 "msm_serial%d", port->line);
1204
1205	msm_init_clock(port);
1206
1207	if (likely(port->fifosize > 12))
1208		rfr_level = port->fifosize - 12;
1209	else
1210		rfr_level = port->fifosize;
1211
1212	/* set automatic RFR level */
1213	data = msm_read(port, MSM_UART_MR1);
1214
1215	if (msm_port->is_uartdm)
1216		mask = MSM_UART_DM_MR1_AUTO_RFR_LEVEL1;
1217	else
1218		mask = MSM_UART_MR1_AUTO_RFR_LEVEL1;
1219
1220	data &= ~mask;
1221	data &= ~MSM_UART_MR1_AUTO_RFR_LEVEL0;
1222	data |= mask & (rfr_level << 2);
1223	data |= MSM_UART_MR1_AUTO_RFR_LEVEL0 & rfr_level;
1224	msm_write(port, data, MSM_UART_MR1);
1225
1226	if (msm_port->is_uartdm) {
1227		msm_request_tx_dma(msm_port, msm_port->uart.mapbase);
1228		msm_request_rx_dma(msm_port, msm_port->uart.mapbase);
1229	}
1230
1231	ret = request_irq(port->irq, msm_uart_irq, IRQF_TRIGGER_HIGH,
1232			  msm_port->name, port);
1233	if (unlikely(ret))
1234		goto err_irq;
1235
1236	return 0;
1237
1238err_irq:
1239	if (msm_port->is_uartdm)
1240		msm_release_dma(msm_port);
1241
1242	clk_disable_unprepare(msm_port->pclk);
1243	clk_disable_unprepare(msm_port->clk);
1244	dev_pm_opp_set_rate(port->dev, 0);
1245
1246	return ret;
1247}
1248
1249static void msm_shutdown(struct uart_port *port)
1250{
1251	struct msm_port *msm_port = to_msm_port(port);
1252
1253	msm_port->imr = 0;
1254	msm_write(port, 0, MSM_UART_IMR); /* disable interrupts */
1255
1256	if (msm_port->is_uartdm)
1257		msm_release_dma(msm_port);
1258
1259	clk_disable_unprepare(msm_port->clk);
1260	dev_pm_opp_set_rate(port->dev, 0);
1261
1262	free_irq(port->irq, port);
1263}
1264
1265static void msm_set_termios(struct uart_port *port, struct ktermios *termios,
1266			    const struct ktermios *old)
1267{
1268	struct msm_port *msm_port = to_msm_port(port);
1269	struct msm_dma *dma = &msm_port->rx_dma;
1270	unsigned long flags;
1271	unsigned int baud, mr;
1272
1273	uart_port_lock_irqsave(port, &flags);
1274
1275	if (dma->chan) /* Terminate if any */
1276		msm_stop_dma(port, dma);
1277
1278	/* calculate and set baud rate */
1279	baud = uart_get_baud_rate(port, termios, old, 300, 4000000);
1280	baud = msm_set_baud_rate(port, baud, &flags);
1281	if (tty_termios_baud_rate(termios))
1282		tty_termios_encode_baud_rate(termios, baud, baud);
1283
1284	/* calculate parity */
1285	mr = msm_read(port, MSM_UART_MR2);
1286	mr &= ~MSM_UART_MR2_PARITY_MODE;
1287	if (termios->c_cflag & PARENB) {
1288		if (termios->c_cflag & PARODD)
1289			mr |= MSM_UART_MR2_PARITY_MODE_ODD;
1290		else if (termios->c_cflag & CMSPAR)
1291			mr |= MSM_UART_MR2_PARITY_MODE_SPACE;
1292		else
1293			mr |= MSM_UART_MR2_PARITY_MODE_EVEN;
1294	}
1295
1296	/* calculate bits per char */
1297	mr &= ~MSM_UART_MR2_BITS_PER_CHAR;
1298	switch (termios->c_cflag & CSIZE) {
1299	case CS5:
1300		mr |= MSM_UART_MR2_BITS_PER_CHAR_5;
1301		break;
1302	case CS6:
1303		mr |= MSM_UART_MR2_BITS_PER_CHAR_6;
1304		break;
1305	case CS7:
1306		mr |= MSM_UART_MR2_BITS_PER_CHAR_7;
1307		break;
1308	case CS8:
1309	default:
1310		mr |= MSM_UART_MR2_BITS_PER_CHAR_8;
1311		break;
1312	}
1313
1314	/* calculate stop bits */
1315	mr &= ~(MSM_UART_MR2_STOP_BIT_LEN_ONE | MSM_UART_MR2_STOP_BIT_LEN_TWO);
1316	if (termios->c_cflag & CSTOPB)
1317		mr |= MSM_UART_MR2_STOP_BIT_LEN_TWO;
1318	else
1319		mr |= MSM_UART_MR2_STOP_BIT_LEN_ONE;
1320
1321	/* set parity, bits per char, and stop bit */
1322	msm_write(port, mr, MSM_UART_MR2);
1323
1324	/* calculate and set hardware flow control */
1325	mr = msm_read(port, MSM_UART_MR1);
1326	mr &= ~(MSM_UART_MR1_CTS_CTL | MSM_UART_MR1_RX_RDY_CTL);
1327	if (termios->c_cflag & CRTSCTS) {
1328		mr |= MSM_UART_MR1_CTS_CTL;
1329		mr |= MSM_UART_MR1_RX_RDY_CTL;
1330	}
1331	msm_write(port, mr, MSM_UART_MR1);
1332
1333	/* Configure status bits to ignore based on termio flags. */
1334	port->read_status_mask = 0;
1335	if (termios->c_iflag & INPCK)
1336		port->read_status_mask |= MSM_UART_SR_PAR_FRAME_ERR;
1337	if (termios->c_iflag & (IGNBRK | BRKINT | PARMRK))
1338		port->read_status_mask |= MSM_UART_SR_RX_BREAK;
1339
1340	uart_update_timeout(port, termios->c_cflag, baud);
1341
1342	/* Try to use DMA */
1343	msm_start_rx_dma(msm_port);
1344
1345	uart_port_unlock_irqrestore(port, flags);
1346}
1347
1348static const char *msm_type(struct uart_port *port)
1349{
1350	return "MSM";
1351}
1352
1353static void msm_release_port(struct uart_port *port)
1354{
1355	struct platform_device *pdev = to_platform_device(port->dev);
1356	struct resource *uart_resource;
1357	resource_size_t size;
1358
1359	uart_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1360	if (unlikely(!uart_resource))
1361		return;
1362	size = resource_size(uart_resource);
1363
1364	release_mem_region(port->mapbase, size);
1365	iounmap(port->membase);
1366	port->membase = NULL;
1367}
1368
1369static int msm_request_port(struct uart_port *port)
1370{
1371	struct platform_device *pdev = to_platform_device(port->dev);
1372	struct resource *uart_resource;
1373	resource_size_t size;
1374	int ret;
1375
1376	uart_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1377	if (unlikely(!uart_resource))
1378		return -ENXIO;
1379
1380	size = resource_size(uart_resource);
1381
1382	if (!request_mem_region(port->mapbase, size, "msm_serial"))
1383		return -EBUSY;
1384
1385	port->membase = ioremap(port->mapbase, size);
1386	if (!port->membase) {
1387		ret = -EBUSY;
1388		goto fail_release_port;
1389	}
1390
1391	return 0;
1392
1393fail_release_port:
1394	release_mem_region(port->mapbase, size);
1395	return ret;
1396}
1397
1398static void msm_config_port(struct uart_port *port, int flags)
1399{
1400	int ret;
1401
1402	if (flags & UART_CONFIG_TYPE) {
1403		port->type = PORT_MSM;
1404		ret = msm_request_port(port);
1405		if (ret)
1406			return;
1407	}
1408}
1409
1410static int msm_verify_port(struct uart_port *port, struct serial_struct *ser)
1411{
1412	if (unlikely(ser->type != PORT_UNKNOWN && ser->type != PORT_MSM))
1413		return -EINVAL;
1414	if (unlikely(port->irq != ser->irq))
1415		return -EINVAL;
1416	return 0;
1417}
1418
1419static void msm_power(struct uart_port *port, unsigned int state,
1420		      unsigned int oldstate)
1421{
1422	struct msm_port *msm_port = to_msm_port(port);
1423
1424	switch (state) {
1425	case 0:
1426		dev_pm_opp_set_rate(port->dev, port->uartclk);
1427		clk_prepare_enable(msm_port->clk);
1428		clk_prepare_enable(msm_port->pclk);
1429		break;
1430	case 3:
1431		clk_disable_unprepare(msm_port->clk);
1432		dev_pm_opp_set_rate(port->dev, 0);
1433		clk_disable_unprepare(msm_port->pclk);
1434		break;
1435	default:
1436		pr_err("msm_serial: Unknown PM state %d\n", state);
1437	}
1438}
1439
1440#ifdef CONFIG_CONSOLE_POLL
1441static int msm_poll_get_char_single(struct uart_port *port)
1442{
1443	struct msm_port *msm_port = to_msm_port(port);
1444	unsigned int rf_reg = msm_port->is_uartdm ? UARTDM_RF : MSM_UART_RF;
1445
1446	if (!(msm_read(port, MSM_UART_SR) & MSM_UART_SR_RX_READY))
1447		return NO_POLL_CHAR;
1448
1449	return msm_read(port, rf_reg) & 0xff;
1450}
1451
1452static int msm_poll_get_char_dm(struct uart_port *port)
1453{
1454	int c;
1455	static u32 slop;
1456	static int count;
1457	unsigned char *sp = (unsigned char *)&slop;
1458
1459	/* Check if a previous read had more than one char */
1460	if (count) {
1461		c = sp[sizeof(slop) - count];
1462		count--;
1463	/* Or if FIFO is empty */
1464	} else if (!(msm_read(port, MSM_UART_SR) & MSM_UART_SR_RX_READY)) {
1465		/*
1466		 * If RX packing buffer has less than a word, force stale to
1467		 * push contents into RX FIFO
1468		 */
1469		count = msm_read(port, UARTDM_RXFS);
1470		count = (count >> UARTDM_RXFS_BUF_SHIFT) & UARTDM_RXFS_BUF_MASK;
1471		if (count) {
1472			msm_write(port, MSM_UART_CR_CMD_FORCE_STALE, MSM_UART_CR);
1473			slop = msm_read(port, UARTDM_RF);
1474			c = sp[0];
1475			count--;
1476			msm_write(port, MSM_UART_CR_CMD_RESET_STALE_INT, MSM_UART_CR);
1477			msm_write(port, 0xFFFFFF, UARTDM_DMRX);
1478			msm_write(port, MSM_UART_CR_CMD_STALE_EVENT_ENABLE, MSM_UART_CR);
1479		} else {
1480			c = NO_POLL_CHAR;
1481		}
1482	/* FIFO has a word */
1483	} else {
1484		slop = msm_read(port, UARTDM_RF);
1485		c = sp[0];
1486		count = sizeof(slop) - 1;
1487	}
1488
1489	return c;
1490}
1491
1492static int msm_poll_get_char(struct uart_port *port)
1493{
1494	u32 imr;
1495	int c;
1496	struct msm_port *msm_port = to_msm_port(port);
1497
1498	/* Disable all interrupts */
1499	imr = msm_read(port, MSM_UART_IMR);
1500	msm_write(port, 0, MSM_UART_IMR);
1501
1502	if (msm_port->is_uartdm)
1503		c = msm_poll_get_char_dm(port);
1504	else
1505		c = msm_poll_get_char_single(port);
1506
1507	/* Enable interrupts */
1508	msm_write(port, imr, MSM_UART_IMR);
1509
1510	return c;
1511}
1512
1513static void msm_poll_put_char(struct uart_port *port, unsigned char c)
1514{
1515	u32 imr;
1516	struct msm_port *msm_port = to_msm_port(port);
1517
1518	/* Disable all interrupts */
1519	imr = msm_read(port, MSM_UART_IMR);
1520	msm_write(port, 0, MSM_UART_IMR);
1521
1522	if (msm_port->is_uartdm)
1523		msm_reset_dm_count(port, 1);
1524
1525	/* Wait until FIFO is empty */
1526	while (!(msm_read(port, MSM_UART_SR) & MSM_UART_SR_TX_READY))
1527		cpu_relax();
1528
1529	/* Write a character */
1530	msm_write(port, c, msm_port->is_uartdm ? UARTDM_TF : MSM_UART_TF);
1531
1532	/* Wait until FIFO is empty */
1533	while (!(msm_read(port, MSM_UART_SR) & MSM_UART_SR_TX_READY))
1534		cpu_relax();
1535
1536	/* Enable interrupts */
1537	msm_write(port, imr, MSM_UART_IMR);
1538}
1539#endif
1540
1541static const struct uart_ops msm_uart_pops = {
1542	.tx_empty = msm_tx_empty,
1543	.set_mctrl = msm_set_mctrl,
1544	.get_mctrl = msm_get_mctrl,
1545	.stop_tx = msm_stop_tx,
1546	.start_tx = msm_start_tx,
1547	.stop_rx = msm_stop_rx,
1548	.enable_ms = msm_enable_ms,
1549	.break_ctl = msm_break_ctl,
1550	.startup = msm_startup,
1551	.shutdown = msm_shutdown,
1552	.set_termios = msm_set_termios,
1553	.type = msm_type,
1554	.release_port = msm_release_port,
1555	.request_port = msm_request_port,
1556	.config_port = msm_config_port,
1557	.verify_port = msm_verify_port,
1558	.pm = msm_power,
1559#ifdef CONFIG_CONSOLE_POLL
1560	.poll_get_char	= msm_poll_get_char,
1561	.poll_put_char	= msm_poll_put_char,
1562#endif
1563};
1564
1565static struct msm_port msm_uart_ports[] = {
1566	{
1567		.uart = {
1568			.iotype = UPIO_MEM,
1569			.ops = &msm_uart_pops,
1570			.flags = UPF_BOOT_AUTOCONF,
1571			.fifosize = 64,
1572			.line = 0,
1573		},
1574	},
1575	{
1576		.uart = {
1577			.iotype = UPIO_MEM,
1578			.ops = &msm_uart_pops,
1579			.flags = UPF_BOOT_AUTOCONF,
1580			.fifosize = 64,
1581			.line = 1,
1582		},
1583	},
1584	{
1585		.uart = {
1586			.iotype = UPIO_MEM,
1587			.ops = &msm_uart_pops,
1588			.flags = UPF_BOOT_AUTOCONF,
1589			.fifosize = 64,
1590			.line = 2,
1591		},
1592	},
1593};
1594
1595#define MSM_UART_NR	ARRAY_SIZE(msm_uart_ports)
1596
1597static inline struct uart_port *msm_get_port_from_line(unsigned int line)
1598{
1599	return &msm_uart_ports[line].uart;
1600}
1601
1602#ifdef CONFIG_SERIAL_MSM_CONSOLE
1603static void __msm_console_write(struct uart_port *port, const char *s,
1604				unsigned int count, bool is_uartdm)
1605{
1606	unsigned long flags;
1607	int i;
1608	int num_newlines = 0;
1609	bool replaced = false;
1610	void __iomem *tf;
1611	int locked = 1;
1612
1613	if (is_uartdm)
1614		tf = port->membase + UARTDM_TF;
1615	else
1616		tf = port->membase + MSM_UART_TF;
1617
1618	/* Account for newlines that will get a carriage return added */
1619	for (i = 0; i < count; i++)
1620		if (s[i] == '\n')
1621			num_newlines++;
1622	count += num_newlines;
1623
1624	local_irq_save(flags);
1625
1626	if (port->sysrq)
1627		locked = 0;
1628	else if (oops_in_progress)
1629		locked = uart_port_trylock(port);
1630	else
1631		uart_port_lock(port);
1632
1633	if (is_uartdm)
1634		msm_reset_dm_count(port, count);
1635
1636	i = 0;
1637	while (i < count) {
1638		int j;
1639		unsigned int num_chars;
1640		char buf[4] = { 0 };
1641
1642		if (is_uartdm)
1643			num_chars = min(count - i, (unsigned int)sizeof(buf));
1644		else
1645			num_chars = 1;
1646
1647		for (j = 0; j < num_chars; j++) {
1648			char c = *s;
1649
1650			if (c == '\n' && !replaced) {
1651				buf[j] = '\r';
1652				j++;
1653				replaced = true;
1654			}
1655			if (j < num_chars) {
1656				buf[j] = c;
1657				s++;
1658				replaced = false;
1659			}
1660		}
1661
1662		while (!(msm_read(port, MSM_UART_SR) & MSM_UART_SR_TX_READY))
1663			cpu_relax();
1664
1665		iowrite32_rep(tf, buf, 1);
1666		i += num_chars;
1667	}
1668
1669	if (locked)
1670		uart_port_unlock(port);
1671
1672	local_irq_restore(flags);
1673}
1674
1675static void msm_console_write(struct console *co, const char *s,
1676			      unsigned int count)
1677{
1678	struct uart_port *port;
1679	struct msm_port *msm_port;
1680
1681	BUG_ON(co->index < 0 || co->index >= MSM_UART_NR);
1682
1683	port = msm_get_port_from_line(co->index);
1684	msm_port = to_msm_port(port);
1685
1686	__msm_console_write(port, s, count, msm_port->is_uartdm);
1687}
1688
1689static int msm_console_setup(struct console *co, char *options)
1690{
1691	struct uart_port *port;
1692	int baud = 115200;
1693	int bits = 8;
1694	int parity = 'n';
1695	int flow = 'n';
1696
1697	if (unlikely(co->index >= MSM_UART_NR || co->index < 0))
1698		return -ENXIO;
1699
1700	port = msm_get_port_from_line(co->index);
1701
1702	if (unlikely(!port->membase))
1703		return -ENXIO;
1704
1705	msm_init_clock(port);
1706
1707	if (options)
1708		uart_parse_options(options, &baud, &parity, &bits, &flow);
1709
1710	pr_info("msm_serial: console setup on port #%d\n", port->line);
1711
1712	return uart_set_options(port, co, baud, parity, bits, flow);
1713}
1714
1715static void
1716msm_serial_early_write(struct console *con, const char *s, unsigned n)
1717{
1718	struct earlycon_device *dev = con->data;
1719
1720	__msm_console_write(&dev->port, s, n, false);
1721}
1722
1723static int __init
1724msm_serial_early_console_setup(struct earlycon_device *device, const char *opt)
1725{
1726	if (!device->port.membase)
1727		return -ENODEV;
1728
1729	device->con->write = msm_serial_early_write;
1730	return 0;
1731}
1732OF_EARLYCON_DECLARE(msm_serial, "qcom,msm-uart",
1733		    msm_serial_early_console_setup);
1734
1735static void
1736msm_serial_early_write_dm(struct console *con, const char *s, unsigned n)
1737{
1738	struct earlycon_device *dev = con->data;
1739
1740	__msm_console_write(&dev->port, s, n, true);
1741}
1742
1743static int __init
1744msm_serial_early_console_setup_dm(struct earlycon_device *device,
1745				  const char *opt)
1746{
1747	if (!device->port.membase)
1748		return -ENODEV;
1749
1750	device->con->write = msm_serial_early_write_dm;
1751	return 0;
1752}
1753OF_EARLYCON_DECLARE(msm_serial_dm, "qcom,msm-uartdm",
1754		    msm_serial_early_console_setup_dm);
1755
1756static struct uart_driver msm_uart_driver;
1757
1758static struct console msm_console = {
1759	.name = "ttyMSM",
1760	.write = msm_console_write,
1761	.device = uart_console_device,
1762	.setup = msm_console_setup,
1763	.flags = CON_PRINTBUFFER,
1764	.index = -1,
1765	.data = &msm_uart_driver,
1766};
1767
1768#define MSM_CONSOLE	(&msm_console)
1769
1770#else
1771#define MSM_CONSOLE	NULL
1772#endif
1773
1774static struct uart_driver msm_uart_driver = {
1775	.owner = THIS_MODULE,
1776	.driver_name = "msm_serial",
1777	.dev_name = "ttyMSM",
1778	.nr = MSM_UART_NR,
1779	.cons = MSM_CONSOLE,
1780};
1781
1782static atomic_t msm_uart_next_id = ATOMIC_INIT(0);
1783
1784static const struct of_device_id msm_uartdm_table[] = {
1785	{ .compatible = "qcom,msm-uartdm-v1.1", .data = (void *)UARTDM_1P1 },
1786	{ .compatible = "qcom,msm-uartdm-v1.2", .data = (void *)UARTDM_1P2 },
1787	{ .compatible = "qcom,msm-uartdm-v1.3", .data = (void *)UARTDM_1P3 },
1788	{ .compatible = "qcom,msm-uartdm-v1.4", .data = (void *)UARTDM_1P4 },
1789	{ }
1790};
1791
1792static int msm_serial_probe(struct platform_device *pdev)
1793{
1794	struct msm_port *msm_port;
1795	struct resource *resource;
1796	struct uart_port *port;
1797	const struct of_device_id *id;
1798	int irq, line, ret;
1799
1800	if (pdev->dev.of_node)
1801		line = of_alias_get_id(pdev->dev.of_node, "serial");
1802	else
1803		line = pdev->id;
1804
1805	if (line < 0)
1806		line = atomic_inc_return(&msm_uart_next_id) - 1;
1807
1808	if (unlikely(line < 0 || line >= MSM_UART_NR))
1809		return -ENXIO;
1810
1811	dev_info(&pdev->dev, "msm_serial: detected port #%d\n", line);
1812
1813	port = msm_get_port_from_line(line);
1814	port->dev = &pdev->dev;
1815	msm_port = to_msm_port(port);
1816
1817	id = of_match_device(msm_uartdm_table, &pdev->dev);
1818	if (id)
1819		msm_port->is_uartdm = (unsigned long)id->data;
1820	else
1821		msm_port->is_uartdm = 0;
1822
1823	msm_port->clk = devm_clk_get(&pdev->dev, "core");
1824	if (IS_ERR(msm_port->clk))
1825		return PTR_ERR(msm_port->clk);
1826
1827	if (msm_port->is_uartdm) {
1828		msm_port->pclk = devm_clk_get(&pdev->dev, "iface");
1829		if (IS_ERR(msm_port->pclk))
1830			return PTR_ERR(msm_port->pclk);
1831	}
1832
1833	ret = devm_pm_opp_set_clkname(&pdev->dev, "core");
1834	if (ret)
1835		return ret;
1836
1837	/* OPP table is optional */
1838	ret = devm_pm_opp_of_add_table(&pdev->dev);
1839	if (ret && ret != -ENODEV)
1840		return dev_err_probe(&pdev->dev, ret, "invalid OPP table\n");
1841
1842	port->uartclk = clk_get_rate(msm_port->clk);
1843	dev_info(&pdev->dev, "uartclk = %d\n", port->uartclk);
1844
1845	resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1846	if (unlikely(!resource))
1847		return -ENXIO;
1848	port->mapbase = resource->start;
1849
1850	irq = platform_get_irq(pdev, 0);
1851	if (unlikely(irq < 0))
1852		return -ENXIO;
1853	port->irq = irq;
1854	port->has_sysrq = IS_ENABLED(CONFIG_SERIAL_MSM_CONSOLE);
1855
1856	platform_set_drvdata(pdev, port);
1857
1858	return uart_add_one_port(&msm_uart_driver, port);
1859}
1860
1861static void msm_serial_remove(struct platform_device *pdev)
1862{
1863	struct uart_port *port = platform_get_drvdata(pdev);
1864
1865	uart_remove_one_port(&msm_uart_driver, port);
1866}
1867
1868static const struct of_device_id msm_match_table[] = {
1869	{ .compatible = "qcom,msm-uart" },
1870	{ .compatible = "qcom,msm-uartdm" },
1871	{}
1872};
1873MODULE_DEVICE_TABLE(of, msm_match_table);
1874
1875static int __maybe_unused msm_serial_suspend(struct device *dev)
1876{
1877	struct msm_port *port = dev_get_drvdata(dev);
1878
1879	uart_suspend_port(&msm_uart_driver, &port->uart);
1880
1881	return 0;
1882}
1883
1884static int __maybe_unused msm_serial_resume(struct device *dev)
1885{
1886	struct msm_port *port = dev_get_drvdata(dev);
1887
1888	uart_resume_port(&msm_uart_driver, &port->uart);
1889
1890	return 0;
1891}
1892
1893static const struct dev_pm_ops msm_serial_dev_pm_ops = {
1894	SET_SYSTEM_SLEEP_PM_OPS(msm_serial_suspend, msm_serial_resume)
1895};
1896
1897static struct platform_driver msm_platform_driver = {
1898	.remove_new = msm_serial_remove,
1899	.probe = msm_serial_probe,
1900	.driver = {
1901		.name = "msm_serial",
1902		.pm = &msm_serial_dev_pm_ops,
1903		.of_match_table = msm_match_table,
1904	},
1905};
1906
1907static int __init msm_serial_init(void)
1908{
1909	int ret;
1910
1911	ret = uart_register_driver(&msm_uart_driver);
1912	if (unlikely(ret))
1913		return ret;
1914
1915	ret = platform_driver_register(&msm_platform_driver);
1916	if (unlikely(ret))
1917		uart_unregister_driver(&msm_uart_driver);
1918
1919	pr_info("msm_serial: driver initialized\n");
1920
1921	return ret;
1922}
1923
1924static void __exit msm_serial_exit(void)
1925{
1926	platform_driver_unregister(&msm_platform_driver);
1927	uart_unregister_driver(&msm_uart_driver);
1928}
1929
1930module_init(msm_serial_init);
1931module_exit(msm_serial_exit);
1932
1933MODULE_AUTHOR("Robert Love <rlove@google.com>");
1934MODULE_DESCRIPTION("Driver for msm7x serial device");
1935MODULE_LICENSE("GPL");