1// SPDX-License-Identifier: GPL-2.0
   2// Copyright (c) 2017-2018, The Linux foundation. All rights reserved.
   3
   4/* Disable MMIO tracing to prevent excessive logging of unwanted MMIO traces */
   5#define __DISABLE_TRACE_MMIO__
   6
   7#include <linux/clk.h>
   8#include <linux/console.h>
   9#include <linux/io.h>
  10#include <linux/iopoll.h>
  11#include <linux/irq.h>
  12#include <linux/module.h>
  13#include <linux/of.h>
  14#include <linux/pm_opp.h>
  15#include <linux/platform_device.h>
  16#include <linux/pm_runtime.h>
  17#include <linux/pm_wakeirq.h>
  18#include <linux/soc/qcom/geni-se.h>
  19#include <linux/serial.h>
  20#include <linux/serial_core.h>
  21#include <linux/slab.h>
  22#include <linux/tty.h>
  23#include <linux/tty_flip.h>
  24#include <dt-bindings/interconnect/qcom,icc.h>
  25
  26/* UART specific GENI registers */
  27#define SE_UART_LOOPBACK_CFG		0x22c
  28#define SE_UART_IO_MACRO_CTRL		0x240
  29#define SE_UART_TX_TRANS_CFG		0x25c
  30#define SE_UART_TX_WORD_LEN		0x268
  31#define SE_UART_TX_STOP_BIT_LEN		0x26c
  32#define SE_UART_TX_TRANS_LEN		0x270
  33#define SE_UART_RX_TRANS_CFG		0x280
  34#define SE_UART_RX_WORD_LEN		0x28c
  35#define SE_UART_RX_STALE_CNT		0x294
  36#define SE_UART_TX_PARITY_CFG		0x2a4
  37#define SE_UART_RX_PARITY_CFG		0x2a8
  38#define SE_UART_MANUAL_RFR		0x2ac
  39
  40/* SE_UART_TRANS_CFG */
  41#define UART_TX_PAR_EN			BIT(0)
  42#define UART_CTS_MASK			BIT(1)
  43
  44/* SE_UART_TX_STOP_BIT_LEN */
  45#define TX_STOP_BIT_LEN_1		0
  46#define TX_STOP_BIT_LEN_2		2
  47
  48/* SE_UART_RX_TRANS_CFG */
  49#define UART_RX_PAR_EN			BIT(3)
  50
  51/* SE_UART_RX_WORD_LEN */
  52#define RX_WORD_LEN_MASK		GENMASK(9, 0)
  53
  54/* SE_UART_RX_STALE_CNT */
  55#define RX_STALE_CNT			GENMASK(23, 0)
  56
  57/* SE_UART_TX_PARITY_CFG/RX_PARITY_CFG */
  58#define PAR_CALC_EN			BIT(0)
  59#define PAR_EVEN			0x00
  60#define PAR_ODD				0x01
  61#define PAR_SPACE			0x10
  62
  63/* SE_UART_MANUAL_RFR register fields */
  64#define UART_MANUAL_RFR_EN		BIT(31)
  65#define UART_RFR_NOT_READY		BIT(1)
  66#define UART_RFR_READY			BIT(0)
  67
  68/* UART M_CMD OP codes */
  69#define UART_START_TX			0x1
  70/* UART S_CMD OP codes */
  71#define UART_START_READ			0x1
  72#define UART_PARAM			0x1
  73#define UART_PARAM_RFR_OPEN		BIT(7)
  74
  75#define UART_OVERSAMPLING		32
  76#define STALE_TIMEOUT			16
  77#define DEFAULT_BITS_PER_CHAR		10
  78#define GENI_UART_CONS_PORTS		1
  79#define GENI_UART_PORTS			3
  80#define DEF_FIFO_DEPTH_WORDS		16
  81#define DEF_TX_WM			2
  82#define DEF_FIFO_WIDTH_BITS		32
  83#define UART_RX_WM			2
  84
  85/* SE_UART_LOOPBACK_CFG */
  86#define RX_TX_SORTED			BIT(0)
  87#define CTS_RTS_SORTED			BIT(1)
  88#define RX_TX_CTS_RTS_SORTED		(RX_TX_SORTED | CTS_RTS_SORTED)
  89
  90/* UART pin swap value */
  91#define DEFAULT_IO_MACRO_IO0_IO1_MASK	GENMASK(3, 0)
  92#define IO_MACRO_IO0_SEL		0x3
  93#define DEFAULT_IO_MACRO_IO2_IO3_MASK	GENMASK(15, 4)
  94#define IO_MACRO_IO2_IO3_SWAP		0x4640
  95
  96/* We always configure 4 bytes per FIFO word */
  97#define BYTES_PER_FIFO_WORD		4U
  98
  99#define DMA_RX_BUF_SIZE		2048
 100
 101struct qcom_geni_device_data {
 102	bool console;
 103	enum geni_se_xfer_mode mode;
 104};
 105
 106struct qcom_geni_private_data {
 107	/* NOTE: earlycon port will have NULL here */
 108	struct uart_driver *drv;
 109
 110	u32 poll_cached_bytes;
 111	unsigned int poll_cached_bytes_cnt;
 112
 113	u32 write_cached_bytes;
 114	unsigned int write_cached_bytes_cnt;
 115};
 116
 117struct qcom_geni_serial_port {
 118	struct uart_port uport;
 119	struct geni_se se;
 120	const char *name;
 121	u32 tx_fifo_depth;
 122	u32 tx_fifo_width;
 123	u32 rx_fifo_depth;
 124	dma_addr_t tx_dma_addr;
 125	dma_addr_t rx_dma_addr;
 126	bool setup;
 127	unsigned int baud;
 128	unsigned long clk_rate;
 129	void *rx_buf;
 130	u32 loopback;
 131	bool brk;
 132
 133	unsigned int tx_remaining;
 134	int wakeup_irq;
 135	bool rx_tx_swap;
 136	bool cts_rts_swap;
 137
 138	struct qcom_geni_private_data private_data;
 139	const struct qcom_geni_device_data *dev_data;
 140};
 141
 142static const struct uart_ops qcom_geni_console_pops;
 143static const struct uart_ops qcom_geni_uart_pops;
 144static struct uart_driver qcom_geni_console_driver;
 145static struct uart_driver qcom_geni_uart_driver;
 146
 147static inline struct qcom_geni_serial_port *to_dev_port(struct uart_port *uport)
 148{
 149	return container_of(uport, struct qcom_geni_serial_port, uport);
 150}
 151
 152static struct qcom_geni_serial_port qcom_geni_uart_ports[GENI_UART_PORTS] = {
 153	[0] = {
 154		.uport = {
 155			.iotype = UPIO_MEM,
 156			.ops = &qcom_geni_uart_pops,
 157			.flags = UPF_BOOT_AUTOCONF,
 158			.line = 0,
 159		},
 160	},
 161	[1] = {
 162		.uport = {
 163			.iotype = UPIO_MEM,
 164			.ops = &qcom_geni_uart_pops,
 165			.flags = UPF_BOOT_AUTOCONF,
 166			.line = 1,
 167		},
 168	},
 169	[2] = {
 170		.uport = {
 171			.iotype = UPIO_MEM,
 172			.ops = &qcom_geni_uart_pops,
 173			.flags = UPF_BOOT_AUTOCONF,
 174			.line = 2,
 175		},
 176	},
 177};
 178
 179static struct qcom_geni_serial_port qcom_geni_console_port = {
 180	.uport = {
 181		.iotype = UPIO_MEM,
 182		.ops = &qcom_geni_console_pops,
 183		.flags = UPF_BOOT_AUTOCONF,
 184		.line = 0,
 185	},
 186};
 187
 188static int qcom_geni_serial_request_port(struct uart_port *uport)
 189{
 190	struct platform_device *pdev = to_platform_device(uport->dev);
 191	struct qcom_geni_serial_port *port = to_dev_port(uport);
 192
 193	uport->membase = devm_platform_ioremap_resource(pdev, 0);
 194	if (IS_ERR(uport->membase))
 195		return PTR_ERR(uport->membase);
 196	port->se.base = uport->membase;
 197	return 0;
 198}
 199
 200static void qcom_geni_serial_config_port(struct uart_port *uport, int cfg_flags)
 201{
 202	if (cfg_flags & UART_CONFIG_TYPE) {
 203		uport->type = PORT_MSM;
 204		qcom_geni_serial_request_port(uport);
 205	}
 206}
 207
 208static unsigned int qcom_geni_serial_get_mctrl(struct uart_port *uport)
 209{
 210	unsigned int mctrl = TIOCM_DSR | TIOCM_CAR;
 211	u32 geni_ios;
 212
 213	if (uart_console(uport)) {
 214		mctrl |= TIOCM_CTS;
 215	} else {
 216		geni_ios = readl(uport->membase + SE_GENI_IOS);
 217		if (!(geni_ios & IO2_DATA_IN))
 218			mctrl |= TIOCM_CTS;
 219	}
 220
 221	return mctrl;
 222}
 223
 224static void qcom_geni_serial_set_mctrl(struct uart_port *uport,
 225							unsigned int mctrl)
 226{
 227	u32 uart_manual_rfr = 0;
 228	struct qcom_geni_serial_port *port = to_dev_port(uport);
 229
 230	if (uart_console(uport))
 231		return;
 232
 233	if (mctrl & TIOCM_LOOP)
 234		port->loopback = RX_TX_CTS_RTS_SORTED;
 235
 236	if (!(mctrl & TIOCM_RTS) && !uport->suspended)
 237		uart_manual_rfr = UART_MANUAL_RFR_EN | UART_RFR_NOT_READY;
 238	writel(uart_manual_rfr, uport->membase + SE_UART_MANUAL_RFR);
 239}
 240
 241static const char *qcom_geni_serial_get_type(struct uart_port *uport)
 242{
 243	return "MSM";
 244}
 245
 246static struct qcom_geni_serial_port *get_port_from_line(int line, bool console)
 247{
 248	struct qcom_geni_serial_port *port;
 249	int nr_ports = console ? GENI_UART_CONS_PORTS : GENI_UART_PORTS;
 250
 251	if (line < 0 || line >= nr_ports)
 252		return ERR_PTR(-ENXIO);
 253
 254	port = console ? &qcom_geni_console_port : &qcom_geni_uart_ports[line];
 255	return port;
 256}
 257
 258static bool qcom_geni_serial_main_active(struct uart_port *uport)
 259{
 260	return readl(uport->membase + SE_GENI_STATUS) & M_GENI_CMD_ACTIVE;
 261}
 262
 263static bool qcom_geni_serial_secondary_active(struct uart_port *uport)
 264{
 265	return readl(uport->membase + SE_GENI_STATUS) & S_GENI_CMD_ACTIVE;
 266}
 267
 268static bool qcom_geni_serial_poll_bit(struct uart_port *uport,
 269				int offset, int field, bool set)
 270{
 271	u32 reg;
 272	struct qcom_geni_serial_port *port;
 273	unsigned int baud;
 274	unsigned int fifo_bits;
 275	unsigned long timeout_us = 20000;
 276	struct qcom_geni_private_data *private_data = uport->private_data;
 277
 278	if (private_data->drv) {
 279		port = to_dev_port(uport);
 280		baud = port->baud;
 281		if (!baud)
 282			baud = 115200;
 283		fifo_bits = port->tx_fifo_depth * port->tx_fifo_width;
 284		/*
 285		 * Total polling iterations based on FIFO worth of bytes to be
 286		 * sent at current baud. Add a little fluff to the wait.
 287		 */
 288		timeout_us = ((fifo_bits * USEC_PER_SEC) / baud) + 500;
 289	}
 290
 291	/*
 292	 * Use custom implementation instead of readl_poll_atomic since ktimer
 293	 * is not ready at the time of early console.
 294	 */
 295	timeout_us = DIV_ROUND_UP(timeout_us, 10) * 10;
 296	while (timeout_us) {
 297		reg = readl(uport->membase + offset);
 298		if ((bool)(reg & field) == set)
 299			return true;
 300		udelay(10);
 301		timeout_us -= 10;
 302	}
 303	return false;
 304}
 305
 306static void qcom_geni_serial_setup_tx(struct uart_port *uport, u32 xmit_size)
 307{
 308	u32 m_cmd;
 309
 310	writel(xmit_size, uport->membase + SE_UART_TX_TRANS_LEN);
 311	m_cmd = UART_START_TX << M_OPCODE_SHFT;
 312	writel(m_cmd, uport->membase + SE_GENI_M_CMD0);
 313}
 314
 315static void qcom_geni_serial_poll_tx_done(struct uart_port *uport)
 316{
 317	int done;
 318	u32 irq_clear = M_CMD_DONE_EN;
 319
 320	done = qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 321						M_CMD_DONE_EN, true);
 322	if (!done) {
 323		writel(M_GENI_CMD_ABORT, uport->membase +
 324						SE_GENI_M_CMD_CTRL_REG);
 325		irq_clear |= M_CMD_ABORT_EN;
 326		qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 327							M_CMD_ABORT_EN, true);
 328	}
 329	writel(irq_clear, uport->membase + SE_GENI_M_IRQ_CLEAR);
 330}
 331
 332static void qcom_geni_serial_abort_rx(struct uart_port *uport)
 333{
 334	u32 irq_clear = S_CMD_DONE_EN | S_CMD_ABORT_EN;
 335
 336	writel(S_GENI_CMD_ABORT, uport->membase + SE_GENI_S_CMD_CTRL_REG);
 337	qcom_geni_serial_poll_bit(uport, SE_GENI_S_CMD_CTRL_REG,
 338					S_GENI_CMD_ABORT, false);
 339	writel(irq_clear, uport->membase + SE_GENI_S_IRQ_CLEAR);
 340	writel(FORCE_DEFAULT, uport->membase + GENI_FORCE_DEFAULT_REG);
 341}
 342
 343#ifdef CONFIG_CONSOLE_POLL
 344static int qcom_geni_serial_get_char(struct uart_port *uport)
 345{
 346	struct qcom_geni_private_data *private_data = uport->private_data;
 347	u32 status;
 348	u32 word_cnt;
 349	int ret;
 350
 351	if (!private_data->poll_cached_bytes_cnt) {
 352		status = readl(uport->membase + SE_GENI_M_IRQ_STATUS);
 353		writel(status, uport->membase + SE_GENI_M_IRQ_CLEAR);
 354
 355		status = readl(uport->membase + SE_GENI_S_IRQ_STATUS);
 356		writel(status, uport->membase + SE_GENI_S_IRQ_CLEAR);
 357
 358		status = readl(uport->membase + SE_GENI_RX_FIFO_STATUS);
 359		word_cnt = status & RX_FIFO_WC_MSK;
 360		if (!word_cnt)
 361			return NO_POLL_CHAR;
 362
 363		if (word_cnt == 1 && (status & RX_LAST))
 364			/*
 365			 * NOTE: If RX_LAST_BYTE_VALID is 0 it needs to be
 366			 * treated as if it was BYTES_PER_FIFO_WORD.
 367			 */
 368			private_data->poll_cached_bytes_cnt =
 369				(status & RX_LAST_BYTE_VALID_MSK) >>
 370				RX_LAST_BYTE_VALID_SHFT;
 371
 372		if (private_data->poll_cached_bytes_cnt == 0)
 373			private_data->poll_cached_bytes_cnt = BYTES_PER_FIFO_WORD;
 374
 375		private_data->poll_cached_bytes =
 376			readl(uport->membase + SE_GENI_RX_FIFOn);
 377	}
 378
 379	private_data->poll_cached_bytes_cnt--;
 380	ret = private_data->poll_cached_bytes & 0xff;
 381	private_data->poll_cached_bytes >>= 8;
 382
 383	return ret;
 384}
 385
 386static void qcom_geni_serial_poll_put_char(struct uart_port *uport,
 387							unsigned char c)
 388{
 389	writel(DEF_TX_WM, uport->membase + SE_GENI_TX_WATERMARK_REG);
 390	qcom_geni_serial_setup_tx(uport, 1);
 391	WARN_ON(!qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 392						M_TX_FIFO_WATERMARK_EN, true));
 393	writel(c, uport->membase + SE_GENI_TX_FIFOn);
 394	writel(M_TX_FIFO_WATERMARK_EN, uport->membase + SE_GENI_M_IRQ_CLEAR);
 395	qcom_geni_serial_poll_tx_done(uport);
 396}
 397#endif
 398
 399#ifdef CONFIG_SERIAL_QCOM_GENI_CONSOLE
 400static void qcom_geni_serial_wr_char(struct uart_port *uport, unsigned char ch)
 401{
 402	struct qcom_geni_private_data *private_data = uport->private_data;
 403
 404	private_data->write_cached_bytes =
 405		(private_data->write_cached_bytes >> 8) | (ch << 24);
 406	private_data->write_cached_bytes_cnt++;
 407
 408	if (private_data->write_cached_bytes_cnt == BYTES_PER_FIFO_WORD) {
 409		writel(private_data->write_cached_bytes,
 410		       uport->membase + SE_GENI_TX_FIFOn);
 411		private_data->write_cached_bytes_cnt = 0;
 412	}
 413}
 414
 415static void
 416__qcom_geni_serial_console_write(struct uart_port *uport, const char *s,
 417				 unsigned int count)
 418{
 419	struct qcom_geni_private_data *private_data = uport->private_data;
 420
 421	int i;
 422	u32 bytes_to_send = count;
 423
 424	for (i = 0; i < count; i++) {
 425		/*
 426		 * uart_console_write() adds a carriage return for each newline.
 427		 * Account for additional bytes to be written.
 428		 */
 429		if (s[i] == '\n')
 430			bytes_to_send++;
 431	}
 432
 433	writel(DEF_TX_WM, uport->membase + SE_GENI_TX_WATERMARK_REG);
 434	qcom_geni_serial_setup_tx(uport, bytes_to_send);
 435	for (i = 0; i < count; ) {
 436		size_t chars_to_write = 0;
 437		size_t avail = DEF_FIFO_DEPTH_WORDS - DEF_TX_WM;
 438
 439		/*
 440		 * If the WM bit never set, then the Tx state machine is not
 441		 * in a valid state, so break, cancel/abort any existing
 442		 * command. Unfortunately the current data being written is
 443		 * lost.
 444		 */
 445		if (!qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 446						M_TX_FIFO_WATERMARK_EN, true))
 447			break;
 448		chars_to_write = min_t(size_t, count - i, avail / 2);
 449		uart_console_write(uport, s + i, chars_to_write,
 450						qcom_geni_serial_wr_char);
 451		writel(M_TX_FIFO_WATERMARK_EN, uport->membase +
 452							SE_GENI_M_IRQ_CLEAR);
 453		i += chars_to_write;
 454	}
 455
 456	if (private_data->write_cached_bytes_cnt) {
 457		private_data->write_cached_bytes >>= BITS_PER_BYTE *
 458			(BYTES_PER_FIFO_WORD - private_data->write_cached_bytes_cnt);
 459		writel(private_data->write_cached_bytes,
 460		       uport->membase + SE_GENI_TX_FIFOn);
 461		private_data->write_cached_bytes_cnt = 0;
 462	}
 463
 464	qcom_geni_serial_poll_tx_done(uport);
 465}
 466
 467static void qcom_geni_serial_console_write(struct console *co, const char *s,
 468			      unsigned int count)
 469{
 470	struct uart_port *uport;
 471	struct qcom_geni_serial_port *port;
 472	bool locked = true;
 473	unsigned long flags;
 474	u32 geni_status;
 475	u32 irq_en;
 476
 477	WARN_ON(co->index < 0 || co->index >= GENI_UART_CONS_PORTS);
 478
 479	port = get_port_from_line(co->index, true);
 480	if (IS_ERR(port))
 481		return;
 482
 483	uport = &port->uport;
 484	if (oops_in_progress)
 485		locked = uart_port_trylock_irqsave(uport, &flags);
 486	else
 487		uart_port_lock_irqsave(uport, &flags);
 488
 489	geni_status = readl(uport->membase + SE_GENI_STATUS);
 490
 491	/* Cancel the current write to log the fault */
 492	if (!locked) {
 493		geni_se_cancel_m_cmd(&port->se);
 494		if (!qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 495						M_CMD_CANCEL_EN, true)) {
 496			geni_se_abort_m_cmd(&port->se);
 497			qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 498							M_CMD_ABORT_EN, true);
 499			writel(M_CMD_ABORT_EN, uport->membase +
 500							SE_GENI_M_IRQ_CLEAR);
 501		}
 502		writel(M_CMD_CANCEL_EN, uport->membase + SE_GENI_M_IRQ_CLEAR);
 503	} else if ((geni_status & M_GENI_CMD_ACTIVE) && !port->tx_remaining) {
 504		/*
 505		 * It seems we can't interrupt existing transfers if all data
 506		 * has been sent, in which case we need to look for done first.
 507		 */
 508		qcom_geni_serial_poll_tx_done(uport);
 509
 510		if (!uart_circ_empty(&uport->state->xmit)) {
 511			irq_en = readl(uport->membase + SE_GENI_M_IRQ_EN);
 512			writel(irq_en | M_TX_FIFO_WATERMARK_EN,
 513					uport->membase + SE_GENI_M_IRQ_EN);
 514		}
 515	}
 516
 517	__qcom_geni_serial_console_write(uport, s, count);
 518
 519	if (port->tx_remaining)
 520		qcom_geni_serial_setup_tx(uport, port->tx_remaining);
 521
 522	if (locked)
 
 
 523		uart_port_unlock_irqrestore(uport, flags);
 
 524}
 525
 526static void handle_rx_console(struct uart_port *uport, u32 bytes, bool drop)
 527{
 528	u32 i;
 529	unsigned char buf[sizeof(u32)];
 530	struct tty_port *tport;
 531	struct qcom_geni_serial_port *port = to_dev_port(uport);
 532
 533	tport = &uport->state->port;
 534	for (i = 0; i < bytes; ) {
 535		int c;
 536		int chunk = min_t(int, bytes - i, BYTES_PER_FIFO_WORD);
 537
 538		ioread32_rep(uport->membase + SE_GENI_RX_FIFOn, buf, 1);
 539		i += chunk;
 540		if (drop)
 541			continue;
 542
 543		for (c = 0; c < chunk; c++) {
 544			int sysrq;
 545
 546			uport->icount.rx++;
 547			if (port->brk && buf[c] == 0) {
 548				port->brk = false;
 549				if (uart_handle_break(uport))
 550					continue;
 551			}
 552
 553			sysrq = uart_prepare_sysrq_char(uport, buf[c]);
 554
 555			if (!sysrq)
 556				tty_insert_flip_char(tport, buf[c], TTY_NORMAL);
 557		}
 558	}
 559	if (!drop)
 560		tty_flip_buffer_push(tport);
 561}
 562#else
 563static void handle_rx_console(struct uart_port *uport, u32 bytes, bool drop)
 564{
 565
 566}
 567#endif /* CONFIG_SERIAL_QCOM_GENI_CONSOLE */
 568
 569static void handle_rx_uart(struct uart_port *uport, u32 bytes, bool drop)
 570{
 571	struct qcom_geni_serial_port *port = to_dev_port(uport);
 572	struct tty_port *tport = &uport->state->port;
 573	int ret;
 574
 575	ret = tty_insert_flip_string(tport, port->rx_buf, bytes);
 576	if (ret != bytes) {
 577		dev_err(uport->dev, "%s:Unable to push data ret %d_bytes %d\n",
 578				__func__, ret, bytes);
 579		WARN_ON_ONCE(1);
 580	}
 581	uport->icount.rx += ret;
 582	tty_flip_buffer_push(tport);
 583}
 584
 585static unsigned int qcom_geni_serial_tx_empty(struct uart_port *uport)
 586{
 587	return !readl(uport->membase + SE_GENI_TX_FIFO_STATUS);
 588}
 589
 590static void qcom_geni_serial_stop_tx_dma(struct uart_port *uport)
 591{
 592	struct qcom_geni_serial_port *port = to_dev_port(uport);
 593	bool done;
 594
 595	if (!qcom_geni_serial_main_active(uport))
 596		return;
 597
 598	if (port->tx_dma_addr) {
 599		geni_se_tx_dma_unprep(&port->se, port->tx_dma_addr,
 600				      port->tx_remaining);
 601		port->tx_dma_addr = 0;
 602		port->tx_remaining = 0;
 603	}
 604
 605	geni_se_cancel_m_cmd(&port->se);
 606
 607	done = qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 608					 M_CMD_CANCEL_EN, true);
 609	if (!done) {
 610		geni_se_abort_m_cmd(&port->se);
 611		done = qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 612						 M_CMD_ABORT_EN, true);
 613		if (!done)
 614			dev_err_ratelimited(uport->dev, "M_CMD_ABORT_EN not set");
 615		writel(M_CMD_ABORT_EN, uport->membase + SE_GENI_M_IRQ_CLEAR);
 616	}
 617
 618	writel(M_CMD_CANCEL_EN, uport->membase + SE_GENI_M_IRQ_CLEAR);
 619}
 620
 621static void qcom_geni_serial_start_tx_dma(struct uart_port *uport)
 622{
 623	struct qcom_geni_serial_port *port = to_dev_port(uport);
 624	struct circ_buf *xmit = &uport->state->xmit;
 625	unsigned int xmit_size;
 626	int ret;
 627
 628	if (port->tx_dma_addr)
 629		return;
 630
 631	if (uart_circ_empty(xmit))
 632		return;
 633
 634	xmit_size = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
 635
 636	qcom_geni_serial_setup_tx(uport, xmit_size);
 637
 638	ret = geni_se_tx_dma_prep(&port->se, &xmit->buf[xmit->tail],
 639				  xmit_size, &port->tx_dma_addr);
 640	if (ret) {
 641		dev_err(uport->dev, "unable to start TX SE DMA: %d\n", ret);
 642		qcom_geni_serial_stop_tx_dma(uport);
 643		return;
 644	}
 645
 646	port->tx_remaining = xmit_size;
 647}
 648
 649static void qcom_geni_serial_start_tx_fifo(struct uart_port *uport)
 650{
 651	u32 irq_en;
 652
 653	if (qcom_geni_serial_main_active(uport) ||
 654	    !qcom_geni_serial_tx_empty(uport))
 655		return;
 656
 657	irq_en = readl(uport->membase +	SE_GENI_M_IRQ_EN);
 658	irq_en |= M_TX_FIFO_WATERMARK_EN | M_CMD_DONE_EN;
 659
 660	writel(DEF_TX_WM, uport->membase + SE_GENI_TX_WATERMARK_REG);
 661	writel(irq_en, uport->membase +	SE_GENI_M_IRQ_EN);
 662}
 663
 664static void qcom_geni_serial_stop_tx_fifo(struct uart_port *uport)
 665{
 666	u32 irq_en;
 667	struct qcom_geni_serial_port *port = to_dev_port(uport);
 668
 669	irq_en = readl(uport->membase + SE_GENI_M_IRQ_EN);
 670	irq_en &= ~(M_CMD_DONE_EN | M_TX_FIFO_WATERMARK_EN);
 671	writel(0, uport->membase + SE_GENI_TX_WATERMARK_REG);
 672	writel(irq_en, uport->membase + SE_GENI_M_IRQ_EN);
 673	/* Possible stop tx is called multiple times. */
 674	if (!qcom_geni_serial_main_active(uport))
 675		return;
 676
 677	geni_se_cancel_m_cmd(&port->se);
 678	if (!qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 679						M_CMD_CANCEL_EN, true)) {
 680		geni_se_abort_m_cmd(&port->se);
 681		qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 682						M_CMD_ABORT_EN, true);
 683		writel(M_CMD_ABORT_EN, uport->membase + SE_GENI_M_IRQ_CLEAR);
 684	}
 685	writel(M_CMD_CANCEL_EN, uport->membase + SE_GENI_M_IRQ_CLEAR);
 686}
 687
 688static void qcom_geni_serial_handle_rx_fifo(struct uart_port *uport, bool drop)
 689{
 690	u32 status;
 691	u32 word_cnt;
 692	u32 last_word_byte_cnt;
 693	u32 last_word_partial;
 694	u32 total_bytes;
 695
 696	status = readl(uport->membase +	SE_GENI_RX_FIFO_STATUS);
 697	word_cnt = status & RX_FIFO_WC_MSK;
 698	last_word_partial = status & RX_LAST;
 699	last_word_byte_cnt = (status & RX_LAST_BYTE_VALID_MSK) >>
 700						RX_LAST_BYTE_VALID_SHFT;
 701
 702	if (!word_cnt)
 703		return;
 704	total_bytes = BYTES_PER_FIFO_WORD * (word_cnt - 1);
 705	if (last_word_partial && last_word_byte_cnt)
 706		total_bytes += last_word_byte_cnt;
 707	else
 708		total_bytes += BYTES_PER_FIFO_WORD;
 709	handle_rx_console(uport, total_bytes, drop);
 710}
 711
 712static void qcom_geni_serial_stop_rx_fifo(struct uart_port *uport)
 713{
 714	u32 irq_en;
 715	struct qcom_geni_serial_port *port = to_dev_port(uport);
 716	u32 s_irq_status;
 717
 718	irq_en = readl(uport->membase + SE_GENI_S_IRQ_EN);
 719	irq_en &= ~(S_RX_FIFO_WATERMARK_EN | S_RX_FIFO_LAST_EN);
 720	writel(irq_en, uport->membase + SE_GENI_S_IRQ_EN);
 721
 722	irq_en = readl(uport->membase + SE_GENI_M_IRQ_EN);
 723	irq_en &= ~(M_RX_FIFO_WATERMARK_EN | M_RX_FIFO_LAST_EN);
 724	writel(irq_en, uport->membase + SE_GENI_M_IRQ_EN);
 725
 726	if (!qcom_geni_serial_secondary_active(uport))
 727		return;
 728
 729	geni_se_cancel_s_cmd(&port->se);
 730	qcom_geni_serial_poll_bit(uport, SE_GENI_S_IRQ_STATUS,
 731					S_CMD_CANCEL_EN, true);
 732	/*
 733	 * If timeout occurs secondary engine remains active
 734	 * and Abort sequence is executed.
 735	 */
 736	s_irq_status = readl(uport->membase + SE_GENI_S_IRQ_STATUS);
 737	/* Flush the Rx buffer */
 738	if (s_irq_status & S_RX_FIFO_LAST_EN)
 739		qcom_geni_serial_handle_rx_fifo(uport, true);
 740	writel(s_irq_status, uport->membase + SE_GENI_S_IRQ_CLEAR);
 741
 742	if (qcom_geni_serial_secondary_active(uport))
 743		qcom_geni_serial_abort_rx(uport);
 744}
 745
 746static void qcom_geni_serial_start_rx_fifo(struct uart_port *uport)
 747{
 748	u32 irq_en;
 749	struct qcom_geni_serial_port *port = to_dev_port(uport);
 750
 751	if (qcom_geni_serial_secondary_active(uport))
 752		qcom_geni_serial_stop_rx_fifo(uport);
 753
 754	geni_se_setup_s_cmd(&port->se, UART_START_READ, 0);
 755
 756	irq_en = readl(uport->membase + SE_GENI_S_IRQ_EN);
 757	irq_en |= S_RX_FIFO_WATERMARK_EN | S_RX_FIFO_LAST_EN;
 758	writel(irq_en, uport->membase + SE_GENI_S_IRQ_EN);
 759
 760	irq_en = readl(uport->membase + SE_GENI_M_IRQ_EN);
 761	irq_en |= M_RX_FIFO_WATERMARK_EN | M_RX_FIFO_LAST_EN;
 762	writel(irq_en, uport->membase + SE_GENI_M_IRQ_EN);
 763}
 764
 765static void qcom_geni_serial_stop_rx_dma(struct uart_port *uport)
 766{
 767	struct qcom_geni_serial_port *port = to_dev_port(uport);
 768
 769	if (!qcom_geni_serial_secondary_active(uport))
 770		return;
 771
 772	geni_se_cancel_s_cmd(&port->se);
 773	qcom_geni_serial_poll_bit(uport, SE_GENI_S_IRQ_STATUS,
 774				  S_CMD_CANCEL_EN, true);
 775
 776	if (qcom_geni_serial_secondary_active(uport))
 777		qcom_geni_serial_abort_rx(uport);
 778
 779	if (port->rx_dma_addr) {
 780		geni_se_rx_dma_unprep(&port->se, port->rx_dma_addr,
 781				      DMA_RX_BUF_SIZE);
 782		port->rx_dma_addr = 0;
 783	}
 784}
 785
 786static void qcom_geni_serial_start_rx_dma(struct uart_port *uport)
 787{
 788	struct qcom_geni_serial_port *port = to_dev_port(uport);
 789	int ret;
 790
 791	if (qcom_geni_serial_secondary_active(uport))
 792		qcom_geni_serial_stop_rx_dma(uport);
 793
 794	geni_se_setup_s_cmd(&port->se, UART_START_READ, UART_PARAM_RFR_OPEN);
 795
 796	ret = geni_se_rx_dma_prep(&port->se, port->rx_buf,
 797				  DMA_RX_BUF_SIZE,
 798				  &port->rx_dma_addr);
 799	if (ret) {
 800		dev_err(uport->dev, "unable to start RX SE DMA: %d\n", ret);
 801		qcom_geni_serial_stop_rx_dma(uport);
 802	}
 803}
 804
 805static void qcom_geni_serial_handle_rx_dma(struct uart_port *uport, bool drop)
 806{
 807	struct qcom_geni_serial_port *port = to_dev_port(uport);
 808	u32 rx_in;
 809	int ret;
 810
 811	if (!qcom_geni_serial_secondary_active(uport))
 812		return;
 813
 814	if (!port->rx_dma_addr)
 815		return;
 816
 817	geni_se_rx_dma_unprep(&port->se, port->rx_dma_addr, DMA_RX_BUF_SIZE);
 818	port->rx_dma_addr = 0;
 819
 820	rx_in = readl(uport->membase + SE_DMA_RX_LEN_IN);
 821	if (!rx_in) {
 822		dev_warn(uport->dev, "serial engine reports 0 RX bytes in!\n");
 823		return;
 824	}
 825
 826	if (!drop)
 827		handle_rx_uart(uport, rx_in, drop);
 828
 829	ret = geni_se_rx_dma_prep(&port->se, port->rx_buf,
 830				  DMA_RX_BUF_SIZE,
 831				  &port->rx_dma_addr);
 832	if (ret) {
 833		dev_err(uport->dev, "unable to start RX SE DMA: %d\n", ret);
 834		qcom_geni_serial_stop_rx_dma(uport);
 835	}
 836}
 837
 838static void qcom_geni_serial_start_rx(struct uart_port *uport)
 839{
 840	uport->ops->start_rx(uport);
 841}
 842
 843static void qcom_geni_serial_stop_rx(struct uart_port *uport)
 844{
 845	uport->ops->stop_rx(uport);
 846}
 847
 848static void qcom_geni_serial_stop_tx(struct uart_port *uport)
 849{
 850	uport->ops->stop_tx(uport);
 851}
 852
 853static void qcom_geni_serial_send_chunk_fifo(struct uart_port *uport,
 854					     unsigned int chunk)
 855{
 856	struct qcom_geni_serial_port *port = to_dev_port(uport);
 857	struct circ_buf *xmit = &uport->state->xmit;
 858	unsigned int tx_bytes, c, remaining = chunk;
 859	u8 buf[BYTES_PER_FIFO_WORD];
 860
 861	while (remaining) {
 862		memset(buf, 0, sizeof(buf));
 863		tx_bytes = min(remaining, BYTES_PER_FIFO_WORD);
 864
 865		for (c = 0; c < tx_bytes ; c++) {
 866			buf[c] = xmit->buf[xmit->tail];
 867			uart_xmit_advance(uport, 1);
 868		}
 869
 870		iowrite32_rep(uport->membase + SE_GENI_TX_FIFOn, buf, 1);
 871
 872		remaining -= tx_bytes;
 873		port->tx_remaining -= tx_bytes;
 874	}
 875}
 876
 877static void qcom_geni_serial_handle_tx_fifo(struct uart_port *uport,
 878					    bool done, bool active)
 879{
 880	struct qcom_geni_serial_port *port = to_dev_port(uport);
 881	struct circ_buf *xmit = &uport->state->xmit;
 882	size_t avail;
 883	size_t pending;
 884	u32 status;
 885	u32 irq_en;
 886	unsigned int chunk;
 887
 888	status = readl(uport->membase + SE_GENI_TX_FIFO_STATUS);
 889
 890	/* Complete the current tx command before taking newly added data */
 891	if (active)
 892		pending = port->tx_remaining;
 893	else
 894		pending = uart_circ_chars_pending(xmit);
 895
 896	/* All data has been transmitted and acknowledged as received */
 897	if (!pending && !status && done) {
 898		qcom_geni_serial_stop_tx_fifo(uport);
 899		goto out_write_wakeup;
 900	}
 901
 902	avail = port->tx_fifo_depth - (status & TX_FIFO_WC);
 903	avail *= BYTES_PER_FIFO_WORD;
 904
 905	chunk = min(avail, pending);
 906	if (!chunk)
 907		goto out_write_wakeup;
 908
 909	if (!port->tx_remaining) {
 910		qcom_geni_serial_setup_tx(uport, pending);
 911		port->tx_remaining = pending;
 912
 913		irq_en = readl(uport->membase + SE_GENI_M_IRQ_EN);
 914		if (!(irq_en & M_TX_FIFO_WATERMARK_EN))
 915			writel(irq_en | M_TX_FIFO_WATERMARK_EN,
 916					uport->membase + SE_GENI_M_IRQ_EN);
 917	}
 918
 919	qcom_geni_serial_send_chunk_fifo(uport, chunk);
 920
 921	/*
 922	 * The tx fifo watermark is level triggered and latched. Though we had
 923	 * cleared it in qcom_geni_serial_isr it will have already reasserted
 924	 * so we must clear it again here after our writes.
 925	 */
 926	writel(M_TX_FIFO_WATERMARK_EN,
 927			uport->membase + SE_GENI_M_IRQ_CLEAR);
 928
 929out_write_wakeup:
 930	if (!port->tx_remaining) {
 931		irq_en = readl(uport->membase + SE_GENI_M_IRQ_EN);
 932		if (irq_en & M_TX_FIFO_WATERMARK_EN)
 933			writel(irq_en & ~M_TX_FIFO_WATERMARK_EN,
 934					uport->membase + SE_GENI_M_IRQ_EN);
 935	}
 936
 937	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 938		uart_write_wakeup(uport);
 939}
 940
 941static void qcom_geni_serial_handle_tx_dma(struct uart_port *uport)
 942{
 943	struct qcom_geni_serial_port *port = to_dev_port(uport);
 944	struct circ_buf *xmit = &uport->state->xmit;
 945
 946	uart_xmit_advance(uport, port->tx_remaining);
 947	geni_se_tx_dma_unprep(&port->se, port->tx_dma_addr, port->tx_remaining);
 948	port->tx_dma_addr = 0;
 949	port->tx_remaining = 0;
 950
 951	if (!uart_circ_empty(xmit))
 952		qcom_geni_serial_start_tx_dma(uport);
 953
 954	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 955		uart_write_wakeup(uport);
 956}
 957
 958static irqreturn_t qcom_geni_serial_isr(int isr, void *dev)
 959{
 960	u32 m_irq_en;
 961	u32 m_irq_status;
 962	u32 s_irq_status;
 963	u32 geni_status;
 964	u32 dma;
 965	u32 dma_tx_status;
 966	u32 dma_rx_status;
 967	struct uart_port *uport = dev;
 968	bool drop_rx = false;
 969	struct tty_port *tport = &uport->state->port;
 970	struct qcom_geni_serial_port *port = to_dev_port(uport);
 971
 972	if (uport->suspended)
 973		return IRQ_NONE;
 974
 975	uart_port_lock(uport);
 976
 977	m_irq_status = readl(uport->membase + SE_GENI_M_IRQ_STATUS);
 978	s_irq_status = readl(uport->membase + SE_GENI_S_IRQ_STATUS);
 979	dma_tx_status = readl(uport->membase + SE_DMA_TX_IRQ_STAT);
 980	dma_rx_status = readl(uport->membase + SE_DMA_RX_IRQ_STAT);
 981	geni_status = readl(uport->membase + SE_GENI_STATUS);
 982	dma = readl(uport->membase + SE_GENI_DMA_MODE_EN);
 983	m_irq_en = readl(uport->membase + SE_GENI_M_IRQ_EN);
 984	writel(m_irq_status, uport->membase + SE_GENI_M_IRQ_CLEAR);
 985	writel(s_irq_status, uport->membase + SE_GENI_S_IRQ_CLEAR);
 986	writel(dma_tx_status, uport->membase + SE_DMA_TX_IRQ_CLR);
 987	writel(dma_rx_status, uport->membase + SE_DMA_RX_IRQ_CLR);
 988
 989	if (WARN_ON(m_irq_status & M_ILLEGAL_CMD_EN))
 990		goto out_unlock;
 991
 992	if (s_irq_status & S_RX_FIFO_WR_ERR_EN) {
 993		uport->icount.overrun++;
 994		tty_insert_flip_char(tport, 0, TTY_OVERRUN);
 995	}
 996
 997	if (s_irq_status & (S_GP_IRQ_0_EN | S_GP_IRQ_1_EN)) {
 998		if (s_irq_status & S_GP_IRQ_0_EN)
 999			uport->icount.parity++;
1000		drop_rx = true;
1001	} else if (s_irq_status & (S_GP_IRQ_2_EN | S_GP_IRQ_3_EN)) {
1002		uport->icount.brk++;
1003		port->brk = true;
1004	}
1005
1006	if (dma) {
1007		if (dma_tx_status & TX_DMA_DONE)
1008			qcom_geni_serial_handle_tx_dma(uport);
1009
1010		if (dma_rx_status) {
1011			if (dma_rx_status & RX_RESET_DONE)
1012				goto out_unlock;
1013
1014			if (dma_rx_status & RX_DMA_PARITY_ERR) {
1015				uport->icount.parity++;
1016				drop_rx = true;
1017			}
1018
1019			if (dma_rx_status & RX_DMA_BREAK)
1020				uport->icount.brk++;
1021
1022			if (dma_rx_status & (RX_DMA_DONE | RX_EOT))
1023				qcom_geni_serial_handle_rx_dma(uport, drop_rx);
1024		}
1025	} else {
1026		if (m_irq_status & m_irq_en &
1027		    (M_TX_FIFO_WATERMARK_EN | M_CMD_DONE_EN))
1028			qcom_geni_serial_handle_tx_fifo(uport,
1029					m_irq_status & M_CMD_DONE_EN,
1030					geni_status & M_GENI_CMD_ACTIVE);
1031
1032		if (s_irq_status & (S_RX_FIFO_WATERMARK_EN | S_RX_FIFO_LAST_EN))
1033			qcom_geni_serial_handle_rx_fifo(uport, drop_rx);
1034	}
1035
1036out_unlock:
1037	uart_unlock_and_check_sysrq(uport);
1038
1039	return IRQ_HANDLED;
1040}
1041
1042static int setup_fifos(struct qcom_geni_serial_port *port)
1043{
1044	struct uart_port *uport;
1045	u32 old_rx_fifo_depth = port->rx_fifo_depth;
1046
1047	uport = &port->uport;
1048	port->tx_fifo_depth = geni_se_get_tx_fifo_depth(&port->se);
1049	port->tx_fifo_width = geni_se_get_tx_fifo_width(&port->se);
1050	port->rx_fifo_depth = geni_se_get_rx_fifo_depth(&port->se);
1051	uport->fifosize =
1052		(port->tx_fifo_depth * port->tx_fifo_width) / BITS_PER_BYTE;
1053
1054	if (port->rx_buf && (old_rx_fifo_depth != port->rx_fifo_depth) && port->rx_fifo_depth) {
1055		/*
1056		 * Use krealloc rather than krealloc_array because rx_buf is
1057		 * accessed as 1 byte entries as well as 4 byte entries so it's
1058		 * not necessarily an array.
1059		 */
1060		port->rx_buf = devm_krealloc(uport->dev, port->rx_buf,
1061					     port->rx_fifo_depth * sizeof(u32),
1062					     GFP_KERNEL);
1063		if (!port->rx_buf)
1064			return -ENOMEM;
1065	}
1066
1067	return 0;
1068}
1069
1070
1071static void qcom_geni_serial_shutdown(struct uart_port *uport)
1072{
1073	disable_irq(uport->irq);
1074
1075	if (uart_console(uport))
1076		return;
1077
1078	qcom_geni_serial_stop_tx(uport);
1079	qcom_geni_serial_stop_rx(uport);
1080}
1081
1082static int qcom_geni_serial_port_setup(struct uart_port *uport)
1083{
1084	struct qcom_geni_serial_port *port = to_dev_port(uport);
1085	u32 rxstale = DEFAULT_BITS_PER_CHAR * STALE_TIMEOUT;
1086	u32 proto;
1087	u32 pin_swap;
1088	int ret;
1089
1090	proto = geni_se_read_proto(&port->se);
1091	if (proto != GENI_SE_UART) {
1092		dev_err(uport->dev, "Invalid FW loaded, proto: %d\n", proto);
1093		return -ENXIO;
1094	}
1095
1096	qcom_geni_serial_stop_rx(uport);
1097
1098	ret = setup_fifos(port);
1099	if (ret)
1100		return ret;
1101
1102	writel(rxstale, uport->membase + SE_UART_RX_STALE_CNT);
1103
1104	pin_swap = readl(uport->membase + SE_UART_IO_MACRO_CTRL);
1105	if (port->rx_tx_swap) {
1106		pin_swap &= ~DEFAULT_IO_MACRO_IO2_IO3_MASK;
1107		pin_swap |= IO_MACRO_IO2_IO3_SWAP;
1108	}
1109	if (port->cts_rts_swap) {
1110		pin_swap &= ~DEFAULT_IO_MACRO_IO0_IO1_MASK;
1111		pin_swap |= IO_MACRO_IO0_SEL;
1112	}
1113	/* Configure this register if RX-TX, CTS-RTS pins are swapped */
1114	if (port->rx_tx_swap || port->cts_rts_swap)
1115		writel(pin_swap, uport->membase + SE_UART_IO_MACRO_CTRL);
1116
1117	/*
1118	 * Make an unconditional cancel on the main sequencer to reset
1119	 * it else we could end up in data loss scenarios.
1120	 */
1121	if (uart_console(uport))
1122		qcom_geni_serial_poll_tx_done(uport);
1123	geni_se_config_packing(&port->se, BITS_PER_BYTE, BYTES_PER_FIFO_WORD,
1124			       false, true, true);
1125	geni_se_init(&port->se, UART_RX_WM, port->rx_fifo_depth - 2);
1126	geni_se_select_mode(&port->se, port->dev_data->mode);
1127	qcom_geni_serial_start_rx(uport);
1128	port->setup = true;
1129
1130	return 0;
1131}
1132
1133static int qcom_geni_serial_startup(struct uart_port *uport)
1134{
1135	int ret;
1136	struct qcom_geni_serial_port *port = to_dev_port(uport);
1137
1138	if (!port->setup) {
1139		ret = qcom_geni_serial_port_setup(uport);
1140		if (ret)
1141			return ret;
1142	}
1143	enable_irq(uport->irq);
1144
1145	return 0;
1146}
1147
1148static unsigned long find_clk_rate_in_tol(struct clk *clk, unsigned int desired_clk,
1149			unsigned int *clk_div, unsigned int percent_tol)
1150{
1151	unsigned long freq;
1152	unsigned long div, maxdiv;
1153	u64 mult;
1154	unsigned long offset, abs_tol, achieved;
1155
1156	abs_tol = div_u64((u64)desired_clk * percent_tol, 100);
1157	maxdiv = CLK_DIV_MSK >> CLK_DIV_SHFT;
1158	div = 1;
1159	while (div <= maxdiv) {
1160		mult = (u64)div * desired_clk;
1161		if (mult != (unsigned long)mult)
1162			break;
1163
1164		offset = div * abs_tol;
1165		freq = clk_round_rate(clk, mult - offset);
1166
1167		/* Can only get lower if we're done */
1168		if (freq < mult - offset)
1169			break;
1170
1171		/*
1172		 * Re-calculate div in case rounding skipped rates but we
1173		 * ended up at a good one, then check for a match.
1174		 */
1175		div = DIV_ROUND_CLOSEST(freq, desired_clk);
1176		achieved = DIV_ROUND_CLOSEST(freq, div);
1177		if (achieved <= desired_clk + abs_tol &&
1178		    achieved >= desired_clk - abs_tol) {
1179			*clk_div = div;
1180			return freq;
1181		}
1182
1183		div = DIV_ROUND_UP(freq, desired_clk);
1184	}
1185
1186	return 0;
1187}
1188
1189static unsigned long get_clk_div_rate(struct clk *clk, unsigned int baud,
1190			unsigned int sampling_rate, unsigned int *clk_div)
1191{
1192	unsigned long ser_clk;
1193	unsigned long desired_clk;
1194
1195	desired_clk = baud * sampling_rate;
1196	if (!desired_clk)
1197		return 0;
1198
1199	/*
1200	 * try to find a clock rate within 2% tolerance, then within 5%
1201	 */
1202	ser_clk = find_clk_rate_in_tol(clk, desired_clk, clk_div, 2);
1203	if (!ser_clk)
1204		ser_clk = find_clk_rate_in_tol(clk, desired_clk, clk_div, 5);
1205
1206	return ser_clk;
1207}
1208
1209static void qcom_geni_serial_set_termios(struct uart_port *uport,
1210					 struct ktermios *termios,
1211					 const struct ktermios *old)
1212{
1213	unsigned int baud;
1214	u32 bits_per_char;
1215	u32 tx_trans_cfg;
1216	u32 tx_parity_cfg;
1217	u32 rx_trans_cfg;
1218	u32 rx_parity_cfg;
1219	u32 stop_bit_len;
1220	unsigned int clk_div;
1221	u32 ser_clk_cfg;
1222	struct qcom_geni_serial_port *port = to_dev_port(uport);
1223	unsigned long clk_rate;
1224	u32 ver, sampling_rate;
1225	unsigned int avg_bw_core;
1226
1227	qcom_geni_serial_stop_rx(uport);
1228	/* baud rate */
1229	baud = uart_get_baud_rate(uport, termios, old, 300, 4000000);
1230	port->baud = baud;
1231
1232	sampling_rate = UART_OVERSAMPLING;
1233	/* Sampling rate is halved for IP versions >= 2.5 */
1234	ver = geni_se_get_qup_hw_version(&port->se);
1235	if (ver >= QUP_SE_VERSION_2_5)
1236		sampling_rate /= 2;
1237
1238	clk_rate = get_clk_div_rate(port->se.clk, baud,
1239		sampling_rate, &clk_div);
1240	if (!clk_rate) {
1241		dev_err(port->se.dev,
1242			"Couldn't find suitable clock rate for %u\n",
1243			baud * sampling_rate);
1244		goto out_restart_rx;
1245	}
1246
1247	dev_dbg(port->se.dev, "desired_rate = %u, clk_rate = %lu, clk_div = %u\n",
1248			baud * sampling_rate, clk_rate, clk_div);
1249
1250	uport->uartclk = clk_rate;
1251	port->clk_rate = clk_rate;
1252	dev_pm_opp_set_rate(uport->dev, clk_rate);
1253	ser_clk_cfg = SER_CLK_EN;
1254	ser_clk_cfg |= clk_div << CLK_DIV_SHFT;
1255
1256	/*
1257	 * Bump up BW vote on CPU and CORE path as driver supports FIFO mode
1258	 * only.
1259	 */
1260	avg_bw_core = (baud > 115200) ? Bps_to_icc(CORE_2X_50_MHZ)
1261						: GENI_DEFAULT_BW;
1262	port->se.icc_paths[GENI_TO_CORE].avg_bw = avg_bw_core;
1263	port->se.icc_paths[CPU_TO_GENI].avg_bw = Bps_to_icc(baud);
1264	geni_icc_set_bw(&port->se);
1265
1266	/* parity */
1267	tx_trans_cfg = readl(uport->membase + SE_UART_TX_TRANS_CFG);
1268	tx_parity_cfg = readl(uport->membase + SE_UART_TX_PARITY_CFG);
1269	rx_trans_cfg = readl(uport->membase + SE_UART_RX_TRANS_CFG);
1270	rx_parity_cfg = readl(uport->membase + SE_UART_RX_PARITY_CFG);
1271	if (termios->c_cflag & PARENB) {
1272		tx_trans_cfg |= UART_TX_PAR_EN;
1273		rx_trans_cfg |= UART_RX_PAR_EN;
1274		tx_parity_cfg |= PAR_CALC_EN;
1275		rx_parity_cfg |= PAR_CALC_EN;
1276		if (termios->c_cflag & PARODD) {
1277			tx_parity_cfg |= PAR_ODD;
1278			rx_parity_cfg |= PAR_ODD;
1279		} else if (termios->c_cflag & CMSPAR) {
1280			tx_parity_cfg |= PAR_SPACE;
1281			rx_parity_cfg |= PAR_SPACE;
1282		} else {
1283			tx_parity_cfg |= PAR_EVEN;
1284			rx_parity_cfg |= PAR_EVEN;
1285		}
1286	} else {
1287		tx_trans_cfg &= ~UART_TX_PAR_EN;
1288		rx_trans_cfg &= ~UART_RX_PAR_EN;
1289		tx_parity_cfg &= ~PAR_CALC_EN;
1290		rx_parity_cfg &= ~PAR_CALC_EN;
1291	}
1292
1293	/* bits per char */
1294	bits_per_char = tty_get_char_size(termios->c_cflag);
1295
1296	/* stop bits */
1297	if (termios->c_cflag & CSTOPB)
1298		stop_bit_len = TX_STOP_BIT_LEN_2;
1299	else
1300		stop_bit_len = TX_STOP_BIT_LEN_1;
1301
1302	/* flow control, clear the CTS_MASK bit if using flow control. */
1303	if (termios->c_cflag & CRTSCTS)
1304		tx_trans_cfg &= ~UART_CTS_MASK;
1305	else
1306		tx_trans_cfg |= UART_CTS_MASK;
1307
1308	if (baud)
1309		uart_update_timeout(uport, termios->c_cflag, baud);
1310
1311	if (!uart_console(uport))
1312		writel(port->loopback,
1313				uport->membase + SE_UART_LOOPBACK_CFG);
1314	writel(tx_trans_cfg, uport->membase + SE_UART_TX_TRANS_CFG);
1315	writel(tx_parity_cfg, uport->membase + SE_UART_TX_PARITY_CFG);
1316	writel(rx_trans_cfg, uport->membase + SE_UART_RX_TRANS_CFG);
1317	writel(rx_parity_cfg, uport->membase + SE_UART_RX_PARITY_CFG);
1318	writel(bits_per_char, uport->membase + SE_UART_TX_WORD_LEN);
1319	writel(bits_per_char, uport->membase + SE_UART_RX_WORD_LEN);
1320	writel(stop_bit_len, uport->membase + SE_UART_TX_STOP_BIT_LEN);
1321	writel(ser_clk_cfg, uport->membase + GENI_SER_M_CLK_CFG);
1322	writel(ser_clk_cfg, uport->membase + GENI_SER_S_CLK_CFG);
1323out_restart_rx:
1324	qcom_geni_serial_start_rx(uport);
1325}
1326
1327#ifdef CONFIG_SERIAL_QCOM_GENI_CONSOLE
1328static int qcom_geni_console_setup(struct console *co, char *options)
1329{
1330	struct uart_port *uport;
1331	struct qcom_geni_serial_port *port;
1332	int baud = 115200;
1333	int bits = 8;
1334	int parity = 'n';
1335	int flow = 'n';
1336	int ret;
1337
1338	if (co->index >= GENI_UART_CONS_PORTS  || co->index < 0)
1339		return -ENXIO;
1340
1341	port = get_port_from_line(co->index, true);
1342	if (IS_ERR(port)) {
1343		pr_err("Invalid line %d\n", co->index);
1344		return PTR_ERR(port);
1345	}
1346
1347	uport = &port->uport;
1348
1349	if (unlikely(!uport->membase))
1350		return -ENXIO;
1351
1352	if (!port->setup) {
1353		ret = qcom_geni_serial_port_setup(uport);
1354		if (ret)
1355			return ret;
1356	}
1357
1358	if (options)
1359		uart_parse_options(options, &baud, &parity, &bits, &flow);
1360
1361	return uart_set_options(uport, co, baud, parity, bits, flow);
1362}
1363
1364static void qcom_geni_serial_earlycon_write(struct console *con,
1365					const char *s, unsigned int n)
1366{
1367	struct earlycon_device *dev = con->data;
1368
1369	__qcom_geni_serial_console_write(&dev->port, s, n);
1370}
1371
1372#ifdef CONFIG_CONSOLE_POLL
1373static int qcom_geni_serial_earlycon_read(struct console *con,
1374					  char *s, unsigned int n)
1375{
1376	struct earlycon_device *dev = con->data;
1377	struct uart_port *uport = &dev->port;
1378	int num_read = 0;
1379	int ch;
1380
1381	while (num_read < n) {
1382		ch = qcom_geni_serial_get_char(uport);
1383		if (ch == NO_POLL_CHAR)
1384			break;
1385		s[num_read++] = ch;
1386	}
1387
1388	return num_read;
1389}
1390
1391static void __init qcom_geni_serial_enable_early_read(struct geni_se *se,
1392						      struct console *con)
1393{
1394	geni_se_setup_s_cmd(se, UART_START_READ, 0);
1395	con->read = qcom_geni_serial_earlycon_read;
1396}
1397#else
1398static inline void qcom_geni_serial_enable_early_read(struct geni_se *se,
1399						      struct console *con) { }
1400#endif
1401
1402static struct qcom_geni_private_data earlycon_private_data;
1403
1404static int __init qcom_geni_serial_earlycon_setup(struct earlycon_device *dev,
1405								const char *opt)
1406{
1407	struct uart_port *uport = &dev->port;
1408	u32 tx_trans_cfg;
1409	u32 tx_parity_cfg = 0;	/* Disable Tx Parity */
1410	u32 rx_trans_cfg = 0;
1411	u32 rx_parity_cfg = 0;	/* Disable Rx Parity */
1412	u32 stop_bit_len = 0;	/* Default stop bit length - 1 bit */
1413	u32 bits_per_char;
1414	struct geni_se se;
1415
1416	if (!uport->membase)
1417		return -EINVAL;
1418
1419	uport->private_data = &earlycon_private_data;
1420
1421	memset(&se, 0, sizeof(se));
1422	se.base = uport->membase;
1423	if (geni_se_read_proto(&se) != GENI_SE_UART)
1424		return -ENXIO;
1425	/*
1426	 * Ignore Flow control.
1427	 * n = 8.
1428	 */
1429	tx_trans_cfg = UART_CTS_MASK;
1430	bits_per_char = BITS_PER_BYTE;
1431
1432	/*
1433	 * Make an unconditional cancel on the main sequencer to reset
1434	 * it else we could end up in data loss scenarios.
1435	 */
1436	qcom_geni_serial_poll_tx_done(uport);
1437	qcom_geni_serial_abort_rx(uport);
1438	geni_se_config_packing(&se, BITS_PER_BYTE, BYTES_PER_FIFO_WORD,
1439			       false, true, true);
1440	geni_se_init(&se, DEF_FIFO_DEPTH_WORDS / 2, DEF_FIFO_DEPTH_WORDS - 2);
1441	geni_se_select_mode(&se, GENI_SE_FIFO);
1442
1443	writel(tx_trans_cfg, uport->membase + SE_UART_TX_TRANS_CFG);
1444	writel(tx_parity_cfg, uport->membase + SE_UART_TX_PARITY_CFG);
1445	writel(rx_trans_cfg, uport->membase + SE_UART_RX_TRANS_CFG);
1446	writel(rx_parity_cfg, uport->membase + SE_UART_RX_PARITY_CFG);
1447	writel(bits_per_char, uport->membase + SE_UART_TX_WORD_LEN);
1448	writel(bits_per_char, uport->membase + SE_UART_RX_WORD_LEN);
1449	writel(stop_bit_len, uport->membase + SE_UART_TX_STOP_BIT_LEN);
1450
1451	dev->con->write = qcom_geni_serial_earlycon_write;
1452	dev->con->setup = NULL;
1453	qcom_geni_serial_enable_early_read(&se, dev->con);
1454
1455	return 0;
1456}
1457OF_EARLYCON_DECLARE(qcom_geni, "qcom,geni-debug-uart",
1458				qcom_geni_serial_earlycon_setup);
1459
1460static int __init console_register(struct uart_driver *drv)
1461{
1462	return uart_register_driver(drv);
1463}
1464
1465static void console_unregister(struct uart_driver *drv)
1466{
1467	uart_unregister_driver(drv);
1468}
1469
1470static struct console cons_ops = {
1471	.name = "ttyMSM",
1472	.write = qcom_geni_serial_console_write,
1473	.device = uart_console_device,
1474	.setup = qcom_geni_console_setup,
1475	.flags = CON_PRINTBUFFER,
1476	.index = -1,
1477	.data = &qcom_geni_console_driver,
1478};
1479
1480static struct uart_driver qcom_geni_console_driver = {
1481	.owner = THIS_MODULE,
1482	.driver_name = "qcom_geni_console",
1483	.dev_name = "ttyMSM",
1484	.nr =  GENI_UART_CONS_PORTS,
1485	.cons = &cons_ops,
1486};
1487#else
1488static int console_register(struct uart_driver *drv)
1489{
1490	return 0;
1491}
1492
1493static void console_unregister(struct uart_driver *drv)
1494{
1495}
1496#endif /* CONFIG_SERIAL_QCOM_GENI_CONSOLE */
1497
1498static struct uart_driver qcom_geni_uart_driver = {
1499	.owner = THIS_MODULE,
1500	.driver_name = "qcom_geni_uart",
1501	.dev_name = "ttyHS",
1502	.nr =  GENI_UART_PORTS,
1503};
1504
1505static void qcom_geni_serial_pm(struct uart_port *uport,
1506		unsigned int new_state, unsigned int old_state)
1507{
1508	struct qcom_geni_serial_port *port = to_dev_port(uport);
1509
1510	/* If we've never been called, treat it as off */
1511	if (old_state == UART_PM_STATE_UNDEFINED)
1512		old_state = UART_PM_STATE_OFF;
1513
1514	if (new_state == UART_PM_STATE_ON && old_state == UART_PM_STATE_OFF) {
1515		geni_icc_enable(&port->se);
1516		if (port->clk_rate)
1517			dev_pm_opp_set_rate(uport->dev, port->clk_rate);
1518		geni_se_resources_on(&port->se);
1519	} else if (new_state == UART_PM_STATE_OFF &&
1520			old_state == UART_PM_STATE_ON) {
1521		geni_se_resources_off(&port->se);
1522		dev_pm_opp_set_rate(uport->dev, 0);
1523		geni_icc_disable(&port->se);
1524	}
1525}
1526
1527static const struct uart_ops qcom_geni_console_pops = {
1528	.tx_empty = qcom_geni_serial_tx_empty,
1529	.stop_tx = qcom_geni_serial_stop_tx_fifo,
1530	.start_tx = qcom_geni_serial_start_tx_fifo,
1531	.stop_rx = qcom_geni_serial_stop_rx_fifo,
1532	.start_rx = qcom_geni_serial_start_rx_fifo,
1533	.set_termios = qcom_geni_serial_set_termios,
1534	.startup = qcom_geni_serial_startup,
1535	.request_port = qcom_geni_serial_request_port,
1536	.config_port = qcom_geni_serial_config_port,
1537	.shutdown = qcom_geni_serial_shutdown,
1538	.type = qcom_geni_serial_get_type,
1539	.set_mctrl = qcom_geni_serial_set_mctrl,
1540	.get_mctrl = qcom_geni_serial_get_mctrl,
1541#ifdef CONFIG_CONSOLE_POLL
1542	.poll_get_char	= qcom_geni_serial_get_char,
1543	.poll_put_char	= qcom_geni_serial_poll_put_char,
1544	.poll_init = qcom_geni_serial_port_setup,
1545#endif
1546	.pm = qcom_geni_serial_pm,
1547};
1548
1549static const struct uart_ops qcom_geni_uart_pops = {
1550	.tx_empty = qcom_geni_serial_tx_empty,
1551	.stop_tx = qcom_geni_serial_stop_tx_dma,
1552	.start_tx = qcom_geni_serial_start_tx_dma,
1553	.start_rx = qcom_geni_serial_start_rx_dma,
1554	.stop_rx = qcom_geni_serial_stop_rx_dma,
1555	.set_termios = qcom_geni_serial_set_termios,
1556	.startup = qcom_geni_serial_startup,
1557	.request_port = qcom_geni_serial_request_port,
1558	.config_port = qcom_geni_serial_config_port,
1559	.shutdown = qcom_geni_serial_shutdown,
1560	.type = qcom_geni_serial_get_type,
1561	.set_mctrl = qcom_geni_serial_set_mctrl,
1562	.get_mctrl = qcom_geni_serial_get_mctrl,
1563	.pm = qcom_geni_serial_pm,
1564};
1565
1566static int qcom_geni_serial_probe(struct platform_device *pdev)
1567{
1568	int ret = 0;
1569	int line;
1570	struct qcom_geni_serial_port *port;
1571	struct uart_port *uport;
1572	struct resource *res;
1573	int irq;
1574	struct uart_driver *drv;
1575	const struct qcom_geni_device_data *data;
1576
1577	data = of_device_get_match_data(&pdev->dev);
1578	if (!data)
1579		return -EINVAL;
1580
1581	if (data->console) {
1582		drv = &qcom_geni_console_driver;
1583		line = of_alias_get_id(pdev->dev.of_node, "serial");
1584	} else {
1585		drv = &qcom_geni_uart_driver;
1586		line = of_alias_get_id(pdev->dev.of_node, "serial");
1587		if (line == -ENODEV) /* compat with non-standard aliases */
1588			line = of_alias_get_id(pdev->dev.of_node, "hsuart");
1589	}
1590
1591	port = get_port_from_line(line, data->console);
1592	if (IS_ERR(port)) {
1593		dev_err(&pdev->dev, "Invalid line %d\n", line);
1594		return PTR_ERR(port);
1595	}
1596
1597	uport = &port->uport;
1598	/* Don't allow 2 drivers to access the same port */
1599	if (uport->private_data)
1600		return -ENODEV;
1601
1602	uport->dev = &pdev->dev;
1603	port->dev_data = data;
1604	port->se.dev = &pdev->dev;
1605	port->se.wrapper = dev_get_drvdata(pdev->dev.parent);
1606	port->se.clk = devm_clk_get(&pdev->dev, "se");
1607	if (IS_ERR(port->se.clk)) {
1608		ret = PTR_ERR(port->se.clk);
1609		dev_err(&pdev->dev, "Err getting SE Core clk %d\n", ret);
1610		return ret;
1611	}
1612
1613	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1614	if (!res)
1615		return -EINVAL;
1616	uport->mapbase = res->start;
1617
1618	port->tx_fifo_depth = DEF_FIFO_DEPTH_WORDS;
1619	port->rx_fifo_depth = DEF_FIFO_DEPTH_WORDS;
1620	port->tx_fifo_width = DEF_FIFO_WIDTH_BITS;
1621
1622	if (!data->console) {
1623		port->rx_buf = devm_kzalloc(uport->dev,
1624					    DMA_RX_BUF_SIZE, GFP_KERNEL);
1625		if (!port->rx_buf)
1626			return -ENOMEM;
1627	}
1628
1629	ret = geni_icc_get(&port->se, NULL);
1630	if (ret)
1631		return ret;
1632	port->se.icc_paths[GENI_TO_CORE].avg_bw = GENI_DEFAULT_BW;
1633	port->se.icc_paths[CPU_TO_GENI].avg_bw = GENI_DEFAULT_BW;
1634
1635	/* Set BW for register access */
1636	ret = geni_icc_set_bw(&port->se);
1637	if (ret)
1638		return ret;
1639
1640	port->name = devm_kasprintf(uport->dev, GFP_KERNEL,
1641			"qcom_geni_serial_%s%d",
1642			uart_console(uport) ? "console" : "uart", uport->line);
1643	if (!port->name)
1644		return -ENOMEM;
1645
1646	irq = platform_get_irq(pdev, 0);
1647	if (irq < 0)
1648		return irq;
1649	uport->irq = irq;
1650	uport->has_sysrq = IS_ENABLED(CONFIG_SERIAL_QCOM_GENI_CONSOLE);
1651
1652	if (!data->console)
1653		port->wakeup_irq = platform_get_irq_optional(pdev, 1);
1654
1655	if (of_property_read_bool(pdev->dev.of_node, "rx-tx-swap"))
1656		port->rx_tx_swap = true;
1657
1658	if (of_property_read_bool(pdev->dev.of_node, "cts-rts-swap"))
1659		port->cts_rts_swap = true;
1660
1661	ret = devm_pm_opp_set_clkname(&pdev->dev, "se");
1662	if (ret)
1663		return ret;
1664	/* OPP table is optional */
1665	ret = devm_pm_opp_of_add_table(&pdev->dev);
1666	if (ret && ret != -ENODEV) {
1667		dev_err(&pdev->dev, "invalid OPP table in device tree\n");
1668		return ret;
1669	}
1670
1671	port->private_data.drv = drv;
1672	uport->private_data = &port->private_data;
1673	platform_set_drvdata(pdev, port);
1674
1675	irq_set_status_flags(uport->irq, IRQ_NOAUTOEN);
1676	ret = devm_request_irq(uport->dev, uport->irq, qcom_geni_serial_isr,
1677			IRQF_TRIGGER_HIGH, port->name, uport);
1678	if (ret) {
1679		dev_err(uport->dev, "Failed to get IRQ ret %d\n", ret);
1680		return ret;
1681	}
1682
1683	ret = uart_add_one_port(drv, uport);
1684	if (ret)
1685		return ret;
1686
1687	if (port->wakeup_irq > 0) {
1688		device_init_wakeup(&pdev->dev, true);
1689		ret = dev_pm_set_dedicated_wake_irq(&pdev->dev,
1690						port->wakeup_irq);
1691		if (ret) {
1692			device_init_wakeup(&pdev->dev, false);
1693			uart_remove_one_port(drv, uport);
1694			return ret;
1695		}
1696	}
1697
1698	return 0;
1699}
1700
1701static void qcom_geni_serial_remove(struct platform_device *pdev)
1702{
1703	struct qcom_geni_serial_port *port = platform_get_drvdata(pdev);
1704	struct uart_driver *drv = port->private_data.drv;
1705
1706	dev_pm_clear_wake_irq(&pdev->dev);
1707	device_init_wakeup(&pdev->dev, false);
1708	uart_remove_one_port(drv, &port->uport);
1709}
1710
1711static int qcom_geni_serial_sys_suspend(struct device *dev)
1712{
1713	struct qcom_geni_serial_port *port = dev_get_drvdata(dev);
1714	struct uart_port *uport = &port->uport;
1715	struct qcom_geni_private_data *private_data = uport->private_data;
1716
1717	/*
1718	 * This is done so we can hit the lowest possible state in suspend
1719	 * even with no_console_suspend
1720	 */
1721	if (uart_console(uport)) {
1722		geni_icc_set_tag(&port->se, QCOM_ICC_TAG_ACTIVE_ONLY);
1723		geni_icc_set_bw(&port->se);
1724	}
1725	return uart_suspend_port(private_data->drv, uport);
1726}
1727
1728static int qcom_geni_serial_sys_resume(struct device *dev)
1729{
1730	int ret;
1731	struct qcom_geni_serial_port *port = dev_get_drvdata(dev);
1732	struct uart_port *uport = &port->uport;
1733	struct qcom_geni_private_data *private_data = uport->private_data;
1734
1735	ret = uart_resume_port(private_data->drv, uport);
1736	if (uart_console(uport)) {
1737		geni_icc_set_tag(&port->se, QCOM_ICC_TAG_ALWAYS);
1738		geni_icc_set_bw(&port->se);
1739	}
1740	return ret;
1741}
1742
1743static int qcom_geni_serial_sys_hib_resume(struct device *dev)
1744{
1745	int ret = 0;
1746	struct uart_port *uport;
1747	struct qcom_geni_private_data *private_data;
1748	struct qcom_geni_serial_port *port = dev_get_drvdata(dev);
1749
1750	uport = &port->uport;
1751	private_data = uport->private_data;
1752
1753	if (uart_console(uport)) {
1754		geni_icc_set_tag(&port->se, QCOM_ICC_TAG_ALWAYS);
1755		geni_icc_set_bw(&port->se);
1756		ret = uart_resume_port(private_data->drv, uport);
1757		/*
1758		 * For hibernation usecase clients for
1759		 * console UART won't call port setup during restore,
1760		 * hence call port setup for console uart.
1761		 */
1762		qcom_geni_serial_port_setup(uport);
1763	} else {
1764		/*
1765		 * Peripheral register settings are lost during hibernation.
1766		 * Update setup flag such that port setup happens again
1767		 * during next session. Clients of HS-UART will close and
1768		 * open the port during hibernation.
1769		 */
1770		port->setup = false;
1771	}
1772	return ret;
1773}
1774
1775static const struct qcom_geni_device_data qcom_geni_console_data = {
1776	.console = true,
1777	.mode = GENI_SE_FIFO,
1778};
1779
1780static const struct qcom_geni_device_data qcom_geni_uart_data = {
1781	.console = false,
1782	.mode = GENI_SE_DMA,
1783};
1784
1785static const struct dev_pm_ops qcom_geni_serial_pm_ops = {
1786	.suspend = pm_sleep_ptr(qcom_geni_serial_sys_suspend),
1787	.resume = pm_sleep_ptr(qcom_geni_serial_sys_resume),
1788	.freeze = pm_sleep_ptr(qcom_geni_serial_sys_suspend),
1789	.poweroff = pm_sleep_ptr(qcom_geni_serial_sys_suspend),
1790	.restore = pm_sleep_ptr(qcom_geni_serial_sys_hib_resume),
1791	.thaw = pm_sleep_ptr(qcom_geni_serial_sys_hib_resume),
1792};
1793
1794static const struct of_device_id qcom_geni_serial_match_table[] = {
1795	{
1796		.compatible = "qcom,geni-debug-uart",
1797		.data = &qcom_geni_console_data,
1798	},
1799	{
1800		.compatible = "qcom,geni-uart",
1801		.data = &qcom_geni_uart_data,
1802	},
1803	{}
1804};
1805MODULE_DEVICE_TABLE(of, qcom_geni_serial_match_table);
1806
1807static struct platform_driver qcom_geni_serial_platform_driver = {
1808	.remove_new = qcom_geni_serial_remove,
1809	.probe = qcom_geni_serial_probe,
1810	.driver = {
1811		.name = "qcom_geni_serial",
1812		.of_match_table = qcom_geni_serial_match_table,
1813		.pm = &qcom_geni_serial_pm_ops,
1814	},
1815};
1816
1817static int __init qcom_geni_serial_init(void)
1818{
1819	int ret;
1820
1821	ret = console_register(&qcom_geni_console_driver);
1822	if (ret)
1823		return ret;
1824
1825	ret = uart_register_driver(&qcom_geni_uart_driver);
1826	if (ret) {
1827		console_unregister(&qcom_geni_console_driver);
1828		return ret;
1829	}
1830
1831	ret = platform_driver_register(&qcom_geni_serial_platform_driver);
1832	if (ret) {
1833		console_unregister(&qcom_geni_console_driver);
1834		uart_unregister_driver(&qcom_geni_uart_driver);
1835	}
1836	return ret;
1837}
1838module_init(qcom_geni_serial_init);
1839
1840static void __exit qcom_geni_serial_exit(void)
1841{
1842	platform_driver_unregister(&qcom_geni_serial_platform_driver);
1843	console_unregister(&qcom_geni_console_driver);
1844	uart_unregister_driver(&qcom_geni_uart_driver);
1845}
1846module_exit(qcom_geni_serial_exit);
1847
1848MODULE_DESCRIPTION("Serial driver for GENI based QUP cores");
1849MODULE_LICENSE("GPL v2");