1// SPDX-License-Identifier: GPL-2.0-or-later
   2/* Realtek PCI-Express SD/MMC Card Interface driver
   3 *
   4 * Copyright(c) 2009-2013 Realtek Semiconductor Corp. All rights reserved.
   5 *
   6 * Author:
   7 *   Wei WANG <wei_wang@realsil.com.cn>
   8 */
   9
  10#include <linux/module.h>
  11#include <linux/slab.h>
  12#include <linux/highmem.h>
  13#include <linux/delay.h>
  14#include <linux/platform_device.h>
  15#include <linux/workqueue.h>
  16#include <linux/mmc/host.h>
  17#include <linux/mmc/mmc.h>
  18#include <linux/mmc/sd.h>
  19#include <linux/mmc/sdio.h>
  20#include <linux/mmc/card.h>
  21#include <linux/rtsx_pci.h>
  22#include <asm/unaligned.h>
  23#include <linux/pm_runtime.h>
  24
  25struct realtek_pci_sdmmc {
  26	struct platform_device	*pdev;
  27	struct rtsx_pcr		*pcr;
  28	struct mmc_host		*mmc;
  29	struct mmc_request	*mrq;
  30#define SDMMC_WORKQ_NAME	"rtsx_pci_sdmmc_workq"
  31
  32	struct work_struct	work;
  33	struct mutex		host_mutex;
  34
  35	u8			ssc_depth;
  36	unsigned int		clock;
  37	bool			vpclk;
  38	bool			double_clk;
  39	bool			eject;
  40	bool			initial_mode;
  41	int			prev_power_state;
 
 
 
  42	int			sg_count;
  43	s32			cookie;
  44	int			cookie_sg_count;
  45	bool			using_cookie;
  46};
  47
  48static int sdmmc_init_sd_express(struct mmc_host *mmc, struct mmc_ios *ios);
  49
  50static inline struct device *sdmmc_dev(struct realtek_pci_sdmmc *host)
  51{
  52	return &(host->pdev->dev);
  53}
  54
  55static inline void sd_clear_error(struct realtek_pci_sdmmc *host)
  56{
  57	rtsx_pci_write_register(host->pcr, CARD_STOP,
  58			SD_STOP | SD_CLR_ERR, SD_STOP | SD_CLR_ERR);
  59}
  60
  61#ifdef DEBUG
  62static void dump_reg_range(struct realtek_pci_sdmmc *host, u16 start, u16 end)
  63{
  64	u16 len = end - start + 1;
  65	int i;
  66	u8 data[8];
  67
  68	for (i = 0; i < len; i += 8) {
  69		int j;
  70		int n = min(8, len - i);
  71
  72		memset(&data, 0, sizeof(data));
  73		for (j = 0; j < n; j++)
  74			rtsx_pci_read_register(host->pcr, start + i + j,
  75				data + j);
  76		dev_dbg(sdmmc_dev(host), "0x%04X(%d): %8ph\n",
  77			start + i, n, data);
  78	}
  79}
  80
  81static void sd_print_debug_regs(struct realtek_pci_sdmmc *host)
  82{
  83	dump_reg_range(host, 0xFDA0, 0xFDB3);
  84	dump_reg_range(host, 0xFD52, 0xFD69);
  85}
  86#else
  87#define sd_print_debug_regs(host)
  88#endif /* DEBUG */
  89
  90static inline int sd_get_cd_int(struct realtek_pci_sdmmc *host)
  91{
  92	return rtsx_pci_readl(host->pcr, RTSX_BIPR) & SD_EXIST;
  93}
  94
  95static void sd_cmd_set_sd_cmd(struct rtsx_pcr *pcr, struct mmc_command *cmd)
  96{
  97	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CMD0, 0xFF,
  98		SD_CMD_START | cmd->opcode);
  99	rtsx_pci_write_be32(pcr, SD_CMD1, cmd->arg);
 100}
 101
 102static void sd_cmd_set_data_len(struct rtsx_pcr *pcr, u16 blocks, u16 blksz)
 103{
 104	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BLOCK_CNT_L, 0xFF, blocks);
 105	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BLOCK_CNT_H, 0xFF, blocks >> 8);
 106	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BYTE_CNT_L, 0xFF, blksz);
 107	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BYTE_CNT_H, 0xFF, blksz >> 8);
 108}
 109
 110static int sd_response_type(struct mmc_command *cmd)
 111{
 112	switch (mmc_resp_type(cmd)) {
 113	case MMC_RSP_NONE:
 114		return SD_RSP_TYPE_R0;
 115	case MMC_RSP_R1:
 116		return SD_RSP_TYPE_R1;
 117	case MMC_RSP_R1_NO_CRC:
 118		return SD_RSP_TYPE_R1 | SD_NO_CHECK_CRC7;
 119	case MMC_RSP_R1B:
 120		return SD_RSP_TYPE_R1b;
 121	case MMC_RSP_R2:
 122		return SD_RSP_TYPE_R2;
 123	case MMC_RSP_R3:
 124		return SD_RSP_TYPE_R3;
 125	default:
 126		return -EINVAL;
 127	}
 128}
 129
 130static int sd_status_index(int resp_type)
 131{
 132	if (resp_type == SD_RSP_TYPE_R0)
 133		return 0;
 134	else if (resp_type == SD_RSP_TYPE_R2)
 135		return 16;
 136
 137	return 5;
 138}
 139/*
 140 * sd_pre_dma_transfer - do dma_map_sg() or using cookie
 141 *
 142 * @pre: if called in pre_req()
 143 * return:
 144 *	0 - do dma_map_sg()
 145 *	1 - using cookie
 146 */
 147static int sd_pre_dma_transfer(struct realtek_pci_sdmmc *host,
 148		struct mmc_data *data, bool pre)
 149{
 150	struct rtsx_pcr *pcr = host->pcr;
 151	int read = data->flags & MMC_DATA_READ;
 152	int count = 0;
 153	int using_cookie = 0;
 154
 155	if (!pre && data->host_cookie && data->host_cookie != host->cookie) {
 156		dev_err(sdmmc_dev(host),
 157			"error: data->host_cookie = %d, host->cookie = %d\n",
 158			data->host_cookie, host->cookie);
 159		data->host_cookie = 0;
 160	}
 161
 162	if (pre || data->host_cookie != host->cookie) {
 163		count = rtsx_pci_dma_map_sg(pcr, data->sg, data->sg_len, read);
 164	} else {
 165		count = host->cookie_sg_count;
 166		using_cookie = 1;
 167	}
 168
 169	if (pre) {
 170		host->cookie_sg_count = count;
 171		if (++host->cookie < 0)
 172			host->cookie = 1;
 173		data->host_cookie = host->cookie;
 174	} else {
 175		host->sg_count = count;
 176	}
 177
 178	return using_cookie;
 179}
 180
 181static void sdmmc_pre_req(struct mmc_host *mmc, struct mmc_request *mrq)
 182{
 183	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
 184	struct mmc_data *data = mrq->data;
 185
 186	if (data->host_cookie) {
 187		dev_err(sdmmc_dev(host),
 188			"error: reset data->host_cookie = %d\n",
 189			data->host_cookie);
 190		data->host_cookie = 0;
 191	}
 192
 193	sd_pre_dma_transfer(host, data, true);
 194	dev_dbg(sdmmc_dev(host), "pre dma sg: %d\n", host->cookie_sg_count);
 195}
 196
 197static void sdmmc_post_req(struct mmc_host *mmc, struct mmc_request *mrq,
 198		int err)
 199{
 200	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
 201	struct rtsx_pcr *pcr = host->pcr;
 202	struct mmc_data *data = mrq->data;
 203	int read = data->flags & MMC_DATA_READ;
 204
 205	rtsx_pci_dma_unmap_sg(pcr, data->sg, data->sg_len, read);
 206	data->host_cookie = 0;
 207}
 208
 209static void sd_send_cmd_get_rsp(struct realtek_pci_sdmmc *host,
 210		struct mmc_command *cmd)
 211{
 212	struct rtsx_pcr *pcr = host->pcr;
 213	u8 cmd_idx = (u8)cmd->opcode;
 214	u32 arg = cmd->arg;
 215	int err = 0;
 216	int timeout = 100;
 217	int i;
 218	u8 *ptr;
 219	int rsp_type;
 220	int stat_idx;
 221	bool clock_toggled = false;
 222
 223	dev_dbg(sdmmc_dev(host), "%s: SD/MMC CMD %d, arg = 0x%08x\n",
 224			__func__, cmd_idx, arg);
 225
 226	rsp_type = sd_response_type(cmd);
 227	if (rsp_type < 0)
 228		goto out;
 229
 230	stat_idx = sd_status_index(rsp_type);
 231
 232	if (rsp_type == SD_RSP_TYPE_R1b)
 233		timeout = cmd->busy_timeout ? cmd->busy_timeout : 3000;
 234
 235	if (cmd->opcode == SD_SWITCH_VOLTAGE) {
 236		err = rtsx_pci_write_register(pcr, SD_BUS_STAT,
 237				0xFF, SD_CLK_TOGGLE_EN);
 238		if (err < 0)
 239			goto out;
 240
 241		clock_toggled = true;
 242	}
 243
 244	rtsx_pci_init_cmd(pcr);
 245	sd_cmd_set_sd_cmd(pcr, cmd);
 246	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG2, 0xFF, rsp_type);
 247	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_DATA_SOURCE,
 248			0x01, PINGPONG_BUFFER);
 249	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_TRANSFER,
 250			0xFF, SD_TM_CMD_RSP | SD_TRANSFER_START);
 251	rtsx_pci_add_cmd(pcr, CHECK_REG_CMD, SD_TRANSFER,
 252		     SD_TRANSFER_END | SD_STAT_IDLE,
 253		     SD_TRANSFER_END | SD_STAT_IDLE);
 254
 255	if (rsp_type == SD_RSP_TYPE_R2) {
 256		/* Read data from ping-pong buffer */
 257		for (i = PPBUF_BASE2; i < PPBUF_BASE2 + 16; i++)
 258			rtsx_pci_add_cmd(pcr, READ_REG_CMD, (u16)i, 0, 0);
 259	} else if (rsp_type != SD_RSP_TYPE_R0) {
 260		/* Read data from SD_CMDx registers */
 261		for (i = SD_CMD0; i <= SD_CMD4; i++)
 262			rtsx_pci_add_cmd(pcr, READ_REG_CMD, (u16)i, 0, 0);
 263	}
 264
 265	rtsx_pci_add_cmd(pcr, READ_REG_CMD, SD_STAT1, 0, 0);
 266
 267	err = rtsx_pci_send_cmd(pcr, timeout);
 268	if (err < 0) {
 269		sd_print_debug_regs(host);
 270		sd_clear_error(host);
 271		dev_dbg(sdmmc_dev(host),
 272			"rtsx_pci_send_cmd error (err = %d)\n", err);
 273		goto out;
 274	}
 275
 276	if (rsp_type == SD_RSP_TYPE_R0) {
 277		err = 0;
 278		goto out;
 279	}
 280
 281	/* Eliminate returned value of CHECK_REG_CMD */
 282	ptr = rtsx_pci_get_cmd_data(pcr) + 1;
 283
 284	/* Check (Start,Transmission) bit of Response */
 285	if ((ptr[0] & 0xC0) != 0) {
 286		err = -EILSEQ;
 287		dev_dbg(sdmmc_dev(host), "Invalid response bit\n");
 288		goto out;
 289	}
 290
 291	/* Check CRC7 */
 292	if (!(rsp_type & SD_NO_CHECK_CRC7)) {
 293		if (ptr[stat_idx] & SD_CRC7_ERR) {
 294			err = -EILSEQ;
 295			dev_dbg(sdmmc_dev(host), "CRC7 error\n");
 296			goto out;
 297		}
 298	}
 299
 300	if (rsp_type == SD_RSP_TYPE_R2) {
 301		/*
 302		 * The controller offloads the last byte {CRC-7, end bit 1'b1}
 303		 * of response type R2. Assign dummy CRC, 0, and end bit to the
 304		 * byte(ptr[16], goes into the LSB of resp[3] later).
 305		 */
 306		ptr[16] = 1;
 307
 308		for (i = 0; i < 4; i++) {
 309			cmd->resp[i] = get_unaligned_be32(ptr + 1 + i * 4);
 310			dev_dbg(sdmmc_dev(host), "cmd->resp[%d] = 0x%08x\n",
 311					i, cmd->resp[i]);
 312		}
 313	} else {
 314		cmd->resp[0] = get_unaligned_be32(ptr + 1);
 315		dev_dbg(sdmmc_dev(host), "cmd->resp[0] = 0x%08x\n",
 316				cmd->resp[0]);
 317	}
 318
 319out:
 320	cmd->error = err;
 321
 322	if (err && clock_toggled)
 323		rtsx_pci_write_register(pcr, SD_BUS_STAT,
 324				SD_CLK_TOGGLE_EN | SD_CLK_FORCE_STOP, 0);
 325}
 326
 327static int sd_read_data(struct realtek_pci_sdmmc *host, struct mmc_command *cmd,
 328	u16 byte_cnt, u8 *buf, int buf_len, int timeout)
 329{
 330	struct rtsx_pcr *pcr = host->pcr;
 331	int err;
 332	u8 trans_mode;
 333
 334	dev_dbg(sdmmc_dev(host), "%s: SD/MMC CMD %d, arg = 0x%08x\n",
 335		__func__, cmd->opcode, cmd->arg);
 336
 337	if (!buf)
 338		buf_len = 0;
 339
 340	if (cmd->opcode == MMC_SEND_TUNING_BLOCK)
 341		trans_mode = SD_TM_AUTO_TUNING;
 342	else
 343		trans_mode = SD_TM_NORMAL_READ;
 344
 345	rtsx_pci_init_cmd(pcr);
 346	sd_cmd_set_sd_cmd(pcr, cmd);
 347	sd_cmd_set_data_len(pcr, 1, byte_cnt);
 348	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG2, 0xFF,
 349			SD_CALCULATE_CRC7 | SD_CHECK_CRC16 |
 350			SD_NO_WAIT_BUSY_END | SD_CHECK_CRC7 | SD_RSP_LEN_6);
 351	if (trans_mode != SD_TM_AUTO_TUNING)
 352		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD,
 353				CARD_DATA_SOURCE, 0x01, PINGPONG_BUFFER);
 354
 355	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_TRANSFER,
 356			0xFF, trans_mode | SD_TRANSFER_START);
 357	rtsx_pci_add_cmd(pcr, CHECK_REG_CMD, SD_TRANSFER,
 358			SD_TRANSFER_END, SD_TRANSFER_END);
 359
 360	err = rtsx_pci_send_cmd(pcr, timeout);
 361	if (err < 0) {
 362		sd_print_debug_regs(host);
 363		dev_dbg(sdmmc_dev(host),
 364			"rtsx_pci_send_cmd fail (err = %d)\n", err);
 365		return err;
 366	}
 367
 368	if (buf && buf_len) {
 369		err = rtsx_pci_read_ppbuf(pcr, buf, buf_len);
 370		if (err < 0) {
 371			dev_dbg(sdmmc_dev(host),
 372				"rtsx_pci_read_ppbuf fail (err = %d)\n", err);
 373			return err;
 374		}
 375	}
 376
 377	return 0;
 378}
 379
 380static int sd_write_data(struct realtek_pci_sdmmc *host,
 381	struct mmc_command *cmd, u16 byte_cnt, u8 *buf, int buf_len,
 382	int timeout)
 383{
 384	struct rtsx_pcr *pcr = host->pcr;
 385	int err;
 386
 387	dev_dbg(sdmmc_dev(host), "%s: SD/MMC CMD %d, arg = 0x%08x\n",
 388		__func__, cmd->opcode, cmd->arg);
 389
 390	if (!buf)
 391		buf_len = 0;
 392
 393	sd_send_cmd_get_rsp(host, cmd);
 394	if (cmd->error)
 395		return cmd->error;
 396
 397	if (buf && buf_len) {
 398		err = rtsx_pci_write_ppbuf(pcr, buf, buf_len);
 399		if (err < 0) {
 400			dev_dbg(sdmmc_dev(host),
 401				"rtsx_pci_write_ppbuf fail (err = %d)\n", err);
 402			return err;
 403		}
 404	}
 405
 406	rtsx_pci_init_cmd(pcr);
 407	sd_cmd_set_data_len(pcr, 1, byte_cnt);
 408	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG2, 0xFF,
 409		SD_CALCULATE_CRC7 | SD_CHECK_CRC16 |
 410		SD_NO_WAIT_BUSY_END | SD_CHECK_CRC7 | SD_RSP_LEN_0);
 411	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_TRANSFER, 0xFF,
 412			SD_TRANSFER_START | SD_TM_AUTO_WRITE_3);
 413	rtsx_pci_add_cmd(pcr, CHECK_REG_CMD, SD_TRANSFER,
 414			SD_TRANSFER_END, SD_TRANSFER_END);
 415
 416	err = rtsx_pci_send_cmd(pcr, timeout);
 417	if (err < 0) {
 418		sd_print_debug_regs(host);
 419		dev_dbg(sdmmc_dev(host),
 420			"rtsx_pci_send_cmd fail (err = %d)\n", err);
 421		return err;
 422	}
 423
 424	return 0;
 425}
 426
 427static int sd_read_long_data(struct realtek_pci_sdmmc *host,
 428	struct mmc_request *mrq)
 429{
 430	struct rtsx_pcr *pcr = host->pcr;
 431	struct mmc_host *mmc = host->mmc;
 432	struct mmc_card *card = mmc->card;
 433	struct mmc_command *cmd = mrq->cmd;
 434	struct mmc_data *data = mrq->data;
 435	int uhs = mmc_card_uhs(card);
 436	u8 cfg2 = 0;
 437	int err;
 438	int resp_type;
 439	size_t data_len = data->blksz * data->blocks;
 440
 441	dev_dbg(sdmmc_dev(host), "%s: SD/MMC CMD %d, arg = 0x%08x\n",
 442		__func__, cmd->opcode, cmd->arg);
 443
 444	resp_type = sd_response_type(cmd);
 445	if (resp_type < 0)
 446		return resp_type;
 447
 448	if (!uhs)
 449		cfg2 |= SD_NO_CHECK_WAIT_CRC_TO;
 450
 451	rtsx_pci_init_cmd(pcr);
 452	sd_cmd_set_sd_cmd(pcr, cmd);
 453	sd_cmd_set_data_len(pcr, data->blocks, data->blksz);
 454	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, IRQSTAT0,
 455			DMA_DONE_INT, DMA_DONE_INT);
 456	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC3,
 457		0xFF, (u8)(data_len >> 24));
 458	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC2,
 459		0xFF, (u8)(data_len >> 16));
 460	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC1,
 461		0xFF, (u8)(data_len >> 8));
 462	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC0, 0xFF, (u8)data_len);
 463	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMACTL,
 464		0x03 | DMA_PACK_SIZE_MASK,
 465		DMA_DIR_FROM_CARD | DMA_EN | DMA_512);
 466	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_DATA_SOURCE,
 467			0x01, RING_BUFFER);
 468	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG2, 0xFF, cfg2 | resp_type);
 469	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_TRANSFER, 0xFF,
 470			SD_TRANSFER_START | SD_TM_AUTO_READ_2);
 471	rtsx_pci_add_cmd(pcr, CHECK_REG_CMD, SD_TRANSFER,
 472			SD_TRANSFER_END, SD_TRANSFER_END);
 473	rtsx_pci_send_cmd_no_wait(pcr);
 474
 475	err = rtsx_pci_dma_transfer(pcr, data->sg, host->sg_count, 1, 10000);
 476	if (err < 0) {
 477		sd_print_debug_regs(host);
 478		sd_clear_error(host);
 479		return err;
 480	}
 481
 482	return 0;
 483}
 484
 485static int sd_write_long_data(struct realtek_pci_sdmmc *host,
 486	struct mmc_request *mrq)
 487{
 488	struct rtsx_pcr *pcr = host->pcr;
 489	struct mmc_host *mmc = host->mmc;
 490	struct mmc_card *card = mmc->card;
 491	struct mmc_command *cmd = mrq->cmd;
 492	struct mmc_data *data = mrq->data;
 493	int uhs = mmc_card_uhs(card);
 494	u8 cfg2;
 495	int err;
 496	size_t data_len = data->blksz * data->blocks;
 497
 498	sd_send_cmd_get_rsp(host, cmd);
 499	if (cmd->error)
 500		return cmd->error;
 501
 502	dev_dbg(sdmmc_dev(host), "%s: SD/MMC CMD %d, arg = 0x%08x\n",
 503		__func__, cmd->opcode, cmd->arg);
 504
 505	cfg2 = SD_NO_CALCULATE_CRC7 | SD_CHECK_CRC16 |
 506		SD_NO_WAIT_BUSY_END | SD_NO_CHECK_CRC7 | SD_RSP_LEN_0;
 507
 508	if (!uhs)
 509		cfg2 |= SD_NO_CHECK_WAIT_CRC_TO;
 510
 511	rtsx_pci_init_cmd(pcr);
 512	sd_cmd_set_data_len(pcr, data->blocks, data->blksz);
 513	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, IRQSTAT0,
 514			DMA_DONE_INT, DMA_DONE_INT);
 515	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC3,
 516		0xFF, (u8)(data_len >> 24));
 517	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC2,
 518		0xFF, (u8)(data_len >> 16));
 519	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC1,
 520		0xFF, (u8)(data_len >> 8));
 521	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC0, 0xFF, (u8)data_len);
 522	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMACTL,
 523		0x03 | DMA_PACK_SIZE_MASK,
 524		DMA_DIR_TO_CARD | DMA_EN | DMA_512);
 525	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_DATA_SOURCE,
 526			0x01, RING_BUFFER);
 527	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG2, 0xFF, cfg2);
 528	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_TRANSFER, 0xFF,
 529			SD_TRANSFER_START | SD_TM_AUTO_WRITE_3);
 530	rtsx_pci_add_cmd(pcr, CHECK_REG_CMD, SD_TRANSFER,
 531			SD_TRANSFER_END, SD_TRANSFER_END);
 532	rtsx_pci_send_cmd_no_wait(pcr);
 533	err = rtsx_pci_dma_transfer(pcr, data->sg, host->sg_count, 0, 10000);
 534	if (err < 0) {
 535		sd_clear_error(host);
 536		return err;
 537	}
 538
 539	return 0;
 540}
 541
 542static inline void sd_enable_initial_mode(struct realtek_pci_sdmmc *host)
 543{
 544	rtsx_pci_write_register(host->pcr, SD_CFG1,
 545			SD_CLK_DIVIDE_MASK, SD_CLK_DIVIDE_128);
 546}
 547
 548static inline void sd_disable_initial_mode(struct realtek_pci_sdmmc *host)
 549{
 550	rtsx_pci_write_register(host->pcr, SD_CFG1,
 551			SD_CLK_DIVIDE_MASK, SD_CLK_DIVIDE_0);
 552}
 553
 554static int sd_rw_multi(struct realtek_pci_sdmmc *host, struct mmc_request *mrq)
 555{
 556	struct mmc_data *data = mrq->data;
 557	int err;
 558
 559	if (host->sg_count < 0) {
 560		data->error = host->sg_count;
 561		dev_dbg(sdmmc_dev(host), "%s: sg_count = %d is invalid\n",
 562			__func__, host->sg_count);
 563		return data->error;
 564	}
 565
 566	if (data->flags & MMC_DATA_READ) {
 567		if (host->initial_mode)
 568			sd_disable_initial_mode(host);
 569
 570		err = sd_read_long_data(host, mrq);
 571
 572		if (host->initial_mode)
 573			sd_enable_initial_mode(host);
 574
 575		return err;
 576	}
 
 
 
 577
 578	return sd_write_long_data(host, mrq);
 
 
 
 579}
 580
 581static void sd_normal_rw(struct realtek_pci_sdmmc *host,
 582		struct mmc_request *mrq)
 583{
 584	struct mmc_command *cmd = mrq->cmd;
 585	struct mmc_data *data = mrq->data;
 586	u8 *buf;
 587
 588	buf = kzalloc(data->blksz, GFP_NOIO);
 589	if (!buf) {
 590		cmd->error = -ENOMEM;
 591		return;
 592	}
 593
 594	if (data->flags & MMC_DATA_READ) {
 595		if (host->initial_mode)
 596			sd_disable_initial_mode(host);
 597
 598		cmd->error = sd_read_data(host, cmd, (u16)data->blksz, buf,
 599				data->blksz, 200);
 600
 601		if (host->initial_mode)
 602			sd_enable_initial_mode(host);
 603
 604		sg_copy_from_buffer(data->sg, data->sg_len, buf, data->blksz);
 605	} else {
 606		sg_copy_to_buffer(data->sg, data->sg_len, buf, data->blksz);
 607
 608		cmd->error = sd_write_data(host, cmd, (u16)data->blksz, buf,
 609				data->blksz, 200);
 610	}
 611
 612	kfree(buf);
 613}
 614
 615static int sd_change_phase(struct realtek_pci_sdmmc *host,
 616		u8 sample_point, bool rx)
 617{
 618	struct rtsx_pcr *pcr = host->pcr;
 619	u16 SD_VP_CTL = 0;
 620	dev_dbg(sdmmc_dev(host), "%s(%s): sample_point = %d\n",
 621			__func__, rx ? "RX" : "TX", sample_point);
 622
 623	rtsx_pci_write_register(pcr, CLK_CTL, CHANGE_CLK, CHANGE_CLK);
 624	if (rx) {
 625		SD_VP_CTL = SD_VPRX_CTL;
 626		rtsx_pci_write_register(pcr, SD_VPRX_CTL,
 627			PHASE_SELECT_MASK, sample_point);
 628	} else {
 629		SD_VP_CTL = SD_VPTX_CTL;
 630		rtsx_pci_write_register(pcr, SD_VPTX_CTL,
 631			PHASE_SELECT_MASK, sample_point);
 632	}
 633	rtsx_pci_write_register(pcr, SD_VP_CTL, PHASE_NOT_RESET, 0);
 634	rtsx_pci_write_register(pcr, SD_VP_CTL, PHASE_NOT_RESET,
 635				PHASE_NOT_RESET);
 636	rtsx_pci_write_register(pcr, CLK_CTL, CHANGE_CLK, 0);
 637	rtsx_pci_write_register(pcr, SD_CFG1, SD_ASYNC_FIFO_NOT_RST, 0);
 638
 639	return 0;
 640}
 641
 642static inline u32 test_phase_bit(u32 phase_map, unsigned int bit)
 643{
 644	bit %= RTSX_PHASE_MAX;
 645	return phase_map & (1 << bit);
 646}
 647
 648static int sd_get_phase_len(u32 phase_map, unsigned int start_bit)
 649{
 650	int i;
 651
 652	for (i = 0; i < RTSX_PHASE_MAX; i++) {
 653		if (test_phase_bit(phase_map, start_bit + i) == 0)
 654			return i;
 655	}
 656	return RTSX_PHASE_MAX;
 657}
 658
 659static u8 sd_search_final_phase(struct realtek_pci_sdmmc *host, u32 phase_map)
 660{
 661	int start = 0, len = 0;
 662	int start_final = 0, len_final = 0;
 663	u8 final_phase = 0xFF;
 664
 665	if (phase_map == 0) {
 666		dev_err(sdmmc_dev(host), "phase error: [map:%x]\n", phase_map);
 667		return final_phase;
 668	}
 669
 670	while (start < RTSX_PHASE_MAX) {
 671		len = sd_get_phase_len(phase_map, start);
 672		if (len_final < len) {
 673			start_final = start;
 674			len_final = len;
 675		}
 676		start += len ? len : 1;
 677	}
 678
 679	final_phase = (start_final + len_final / 2) % RTSX_PHASE_MAX;
 680	dev_dbg(sdmmc_dev(host), "phase: [map:%x] [maxlen:%d] [final:%d]\n",
 681		phase_map, len_final, final_phase);
 682
 683	return final_phase;
 684}
 685
 686static void sd_wait_data_idle(struct realtek_pci_sdmmc *host)
 687{
 688	int i;
 689	u8 val = 0;
 690
 691	for (i = 0; i < 100; i++) {
 692		rtsx_pci_read_register(host->pcr, SD_DATA_STATE, &val);
 693		if (val & SD_DATA_IDLE)
 694			return;
 695
 696		udelay(100);
 697	}
 698}
 699
 700static int sd_tuning_rx_cmd(struct realtek_pci_sdmmc *host,
 701		u8 opcode, u8 sample_point)
 702{
 703	int err;
 704	struct mmc_command cmd = {};
 705	struct rtsx_pcr *pcr = host->pcr;
 706
 707	sd_change_phase(host, sample_point, true);
 708
 709	rtsx_pci_write_register(pcr, SD_CFG3, SD_RSP_80CLK_TIMEOUT_EN,
 710		SD_RSP_80CLK_TIMEOUT_EN);
 711
 712	cmd.opcode = opcode;
 713	err = sd_read_data(host, &cmd, 0x40, NULL, 0, 100);
 714	if (err < 0) {
 715		/* Wait till SD DATA IDLE */
 716		sd_wait_data_idle(host);
 717		sd_clear_error(host);
 718		rtsx_pci_write_register(pcr, SD_CFG3,
 719			SD_RSP_80CLK_TIMEOUT_EN, 0);
 720		return err;
 721	}
 722
 723	rtsx_pci_write_register(pcr, SD_CFG3, SD_RSP_80CLK_TIMEOUT_EN, 0);
 724	return 0;
 725}
 726
 727static int sd_tuning_phase(struct realtek_pci_sdmmc *host,
 728		u8 opcode, u32 *phase_map)
 729{
 730	int err, i;
 731	u32 raw_phase_map = 0;
 732
 733	for (i = 0; i < RTSX_PHASE_MAX; i++) {
 734		err = sd_tuning_rx_cmd(host, opcode, (u8)i);
 735		if (err == 0)
 736			raw_phase_map |= 1 << i;
 737	}
 738
 739	if (phase_map)
 740		*phase_map = raw_phase_map;
 741
 742	return 0;
 743}
 744
 745static int sd_tuning_rx(struct realtek_pci_sdmmc *host, u8 opcode)
 746{
 747	int err, i;
 748	u32 raw_phase_map[RX_TUNING_CNT] = {0}, phase_map;
 749	u8 final_phase;
 750
 751	for (i = 0; i < RX_TUNING_CNT; i++) {
 752		err = sd_tuning_phase(host, opcode, &(raw_phase_map[i]));
 753		if (err < 0)
 754			return err;
 755
 756		if (raw_phase_map[i] == 0)
 757			break;
 758	}
 759
 760	phase_map = 0xFFFFFFFF;
 761	for (i = 0; i < RX_TUNING_CNT; i++) {
 762		dev_dbg(sdmmc_dev(host), "RX raw_phase_map[%d] = 0x%08x\n",
 763				i, raw_phase_map[i]);
 764		phase_map &= raw_phase_map[i];
 765	}
 766	dev_dbg(sdmmc_dev(host), "RX phase_map = 0x%08x\n", phase_map);
 767
 768	if (phase_map) {
 769		final_phase = sd_search_final_phase(host, phase_map);
 770		if (final_phase == 0xFF)
 771			return -EINVAL;
 772
 773		err = sd_change_phase(host, final_phase, true);
 774		if (err < 0)
 775			return err;
 776	} else {
 777		return -EINVAL;
 778	}
 779
 780	return 0;
 781}
 782
 783static inline int sdio_extblock_cmd(struct mmc_command *cmd,
 784	struct mmc_data *data)
 785{
 786	return (cmd->opcode == SD_IO_RW_EXTENDED) && (data->blksz == 512);
 787}
 788
 789static inline int sd_rw_cmd(struct mmc_command *cmd)
 790{
 791	return mmc_op_multi(cmd->opcode) ||
 792		(cmd->opcode == MMC_READ_SINGLE_BLOCK) ||
 793		(cmd->opcode == MMC_WRITE_BLOCK);
 794}
 795
 796static void sd_request(struct work_struct *work)
 797{
 798	struct realtek_pci_sdmmc *host = container_of(work,
 799			struct realtek_pci_sdmmc, work);
 800	struct rtsx_pcr *pcr = host->pcr;
 801
 802	struct mmc_host *mmc = host->mmc;
 803	struct mmc_request *mrq = host->mrq;
 804	struct mmc_command *cmd = mrq->cmd;
 805	struct mmc_data *data = mrq->data;
 806
 807	unsigned int data_size = 0;
 808	int err;
 809
 810	if (host->eject || !sd_get_cd_int(host)) {
 811		cmd->error = -ENOMEDIUM;
 812		goto finish;
 813	}
 814
 815	err = rtsx_pci_card_exclusive_check(host->pcr, RTSX_SD_CARD);
 816	if (err) {
 817		cmd->error = err;
 818		goto finish;
 819	}
 820
 821	mutex_lock(&pcr->pcr_mutex);
 822
 823	rtsx_pci_start_run(pcr);
 824
 825	rtsx_pci_switch_clock(pcr, host->clock, host->ssc_depth,
 826			host->initial_mode, host->double_clk, host->vpclk);
 827	rtsx_pci_write_register(pcr, CARD_SELECT, 0x07, SD_MOD_SEL);
 828	rtsx_pci_write_register(pcr, CARD_SHARE_MODE,
 829			CARD_SHARE_MASK, CARD_SHARE_48_SD);
 830
 831	mutex_lock(&host->host_mutex);
 832	host->mrq = mrq;
 833	mutex_unlock(&host->host_mutex);
 834
 835	if (mrq->data)
 836		data_size = data->blocks * data->blksz;
 837
 838	if (!data_size) {
 839		sd_send_cmd_get_rsp(host, cmd);
 840	} else if (sd_rw_cmd(cmd) || sdio_extblock_cmd(cmd, data)) {
 841		cmd->error = sd_rw_multi(host, mrq);
 842		if (!host->using_cookie)
 843			sdmmc_post_req(host->mmc, host->mrq, 0);
 844
 845		if (mmc_op_multi(cmd->opcode) && mrq->stop)
 846			sd_send_cmd_get_rsp(host, mrq->stop);
 847	} else {
 848		sd_normal_rw(host, mrq);
 849	}
 850
 851	if (mrq->data) {
 852		if (cmd->error || data->error)
 853			data->bytes_xfered = 0;
 854		else
 855			data->bytes_xfered = data->blocks * data->blksz;
 856	}
 857
 858	mutex_unlock(&pcr->pcr_mutex);
 859
 860finish:
 861	if (cmd->error) {
 862		dev_dbg(sdmmc_dev(host), "CMD %d 0x%08x error(%d)\n",
 863			cmd->opcode, cmd->arg, cmd->error);
 864	}
 865
 866	mutex_lock(&host->host_mutex);
 867	host->mrq = NULL;
 868	mutex_unlock(&host->host_mutex);
 869
 870	mmc_request_done(mmc, mrq);
 871}
 872
 873static void sdmmc_request(struct mmc_host *mmc, struct mmc_request *mrq)
 874{
 875	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
 876	struct mmc_data *data = mrq->data;
 877
 878	mutex_lock(&host->host_mutex);
 879	host->mrq = mrq;
 880	mutex_unlock(&host->host_mutex);
 881
 882	if (sd_rw_cmd(mrq->cmd) || sdio_extblock_cmd(mrq->cmd, data))
 883		host->using_cookie = sd_pre_dma_transfer(host, data, false);
 884
 885	schedule_work(&host->work);
 886}
 887
 888static int sd_set_bus_width(struct realtek_pci_sdmmc *host,
 889		unsigned char bus_width)
 890{
 891	int err = 0;
 892	u8 width[] = {
 893		[MMC_BUS_WIDTH_1] = SD_BUS_WIDTH_1BIT,
 894		[MMC_BUS_WIDTH_4] = SD_BUS_WIDTH_4BIT,
 895		[MMC_BUS_WIDTH_8] = SD_BUS_WIDTH_8BIT,
 896	};
 897
 898	if (bus_width <= MMC_BUS_WIDTH_8)
 899		err = rtsx_pci_write_register(host->pcr, SD_CFG1,
 900				0x03, width[bus_width]);
 901
 902	return err;
 903}
 904
 905static int sd_power_on(struct realtek_pci_sdmmc *host, unsigned char power_mode)
 906{
 907	struct rtsx_pcr *pcr = host->pcr;
 908	struct mmc_host *mmc = host->mmc;
 909	int err;
 910	u32 val;
 911	u8 test_mode;
 912
 913	if (host->prev_power_state == MMC_POWER_ON)
 914		return 0;
 915
 916	if (host->prev_power_state == MMC_POWER_UP) {
 917		rtsx_pci_write_register(pcr, SD_BUS_STAT, SD_CLK_TOGGLE_EN, 0);
 918		goto finish;
 919	}
 920
 921	msleep(100);
 922
 923	rtsx_pci_init_cmd(pcr);
 924	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_SELECT, 0x07, SD_MOD_SEL);
 925	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_SHARE_MODE,
 926			CARD_SHARE_MASK, CARD_SHARE_48_SD);
 927	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_EN,
 928			SD_CLK_EN, SD_CLK_EN);
 929	err = rtsx_pci_send_cmd(pcr, 100);
 930	if (err < 0)
 931		return err;
 932
 933	err = rtsx_pci_card_pull_ctl_enable(pcr, RTSX_SD_CARD);
 934	if (err < 0)
 935		return err;
 936
 937	err = rtsx_pci_card_power_on(pcr, RTSX_SD_CARD);
 938	if (err < 0)
 939		return err;
 940
 941	mdelay(1);
 942
 943	err = rtsx_pci_write_register(pcr, CARD_OE, SD_OUTPUT_EN, SD_OUTPUT_EN);
 944	if (err < 0)
 945		return err;
 946
 947	/* send at least 74 clocks */
 948	rtsx_pci_write_register(pcr, SD_BUS_STAT, SD_CLK_TOGGLE_EN, SD_CLK_TOGGLE_EN);
 949
 950	if (PCI_PID(pcr) == PID_5261) {
 951		/*
 952		 * If test mode is set switch to SD Express mandatorily,
 953		 * this is only for factory testing.
 954		 */
 955		rtsx_pci_read_register(pcr, RTS5261_FW_CFG_INFO0, &test_mode);
 956		if (test_mode & RTS5261_FW_EXPRESS_TEST_MASK) {
 957			sdmmc_init_sd_express(mmc, NULL);
 958			return 0;
 959		}
 960		if (pcr->extra_caps & EXTRA_CAPS_SD_EXPRESS)
 961			mmc->caps2 |= MMC_CAP2_SD_EXP | MMC_CAP2_SD_EXP_1_2V;
 962		/*
 963		 * HW read wp status when resuming from S3/S4,
 964		 * and then picks SD legacy interface if it's set
 965		 * in read-only mode.
 966		 */
 967		val = rtsx_pci_readl(pcr, RTSX_BIPR);
 968		if (val & SD_WRITE_PROTECT) {
 969			pcr->extra_caps &= ~EXTRA_CAPS_SD_EXPRESS;
 970			mmc->caps2 &= ~(MMC_CAP2_SD_EXP | MMC_CAP2_SD_EXP_1_2V);
 971		}
 972	}
 973
 974finish:
 975	host->prev_power_state = power_mode;
 976	return 0;
 977}
 978
 979static int sd_power_off(struct realtek_pci_sdmmc *host)
 980{
 981	struct rtsx_pcr *pcr = host->pcr;
 982	int err;
 983
 984	host->prev_power_state = MMC_POWER_OFF;
 985
 986	rtsx_pci_init_cmd(pcr);
 987
 988	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_EN, SD_CLK_EN, 0);
 989	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_OE, SD_OUTPUT_EN, 0);
 990
 991	err = rtsx_pci_send_cmd(pcr, 100);
 992	if (err < 0)
 993		return err;
 994
 995	err = rtsx_pci_card_power_off(pcr, RTSX_SD_CARD);
 996	if (err < 0)
 997		return err;
 998
 999	return rtsx_pci_card_pull_ctl_disable(pcr, RTSX_SD_CARD);
1000}
1001
1002static int sd_set_power_mode(struct realtek_pci_sdmmc *host,
1003		unsigned char power_mode)
1004{
1005	int err;
1006
1007	if (power_mode == MMC_POWER_OFF)
1008		err = sd_power_off(host);
1009	else
1010		err = sd_power_on(host, power_mode);
1011
1012	return err;
1013}
1014
1015static int sd_set_timing(struct realtek_pci_sdmmc *host, unsigned char timing)
1016{
1017	struct rtsx_pcr *pcr = host->pcr;
1018	int err = 0;
1019
1020	rtsx_pci_init_cmd(pcr);
1021
1022	switch (timing) {
1023	case MMC_TIMING_UHS_SDR104:
1024	case MMC_TIMING_UHS_SDR50:
1025		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG1,
1026				0x0C | SD_ASYNC_FIFO_NOT_RST,
1027				SD_30_MODE | SD_ASYNC_FIFO_NOT_RST);
1028		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL,
1029				CLK_LOW_FREQ, CLK_LOW_FREQ);
1030		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_SOURCE, 0xFF,
1031				CRC_VAR_CLK0 | SD30_FIX_CLK | SAMPLE_VAR_CLK1);
1032		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL, CLK_LOW_FREQ, 0);
1033		break;
1034
1035	case MMC_TIMING_MMC_DDR52:
1036	case MMC_TIMING_UHS_DDR50:
1037		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG1,
1038				0x0C | SD_ASYNC_FIFO_NOT_RST,
1039				SD_DDR_MODE | SD_ASYNC_FIFO_NOT_RST);
1040		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL,
1041				CLK_LOW_FREQ, CLK_LOW_FREQ);
1042		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_SOURCE, 0xFF,
1043				CRC_VAR_CLK0 | SD30_FIX_CLK | SAMPLE_VAR_CLK1);
1044		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL, CLK_LOW_FREQ, 0);
1045		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_PUSH_POINT_CTL,
1046				DDR_VAR_TX_CMD_DAT, DDR_VAR_TX_CMD_DAT);
1047		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_SAMPLE_POINT_CTL,
1048				DDR_VAR_RX_DAT | DDR_VAR_RX_CMD,
1049				DDR_VAR_RX_DAT | DDR_VAR_RX_CMD);
1050		break;
1051
1052	case MMC_TIMING_MMC_HS:
1053	case MMC_TIMING_SD_HS:
1054		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG1,
1055				0x0C, SD_20_MODE);
1056		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL,
1057				CLK_LOW_FREQ, CLK_LOW_FREQ);
1058		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_SOURCE, 0xFF,
1059				CRC_FIX_CLK | SD30_VAR_CLK0 | SAMPLE_VAR_CLK1);
1060		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL, CLK_LOW_FREQ, 0);
1061		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_PUSH_POINT_CTL,
1062				SD20_TX_SEL_MASK, SD20_TX_14_AHEAD);
1063		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_SAMPLE_POINT_CTL,
1064				SD20_RX_SEL_MASK, SD20_RX_14_DELAY);
1065		break;
1066
1067	default:
1068		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD,
1069				SD_CFG1, 0x0C, SD_20_MODE);
1070		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL,
1071				CLK_LOW_FREQ, CLK_LOW_FREQ);
1072		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_SOURCE, 0xFF,
1073				CRC_FIX_CLK | SD30_VAR_CLK0 | SAMPLE_VAR_CLK1);
1074		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL, CLK_LOW_FREQ, 0);
1075		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD,
1076				SD_PUSH_POINT_CTL, 0xFF, 0);
1077		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_SAMPLE_POINT_CTL,
1078				SD20_RX_SEL_MASK, SD20_RX_POS_EDGE);
1079		break;
1080	}
1081
1082	err = rtsx_pci_send_cmd(pcr, 100);
1083
1084	return err;
1085}
1086
1087static void sdmmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
1088{
1089	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1090	struct rtsx_pcr *pcr = host->pcr;
1091
1092	if (host->eject)
1093		return;
1094
1095	if (rtsx_pci_card_exclusive_check(host->pcr, RTSX_SD_CARD))
1096		return;
1097
1098	mutex_lock(&pcr->pcr_mutex);
1099
1100	rtsx_pci_start_run(pcr);
1101
1102	sd_set_bus_width(host, ios->bus_width);
1103	sd_set_power_mode(host, ios->power_mode);
1104	sd_set_timing(host, ios->timing);
1105
1106	host->vpclk = false;
1107	host->double_clk = true;
1108
1109	switch (ios->timing) {
1110	case MMC_TIMING_UHS_SDR104:
1111	case MMC_TIMING_UHS_SDR50:
1112		host->ssc_depth = RTSX_SSC_DEPTH_2M;
1113		host->vpclk = true;
1114		host->double_clk = false;
1115		break;
1116	case MMC_TIMING_MMC_DDR52:
1117	case MMC_TIMING_UHS_DDR50:
1118	case MMC_TIMING_UHS_SDR25:
1119		host->ssc_depth = RTSX_SSC_DEPTH_1M;
1120		break;
1121	default:
1122		host->ssc_depth = RTSX_SSC_DEPTH_500K;
1123		break;
1124	}
1125
1126	host->initial_mode = (ios->clock <= 1000000) ? true : false;
1127
1128	host->clock = ios->clock;
1129	rtsx_pci_switch_clock(pcr, ios->clock, host->ssc_depth,
1130			host->initial_mode, host->double_clk, host->vpclk);
1131
1132	mutex_unlock(&pcr->pcr_mutex);
1133}
1134
1135static int sdmmc_get_ro(struct mmc_host *mmc)
1136{
1137	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1138	struct rtsx_pcr *pcr = host->pcr;
1139	int ro = 0;
1140	u32 val;
1141
1142	if (host->eject)
1143		return -ENOMEDIUM;
1144
1145	mutex_lock(&pcr->pcr_mutex);
1146
1147	rtsx_pci_start_run(pcr);
1148
1149	/* Check SD mechanical write-protect switch */
1150	val = rtsx_pci_readl(pcr, RTSX_BIPR);
1151	dev_dbg(sdmmc_dev(host), "%s: RTSX_BIPR = 0x%08x\n", __func__, val);
1152	if (val & SD_WRITE_PROTECT)
1153		ro = 1;
1154
1155	mutex_unlock(&pcr->pcr_mutex);
1156
1157	return ro;
1158}
1159
1160static int sdmmc_get_cd(struct mmc_host *mmc)
1161{
1162	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1163	struct rtsx_pcr *pcr = host->pcr;
1164	int cd = 0;
1165	u32 val;
1166
1167	if (host->eject)
1168		return cd;
1169
1170	mutex_lock(&pcr->pcr_mutex);
1171
1172	rtsx_pci_start_run(pcr);
1173
1174	/* Check SD card detect */
1175	val = rtsx_pci_card_exist(pcr);
1176	dev_dbg(sdmmc_dev(host), "%s: RTSX_BIPR = 0x%08x\n", __func__, val);
1177	if (val & SD_EXIST)
1178		cd = 1;
1179
1180	mutex_unlock(&pcr->pcr_mutex);
1181
1182	return cd;
1183}
1184
1185static int sd_wait_voltage_stable_1(struct realtek_pci_sdmmc *host)
1186{
1187	struct rtsx_pcr *pcr = host->pcr;
1188	int err;
1189	u8 stat;
1190
1191	/* Reference to Signal Voltage Switch Sequence in SD spec.
1192	 * Wait for a period of time so that the card can drive SD_CMD and
1193	 * SD_DAT[3:0] to low after sending back CMD11 response.
1194	 */
1195	mdelay(1);
1196
1197	/* SD_CMD, SD_DAT[3:0] should be driven to low by card;
1198	 * If either one of SD_CMD,SD_DAT[3:0] is not low,
1199	 * abort the voltage switch sequence;
1200	 */
1201	err = rtsx_pci_read_register(pcr, SD_BUS_STAT, &stat);
1202	if (err < 0)
1203		return err;
1204
1205	if (stat & (SD_CMD_STATUS | SD_DAT3_STATUS | SD_DAT2_STATUS |
1206				SD_DAT1_STATUS | SD_DAT0_STATUS))
1207		return -EINVAL;
1208
1209	/* Stop toggle SD clock */
1210	err = rtsx_pci_write_register(pcr, SD_BUS_STAT,
1211			0xFF, SD_CLK_FORCE_STOP);
1212	if (err < 0)
1213		return err;
1214
1215	return 0;
1216}
1217
1218static int sd_wait_voltage_stable_2(struct realtek_pci_sdmmc *host)
1219{
1220	struct rtsx_pcr *pcr = host->pcr;
1221	int err;
1222	u8 stat, mask, val;
1223
1224	/* Wait 1.8V output of voltage regulator in card stable */
1225	msleep(50);
1226
1227	/* Toggle SD clock again */
1228	err = rtsx_pci_write_register(pcr, SD_BUS_STAT, 0xFF, SD_CLK_TOGGLE_EN);
1229	if (err < 0)
1230		return err;
1231
1232	/* Wait for a period of time so that the card can drive
1233	 * SD_DAT[3:0] to high at 1.8V
1234	 */
1235	msleep(20);
1236
1237	/* SD_CMD, SD_DAT[3:0] should be pulled high by host */
1238	err = rtsx_pci_read_register(pcr, SD_BUS_STAT, &stat);
1239	if (err < 0)
1240		return err;
1241
1242	mask = SD_CMD_STATUS | SD_DAT3_STATUS | SD_DAT2_STATUS |
1243		SD_DAT1_STATUS | SD_DAT0_STATUS;
1244	val = SD_CMD_STATUS | SD_DAT3_STATUS | SD_DAT2_STATUS |
1245		SD_DAT1_STATUS | SD_DAT0_STATUS;
1246	if ((stat & mask) != val) {
1247		dev_dbg(sdmmc_dev(host),
1248			"%s: SD_BUS_STAT = 0x%x\n", __func__, stat);
1249		rtsx_pci_write_register(pcr, SD_BUS_STAT,
1250				SD_CLK_TOGGLE_EN | SD_CLK_FORCE_STOP, 0);
1251		rtsx_pci_write_register(pcr, CARD_CLK_EN, 0xFF, 0);
1252		return -EINVAL;
1253	}
1254
1255	return 0;
1256}
1257
1258static int sdmmc_switch_voltage(struct mmc_host *mmc, struct mmc_ios *ios)
1259{
1260	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1261	struct rtsx_pcr *pcr = host->pcr;
1262	int err = 0;
1263	u8 voltage;
1264
1265	dev_dbg(sdmmc_dev(host), "%s: signal_voltage = %d\n",
1266			__func__, ios->signal_voltage);
1267
1268	if (host->eject)
1269		return -ENOMEDIUM;
1270
1271	err = rtsx_pci_card_exclusive_check(host->pcr, RTSX_SD_CARD);
1272	if (err)
1273		return err;
1274
1275	mutex_lock(&pcr->pcr_mutex);
1276
1277	rtsx_pci_start_run(pcr);
1278
1279	if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_330)
1280		voltage = OUTPUT_3V3;
1281	else
1282		voltage = OUTPUT_1V8;
1283
1284	if (voltage == OUTPUT_1V8) {
1285		err = sd_wait_voltage_stable_1(host);
1286		if (err < 0)
1287			goto out;
1288	}
1289
1290	err = rtsx_pci_switch_output_voltage(pcr, voltage);
1291	if (err < 0)
1292		goto out;
1293
1294	if (voltage == OUTPUT_1V8) {
1295		err = sd_wait_voltage_stable_2(host);
1296		if (err < 0)
1297			goto out;
1298	}
1299
1300out:
1301	/* Stop toggle SD clock in idle */
1302	err = rtsx_pci_write_register(pcr, SD_BUS_STAT,
1303			SD_CLK_TOGGLE_EN | SD_CLK_FORCE_STOP, 0);
1304
1305	mutex_unlock(&pcr->pcr_mutex);
1306
1307	return err;
1308}
1309
1310static int sdmmc_execute_tuning(struct mmc_host *mmc, u32 opcode)
1311{
1312	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1313	struct rtsx_pcr *pcr = host->pcr;
1314	int err = 0;
1315
1316	if (host->eject)
1317		return -ENOMEDIUM;
1318
1319	err = rtsx_pci_card_exclusive_check(host->pcr, RTSX_SD_CARD);
1320	if (err)
1321		return err;
1322
1323	mutex_lock(&pcr->pcr_mutex);
1324
1325	rtsx_pci_start_run(pcr);
1326
1327	/* Set initial TX phase */
1328	switch (mmc->ios.timing) {
1329	case MMC_TIMING_UHS_SDR104:
1330		err = sd_change_phase(host, SDR104_TX_PHASE(pcr), false);
1331		break;
1332
1333	case MMC_TIMING_UHS_SDR50:
1334		err = sd_change_phase(host, SDR50_TX_PHASE(pcr), false);
1335		break;
1336
1337	case MMC_TIMING_UHS_DDR50:
1338		err = sd_change_phase(host, DDR50_TX_PHASE(pcr), false);
1339		break;
1340
1341	default:
1342		err = 0;
1343	}
1344
1345	if (err)
1346		goto out;
1347
1348	/* Tuning RX phase */
1349	if ((mmc->ios.timing == MMC_TIMING_UHS_SDR104) ||
1350			(mmc->ios.timing == MMC_TIMING_UHS_SDR50))
1351		err = sd_tuning_rx(host, opcode);
1352	else if (mmc->ios.timing == MMC_TIMING_UHS_DDR50)
1353		err = sd_change_phase(host, DDR50_RX_PHASE(pcr), true);
1354
1355out:
1356	mutex_unlock(&pcr->pcr_mutex);
1357
1358	return err;
1359}
1360
1361static int sdmmc_init_sd_express(struct mmc_host *mmc, struct mmc_ios *ios)
1362{
1363	u32 relink_time;
1364	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1365	struct rtsx_pcr *pcr = host->pcr;
1366
1367	/* Set relink_time for changing to PCIe card */
1368	relink_time = 0x8FFF;
1369
1370	rtsx_pci_write_register(pcr, 0xFF01, 0xFF, relink_time);
1371	rtsx_pci_write_register(pcr, 0xFF02, 0xFF, relink_time >> 8);
1372	rtsx_pci_write_register(pcr, 0xFF03, 0x01, relink_time >> 16);
1373
1374	rtsx_pci_write_register(pcr, PETXCFG, 0x80, 0x80);
1375	rtsx_pci_write_register(pcr, LDO_VCC_CFG0,
1376		RTS5261_LDO1_OCP_THD_MASK,
1377		pcr->option.sd_800mA_ocp_thd);
1378
1379	if (pcr->ops->disable_auto_blink)
1380		pcr->ops->disable_auto_blink(pcr);
1381
1382	/* For PCIe/NVMe mode can't enter delink issue */
1383	pcr->hw_param.interrupt_en &= ~(SD_INT_EN);
1384	rtsx_pci_writel(pcr, RTSX_BIER, pcr->hw_param.interrupt_en);
1385
1386	rtsx_pci_write_register(pcr, RTS5260_AUTOLOAD_CFG4,
1387		RTS5261_AUX_CLK_16M_EN, RTS5261_AUX_CLK_16M_EN);
1388	rtsx_pci_write_register(pcr, RTS5261_FW_CFG0,
1389		RTS5261_FW_ENTER_EXPRESS, RTS5261_FW_ENTER_EXPRESS);
1390	rtsx_pci_write_register(pcr, RTS5261_FW_CFG1,
1391		RTS5261_MCU_CLOCK_GATING, RTS5261_MCU_CLOCK_GATING);
1392	rtsx_pci_write_register(pcr, RTS5261_FW_CFG1,
1393		RTS5261_MCU_BUS_SEL_MASK | RTS5261_MCU_CLOCK_SEL_MASK
1394		| RTS5261_DRIVER_ENABLE_FW,
1395		RTS5261_MCU_CLOCK_SEL_16M | RTS5261_DRIVER_ENABLE_FW);
1396	host->eject = true;
1397	return 0;
1398}
1399
1400static const struct mmc_host_ops realtek_pci_sdmmc_ops = {
1401	.pre_req = sdmmc_pre_req,
1402	.post_req = sdmmc_post_req,
1403	.request = sdmmc_request,
1404	.set_ios = sdmmc_set_ios,
1405	.get_ro = sdmmc_get_ro,
1406	.get_cd = sdmmc_get_cd,
1407	.start_signal_voltage_switch = sdmmc_switch_voltage,
1408	.execute_tuning = sdmmc_execute_tuning,
1409	.init_sd_express = sdmmc_init_sd_express,
1410};
1411
1412static void init_extra_caps(struct realtek_pci_sdmmc *host)
1413{
1414	struct mmc_host *mmc = host->mmc;
1415	struct rtsx_pcr *pcr = host->pcr;
1416
1417	dev_dbg(sdmmc_dev(host), "pcr->extra_caps = 0x%x\n", pcr->extra_caps);
1418
1419	if (pcr->extra_caps & EXTRA_CAPS_SD_SDR50)
1420		mmc->caps |= MMC_CAP_UHS_SDR50;
1421	if (pcr->extra_caps & EXTRA_CAPS_SD_SDR104)
1422		mmc->caps |= MMC_CAP_UHS_SDR104;
1423	if (pcr->extra_caps & EXTRA_CAPS_SD_DDR50)
1424		mmc->caps |= MMC_CAP_UHS_DDR50;
1425	if (pcr->extra_caps & EXTRA_CAPS_MMC_HSDDR)
1426		mmc->caps |= MMC_CAP_1_8V_DDR;
1427	if (pcr->extra_caps & EXTRA_CAPS_MMC_8BIT)
1428		mmc->caps |= MMC_CAP_8_BIT_DATA;
1429	if (pcr->extra_caps & EXTRA_CAPS_NO_MMC)
1430		mmc->caps2 |= MMC_CAP2_NO_MMC;
1431	if (pcr->extra_caps & EXTRA_CAPS_SD_EXPRESS)
1432		mmc->caps2 |= MMC_CAP2_SD_EXP | MMC_CAP2_SD_EXP_1_2V;
1433}
1434
1435static void realtek_init_host(struct realtek_pci_sdmmc *host)
1436{
1437	struct mmc_host *mmc = host->mmc;
1438	struct rtsx_pcr *pcr = host->pcr;
1439
1440	mmc->f_min = 250000;
1441	mmc->f_max = 208000000;
1442	mmc->ocr_avail = MMC_VDD_32_33 | MMC_VDD_33_34 | MMC_VDD_165_195;
1443	mmc->caps = MMC_CAP_4_BIT_DATA | MMC_CAP_SD_HIGHSPEED |
1444		MMC_CAP_MMC_HIGHSPEED | MMC_CAP_BUS_WIDTH_TEST |
1445		MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25;
1446	if (pcr->rtd3_en)
1447		mmc->caps = mmc->caps | MMC_CAP_AGGRESSIVE_PM;
1448	mmc->caps2 = MMC_CAP2_NO_PRESCAN_POWERUP | MMC_CAP2_FULL_PWR_CYCLE |
1449		MMC_CAP2_NO_SDIO;
1450	mmc->max_current_330 = 400;
1451	mmc->max_current_180 = 800;
1452	mmc->ops = &realtek_pci_sdmmc_ops;
1453
1454	init_extra_caps(host);
1455
1456	mmc->max_segs = 256;
1457	mmc->max_seg_size = 65536;
1458	mmc->max_blk_size = 512;
1459	mmc->max_blk_count = 65535;
1460	mmc->max_req_size = 524288;
1461}
1462
1463static void rtsx_pci_sdmmc_card_event(struct platform_device *pdev)
1464{
1465	struct realtek_pci_sdmmc *host = platform_get_drvdata(pdev);
1466
1467	host->cookie = -1;
1468	mmc_detect_change(host->mmc, 0);
1469}
1470
1471static int rtsx_pci_sdmmc_drv_probe(struct platform_device *pdev)
1472{
1473	struct mmc_host *mmc;
1474	struct realtek_pci_sdmmc *host;
1475	struct rtsx_pcr *pcr;
1476	struct pcr_handle *handle = pdev->dev.platform_data;
1477	int ret;
1478
1479	if (!handle)
1480		return -ENXIO;
1481
1482	pcr = handle->pcr;
1483	if (!pcr)
1484		return -ENXIO;
1485
1486	dev_dbg(&(pdev->dev), ": Realtek PCI-E SDMMC controller found\n");
1487
1488	mmc = mmc_alloc_host(sizeof(*host), &pdev->dev);
1489	if (!mmc)
1490		return -ENOMEM;
1491
1492	host = mmc_priv(mmc);
1493	host->pcr = pcr;
1494	mmc->ios.power_delay_ms = 5;
1495	host->mmc = mmc;
1496	host->pdev = pdev;
1497	host->cookie = -1;
1498	host->prev_power_state = MMC_POWER_OFF;
1499	INIT_WORK(&host->work, sd_request);
1500	platform_set_drvdata(pdev, host);
1501	pcr->slots[RTSX_SD_CARD].p_dev = pdev;
1502	pcr->slots[RTSX_SD_CARD].card_event = rtsx_pci_sdmmc_card_event;
1503
1504	mutex_init(&host->host_mutex);
1505
1506	realtek_init_host(host);
1507
1508	pm_runtime_no_callbacks(&pdev->dev);
1509	pm_runtime_set_active(&pdev->dev);
1510	pm_runtime_enable(&pdev->dev);
1511	pm_runtime_set_autosuspend_delay(&pdev->dev, 200);
1512	pm_runtime_mark_last_busy(&pdev->dev);
1513	pm_runtime_use_autosuspend(&pdev->dev);
1514
1515	ret = mmc_add_host(mmc);
1516	if (ret) {
1517		pm_runtime_dont_use_autosuspend(&pdev->dev);
1518		pm_runtime_disable(&pdev->dev);
1519		mmc_free_host(mmc);
1520		return ret;
1521	}
1522
1523	return 0;
1524}
1525
1526static int rtsx_pci_sdmmc_drv_remove(struct platform_device *pdev)
1527{
1528	struct realtek_pci_sdmmc *host = platform_get_drvdata(pdev);
1529	struct rtsx_pcr *pcr;
1530	struct mmc_host *mmc;
1531
1532	if (!host)
1533		return 0;
1534
1535	pcr = host->pcr;
1536	pcr->slots[RTSX_SD_CARD].p_dev = NULL;
1537	pcr->slots[RTSX_SD_CARD].card_event = NULL;
1538	mmc = host->mmc;
1539
1540	cancel_work_sync(&host->work);
1541
1542	mutex_lock(&host->host_mutex);
1543	if (host->mrq) {
1544		dev_dbg(&(pdev->dev),
1545			"%s: Controller removed during transfer\n",
1546			mmc_hostname(mmc));
1547
1548		rtsx_pci_complete_unfinished_transfer(pcr);
1549
1550		host->mrq->cmd->error = -ENOMEDIUM;
1551		if (host->mrq->stop)
1552			host->mrq->stop->error = -ENOMEDIUM;
1553		mmc_request_done(mmc, host->mrq);
1554	}
1555	mutex_unlock(&host->host_mutex);
1556
1557	mmc_remove_host(mmc);
1558	host->eject = true;
1559
1560	flush_work(&host->work);
1561
1562	pm_runtime_dont_use_autosuspend(&pdev->dev);
1563	pm_runtime_disable(&pdev->dev);
1564
1565	mmc_free_host(mmc);
1566
1567	dev_dbg(&(pdev->dev),
1568		": Realtek PCI-E SDMMC controller has been removed\n");
1569
1570	return 0;
1571}
1572
1573static const struct platform_device_id rtsx_pci_sdmmc_ids[] = {
1574	{
1575		.name = DRV_NAME_RTSX_PCI_SDMMC,
1576	}, {
1577		/* sentinel */
1578	}
1579};
1580MODULE_DEVICE_TABLE(platform, rtsx_pci_sdmmc_ids);
1581
1582static struct platform_driver rtsx_pci_sdmmc_driver = {
1583	.probe		= rtsx_pci_sdmmc_drv_probe,
1584	.remove		= rtsx_pci_sdmmc_drv_remove,
1585	.id_table       = rtsx_pci_sdmmc_ids,
1586	.driver		= {
1587		.name	= DRV_NAME_RTSX_PCI_SDMMC,
1588		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
1589	},
1590};
1591module_platform_driver(rtsx_pci_sdmmc_driver);
1592
1593MODULE_LICENSE("GPL");
1594MODULE_AUTHOR("Wei WANG <wei_wang@realsil.com.cn>");
1595MODULE_DESCRIPTION("Realtek PCI-E SD/MMC Card Host Driver");