1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* Realtek PCI-Express SD/MMC Card Interface driver
3  *
4  * Copyright(c) 2009-2013 Realtek Semiconductor Corp. All rights reserved.
5  *
6  * Author:
7  *   Wei WANG <wei_wang@realsil.com.cn>
8  */
9 
10 #include <linux/module.h>
11 #include <linux/slab.h>
12 #include <linux/highmem.h>
13 #include <linux/delay.h>
14 #include <linux/platform_device.h>
15 #include <linux/workqueue.h>
16 #include <linux/mmc/host.h>
17 #include <linux/mmc/mmc.h>
18 #include <linux/mmc/sd.h>
19 #include <linux/mmc/sdio.h>
20 #include <linux/mmc/card.h>
21 #include <linux/rtsx_pci.h>
22 #include <asm/unaligned.h>
23 #include <linux/pm_runtime.h>
24 
25 struct realtek_pci_sdmmc {
26 	struct platform_device	*pdev;
27 	struct rtsx_pcr		*pcr;
28 	struct mmc_host		*mmc;
29 	struct mmc_request	*mrq;
30 #define SDMMC_WORKQ_NAME	"rtsx_pci_sdmmc_workq"
31 
32 	struct work_struct	work;
33 	struct mutex		host_mutex;
34 
35 	u8			ssc_depth;
36 	unsigned int		clock;
37 	bool			vpclk;
38 	bool			double_clk;
39 	bool			eject;
40 	bool			initial_mode;
41 	int			power_state;
42 #define SDMMC_POWER_ON		1
43 #define SDMMC_POWER_OFF		0
44 
45 	int			sg_count;
46 	s32			cookie;
47 	int			cookie_sg_count;
48 	bool			using_cookie;
49 };
50 
51 static int sdmmc_init_sd_express(struct mmc_host *mmc, struct mmc_ios *ios);
52 
53 static inline struct device *sdmmc_dev(struct realtek_pci_sdmmc *host)
54 {
55 	return &(host->pdev->dev);
56 }
57 
58 static inline void sd_clear_error(struct realtek_pci_sdmmc *host)
59 {
60 	rtsx_pci_write_register(host->pcr, CARD_STOP,
61 			SD_STOP | SD_CLR_ERR, SD_STOP | SD_CLR_ERR);
62 }
63 
64 #ifdef DEBUG
65 static void dump_reg_range(struct realtek_pci_sdmmc *host, u16 start, u16 end)
66 {
67 	u16 len = end - start + 1;
68 	int i;
69 	u8 data[8];
70 
71 	for (i = 0; i < len; i += 8) {
72 		int j;
73 		int n = min(8, len - i);
74 
75 		memset(&data, 0, sizeof(data));
76 		for (j = 0; j < n; j++)
77 			rtsx_pci_read_register(host->pcr, start + i + j,
78 				data + j);
79 		dev_dbg(sdmmc_dev(host), "0x%04X(%d): %8ph\n",
80 			start + i, n, data);
81 	}
82 }
83 
84 static void sd_print_debug_regs(struct realtek_pci_sdmmc *host)
85 {
86 	dump_reg_range(host, 0xFDA0, 0xFDB3);
87 	dump_reg_range(host, 0xFD52, 0xFD69);
88 }
89 #else
90 #define sd_print_debug_regs(host)
91 #endif /* DEBUG */
92 
93 static inline int sd_get_cd_int(struct realtek_pci_sdmmc *host)
94 {
95 	return rtsx_pci_readl(host->pcr, RTSX_BIPR) & SD_EXIST;
96 }
97 
98 static void sd_cmd_set_sd_cmd(struct rtsx_pcr *pcr, struct mmc_command *cmd)
99 {
100 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CMD0, 0xFF,
101 		SD_CMD_START | cmd->opcode);
102 	rtsx_pci_write_be32(pcr, SD_CMD1, cmd->arg);
103 }
104 
105 static void sd_cmd_set_data_len(struct rtsx_pcr *pcr, u16 blocks, u16 blksz)
106 {
107 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BLOCK_CNT_L, 0xFF, blocks);
108 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BLOCK_CNT_H, 0xFF, blocks >> 8);
109 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BYTE_CNT_L, 0xFF, blksz);
110 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BYTE_CNT_H, 0xFF, blksz >> 8);
111 }
112 
113 static int sd_response_type(struct mmc_command *cmd)
114 {
115 	switch (mmc_resp_type(cmd)) {
116 	case MMC_RSP_NONE:
117 		return SD_RSP_TYPE_R0;
118 	case MMC_RSP_R1:
119 		return SD_RSP_TYPE_R1;
120 	case MMC_RSP_R1_NO_CRC:
121 		return SD_RSP_TYPE_R1 | SD_NO_CHECK_CRC7;
122 	case MMC_RSP_R1B:
123 		return SD_RSP_TYPE_R1b;
124 	case MMC_RSP_R2:
125 		return SD_RSP_TYPE_R2;
126 	case MMC_RSP_R3:
127 		return SD_RSP_TYPE_R3;
128 	default:
129 		return -EINVAL;
130 	}
131 }
132 
133 static int sd_status_index(int resp_type)
134 {
135 	if (resp_type == SD_RSP_TYPE_R0)
136 		return 0;
137 	else if (resp_type == SD_RSP_TYPE_R2)
138 		return 16;
139 
140 	return 5;
141 }
142 /*
143  * sd_pre_dma_transfer - do dma_map_sg() or using cookie
144  *
145  * @pre: if called in pre_req()
146  * return:
147  *	0 - do dma_map_sg()
148  *	1 - using cookie
149  */
150 static int sd_pre_dma_transfer(struct realtek_pci_sdmmc *host,
151 		struct mmc_data *data, bool pre)
152 {
153 	struct rtsx_pcr *pcr = host->pcr;
154 	int read = data->flags & MMC_DATA_READ;
155 	int count = 0;
156 	int using_cookie = 0;
157 
158 	if (!pre && data->host_cookie && data->host_cookie != host->cookie) {
159 		dev_err(sdmmc_dev(host),
160 			"error: data->host_cookie = %d, host->cookie = %d\n",
161 			data->host_cookie, host->cookie);
162 		data->host_cookie = 0;
163 	}
164 
165 	if (pre || data->host_cookie != host->cookie) {
166 		count = rtsx_pci_dma_map_sg(pcr, data->sg, data->sg_len, read);
167 	} else {
168 		count = host->cookie_sg_count;
169 		using_cookie = 1;
170 	}
171 
172 	if (pre) {
173 		host->cookie_sg_count = count;
174 		if (++host->cookie < 0)
175 			host->cookie = 1;
176 		data->host_cookie = host->cookie;
177 	} else {
178 		host->sg_count = count;
179 	}
180 
181 	return using_cookie;
182 }
183 
184 static void sdmmc_pre_req(struct mmc_host *mmc, struct mmc_request *mrq)
185 {
186 	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
187 	struct mmc_data *data = mrq->data;
188 
189 	if (data->host_cookie) {
190 		dev_err(sdmmc_dev(host),
191 			"error: reset data->host_cookie = %d\n",
192 			data->host_cookie);
193 		data->host_cookie = 0;
194 	}
195 
196 	sd_pre_dma_transfer(host, data, true);
197 	dev_dbg(sdmmc_dev(host), "pre dma sg: %d\n", host->cookie_sg_count);
198 }
199 
200 static void sdmmc_post_req(struct mmc_host *mmc, struct mmc_request *mrq,
201 		int err)
202 {
203 	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
204 	struct rtsx_pcr *pcr = host->pcr;
205 	struct mmc_data *data = mrq->data;
206 	int read = data->flags & MMC_DATA_READ;
207 
208 	rtsx_pci_dma_unmap_sg(pcr, data->sg, data->sg_len, read);
209 	data->host_cookie = 0;
210 }
211 
212 static void sd_send_cmd_get_rsp(struct realtek_pci_sdmmc *host,
213 		struct mmc_command *cmd)
214 {
215 	struct rtsx_pcr *pcr = host->pcr;
216 	u8 cmd_idx = (u8)cmd->opcode;
217 	u32 arg = cmd->arg;
218 	int err = 0;
219 	int timeout = 100;
220 	int i;
221 	u8 *ptr;
222 	int rsp_type;
223 	int stat_idx;
224 	bool clock_toggled = false;
225 
226 	dev_dbg(sdmmc_dev(host), "%s: SD/MMC CMD %d, arg = 0x%08x\n",
227 			__func__, cmd_idx, arg);
228 
229 	rsp_type = sd_response_type(cmd);
230 	if (rsp_type < 0)
231 		goto out;
232 
233 	stat_idx = sd_status_index(rsp_type);
234 
235 	if (rsp_type == SD_RSP_TYPE_R1b)
236 		timeout = cmd->busy_timeout ? cmd->busy_timeout : 3000;
237 
238 	if (cmd->opcode == SD_SWITCH_VOLTAGE) {
239 		err = rtsx_pci_write_register(pcr, SD_BUS_STAT,
240 				0xFF, SD_CLK_TOGGLE_EN);
241 		if (err < 0)
242 			goto out;
243 
244 		clock_toggled = true;
245 	}
246 
247 	rtsx_pci_init_cmd(pcr);
248 	sd_cmd_set_sd_cmd(pcr, cmd);
249 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG2, 0xFF, rsp_type);
250 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_DATA_SOURCE,
251 			0x01, PINGPONG_BUFFER);
252 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_TRANSFER,
253 			0xFF, SD_TM_CMD_RSP | SD_TRANSFER_START);
254 	rtsx_pci_add_cmd(pcr, CHECK_REG_CMD, SD_TRANSFER,
255 		     SD_TRANSFER_END | SD_STAT_IDLE,
256 		     SD_TRANSFER_END | SD_STAT_IDLE);
257 
258 	if (rsp_type == SD_RSP_TYPE_R2) {
259 		/* Read data from ping-pong buffer */
260 		for (i = PPBUF_BASE2; i < PPBUF_BASE2 + 16; i++)
261 			rtsx_pci_add_cmd(pcr, READ_REG_CMD, (u16)i, 0, 0);
262 	} else if (rsp_type != SD_RSP_TYPE_R0) {
263 		/* Read data from SD_CMDx registers */
264 		for (i = SD_CMD0; i <= SD_CMD4; i++)
265 			rtsx_pci_add_cmd(pcr, READ_REG_CMD, (u16)i, 0, 0);
266 	}
267 
268 	rtsx_pci_add_cmd(pcr, READ_REG_CMD, SD_STAT1, 0, 0);
269 
270 	err = rtsx_pci_send_cmd(pcr, timeout);
271 	if (err < 0) {
272 		sd_print_debug_regs(host);
273 		sd_clear_error(host);
274 		dev_dbg(sdmmc_dev(host),
275 			"rtsx_pci_send_cmd error (err = %d)\n", err);
276 		goto out;
277 	}
278 
279 	if (rsp_type == SD_RSP_TYPE_R0) {
280 		err = 0;
281 		goto out;
282 	}
283 
284 	/* Eliminate returned value of CHECK_REG_CMD */
285 	ptr = rtsx_pci_get_cmd_data(pcr) + 1;
286 
287 	/* Check (Start,Transmission) bit of Response */
288 	if ((ptr[0] & 0xC0) != 0) {
289 		err = -EILSEQ;
290 		dev_dbg(sdmmc_dev(host), "Invalid response bit\n");
291 		goto out;
292 	}
293 
294 	/* Check CRC7 */
295 	if (!(rsp_type & SD_NO_CHECK_CRC7)) {
296 		if (ptr[stat_idx] & SD_CRC7_ERR) {
297 			err = -EILSEQ;
298 			dev_dbg(sdmmc_dev(host), "CRC7 error\n");
299 			goto out;
300 		}
301 	}
302 
303 	if (rsp_type == SD_RSP_TYPE_R2) {
304 		/*
305 		 * The controller offloads the last byte {CRC-7, end bit 1'b1}
306 		 * of response type R2. Assign dummy CRC, 0, and end bit to the
307 		 * byte(ptr[16], goes into the LSB of resp[3] later).
308 		 */
309 		ptr[16] = 1;
310 
311 		for (i = 0; i < 4; i++) {
312 			cmd->resp[i] = get_unaligned_be32(ptr + 1 + i * 4);
313 			dev_dbg(sdmmc_dev(host), "cmd->resp[%d] = 0x%08x\n",
314 					i, cmd->resp[i]);
315 		}
316 	} else {
317 		cmd->resp[0] = get_unaligned_be32(ptr + 1);
318 		dev_dbg(sdmmc_dev(host), "cmd->resp[0] = 0x%08x\n",
319 				cmd->resp[0]);
320 	}
321 
322 out:
323 	cmd->error = err;
324 
325 	if (err && clock_toggled)
326 		rtsx_pci_write_register(pcr, SD_BUS_STAT,
327 				SD_CLK_TOGGLE_EN | SD_CLK_FORCE_STOP, 0);
328 }
329 
330 static int sd_read_data(struct realtek_pci_sdmmc *host, struct mmc_command *cmd,
331 	u16 byte_cnt, u8 *buf, int buf_len, int timeout)
332 {
333 	struct rtsx_pcr *pcr = host->pcr;
334 	int err;
335 	u8 trans_mode;
336 
337 	dev_dbg(sdmmc_dev(host), "%s: SD/MMC CMD %d, arg = 0x%08x\n",
338 		__func__, cmd->opcode, cmd->arg);
339 
340 	if (!buf)
341 		buf_len = 0;
342 
343 	if (cmd->opcode == MMC_SEND_TUNING_BLOCK)
344 		trans_mode = SD_TM_AUTO_TUNING;
345 	else
346 		trans_mode = SD_TM_NORMAL_READ;
347 
348 	rtsx_pci_init_cmd(pcr);
349 	sd_cmd_set_sd_cmd(pcr, cmd);
350 	sd_cmd_set_data_len(pcr, 1, byte_cnt);
351 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG2, 0xFF,
352 			SD_CALCULATE_CRC7 | SD_CHECK_CRC16 |
353 			SD_NO_WAIT_BUSY_END | SD_CHECK_CRC7 | SD_RSP_LEN_6);
354 	if (trans_mode != SD_TM_AUTO_TUNING)
355 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD,
356 				CARD_DATA_SOURCE, 0x01, PINGPONG_BUFFER);
357 
358 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_TRANSFER,
359 			0xFF, trans_mode | SD_TRANSFER_START);
360 	rtsx_pci_add_cmd(pcr, CHECK_REG_CMD, SD_TRANSFER,
361 			SD_TRANSFER_END, SD_TRANSFER_END);
362 
363 	err = rtsx_pci_send_cmd(pcr, timeout);
364 	if (err < 0) {
365 		sd_print_debug_regs(host);
366 		dev_dbg(sdmmc_dev(host),
367 			"rtsx_pci_send_cmd fail (err = %d)\n", err);
368 		return err;
369 	}
370 
371 	if (buf && buf_len) {
372 		err = rtsx_pci_read_ppbuf(pcr, buf, buf_len);
373 		if (err < 0) {
374 			dev_dbg(sdmmc_dev(host),
375 				"rtsx_pci_read_ppbuf fail (err = %d)\n", err);
376 			return err;
377 		}
378 	}
379 
380 	return 0;
381 }
382 
383 static int sd_write_data(struct realtek_pci_sdmmc *host,
384 	struct mmc_command *cmd, u16 byte_cnt, u8 *buf, int buf_len,
385 	int timeout)
386 {
387 	struct rtsx_pcr *pcr = host->pcr;
388 	int err;
389 
390 	dev_dbg(sdmmc_dev(host), "%s: SD/MMC CMD %d, arg = 0x%08x\n",
391 		__func__, cmd->opcode, cmd->arg);
392 
393 	if (!buf)
394 		buf_len = 0;
395 
396 	sd_send_cmd_get_rsp(host, cmd);
397 	if (cmd->error)
398 		return cmd->error;
399 
400 	if (buf && buf_len) {
401 		err = rtsx_pci_write_ppbuf(pcr, buf, buf_len);
402 		if (err < 0) {
403 			dev_dbg(sdmmc_dev(host),
404 				"rtsx_pci_write_ppbuf fail (err = %d)\n", err);
405 			return err;
406 		}
407 	}
408 
409 	rtsx_pci_init_cmd(pcr);
410 	sd_cmd_set_data_len(pcr, 1, byte_cnt);
411 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG2, 0xFF,
412 		SD_CALCULATE_CRC7 | SD_CHECK_CRC16 |
413 		SD_NO_WAIT_BUSY_END | SD_CHECK_CRC7 | SD_RSP_LEN_0);
414 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_TRANSFER, 0xFF,
415 			SD_TRANSFER_START | SD_TM_AUTO_WRITE_3);
416 	rtsx_pci_add_cmd(pcr, CHECK_REG_CMD, SD_TRANSFER,
417 			SD_TRANSFER_END, SD_TRANSFER_END);
418 
419 	err = rtsx_pci_send_cmd(pcr, timeout);
420 	if (err < 0) {
421 		sd_print_debug_regs(host);
422 		dev_dbg(sdmmc_dev(host),
423 			"rtsx_pci_send_cmd fail (err = %d)\n", err);
424 		return err;
425 	}
426 
427 	return 0;
428 }
429 
430 static int sd_read_long_data(struct realtek_pci_sdmmc *host,
431 	struct mmc_request *mrq)
432 {
433 	struct rtsx_pcr *pcr = host->pcr;
434 	struct mmc_host *mmc = host->mmc;
435 	struct mmc_card *card = mmc->card;
436 	struct mmc_command *cmd = mrq->cmd;
437 	struct mmc_data *data = mrq->data;
438 	int uhs = mmc_card_uhs(card);
439 	u8 cfg2 = 0;
440 	int err;
441 	int resp_type;
442 	size_t data_len = data->blksz * data->blocks;
443 
444 	dev_dbg(sdmmc_dev(host), "%s: SD/MMC CMD %d, arg = 0x%08x\n",
445 		__func__, cmd->opcode, cmd->arg);
446 
447 	resp_type = sd_response_type(cmd);
448 	if (resp_type < 0)
449 		return resp_type;
450 
451 	if (!uhs)
452 		cfg2 |= SD_NO_CHECK_WAIT_CRC_TO;
453 
454 	rtsx_pci_init_cmd(pcr);
455 	sd_cmd_set_sd_cmd(pcr, cmd);
456 	sd_cmd_set_data_len(pcr, data->blocks, data->blksz);
457 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, IRQSTAT0,
458 			DMA_DONE_INT, DMA_DONE_INT);
459 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC3,
460 		0xFF, (u8)(data_len >> 24));
461 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC2,
462 		0xFF, (u8)(data_len >> 16));
463 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC1,
464 		0xFF, (u8)(data_len >> 8));
465 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC0, 0xFF, (u8)data_len);
466 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMACTL,
467 		0x03 | DMA_PACK_SIZE_MASK,
468 		DMA_DIR_FROM_CARD | DMA_EN | DMA_512);
469 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_DATA_SOURCE,
470 			0x01, RING_BUFFER);
471 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG2, 0xFF, cfg2 | resp_type);
472 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_TRANSFER, 0xFF,
473 			SD_TRANSFER_START | SD_TM_AUTO_READ_2);
474 	rtsx_pci_add_cmd(pcr, CHECK_REG_CMD, SD_TRANSFER,
475 			SD_TRANSFER_END, SD_TRANSFER_END);
476 	rtsx_pci_send_cmd_no_wait(pcr);
477 
478 	err = rtsx_pci_dma_transfer(pcr, data->sg, host->sg_count, 1, 10000);
479 	if (err < 0) {
480 		sd_print_debug_regs(host);
481 		sd_clear_error(host);
482 		return err;
483 	}
484 
485 	return 0;
486 }
487 
488 static int sd_write_long_data(struct realtek_pci_sdmmc *host,
489 	struct mmc_request *mrq)
490 {
491 	struct rtsx_pcr *pcr = host->pcr;
492 	struct mmc_host *mmc = host->mmc;
493 	struct mmc_card *card = mmc->card;
494 	struct mmc_command *cmd = mrq->cmd;
495 	struct mmc_data *data = mrq->data;
496 	int uhs = mmc_card_uhs(card);
497 	u8 cfg2;
498 	int err;
499 	size_t data_len = data->blksz * data->blocks;
500 
501 	sd_send_cmd_get_rsp(host, cmd);
502 	if (cmd->error)
503 		return cmd->error;
504 
505 	dev_dbg(sdmmc_dev(host), "%s: SD/MMC CMD %d, arg = 0x%08x\n",
506 		__func__, cmd->opcode, cmd->arg);
507 
508 	cfg2 = SD_NO_CALCULATE_CRC7 | SD_CHECK_CRC16 |
509 		SD_NO_WAIT_BUSY_END | SD_NO_CHECK_CRC7 | SD_RSP_LEN_0;
510 
511 	if (!uhs)
512 		cfg2 |= SD_NO_CHECK_WAIT_CRC_TO;
513 
514 	rtsx_pci_init_cmd(pcr);
515 	sd_cmd_set_data_len(pcr, data->blocks, data->blksz);
516 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, IRQSTAT0,
517 			DMA_DONE_INT, DMA_DONE_INT);
518 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC3,
519 		0xFF, (u8)(data_len >> 24));
520 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC2,
521 		0xFF, (u8)(data_len >> 16));
522 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC1,
523 		0xFF, (u8)(data_len >> 8));
524 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC0, 0xFF, (u8)data_len);
525 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMACTL,
526 		0x03 | DMA_PACK_SIZE_MASK,
527 		DMA_DIR_TO_CARD | DMA_EN | DMA_512);
528 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_DATA_SOURCE,
529 			0x01, RING_BUFFER);
530 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG2, 0xFF, cfg2);
531 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_TRANSFER, 0xFF,
532 			SD_TRANSFER_START | SD_TM_AUTO_WRITE_3);
533 	rtsx_pci_add_cmd(pcr, CHECK_REG_CMD, SD_TRANSFER,
534 			SD_TRANSFER_END, SD_TRANSFER_END);
535 	rtsx_pci_send_cmd_no_wait(pcr);
536 	err = rtsx_pci_dma_transfer(pcr, data->sg, host->sg_count, 0, 10000);
537 	if (err < 0) {
538 		sd_clear_error(host);
539 		return err;
540 	}
541 
542 	return 0;
543 }
544 
545 static int sd_rw_multi(struct realtek_pci_sdmmc *host, struct mmc_request *mrq)
546 {
547 	struct mmc_data *data = mrq->data;
548 
549 	if (host->sg_count < 0) {
550 		data->error = host->sg_count;
551 		dev_dbg(sdmmc_dev(host), "%s: sg_count = %d is invalid\n",
552 			__func__, host->sg_count);
553 		return data->error;
554 	}
555 
556 	if (data->flags & MMC_DATA_READ)
557 		return sd_read_long_data(host, mrq);
558 
559 	return sd_write_long_data(host, mrq);
560 }
561 
562 static inline void sd_enable_initial_mode(struct realtek_pci_sdmmc *host)
563 {
564 	rtsx_pci_write_register(host->pcr, SD_CFG1,
565 			SD_CLK_DIVIDE_MASK, SD_CLK_DIVIDE_128);
566 }
567 
568 static inline void sd_disable_initial_mode(struct realtek_pci_sdmmc *host)
569 {
570 	rtsx_pci_write_register(host->pcr, SD_CFG1,
571 			SD_CLK_DIVIDE_MASK, SD_CLK_DIVIDE_0);
572 }
573 
574 static void sd_normal_rw(struct realtek_pci_sdmmc *host,
575 		struct mmc_request *mrq)
576 {
577 	struct mmc_command *cmd = mrq->cmd;
578 	struct mmc_data *data = mrq->data;
579 	u8 *buf;
580 
581 	buf = kzalloc(data->blksz, GFP_NOIO);
582 	if (!buf) {
583 		cmd->error = -ENOMEM;
584 		return;
585 	}
586 
587 	if (data->flags & MMC_DATA_READ) {
588 		if (host->initial_mode)
589 			sd_disable_initial_mode(host);
590 
591 		cmd->error = sd_read_data(host, cmd, (u16)data->blksz, buf,
592 				data->blksz, 200);
593 
594 		if (host->initial_mode)
595 			sd_enable_initial_mode(host);
596 
597 		sg_copy_from_buffer(data->sg, data->sg_len, buf, data->blksz);
598 	} else {
599 		sg_copy_to_buffer(data->sg, data->sg_len, buf, data->blksz);
600 
601 		cmd->error = sd_write_data(host, cmd, (u16)data->blksz, buf,
602 				data->blksz, 200);
603 	}
604 
605 	kfree(buf);
606 }
607 
608 static int sd_change_phase(struct realtek_pci_sdmmc *host,
609 		u8 sample_point, bool rx)
610 {
611 	struct rtsx_pcr *pcr = host->pcr;
612 	u16 SD_VP_CTL = 0;
613 	dev_dbg(sdmmc_dev(host), "%s(%s): sample_point = %d\n",
614 			__func__, rx ? "RX" : "TX", sample_point);
615 
616 	rtsx_pci_write_register(pcr, CLK_CTL, CHANGE_CLK, CHANGE_CLK);
617 	if (rx) {
618 		SD_VP_CTL = SD_VPRX_CTL;
619 		rtsx_pci_write_register(pcr, SD_VPRX_CTL,
620 			PHASE_SELECT_MASK, sample_point);
621 	} else {
622 		SD_VP_CTL = SD_VPTX_CTL;
623 		rtsx_pci_write_register(pcr, SD_VPTX_CTL,
624 			PHASE_SELECT_MASK, sample_point);
625 	}
626 	rtsx_pci_write_register(pcr, SD_VP_CTL, PHASE_NOT_RESET, 0);
627 	rtsx_pci_write_register(pcr, SD_VP_CTL, PHASE_NOT_RESET,
628 				PHASE_NOT_RESET);
629 	rtsx_pci_write_register(pcr, CLK_CTL, CHANGE_CLK, 0);
630 	rtsx_pci_write_register(pcr, SD_CFG1, SD_ASYNC_FIFO_NOT_RST, 0);
631 
632 	return 0;
633 }
634 
635 static inline u32 test_phase_bit(u32 phase_map, unsigned int bit)
636 {
637 	bit %= RTSX_PHASE_MAX;
638 	return phase_map & (1 << bit);
639 }
640 
641 static int sd_get_phase_len(u32 phase_map, unsigned int start_bit)
642 {
643 	int i;
644 
645 	for (i = 0; i < RTSX_PHASE_MAX; i++) {
646 		if (test_phase_bit(phase_map, start_bit + i) == 0)
647 			return i;
648 	}
649 	return RTSX_PHASE_MAX;
650 }
651 
652 static u8 sd_search_final_phase(struct realtek_pci_sdmmc *host, u32 phase_map)
653 {
654 	int start = 0, len = 0;
655 	int start_final = 0, len_final = 0;
656 	u8 final_phase = 0xFF;
657 
658 	if (phase_map == 0) {
659 		dev_err(sdmmc_dev(host), "phase error: [map:%x]\n", phase_map);
660 		return final_phase;
661 	}
662 
663 	while (start < RTSX_PHASE_MAX) {
664 		len = sd_get_phase_len(phase_map, start);
665 		if (len_final < len) {
666 			start_final = start;
667 			len_final = len;
668 		}
669 		start += len ? len : 1;
670 	}
671 
672 	final_phase = (start_final + len_final / 2) % RTSX_PHASE_MAX;
673 	dev_dbg(sdmmc_dev(host), "phase: [map:%x] [maxlen:%d] [final:%d]\n",
674 		phase_map, len_final, final_phase);
675 
676 	return final_phase;
677 }
678 
679 static void sd_wait_data_idle(struct realtek_pci_sdmmc *host)
680 {
681 	int i;
682 	u8 val = 0;
683 
684 	for (i = 0; i < 100; i++) {
685 		rtsx_pci_read_register(host->pcr, SD_DATA_STATE, &val);
686 		if (val & SD_DATA_IDLE)
687 			return;
688 
689 		udelay(100);
690 	}
691 }
692 
693 static int sd_tuning_rx_cmd(struct realtek_pci_sdmmc *host,
694 		u8 opcode, u8 sample_point)
695 {
696 	int err;
697 	struct mmc_command cmd = {};
698 	struct rtsx_pcr *pcr = host->pcr;
699 
700 	sd_change_phase(host, sample_point, true);
701 
702 	rtsx_pci_write_register(pcr, SD_CFG3, SD_RSP_80CLK_TIMEOUT_EN,
703 		SD_RSP_80CLK_TIMEOUT_EN);
704 
705 	cmd.opcode = opcode;
706 	err = sd_read_data(host, &cmd, 0x40, NULL, 0, 100);
707 	if (err < 0) {
708 		/* Wait till SD DATA IDLE */
709 		sd_wait_data_idle(host);
710 		sd_clear_error(host);
711 		rtsx_pci_write_register(pcr, SD_CFG3,
712 			SD_RSP_80CLK_TIMEOUT_EN, 0);
713 		return err;
714 	}
715 
716 	rtsx_pci_write_register(pcr, SD_CFG3, SD_RSP_80CLK_TIMEOUT_EN, 0);
717 	return 0;
718 }
719 
720 static int sd_tuning_phase(struct realtek_pci_sdmmc *host,
721 		u8 opcode, u32 *phase_map)
722 {
723 	int err, i;
724 	u32 raw_phase_map = 0;
725 
726 	for (i = 0; i < RTSX_PHASE_MAX; i++) {
727 		err = sd_tuning_rx_cmd(host, opcode, (u8)i);
728 		if (err == 0)
729 			raw_phase_map |= 1 << i;
730 	}
731 
732 	if (phase_map)
733 		*phase_map = raw_phase_map;
734 
735 	return 0;
736 }
737 
738 static int sd_tuning_rx(struct realtek_pci_sdmmc *host, u8 opcode)
739 {
740 	int err, i;
741 	u32 raw_phase_map[RX_TUNING_CNT] = {0}, phase_map;
742 	u8 final_phase;
743 
744 	for (i = 0; i < RX_TUNING_CNT; i++) {
745 		err = sd_tuning_phase(host, opcode, &(raw_phase_map[i]));
746 		if (err < 0)
747 			return err;
748 
749 		if (raw_phase_map[i] == 0)
750 			break;
751 	}
752 
753 	phase_map = 0xFFFFFFFF;
754 	for (i = 0; i < RX_TUNING_CNT; i++) {
755 		dev_dbg(sdmmc_dev(host), "RX raw_phase_map[%d] = 0x%08x\n",
756 				i, raw_phase_map[i]);
757 		phase_map &= raw_phase_map[i];
758 	}
759 	dev_dbg(sdmmc_dev(host), "RX phase_map = 0x%08x\n", phase_map);
760 
761 	if (phase_map) {
762 		final_phase = sd_search_final_phase(host, phase_map);
763 		if (final_phase == 0xFF)
764 			return -EINVAL;
765 
766 		err = sd_change_phase(host, final_phase, true);
767 		if (err < 0)
768 			return err;
769 	} else {
770 		return -EINVAL;
771 	}
772 
773 	return 0;
774 }
775 
776 static inline int sdio_extblock_cmd(struct mmc_command *cmd,
777 	struct mmc_data *data)
778 {
779 	return (cmd->opcode == SD_IO_RW_EXTENDED) && (data->blksz == 512);
780 }
781 
782 static inline int sd_rw_cmd(struct mmc_command *cmd)
783 {
784 	return mmc_op_multi(cmd->opcode) ||
785 		(cmd->opcode == MMC_READ_SINGLE_BLOCK) ||
786 		(cmd->opcode == MMC_WRITE_BLOCK);
787 }
788 
789 static void sd_request(struct work_struct *work)
790 {
791 	struct realtek_pci_sdmmc *host = container_of(work,
792 			struct realtek_pci_sdmmc, work);
793 	struct rtsx_pcr *pcr = host->pcr;
794 
795 	struct mmc_host *mmc = host->mmc;
796 	struct mmc_request *mrq = host->mrq;
797 	struct mmc_command *cmd = mrq->cmd;
798 	struct mmc_data *data = mrq->data;
799 
800 	unsigned int data_size = 0;
801 	int err;
802 
803 	if (host->eject || !sd_get_cd_int(host)) {
804 		cmd->error = -ENOMEDIUM;
805 		goto finish;
806 	}
807 
808 	err = rtsx_pci_card_exclusive_check(host->pcr, RTSX_SD_CARD);
809 	if (err) {
810 		cmd->error = err;
811 		goto finish;
812 	}
813 
814 	mutex_lock(&pcr->pcr_mutex);
815 
816 	rtsx_pci_start_run(pcr);
817 
818 	rtsx_pci_switch_clock(pcr, host->clock, host->ssc_depth,
819 			host->initial_mode, host->double_clk, host->vpclk);
820 	rtsx_pci_write_register(pcr, CARD_SELECT, 0x07, SD_MOD_SEL);
821 	rtsx_pci_write_register(pcr, CARD_SHARE_MODE,
822 			CARD_SHARE_MASK, CARD_SHARE_48_SD);
823 
824 	mutex_lock(&host->host_mutex);
825 	host->mrq = mrq;
826 	mutex_unlock(&host->host_mutex);
827 
828 	if (mrq->data)
829 		data_size = data->blocks * data->blksz;
830 
831 	if (!data_size) {
832 		sd_send_cmd_get_rsp(host, cmd);
833 	} else if (sd_rw_cmd(cmd) || sdio_extblock_cmd(cmd, data)) {
834 		cmd->error = sd_rw_multi(host, mrq);
835 		if (!host->using_cookie)
836 			sdmmc_post_req(host->mmc, host->mrq, 0);
837 
838 		if (mmc_op_multi(cmd->opcode) && mrq->stop)
839 			sd_send_cmd_get_rsp(host, mrq->stop);
840 	} else {
841 		sd_normal_rw(host, mrq);
842 	}
843 
844 	if (mrq->data) {
845 		if (cmd->error || data->error)
846 			data->bytes_xfered = 0;
847 		else
848 			data->bytes_xfered = data->blocks * data->blksz;
849 	}
850 
851 	mutex_unlock(&pcr->pcr_mutex);
852 
853 finish:
854 	if (cmd->error) {
855 		dev_dbg(sdmmc_dev(host), "CMD %d 0x%08x error(%d)\n",
856 			cmd->opcode, cmd->arg, cmd->error);
857 	}
858 
859 	mutex_lock(&host->host_mutex);
860 	host->mrq = NULL;
861 	mutex_unlock(&host->host_mutex);
862 
863 	mmc_request_done(mmc, mrq);
864 }
865 
866 static void sdmmc_request(struct mmc_host *mmc, struct mmc_request *mrq)
867 {
868 	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
869 	struct mmc_data *data = mrq->data;
870 
871 	mutex_lock(&host->host_mutex);
872 	host->mrq = mrq;
873 	mutex_unlock(&host->host_mutex);
874 
875 	if (sd_rw_cmd(mrq->cmd) || sdio_extblock_cmd(mrq->cmd, data))
876 		host->using_cookie = sd_pre_dma_transfer(host, data, false);
877 
878 	schedule_work(&host->work);
879 }
880 
881 static int sd_set_bus_width(struct realtek_pci_sdmmc *host,
882 		unsigned char bus_width)
883 {
884 	int err = 0;
885 	u8 width[] = {
886 		[MMC_BUS_WIDTH_1] = SD_BUS_WIDTH_1BIT,
887 		[MMC_BUS_WIDTH_4] = SD_BUS_WIDTH_4BIT,
888 		[MMC_BUS_WIDTH_8] = SD_BUS_WIDTH_8BIT,
889 	};
890 
891 	if (bus_width <= MMC_BUS_WIDTH_8)
892 		err = rtsx_pci_write_register(host->pcr, SD_CFG1,
893 				0x03, width[bus_width]);
894 
895 	return err;
896 }
897 
898 static int sd_power_on(struct realtek_pci_sdmmc *host)
899 {
900 	struct rtsx_pcr *pcr = host->pcr;
901 	struct mmc_host *mmc = host->mmc;
902 	int err;
903 	u32 val;
904 	u8 test_mode;
905 
906 	if (host->power_state == SDMMC_POWER_ON)
907 		return 0;
908 
909 	msleep(100);
910 
911 	rtsx_pci_init_cmd(pcr);
912 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_SELECT, 0x07, SD_MOD_SEL);
913 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_SHARE_MODE,
914 			CARD_SHARE_MASK, CARD_SHARE_48_SD);
915 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_EN,
916 			SD_CLK_EN, SD_CLK_EN);
917 	err = rtsx_pci_send_cmd(pcr, 100);
918 	if (err < 0)
919 		return err;
920 
921 	err = rtsx_pci_card_pull_ctl_enable(pcr, RTSX_SD_CARD);
922 	if (err < 0)
923 		return err;
924 
925 	err = rtsx_pci_card_power_on(pcr, RTSX_SD_CARD);
926 	if (err < 0)
927 		return err;
928 
929 	err = rtsx_pci_write_register(pcr, CARD_OE, SD_OUTPUT_EN, SD_OUTPUT_EN);
930 	if (err < 0)
931 		return err;
932 
933 	if (PCI_PID(pcr) == PID_5261) {
934 		/*
935 		 * If test mode is set switch to SD Express mandatorily,
936 		 * this is only for factory testing.
937 		 */
938 		rtsx_pci_read_register(pcr, RTS5261_FW_CFG_INFO0, &test_mode);
939 		if (test_mode & RTS5261_FW_EXPRESS_TEST_MASK) {
940 			sdmmc_init_sd_express(mmc, NULL);
941 			return 0;
942 		}
943 		if (pcr->extra_caps & EXTRA_CAPS_SD_EXPRESS)
944 			mmc->caps2 |= MMC_CAP2_SD_EXP | MMC_CAP2_SD_EXP_1_2V;
945 		/*
946 		 * HW read wp status when resuming from S3/S4,
947 		 * and then picks SD legacy interface if it's set
948 		 * in read-only mode.
949 		 */
950 		val = rtsx_pci_readl(pcr, RTSX_BIPR);
951 		if (val & SD_WRITE_PROTECT) {
952 			pcr->extra_caps &= ~EXTRA_CAPS_SD_EXPRESS;
953 			mmc->caps2 &= ~(MMC_CAP2_SD_EXP | MMC_CAP2_SD_EXP_1_2V);
954 		}
955 	}
956 
957 	host->power_state = SDMMC_POWER_ON;
958 	return 0;
959 }
960 
961 static int sd_power_off(struct realtek_pci_sdmmc *host)
962 {
963 	struct rtsx_pcr *pcr = host->pcr;
964 	int err;
965 
966 	host->power_state = SDMMC_POWER_OFF;
967 
968 	rtsx_pci_init_cmd(pcr);
969 
970 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_EN, SD_CLK_EN, 0);
971 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_OE, SD_OUTPUT_EN, 0);
972 
973 	err = rtsx_pci_send_cmd(pcr, 100);
974 	if (err < 0)
975 		return err;
976 
977 	err = rtsx_pci_card_power_off(pcr, RTSX_SD_CARD);
978 	if (err < 0)
979 		return err;
980 
981 	return rtsx_pci_card_pull_ctl_disable(pcr, RTSX_SD_CARD);
982 }
983 
984 static int sd_set_power_mode(struct realtek_pci_sdmmc *host,
985 		unsigned char power_mode)
986 {
987 	int err;
988 
989 	if (power_mode == MMC_POWER_OFF)
990 		err = sd_power_off(host);
991 	else
992 		err = sd_power_on(host);
993 
994 	return err;
995 }
996 
997 static int sd_set_timing(struct realtek_pci_sdmmc *host, unsigned char timing)
998 {
999 	struct rtsx_pcr *pcr = host->pcr;
1000 	int err = 0;
1001 
1002 	rtsx_pci_init_cmd(pcr);
1003 
1004 	switch (timing) {
1005 	case MMC_TIMING_UHS_SDR104:
1006 	case MMC_TIMING_UHS_SDR50:
1007 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG1,
1008 				0x0C | SD_ASYNC_FIFO_NOT_RST,
1009 				SD_30_MODE | SD_ASYNC_FIFO_NOT_RST);
1010 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL,
1011 				CLK_LOW_FREQ, CLK_LOW_FREQ);
1012 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_SOURCE, 0xFF,
1013 				CRC_VAR_CLK0 | SD30_FIX_CLK | SAMPLE_VAR_CLK1);
1014 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL, CLK_LOW_FREQ, 0);
1015 		break;
1016 
1017 	case MMC_TIMING_MMC_DDR52:
1018 	case MMC_TIMING_UHS_DDR50:
1019 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG1,
1020 				0x0C | SD_ASYNC_FIFO_NOT_RST,
1021 				SD_DDR_MODE | SD_ASYNC_FIFO_NOT_RST);
1022 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL,
1023 				CLK_LOW_FREQ, CLK_LOW_FREQ);
1024 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_SOURCE, 0xFF,
1025 				CRC_VAR_CLK0 | SD30_FIX_CLK | SAMPLE_VAR_CLK1);
1026 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL, CLK_LOW_FREQ, 0);
1027 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_PUSH_POINT_CTL,
1028 				DDR_VAR_TX_CMD_DAT, DDR_VAR_TX_CMD_DAT);
1029 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_SAMPLE_POINT_CTL,
1030 				DDR_VAR_RX_DAT | DDR_VAR_RX_CMD,
1031 				DDR_VAR_RX_DAT | DDR_VAR_RX_CMD);
1032 		break;
1033 
1034 	case MMC_TIMING_MMC_HS:
1035 	case MMC_TIMING_SD_HS:
1036 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG1,
1037 				0x0C, SD_20_MODE);
1038 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL,
1039 				CLK_LOW_FREQ, CLK_LOW_FREQ);
1040 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_SOURCE, 0xFF,
1041 				CRC_FIX_CLK | SD30_VAR_CLK0 | SAMPLE_VAR_CLK1);
1042 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL, CLK_LOW_FREQ, 0);
1043 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_PUSH_POINT_CTL,
1044 				SD20_TX_SEL_MASK, SD20_TX_14_AHEAD);
1045 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_SAMPLE_POINT_CTL,
1046 				SD20_RX_SEL_MASK, SD20_RX_14_DELAY);
1047 		break;
1048 
1049 	default:
1050 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD,
1051 				SD_CFG1, 0x0C, SD_20_MODE);
1052 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL,
1053 				CLK_LOW_FREQ, CLK_LOW_FREQ);
1054 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_SOURCE, 0xFF,
1055 				CRC_FIX_CLK | SD30_VAR_CLK0 | SAMPLE_VAR_CLK1);
1056 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL, CLK_LOW_FREQ, 0);
1057 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD,
1058 				SD_PUSH_POINT_CTL, 0xFF, 0);
1059 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_SAMPLE_POINT_CTL,
1060 				SD20_RX_SEL_MASK, SD20_RX_POS_EDGE);
1061 		break;
1062 	}
1063 
1064 	err = rtsx_pci_send_cmd(pcr, 100);
1065 
1066 	return err;
1067 }
1068 
1069 static void sdmmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
1070 {
1071 	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1072 	struct rtsx_pcr *pcr = host->pcr;
1073 
1074 	if (host->eject)
1075 		return;
1076 
1077 	if (rtsx_pci_card_exclusive_check(host->pcr, RTSX_SD_CARD))
1078 		return;
1079 
1080 	mutex_lock(&pcr->pcr_mutex);
1081 
1082 	rtsx_pci_start_run(pcr);
1083 
1084 	sd_set_bus_width(host, ios->bus_width);
1085 	sd_set_power_mode(host, ios->power_mode);
1086 	sd_set_timing(host, ios->timing);
1087 
1088 	host->vpclk = false;
1089 	host->double_clk = true;
1090 
1091 	switch (ios->timing) {
1092 	case MMC_TIMING_UHS_SDR104:
1093 	case MMC_TIMING_UHS_SDR50:
1094 		host->ssc_depth = RTSX_SSC_DEPTH_2M;
1095 		host->vpclk = true;
1096 		host->double_clk = false;
1097 		break;
1098 	case MMC_TIMING_MMC_DDR52:
1099 	case MMC_TIMING_UHS_DDR50:
1100 	case MMC_TIMING_UHS_SDR25:
1101 		host->ssc_depth = RTSX_SSC_DEPTH_1M;
1102 		break;
1103 	default:
1104 		host->ssc_depth = RTSX_SSC_DEPTH_500K;
1105 		break;
1106 	}
1107 
1108 	host->initial_mode = (ios->clock <= 1000000) ? true : false;
1109 
1110 	host->clock = ios->clock;
1111 	rtsx_pci_switch_clock(pcr, ios->clock, host->ssc_depth,
1112 			host->initial_mode, host->double_clk, host->vpclk);
1113 
1114 	mutex_unlock(&pcr->pcr_mutex);
1115 }
1116 
1117 static int sdmmc_get_ro(struct mmc_host *mmc)
1118 {
1119 	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1120 	struct rtsx_pcr *pcr = host->pcr;
1121 	int ro = 0;
1122 	u32 val;
1123 
1124 	if (host->eject)
1125 		return -ENOMEDIUM;
1126 
1127 	mutex_lock(&pcr->pcr_mutex);
1128 
1129 	rtsx_pci_start_run(pcr);
1130 
1131 	/* Check SD mechanical write-protect switch */
1132 	val = rtsx_pci_readl(pcr, RTSX_BIPR);
1133 	dev_dbg(sdmmc_dev(host), "%s: RTSX_BIPR = 0x%08x\n", __func__, val);
1134 	if (val & SD_WRITE_PROTECT)
1135 		ro = 1;
1136 
1137 	mutex_unlock(&pcr->pcr_mutex);
1138 
1139 	return ro;
1140 }
1141 
1142 static int sdmmc_get_cd(struct mmc_host *mmc)
1143 {
1144 	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1145 	struct rtsx_pcr *pcr = host->pcr;
1146 	int cd = 0;
1147 	u32 val;
1148 
1149 	if (host->eject)
1150 		return cd;
1151 
1152 	mutex_lock(&pcr->pcr_mutex);
1153 
1154 	rtsx_pci_start_run(pcr);
1155 
1156 	/* Check SD card detect */
1157 	val = rtsx_pci_card_exist(pcr);
1158 	dev_dbg(sdmmc_dev(host), "%s: RTSX_BIPR = 0x%08x\n", __func__, val);
1159 	if (val & SD_EXIST)
1160 		cd = 1;
1161 
1162 	mutex_unlock(&pcr->pcr_mutex);
1163 
1164 	return cd;
1165 }
1166 
1167 static int sd_wait_voltage_stable_1(struct realtek_pci_sdmmc *host)
1168 {
1169 	struct rtsx_pcr *pcr = host->pcr;
1170 	int err;
1171 	u8 stat;
1172 
1173 	/* Reference to Signal Voltage Switch Sequence in SD spec.
1174 	 * Wait for a period of time so that the card can drive SD_CMD and
1175 	 * SD_DAT[3:0] to low after sending back CMD11 response.
1176 	 */
1177 	mdelay(1);
1178 
1179 	/* SD_CMD, SD_DAT[3:0] should be driven to low by card;
1180 	 * If either one of SD_CMD,SD_DAT[3:0] is not low,
1181 	 * abort the voltage switch sequence;
1182 	 */
1183 	err = rtsx_pci_read_register(pcr, SD_BUS_STAT, &stat);
1184 	if (err < 0)
1185 		return err;
1186 
1187 	if (stat & (SD_CMD_STATUS | SD_DAT3_STATUS | SD_DAT2_STATUS |
1188 				SD_DAT1_STATUS | SD_DAT0_STATUS))
1189 		return -EINVAL;
1190 
1191 	/* Stop toggle SD clock */
1192 	err = rtsx_pci_write_register(pcr, SD_BUS_STAT,
1193 			0xFF, SD_CLK_FORCE_STOP);
1194 	if (err < 0)
1195 		return err;
1196 
1197 	return 0;
1198 }
1199 
1200 static int sd_wait_voltage_stable_2(struct realtek_pci_sdmmc *host)
1201 {
1202 	struct rtsx_pcr *pcr = host->pcr;
1203 	int err;
1204 	u8 stat, mask, val;
1205 
1206 	/* Wait 1.8V output of voltage regulator in card stable */
1207 	msleep(50);
1208 
1209 	/* Toggle SD clock again */
1210 	err = rtsx_pci_write_register(pcr, SD_BUS_STAT, 0xFF, SD_CLK_TOGGLE_EN);
1211 	if (err < 0)
1212 		return err;
1213 
1214 	/* Wait for a period of time so that the card can drive
1215 	 * SD_DAT[3:0] to high at 1.8V
1216 	 */
1217 	msleep(20);
1218 
1219 	/* SD_CMD, SD_DAT[3:0] should be pulled high by host */
1220 	err = rtsx_pci_read_register(pcr, SD_BUS_STAT, &stat);
1221 	if (err < 0)
1222 		return err;
1223 
1224 	mask = SD_CMD_STATUS | SD_DAT3_STATUS | SD_DAT2_STATUS |
1225 		SD_DAT1_STATUS | SD_DAT0_STATUS;
1226 	val = SD_CMD_STATUS | SD_DAT3_STATUS | SD_DAT2_STATUS |
1227 		SD_DAT1_STATUS | SD_DAT0_STATUS;
1228 	if ((stat & mask) != val) {
1229 		dev_dbg(sdmmc_dev(host),
1230 			"%s: SD_BUS_STAT = 0x%x\n", __func__, stat);
1231 		rtsx_pci_write_register(pcr, SD_BUS_STAT,
1232 				SD_CLK_TOGGLE_EN | SD_CLK_FORCE_STOP, 0);
1233 		rtsx_pci_write_register(pcr, CARD_CLK_EN, 0xFF, 0);
1234 		return -EINVAL;
1235 	}
1236 
1237 	return 0;
1238 }
1239 
1240 static int sdmmc_switch_voltage(struct mmc_host *mmc, struct mmc_ios *ios)
1241 {
1242 	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1243 	struct rtsx_pcr *pcr = host->pcr;
1244 	int err = 0;
1245 	u8 voltage;
1246 
1247 	dev_dbg(sdmmc_dev(host), "%s: signal_voltage = %d\n",
1248 			__func__, ios->signal_voltage);
1249 
1250 	if (host->eject)
1251 		return -ENOMEDIUM;
1252 
1253 	err = rtsx_pci_card_exclusive_check(host->pcr, RTSX_SD_CARD);
1254 	if (err)
1255 		return err;
1256 
1257 	mutex_lock(&pcr->pcr_mutex);
1258 
1259 	rtsx_pci_start_run(pcr);
1260 
1261 	if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_330)
1262 		voltage = OUTPUT_3V3;
1263 	else
1264 		voltage = OUTPUT_1V8;
1265 
1266 	if (voltage == OUTPUT_1V8) {
1267 		err = sd_wait_voltage_stable_1(host);
1268 		if (err < 0)
1269 			goto out;
1270 	}
1271 
1272 	err = rtsx_pci_switch_output_voltage(pcr, voltage);
1273 	if (err < 0)
1274 		goto out;
1275 
1276 	if (voltage == OUTPUT_1V8) {
1277 		err = sd_wait_voltage_stable_2(host);
1278 		if (err < 0)
1279 			goto out;
1280 	}
1281 
1282 out:
1283 	/* Stop toggle SD clock in idle */
1284 	err = rtsx_pci_write_register(pcr, SD_BUS_STAT,
1285 			SD_CLK_TOGGLE_EN | SD_CLK_FORCE_STOP, 0);
1286 
1287 	mutex_unlock(&pcr->pcr_mutex);
1288 
1289 	return err;
1290 }
1291 
1292 static int sdmmc_execute_tuning(struct mmc_host *mmc, u32 opcode)
1293 {
1294 	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1295 	struct rtsx_pcr *pcr = host->pcr;
1296 	int err = 0;
1297 
1298 	if (host->eject)
1299 		return -ENOMEDIUM;
1300 
1301 	err = rtsx_pci_card_exclusive_check(host->pcr, RTSX_SD_CARD);
1302 	if (err)
1303 		return err;
1304 
1305 	mutex_lock(&pcr->pcr_mutex);
1306 
1307 	rtsx_pci_start_run(pcr);
1308 
1309 	/* Set initial TX phase */
1310 	switch (mmc->ios.timing) {
1311 	case MMC_TIMING_UHS_SDR104:
1312 		err = sd_change_phase(host, SDR104_TX_PHASE(pcr), false);
1313 		break;
1314 
1315 	case MMC_TIMING_UHS_SDR50:
1316 		err = sd_change_phase(host, SDR50_TX_PHASE(pcr), false);
1317 		break;
1318 
1319 	case MMC_TIMING_UHS_DDR50:
1320 		err = sd_change_phase(host, DDR50_TX_PHASE(pcr), false);
1321 		break;
1322 
1323 	default:
1324 		err = 0;
1325 	}
1326 
1327 	if (err)
1328 		goto out;
1329 
1330 	/* Tuning RX phase */
1331 	if ((mmc->ios.timing == MMC_TIMING_UHS_SDR104) ||
1332 			(mmc->ios.timing == MMC_TIMING_UHS_SDR50))
1333 		err = sd_tuning_rx(host, opcode);
1334 	else if (mmc->ios.timing == MMC_TIMING_UHS_DDR50)
1335 		err = sd_change_phase(host, DDR50_RX_PHASE(pcr), true);
1336 
1337 out:
1338 	mutex_unlock(&pcr->pcr_mutex);
1339 
1340 	return err;
1341 }
1342 
1343 static int sdmmc_init_sd_express(struct mmc_host *mmc, struct mmc_ios *ios)
1344 {
1345 	u32 relink_time;
1346 	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1347 	struct rtsx_pcr *pcr = host->pcr;
1348 
1349 	/* Set relink_time for changing to PCIe card */
1350 	relink_time = 0x8FFF;
1351 
1352 	rtsx_pci_write_register(pcr, 0xFF01, 0xFF, relink_time);
1353 	rtsx_pci_write_register(pcr, 0xFF02, 0xFF, relink_time >> 8);
1354 	rtsx_pci_write_register(pcr, 0xFF03, 0x01, relink_time >> 16);
1355 
1356 	rtsx_pci_write_register(pcr, PETXCFG, 0x80, 0x80);
1357 	rtsx_pci_write_register(pcr, LDO_VCC_CFG0,
1358 		RTS5261_LDO1_OCP_THD_MASK,
1359 		pcr->option.sd_800mA_ocp_thd);
1360 
1361 	if (pcr->ops->disable_auto_blink)
1362 		pcr->ops->disable_auto_blink(pcr);
1363 
1364 	/* For PCIe/NVMe mode can't enter delink issue */
1365 	pcr->hw_param.interrupt_en &= ~(SD_INT_EN);
1366 	rtsx_pci_writel(pcr, RTSX_BIER, pcr->hw_param.interrupt_en);
1367 
1368 	rtsx_pci_write_register(pcr, RTS5260_AUTOLOAD_CFG4,
1369 		RTS5261_AUX_CLK_16M_EN, RTS5261_AUX_CLK_16M_EN);
1370 	rtsx_pci_write_register(pcr, RTS5261_FW_CFG0,
1371 		RTS5261_FW_ENTER_EXPRESS, RTS5261_FW_ENTER_EXPRESS);
1372 	rtsx_pci_write_register(pcr, RTS5261_FW_CFG1,
1373 		RTS5261_MCU_CLOCK_GATING, RTS5261_MCU_CLOCK_GATING);
1374 	rtsx_pci_write_register(pcr, RTS5261_FW_CFG1,
1375 		RTS5261_MCU_BUS_SEL_MASK | RTS5261_MCU_CLOCK_SEL_MASK
1376 		| RTS5261_DRIVER_ENABLE_FW,
1377 		RTS5261_MCU_CLOCK_SEL_16M | RTS5261_DRIVER_ENABLE_FW);
1378 	host->eject = true;
1379 	return 0;
1380 }
1381 
1382 static const struct mmc_host_ops realtek_pci_sdmmc_ops = {
1383 	.pre_req = sdmmc_pre_req,
1384 	.post_req = sdmmc_post_req,
1385 	.request = sdmmc_request,
1386 	.set_ios = sdmmc_set_ios,
1387 	.get_ro = sdmmc_get_ro,
1388 	.get_cd = sdmmc_get_cd,
1389 	.start_signal_voltage_switch = sdmmc_switch_voltage,
1390 	.execute_tuning = sdmmc_execute_tuning,
1391 	.init_sd_express = sdmmc_init_sd_express,
1392 };
1393 
1394 static void init_extra_caps(struct realtek_pci_sdmmc *host)
1395 {
1396 	struct mmc_host *mmc = host->mmc;
1397 	struct rtsx_pcr *pcr = host->pcr;
1398 
1399 	dev_dbg(sdmmc_dev(host), "pcr->extra_caps = 0x%x\n", pcr->extra_caps);
1400 
1401 	if (pcr->extra_caps & EXTRA_CAPS_SD_SDR50)
1402 		mmc->caps |= MMC_CAP_UHS_SDR50;
1403 	if (pcr->extra_caps & EXTRA_CAPS_SD_SDR104)
1404 		mmc->caps |= MMC_CAP_UHS_SDR104;
1405 	if (pcr->extra_caps & EXTRA_CAPS_SD_DDR50)
1406 		mmc->caps |= MMC_CAP_UHS_DDR50;
1407 	if (pcr->extra_caps & EXTRA_CAPS_MMC_HSDDR)
1408 		mmc->caps |= MMC_CAP_1_8V_DDR;
1409 	if (pcr->extra_caps & EXTRA_CAPS_MMC_8BIT)
1410 		mmc->caps |= MMC_CAP_8_BIT_DATA;
1411 	if (pcr->extra_caps & EXTRA_CAPS_NO_MMC)
1412 		mmc->caps2 |= MMC_CAP2_NO_MMC;
1413 	if (pcr->extra_caps & EXTRA_CAPS_SD_EXPRESS)
1414 		mmc->caps2 |= MMC_CAP2_SD_EXP | MMC_CAP2_SD_EXP_1_2V;
1415 }
1416 
1417 static void realtek_init_host(struct realtek_pci_sdmmc *host)
1418 {
1419 	struct mmc_host *mmc = host->mmc;
1420 	struct rtsx_pcr *pcr = host->pcr;
1421 
1422 	mmc->f_min = 250000;
1423 	mmc->f_max = 208000000;
1424 	mmc->ocr_avail = MMC_VDD_32_33 | MMC_VDD_33_34 | MMC_VDD_165_195;
1425 	mmc->caps = MMC_CAP_4_BIT_DATA | MMC_CAP_SD_HIGHSPEED |
1426 		MMC_CAP_MMC_HIGHSPEED | MMC_CAP_BUS_WIDTH_TEST |
1427 		MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25;
1428 	if (pcr->rtd3_en)
1429 		mmc->caps = mmc->caps | MMC_CAP_AGGRESSIVE_PM;
1430 	mmc->caps2 = MMC_CAP2_NO_PRESCAN_POWERUP | MMC_CAP2_FULL_PWR_CYCLE |
1431 		MMC_CAP2_NO_SDIO;
1432 	mmc->max_current_330 = 400;
1433 	mmc->max_current_180 = 800;
1434 	mmc->ops = &realtek_pci_sdmmc_ops;
1435 
1436 	init_extra_caps(host);
1437 
1438 	mmc->max_segs = 256;
1439 	mmc->max_seg_size = 65536;
1440 	mmc->max_blk_size = 512;
1441 	mmc->max_blk_count = 65535;
1442 	mmc->max_req_size = 524288;
1443 }
1444 
1445 static void rtsx_pci_sdmmc_card_event(struct platform_device *pdev)
1446 {
1447 	struct realtek_pci_sdmmc *host = platform_get_drvdata(pdev);
1448 
1449 	host->cookie = -1;
1450 	mmc_detect_change(host->mmc, 0);
1451 }
1452 
1453 static int rtsx_pci_sdmmc_drv_probe(struct platform_device *pdev)
1454 {
1455 	struct mmc_host *mmc;
1456 	struct realtek_pci_sdmmc *host;
1457 	struct rtsx_pcr *pcr;
1458 	struct pcr_handle *handle = pdev->dev.platform_data;
1459 
1460 	if (!handle)
1461 		return -ENXIO;
1462 
1463 	pcr = handle->pcr;
1464 	if (!pcr)
1465 		return -ENXIO;
1466 
1467 	dev_dbg(&(pdev->dev), ": Realtek PCI-E SDMMC controller found\n");
1468 
1469 	mmc = mmc_alloc_host(sizeof(*host), &pdev->dev);
1470 	if (!mmc)
1471 		return -ENOMEM;
1472 
1473 	host = mmc_priv(mmc);
1474 	host->pcr = pcr;
1475 	host->mmc = mmc;
1476 	host->pdev = pdev;
1477 	host->cookie = -1;
1478 	host->power_state = SDMMC_POWER_OFF;
1479 	INIT_WORK(&host->work, sd_request);
1480 	platform_set_drvdata(pdev, host);
1481 	pcr->slots[RTSX_SD_CARD].p_dev = pdev;
1482 	pcr->slots[RTSX_SD_CARD].card_event = rtsx_pci_sdmmc_card_event;
1483 
1484 	mutex_init(&host->host_mutex);
1485 
1486 	realtek_init_host(host);
1487 
1488 	if (pcr->rtd3_en) {
1489 		pm_runtime_set_autosuspend_delay(&pdev->dev, 5000);
1490 		pm_runtime_use_autosuspend(&pdev->dev);
1491 		pm_runtime_enable(&pdev->dev);
1492 	}
1493 
1494 
1495 	mmc_add_host(mmc);
1496 
1497 	return 0;
1498 }
1499 
1500 static int rtsx_pci_sdmmc_drv_remove(struct platform_device *pdev)
1501 {
1502 	struct realtek_pci_sdmmc *host = platform_get_drvdata(pdev);
1503 	struct rtsx_pcr *pcr;
1504 	struct mmc_host *mmc;
1505 
1506 	if (!host)
1507 		return 0;
1508 
1509 	pcr = host->pcr;
1510 	pcr->slots[RTSX_SD_CARD].p_dev = NULL;
1511 	pcr->slots[RTSX_SD_CARD].card_event = NULL;
1512 	mmc = host->mmc;
1513 
1514 	if (pcr->rtd3_en) {
1515 		pm_runtime_dont_use_autosuspend(&pdev->dev);
1516 		pm_runtime_disable(&pdev->dev);
1517 	}
1518 
1519 	cancel_work_sync(&host->work);
1520 
1521 	mutex_lock(&host->host_mutex);
1522 	if (host->mrq) {
1523 		dev_dbg(&(pdev->dev),
1524 			"%s: Controller removed during transfer\n",
1525 			mmc_hostname(mmc));
1526 
1527 		rtsx_pci_complete_unfinished_transfer(pcr);
1528 
1529 		host->mrq->cmd->error = -ENOMEDIUM;
1530 		if (host->mrq->stop)
1531 			host->mrq->stop->error = -ENOMEDIUM;
1532 		mmc_request_done(mmc, host->mrq);
1533 	}
1534 	mutex_unlock(&host->host_mutex);
1535 
1536 	mmc_remove_host(mmc);
1537 	host->eject = true;
1538 
1539 	flush_work(&host->work);
1540 
1541 	mmc_free_host(mmc);
1542 
1543 	dev_dbg(&(pdev->dev),
1544 		": Realtek PCI-E SDMMC controller has been removed\n");
1545 
1546 	return 0;
1547 }
1548 
1549 static const struct platform_device_id rtsx_pci_sdmmc_ids[] = {
1550 	{
1551 		.name = DRV_NAME_RTSX_PCI_SDMMC,
1552 	}, {
1553 		/* sentinel */
1554 	}
1555 };
1556 MODULE_DEVICE_TABLE(platform, rtsx_pci_sdmmc_ids);
1557 
1558 static struct platform_driver rtsx_pci_sdmmc_driver = {
1559 	.probe		= rtsx_pci_sdmmc_drv_probe,
1560 	.remove		= rtsx_pci_sdmmc_drv_remove,
1561 	.id_table       = rtsx_pci_sdmmc_ids,
1562 	.driver		= {
1563 		.name	= DRV_NAME_RTSX_PCI_SDMMC,
1564 		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
1565 	},
1566 };
1567 module_platform_driver(rtsx_pci_sdmmc_driver);
1568 
1569 MODULE_LICENSE("GPL");
1570 MODULE_AUTHOR("Wei WANG <wei_wang@realsil.com.cn>");
1571 MODULE_DESCRIPTION("Realtek PCI-E SD/MMC Card Host Driver");
1572