1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * Amlogic Meson6/Meson8/Meson8b/Meson8m2 SDHC MMC host controller driver.
4  *
5  * Copyright (C) 2020 Martin Blumenstingl <martin.blumenstingl@googlemail.com>
6  */
7 
8 #include <linux/clk.h>
9 #include <linux/device.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/interrupt.h>
12 #include <linux/iopoll.h>
13 #include <linux/module.h>
14 #include <linux/of.h>
15 #include <linux/platform_device.h>
16 #include <linux/property.h>
17 #include <linux/regmap.h>
18 #include <linux/regulator/consumer.h>
19 #include <linux/types.h>
20 
21 #include <linux/mmc/host.h>
22 #include <linux/mmc/mmc.h>
23 #include <linux/mmc/sdio.h>
24 #include <linux/mmc/slot-gpio.h>
25 
26 #include "meson-mx-sdhc.h"
27 
28 #define MESON_SDHC_NUM_BULK_CLKS				4
29 #define MESON_SDHC_MAX_BLK_SIZE					512
30 #define MESON_SDHC_NUM_TUNING_TRIES				10
31 
32 #define MESON_SDHC_WAIT_CMD_READY_SLEEP_US			1
33 #define MESON_SDHC_WAIT_CMD_READY_TIMEOUT_US			100000
34 #define MESON_SDHC_WAIT_BEFORE_SEND_SLEEP_US			1
35 #define MESON_SDHC_WAIT_BEFORE_SEND_TIMEOUT_US			200
36 
37 struct meson_mx_sdhc_data {
38 	void		(*init_hw)(struct mmc_host *mmc);
39 	void		(*set_pdma)(struct mmc_host *mmc);
40 	void		(*wait_before_send)(struct mmc_host *mmc);
41 	bool		hardware_flush_all_cmds;
42 };
43 
44 struct meson_mx_sdhc_host {
45 	struct mmc_host			*mmc;
46 
47 	struct mmc_request		*mrq;
48 	struct mmc_command		*cmd;
49 	int				error;
50 
51 	struct regmap			*regmap;
52 
53 	struct clk			*pclk;
54 	struct clk			*sd_clk;
55 	struct clk_bulk_data		bulk_clks[MESON_SDHC_NUM_BULK_CLKS];
56 	bool				bulk_clks_enabled;
57 
58 	const struct meson_mx_sdhc_data	*platform;
59 };
60 
61 static const struct regmap_config meson_mx_sdhc_regmap_config = {
62 	.reg_bits = 8,
63 	.val_bits = 32,
64 	.reg_stride = 4,
65 	.max_register = MESON_SDHC_CLK2,
66 };
67 
68 static void meson_mx_sdhc_hw_reset(struct mmc_host *mmc)
69 {
70 	struct meson_mx_sdhc_host *host = mmc_priv(mmc);
71 
72 	regmap_write(host->regmap, MESON_SDHC_SRST, MESON_SDHC_SRST_MAIN_CTRL |
73 		     MESON_SDHC_SRST_RXFIFO | MESON_SDHC_SRST_TXFIFO |
74 		     MESON_SDHC_SRST_DPHY_RX | MESON_SDHC_SRST_DPHY_TX |
75 		     MESON_SDHC_SRST_DMA_IF);
76 	usleep_range(10, 100);
77 
78 	regmap_write(host->regmap, MESON_SDHC_SRST, 0);
79 	usleep_range(10, 100);
80 }
81 
82 static void meson_mx_sdhc_clear_fifo(struct mmc_host *mmc)
83 {
84 	struct meson_mx_sdhc_host *host = mmc_priv(mmc);
85 	u32 stat;
86 
87 	regmap_read(host->regmap, MESON_SDHC_STAT, &stat);
88 	if (!FIELD_GET(MESON_SDHC_STAT_RXFIFO_CNT, stat) &&
89 	    !FIELD_GET(MESON_SDHC_STAT_TXFIFO_CNT, stat))
90 		return;
91 
92 	regmap_write(host->regmap, MESON_SDHC_SRST, MESON_SDHC_SRST_RXFIFO |
93 		     MESON_SDHC_SRST_TXFIFO | MESON_SDHC_SRST_MAIN_CTRL);
94 	udelay(5);
95 
96 	regmap_read(host->regmap, MESON_SDHC_STAT, &stat);
97 	if (FIELD_GET(MESON_SDHC_STAT_RXFIFO_CNT, stat) ||
98 	    FIELD_GET(MESON_SDHC_STAT_TXFIFO_CNT, stat))
99 		dev_warn(mmc_dev(host->mmc),
100 			 "Failed to clear FIFOs, RX: %lu, TX: %lu\n",
101 			 FIELD_GET(MESON_SDHC_STAT_RXFIFO_CNT, stat),
102 			 FIELD_GET(MESON_SDHC_STAT_TXFIFO_CNT, stat));
103 }
104 
105 static void meson_mx_sdhc_wait_cmd_ready(struct mmc_host *mmc)
106 {
107 	struct meson_mx_sdhc_host *host = mmc_priv(mmc);
108 	u32 stat, esta;
109 	int ret;
110 
111 	ret = regmap_read_poll_timeout(host->regmap, MESON_SDHC_STAT, stat,
112 				       !(stat & MESON_SDHC_STAT_CMD_BUSY),
113 				       MESON_SDHC_WAIT_CMD_READY_SLEEP_US,
114 				       MESON_SDHC_WAIT_CMD_READY_TIMEOUT_US);
115 	if (ret) {
116 		dev_warn(mmc_dev(mmc),
117 			 "Failed to poll for CMD_BUSY while processing CMD%d\n",
118 			 host->cmd->opcode);
119 		meson_mx_sdhc_hw_reset(mmc);
120 	}
121 
122 	ret = regmap_read_poll_timeout(host->regmap, MESON_SDHC_ESTA, esta,
123 				       !(esta & MESON_SDHC_ESTA_11_13),
124 				       MESON_SDHC_WAIT_CMD_READY_SLEEP_US,
125 				       MESON_SDHC_WAIT_CMD_READY_TIMEOUT_US);
126 	if (ret) {
127 		dev_warn(mmc_dev(mmc),
128 			 "Failed to poll for ESTA[13:11] while processing CMD%d\n",
129 			 host->cmd->opcode);
130 		meson_mx_sdhc_hw_reset(mmc);
131 	}
132 }
133 
134 static void meson_mx_sdhc_start_cmd(struct mmc_host *mmc,
135 				    struct mmc_command *cmd)
136 {
137 	struct meson_mx_sdhc_host *host = mmc_priv(mmc);
138 	bool manual_stop = false;
139 	u32 ictl, send;
140 	int pack_len;
141 
142 	host->cmd = cmd;
143 
144 	ictl = MESON_SDHC_ICTL_DATA_TIMEOUT | MESON_SDHC_ICTL_DATA_ERR_CRC |
145 	       MESON_SDHC_ICTL_RXFIFO_FULL | MESON_SDHC_ICTL_TXFIFO_EMPTY |
146 	       MESON_SDHC_ICTL_RESP_TIMEOUT | MESON_SDHC_ICTL_RESP_ERR_CRC;
147 
148 	send = FIELD_PREP(MESON_SDHC_SEND_CMD_INDEX, cmd->opcode);
149 
150 	if (cmd->data) {
151 		send |= MESON_SDHC_SEND_CMD_HAS_DATA;
152 		send |= FIELD_PREP(MESON_SDHC_SEND_TOTAL_PACK,
153 				   cmd->data->blocks - 1);
154 
155 		if (cmd->data->blksz < MESON_SDHC_MAX_BLK_SIZE)
156 			pack_len = cmd->data->blksz;
157 		else
158 			pack_len = 0;
159 
160 		if (cmd->data->flags & MMC_DATA_WRITE)
161 			send |= MESON_SDHC_SEND_DATA_DIR;
162 
163 		/*
164 		 * If command with no data, just wait response done
165 		 * interrupt(int[0]), and if command with data transfer, just
166 		 * wait dma done interrupt(int[11]), don't need care about
167 		 * dat0 busy or not.
168 		 */
169 		if (host->platform->hardware_flush_all_cmds ||
170 		    cmd->data->flags & MMC_DATA_WRITE)
171 			/* hardware flush: */
172 			ictl |= MESON_SDHC_ICTL_DMA_DONE;
173 		else
174 			/* software flush: */
175 			ictl |= MESON_SDHC_ICTL_DATA_XFER_OK;
176 
177 		/*
178 		 * Mimic the logic from the vendor driver where (only)
179 		 * SD_IO_RW_EXTENDED commands with more than one block set the
180 		 * MESON_SDHC_MISC_MANUAL_STOP bit. This fixes the firmware
181 		 * download in the brcmfmac driver for a BCM43362/1 card.
182 		 * Without this sdio_memcpy_toio() (with a size of 219557
183 		 * bytes) times out if MESON_SDHC_MISC_MANUAL_STOP is not set.
184 		 */
185 		manual_stop = cmd->data->blocks > 1 &&
186 			      cmd->opcode == SD_IO_RW_EXTENDED;
187 	} else {
188 		pack_len = 0;
189 
190 		ictl |= MESON_SDHC_ICTL_RESP_OK;
191 	}
192 
193 	regmap_update_bits(host->regmap, MESON_SDHC_MISC,
194 			   MESON_SDHC_MISC_MANUAL_STOP,
195 			   manual_stop ? MESON_SDHC_MISC_MANUAL_STOP : 0);
196 
197 	if (cmd->opcode == MMC_STOP_TRANSMISSION)
198 		send |= MESON_SDHC_SEND_DATA_STOP;
199 
200 	if (cmd->flags & MMC_RSP_PRESENT)
201 		send |= MESON_SDHC_SEND_CMD_HAS_RESP;
202 
203 	if (cmd->flags & MMC_RSP_136) {
204 		send |= MESON_SDHC_SEND_RESP_LEN;
205 		send |= MESON_SDHC_SEND_RESP_NO_CRC;
206 	}
207 
208 	if (!(cmd->flags & MMC_RSP_CRC))
209 		send |= MESON_SDHC_SEND_RESP_NO_CRC;
210 
211 	if (cmd->flags & MMC_RSP_BUSY)
212 		send |= MESON_SDHC_SEND_R1B;
213 
214 	/* enable the new IRQs and mask all pending ones */
215 	regmap_write(host->regmap, MESON_SDHC_ICTL, ictl);
216 	regmap_write(host->regmap, MESON_SDHC_ISTA, MESON_SDHC_ISTA_ALL_IRQS);
217 
218 	regmap_write(host->regmap, MESON_SDHC_ARGU, cmd->arg);
219 
220 	regmap_update_bits(host->regmap, MESON_SDHC_CTRL,
221 			   MESON_SDHC_CTRL_PACK_LEN,
222 			   FIELD_PREP(MESON_SDHC_CTRL_PACK_LEN, pack_len));
223 
224 	if (cmd->data)
225 		regmap_write(host->regmap, MESON_SDHC_ADDR,
226 			     sg_dma_address(cmd->data->sg));
227 
228 	meson_mx_sdhc_wait_cmd_ready(mmc);
229 
230 	if (cmd->data)
231 		host->platform->set_pdma(mmc);
232 
233 	if (host->platform->wait_before_send)
234 		host->platform->wait_before_send(mmc);
235 
236 	regmap_write(host->regmap, MESON_SDHC_SEND, send);
237 }
238 
239 static void meson_mx_sdhc_disable_clks(struct mmc_host *mmc)
240 {
241 	struct meson_mx_sdhc_host *host = mmc_priv(mmc);
242 
243 	if (!host->bulk_clks_enabled)
244 		return;
245 
246 	clk_bulk_disable_unprepare(MESON_SDHC_NUM_BULK_CLKS, host->bulk_clks);
247 
248 	host->bulk_clks_enabled = false;
249 }
250 
251 static int meson_mx_sdhc_enable_clks(struct mmc_host *mmc)
252 {
253 	struct meson_mx_sdhc_host *host = mmc_priv(mmc);
254 	int ret;
255 
256 	if (host->bulk_clks_enabled)
257 		return 0;
258 
259 	ret = clk_bulk_prepare_enable(MESON_SDHC_NUM_BULK_CLKS,
260 				      host->bulk_clks);
261 	if (ret)
262 		return ret;
263 
264 	host->bulk_clks_enabled = true;
265 
266 	return 0;
267 }
268 
269 static int meson_mx_sdhc_set_clk(struct mmc_host *mmc, struct mmc_ios *ios)
270 {
271 	struct meson_mx_sdhc_host *host = mmc_priv(mmc);
272 	u32 rx_clk_phase;
273 	int ret;
274 
275 	meson_mx_sdhc_disable_clks(mmc);
276 
277 	if (ios->clock) {
278 		ret = clk_set_rate(host->sd_clk, ios->clock);
279 		if (ret) {
280 			dev_warn(mmc_dev(mmc),
281 				 "Failed to set MMC clock to %uHz: %d\n",
282 				 ios->clock, host->error);
283 			return ret;
284 		}
285 
286 		ret = meson_mx_sdhc_enable_clks(mmc);
287 		if (ret)
288 			return ret;
289 
290 		mmc->actual_clock = clk_get_rate(host->sd_clk);
291 
292 		/*
293 		 * according to Amlogic the following latching points are
294 		 * selected with empirical values, there is no (known) formula
295 		 * to calculate these.
296 		 */
297 		if (mmc->actual_clock > 100000000) {
298 			rx_clk_phase = 1;
299 		} else if (mmc->actual_clock > 45000000) {
300 			if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_330)
301 				rx_clk_phase = 15;
302 			else
303 				rx_clk_phase = 11;
304 		} else if (mmc->actual_clock >= 25000000) {
305 			rx_clk_phase = 15;
306 		} else if (mmc->actual_clock > 5000000) {
307 			rx_clk_phase = 23;
308 		} else if (mmc->actual_clock > 1000000) {
309 			rx_clk_phase = 55;
310 		} else {
311 			rx_clk_phase = 1061;
312 		}
313 
314 		regmap_update_bits(host->regmap, MESON_SDHC_CLK2,
315 				   MESON_SDHC_CLK2_RX_CLK_PHASE,
316 				   FIELD_PREP(MESON_SDHC_CLK2_RX_CLK_PHASE,
317 					      rx_clk_phase));
318 	} else {
319 		mmc->actual_clock = 0;
320 	}
321 
322 	return 0;
323 }
324 
325 static void meson_mx_sdhc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
326 {
327 	struct meson_mx_sdhc_host *host = mmc_priv(mmc);
328 	unsigned short vdd = ios->vdd;
329 
330 	switch (ios->power_mode) {
331 	case MMC_POWER_OFF:
332 		vdd = 0;
333 		fallthrough;
334 
335 	case MMC_POWER_UP:
336 		if (!IS_ERR(mmc->supply.vmmc)) {
337 			host->error = mmc_regulator_set_ocr(mmc,
338 							    mmc->supply.vmmc,
339 							    vdd);
340 			if (host->error)
341 				return;
342 		}
343 
344 		break;
345 
346 	case MMC_POWER_ON:
347 		break;
348 	}
349 
350 	host->error = meson_mx_sdhc_set_clk(mmc, ios);
351 	if (host->error)
352 		return;
353 
354 	switch (ios->bus_width) {
355 	case MMC_BUS_WIDTH_1:
356 		regmap_update_bits(host->regmap, MESON_SDHC_CTRL,
357 				   MESON_SDHC_CTRL_DAT_TYPE,
358 				   FIELD_PREP(MESON_SDHC_CTRL_DAT_TYPE, 0));
359 		break;
360 
361 	case MMC_BUS_WIDTH_4:
362 		regmap_update_bits(host->regmap, MESON_SDHC_CTRL,
363 				   MESON_SDHC_CTRL_DAT_TYPE,
364 				   FIELD_PREP(MESON_SDHC_CTRL_DAT_TYPE, 1));
365 		break;
366 
367 	case MMC_BUS_WIDTH_8:
368 		regmap_update_bits(host->regmap, MESON_SDHC_CTRL,
369 				   MESON_SDHC_CTRL_DAT_TYPE,
370 				   FIELD_PREP(MESON_SDHC_CTRL_DAT_TYPE, 2));
371 		break;
372 
373 	default:
374 		dev_err(mmc_dev(mmc), "unsupported bus width: %d\n",
375 			ios->bus_width);
376 		host->error = -EINVAL;
377 		return;
378 	}
379 }
380 
381 static int meson_mx_sdhc_map_dma(struct mmc_host *mmc, struct mmc_request *mrq)
382 {
383 	struct mmc_data *data = mrq->data;
384 	int dma_len;
385 
386 	if (!data)
387 		return 0;
388 
389 	dma_len = dma_map_sg(mmc_dev(mmc), data->sg, data->sg_len,
390 			     mmc_get_dma_dir(data));
391 	if (dma_len <= 0) {
392 		dev_err(mmc_dev(mmc), "dma_map_sg failed\n");
393 		return -ENOMEM;
394 	}
395 
396 	return 0;
397 }
398 
399 static void meson_mx_sdhc_request(struct mmc_host *mmc, struct mmc_request *mrq)
400 {
401 	struct meson_mx_sdhc_host *host = mmc_priv(mmc);
402 	struct mmc_command *cmd = mrq->cmd;
403 
404 	if (!host->error)
405 		host->error = meson_mx_sdhc_map_dma(mmc, mrq);
406 
407 	if (host->error) {
408 		cmd->error = host->error;
409 		mmc_request_done(mmc, mrq);
410 		return;
411 	}
412 
413 	host->mrq = mrq;
414 
415 	meson_mx_sdhc_start_cmd(mmc, mrq->cmd);
416 }
417 
418 static int meson_mx_sdhc_card_busy(struct mmc_host *mmc)
419 {
420 	struct meson_mx_sdhc_host *host = mmc_priv(mmc);
421 	u32 stat;
422 
423 	regmap_read(host->regmap, MESON_SDHC_STAT, &stat);
424 	return FIELD_GET(MESON_SDHC_STAT_DAT3_0, stat) == 0;
425 }
426 
427 static bool meson_mx_sdhc_tuning_point_matches(struct mmc_host *mmc,
428 					       u32 opcode)
429 {
430 	unsigned int i, num_matches = 0;
431 	int ret;
432 
433 	for (i = 0; i < MESON_SDHC_NUM_TUNING_TRIES; i++) {
434 		ret = mmc_send_tuning(mmc, opcode, NULL);
435 		if (!ret)
436 			num_matches++;
437 	}
438 
439 	return num_matches == MESON_SDHC_NUM_TUNING_TRIES;
440 }
441 
442 static int meson_mx_sdhc_execute_tuning(struct mmc_host *mmc, u32 opcode)
443 {
444 	struct meson_mx_sdhc_host *host = mmc_priv(mmc);
445 	int div, start, len, best_start, best_len;
446 	int curr_phase, old_phase, new_phase;
447 	u32 val;
448 
449 	len = 0;
450 	start = 0;
451 	best_len = 0;
452 
453 	regmap_read(host->regmap, MESON_SDHC_CLK2, &val);
454 	old_phase = FIELD_GET(MESON_SDHC_CLK2_RX_CLK_PHASE, val);
455 
456 	regmap_read(host->regmap, MESON_SDHC_CLKC, &val);
457 	div = FIELD_GET(MESON_SDHC_CLKC_CLK_DIV, val);
458 
459 	for (curr_phase = 0; curr_phase <= div; curr_phase++) {
460 		regmap_update_bits(host->regmap, MESON_SDHC_CLK2,
461 				   MESON_SDHC_CLK2_RX_CLK_PHASE,
462 				   FIELD_PREP(MESON_SDHC_CLK2_RX_CLK_PHASE,
463 					      curr_phase));
464 
465 		if (meson_mx_sdhc_tuning_point_matches(mmc, opcode)) {
466 			if (!len) {
467 				start = curr_phase;
468 
469 				dev_dbg(mmc_dev(mmc),
470 					"New RX phase window starts at %u\n",
471 					start);
472 			}
473 
474 			len++;
475 		} else {
476 			if (len > best_len) {
477 				best_start = start;
478 				best_len = len;
479 
480 				dev_dbg(mmc_dev(mmc),
481 					"New best RX phase window: %u - %u\n",
482 					best_start, best_start + best_len);
483 			}
484 
485 			/* reset the current window */
486 			len = 0;
487 		}
488 	}
489 
490 	if (len > best_len)
491 		/* the last window is the best (or possibly only) window */
492 		new_phase = start + (len / 2);
493 	else if (best_len)
494 		/* there was a better window than the last */
495 		new_phase = best_start + (best_len / 2);
496 	else
497 		/* no window was found at all, reset to the original phase */
498 		new_phase = old_phase;
499 
500 	regmap_update_bits(host->regmap, MESON_SDHC_CLK2,
501 			   MESON_SDHC_CLK2_RX_CLK_PHASE,
502 			   FIELD_PREP(MESON_SDHC_CLK2_RX_CLK_PHASE,
503 				      new_phase));
504 
505 	if (!len && !best_len)
506 		return -EIO;
507 
508 	dev_dbg(mmc_dev(mmc), "Tuned RX clock phase to %u\n", new_phase);
509 
510 	return 0;
511 }
512 
513 static const struct mmc_host_ops meson_mx_sdhc_ops = {
514 	.hw_reset			= meson_mx_sdhc_hw_reset,
515 	.request			= meson_mx_sdhc_request,
516 	.set_ios			= meson_mx_sdhc_set_ios,
517 	.card_busy			= meson_mx_sdhc_card_busy,
518 	.execute_tuning			= meson_mx_sdhc_execute_tuning,
519 	.get_cd				= mmc_gpio_get_cd,
520 	.get_ro				= mmc_gpio_get_ro,
521 };
522 
523 static void meson_mx_sdhc_request_done(struct meson_mx_sdhc_host *host)
524 {
525 	struct mmc_request *mrq = host->mrq;
526 	struct mmc_host *mmc = host->mmc;
527 
528 	/* disable interrupts and mask all pending ones */
529 	regmap_update_bits(host->regmap, MESON_SDHC_ICTL,
530 			   MESON_SDHC_ICTL_ALL_IRQS, 0);
531 	regmap_update_bits(host->regmap, MESON_SDHC_ISTA,
532 			   MESON_SDHC_ISTA_ALL_IRQS, MESON_SDHC_ISTA_ALL_IRQS);
533 
534 	host->mrq = NULL;
535 	host->cmd = NULL;
536 
537 	mmc_request_done(mmc, mrq);
538 }
539 
540 static u32 meson_mx_sdhc_read_response(struct meson_mx_sdhc_host *host, u8 idx)
541 {
542 	u32 val;
543 
544 	regmap_update_bits(host->regmap, MESON_SDHC_PDMA,
545 			   MESON_SDHC_PDMA_DMA_MODE, 0);
546 
547 	regmap_update_bits(host->regmap, MESON_SDHC_PDMA,
548 			   MESON_SDHC_PDMA_PIO_RDRESP,
549 			   FIELD_PREP(MESON_SDHC_PDMA_PIO_RDRESP, idx));
550 
551 	regmap_read(host->regmap, MESON_SDHC_ARGU, &val);
552 
553 	return val;
554 }
555 
556 static irqreturn_t meson_mx_sdhc_irq(int irq, void *data)
557 {
558 	struct meson_mx_sdhc_host *host = data;
559 	struct mmc_command *cmd = host->cmd;
560 	u32 ictl, ista;
561 
562 	regmap_read(host->regmap, MESON_SDHC_ICTL, &ictl);
563 	regmap_read(host->regmap, MESON_SDHC_ISTA, &ista);
564 
565 	if (!(ictl & ista))
566 		return IRQ_NONE;
567 
568 	if (ista & MESON_SDHC_ISTA_RXFIFO_FULL ||
569 	    ista & MESON_SDHC_ISTA_TXFIFO_EMPTY)
570 		cmd->error = -EIO;
571 	else if (ista & MESON_SDHC_ISTA_RESP_ERR_CRC)
572 		cmd->error = -EILSEQ;
573 	else if (ista & MESON_SDHC_ISTA_RESP_TIMEOUT)
574 		cmd->error = -ETIMEDOUT;
575 
576 	if (cmd->data) {
577 		if (ista & MESON_SDHC_ISTA_DATA_ERR_CRC)
578 			cmd->data->error = -EILSEQ;
579 		else if (ista & MESON_SDHC_ISTA_DATA_TIMEOUT)
580 			cmd->data->error = -ETIMEDOUT;
581 	}
582 
583 	if (cmd->error || (cmd->data && cmd->data->error))
584 		dev_dbg(mmc_dev(host->mmc), "CMD%d error, ISTA: 0x%08x\n",
585 			cmd->opcode, ista);
586 
587 	return IRQ_WAKE_THREAD;
588 }
589 
590 static irqreturn_t meson_mx_sdhc_irq_thread(int irq, void *irq_data)
591 {
592 	struct meson_mx_sdhc_host *host = irq_data;
593 	struct mmc_command *cmd;
594 	u32 val;
595 
596 	cmd = host->cmd;
597 	if (WARN_ON(!cmd))
598 		return IRQ_HANDLED;
599 
600 	if (cmd->data && !cmd->data->error) {
601 		if (!host->platform->hardware_flush_all_cmds &&
602 		    cmd->data->flags & MMC_DATA_READ) {
603 			meson_mx_sdhc_wait_cmd_ready(host->mmc);
604 
605 			/*
606 			 * If MESON_SDHC_PDMA_RXFIFO_MANUAL_FLUSH was
607 			 * previously 0x1 then it has to be set to 0x3. If it
608 			 * was 0x0 before then it has to be set to 0x2. Without
609 			 * this reading SD cards sometimes transfers garbage,
610 			 * which results in cards not being detected due to:
611 			 *   unrecognised SCR structure version <random number>
612 			 */
613 			val = FIELD_PREP(MESON_SDHC_PDMA_RXFIFO_MANUAL_FLUSH,
614 					 2);
615 			regmap_update_bits(host->regmap, MESON_SDHC_PDMA, val,
616 					   val);
617 		}
618 
619 		dma_unmap_sg(mmc_dev(host->mmc), cmd->data->sg,
620 			     cmd->data->sg_len, mmc_get_dma_dir(cmd->data));
621 
622 		cmd->data->bytes_xfered = cmd->data->blksz * cmd->data->blocks;
623 	}
624 
625 	meson_mx_sdhc_wait_cmd_ready(host->mmc);
626 
627 	if (cmd->flags & MMC_RSP_136) {
628 		cmd->resp[0] = meson_mx_sdhc_read_response(host, 4);
629 		cmd->resp[1] = meson_mx_sdhc_read_response(host, 3);
630 		cmd->resp[2] = meson_mx_sdhc_read_response(host, 2);
631 		cmd->resp[3] = meson_mx_sdhc_read_response(host, 1);
632 	} else {
633 		cmd->resp[0] = meson_mx_sdhc_read_response(host, 0);
634 	}
635 
636 	if (cmd->error == -EIO || cmd->error == -ETIMEDOUT)
637 		meson_mx_sdhc_hw_reset(host->mmc);
638 	else if (cmd->data)
639 		/*
640 		 * Clear the FIFOs after completing data transfers to prevent
641 		 * corrupting data on write access. It's not clear why this is
642 		 * needed (for reads and writes), but it mimics what the BSP
643 		 * kernel did.
644 		 */
645 		meson_mx_sdhc_clear_fifo(host->mmc);
646 
647 	meson_mx_sdhc_request_done(host);
648 
649 	return IRQ_HANDLED;
650 }
651 
652 static void meson_mx_sdhc_init_hw_meson8(struct mmc_host *mmc)
653 {
654 	struct meson_mx_sdhc_host *host = mmc_priv(mmc);
655 
656 	regmap_write(host->regmap, MESON_SDHC_MISC,
657 		     FIELD_PREP(MESON_SDHC_MISC_TXSTART_THRES, 7) |
658 		     FIELD_PREP(MESON_SDHC_MISC_WCRC_ERR_PATT, 5) |
659 		     FIELD_PREP(MESON_SDHC_MISC_WCRC_OK_PATT, 2));
660 
661 	regmap_write(host->regmap, MESON_SDHC_ENHC,
662 		     FIELD_PREP(MESON_SDHC_ENHC_RXFIFO_TH, 63) |
663 		     MESON_SDHC_ENHC_MESON6_DMA_WR_RESP |
664 		     FIELD_PREP(MESON_SDHC_ENHC_MESON6_RX_TIMEOUT, 255) |
665 		     FIELD_PREP(MESON_SDHC_ENHC_SDIO_IRQ_PERIOD, 12));
666 };
667 
668 static void meson_mx_sdhc_set_pdma_meson8(struct mmc_host *mmc)
669 {
670 	struct meson_mx_sdhc_host *host = mmc_priv(mmc);
671 
672 	if (host->cmd->data->flags & MMC_DATA_WRITE)
673 		regmap_update_bits(host->regmap, MESON_SDHC_PDMA,
674 				   MESON_SDHC_PDMA_DMA_MODE |
675 				   MESON_SDHC_PDMA_RD_BURST |
676 				   MESON_SDHC_PDMA_TXFIFO_FILL,
677 				   MESON_SDHC_PDMA_DMA_MODE |
678 				   FIELD_PREP(MESON_SDHC_PDMA_RD_BURST, 31) |
679 				   MESON_SDHC_PDMA_TXFIFO_FILL);
680 	else
681 		regmap_update_bits(host->regmap, MESON_SDHC_PDMA,
682 				   MESON_SDHC_PDMA_DMA_MODE |
683 				   MESON_SDHC_PDMA_RXFIFO_MANUAL_FLUSH,
684 				   MESON_SDHC_PDMA_DMA_MODE |
685 				   FIELD_PREP(MESON_SDHC_PDMA_RXFIFO_MANUAL_FLUSH,
686 					      1));
687 
688 	if (host->cmd->data->flags & MMC_DATA_WRITE)
689 		regmap_update_bits(host->regmap, MESON_SDHC_PDMA,
690 				   MESON_SDHC_PDMA_RD_BURST,
691 				   FIELD_PREP(MESON_SDHC_PDMA_RD_BURST, 15));
692 }
693 
694 static void meson_mx_sdhc_wait_before_send_meson8(struct mmc_host *mmc)
695 {
696 	struct meson_mx_sdhc_host *host = mmc_priv(mmc);
697 	u32 val;
698 	int ret;
699 
700 	ret = regmap_read_poll_timeout(host->regmap, MESON_SDHC_ESTA, val,
701 				       val == 0,
702 				       MESON_SDHC_WAIT_BEFORE_SEND_SLEEP_US,
703 				       MESON_SDHC_WAIT_BEFORE_SEND_TIMEOUT_US);
704 	if (ret)
705 		dev_warn(mmc_dev(mmc),
706 			 "Failed to wait for ESTA to clear: 0x%08x\n", val);
707 
708 	if (host->cmd->data && host->cmd->data->flags & MMC_DATA_WRITE) {
709 		ret = regmap_read_poll_timeout(host->regmap, MESON_SDHC_STAT,
710 					val, val & MESON_SDHC_STAT_TXFIFO_CNT,
711 					MESON_SDHC_WAIT_BEFORE_SEND_SLEEP_US,
712 					MESON_SDHC_WAIT_BEFORE_SEND_TIMEOUT_US);
713 		if (ret)
714 			dev_warn(mmc_dev(mmc),
715 				 "Failed to wait for TX FIFO to fill\n");
716 	}
717 }
718 
719 static void meson_mx_sdhc_init_hw_meson8m2(struct mmc_host *mmc)
720 {
721 	struct meson_mx_sdhc_host *host = mmc_priv(mmc);
722 
723 	regmap_write(host->regmap, MESON_SDHC_MISC,
724 		     FIELD_PREP(MESON_SDHC_MISC_TXSTART_THRES, 6) |
725 		     FIELD_PREP(MESON_SDHC_MISC_WCRC_ERR_PATT, 5) |
726 		     FIELD_PREP(MESON_SDHC_MISC_WCRC_OK_PATT, 2));
727 
728 	regmap_write(host->regmap, MESON_SDHC_ENHC,
729 		     FIELD_PREP(MESON_SDHC_ENHC_RXFIFO_TH, 64) |
730 		     FIELD_PREP(MESON_SDHC_ENHC_MESON8M2_DEBUG, 1) |
731 		     MESON_SDHC_ENHC_MESON8M2_WRRSP_MODE |
732 		     FIELD_PREP(MESON_SDHC_ENHC_SDIO_IRQ_PERIOD, 12));
733 }
734 
735 static void meson_mx_sdhc_set_pdma_meson8m2(struct mmc_host *mmc)
736 {
737 	struct meson_mx_sdhc_host *host = mmc_priv(mmc);
738 
739 	regmap_update_bits(host->regmap, MESON_SDHC_PDMA,
740 			   MESON_SDHC_PDMA_DMA_MODE, MESON_SDHC_PDMA_DMA_MODE);
741 }
742 
743 static void meson_mx_sdhc_init_hw(struct mmc_host *mmc)
744 {
745 	struct meson_mx_sdhc_host *host = mmc_priv(mmc);
746 
747 	meson_mx_sdhc_hw_reset(mmc);
748 
749 	regmap_write(host->regmap, MESON_SDHC_CTRL,
750 		     FIELD_PREP(MESON_SDHC_CTRL_RX_PERIOD, 0xf) |
751 		     FIELD_PREP(MESON_SDHC_CTRL_RX_TIMEOUT, 0x7f) |
752 		     FIELD_PREP(MESON_SDHC_CTRL_RX_ENDIAN, 0x7) |
753 		     FIELD_PREP(MESON_SDHC_CTRL_TX_ENDIAN, 0x7));
754 
755 	/*
756 	 * start with a valid divider and enable the memory (un-setting
757 	 * MESON_SDHC_CLKC_MEM_PWR_OFF).
758 	 */
759 	regmap_write(host->regmap, MESON_SDHC_CLKC, MESON_SDHC_CLKC_CLK_DIV);
760 
761 	regmap_write(host->regmap, MESON_SDHC_CLK2,
762 		     FIELD_PREP(MESON_SDHC_CLK2_SD_CLK_PHASE, 1));
763 
764 	regmap_write(host->regmap, MESON_SDHC_PDMA,
765 		     MESON_SDHC_PDMA_DMA_URGENT |
766 		     FIELD_PREP(MESON_SDHC_PDMA_WR_BURST, 7) |
767 		     FIELD_PREP(MESON_SDHC_PDMA_TXFIFO_TH, 49) |
768 		     FIELD_PREP(MESON_SDHC_PDMA_RD_BURST, 15) |
769 		     FIELD_PREP(MESON_SDHC_PDMA_RXFIFO_TH, 7));
770 
771 	/* some initialization bits depend on the SoC: */
772 	host->platform->init_hw(mmc);
773 
774 	/* disable and mask all interrupts: */
775 	regmap_write(host->regmap, MESON_SDHC_ICTL, 0);
776 	regmap_write(host->regmap, MESON_SDHC_ISTA, MESON_SDHC_ISTA_ALL_IRQS);
777 }
778 
779 static int meson_mx_sdhc_probe(struct platform_device *pdev)
780 {
781 	struct device *dev = &pdev->dev;
782 	struct meson_mx_sdhc_host *host;
783 	struct mmc_host *mmc;
784 	void __iomem *base;
785 	int ret, irq;
786 
787 	mmc = mmc_alloc_host(sizeof(*host), dev);
788 	if (!mmc)
789 		return -ENOMEM;
790 
791 	ret = devm_add_action_or_reset(dev, (void(*)(void *))mmc_free_host,
792 				       mmc);
793 	if (ret) {
794 		dev_err(dev, "Failed to register mmc_free_host action\n");
795 		return ret;
796 	}
797 
798 	host = mmc_priv(mmc);
799 	host->mmc = mmc;
800 
801 	platform_set_drvdata(pdev, host);
802 
803 	host->platform = device_get_match_data(dev);
804 	if (!host->platform)
805 		return -EINVAL;
806 
807 	base = devm_platform_ioremap_resource(pdev, 0);
808 	if (IS_ERR(base))
809 		return PTR_ERR(base);
810 
811 	host->regmap = devm_regmap_init_mmio(dev, base,
812 					     &meson_mx_sdhc_regmap_config);
813 	if (IS_ERR(host->regmap))
814 		return PTR_ERR(host->regmap);
815 
816 	host->pclk = devm_clk_get(dev, "pclk");
817 	if (IS_ERR(host->pclk))
818 		return PTR_ERR(host->pclk);
819 
820 	/* accessing any register requires the module clock to be enabled: */
821 	ret = clk_prepare_enable(host->pclk);
822 	if (ret) {
823 		dev_err(dev, "Failed to enable 'pclk' clock\n");
824 		return ret;
825 	}
826 
827 	meson_mx_sdhc_init_hw(mmc);
828 
829 	ret = meson_mx_sdhc_register_clkc(dev, base, host->bulk_clks);
830 	if (ret)
831 		goto err_disable_pclk;
832 
833 	host->sd_clk = host->bulk_clks[1].clk;
834 
835 	/* Get regulators and the supported OCR mask */
836 	ret = mmc_regulator_get_supply(mmc);
837 	if (ret)
838 		goto err_disable_pclk;
839 
840 	mmc->max_req_size = SZ_128K;
841 	mmc->max_seg_size = mmc->max_req_size;
842 	mmc->max_blk_count = FIELD_GET(MESON_SDHC_SEND_TOTAL_PACK, ~0);
843 	mmc->max_blk_size = MESON_SDHC_MAX_BLK_SIZE;
844 	mmc->max_busy_timeout = 30 * MSEC_PER_SEC;
845 	mmc->f_min = clk_round_rate(host->sd_clk, 1);
846 	mmc->f_max = clk_round_rate(host->sd_clk, ULONG_MAX);
847 	mmc->max_current_180 = 300;
848 	mmc->max_current_330 = 300;
849 	mmc->caps |= MMC_CAP_WAIT_WHILE_BUSY | MMC_CAP_HW_RESET;
850 	mmc->ops = &meson_mx_sdhc_ops;
851 
852 	ret = mmc_of_parse(mmc);
853 	if (ret)
854 		goto err_disable_pclk;
855 
856 	irq = platform_get_irq(pdev, 0);
857 	if (irq < 0) {
858 		ret = irq;
859 		goto err_disable_pclk;
860 	}
861 
862 	ret = devm_request_threaded_irq(dev, irq, meson_mx_sdhc_irq,
863 					meson_mx_sdhc_irq_thread, IRQF_ONESHOT,
864 					NULL, host);
865 	if (ret)
866 		goto err_disable_pclk;
867 
868 	ret = mmc_add_host(mmc);
869 	if (ret)
870 		goto err_disable_pclk;
871 
872 	return 0;
873 
874 err_disable_pclk:
875 	clk_disable_unprepare(host->pclk);
876 	return ret;
877 }
878 
879 static int meson_mx_sdhc_remove(struct platform_device *pdev)
880 {
881 	struct meson_mx_sdhc_host *host = platform_get_drvdata(pdev);
882 
883 	mmc_remove_host(host->mmc);
884 
885 	meson_mx_sdhc_disable_clks(host->mmc);
886 
887 	clk_disable_unprepare(host->pclk);
888 
889 	return 0;
890 }
891 
892 static const struct meson_mx_sdhc_data meson_mx_sdhc_data_meson8 = {
893 	.init_hw			= meson_mx_sdhc_init_hw_meson8,
894 	.set_pdma			= meson_mx_sdhc_set_pdma_meson8,
895 	.wait_before_send		= meson_mx_sdhc_wait_before_send_meson8,
896 	.hardware_flush_all_cmds	= false,
897 };
898 
899 static const struct meson_mx_sdhc_data meson_mx_sdhc_data_meson8m2 = {
900 	.init_hw			= meson_mx_sdhc_init_hw_meson8m2,
901 	.set_pdma			= meson_mx_sdhc_set_pdma_meson8m2,
902 	.hardware_flush_all_cmds	= true,
903 };
904 
905 static const struct of_device_id meson_mx_sdhc_of_match[] = {
906 	{
907 		.compatible = "amlogic,meson8-sdhc",
908 		.data = &meson_mx_sdhc_data_meson8
909 	},
910 	{
911 		.compatible = "amlogic,meson8b-sdhc",
912 		.data = &meson_mx_sdhc_data_meson8
913 	},
914 	{
915 		.compatible = "amlogic,meson8m2-sdhc",
916 		.data = &meson_mx_sdhc_data_meson8m2
917 	},
918 	{ /* sentinel */ }
919 };
920 MODULE_DEVICE_TABLE(of, meson_mx_sdhc_of_match);
921 
922 static struct platform_driver meson_mx_sdhc_driver = {
923 	.probe   = meson_mx_sdhc_probe,
924 	.remove  = meson_mx_sdhc_remove,
925 	.driver  = {
926 		.name = "meson-mx-sdhc",
927 		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
928 		.of_match_table = of_match_ptr(meson_mx_sdhc_of_match),
929 	},
930 };
931 
932 module_platform_driver(meson_mx_sdhc_driver);
933 
934 MODULE_DESCRIPTION("Meson6, Meson8, Meson8b and Meson8m2 SDHC Host Driver");
935 MODULE_AUTHOR("Martin Blumenstingl <martin.blumenstingl@googlemail.com>");
936 MODULE_LICENSE("GPL v2");
937