xref: /openbmc/linux/drivers/mmc/host/sdhci.c (revision bb3982b4)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  *  linux/drivers/mmc/host/sdhci.c - Secure Digital Host Controller Interface driver
4  *
5  *  Copyright (C) 2005-2008 Pierre Ossman, All Rights Reserved.
6  *
7  * Thanks to the following companies for their support:
8  *
9  *     - JMicron (hardware and technical support)
10  */
11 
12 #include <linux/delay.h>
13 #include <linux/ktime.h>
14 #include <linux/highmem.h>
15 #include <linux/io.h>
16 #include <linux/module.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/slab.h>
19 #include <linux/scatterlist.h>
20 #include <linux/sizes.h>
21 #include <linux/swiotlb.h>
22 #include <linux/regulator/consumer.h>
23 #include <linux/pm_runtime.h>
24 #include <linux/of.h>
25 
26 #include <linux/leds.h>
27 
28 #include <linux/mmc/mmc.h>
29 #include <linux/mmc/host.h>
30 #include <linux/mmc/card.h>
31 #include <linux/mmc/sdio.h>
32 #include <linux/mmc/slot-gpio.h>
33 
34 #include "sdhci.h"
35 
36 #define DRIVER_NAME "sdhci"
37 
38 #define DBG(f, x...) \
39 	pr_debug("%s: " DRIVER_NAME ": " f, mmc_hostname(host->mmc), ## x)
40 
41 #define SDHCI_DUMP(f, x...) \
42 	pr_err("%s: " DRIVER_NAME ": " f, mmc_hostname(host->mmc), ## x)
43 
44 #define MAX_TUNING_LOOP 40
45 
46 static unsigned int debug_quirks = 0;
47 static unsigned int debug_quirks2;
48 
49 static void sdhci_finish_data(struct sdhci_host *);
50 
51 static void sdhci_enable_preset_value(struct sdhci_host *host, bool enable);
52 
53 void sdhci_dumpregs(struct sdhci_host *host)
54 {
55 	SDHCI_DUMP("============ SDHCI REGISTER DUMP ===========\n");
56 
57 	SDHCI_DUMP("Sys addr:  0x%08x | Version:  0x%08x\n",
58 		   sdhci_readl(host, SDHCI_DMA_ADDRESS),
59 		   sdhci_readw(host, SDHCI_HOST_VERSION));
60 	SDHCI_DUMP("Blk size:  0x%08x | Blk cnt:  0x%08x\n",
61 		   sdhci_readw(host, SDHCI_BLOCK_SIZE),
62 		   sdhci_readw(host, SDHCI_BLOCK_COUNT));
63 	SDHCI_DUMP("Argument:  0x%08x | Trn mode: 0x%08x\n",
64 		   sdhci_readl(host, SDHCI_ARGUMENT),
65 		   sdhci_readw(host, SDHCI_TRANSFER_MODE));
66 	SDHCI_DUMP("Present:   0x%08x | Host ctl: 0x%08x\n",
67 		   sdhci_readl(host, SDHCI_PRESENT_STATE),
68 		   sdhci_readb(host, SDHCI_HOST_CONTROL));
69 	SDHCI_DUMP("Power:     0x%08x | Blk gap:  0x%08x\n",
70 		   sdhci_readb(host, SDHCI_POWER_CONTROL),
71 		   sdhci_readb(host, SDHCI_BLOCK_GAP_CONTROL));
72 	SDHCI_DUMP("Wake-up:   0x%08x | Clock:    0x%08x\n",
73 		   sdhci_readb(host, SDHCI_WAKE_UP_CONTROL),
74 		   sdhci_readw(host, SDHCI_CLOCK_CONTROL));
75 	SDHCI_DUMP("Timeout:   0x%08x | Int stat: 0x%08x\n",
76 		   sdhci_readb(host, SDHCI_TIMEOUT_CONTROL),
77 		   sdhci_readl(host, SDHCI_INT_STATUS));
78 	SDHCI_DUMP("Int enab:  0x%08x | Sig enab: 0x%08x\n",
79 		   sdhci_readl(host, SDHCI_INT_ENABLE),
80 		   sdhci_readl(host, SDHCI_SIGNAL_ENABLE));
81 	SDHCI_DUMP("ACmd stat: 0x%08x | Slot int: 0x%08x\n",
82 		   sdhci_readw(host, SDHCI_AUTO_CMD_STATUS),
83 		   sdhci_readw(host, SDHCI_SLOT_INT_STATUS));
84 	SDHCI_DUMP("Caps:      0x%08x | Caps_1:   0x%08x\n",
85 		   sdhci_readl(host, SDHCI_CAPABILITIES),
86 		   sdhci_readl(host, SDHCI_CAPABILITIES_1));
87 	SDHCI_DUMP("Cmd:       0x%08x | Max curr: 0x%08x\n",
88 		   sdhci_readw(host, SDHCI_COMMAND),
89 		   sdhci_readl(host, SDHCI_MAX_CURRENT));
90 	SDHCI_DUMP("Resp[0]:   0x%08x | Resp[1]:  0x%08x\n",
91 		   sdhci_readl(host, SDHCI_RESPONSE),
92 		   sdhci_readl(host, SDHCI_RESPONSE + 4));
93 	SDHCI_DUMP("Resp[2]:   0x%08x | Resp[3]:  0x%08x\n",
94 		   sdhci_readl(host, SDHCI_RESPONSE + 8),
95 		   sdhci_readl(host, SDHCI_RESPONSE + 12));
96 	SDHCI_DUMP("Host ctl2: 0x%08x\n",
97 		   sdhci_readw(host, SDHCI_HOST_CONTROL2));
98 
99 	if (host->flags & SDHCI_USE_ADMA) {
100 		if (host->flags & SDHCI_USE_64_BIT_DMA) {
101 			SDHCI_DUMP("ADMA Err:  0x%08x | ADMA Ptr: 0x%08x%08x\n",
102 				   sdhci_readl(host, SDHCI_ADMA_ERROR),
103 				   sdhci_readl(host, SDHCI_ADMA_ADDRESS_HI),
104 				   sdhci_readl(host, SDHCI_ADMA_ADDRESS));
105 		} else {
106 			SDHCI_DUMP("ADMA Err:  0x%08x | ADMA Ptr: 0x%08x\n",
107 				   sdhci_readl(host, SDHCI_ADMA_ERROR),
108 				   sdhci_readl(host, SDHCI_ADMA_ADDRESS));
109 		}
110 	}
111 
112 	SDHCI_DUMP("============================================\n");
113 }
114 EXPORT_SYMBOL_GPL(sdhci_dumpregs);
115 
116 /*****************************************************************************\
117  *                                                                           *
118  * Low level functions                                                       *
119  *                                                                           *
120 \*****************************************************************************/
121 
122 static void sdhci_do_enable_v4_mode(struct sdhci_host *host)
123 {
124 	u16 ctrl2;
125 
126 	ctrl2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
127 	if (ctrl2 & SDHCI_CTRL_V4_MODE)
128 		return;
129 
130 	ctrl2 |= SDHCI_CTRL_V4_MODE;
131 	sdhci_writew(host, ctrl2, SDHCI_HOST_CONTROL2);
132 }
133 
134 /*
135  * This can be called before sdhci_add_host() by Vendor's host controller
136  * driver to enable v4 mode if supported.
137  */
138 void sdhci_enable_v4_mode(struct sdhci_host *host)
139 {
140 	host->v4_mode = true;
141 	sdhci_do_enable_v4_mode(host);
142 }
143 EXPORT_SYMBOL_GPL(sdhci_enable_v4_mode);
144 
145 static inline bool sdhci_data_line_cmd(struct mmc_command *cmd)
146 {
147 	return cmd->data || cmd->flags & MMC_RSP_BUSY;
148 }
149 
150 static void sdhci_set_card_detection(struct sdhci_host *host, bool enable)
151 {
152 	u32 present;
153 
154 	if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) ||
155 	    !mmc_card_is_removable(host->mmc))
156 		return;
157 
158 	if (enable) {
159 		present = sdhci_readl(host, SDHCI_PRESENT_STATE) &
160 				      SDHCI_CARD_PRESENT;
161 
162 		host->ier |= present ? SDHCI_INT_CARD_REMOVE :
163 				       SDHCI_INT_CARD_INSERT;
164 	} else {
165 		host->ier &= ~(SDHCI_INT_CARD_REMOVE | SDHCI_INT_CARD_INSERT);
166 	}
167 
168 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
169 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
170 }
171 
172 static void sdhci_enable_card_detection(struct sdhci_host *host)
173 {
174 	sdhci_set_card_detection(host, true);
175 }
176 
177 static void sdhci_disable_card_detection(struct sdhci_host *host)
178 {
179 	sdhci_set_card_detection(host, false);
180 }
181 
182 static void sdhci_runtime_pm_bus_on(struct sdhci_host *host)
183 {
184 	if (host->bus_on)
185 		return;
186 	host->bus_on = true;
187 	pm_runtime_get_noresume(host->mmc->parent);
188 }
189 
190 static void sdhci_runtime_pm_bus_off(struct sdhci_host *host)
191 {
192 	if (!host->bus_on)
193 		return;
194 	host->bus_on = false;
195 	pm_runtime_put_noidle(host->mmc->parent);
196 }
197 
198 void sdhci_reset(struct sdhci_host *host, u8 mask)
199 {
200 	ktime_t timeout;
201 
202 	sdhci_writeb(host, mask, SDHCI_SOFTWARE_RESET);
203 
204 	if (mask & SDHCI_RESET_ALL) {
205 		host->clock = 0;
206 		/* Reset-all turns off SD Bus Power */
207 		if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
208 			sdhci_runtime_pm_bus_off(host);
209 	}
210 
211 	/* Wait max 100 ms */
212 	timeout = ktime_add_ms(ktime_get(), 100);
213 
214 	/* hw clears the bit when it's done */
215 	while (1) {
216 		bool timedout = ktime_after(ktime_get(), timeout);
217 
218 		if (!(sdhci_readb(host, SDHCI_SOFTWARE_RESET) & mask))
219 			break;
220 		if (timedout) {
221 			pr_err("%s: Reset 0x%x never completed.\n",
222 				mmc_hostname(host->mmc), (int)mask);
223 			sdhci_dumpregs(host);
224 			return;
225 		}
226 		udelay(10);
227 	}
228 }
229 EXPORT_SYMBOL_GPL(sdhci_reset);
230 
231 static void sdhci_do_reset(struct sdhci_host *host, u8 mask)
232 {
233 	if (host->quirks & SDHCI_QUIRK_NO_CARD_NO_RESET) {
234 		struct mmc_host *mmc = host->mmc;
235 
236 		if (!mmc->ops->get_cd(mmc))
237 			return;
238 	}
239 
240 	host->ops->reset(host, mask);
241 
242 	if (mask & SDHCI_RESET_ALL) {
243 		if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
244 			if (host->ops->enable_dma)
245 				host->ops->enable_dma(host);
246 		}
247 
248 		/* Resetting the controller clears many */
249 		host->preset_enabled = false;
250 	}
251 }
252 
253 static void sdhci_set_default_irqs(struct sdhci_host *host)
254 {
255 	host->ier = SDHCI_INT_BUS_POWER | SDHCI_INT_DATA_END_BIT |
256 		    SDHCI_INT_DATA_CRC | SDHCI_INT_DATA_TIMEOUT |
257 		    SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC |
258 		    SDHCI_INT_TIMEOUT | SDHCI_INT_DATA_END |
259 		    SDHCI_INT_RESPONSE;
260 
261 	if (host->tuning_mode == SDHCI_TUNING_MODE_2 ||
262 	    host->tuning_mode == SDHCI_TUNING_MODE_3)
263 		host->ier |= SDHCI_INT_RETUNE;
264 
265 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
266 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
267 }
268 
269 static void sdhci_config_dma(struct sdhci_host *host)
270 {
271 	u8 ctrl;
272 	u16 ctrl2;
273 
274 	if (host->version < SDHCI_SPEC_200)
275 		return;
276 
277 	ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
278 
279 	/*
280 	 * Always adjust the DMA selection as some controllers
281 	 * (e.g. JMicron) can't do PIO properly when the selection
282 	 * is ADMA.
283 	 */
284 	ctrl &= ~SDHCI_CTRL_DMA_MASK;
285 	if (!(host->flags & SDHCI_REQ_USE_DMA))
286 		goto out;
287 
288 	/* Note if DMA Select is zero then SDMA is selected */
289 	if (host->flags & SDHCI_USE_ADMA)
290 		ctrl |= SDHCI_CTRL_ADMA32;
291 
292 	if (host->flags & SDHCI_USE_64_BIT_DMA) {
293 		/*
294 		 * If v4 mode, all supported DMA can be 64-bit addressing if
295 		 * controller supports 64-bit system address, otherwise only
296 		 * ADMA can support 64-bit addressing.
297 		 */
298 		if (host->v4_mode) {
299 			ctrl2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
300 			ctrl2 |= SDHCI_CTRL_64BIT_ADDR;
301 			sdhci_writew(host, ctrl2, SDHCI_HOST_CONTROL2);
302 		} else if (host->flags & SDHCI_USE_ADMA) {
303 			/*
304 			 * Don't need to undo SDHCI_CTRL_ADMA32 in order to
305 			 * set SDHCI_CTRL_ADMA64.
306 			 */
307 			ctrl |= SDHCI_CTRL_ADMA64;
308 		}
309 	}
310 
311 out:
312 	sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
313 }
314 
315 static void sdhci_init(struct sdhci_host *host, int soft)
316 {
317 	struct mmc_host *mmc = host->mmc;
318 
319 	if (soft)
320 		sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
321 	else
322 		sdhci_do_reset(host, SDHCI_RESET_ALL);
323 
324 	if (host->v4_mode)
325 		sdhci_do_enable_v4_mode(host);
326 
327 	sdhci_set_default_irqs(host);
328 
329 	host->cqe_on = false;
330 
331 	if (soft) {
332 		/* force clock reconfiguration */
333 		host->clock = 0;
334 		mmc->ops->set_ios(mmc, &mmc->ios);
335 	}
336 }
337 
338 static void sdhci_reinit(struct sdhci_host *host)
339 {
340 	sdhci_init(host, 0);
341 	sdhci_enable_card_detection(host);
342 }
343 
344 static void __sdhci_led_activate(struct sdhci_host *host)
345 {
346 	u8 ctrl;
347 
348 	if (host->quirks & SDHCI_QUIRK_NO_LED)
349 		return;
350 
351 	ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
352 	ctrl |= SDHCI_CTRL_LED;
353 	sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
354 }
355 
356 static void __sdhci_led_deactivate(struct sdhci_host *host)
357 {
358 	u8 ctrl;
359 
360 	if (host->quirks & SDHCI_QUIRK_NO_LED)
361 		return;
362 
363 	ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
364 	ctrl &= ~SDHCI_CTRL_LED;
365 	sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
366 }
367 
368 #if IS_REACHABLE(CONFIG_LEDS_CLASS)
369 static void sdhci_led_control(struct led_classdev *led,
370 			      enum led_brightness brightness)
371 {
372 	struct sdhci_host *host = container_of(led, struct sdhci_host, led);
373 	unsigned long flags;
374 
375 	spin_lock_irqsave(&host->lock, flags);
376 
377 	if (host->runtime_suspended)
378 		goto out;
379 
380 	if (brightness == LED_OFF)
381 		__sdhci_led_deactivate(host);
382 	else
383 		__sdhci_led_activate(host);
384 out:
385 	spin_unlock_irqrestore(&host->lock, flags);
386 }
387 
388 static int sdhci_led_register(struct sdhci_host *host)
389 {
390 	struct mmc_host *mmc = host->mmc;
391 
392 	if (host->quirks & SDHCI_QUIRK_NO_LED)
393 		return 0;
394 
395 	snprintf(host->led_name, sizeof(host->led_name),
396 		 "%s::", mmc_hostname(mmc));
397 
398 	host->led.name = host->led_name;
399 	host->led.brightness = LED_OFF;
400 	host->led.default_trigger = mmc_hostname(mmc);
401 	host->led.brightness_set = sdhci_led_control;
402 
403 	return led_classdev_register(mmc_dev(mmc), &host->led);
404 }
405 
406 static void sdhci_led_unregister(struct sdhci_host *host)
407 {
408 	if (host->quirks & SDHCI_QUIRK_NO_LED)
409 		return;
410 
411 	led_classdev_unregister(&host->led);
412 }
413 
414 static inline void sdhci_led_activate(struct sdhci_host *host)
415 {
416 }
417 
418 static inline void sdhci_led_deactivate(struct sdhci_host *host)
419 {
420 }
421 
422 #else
423 
424 static inline int sdhci_led_register(struct sdhci_host *host)
425 {
426 	return 0;
427 }
428 
429 static inline void sdhci_led_unregister(struct sdhci_host *host)
430 {
431 }
432 
433 static inline void sdhci_led_activate(struct sdhci_host *host)
434 {
435 	__sdhci_led_activate(host);
436 }
437 
438 static inline void sdhci_led_deactivate(struct sdhci_host *host)
439 {
440 	__sdhci_led_deactivate(host);
441 }
442 
443 #endif
444 
445 static void sdhci_mod_timer(struct sdhci_host *host, struct mmc_request *mrq,
446 			    unsigned long timeout)
447 {
448 	if (sdhci_data_line_cmd(mrq->cmd))
449 		mod_timer(&host->data_timer, timeout);
450 	else
451 		mod_timer(&host->timer, timeout);
452 }
453 
454 static void sdhci_del_timer(struct sdhci_host *host, struct mmc_request *mrq)
455 {
456 	if (sdhci_data_line_cmd(mrq->cmd))
457 		del_timer(&host->data_timer);
458 	else
459 		del_timer(&host->timer);
460 }
461 
462 static inline bool sdhci_has_requests(struct sdhci_host *host)
463 {
464 	return host->cmd || host->data_cmd;
465 }
466 
467 /*****************************************************************************\
468  *                                                                           *
469  * Core functions                                                            *
470  *                                                                           *
471 \*****************************************************************************/
472 
473 static void sdhci_read_block_pio(struct sdhci_host *host)
474 {
475 	unsigned long flags;
476 	size_t blksize, len, chunk;
477 	u32 uninitialized_var(scratch);
478 	u8 *buf;
479 
480 	DBG("PIO reading\n");
481 
482 	blksize = host->data->blksz;
483 	chunk = 0;
484 
485 	local_irq_save(flags);
486 
487 	while (blksize) {
488 		BUG_ON(!sg_miter_next(&host->sg_miter));
489 
490 		len = min(host->sg_miter.length, blksize);
491 
492 		blksize -= len;
493 		host->sg_miter.consumed = len;
494 
495 		buf = host->sg_miter.addr;
496 
497 		while (len) {
498 			if (chunk == 0) {
499 				scratch = sdhci_readl(host, SDHCI_BUFFER);
500 				chunk = 4;
501 			}
502 
503 			*buf = scratch & 0xFF;
504 
505 			buf++;
506 			scratch >>= 8;
507 			chunk--;
508 			len--;
509 		}
510 	}
511 
512 	sg_miter_stop(&host->sg_miter);
513 
514 	local_irq_restore(flags);
515 }
516 
517 static void sdhci_write_block_pio(struct sdhci_host *host)
518 {
519 	unsigned long flags;
520 	size_t blksize, len, chunk;
521 	u32 scratch;
522 	u8 *buf;
523 
524 	DBG("PIO writing\n");
525 
526 	blksize = host->data->blksz;
527 	chunk = 0;
528 	scratch = 0;
529 
530 	local_irq_save(flags);
531 
532 	while (blksize) {
533 		BUG_ON(!sg_miter_next(&host->sg_miter));
534 
535 		len = min(host->sg_miter.length, blksize);
536 
537 		blksize -= len;
538 		host->sg_miter.consumed = len;
539 
540 		buf = host->sg_miter.addr;
541 
542 		while (len) {
543 			scratch |= (u32)*buf << (chunk * 8);
544 
545 			buf++;
546 			chunk++;
547 			len--;
548 
549 			if ((chunk == 4) || ((len == 0) && (blksize == 0))) {
550 				sdhci_writel(host, scratch, SDHCI_BUFFER);
551 				chunk = 0;
552 				scratch = 0;
553 			}
554 		}
555 	}
556 
557 	sg_miter_stop(&host->sg_miter);
558 
559 	local_irq_restore(flags);
560 }
561 
562 static void sdhci_transfer_pio(struct sdhci_host *host)
563 {
564 	u32 mask;
565 
566 	if (host->blocks == 0)
567 		return;
568 
569 	if (host->data->flags & MMC_DATA_READ)
570 		mask = SDHCI_DATA_AVAILABLE;
571 	else
572 		mask = SDHCI_SPACE_AVAILABLE;
573 
574 	/*
575 	 * Some controllers (JMicron JMB38x) mess up the buffer bits
576 	 * for transfers < 4 bytes. As long as it is just one block,
577 	 * we can ignore the bits.
578 	 */
579 	if ((host->quirks & SDHCI_QUIRK_BROKEN_SMALL_PIO) &&
580 		(host->data->blocks == 1))
581 		mask = ~0;
582 
583 	while (sdhci_readl(host, SDHCI_PRESENT_STATE) & mask) {
584 		if (host->quirks & SDHCI_QUIRK_PIO_NEEDS_DELAY)
585 			udelay(100);
586 
587 		if (host->data->flags & MMC_DATA_READ)
588 			sdhci_read_block_pio(host);
589 		else
590 			sdhci_write_block_pio(host);
591 
592 		host->blocks--;
593 		if (host->blocks == 0)
594 			break;
595 	}
596 
597 	DBG("PIO transfer complete.\n");
598 }
599 
600 static int sdhci_pre_dma_transfer(struct sdhci_host *host,
601 				  struct mmc_data *data, int cookie)
602 {
603 	int sg_count;
604 
605 	/*
606 	 * If the data buffers are already mapped, return the previous
607 	 * dma_map_sg() result.
608 	 */
609 	if (data->host_cookie == COOKIE_PRE_MAPPED)
610 		return data->sg_count;
611 
612 	/* Bounce write requests to the bounce buffer */
613 	if (host->bounce_buffer) {
614 		unsigned int length = data->blksz * data->blocks;
615 
616 		if (length > host->bounce_buffer_size) {
617 			pr_err("%s: asked for transfer of %u bytes exceeds bounce buffer %u bytes\n",
618 			       mmc_hostname(host->mmc), length,
619 			       host->bounce_buffer_size);
620 			return -EIO;
621 		}
622 		if (mmc_get_dma_dir(data) == DMA_TO_DEVICE) {
623 			/* Copy the data to the bounce buffer */
624 			sg_copy_to_buffer(data->sg, data->sg_len,
625 					  host->bounce_buffer,
626 					  length);
627 		}
628 		/* Switch ownership to the DMA */
629 		dma_sync_single_for_device(host->mmc->parent,
630 					   host->bounce_addr,
631 					   host->bounce_buffer_size,
632 					   mmc_get_dma_dir(data));
633 		/* Just a dummy value */
634 		sg_count = 1;
635 	} else {
636 		/* Just access the data directly from memory */
637 		sg_count = dma_map_sg(mmc_dev(host->mmc),
638 				      data->sg, data->sg_len,
639 				      mmc_get_dma_dir(data));
640 	}
641 
642 	if (sg_count == 0)
643 		return -ENOSPC;
644 
645 	data->sg_count = sg_count;
646 	data->host_cookie = cookie;
647 
648 	return sg_count;
649 }
650 
651 static char *sdhci_kmap_atomic(struct scatterlist *sg, unsigned long *flags)
652 {
653 	local_irq_save(*flags);
654 	return kmap_atomic(sg_page(sg)) + sg->offset;
655 }
656 
657 static void sdhci_kunmap_atomic(void *buffer, unsigned long *flags)
658 {
659 	kunmap_atomic(buffer);
660 	local_irq_restore(*flags);
661 }
662 
663 void sdhci_adma_write_desc(struct sdhci_host *host, void **desc,
664 			   dma_addr_t addr, int len, unsigned int cmd)
665 {
666 	struct sdhci_adma2_64_desc *dma_desc = *desc;
667 
668 	/* 32-bit and 64-bit descriptors have these members in same position */
669 	dma_desc->cmd = cpu_to_le16(cmd);
670 	dma_desc->len = cpu_to_le16(len);
671 	dma_desc->addr_lo = cpu_to_le32((u32)addr);
672 
673 	if (host->flags & SDHCI_USE_64_BIT_DMA)
674 		dma_desc->addr_hi = cpu_to_le32((u64)addr >> 32);
675 
676 	*desc += host->desc_sz;
677 }
678 EXPORT_SYMBOL_GPL(sdhci_adma_write_desc);
679 
680 static inline void __sdhci_adma_write_desc(struct sdhci_host *host,
681 					   void **desc, dma_addr_t addr,
682 					   int len, unsigned int cmd)
683 {
684 	if (host->ops->adma_write_desc)
685 		host->ops->adma_write_desc(host, desc, addr, len, cmd);
686 	else
687 		sdhci_adma_write_desc(host, desc, addr, len, cmd);
688 }
689 
690 static void sdhci_adma_mark_end(void *desc)
691 {
692 	struct sdhci_adma2_64_desc *dma_desc = desc;
693 
694 	/* 32-bit and 64-bit descriptors have 'cmd' in same position */
695 	dma_desc->cmd |= cpu_to_le16(ADMA2_END);
696 }
697 
698 static void sdhci_adma_table_pre(struct sdhci_host *host,
699 	struct mmc_data *data, int sg_count)
700 {
701 	struct scatterlist *sg;
702 	unsigned long flags;
703 	dma_addr_t addr, align_addr;
704 	void *desc, *align;
705 	char *buffer;
706 	int len, offset, i;
707 
708 	/*
709 	 * The spec does not specify endianness of descriptor table.
710 	 * We currently guess that it is LE.
711 	 */
712 
713 	host->sg_count = sg_count;
714 
715 	desc = host->adma_table;
716 	align = host->align_buffer;
717 
718 	align_addr = host->align_addr;
719 
720 	for_each_sg(data->sg, sg, host->sg_count, i) {
721 		addr = sg_dma_address(sg);
722 		len = sg_dma_len(sg);
723 
724 		/*
725 		 * The SDHCI specification states that ADMA addresses must
726 		 * be 32-bit aligned. If they aren't, then we use a bounce
727 		 * buffer for the (up to three) bytes that screw up the
728 		 * alignment.
729 		 */
730 		offset = (SDHCI_ADMA2_ALIGN - (addr & SDHCI_ADMA2_MASK)) &
731 			 SDHCI_ADMA2_MASK;
732 		if (offset) {
733 			if (data->flags & MMC_DATA_WRITE) {
734 				buffer = sdhci_kmap_atomic(sg, &flags);
735 				memcpy(align, buffer, offset);
736 				sdhci_kunmap_atomic(buffer, &flags);
737 			}
738 
739 			/* tran, valid */
740 			__sdhci_adma_write_desc(host, &desc, align_addr,
741 						offset, ADMA2_TRAN_VALID);
742 
743 			BUG_ON(offset > 65536);
744 
745 			align += SDHCI_ADMA2_ALIGN;
746 			align_addr += SDHCI_ADMA2_ALIGN;
747 
748 			addr += offset;
749 			len -= offset;
750 		}
751 
752 		BUG_ON(len > 65536);
753 
754 		/* tran, valid */
755 		if (len)
756 			__sdhci_adma_write_desc(host, &desc, addr, len,
757 						ADMA2_TRAN_VALID);
758 
759 		/*
760 		 * If this triggers then we have a calculation bug
761 		 * somewhere. :/
762 		 */
763 		WARN_ON((desc - host->adma_table) >= host->adma_table_sz);
764 	}
765 
766 	if (host->quirks & SDHCI_QUIRK_NO_ENDATTR_IN_NOPDESC) {
767 		/* Mark the last descriptor as the terminating descriptor */
768 		if (desc != host->adma_table) {
769 			desc -= host->desc_sz;
770 			sdhci_adma_mark_end(desc);
771 		}
772 	} else {
773 		/* Add a terminating entry - nop, end, valid */
774 		__sdhci_adma_write_desc(host, &desc, 0, 0, ADMA2_NOP_END_VALID);
775 	}
776 }
777 
778 static void sdhci_adma_table_post(struct sdhci_host *host,
779 	struct mmc_data *data)
780 {
781 	struct scatterlist *sg;
782 	int i, size;
783 	void *align;
784 	char *buffer;
785 	unsigned long flags;
786 
787 	if (data->flags & MMC_DATA_READ) {
788 		bool has_unaligned = false;
789 
790 		/* Do a quick scan of the SG list for any unaligned mappings */
791 		for_each_sg(data->sg, sg, host->sg_count, i)
792 			if (sg_dma_address(sg) & SDHCI_ADMA2_MASK) {
793 				has_unaligned = true;
794 				break;
795 			}
796 
797 		if (has_unaligned) {
798 			dma_sync_sg_for_cpu(mmc_dev(host->mmc), data->sg,
799 					    data->sg_len, DMA_FROM_DEVICE);
800 
801 			align = host->align_buffer;
802 
803 			for_each_sg(data->sg, sg, host->sg_count, i) {
804 				if (sg_dma_address(sg) & SDHCI_ADMA2_MASK) {
805 					size = SDHCI_ADMA2_ALIGN -
806 					       (sg_dma_address(sg) & SDHCI_ADMA2_MASK);
807 
808 					buffer = sdhci_kmap_atomic(sg, &flags);
809 					memcpy(buffer, align, size);
810 					sdhci_kunmap_atomic(buffer, &flags);
811 
812 					align += SDHCI_ADMA2_ALIGN;
813 				}
814 			}
815 		}
816 	}
817 }
818 
819 static dma_addr_t sdhci_sdma_address(struct sdhci_host *host)
820 {
821 	if (host->bounce_buffer)
822 		return host->bounce_addr;
823 	else
824 		return sg_dma_address(host->data->sg);
825 }
826 
827 static void sdhci_set_sdma_addr(struct sdhci_host *host, dma_addr_t addr)
828 {
829 	if (host->v4_mode) {
830 		sdhci_writel(host, addr, SDHCI_ADMA_ADDRESS);
831 		if (host->flags & SDHCI_USE_64_BIT_DMA)
832 			sdhci_writel(host, (u64)addr >> 32, SDHCI_ADMA_ADDRESS_HI);
833 	} else {
834 		sdhci_writel(host, addr, SDHCI_DMA_ADDRESS);
835 	}
836 }
837 
838 static unsigned int sdhci_target_timeout(struct sdhci_host *host,
839 					 struct mmc_command *cmd,
840 					 struct mmc_data *data)
841 {
842 	unsigned int target_timeout;
843 
844 	/* timeout in us */
845 	if (!data) {
846 		target_timeout = cmd->busy_timeout * 1000;
847 	} else {
848 		target_timeout = DIV_ROUND_UP(data->timeout_ns, 1000);
849 		if (host->clock && data->timeout_clks) {
850 			unsigned long long val;
851 
852 			/*
853 			 * data->timeout_clks is in units of clock cycles.
854 			 * host->clock is in Hz.  target_timeout is in us.
855 			 * Hence, us = 1000000 * cycles / Hz.  Round up.
856 			 */
857 			val = 1000000ULL * data->timeout_clks;
858 			if (do_div(val, host->clock))
859 				target_timeout++;
860 			target_timeout += val;
861 		}
862 	}
863 
864 	return target_timeout;
865 }
866 
867 static void sdhci_calc_sw_timeout(struct sdhci_host *host,
868 				  struct mmc_command *cmd)
869 {
870 	struct mmc_data *data = cmd->data;
871 	struct mmc_host *mmc = host->mmc;
872 	struct mmc_ios *ios = &mmc->ios;
873 	unsigned char bus_width = 1 << ios->bus_width;
874 	unsigned int blksz;
875 	unsigned int freq;
876 	u64 target_timeout;
877 	u64 transfer_time;
878 
879 	target_timeout = sdhci_target_timeout(host, cmd, data);
880 	target_timeout *= NSEC_PER_USEC;
881 
882 	if (data) {
883 		blksz = data->blksz;
884 		freq = host->mmc->actual_clock ? : host->clock;
885 		transfer_time = (u64)blksz * NSEC_PER_SEC * (8 / bus_width);
886 		do_div(transfer_time, freq);
887 		/* multiply by '2' to account for any unknowns */
888 		transfer_time = transfer_time * 2;
889 		/* calculate timeout for the entire data */
890 		host->data_timeout = data->blocks * target_timeout +
891 				     transfer_time;
892 	} else {
893 		host->data_timeout = target_timeout;
894 	}
895 
896 	if (host->data_timeout)
897 		host->data_timeout += MMC_CMD_TRANSFER_TIME;
898 }
899 
900 static u8 sdhci_calc_timeout(struct sdhci_host *host, struct mmc_command *cmd,
901 			     bool *too_big)
902 {
903 	u8 count;
904 	struct mmc_data *data;
905 	unsigned target_timeout, current_timeout;
906 
907 	*too_big = true;
908 
909 	/*
910 	 * If the host controller provides us with an incorrect timeout
911 	 * value, just skip the check and use 0xE.  The hardware may take
912 	 * longer to time out, but that's much better than having a too-short
913 	 * timeout value.
914 	 */
915 	if (host->quirks & SDHCI_QUIRK_BROKEN_TIMEOUT_VAL)
916 		return 0xE;
917 
918 	/* Unspecified command, asume max */
919 	if (cmd == NULL)
920 		return 0xE;
921 
922 	data = cmd->data;
923 	/* Unspecified timeout, assume max */
924 	if (!data && !cmd->busy_timeout)
925 		return 0xE;
926 
927 	/* timeout in us */
928 	target_timeout = sdhci_target_timeout(host, cmd, data);
929 
930 	/*
931 	 * Figure out needed cycles.
932 	 * We do this in steps in order to fit inside a 32 bit int.
933 	 * The first step is the minimum timeout, which will have a
934 	 * minimum resolution of 6 bits:
935 	 * (1) 2^13*1000 > 2^22,
936 	 * (2) host->timeout_clk < 2^16
937 	 *     =>
938 	 *     (1) / (2) > 2^6
939 	 */
940 	count = 0;
941 	current_timeout = (1 << 13) * 1000 / host->timeout_clk;
942 	while (current_timeout < target_timeout) {
943 		count++;
944 		current_timeout <<= 1;
945 		if (count >= 0xF)
946 			break;
947 	}
948 
949 	if (count >= 0xF) {
950 		if (!(host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT))
951 			DBG("Too large timeout 0x%x requested for CMD%d!\n",
952 			    count, cmd->opcode);
953 		count = 0xE;
954 	} else {
955 		*too_big = false;
956 	}
957 
958 	return count;
959 }
960 
961 static void sdhci_set_transfer_irqs(struct sdhci_host *host)
962 {
963 	u32 pio_irqs = SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL;
964 	u32 dma_irqs = SDHCI_INT_DMA_END | SDHCI_INT_ADMA_ERROR;
965 
966 	if (host->flags & SDHCI_REQ_USE_DMA)
967 		host->ier = (host->ier & ~pio_irqs) | dma_irqs;
968 	else
969 		host->ier = (host->ier & ~dma_irqs) | pio_irqs;
970 
971 	if (host->flags & (SDHCI_AUTO_CMD23 | SDHCI_AUTO_CMD12))
972 		host->ier |= SDHCI_INT_AUTO_CMD_ERR;
973 	else
974 		host->ier &= ~SDHCI_INT_AUTO_CMD_ERR;
975 
976 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
977 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
978 }
979 
980 static void sdhci_set_data_timeout_irq(struct sdhci_host *host, bool enable)
981 {
982 	if (enable)
983 		host->ier |= SDHCI_INT_DATA_TIMEOUT;
984 	else
985 		host->ier &= ~SDHCI_INT_DATA_TIMEOUT;
986 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
987 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
988 }
989 
990 static void sdhci_set_timeout(struct sdhci_host *host, struct mmc_command *cmd)
991 {
992 	u8 count;
993 
994 	if (host->ops->set_timeout) {
995 		host->ops->set_timeout(host, cmd);
996 	} else {
997 		bool too_big = false;
998 
999 		count = sdhci_calc_timeout(host, cmd, &too_big);
1000 
1001 		if (too_big &&
1002 		    host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT) {
1003 			sdhci_calc_sw_timeout(host, cmd);
1004 			sdhci_set_data_timeout_irq(host, false);
1005 		} else if (!(host->ier & SDHCI_INT_DATA_TIMEOUT)) {
1006 			sdhci_set_data_timeout_irq(host, true);
1007 		}
1008 
1009 		sdhci_writeb(host, count, SDHCI_TIMEOUT_CONTROL);
1010 	}
1011 }
1012 
1013 static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd)
1014 {
1015 	struct mmc_data *data = cmd->data;
1016 
1017 	host->data_timeout = 0;
1018 
1019 	if (sdhci_data_line_cmd(cmd))
1020 		sdhci_set_timeout(host, cmd);
1021 
1022 	if (!data)
1023 		return;
1024 
1025 	WARN_ON(host->data);
1026 
1027 	/* Sanity checks */
1028 	BUG_ON(data->blksz * data->blocks > 524288);
1029 	BUG_ON(data->blksz > host->mmc->max_blk_size);
1030 	BUG_ON(data->blocks > 65535);
1031 
1032 	host->data = data;
1033 	host->data_early = 0;
1034 	host->data->bytes_xfered = 0;
1035 
1036 	if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
1037 		struct scatterlist *sg;
1038 		unsigned int length_mask, offset_mask;
1039 		int i;
1040 
1041 		host->flags |= SDHCI_REQ_USE_DMA;
1042 
1043 		/*
1044 		 * FIXME: This doesn't account for merging when mapping the
1045 		 * scatterlist.
1046 		 *
1047 		 * The assumption here being that alignment and lengths are
1048 		 * the same after DMA mapping to device address space.
1049 		 */
1050 		length_mask = 0;
1051 		offset_mask = 0;
1052 		if (host->flags & SDHCI_USE_ADMA) {
1053 			if (host->quirks & SDHCI_QUIRK_32BIT_ADMA_SIZE) {
1054 				length_mask = 3;
1055 				/*
1056 				 * As we use up to 3 byte chunks to work
1057 				 * around alignment problems, we need to
1058 				 * check the offset as well.
1059 				 */
1060 				offset_mask = 3;
1061 			}
1062 		} else {
1063 			if (host->quirks & SDHCI_QUIRK_32BIT_DMA_SIZE)
1064 				length_mask = 3;
1065 			if (host->quirks & SDHCI_QUIRK_32BIT_DMA_ADDR)
1066 				offset_mask = 3;
1067 		}
1068 
1069 		if (unlikely(length_mask | offset_mask)) {
1070 			for_each_sg(data->sg, sg, data->sg_len, i) {
1071 				if (sg->length & length_mask) {
1072 					DBG("Reverting to PIO because of transfer size (%d)\n",
1073 					    sg->length);
1074 					host->flags &= ~SDHCI_REQ_USE_DMA;
1075 					break;
1076 				}
1077 				if (sg->offset & offset_mask) {
1078 					DBG("Reverting to PIO because of bad alignment\n");
1079 					host->flags &= ~SDHCI_REQ_USE_DMA;
1080 					break;
1081 				}
1082 			}
1083 		}
1084 	}
1085 
1086 	if (host->flags & SDHCI_REQ_USE_DMA) {
1087 		int sg_cnt = sdhci_pre_dma_transfer(host, data, COOKIE_MAPPED);
1088 
1089 		if (sg_cnt <= 0) {
1090 			/*
1091 			 * This only happens when someone fed
1092 			 * us an invalid request.
1093 			 */
1094 			WARN_ON(1);
1095 			host->flags &= ~SDHCI_REQ_USE_DMA;
1096 		} else if (host->flags & SDHCI_USE_ADMA) {
1097 			sdhci_adma_table_pre(host, data, sg_cnt);
1098 
1099 			sdhci_writel(host, host->adma_addr, SDHCI_ADMA_ADDRESS);
1100 			if (host->flags & SDHCI_USE_64_BIT_DMA)
1101 				sdhci_writel(host,
1102 					     (u64)host->adma_addr >> 32,
1103 					     SDHCI_ADMA_ADDRESS_HI);
1104 		} else {
1105 			WARN_ON(sg_cnt != 1);
1106 			sdhci_set_sdma_addr(host, sdhci_sdma_address(host));
1107 		}
1108 	}
1109 
1110 	sdhci_config_dma(host);
1111 
1112 	if (!(host->flags & SDHCI_REQ_USE_DMA)) {
1113 		int flags;
1114 
1115 		flags = SG_MITER_ATOMIC;
1116 		if (host->data->flags & MMC_DATA_READ)
1117 			flags |= SG_MITER_TO_SG;
1118 		else
1119 			flags |= SG_MITER_FROM_SG;
1120 		sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags);
1121 		host->blocks = data->blocks;
1122 	}
1123 
1124 	sdhci_set_transfer_irqs(host);
1125 
1126 	/* Set the DMA boundary value and block size */
1127 	sdhci_writew(host, SDHCI_MAKE_BLKSZ(host->sdma_boundary, data->blksz),
1128 		     SDHCI_BLOCK_SIZE);
1129 
1130 	/*
1131 	 * For Version 4.10 onwards, if v4 mode is enabled, 32-bit Block Count
1132 	 * can be supported, in that case 16-bit block count register must be 0.
1133 	 */
1134 	if (host->version >= SDHCI_SPEC_410 && host->v4_mode &&
1135 	    (host->quirks2 & SDHCI_QUIRK2_USE_32BIT_BLK_CNT)) {
1136 		if (sdhci_readw(host, SDHCI_BLOCK_COUNT))
1137 			sdhci_writew(host, 0, SDHCI_BLOCK_COUNT);
1138 		sdhci_writew(host, data->blocks, SDHCI_32BIT_BLK_CNT);
1139 	} else {
1140 		sdhci_writew(host, data->blocks, SDHCI_BLOCK_COUNT);
1141 	}
1142 }
1143 
1144 static inline bool sdhci_auto_cmd12(struct sdhci_host *host,
1145 				    struct mmc_request *mrq)
1146 {
1147 	return !mrq->sbc && (host->flags & SDHCI_AUTO_CMD12) &&
1148 	       !mrq->cap_cmd_during_tfr;
1149 }
1150 
1151 static inline void sdhci_auto_cmd_select(struct sdhci_host *host,
1152 					 struct mmc_command *cmd,
1153 					 u16 *mode)
1154 {
1155 	bool use_cmd12 = sdhci_auto_cmd12(host, cmd->mrq) &&
1156 			 (cmd->opcode != SD_IO_RW_EXTENDED);
1157 	bool use_cmd23 = cmd->mrq->sbc && (host->flags & SDHCI_AUTO_CMD23);
1158 	u16 ctrl2;
1159 
1160 	/*
1161 	 * In case of Version 4.10 or later, use of 'Auto CMD Auto
1162 	 * Select' is recommended rather than use of 'Auto CMD12
1163 	 * Enable' or 'Auto CMD23 Enable'.
1164 	 */
1165 	if (host->version >= SDHCI_SPEC_410 && (use_cmd12 || use_cmd23)) {
1166 		*mode |= SDHCI_TRNS_AUTO_SEL;
1167 
1168 		ctrl2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1169 		if (use_cmd23)
1170 			ctrl2 |= SDHCI_CMD23_ENABLE;
1171 		else
1172 			ctrl2 &= ~SDHCI_CMD23_ENABLE;
1173 		sdhci_writew(host, ctrl2, SDHCI_HOST_CONTROL2);
1174 
1175 		return;
1176 	}
1177 
1178 	/*
1179 	 * If we are sending CMD23, CMD12 never gets sent
1180 	 * on successful completion (so no Auto-CMD12).
1181 	 */
1182 	if (use_cmd12)
1183 		*mode |= SDHCI_TRNS_AUTO_CMD12;
1184 	else if (use_cmd23)
1185 		*mode |= SDHCI_TRNS_AUTO_CMD23;
1186 }
1187 
1188 static void sdhci_set_transfer_mode(struct sdhci_host *host,
1189 	struct mmc_command *cmd)
1190 {
1191 	u16 mode = 0;
1192 	struct mmc_data *data = cmd->data;
1193 
1194 	if (data == NULL) {
1195 		if (host->quirks2 &
1196 			SDHCI_QUIRK2_CLEAR_TRANSFERMODE_REG_BEFORE_CMD) {
1197 			/* must not clear SDHCI_TRANSFER_MODE when tuning */
1198 			if (cmd->opcode != MMC_SEND_TUNING_BLOCK_HS200)
1199 				sdhci_writew(host, 0x0, SDHCI_TRANSFER_MODE);
1200 		} else {
1201 		/* clear Auto CMD settings for no data CMDs */
1202 			mode = sdhci_readw(host, SDHCI_TRANSFER_MODE);
1203 			sdhci_writew(host, mode & ~(SDHCI_TRNS_AUTO_CMD12 |
1204 				SDHCI_TRNS_AUTO_CMD23), SDHCI_TRANSFER_MODE);
1205 		}
1206 		return;
1207 	}
1208 
1209 	WARN_ON(!host->data);
1210 
1211 	if (!(host->quirks2 & SDHCI_QUIRK2_SUPPORT_SINGLE))
1212 		mode = SDHCI_TRNS_BLK_CNT_EN;
1213 
1214 	if (mmc_op_multi(cmd->opcode) || data->blocks > 1) {
1215 		mode = SDHCI_TRNS_BLK_CNT_EN | SDHCI_TRNS_MULTI;
1216 		sdhci_auto_cmd_select(host, cmd, &mode);
1217 		if (cmd->mrq->sbc && (host->flags & SDHCI_AUTO_CMD23))
1218 			sdhci_writel(host, cmd->mrq->sbc->arg, SDHCI_ARGUMENT2);
1219 	}
1220 
1221 	if (data->flags & MMC_DATA_READ)
1222 		mode |= SDHCI_TRNS_READ;
1223 	if (host->flags & SDHCI_REQ_USE_DMA)
1224 		mode |= SDHCI_TRNS_DMA;
1225 
1226 	sdhci_writew(host, mode, SDHCI_TRANSFER_MODE);
1227 }
1228 
1229 static bool sdhci_needs_reset(struct sdhci_host *host, struct mmc_request *mrq)
1230 {
1231 	return (!(host->flags & SDHCI_DEVICE_DEAD) &&
1232 		((mrq->cmd && mrq->cmd->error) ||
1233 		 (mrq->sbc && mrq->sbc->error) ||
1234 		 (mrq->data && mrq->data->stop && mrq->data->stop->error) ||
1235 		 (host->quirks & SDHCI_QUIRK_RESET_AFTER_REQUEST)));
1236 }
1237 
1238 static void __sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq)
1239 {
1240 	int i;
1241 
1242 	if (host->cmd && host->cmd->mrq == mrq)
1243 		host->cmd = NULL;
1244 
1245 	if (host->data_cmd && host->data_cmd->mrq == mrq)
1246 		host->data_cmd = NULL;
1247 
1248 	if (host->data && host->data->mrq == mrq)
1249 		host->data = NULL;
1250 
1251 	if (sdhci_needs_reset(host, mrq))
1252 		host->pending_reset = true;
1253 
1254 	for (i = 0; i < SDHCI_MAX_MRQS; i++) {
1255 		if (host->mrqs_done[i] == mrq) {
1256 			WARN_ON(1);
1257 			return;
1258 		}
1259 	}
1260 
1261 	for (i = 0; i < SDHCI_MAX_MRQS; i++) {
1262 		if (!host->mrqs_done[i]) {
1263 			host->mrqs_done[i] = mrq;
1264 			break;
1265 		}
1266 	}
1267 
1268 	WARN_ON(i >= SDHCI_MAX_MRQS);
1269 
1270 	sdhci_del_timer(host, mrq);
1271 
1272 	if (!sdhci_has_requests(host))
1273 		sdhci_led_deactivate(host);
1274 }
1275 
1276 static void sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq)
1277 {
1278 	__sdhci_finish_mrq(host, mrq);
1279 
1280 	queue_work(host->complete_wq, &host->complete_work);
1281 }
1282 
1283 static void sdhci_finish_data(struct sdhci_host *host)
1284 {
1285 	struct mmc_command *data_cmd = host->data_cmd;
1286 	struct mmc_data *data = host->data;
1287 
1288 	host->data = NULL;
1289 	host->data_cmd = NULL;
1290 
1291 	/*
1292 	 * The controller needs a reset of internal state machines upon error
1293 	 * conditions.
1294 	 */
1295 	if (data->error) {
1296 		if (!host->cmd || host->cmd == data_cmd)
1297 			sdhci_do_reset(host, SDHCI_RESET_CMD);
1298 		sdhci_do_reset(host, SDHCI_RESET_DATA);
1299 	}
1300 
1301 	if ((host->flags & (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA)) ==
1302 	    (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA))
1303 		sdhci_adma_table_post(host, data);
1304 
1305 	/*
1306 	 * The specification states that the block count register must
1307 	 * be updated, but it does not specify at what point in the
1308 	 * data flow. That makes the register entirely useless to read
1309 	 * back so we have to assume that nothing made it to the card
1310 	 * in the event of an error.
1311 	 */
1312 	if (data->error)
1313 		data->bytes_xfered = 0;
1314 	else
1315 		data->bytes_xfered = data->blksz * data->blocks;
1316 
1317 	/*
1318 	 * Need to send CMD12 if -
1319 	 * a) open-ended multiblock transfer (no CMD23)
1320 	 * b) error in multiblock transfer
1321 	 */
1322 	if (data->stop &&
1323 	    (data->error ||
1324 	     !data->mrq->sbc)) {
1325 		/*
1326 		 * 'cap_cmd_during_tfr' request must not use the command line
1327 		 * after mmc_command_done() has been called. It is upper layer's
1328 		 * responsibility to send the stop command if required.
1329 		 */
1330 		if (data->mrq->cap_cmd_during_tfr) {
1331 			__sdhci_finish_mrq(host, data->mrq);
1332 		} else {
1333 			/* Avoid triggering warning in sdhci_send_command() */
1334 			host->cmd = NULL;
1335 			sdhci_send_command(host, data->stop);
1336 		}
1337 	} else {
1338 		__sdhci_finish_mrq(host, data->mrq);
1339 	}
1340 }
1341 
1342 void sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd)
1343 {
1344 	int flags;
1345 	u32 mask;
1346 	unsigned long timeout;
1347 
1348 	WARN_ON(host->cmd);
1349 
1350 	/* Initially, a command has no error */
1351 	cmd->error = 0;
1352 
1353 	if ((host->quirks2 & SDHCI_QUIRK2_STOP_WITH_TC) &&
1354 	    cmd->opcode == MMC_STOP_TRANSMISSION)
1355 		cmd->flags |= MMC_RSP_BUSY;
1356 
1357 	/* Wait max 10 ms */
1358 	timeout = 10;
1359 
1360 	mask = SDHCI_CMD_INHIBIT;
1361 	if (sdhci_data_line_cmd(cmd))
1362 		mask |= SDHCI_DATA_INHIBIT;
1363 
1364 	/* We shouldn't wait for data inihibit for stop commands, even
1365 	   though they might use busy signaling */
1366 	if (cmd->mrq->data && (cmd == cmd->mrq->data->stop))
1367 		mask &= ~SDHCI_DATA_INHIBIT;
1368 
1369 	while (sdhci_readl(host, SDHCI_PRESENT_STATE) & mask) {
1370 		if (timeout == 0) {
1371 			pr_err("%s: Controller never released inhibit bit(s).\n",
1372 			       mmc_hostname(host->mmc));
1373 			sdhci_dumpregs(host);
1374 			cmd->error = -EIO;
1375 			sdhci_finish_mrq(host, cmd->mrq);
1376 			return;
1377 		}
1378 		timeout--;
1379 		mdelay(1);
1380 	}
1381 
1382 	host->cmd = cmd;
1383 	if (sdhci_data_line_cmd(cmd)) {
1384 		WARN_ON(host->data_cmd);
1385 		host->data_cmd = cmd;
1386 	}
1387 
1388 	sdhci_prepare_data(host, cmd);
1389 
1390 	sdhci_writel(host, cmd->arg, SDHCI_ARGUMENT);
1391 
1392 	sdhci_set_transfer_mode(host, cmd);
1393 
1394 	if ((cmd->flags & MMC_RSP_136) && (cmd->flags & MMC_RSP_BUSY)) {
1395 		pr_err("%s: Unsupported response type!\n",
1396 			mmc_hostname(host->mmc));
1397 		cmd->error = -EINVAL;
1398 		sdhci_finish_mrq(host, cmd->mrq);
1399 		return;
1400 	}
1401 
1402 	if (!(cmd->flags & MMC_RSP_PRESENT))
1403 		flags = SDHCI_CMD_RESP_NONE;
1404 	else if (cmd->flags & MMC_RSP_136)
1405 		flags = SDHCI_CMD_RESP_LONG;
1406 	else if (cmd->flags & MMC_RSP_BUSY)
1407 		flags = SDHCI_CMD_RESP_SHORT_BUSY;
1408 	else
1409 		flags = SDHCI_CMD_RESP_SHORT;
1410 
1411 	if (cmd->flags & MMC_RSP_CRC)
1412 		flags |= SDHCI_CMD_CRC;
1413 	if (cmd->flags & MMC_RSP_OPCODE)
1414 		flags |= SDHCI_CMD_INDEX;
1415 
1416 	/* CMD19 is special in that the Data Present Select should be set */
1417 	if (cmd->data || cmd->opcode == MMC_SEND_TUNING_BLOCK ||
1418 	    cmd->opcode == MMC_SEND_TUNING_BLOCK_HS200)
1419 		flags |= SDHCI_CMD_DATA;
1420 
1421 	timeout = jiffies;
1422 	if (host->data_timeout)
1423 		timeout += nsecs_to_jiffies(host->data_timeout);
1424 	else if (!cmd->data && cmd->busy_timeout > 9000)
1425 		timeout += DIV_ROUND_UP(cmd->busy_timeout, 1000) * HZ + HZ;
1426 	else
1427 		timeout += 10 * HZ;
1428 	sdhci_mod_timer(host, cmd->mrq, timeout);
1429 
1430 	sdhci_writew(host, SDHCI_MAKE_CMD(cmd->opcode, flags), SDHCI_COMMAND);
1431 }
1432 EXPORT_SYMBOL_GPL(sdhci_send_command);
1433 
1434 static void sdhci_read_rsp_136(struct sdhci_host *host, struct mmc_command *cmd)
1435 {
1436 	int i, reg;
1437 
1438 	for (i = 0; i < 4; i++) {
1439 		reg = SDHCI_RESPONSE + (3 - i) * 4;
1440 		cmd->resp[i] = sdhci_readl(host, reg);
1441 	}
1442 
1443 	if (host->quirks2 & SDHCI_QUIRK2_RSP_136_HAS_CRC)
1444 		return;
1445 
1446 	/* CRC is stripped so we need to do some shifting */
1447 	for (i = 0; i < 4; i++) {
1448 		cmd->resp[i] <<= 8;
1449 		if (i != 3)
1450 			cmd->resp[i] |= cmd->resp[i + 1] >> 24;
1451 	}
1452 }
1453 
1454 static void sdhci_finish_command(struct sdhci_host *host)
1455 {
1456 	struct mmc_command *cmd = host->cmd;
1457 
1458 	host->cmd = NULL;
1459 
1460 	if (cmd->flags & MMC_RSP_PRESENT) {
1461 		if (cmd->flags & MMC_RSP_136) {
1462 			sdhci_read_rsp_136(host, cmd);
1463 		} else {
1464 			cmd->resp[0] = sdhci_readl(host, SDHCI_RESPONSE);
1465 		}
1466 	}
1467 
1468 	if (cmd->mrq->cap_cmd_during_tfr && cmd == cmd->mrq->cmd)
1469 		mmc_command_done(host->mmc, cmd->mrq);
1470 
1471 	/*
1472 	 * The host can send and interrupt when the busy state has
1473 	 * ended, allowing us to wait without wasting CPU cycles.
1474 	 * The busy signal uses DAT0 so this is similar to waiting
1475 	 * for data to complete.
1476 	 *
1477 	 * Note: The 1.0 specification is a bit ambiguous about this
1478 	 *       feature so there might be some problems with older
1479 	 *       controllers.
1480 	 */
1481 	if (cmd->flags & MMC_RSP_BUSY) {
1482 		if (cmd->data) {
1483 			DBG("Cannot wait for busy signal when also doing a data transfer");
1484 		} else if (!(host->quirks & SDHCI_QUIRK_NO_BUSY_IRQ) &&
1485 			   cmd == host->data_cmd) {
1486 			/* Command complete before busy is ended */
1487 			return;
1488 		}
1489 	}
1490 
1491 	/* Finished CMD23, now send actual command. */
1492 	if (cmd == cmd->mrq->sbc) {
1493 		sdhci_send_command(host, cmd->mrq->cmd);
1494 	} else {
1495 
1496 		/* Processed actual command. */
1497 		if (host->data && host->data_early)
1498 			sdhci_finish_data(host);
1499 
1500 		if (!cmd->data)
1501 			__sdhci_finish_mrq(host, cmd->mrq);
1502 	}
1503 }
1504 
1505 static u16 sdhci_get_preset_value(struct sdhci_host *host)
1506 {
1507 	u16 preset = 0;
1508 
1509 	switch (host->timing) {
1510 	case MMC_TIMING_UHS_SDR12:
1511 		preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR12);
1512 		break;
1513 	case MMC_TIMING_UHS_SDR25:
1514 		preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR25);
1515 		break;
1516 	case MMC_TIMING_UHS_SDR50:
1517 		preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR50);
1518 		break;
1519 	case MMC_TIMING_UHS_SDR104:
1520 	case MMC_TIMING_MMC_HS200:
1521 		preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR104);
1522 		break;
1523 	case MMC_TIMING_UHS_DDR50:
1524 	case MMC_TIMING_MMC_DDR52:
1525 		preset = sdhci_readw(host, SDHCI_PRESET_FOR_DDR50);
1526 		break;
1527 	case MMC_TIMING_MMC_HS400:
1528 		preset = sdhci_readw(host, SDHCI_PRESET_FOR_HS400);
1529 		break;
1530 	default:
1531 		pr_warn("%s: Invalid UHS-I mode selected\n",
1532 			mmc_hostname(host->mmc));
1533 		preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR12);
1534 		break;
1535 	}
1536 	return preset;
1537 }
1538 
1539 u16 sdhci_calc_clk(struct sdhci_host *host, unsigned int clock,
1540 		   unsigned int *actual_clock)
1541 {
1542 	int div = 0; /* Initialized for compiler warning */
1543 	int real_div = div, clk_mul = 1;
1544 	u16 clk = 0;
1545 	bool switch_base_clk = false;
1546 
1547 	if (host->version >= SDHCI_SPEC_300) {
1548 		if (host->preset_enabled) {
1549 			u16 pre_val;
1550 
1551 			clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1552 			pre_val = sdhci_get_preset_value(host);
1553 			div = (pre_val & SDHCI_PRESET_SDCLK_FREQ_MASK)
1554 				>> SDHCI_PRESET_SDCLK_FREQ_SHIFT;
1555 			if (host->clk_mul &&
1556 				(pre_val & SDHCI_PRESET_CLKGEN_SEL_MASK)) {
1557 				clk = SDHCI_PROG_CLOCK_MODE;
1558 				real_div = div + 1;
1559 				clk_mul = host->clk_mul;
1560 			} else {
1561 				real_div = max_t(int, 1, div << 1);
1562 			}
1563 			goto clock_set;
1564 		}
1565 
1566 		/*
1567 		 * Check if the Host Controller supports Programmable Clock
1568 		 * Mode.
1569 		 */
1570 		if (host->clk_mul) {
1571 			for (div = 1; div <= 1024; div++) {
1572 				if ((host->max_clk * host->clk_mul / div)
1573 					<= clock)
1574 					break;
1575 			}
1576 			if ((host->max_clk * host->clk_mul / div) <= clock) {
1577 				/*
1578 				 * Set Programmable Clock Mode in the Clock
1579 				 * Control register.
1580 				 */
1581 				clk = SDHCI_PROG_CLOCK_MODE;
1582 				real_div = div;
1583 				clk_mul = host->clk_mul;
1584 				div--;
1585 			} else {
1586 				/*
1587 				 * Divisor can be too small to reach clock
1588 				 * speed requirement. Then use the base clock.
1589 				 */
1590 				switch_base_clk = true;
1591 			}
1592 		}
1593 
1594 		if (!host->clk_mul || switch_base_clk) {
1595 			/* Version 3.00 divisors must be a multiple of 2. */
1596 			if (host->max_clk <= clock)
1597 				div = 1;
1598 			else {
1599 				for (div = 2; div < SDHCI_MAX_DIV_SPEC_300;
1600 				     div += 2) {
1601 					if ((host->max_clk / div) <= clock)
1602 						break;
1603 				}
1604 			}
1605 			real_div = div;
1606 			div >>= 1;
1607 			if ((host->quirks2 & SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN)
1608 				&& !div && host->max_clk <= 25000000)
1609 				div = 1;
1610 		}
1611 	} else {
1612 		/* Version 2.00 divisors must be a power of 2. */
1613 		for (div = 1; div < SDHCI_MAX_DIV_SPEC_200; div *= 2) {
1614 			if ((host->max_clk / div) <= clock)
1615 				break;
1616 		}
1617 		real_div = div;
1618 		div >>= 1;
1619 	}
1620 
1621 clock_set:
1622 	if (real_div)
1623 		*actual_clock = (host->max_clk * clk_mul) / real_div;
1624 	clk |= (div & SDHCI_DIV_MASK) << SDHCI_DIVIDER_SHIFT;
1625 	clk |= ((div & SDHCI_DIV_HI_MASK) >> SDHCI_DIV_MASK_LEN)
1626 		<< SDHCI_DIVIDER_HI_SHIFT;
1627 
1628 	return clk;
1629 }
1630 EXPORT_SYMBOL_GPL(sdhci_calc_clk);
1631 
1632 void sdhci_enable_clk(struct sdhci_host *host, u16 clk)
1633 {
1634 	ktime_t timeout;
1635 
1636 	clk |= SDHCI_CLOCK_INT_EN;
1637 	sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1638 
1639 	/* Wait max 20 ms */
1640 	timeout = ktime_add_ms(ktime_get(), 20);
1641 	while (1) {
1642 		bool timedout = ktime_after(ktime_get(), timeout);
1643 
1644 		clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1645 		if (clk & SDHCI_CLOCK_INT_STABLE)
1646 			break;
1647 		if (timedout) {
1648 			pr_err("%s: Internal clock never stabilised.\n",
1649 			       mmc_hostname(host->mmc));
1650 			sdhci_dumpregs(host);
1651 			return;
1652 		}
1653 		udelay(10);
1654 	}
1655 
1656 	clk |= SDHCI_CLOCK_CARD_EN;
1657 	sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1658 }
1659 EXPORT_SYMBOL_GPL(sdhci_enable_clk);
1660 
1661 void sdhci_set_clock(struct sdhci_host *host, unsigned int clock)
1662 {
1663 	u16 clk;
1664 
1665 	host->mmc->actual_clock = 0;
1666 
1667 	sdhci_writew(host, 0, SDHCI_CLOCK_CONTROL);
1668 
1669 	if (clock == 0)
1670 		return;
1671 
1672 	clk = sdhci_calc_clk(host, clock, &host->mmc->actual_clock);
1673 	sdhci_enable_clk(host, clk);
1674 }
1675 EXPORT_SYMBOL_GPL(sdhci_set_clock);
1676 
1677 static void sdhci_set_power_reg(struct sdhci_host *host, unsigned char mode,
1678 				unsigned short vdd)
1679 {
1680 	struct mmc_host *mmc = host->mmc;
1681 
1682 	mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, vdd);
1683 
1684 	if (mode != MMC_POWER_OFF)
1685 		sdhci_writeb(host, SDHCI_POWER_ON, SDHCI_POWER_CONTROL);
1686 	else
1687 		sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
1688 }
1689 
1690 void sdhci_set_power_noreg(struct sdhci_host *host, unsigned char mode,
1691 			   unsigned short vdd)
1692 {
1693 	u8 pwr = 0;
1694 
1695 	if (mode != MMC_POWER_OFF) {
1696 		switch (1 << vdd) {
1697 		case MMC_VDD_165_195:
1698 		/*
1699 		 * Without a regulator, SDHCI does not support 2.0v
1700 		 * so we only get here if the driver deliberately
1701 		 * added the 2.0v range to ocr_avail. Map it to 1.8v
1702 		 * for the purpose of turning on the power.
1703 		 */
1704 		case MMC_VDD_20_21:
1705 			pwr = SDHCI_POWER_180;
1706 			break;
1707 		case MMC_VDD_29_30:
1708 		case MMC_VDD_30_31:
1709 			pwr = SDHCI_POWER_300;
1710 			break;
1711 		case MMC_VDD_32_33:
1712 		case MMC_VDD_33_34:
1713 			pwr = SDHCI_POWER_330;
1714 			break;
1715 		default:
1716 			WARN(1, "%s: Invalid vdd %#x\n",
1717 			     mmc_hostname(host->mmc), vdd);
1718 			break;
1719 		}
1720 	}
1721 
1722 	if (host->pwr == pwr)
1723 		return;
1724 
1725 	host->pwr = pwr;
1726 
1727 	if (pwr == 0) {
1728 		sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
1729 		if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
1730 			sdhci_runtime_pm_bus_off(host);
1731 	} else {
1732 		/*
1733 		 * Spec says that we should clear the power reg before setting
1734 		 * a new value. Some controllers don't seem to like this though.
1735 		 */
1736 		if (!(host->quirks & SDHCI_QUIRK_SINGLE_POWER_WRITE))
1737 			sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
1738 
1739 		/*
1740 		 * At least the Marvell CaFe chip gets confused if we set the
1741 		 * voltage and set turn on power at the same time, so set the
1742 		 * voltage first.
1743 		 */
1744 		if (host->quirks & SDHCI_QUIRK_NO_SIMULT_VDD_AND_POWER)
1745 			sdhci_writeb(host, pwr, SDHCI_POWER_CONTROL);
1746 
1747 		pwr |= SDHCI_POWER_ON;
1748 
1749 		sdhci_writeb(host, pwr, SDHCI_POWER_CONTROL);
1750 
1751 		if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
1752 			sdhci_runtime_pm_bus_on(host);
1753 
1754 		/*
1755 		 * Some controllers need an extra 10ms delay of 10ms before
1756 		 * they can apply clock after applying power
1757 		 */
1758 		if (host->quirks & SDHCI_QUIRK_DELAY_AFTER_POWER)
1759 			mdelay(10);
1760 	}
1761 }
1762 EXPORT_SYMBOL_GPL(sdhci_set_power_noreg);
1763 
1764 void sdhci_set_power(struct sdhci_host *host, unsigned char mode,
1765 		     unsigned short vdd)
1766 {
1767 	if (IS_ERR(host->mmc->supply.vmmc))
1768 		sdhci_set_power_noreg(host, mode, vdd);
1769 	else
1770 		sdhci_set_power_reg(host, mode, vdd);
1771 }
1772 EXPORT_SYMBOL_GPL(sdhci_set_power);
1773 
1774 /*****************************************************************************\
1775  *                                                                           *
1776  * MMC callbacks                                                             *
1777  *                                                                           *
1778 \*****************************************************************************/
1779 
1780 void sdhci_request(struct mmc_host *mmc, struct mmc_request *mrq)
1781 {
1782 	struct sdhci_host *host;
1783 	int present;
1784 	unsigned long flags;
1785 
1786 	host = mmc_priv(mmc);
1787 
1788 	/* Firstly check card presence */
1789 	present = mmc->ops->get_cd(mmc);
1790 
1791 	spin_lock_irqsave(&host->lock, flags);
1792 
1793 	sdhci_led_activate(host);
1794 
1795 	/*
1796 	 * Ensure we don't send the STOP for non-SET_BLOCK_COUNTED
1797 	 * requests if Auto-CMD12 is enabled.
1798 	 */
1799 	if (sdhci_auto_cmd12(host, mrq)) {
1800 		if (mrq->stop) {
1801 			mrq->data->stop = NULL;
1802 			mrq->stop = NULL;
1803 		}
1804 	}
1805 
1806 	if (!present || host->flags & SDHCI_DEVICE_DEAD) {
1807 		mrq->cmd->error = -ENOMEDIUM;
1808 		sdhci_finish_mrq(host, mrq);
1809 	} else {
1810 		if (mrq->sbc && !(host->flags & SDHCI_AUTO_CMD23))
1811 			sdhci_send_command(host, mrq->sbc);
1812 		else
1813 			sdhci_send_command(host, mrq->cmd);
1814 	}
1815 
1816 	spin_unlock_irqrestore(&host->lock, flags);
1817 }
1818 EXPORT_SYMBOL_GPL(sdhci_request);
1819 
1820 void sdhci_set_bus_width(struct sdhci_host *host, int width)
1821 {
1822 	u8 ctrl;
1823 
1824 	ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
1825 	if (width == MMC_BUS_WIDTH_8) {
1826 		ctrl &= ~SDHCI_CTRL_4BITBUS;
1827 		ctrl |= SDHCI_CTRL_8BITBUS;
1828 	} else {
1829 		if (host->mmc->caps & MMC_CAP_8_BIT_DATA)
1830 			ctrl &= ~SDHCI_CTRL_8BITBUS;
1831 		if (width == MMC_BUS_WIDTH_4)
1832 			ctrl |= SDHCI_CTRL_4BITBUS;
1833 		else
1834 			ctrl &= ~SDHCI_CTRL_4BITBUS;
1835 	}
1836 	sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1837 }
1838 EXPORT_SYMBOL_GPL(sdhci_set_bus_width);
1839 
1840 void sdhci_set_uhs_signaling(struct sdhci_host *host, unsigned timing)
1841 {
1842 	u16 ctrl_2;
1843 
1844 	ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1845 	/* Select Bus Speed Mode for host */
1846 	ctrl_2 &= ~SDHCI_CTRL_UHS_MASK;
1847 	if ((timing == MMC_TIMING_MMC_HS200) ||
1848 	    (timing == MMC_TIMING_UHS_SDR104))
1849 		ctrl_2 |= SDHCI_CTRL_UHS_SDR104;
1850 	else if (timing == MMC_TIMING_UHS_SDR12)
1851 		ctrl_2 |= SDHCI_CTRL_UHS_SDR12;
1852 	else if (timing == MMC_TIMING_UHS_SDR25)
1853 		ctrl_2 |= SDHCI_CTRL_UHS_SDR25;
1854 	else if (timing == MMC_TIMING_UHS_SDR50)
1855 		ctrl_2 |= SDHCI_CTRL_UHS_SDR50;
1856 	else if ((timing == MMC_TIMING_UHS_DDR50) ||
1857 		 (timing == MMC_TIMING_MMC_DDR52))
1858 		ctrl_2 |= SDHCI_CTRL_UHS_DDR50;
1859 	else if (timing == MMC_TIMING_MMC_HS400)
1860 		ctrl_2 |= SDHCI_CTRL_HS400; /* Non-standard */
1861 	sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
1862 }
1863 EXPORT_SYMBOL_GPL(sdhci_set_uhs_signaling);
1864 
1865 void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
1866 {
1867 	struct sdhci_host *host = mmc_priv(mmc);
1868 	u8 ctrl;
1869 
1870 	if (ios->power_mode == MMC_POWER_UNDEFINED)
1871 		return;
1872 
1873 	if (host->flags & SDHCI_DEVICE_DEAD) {
1874 		if (!IS_ERR(mmc->supply.vmmc) &&
1875 		    ios->power_mode == MMC_POWER_OFF)
1876 			mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0);
1877 		return;
1878 	}
1879 
1880 	/*
1881 	 * Reset the chip on each power off.
1882 	 * Should clear out any weird states.
1883 	 */
1884 	if (ios->power_mode == MMC_POWER_OFF) {
1885 		sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
1886 		sdhci_reinit(host);
1887 	}
1888 
1889 	if (host->version >= SDHCI_SPEC_300 &&
1890 		(ios->power_mode == MMC_POWER_UP) &&
1891 		!(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN))
1892 		sdhci_enable_preset_value(host, false);
1893 
1894 	if (!ios->clock || ios->clock != host->clock) {
1895 		host->ops->set_clock(host, ios->clock);
1896 		host->clock = ios->clock;
1897 
1898 		if (host->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK &&
1899 		    host->clock) {
1900 			host->timeout_clk = host->mmc->actual_clock ?
1901 						host->mmc->actual_clock / 1000 :
1902 						host->clock / 1000;
1903 			host->mmc->max_busy_timeout =
1904 				host->ops->get_max_timeout_count ?
1905 				host->ops->get_max_timeout_count(host) :
1906 				1 << 27;
1907 			host->mmc->max_busy_timeout /= host->timeout_clk;
1908 		}
1909 	}
1910 
1911 	if (host->ops->set_power)
1912 		host->ops->set_power(host, ios->power_mode, ios->vdd);
1913 	else
1914 		sdhci_set_power(host, ios->power_mode, ios->vdd);
1915 
1916 	if (host->ops->platform_send_init_74_clocks)
1917 		host->ops->platform_send_init_74_clocks(host, ios->power_mode);
1918 
1919 	host->ops->set_bus_width(host, ios->bus_width);
1920 
1921 	ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
1922 
1923 	if (!(host->quirks & SDHCI_QUIRK_NO_HISPD_BIT)) {
1924 		if (ios->timing == MMC_TIMING_SD_HS ||
1925 		     ios->timing == MMC_TIMING_MMC_HS ||
1926 		     ios->timing == MMC_TIMING_MMC_HS400 ||
1927 		     ios->timing == MMC_TIMING_MMC_HS200 ||
1928 		     ios->timing == MMC_TIMING_MMC_DDR52 ||
1929 		     ios->timing == MMC_TIMING_UHS_SDR50 ||
1930 		     ios->timing == MMC_TIMING_UHS_SDR104 ||
1931 		     ios->timing == MMC_TIMING_UHS_DDR50 ||
1932 		     ios->timing == MMC_TIMING_UHS_SDR25)
1933 			ctrl |= SDHCI_CTRL_HISPD;
1934 		else
1935 			ctrl &= ~SDHCI_CTRL_HISPD;
1936 	}
1937 
1938 	if (host->version >= SDHCI_SPEC_300) {
1939 		u16 clk, ctrl_2;
1940 
1941 		if (!host->preset_enabled) {
1942 			sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1943 			/*
1944 			 * We only need to set Driver Strength if the
1945 			 * preset value enable is not set.
1946 			 */
1947 			ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1948 			ctrl_2 &= ~SDHCI_CTRL_DRV_TYPE_MASK;
1949 			if (ios->drv_type == MMC_SET_DRIVER_TYPE_A)
1950 				ctrl_2 |= SDHCI_CTRL_DRV_TYPE_A;
1951 			else if (ios->drv_type == MMC_SET_DRIVER_TYPE_B)
1952 				ctrl_2 |= SDHCI_CTRL_DRV_TYPE_B;
1953 			else if (ios->drv_type == MMC_SET_DRIVER_TYPE_C)
1954 				ctrl_2 |= SDHCI_CTRL_DRV_TYPE_C;
1955 			else if (ios->drv_type == MMC_SET_DRIVER_TYPE_D)
1956 				ctrl_2 |= SDHCI_CTRL_DRV_TYPE_D;
1957 			else {
1958 				pr_warn("%s: invalid driver type, default to driver type B\n",
1959 					mmc_hostname(mmc));
1960 				ctrl_2 |= SDHCI_CTRL_DRV_TYPE_B;
1961 			}
1962 
1963 			sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
1964 		} else {
1965 			/*
1966 			 * According to SDHC Spec v3.00, if the Preset Value
1967 			 * Enable in the Host Control 2 register is set, we
1968 			 * need to reset SD Clock Enable before changing High
1969 			 * Speed Enable to avoid generating clock gliches.
1970 			 */
1971 
1972 			/* Reset SD Clock Enable */
1973 			clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1974 			clk &= ~SDHCI_CLOCK_CARD_EN;
1975 			sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1976 
1977 			sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1978 
1979 			/* Re-enable SD Clock */
1980 			host->ops->set_clock(host, host->clock);
1981 		}
1982 
1983 		/* Reset SD Clock Enable */
1984 		clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1985 		clk &= ~SDHCI_CLOCK_CARD_EN;
1986 		sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1987 
1988 		host->ops->set_uhs_signaling(host, ios->timing);
1989 		host->timing = ios->timing;
1990 
1991 		if (!(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN) &&
1992 				((ios->timing == MMC_TIMING_UHS_SDR12) ||
1993 				 (ios->timing == MMC_TIMING_UHS_SDR25) ||
1994 				 (ios->timing == MMC_TIMING_UHS_SDR50) ||
1995 				 (ios->timing == MMC_TIMING_UHS_SDR104) ||
1996 				 (ios->timing == MMC_TIMING_UHS_DDR50) ||
1997 				 (ios->timing == MMC_TIMING_MMC_DDR52))) {
1998 			u16 preset;
1999 
2000 			sdhci_enable_preset_value(host, true);
2001 			preset = sdhci_get_preset_value(host);
2002 			ios->drv_type = (preset & SDHCI_PRESET_DRV_MASK)
2003 				>> SDHCI_PRESET_DRV_SHIFT;
2004 		}
2005 
2006 		/* Re-enable SD Clock */
2007 		host->ops->set_clock(host, host->clock);
2008 	} else
2009 		sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
2010 
2011 	/*
2012 	 * Some (ENE) controllers go apeshit on some ios operation,
2013 	 * signalling timeout and CRC errors even on CMD0. Resetting
2014 	 * it on each ios seems to solve the problem.
2015 	 */
2016 	if (host->quirks & SDHCI_QUIRK_RESET_CMD_DATA_ON_IOS)
2017 		sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
2018 }
2019 EXPORT_SYMBOL_GPL(sdhci_set_ios);
2020 
2021 static int sdhci_get_cd(struct mmc_host *mmc)
2022 {
2023 	struct sdhci_host *host = mmc_priv(mmc);
2024 	int gpio_cd = mmc_gpio_get_cd(mmc);
2025 
2026 	if (host->flags & SDHCI_DEVICE_DEAD)
2027 		return 0;
2028 
2029 	/* If nonremovable, assume that the card is always present. */
2030 	if (!mmc_card_is_removable(host->mmc))
2031 		return 1;
2032 
2033 	/*
2034 	 * Try slot gpio detect, if defined it take precedence
2035 	 * over build in controller functionality
2036 	 */
2037 	if (gpio_cd >= 0)
2038 		return !!gpio_cd;
2039 
2040 	/* If polling, assume that the card is always present. */
2041 	if (host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION)
2042 		return 1;
2043 
2044 	/* Host native card detect */
2045 	return !!(sdhci_readl(host, SDHCI_PRESENT_STATE) & SDHCI_CARD_PRESENT);
2046 }
2047 
2048 static int sdhci_check_ro(struct sdhci_host *host)
2049 {
2050 	unsigned long flags;
2051 	int is_readonly;
2052 
2053 	spin_lock_irqsave(&host->lock, flags);
2054 
2055 	if (host->flags & SDHCI_DEVICE_DEAD)
2056 		is_readonly = 0;
2057 	else if (host->ops->get_ro)
2058 		is_readonly = host->ops->get_ro(host);
2059 	else if (mmc_can_gpio_ro(host->mmc))
2060 		is_readonly = mmc_gpio_get_ro(host->mmc);
2061 	else
2062 		is_readonly = !(sdhci_readl(host, SDHCI_PRESENT_STATE)
2063 				& SDHCI_WRITE_PROTECT);
2064 
2065 	spin_unlock_irqrestore(&host->lock, flags);
2066 
2067 	/* This quirk needs to be replaced by a callback-function later */
2068 	return host->quirks & SDHCI_QUIRK_INVERTED_WRITE_PROTECT ?
2069 		!is_readonly : is_readonly;
2070 }
2071 
2072 #define SAMPLE_COUNT	5
2073 
2074 static int sdhci_get_ro(struct mmc_host *mmc)
2075 {
2076 	struct sdhci_host *host = mmc_priv(mmc);
2077 	int i, ro_count;
2078 
2079 	if (!(host->quirks & SDHCI_QUIRK_UNSTABLE_RO_DETECT))
2080 		return sdhci_check_ro(host);
2081 
2082 	ro_count = 0;
2083 	for (i = 0; i < SAMPLE_COUNT; i++) {
2084 		if (sdhci_check_ro(host)) {
2085 			if (++ro_count > SAMPLE_COUNT / 2)
2086 				return 1;
2087 		}
2088 		msleep(30);
2089 	}
2090 	return 0;
2091 }
2092 
2093 static void sdhci_hw_reset(struct mmc_host *mmc)
2094 {
2095 	struct sdhci_host *host = mmc_priv(mmc);
2096 
2097 	if (host->ops && host->ops->hw_reset)
2098 		host->ops->hw_reset(host);
2099 }
2100 
2101 static void sdhci_enable_sdio_irq_nolock(struct sdhci_host *host, int enable)
2102 {
2103 	if (!(host->flags & SDHCI_DEVICE_DEAD)) {
2104 		if (enable)
2105 			host->ier |= SDHCI_INT_CARD_INT;
2106 		else
2107 			host->ier &= ~SDHCI_INT_CARD_INT;
2108 
2109 		sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
2110 		sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
2111 	}
2112 }
2113 
2114 void sdhci_enable_sdio_irq(struct mmc_host *mmc, int enable)
2115 {
2116 	struct sdhci_host *host = mmc_priv(mmc);
2117 	unsigned long flags;
2118 
2119 	if (enable)
2120 		pm_runtime_get_noresume(host->mmc->parent);
2121 
2122 	spin_lock_irqsave(&host->lock, flags);
2123 	if (enable)
2124 		host->flags |= SDHCI_SDIO_IRQ_ENABLED;
2125 	else
2126 		host->flags &= ~SDHCI_SDIO_IRQ_ENABLED;
2127 
2128 	sdhci_enable_sdio_irq_nolock(host, enable);
2129 	spin_unlock_irqrestore(&host->lock, flags);
2130 
2131 	if (!enable)
2132 		pm_runtime_put_noidle(host->mmc->parent);
2133 }
2134 EXPORT_SYMBOL_GPL(sdhci_enable_sdio_irq);
2135 
2136 int sdhci_start_signal_voltage_switch(struct mmc_host *mmc,
2137 				      struct mmc_ios *ios)
2138 {
2139 	struct sdhci_host *host = mmc_priv(mmc);
2140 	u16 ctrl;
2141 	int ret;
2142 
2143 	/*
2144 	 * Signal Voltage Switching is only applicable for Host Controllers
2145 	 * v3.00 and above.
2146 	 */
2147 	if (host->version < SDHCI_SPEC_300)
2148 		return 0;
2149 
2150 	ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2151 
2152 	switch (ios->signal_voltage) {
2153 	case MMC_SIGNAL_VOLTAGE_330:
2154 		if (!(host->flags & SDHCI_SIGNALING_330))
2155 			return -EINVAL;
2156 		/* Set 1.8V Signal Enable in the Host Control2 register to 0 */
2157 		ctrl &= ~SDHCI_CTRL_VDD_180;
2158 		sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2159 
2160 		if (!IS_ERR(mmc->supply.vqmmc)) {
2161 			ret = mmc_regulator_set_vqmmc(mmc, ios);
2162 			if (ret) {
2163 				pr_warn("%s: Switching to 3.3V signalling voltage failed\n",
2164 					mmc_hostname(mmc));
2165 				return -EIO;
2166 			}
2167 		}
2168 		/* Wait for 5ms */
2169 		usleep_range(5000, 5500);
2170 
2171 		/* 3.3V regulator output should be stable within 5 ms */
2172 		ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2173 		if (!(ctrl & SDHCI_CTRL_VDD_180))
2174 			return 0;
2175 
2176 		pr_warn("%s: 3.3V regulator output did not became stable\n",
2177 			mmc_hostname(mmc));
2178 
2179 		return -EAGAIN;
2180 	case MMC_SIGNAL_VOLTAGE_180:
2181 		if (!(host->flags & SDHCI_SIGNALING_180))
2182 			return -EINVAL;
2183 		if (!IS_ERR(mmc->supply.vqmmc)) {
2184 			ret = mmc_regulator_set_vqmmc(mmc, ios);
2185 			if (ret) {
2186 				pr_warn("%s: Switching to 1.8V signalling voltage failed\n",
2187 					mmc_hostname(mmc));
2188 				return -EIO;
2189 			}
2190 		}
2191 
2192 		/*
2193 		 * Enable 1.8V Signal Enable in the Host Control2
2194 		 * register
2195 		 */
2196 		ctrl |= SDHCI_CTRL_VDD_180;
2197 		sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2198 
2199 		/* Some controller need to do more when switching */
2200 		if (host->ops->voltage_switch)
2201 			host->ops->voltage_switch(host);
2202 
2203 		/* 1.8V regulator output should be stable within 5 ms */
2204 		ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2205 		if (ctrl & SDHCI_CTRL_VDD_180)
2206 			return 0;
2207 
2208 		pr_warn("%s: 1.8V regulator output did not became stable\n",
2209 			mmc_hostname(mmc));
2210 
2211 		return -EAGAIN;
2212 	case MMC_SIGNAL_VOLTAGE_120:
2213 		if (!(host->flags & SDHCI_SIGNALING_120))
2214 			return -EINVAL;
2215 		if (!IS_ERR(mmc->supply.vqmmc)) {
2216 			ret = mmc_regulator_set_vqmmc(mmc, ios);
2217 			if (ret) {
2218 				pr_warn("%s: Switching to 1.2V signalling voltage failed\n",
2219 					mmc_hostname(mmc));
2220 				return -EIO;
2221 			}
2222 		}
2223 		return 0;
2224 	default:
2225 		/* No signal voltage switch required */
2226 		return 0;
2227 	}
2228 }
2229 EXPORT_SYMBOL_GPL(sdhci_start_signal_voltage_switch);
2230 
2231 static int sdhci_card_busy(struct mmc_host *mmc)
2232 {
2233 	struct sdhci_host *host = mmc_priv(mmc);
2234 	u32 present_state;
2235 
2236 	/* Check whether DAT[0] is 0 */
2237 	present_state = sdhci_readl(host, SDHCI_PRESENT_STATE);
2238 
2239 	return !(present_state & SDHCI_DATA_0_LVL_MASK);
2240 }
2241 
2242 static int sdhci_prepare_hs400_tuning(struct mmc_host *mmc, struct mmc_ios *ios)
2243 {
2244 	struct sdhci_host *host = mmc_priv(mmc);
2245 	unsigned long flags;
2246 
2247 	spin_lock_irqsave(&host->lock, flags);
2248 	host->flags |= SDHCI_HS400_TUNING;
2249 	spin_unlock_irqrestore(&host->lock, flags);
2250 
2251 	return 0;
2252 }
2253 
2254 void sdhci_start_tuning(struct sdhci_host *host)
2255 {
2256 	u16 ctrl;
2257 
2258 	ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2259 	ctrl |= SDHCI_CTRL_EXEC_TUNING;
2260 	if (host->quirks2 & SDHCI_QUIRK2_TUNING_WORK_AROUND)
2261 		ctrl |= SDHCI_CTRL_TUNED_CLK;
2262 	sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2263 
2264 	/*
2265 	 * As per the Host Controller spec v3.00, tuning command
2266 	 * generates Buffer Read Ready interrupt, so enable that.
2267 	 *
2268 	 * Note: The spec clearly says that when tuning sequence
2269 	 * is being performed, the controller does not generate
2270 	 * interrupts other than Buffer Read Ready interrupt. But
2271 	 * to make sure we don't hit a controller bug, we _only_
2272 	 * enable Buffer Read Ready interrupt here.
2273 	 */
2274 	sdhci_writel(host, SDHCI_INT_DATA_AVAIL, SDHCI_INT_ENABLE);
2275 	sdhci_writel(host, SDHCI_INT_DATA_AVAIL, SDHCI_SIGNAL_ENABLE);
2276 }
2277 EXPORT_SYMBOL_GPL(sdhci_start_tuning);
2278 
2279 void sdhci_end_tuning(struct sdhci_host *host)
2280 {
2281 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
2282 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
2283 }
2284 EXPORT_SYMBOL_GPL(sdhci_end_tuning);
2285 
2286 void sdhci_reset_tuning(struct sdhci_host *host)
2287 {
2288 	u16 ctrl;
2289 
2290 	ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2291 	ctrl &= ~SDHCI_CTRL_TUNED_CLK;
2292 	ctrl &= ~SDHCI_CTRL_EXEC_TUNING;
2293 	sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2294 }
2295 EXPORT_SYMBOL_GPL(sdhci_reset_tuning);
2296 
2297 static void sdhci_abort_tuning(struct sdhci_host *host, u32 opcode)
2298 {
2299 	sdhci_reset_tuning(host);
2300 
2301 	sdhci_do_reset(host, SDHCI_RESET_CMD);
2302 	sdhci_do_reset(host, SDHCI_RESET_DATA);
2303 
2304 	sdhci_end_tuning(host);
2305 
2306 	mmc_abort_tuning(host->mmc, opcode);
2307 }
2308 
2309 /*
2310  * We use sdhci_send_tuning() because mmc_send_tuning() is not a good fit. SDHCI
2311  * tuning command does not have a data payload (or rather the hardware does it
2312  * automatically) so mmc_send_tuning() will return -EIO. Also the tuning command
2313  * interrupt setup is different to other commands and there is no timeout
2314  * interrupt so special handling is needed.
2315  */
2316 void sdhci_send_tuning(struct sdhci_host *host, u32 opcode)
2317 {
2318 	struct mmc_host *mmc = host->mmc;
2319 	struct mmc_command cmd = {};
2320 	struct mmc_request mrq = {};
2321 	unsigned long flags;
2322 	u32 b = host->sdma_boundary;
2323 
2324 	spin_lock_irqsave(&host->lock, flags);
2325 
2326 	cmd.opcode = opcode;
2327 	cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC;
2328 	cmd.mrq = &mrq;
2329 
2330 	mrq.cmd = &cmd;
2331 	/*
2332 	 * In response to CMD19, the card sends 64 bytes of tuning
2333 	 * block to the Host Controller. So we set the block size
2334 	 * to 64 here.
2335 	 */
2336 	if (cmd.opcode == MMC_SEND_TUNING_BLOCK_HS200 &&
2337 	    mmc->ios.bus_width == MMC_BUS_WIDTH_8)
2338 		sdhci_writew(host, SDHCI_MAKE_BLKSZ(b, 128), SDHCI_BLOCK_SIZE);
2339 	else
2340 		sdhci_writew(host, SDHCI_MAKE_BLKSZ(b, 64), SDHCI_BLOCK_SIZE);
2341 
2342 	/*
2343 	 * The tuning block is sent by the card to the host controller.
2344 	 * So we set the TRNS_READ bit in the Transfer Mode register.
2345 	 * This also takes care of setting DMA Enable and Multi Block
2346 	 * Select in the same register to 0.
2347 	 */
2348 	sdhci_writew(host, SDHCI_TRNS_READ, SDHCI_TRANSFER_MODE);
2349 
2350 	sdhci_send_command(host, &cmd);
2351 
2352 	host->cmd = NULL;
2353 
2354 	sdhci_del_timer(host, &mrq);
2355 
2356 	host->tuning_done = 0;
2357 
2358 	spin_unlock_irqrestore(&host->lock, flags);
2359 
2360 	/* Wait for Buffer Read Ready interrupt */
2361 	wait_event_timeout(host->buf_ready_int, (host->tuning_done == 1),
2362 			   msecs_to_jiffies(50));
2363 
2364 }
2365 EXPORT_SYMBOL_GPL(sdhci_send_tuning);
2366 
2367 static int __sdhci_execute_tuning(struct sdhci_host *host, u32 opcode)
2368 {
2369 	int i;
2370 
2371 	/*
2372 	 * Issue opcode repeatedly till Execute Tuning is set to 0 or the number
2373 	 * of loops reaches tuning loop count.
2374 	 */
2375 	for (i = 0; i < host->tuning_loop_count; i++) {
2376 		u16 ctrl;
2377 
2378 		sdhci_send_tuning(host, opcode);
2379 
2380 		if (!host->tuning_done) {
2381 			pr_info("%s: Tuning timeout, falling back to fixed sampling clock\n",
2382 				mmc_hostname(host->mmc));
2383 			sdhci_abort_tuning(host, opcode);
2384 			return -ETIMEDOUT;
2385 		}
2386 
2387 		/* Spec does not require a delay between tuning cycles */
2388 		if (host->tuning_delay > 0)
2389 			mdelay(host->tuning_delay);
2390 
2391 		ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2392 		if (!(ctrl & SDHCI_CTRL_EXEC_TUNING)) {
2393 			if (ctrl & SDHCI_CTRL_TUNED_CLK)
2394 				return 0; /* Success! */
2395 			break;
2396 		}
2397 
2398 	}
2399 
2400 	pr_info("%s: Tuning failed, falling back to fixed sampling clock\n",
2401 		mmc_hostname(host->mmc));
2402 	sdhci_reset_tuning(host);
2403 	return -EAGAIN;
2404 }
2405 
2406 int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode)
2407 {
2408 	struct sdhci_host *host = mmc_priv(mmc);
2409 	int err = 0;
2410 	unsigned int tuning_count = 0;
2411 	bool hs400_tuning;
2412 
2413 	hs400_tuning = host->flags & SDHCI_HS400_TUNING;
2414 
2415 	if (host->tuning_mode == SDHCI_TUNING_MODE_1)
2416 		tuning_count = host->tuning_count;
2417 
2418 	/*
2419 	 * The Host Controller needs tuning in case of SDR104 and DDR50
2420 	 * mode, and for SDR50 mode when Use Tuning for SDR50 is set in
2421 	 * the Capabilities register.
2422 	 * If the Host Controller supports the HS200 mode then the
2423 	 * tuning function has to be executed.
2424 	 */
2425 	switch (host->timing) {
2426 	/* HS400 tuning is done in HS200 mode */
2427 	case MMC_TIMING_MMC_HS400:
2428 		err = -EINVAL;
2429 		goto out;
2430 
2431 	case MMC_TIMING_MMC_HS200:
2432 		/*
2433 		 * Periodic re-tuning for HS400 is not expected to be needed, so
2434 		 * disable it here.
2435 		 */
2436 		if (hs400_tuning)
2437 			tuning_count = 0;
2438 		break;
2439 
2440 	case MMC_TIMING_UHS_SDR104:
2441 	case MMC_TIMING_UHS_DDR50:
2442 		break;
2443 
2444 	case MMC_TIMING_UHS_SDR50:
2445 		if (host->flags & SDHCI_SDR50_NEEDS_TUNING)
2446 			break;
2447 		/* FALLTHROUGH */
2448 
2449 	default:
2450 		goto out;
2451 	}
2452 
2453 	if (host->ops->platform_execute_tuning) {
2454 		err = host->ops->platform_execute_tuning(host, opcode);
2455 		goto out;
2456 	}
2457 
2458 	host->mmc->retune_period = tuning_count;
2459 
2460 	if (host->tuning_delay < 0)
2461 		host->tuning_delay = opcode == MMC_SEND_TUNING_BLOCK;
2462 
2463 	sdhci_start_tuning(host);
2464 
2465 	host->tuning_err = __sdhci_execute_tuning(host, opcode);
2466 
2467 	sdhci_end_tuning(host);
2468 out:
2469 	host->flags &= ~SDHCI_HS400_TUNING;
2470 
2471 	return err;
2472 }
2473 EXPORT_SYMBOL_GPL(sdhci_execute_tuning);
2474 
2475 static void sdhci_enable_preset_value(struct sdhci_host *host, bool enable)
2476 {
2477 	/* Host Controller v3.00 defines preset value registers */
2478 	if (host->version < SDHCI_SPEC_300)
2479 		return;
2480 
2481 	/*
2482 	 * We only enable or disable Preset Value if they are not already
2483 	 * enabled or disabled respectively. Otherwise, we bail out.
2484 	 */
2485 	if (host->preset_enabled != enable) {
2486 		u16 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2487 
2488 		if (enable)
2489 			ctrl |= SDHCI_CTRL_PRESET_VAL_ENABLE;
2490 		else
2491 			ctrl &= ~SDHCI_CTRL_PRESET_VAL_ENABLE;
2492 
2493 		sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2494 
2495 		if (enable)
2496 			host->flags |= SDHCI_PV_ENABLED;
2497 		else
2498 			host->flags &= ~SDHCI_PV_ENABLED;
2499 
2500 		host->preset_enabled = enable;
2501 	}
2502 }
2503 
2504 static void sdhci_post_req(struct mmc_host *mmc, struct mmc_request *mrq,
2505 				int err)
2506 {
2507 	struct sdhci_host *host = mmc_priv(mmc);
2508 	struct mmc_data *data = mrq->data;
2509 
2510 	if (data->host_cookie != COOKIE_UNMAPPED)
2511 		dma_unmap_sg(mmc_dev(host->mmc), data->sg, data->sg_len,
2512 			     mmc_get_dma_dir(data));
2513 
2514 	data->host_cookie = COOKIE_UNMAPPED;
2515 }
2516 
2517 static void sdhci_pre_req(struct mmc_host *mmc, struct mmc_request *mrq)
2518 {
2519 	struct sdhci_host *host = mmc_priv(mmc);
2520 
2521 	mrq->data->host_cookie = COOKIE_UNMAPPED;
2522 
2523 	/*
2524 	 * No pre-mapping in the pre hook if we're using the bounce buffer,
2525 	 * for that we would need two bounce buffers since one buffer is
2526 	 * in flight when this is getting called.
2527 	 */
2528 	if (host->flags & SDHCI_REQ_USE_DMA && !host->bounce_buffer)
2529 		sdhci_pre_dma_transfer(host, mrq->data, COOKIE_PRE_MAPPED);
2530 }
2531 
2532 static void sdhci_error_out_mrqs(struct sdhci_host *host, int err)
2533 {
2534 	if (host->data_cmd) {
2535 		host->data_cmd->error = err;
2536 		sdhci_finish_mrq(host, host->data_cmd->mrq);
2537 	}
2538 
2539 	if (host->cmd) {
2540 		host->cmd->error = err;
2541 		sdhci_finish_mrq(host, host->cmd->mrq);
2542 	}
2543 }
2544 
2545 static void sdhci_card_event(struct mmc_host *mmc)
2546 {
2547 	struct sdhci_host *host = mmc_priv(mmc);
2548 	unsigned long flags;
2549 	int present;
2550 
2551 	/* First check if client has provided their own card event */
2552 	if (host->ops->card_event)
2553 		host->ops->card_event(host);
2554 
2555 	present = mmc->ops->get_cd(mmc);
2556 
2557 	spin_lock_irqsave(&host->lock, flags);
2558 
2559 	/* Check sdhci_has_requests() first in case we are runtime suspended */
2560 	if (sdhci_has_requests(host) && !present) {
2561 		pr_err("%s: Card removed during transfer!\n",
2562 			mmc_hostname(host->mmc));
2563 		pr_err("%s: Resetting controller.\n",
2564 			mmc_hostname(host->mmc));
2565 
2566 		sdhci_do_reset(host, SDHCI_RESET_CMD);
2567 		sdhci_do_reset(host, SDHCI_RESET_DATA);
2568 
2569 		sdhci_error_out_mrqs(host, -ENOMEDIUM);
2570 	}
2571 
2572 	spin_unlock_irqrestore(&host->lock, flags);
2573 }
2574 
2575 static const struct mmc_host_ops sdhci_ops = {
2576 	.request	= sdhci_request,
2577 	.post_req	= sdhci_post_req,
2578 	.pre_req	= sdhci_pre_req,
2579 	.set_ios	= sdhci_set_ios,
2580 	.get_cd		= sdhci_get_cd,
2581 	.get_ro		= sdhci_get_ro,
2582 	.hw_reset	= sdhci_hw_reset,
2583 	.enable_sdio_irq = sdhci_enable_sdio_irq,
2584 	.start_signal_voltage_switch	= sdhci_start_signal_voltage_switch,
2585 	.prepare_hs400_tuning		= sdhci_prepare_hs400_tuning,
2586 	.execute_tuning			= sdhci_execute_tuning,
2587 	.card_event			= sdhci_card_event,
2588 	.card_busy	= sdhci_card_busy,
2589 };
2590 
2591 /*****************************************************************************\
2592  *                                                                           *
2593  * Request done                                                              *
2594  *                                                                           *
2595 \*****************************************************************************/
2596 
2597 static bool sdhci_request_done(struct sdhci_host *host)
2598 {
2599 	unsigned long flags;
2600 	struct mmc_request *mrq;
2601 	int i;
2602 
2603 	spin_lock_irqsave(&host->lock, flags);
2604 
2605 	for (i = 0; i < SDHCI_MAX_MRQS; i++) {
2606 		mrq = host->mrqs_done[i];
2607 		if (mrq)
2608 			break;
2609 	}
2610 
2611 	if (!mrq) {
2612 		spin_unlock_irqrestore(&host->lock, flags);
2613 		return true;
2614 	}
2615 
2616 	/*
2617 	 * Always unmap the data buffers if they were mapped by
2618 	 * sdhci_prepare_data() whenever we finish with a request.
2619 	 * This avoids leaking DMA mappings on error.
2620 	 */
2621 	if (host->flags & SDHCI_REQ_USE_DMA) {
2622 		struct mmc_data *data = mrq->data;
2623 
2624 		if (data && data->host_cookie == COOKIE_MAPPED) {
2625 			if (host->bounce_buffer) {
2626 				/*
2627 				 * On reads, copy the bounced data into the
2628 				 * sglist
2629 				 */
2630 				if (mmc_get_dma_dir(data) == DMA_FROM_DEVICE) {
2631 					unsigned int length = data->bytes_xfered;
2632 
2633 					if (length > host->bounce_buffer_size) {
2634 						pr_err("%s: bounce buffer is %u bytes but DMA claims to have transferred %u bytes\n",
2635 						       mmc_hostname(host->mmc),
2636 						       host->bounce_buffer_size,
2637 						       data->bytes_xfered);
2638 						/* Cap it down and continue */
2639 						length = host->bounce_buffer_size;
2640 					}
2641 					dma_sync_single_for_cpu(
2642 						host->mmc->parent,
2643 						host->bounce_addr,
2644 						host->bounce_buffer_size,
2645 						DMA_FROM_DEVICE);
2646 					sg_copy_from_buffer(data->sg,
2647 						data->sg_len,
2648 						host->bounce_buffer,
2649 						length);
2650 				} else {
2651 					/* No copying, just switch ownership */
2652 					dma_sync_single_for_cpu(
2653 						host->mmc->parent,
2654 						host->bounce_addr,
2655 						host->bounce_buffer_size,
2656 						mmc_get_dma_dir(data));
2657 				}
2658 			} else {
2659 				/* Unmap the raw data */
2660 				dma_unmap_sg(mmc_dev(host->mmc), data->sg,
2661 					     data->sg_len,
2662 					     mmc_get_dma_dir(data));
2663 			}
2664 			data->host_cookie = COOKIE_UNMAPPED;
2665 		}
2666 	}
2667 
2668 	/*
2669 	 * The controller needs a reset of internal state machines
2670 	 * upon error conditions.
2671 	 */
2672 	if (sdhci_needs_reset(host, mrq)) {
2673 		/*
2674 		 * Do not finish until command and data lines are available for
2675 		 * reset. Note there can only be one other mrq, so it cannot
2676 		 * also be in mrqs_done, otherwise host->cmd and host->data_cmd
2677 		 * would both be null.
2678 		 */
2679 		if (host->cmd || host->data_cmd) {
2680 			spin_unlock_irqrestore(&host->lock, flags);
2681 			return true;
2682 		}
2683 
2684 		/* Some controllers need this kick or reset won't work here */
2685 		if (host->quirks & SDHCI_QUIRK_CLOCK_BEFORE_RESET)
2686 			/* This is to force an update */
2687 			host->ops->set_clock(host, host->clock);
2688 
2689 		/* Spec says we should do both at the same time, but Ricoh
2690 		   controllers do not like that. */
2691 		sdhci_do_reset(host, SDHCI_RESET_CMD);
2692 		sdhci_do_reset(host, SDHCI_RESET_DATA);
2693 
2694 		host->pending_reset = false;
2695 	}
2696 
2697 	host->mrqs_done[i] = NULL;
2698 
2699 	spin_unlock_irqrestore(&host->lock, flags);
2700 
2701 	mmc_request_done(host->mmc, mrq);
2702 
2703 	return false;
2704 }
2705 
2706 static void sdhci_complete_work(struct work_struct *work)
2707 {
2708 	struct sdhci_host *host = container_of(work, struct sdhci_host,
2709 					       complete_work);
2710 
2711 	while (!sdhci_request_done(host))
2712 		;
2713 }
2714 
2715 static void sdhci_timeout_timer(struct timer_list *t)
2716 {
2717 	struct sdhci_host *host;
2718 	unsigned long flags;
2719 
2720 	host = from_timer(host, t, timer);
2721 
2722 	spin_lock_irqsave(&host->lock, flags);
2723 
2724 	if (host->cmd && !sdhci_data_line_cmd(host->cmd)) {
2725 		pr_err("%s: Timeout waiting for hardware cmd interrupt.\n",
2726 		       mmc_hostname(host->mmc));
2727 		sdhci_dumpregs(host);
2728 
2729 		host->cmd->error = -ETIMEDOUT;
2730 		sdhci_finish_mrq(host, host->cmd->mrq);
2731 	}
2732 
2733 	spin_unlock_irqrestore(&host->lock, flags);
2734 }
2735 
2736 static void sdhci_timeout_data_timer(struct timer_list *t)
2737 {
2738 	struct sdhci_host *host;
2739 	unsigned long flags;
2740 
2741 	host = from_timer(host, t, data_timer);
2742 
2743 	spin_lock_irqsave(&host->lock, flags);
2744 
2745 	if (host->data || host->data_cmd ||
2746 	    (host->cmd && sdhci_data_line_cmd(host->cmd))) {
2747 		pr_err("%s: Timeout waiting for hardware interrupt.\n",
2748 		       mmc_hostname(host->mmc));
2749 		sdhci_dumpregs(host);
2750 
2751 		if (host->data) {
2752 			host->data->error = -ETIMEDOUT;
2753 			sdhci_finish_data(host);
2754 			queue_work(host->complete_wq, &host->complete_work);
2755 		} else if (host->data_cmd) {
2756 			host->data_cmd->error = -ETIMEDOUT;
2757 			sdhci_finish_mrq(host, host->data_cmd->mrq);
2758 		} else {
2759 			host->cmd->error = -ETIMEDOUT;
2760 			sdhci_finish_mrq(host, host->cmd->mrq);
2761 		}
2762 	}
2763 
2764 	spin_unlock_irqrestore(&host->lock, flags);
2765 }
2766 
2767 /*****************************************************************************\
2768  *                                                                           *
2769  * Interrupt handling                                                        *
2770  *                                                                           *
2771 \*****************************************************************************/
2772 
2773 static void sdhci_cmd_irq(struct sdhci_host *host, u32 intmask, u32 *intmask_p)
2774 {
2775 	/* Handle auto-CMD12 error */
2776 	if (intmask & SDHCI_INT_AUTO_CMD_ERR && host->data_cmd) {
2777 		struct mmc_request *mrq = host->data_cmd->mrq;
2778 		u16 auto_cmd_status = sdhci_readw(host, SDHCI_AUTO_CMD_STATUS);
2779 		int data_err_bit = (auto_cmd_status & SDHCI_AUTO_CMD_TIMEOUT) ?
2780 				   SDHCI_INT_DATA_TIMEOUT :
2781 				   SDHCI_INT_DATA_CRC;
2782 
2783 		/* Treat auto-CMD12 error the same as data error */
2784 		if (!mrq->sbc && (host->flags & SDHCI_AUTO_CMD12)) {
2785 			*intmask_p |= data_err_bit;
2786 			return;
2787 		}
2788 	}
2789 
2790 	if (!host->cmd) {
2791 		/*
2792 		 * SDHCI recovers from errors by resetting the cmd and data
2793 		 * circuits.  Until that is done, there very well might be more
2794 		 * interrupts, so ignore them in that case.
2795 		 */
2796 		if (host->pending_reset)
2797 			return;
2798 		pr_err("%s: Got command interrupt 0x%08x even though no command operation was in progress.\n",
2799 		       mmc_hostname(host->mmc), (unsigned)intmask);
2800 		sdhci_dumpregs(host);
2801 		return;
2802 	}
2803 
2804 	if (intmask & (SDHCI_INT_TIMEOUT | SDHCI_INT_CRC |
2805 		       SDHCI_INT_END_BIT | SDHCI_INT_INDEX)) {
2806 		if (intmask & SDHCI_INT_TIMEOUT)
2807 			host->cmd->error = -ETIMEDOUT;
2808 		else
2809 			host->cmd->error = -EILSEQ;
2810 
2811 		/* Treat data command CRC error the same as data CRC error */
2812 		if (host->cmd->data &&
2813 		    (intmask & (SDHCI_INT_CRC | SDHCI_INT_TIMEOUT)) ==
2814 		     SDHCI_INT_CRC) {
2815 			host->cmd = NULL;
2816 			*intmask_p |= SDHCI_INT_DATA_CRC;
2817 			return;
2818 		}
2819 
2820 		__sdhci_finish_mrq(host, host->cmd->mrq);
2821 		return;
2822 	}
2823 
2824 	/* Handle auto-CMD23 error */
2825 	if (intmask & SDHCI_INT_AUTO_CMD_ERR) {
2826 		struct mmc_request *mrq = host->cmd->mrq;
2827 		u16 auto_cmd_status = sdhci_readw(host, SDHCI_AUTO_CMD_STATUS);
2828 		int err = (auto_cmd_status & SDHCI_AUTO_CMD_TIMEOUT) ?
2829 			  -ETIMEDOUT :
2830 			  -EILSEQ;
2831 
2832 		if (mrq->sbc && (host->flags & SDHCI_AUTO_CMD23)) {
2833 			mrq->sbc->error = err;
2834 			__sdhci_finish_mrq(host, mrq);
2835 			return;
2836 		}
2837 	}
2838 
2839 	if (intmask & SDHCI_INT_RESPONSE)
2840 		sdhci_finish_command(host);
2841 }
2842 
2843 static void sdhci_adma_show_error(struct sdhci_host *host)
2844 {
2845 	void *desc = host->adma_table;
2846 
2847 	sdhci_dumpregs(host);
2848 
2849 	while (true) {
2850 		struct sdhci_adma2_64_desc *dma_desc = desc;
2851 
2852 		if (host->flags & SDHCI_USE_64_BIT_DMA)
2853 			DBG("%p: DMA 0x%08x%08x, LEN 0x%04x, Attr=0x%02x\n",
2854 			    desc, le32_to_cpu(dma_desc->addr_hi),
2855 			    le32_to_cpu(dma_desc->addr_lo),
2856 			    le16_to_cpu(dma_desc->len),
2857 			    le16_to_cpu(dma_desc->cmd));
2858 		else
2859 			DBG("%p: DMA 0x%08x, LEN 0x%04x, Attr=0x%02x\n",
2860 			    desc, le32_to_cpu(dma_desc->addr_lo),
2861 			    le16_to_cpu(dma_desc->len),
2862 			    le16_to_cpu(dma_desc->cmd));
2863 
2864 		desc += host->desc_sz;
2865 
2866 		if (dma_desc->cmd & cpu_to_le16(ADMA2_END))
2867 			break;
2868 	}
2869 }
2870 
2871 static void sdhci_data_irq(struct sdhci_host *host, u32 intmask)
2872 {
2873 	u32 command;
2874 
2875 	/* CMD19 generates _only_ Buffer Read Ready interrupt */
2876 	if (intmask & SDHCI_INT_DATA_AVAIL) {
2877 		command = SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND));
2878 		if (command == MMC_SEND_TUNING_BLOCK ||
2879 		    command == MMC_SEND_TUNING_BLOCK_HS200) {
2880 			host->tuning_done = 1;
2881 			wake_up(&host->buf_ready_int);
2882 			return;
2883 		}
2884 	}
2885 
2886 	if (!host->data) {
2887 		struct mmc_command *data_cmd = host->data_cmd;
2888 
2889 		/*
2890 		 * The "data complete" interrupt is also used to
2891 		 * indicate that a busy state has ended. See comment
2892 		 * above in sdhci_cmd_irq().
2893 		 */
2894 		if (data_cmd && (data_cmd->flags & MMC_RSP_BUSY)) {
2895 			if (intmask & SDHCI_INT_DATA_TIMEOUT) {
2896 				host->data_cmd = NULL;
2897 				data_cmd->error = -ETIMEDOUT;
2898 				__sdhci_finish_mrq(host, data_cmd->mrq);
2899 				return;
2900 			}
2901 			if (intmask & SDHCI_INT_DATA_END) {
2902 				host->data_cmd = NULL;
2903 				/*
2904 				 * Some cards handle busy-end interrupt
2905 				 * before the command completed, so make
2906 				 * sure we do things in the proper order.
2907 				 */
2908 				if (host->cmd == data_cmd)
2909 					return;
2910 
2911 				__sdhci_finish_mrq(host, data_cmd->mrq);
2912 				return;
2913 			}
2914 		}
2915 
2916 		/*
2917 		 * SDHCI recovers from errors by resetting the cmd and data
2918 		 * circuits. Until that is done, there very well might be more
2919 		 * interrupts, so ignore them in that case.
2920 		 */
2921 		if (host->pending_reset)
2922 			return;
2923 
2924 		pr_err("%s: Got data interrupt 0x%08x even though no data operation was in progress.\n",
2925 		       mmc_hostname(host->mmc), (unsigned)intmask);
2926 		sdhci_dumpregs(host);
2927 
2928 		return;
2929 	}
2930 
2931 	if (intmask & SDHCI_INT_DATA_TIMEOUT)
2932 		host->data->error = -ETIMEDOUT;
2933 	else if (intmask & SDHCI_INT_DATA_END_BIT)
2934 		host->data->error = -EILSEQ;
2935 	else if ((intmask & SDHCI_INT_DATA_CRC) &&
2936 		SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND))
2937 			!= MMC_BUS_TEST_R)
2938 		host->data->error = -EILSEQ;
2939 	else if (intmask & SDHCI_INT_ADMA_ERROR) {
2940 		pr_err("%s: ADMA error\n", mmc_hostname(host->mmc));
2941 		sdhci_adma_show_error(host);
2942 		host->data->error = -EIO;
2943 		if (host->ops->adma_workaround)
2944 			host->ops->adma_workaround(host, intmask);
2945 	}
2946 
2947 	if (host->data->error)
2948 		sdhci_finish_data(host);
2949 	else {
2950 		if (intmask & (SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL))
2951 			sdhci_transfer_pio(host);
2952 
2953 		/*
2954 		 * We currently don't do anything fancy with DMA
2955 		 * boundaries, but as we can't disable the feature
2956 		 * we need to at least restart the transfer.
2957 		 *
2958 		 * According to the spec sdhci_readl(host, SDHCI_DMA_ADDRESS)
2959 		 * should return a valid address to continue from, but as
2960 		 * some controllers are faulty, don't trust them.
2961 		 */
2962 		if (intmask & SDHCI_INT_DMA_END) {
2963 			dma_addr_t dmastart, dmanow;
2964 
2965 			dmastart = sdhci_sdma_address(host);
2966 			dmanow = dmastart + host->data->bytes_xfered;
2967 			/*
2968 			 * Force update to the next DMA block boundary.
2969 			 */
2970 			dmanow = (dmanow &
2971 				~((dma_addr_t)SDHCI_DEFAULT_BOUNDARY_SIZE - 1)) +
2972 				SDHCI_DEFAULT_BOUNDARY_SIZE;
2973 			host->data->bytes_xfered = dmanow - dmastart;
2974 			DBG("DMA base %pad, transferred 0x%06x bytes, next %pad\n",
2975 			    &dmastart, host->data->bytes_xfered, &dmanow);
2976 			sdhci_set_sdma_addr(host, dmanow);
2977 		}
2978 
2979 		if (intmask & SDHCI_INT_DATA_END) {
2980 			if (host->cmd == host->data_cmd) {
2981 				/*
2982 				 * Data managed to finish before the
2983 				 * command completed. Make sure we do
2984 				 * things in the proper order.
2985 				 */
2986 				host->data_early = 1;
2987 			} else {
2988 				sdhci_finish_data(host);
2989 			}
2990 		}
2991 	}
2992 }
2993 
2994 static inline bool sdhci_defer_done(struct sdhci_host *host,
2995 				    struct mmc_request *mrq)
2996 {
2997 	struct mmc_data *data = mrq->data;
2998 
2999 	return host->pending_reset ||
3000 	       ((host->flags & SDHCI_REQ_USE_DMA) && data &&
3001 		data->host_cookie == COOKIE_MAPPED);
3002 }
3003 
3004 static irqreturn_t sdhci_irq(int irq, void *dev_id)
3005 {
3006 	struct mmc_request *mrqs_done[SDHCI_MAX_MRQS] = {0};
3007 	irqreturn_t result = IRQ_NONE;
3008 	struct sdhci_host *host = dev_id;
3009 	u32 intmask, mask, unexpected = 0;
3010 	int max_loops = 16;
3011 	int i;
3012 
3013 	spin_lock(&host->lock);
3014 
3015 	if (host->runtime_suspended && !sdhci_sdio_irq_enabled(host)) {
3016 		spin_unlock(&host->lock);
3017 		return IRQ_NONE;
3018 	}
3019 
3020 	intmask = sdhci_readl(host, SDHCI_INT_STATUS);
3021 	if (!intmask || intmask == 0xffffffff) {
3022 		result = IRQ_NONE;
3023 		goto out;
3024 	}
3025 
3026 	do {
3027 		DBG("IRQ status 0x%08x\n", intmask);
3028 
3029 		if (host->ops->irq) {
3030 			intmask = host->ops->irq(host, intmask);
3031 			if (!intmask)
3032 				goto cont;
3033 		}
3034 
3035 		/* Clear selected interrupts. */
3036 		mask = intmask & (SDHCI_INT_CMD_MASK | SDHCI_INT_DATA_MASK |
3037 				  SDHCI_INT_BUS_POWER);
3038 		sdhci_writel(host, mask, SDHCI_INT_STATUS);
3039 
3040 		if (intmask & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) {
3041 			u32 present = sdhci_readl(host, SDHCI_PRESENT_STATE) &
3042 				      SDHCI_CARD_PRESENT;
3043 
3044 			/*
3045 			 * There is a observation on i.mx esdhc.  INSERT
3046 			 * bit will be immediately set again when it gets
3047 			 * cleared, if a card is inserted.  We have to mask
3048 			 * the irq to prevent interrupt storm which will
3049 			 * freeze the system.  And the REMOVE gets the
3050 			 * same situation.
3051 			 *
3052 			 * More testing are needed here to ensure it works
3053 			 * for other platforms though.
3054 			 */
3055 			host->ier &= ~(SDHCI_INT_CARD_INSERT |
3056 				       SDHCI_INT_CARD_REMOVE);
3057 			host->ier |= present ? SDHCI_INT_CARD_REMOVE :
3058 					       SDHCI_INT_CARD_INSERT;
3059 			sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
3060 			sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
3061 
3062 			sdhci_writel(host, intmask & (SDHCI_INT_CARD_INSERT |
3063 				     SDHCI_INT_CARD_REMOVE), SDHCI_INT_STATUS);
3064 
3065 			host->thread_isr |= intmask & (SDHCI_INT_CARD_INSERT |
3066 						       SDHCI_INT_CARD_REMOVE);
3067 			result = IRQ_WAKE_THREAD;
3068 		}
3069 
3070 		if (intmask & SDHCI_INT_CMD_MASK)
3071 			sdhci_cmd_irq(host, intmask & SDHCI_INT_CMD_MASK, &intmask);
3072 
3073 		if (intmask & SDHCI_INT_DATA_MASK)
3074 			sdhci_data_irq(host, intmask & SDHCI_INT_DATA_MASK);
3075 
3076 		if (intmask & SDHCI_INT_BUS_POWER)
3077 			pr_err("%s: Card is consuming too much power!\n",
3078 				mmc_hostname(host->mmc));
3079 
3080 		if (intmask & SDHCI_INT_RETUNE)
3081 			mmc_retune_needed(host->mmc);
3082 
3083 		if ((intmask & SDHCI_INT_CARD_INT) &&
3084 		    (host->ier & SDHCI_INT_CARD_INT)) {
3085 			sdhci_enable_sdio_irq_nolock(host, false);
3086 			host->thread_isr |= SDHCI_INT_CARD_INT;
3087 			result = IRQ_WAKE_THREAD;
3088 		}
3089 
3090 		intmask &= ~(SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE |
3091 			     SDHCI_INT_CMD_MASK | SDHCI_INT_DATA_MASK |
3092 			     SDHCI_INT_ERROR | SDHCI_INT_BUS_POWER |
3093 			     SDHCI_INT_RETUNE | SDHCI_INT_CARD_INT);
3094 
3095 		if (intmask) {
3096 			unexpected |= intmask;
3097 			sdhci_writel(host, intmask, SDHCI_INT_STATUS);
3098 		}
3099 cont:
3100 		if (result == IRQ_NONE)
3101 			result = IRQ_HANDLED;
3102 
3103 		intmask = sdhci_readl(host, SDHCI_INT_STATUS);
3104 	} while (intmask && --max_loops);
3105 
3106 	/* Determine if mrqs can be completed immediately */
3107 	for (i = 0; i < SDHCI_MAX_MRQS; i++) {
3108 		struct mmc_request *mrq = host->mrqs_done[i];
3109 
3110 		if (!mrq)
3111 			continue;
3112 
3113 		if (sdhci_defer_done(host, mrq)) {
3114 			result = IRQ_WAKE_THREAD;
3115 		} else {
3116 			mrqs_done[i] = mrq;
3117 			host->mrqs_done[i] = NULL;
3118 		}
3119 	}
3120 out:
3121 	spin_unlock(&host->lock);
3122 
3123 	/* Process mrqs ready for immediate completion */
3124 	for (i = 0; i < SDHCI_MAX_MRQS; i++) {
3125 		if (mrqs_done[i])
3126 			mmc_request_done(host->mmc, mrqs_done[i]);
3127 	}
3128 
3129 	if (unexpected) {
3130 		pr_err("%s: Unexpected interrupt 0x%08x.\n",
3131 			   mmc_hostname(host->mmc), unexpected);
3132 		sdhci_dumpregs(host);
3133 	}
3134 
3135 	return result;
3136 }
3137 
3138 static irqreturn_t sdhci_thread_irq(int irq, void *dev_id)
3139 {
3140 	struct sdhci_host *host = dev_id;
3141 	unsigned long flags;
3142 	u32 isr;
3143 
3144 	while (!sdhci_request_done(host))
3145 		;
3146 
3147 	spin_lock_irqsave(&host->lock, flags);
3148 	isr = host->thread_isr;
3149 	host->thread_isr = 0;
3150 	spin_unlock_irqrestore(&host->lock, flags);
3151 
3152 	if (isr & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) {
3153 		struct mmc_host *mmc = host->mmc;
3154 
3155 		mmc->ops->card_event(mmc);
3156 		mmc_detect_change(mmc, msecs_to_jiffies(200));
3157 	}
3158 
3159 	if (isr & SDHCI_INT_CARD_INT) {
3160 		sdio_run_irqs(host->mmc);
3161 
3162 		spin_lock_irqsave(&host->lock, flags);
3163 		if (host->flags & SDHCI_SDIO_IRQ_ENABLED)
3164 			sdhci_enable_sdio_irq_nolock(host, true);
3165 		spin_unlock_irqrestore(&host->lock, flags);
3166 	}
3167 
3168 	return IRQ_HANDLED;
3169 }
3170 
3171 /*****************************************************************************\
3172  *                                                                           *
3173  * Suspend/resume                                                            *
3174  *                                                                           *
3175 \*****************************************************************************/
3176 
3177 #ifdef CONFIG_PM
3178 
3179 static bool sdhci_cd_irq_can_wakeup(struct sdhci_host *host)
3180 {
3181 	return mmc_card_is_removable(host->mmc) &&
3182 	       !(host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) &&
3183 	       !mmc_can_gpio_cd(host->mmc);
3184 }
3185 
3186 /*
3187  * To enable wakeup events, the corresponding events have to be enabled in
3188  * the Interrupt Status Enable register too. See 'Table 1-6: Wakeup Signal
3189  * Table' in the SD Host Controller Standard Specification.
3190  * It is useless to restore SDHCI_INT_ENABLE state in
3191  * sdhci_disable_irq_wakeups() since it will be set by
3192  * sdhci_enable_card_detection() or sdhci_init().
3193  */
3194 static bool sdhci_enable_irq_wakeups(struct sdhci_host *host)
3195 {
3196 	u8 mask = SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE |
3197 		  SDHCI_WAKE_ON_INT;
3198 	u32 irq_val = 0;
3199 	u8 wake_val = 0;
3200 	u8 val;
3201 
3202 	if (sdhci_cd_irq_can_wakeup(host)) {
3203 		wake_val |= SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE;
3204 		irq_val |= SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE;
3205 	}
3206 
3207 	if (mmc_card_wake_sdio_irq(host->mmc)) {
3208 		wake_val |= SDHCI_WAKE_ON_INT;
3209 		irq_val |= SDHCI_INT_CARD_INT;
3210 	}
3211 
3212 	if (!irq_val)
3213 		return false;
3214 
3215 	val = sdhci_readb(host, SDHCI_WAKE_UP_CONTROL);
3216 	val &= ~mask;
3217 	val |= wake_val;
3218 	sdhci_writeb(host, val, SDHCI_WAKE_UP_CONTROL);
3219 
3220 	sdhci_writel(host, irq_val, SDHCI_INT_ENABLE);
3221 
3222 	host->irq_wake_enabled = !enable_irq_wake(host->irq);
3223 
3224 	return host->irq_wake_enabled;
3225 }
3226 
3227 static void sdhci_disable_irq_wakeups(struct sdhci_host *host)
3228 {
3229 	u8 val;
3230 	u8 mask = SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE
3231 			| SDHCI_WAKE_ON_INT;
3232 
3233 	val = sdhci_readb(host, SDHCI_WAKE_UP_CONTROL);
3234 	val &= ~mask;
3235 	sdhci_writeb(host, val, SDHCI_WAKE_UP_CONTROL);
3236 
3237 	disable_irq_wake(host->irq);
3238 
3239 	host->irq_wake_enabled = false;
3240 }
3241 
3242 int sdhci_suspend_host(struct sdhci_host *host)
3243 {
3244 	sdhci_disable_card_detection(host);
3245 
3246 	mmc_retune_timer_stop(host->mmc);
3247 
3248 	if (!device_may_wakeup(mmc_dev(host->mmc)) ||
3249 	    !sdhci_enable_irq_wakeups(host)) {
3250 		host->ier = 0;
3251 		sdhci_writel(host, 0, SDHCI_INT_ENABLE);
3252 		sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
3253 		free_irq(host->irq, host);
3254 	}
3255 
3256 	return 0;
3257 }
3258 
3259 EXPORT_SYMBOL_GPL(sdhci_suspend_host);
3260 
3261 int sdhci_resume_host(struct sdhci_host *host)
3262 {
3263 	struct mmc_host *mmc = host->mmc;
3264 	int ret = 0;
3265 
3266 	if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
3267 		if (host->ops->enable_dma)
3268 			host->ops->enable_dma(host);
3269 	}
3270 
3271 	if ((host->mmc->pm_flags & MMC_PM_KEEP_POWER) &&
3272 	    (host->quirks2 & SDHCI_QUIRK2_HOST_OFF_CARD_ON)) {
3273 		/* Card keeps power but host controller does not */
3274 		sdhci_init(host, 0);
3275 		host->pwr = 0;
3276 		host->clock = 0;
3277 		mmc->ops->set_ios(mmc, &mmc->ios);
3278 	} else {
3279 		sdhci_init(host, (host->mmc->pm_flags & MMC_PM_KEEP_POWER));
3280 	}
3281 
3282 	if (host->irq_wake_enabled) {
3283 		sdhci_disable_irq_wakeups(host);
3284 	} else {
3285 		ret = request_threaded_irq(host->irq, sdhci_irq,
3286 					   sdhci_thread_irq, IRQF_SHARED,
3287 					   mmc_hostname(host->mmc), host);
3288 		if (ret)
3289 			return ret;
3290 	}
3291 
3292 	sdhci_enable_card_detection(host);
3293 
3294 	return ret;
3295 }
3296 
3297 EXPORT_SYMBOL_GPL(sdhci_resume_host);
3298 
3299 int sdhci_runtime_suspend_host(struct sdhci_host *host)
3300 {
3301 	unsigned long flags;
3302 
3303 	mmc_retune_timer_stop(host->mmc);
3304 
3305 	spin_lock_irqsave(&host->lock, flags);
3306 	host->ier &= SDHCI_INT_CARD_INT;
3307 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
3308 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
3309 	spin_unlock_irqrestore(&host->lock, flags);
3310 
3311 	synchronize_hardirq(host->irq);
3312 
3313 	spin_lock_irqsave(&host->lock, flags);
3314 	host->runtime_suspended = true;
3315 	spin_unlock_irqrestore(&host->lock, flags);
3316 
3317 	return 0;
3318 }
3319 EXPORT_SYMBOL_GPL(sdhci_runtime_suspend_host);
3320 
3321 int sdhci_runtime_resume_host(struct sdhci_host *host)
3322 {
3323 	struct mmc_host *mmc = host->mmc;
3324 	unsigned long flags;
3325 	int host_flags = host->flags;
3326 
3327 	if (host_flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
3328 		if (host->ops->enable_dma)
3329 			host->ops->enable_dma(host);
3330 	}
3331 
3332 	sdhci_init(host, 0);
3333 
3334 	if (mmc->ios.power_mode != MMC_POWER_UNDEFINED &&
3335 	    mmc->ios.power_mode != MMC_POWER_OFF) {
3336 		/* Force clock and power re-program */
3337 		host->pwr = 0;
3338 		host->clock = 0;
3339 		mmc->ops->start_signal_voltage_switch(mmc, &mmc->ios);
3340 		mmc->ops->set_ios(mmc, &mmc->ios);
3341 
3342 		if ((host_flags & SDHCI_PV_ENABLED) &&
3343 		    !(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN)) {
3344 			spin_lock_irqsave(&host->lock, flags);
3345 			sdhci_enable_preset_value(host, true);
3346 			spin_unlock_irqrestore(&host->lock, flags);
3347 		}
3348 
3349 		if ((mmc->caps2 & MMC_CAP2_HS400_ES) &&
3350 		    mmc->ops->hs400_enhanced_strobe)
3351 			mmc->ops->hs400_enhanced_strobe(mmc, &mmc->ios);
3352 	}
3353 
3354 	spin_lock_irqsave(&host->lock, flags);
3355 
3356 	host->runtime_suspended = false;
3357 
3358 	/* Enable SDIO IRQ */
3359 	if (host->flags & SDHCI_SDIO_IRQ_ENABLED)
3360 		sdhci_enable_sdio_irq_nolock(host, true);
3361 
3362 	/* Enable Card Detection */
3363 	sdhci_enable_card_detection(host);
3364 
3365 	spin_unlock_irqrestore(&host->lock, flags);
3366 
3367 	return 0;
3368 }
3369 EXPORT_SYMBOL_GPL(sdhci_runtime_resume_host);
3370 
3371 #endif /* CONFIG_PM */
3372 
3373 /*****************************************************************************\
3374  *                                                                           *
3375  * Command Queue Engine (CQE) helpers                                        *
3376  *                                                                           *
3377 \*****************************************************************************/
3378 
3379 void sdhci_cqe_enable(struct mmc_host *mmc)
3380 {
3381 	struct sdhci_host *host = mmc_priv(mmc);
3382 	unsigned long flags;
3383 	u8 ctrl;
3384 
3385 	spin_lock_irqsave(&host->lock, flags);
3386 
3387 	ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
3388 	ctrl &= ~SDHCI_CTRL_DMA_MASK;
3389 	/*
3390 	 * Host from V4.10 supports ADMA3 DMA type.
3391 	 * ADMA3 performs integrated descriptor which is more suitable
3392 	 * for cmd queuing to fetch both command and transfer descriptors.
3393 	 */
3394 	if (host->v4_mode && (host->caps1 & SDHCI_CAN_DO_ADMA3))
3395 		ctrl |= SDHCI_CTRL_ADMA3;
3396 	else if (host->flags & SDHCI_USE_64_BIT_DMA)
3397 		ctrl |= SDHCI_CTRL_ADMA64;
3398 	else
3399 		ctrl |= SDHCI_CTRL_ADMA32;
3400 	sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
3401 
3402 	sdhci_writew(host, SDHCI_MAKE_BLKSZ(host->sdma_boundary, 512),
3403 		     SDHCI_BLOCK_SIZE);
3404 
3405 	/* Set maximum timeout */
3406 	sdhci_set_timeout(host, NULL);
3407 
3408 	host->ier = host->cqe_ier;
3409 
3410 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
3411 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
3412 
3413 	host->cqe_on = true;
3414 
3415 	pr_debug("%s: sdhci: CQE on, IRQ mask %#x, IRQ status %#x\n",
3416 		 mmc_hostname(mmc), host->ier,
3417 		 sdhci_readl(host, SDHCI_INT_STATUS));
3418 
3419 	spin_unlock_irqrestore(&host->lock, flags);
3420 }
3421 EXPORT_SYMBOL_GPL(sdhci_cqe_enable);
3422 
3423 void sdhci_cqe_disable(struct mmc_host *mmc, bool recovery)
3424 {
3425 	struct sdhci_host *host = mmc_priv(mmc);
3426 	unsigned long flags;
3427 
3428 	spin_lock_irqsave(&host->lock, flags);
3429 
3430 	sdhci_set_default_irqs(host);
3431 
3432 	host->cqe_on = false;
3433 
3434 	if (recovery) {
3435 		sdhci_do_reset(host, SDHCI_RESET_CMD);
3436 		sdhci_do_reset(host, SDHCI_RESET_DATA);
3437 	}
3438 
3439 	pr_debug("%s: sdhci: CQE off, IRQ mask %#x, IRQ status %#x\n",
3440 		 mmc_hostname(mmc), host->ier,
3441 		 sdhci_readl(host, SDHCI_INT_STATUS));
3442 
3443 	spin_unlock_irqrestore(&host->lock, flags);
3444 }
3445 EXPORT_SYMBOL_GPL(sdhci_cqe_disable);
3446 
3447 bool sdhci_cqe_irq(struct sdhci_host *host, u32 intmask, int *cmd_error,
3448 		   int *data_error)
3449 {
3450 	u32 mask;
3451 
3452 	if (!host->cqe_on)
3453 		return false;
3454 
3455 	if (intmask & (SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC))
3456 		*cmd_error = -EILSEQ;
3457 	else if (intmask & SDHCI_INT_TIMEOUT)
3458 		*cmd_error = -ETIMEDOUT;
3459 	else
3460 		*cmd_error = 0;
3461 
3462 	if (intmask & (SDHCI_INT_DATA_END_BIT | SDHCI_INT_DATA_CRC))
3463 		*data_error = -EILSEQ;
3464 	else if (intmask & SDHCI_INT_DATA_TIMEOUT)
3465 		*data_error = -ETIMEDOUT;
3466 	else if (intmask & SDHCI_INT_ADMA_ERROR)
3467 		*data_error = -EIO;
3468 	else
3469 		*data_error = 0;
3470 
3471 	/* Clear selected interrupts. */
3472 	mask = intmask & host->cqe_ier;
3473 	sdhci_writel(host, mask, SDHCI_INT_STATUS);
3474 
3475 	if (intmask & SDHCI_INT_BUS_POWER)
3476 		pr_err("%s: Card is consuming too much power!\n",
3477 		       mmc_hostname(host->mmc));
3478 
3479 	intmask &= ~(host->cqe_ier | SDHCI_INT_ERROR);
3480 	if (intmask) {
3481 		sdhci_writel(host, intmask, SDHCI_INT_STATUS);
3482 		pr_err("%s: CQE: Unexpected interrupt 0x%08x.\n",
3483 		       mmc_hostname(host->mmc), intmask);
3484 		sdhci_dumpregs(host);
3485 	}
3486 
3487 	return true;
3488 }
3489 EXPORT_SYMBOL_GPL(sdhci_cqe_irq);
3490 
3491 /*****************************************************************************\
3492  *                                                                           *
3493  * Device allocation/registration                                            *
3494  *                                                                           *
3495 \*****************************************************************************/
3496 
3497 struct sdhci_host *sdhci_alloc_host(struct device *dev,
3498 	size_t priv_size)
3499 {
3500 	struct mmc_host *mmc;
3501 	struct sdhci_host *host;
3502 
3503 	WARN_ON(dev == NULL);
3504 
3505 	mmc = mmc_alloc_host(sizeof(struct sdhci_host) + priv_size, dev);
3506 	if (!mmc)
3507 		return ERR_PTR(-ENOMEM);
3508 
3509 	host = mmc_priv(mmc);
3510 	host->mmc = mmc;
3511 	host->mmc_host_ops = sdhci_ops;
3512 	mmc->ops = &host->mmc_host_ops;
3513 
3514 	host->flags = SDHCI_SIGNALING_330;
3515 
3516 	host->cqe_ier     = SDHCI_CQE_INT_MASK;
3517 	host->cqe_err_ier = SDHCI_CQE_INT_ERR_MASK;
3518 
3519 	host->tuning_delay = -1;
3520 	host->tuning_loop_count = MAX_TUNING_LOOP;
3521 
3522 	host->sdma_boundary = SDHCI_DEFAULT_BOUNDARY_ARG;
3523 
3524 	/*
3525 	 * The DMA table descriptor count is calculated as the maximum
3526 	 * number of segments times 2, to allow for an alignment
3527 	 * descriptor for each segment, plus 1 for a nop end descriptor.
3528 	 */
3529 	host->adma_table_cnt = SDHCI_MAX_SEGS * 2 + 1;
3530 
3531 	return host;
3532 }
3533 
3534 EXPORT_SYMBOL_GPL(sdhci_alloc_host);
3535 
3536 static int sdhci_set_dma_mask(struct sdhci_host *host)
3537 {
3538 	struct mmc_host *mmc = host->mmc;
3539 	struct device *dev = mmc_dev(mmc);
3540 	int ret = -EINVAL;
3541 
3542 	if (host->quirks2 & SDHCI_QUIRK2_BROKEN_64_BIT_DMA)
3543 		host->flags &= ~SDHCI_USE_64_BIT_DMA;
3544 
3545 	/* Try 64-bit mask if hardware is capable  of it */
3546 	if (host->flags & SDHCI_USE_64_BIT_DMA) {
3547 		ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
3548 		if (ret) {
3549 			pr_warn("%s: Failed to set 64-bit DMA mask.\n",
3550 				mmc_hostname(mmc));
3551 			host->flags &= ~SDHCI_USE_64_BIT_DMA;
3552 		}
3553 	}
3554 
3555 	/* 32-bit mask as default & fallback */
3556 	if (ret) {
3557 		ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
3558 		if (ret)
3559 			pr_warn("%s: Failed to set 32-bit DMA mask.\n",
3560 				mmc_hostname(mmc));
3561 	}
3562 
3563 	return ret;
3564 }
3565 
3566 void __sdhci_read_caps(struct sdhci_host *host, u16 *ver, u32 *caps, u32 *caps1)
3567 {
3568 	u16 v;
3569 	u64 dt_caps_mask = 0;
3570 	u64 dt_caps = 0;
3571 
3572 	if (host->read_caps)
3573 		return;
3574 
3575 	host->read_caps = true;
3576 
3577 	if (debug_quirks)
3578 		host->quirks = debug_quirks;
3579 
3580 	if (debug_quirks2)
3581 		host->quirks2 = debug_quirks2;
3582 
3583 	sdhci_do_reset(host, SDHCI_RESET_ALL);
3584 
3585 	if (host->v4_mode)
3586 		sdhci_do_enable_v4_mode(host);
3587 
3588 	of_property_read_u64(mmc_dev(host->mmc)->of_node,
3589 			     "sdhci-caps-mask", &dt_caps_mask);
3590 	of_property_read_u64(mmc_dev(host->mmc)->of_node,
3591 			     "sdhci-caps", &dt_caps);
3592 
3593 	v = ver ? *ver : sdhci_readw(host, SDHCI_HOST_VERSION);
3594 	host->version = (v & SDHCI_SPEC_VER_MASK) >> SDHCI_SPEC_VER_SHIFT;
3595 
3596 	if (host->quirks & SDHCI_QUIRK_MISSING_CAPS)
3597 		return;
3598 
3599 	if (caps) {
3600 		host->caps = *caps;
3601 	} else {
3602 		host->caps = sdhci_readl(host, SDHCI_CAPABILITIES);
3603 		host->caps &= ~lower_32_bits(dt_caps_mask);
3604 		host->caps |= lower_32_bits(dt_caps);
3605 	}
3606 
3607 	if (host->version < SDHCI_SPEC_300)
3608 		return;
3609 
3610 	if (caps1) {
3611 		host->caps1 = *caps1;
3612 	} else {
3613 		host->caps1 = sdhci_readl(host, SDHCI_CAPABILITIES_1);
3614 		host->caps1 &= ~upper_32_bits(dt_caps_mask);
3615 		host->caps1 |= upper_32_bits(dt_caps);
3616 	}
3617 }
3618 EXPORT_SYMBOL_GPL(__sdhci_read_caps);
3619 
3620 static void sdhci_allocate_bounce_buffer(struct sdhci_host *host)
3621 {
3622 	struct mmc_host *mmc = host->mmc;
3623 	unsigned int max_blocks;
3624 	unsigned int bounce_size;
3625 	int ret;
3626 
3627 	/*
3628 	 * Cap the bounce buffer at 64KB. Using a bigger bounce buffer
3629 	 * has diminishing returns, this is probably because SD/MMC
3630 	 * cards are usually optimized to handle this size of requests.
3631 	 */
3632 	bounce_size = SZ_64K;
3633 	/*
3634 	 * Adjust downwards to maximum request size if this is less
3635 	 * than our segment size, else hammer down the maximum
3636 	 * request size to the maximum buffer size.
3637 	 */
3638 	if (mmc->max_req_size < bounce_size)
3639 		bounce_size = mmc->max_req_size;
3640 	max_blocks = bounce_size / 512;
3641 
3642 	/*
3643 	 * When we just support one segment, we can get significant
3644 	 * speedups by the help of a bounce buffer to group scattered
3645 	 * reads/writes together.
3646 	 */
3647 	host->bounce_buffer = devm_kmalloc(mmc->parent,
3648 					   bounce_size,
3649 					   GFP_KERNEL);
3650 	if (!host->bounce_buffer) {
3651 		pr_err("%s: failed to allocate %u bytes for bounce buffer, falling back to single segments\n",
3652 		       mmc_hostname(mmc),
3653 		       bounce_size);
3654 		/*
3655 		 * Exiting with zero here makes sure we proceed with
3656 		 * mmc->max_segs == 1.
3657 		 */
3658 		return;
3659 	}
3660 
3661 	host->bounce_addr = dma_map_single(mmc->parent,
3662 					   host->bounce_buffer,
3663 					   bounce_size,
3664 					   DMA_BIDIRECTIONAL);
3665 	ret = dma_mapping_error(mmc->parent, host->bounce_addr);
3666 	if (ret)
3667 		/* Again fall back to max_segs == 1 */
3668 		return;
3669 	host->bounce_buffer_size = bounce_size;
3670 
3671 	/* Lie about this since we're bouncing */
3672 	mmc->max_segs = max_blocks;
3673 	mmc->max_seg_size = bounce_size;
3674 	mmc->max_req_size = bounce_size;
3675 
3676 	pr_info("%s bounce up to %u segments into one, max segment size %u bytes\n",
3677 		mmc_hostname(mmc), max_blocks, bounce_size);
3678 }
3679 
3680 static inline bool sdhci_can_64bit_dma(struct sdhci_host *host)
3681 {
3682 	/*
3683 	 * According to SD Host Controller spec v4.10, bit[27] added from
3684 	 * version 4.10 in Capabilities Register is used as 64-bit System
3685 	 * Address support for V4 mode.
3686 	 */
3687 	if (host->version >= SDHCI_SPEC_410 && host->v4_mode)
3688 		return host->caps & SDHCI_CAN_64BIT_V4;
3689 
3690 	return host->caps & SDHCI_CAN_64BIT;
3691 }
3692 
3693 int sdhci_setup_host(struct sdhci_host *host)
3694 {
3695 	struct mmc_host *mmc;
3696 	u32 max_current_caps;
3697 	unsigned int ocr_avail;
3698 	unsigned int override_timeout_clk;
3699 	u32 max_clk;
3700 	int ret;
3701 
3702 	WARN_ON(host == NULL);
3703 	if (host == NULL)
3704 		return -EINVAL;
3705 
3706 	mmc = host->mmc;
3707 
3708 	/*
3709 	 * If there are external regulators, get them. Note this must be done
3710 	 * early before resetting the host and reading the capabilities so that
3711 	 * the host can take the appropriate action if regulators are not
3712 	 * available.
3713 	 */
3714 	ret = mmc_regulator_get_supply(mmc);
3715 	if (ret)
3716 		return ret;
3717 
3718 	DBG("Version:   0x%08x | Present:  0x%08x\n",
3719 	    sdhci_readw(host, SDHCI_HOST_VERSION),
3720 	    sdhci_readl(host, SDHCI_PRESENT_STATE));
3721 	DBG("Caps:      0x%08x | Caps_1:   0x%08x\n",
3722 	    sdhci_readl(host, SDHCI_CAPABILITIES),
3723 	    sdhci_readl(host, SDHCI_CAPABILITIES_1));
3724 
3725 	sdhci_read_caps(host);
3726 
3727 	override_timeout_clk = host->timeout_clk;
3728 
3729 	if (host->version > SDHCI_SPEC_420) {
3730 		pr_err("%s: Unknown controller version (%d). You may experience problems.\n",
3731 		       mmc_hostname(mmc), host->version);
3732 	}
3733 
3734 	if (host->quirks & SDHCI_QUIRK_FORCE_DMA)
3735 		host->flags |= SDHCI_USE_SDMA;
3736 	else if (!(host->caps & SDHCI_CAN_DO_SDMA))
3737 		DBG("Controller doesn't have SDMA capability\n");
3738 	else
3739 		host->flags |= SDHCI_USE_SDMA;
3740 
3741 	if ((host->quirks & SDHCI_QUIRK_BROKEN_DMA) &&
3742 		(host->flags & SDHCI_USE_SDMA)) {
3743 		DBG("Disabling DMA as it is marked broken\n");
3744 		host->flags &= ~SDHCI_USE_SDMA;
3745 	}
3746 
3747 	if ((host->version >= SDHCI_SPEC_200) &&
3748 		(host->caps & SDHCI_CAN_DO_ADMA2))
3749 		host->flags |= SDHCI_USE_ADMA;
3750 
3751 	if ((host->quirks & SDHCI_QUIRK_BROKEN_ADMA) &&
3752 		(host->flags & SDHCI_USE_ADMA)) {
3753 		DBG("Disabling ADMA as it is marked broken\n");
3754 		host->flags &= ~SDHCI_USE_ADMA;
3755 	}
3756 
3757 	/*
3758 	 * It is assumed that a 64-bit capable device has set a 64-bit DMA mask
3759 	 * and *must* do 64-bit DMA.  A driver has the opportunity to change
3760 	 * that during the first call to ->enable_dma().  Similarly
3761 	 * SDHCI_QUIRK2_BROKEN_64_BIT_DMA must be left to the drivers to
3762 	 * implement.
3763 	 */
3764 	if (sdhci_can_64bit_dma(host))
3765 		host->flags |= SDHCI_USE_64_BIT_DMA;
3766 
3767 	if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
3768 		ret = sdhci_set_dma_mask(host);
3769 
3770 		if (!ret && host->ops->enable_dma)
3771 			ret = host->ops->enable_dma(host);
3772 
3773 		if (ret) {
3774 			pr_warn("%s: No suitable DMA available - falling back to PIO\n",
3775 				mmc_hostname(mmc));
3776 			host->flags &= ~(SDHCI_USE_SDMA | SDHCI_USE_ADMA);
3777 
3778 			ret = 0;
3779 		}
3780 	}
3781 
3782 	/* SDMA does not support 64-bit DMA if v4 mode not set */
3783 	if ((host->flags & SDHCI_USE_64_BIT_DMA) && !host->v4_mode)
3784 		host->flags &= ~SDHCI_USE_SDMA;
3785 
3786 	if (host->flags & SDHCI_USE_ADMA) {
3787 		dma_addr_t dma;
3788 		void *buf;
3789 
3790 		if (host->flags & SDHCI_USE_64_BIT_DMA) {
3791 			host->adma_table_sz = host->adma_table_cnt *
3792 					      SDHCI_ADMA2_64_DESC_SZ(host);
3793 			host->desc_sz = SDHCI_ADMA2_64_DESC_SZ(host);
3794 		} else {
3795 			host->adma_table_sz = host->adma_table_cnt *
3796 					      SDHCI_ADMA2_32_DESC_SZ;
3797 			host->desc_sz = SDHCI_ADMA2_32_DESC_SZ;
3798 		}
3799 
3800 		host->align_buffer_sz = SDHCI_MAX_SEGS * SDHCI_ADMA2_ALIGN;
3801 		/*
3802 		 * Use zalloc to zero the reserved high 32-bits of 128-bit
3803 		 * descriptors so that they never need to be written.
3804 		 */
3805 		buf = dma_alloc_coherent(mmc_dev(mmc),
3806 					 host->align_buffer_sz + host->adma_table_sz,
3807 					 &dma, GFP_KERNEL);
3808 		if (!buf) {
3809 			pr_warn("%s: Unable to allocate ADMA buffers - falling back to standard DMA\n",
3810 				mmc_hostname(mmc));
3811 			host->flags &= ~SDHCI_USE_ADMA;
3812 		} else if ((dma + host->align_buffer_sz) &
3813 			   (SDHCI_ADMA2_DESC_ALIGN - 1)) {
3814 			pr_warn("%s: unable to allocate aligned ADMA descriptor\n",
3815 				mmc_hostname(mmc));
3816 			host->flags &= ~SDHCI_USE_ADMA;
3817 			dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
3818 					  host->adma_table_sz, buf, dma);
3819 		} else {
3820 			host->align_buffer = buf;
3821 			host->align_addr = dma;
3822 
3823 			host->adma_table = buf + host->align_buffer_sz;
3824 			host->adma_addr = dma + host->align_buffer_sz;
3825 		}
3826 	}
3827 
3828 	/*
3829 	 * If we use DMA, then it's up to the caller to set the DMA
3830 	 * mask, but PIO does not need the hw shim so we set a new
3831 	 * mask here in that case.
3832 	 */
3833 	if (!(host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA))) {
3834 		host->dma_mask = DMA_BIT_MASK(64);
3835 		mmc_dev(mmc)->dma_mask = &host->dma_mask;
3836 	}
3837 
3838 	if (host->version >= SDHCI_SPEC_300)
3839 		host->max_clk = (host->caps & SDHCI_CLOCK_V3_BASE_MASK)
3840 			>> SDHCI_CLOCK_BASE_SHIFT;
3841 	else
3842 		host->max_clk = (host->caps & SDHCI_CLOCK_BASE_MASK)
3843 			>> SDHCI_CLOCK_BASE_SHIFT;
3844 
3845 	host->max_clk *= 1000000;
3846 	if (host->max_clk == 0 || host->quirks &
3847 			SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN) {
3848 		if (!host->ops->get_max_clock) {
3849 			pr_err("%s: Hardware doesn't specify base clock frequency.\n",
3850 			       mmc_hostname(mmc));
3851 			ret = -ENODEV;
3852 			goto undma;
3853 		}
3854 		host->max_clk = host->ops->get_max_clock(host);
3855 	}
3856 
3857 	/*
3858 	 * In case of Host Controller v3.00, find out whether clock
3859 	 * multiplier is supported.
3860 	 */
3861 	host->clk_mul = (host->caps1 & SDHCI_CLOCK_MUL_MASK) >>
3862 			SDHCI_CLOCK_MUL_SHIFT;
3863 
3864 	/*
3865 	 * In case the value in Clock Multiplier is 0, then programmable
3866 	 * clock mode is not supported, otherwise the actual clock
3867 	 * multiplier is one more than the value of Clock Multiplier
3868 	 * in the Capabilities Register.
3869 	 */
3870 	if (host->clk_mul)
3871 		host->clk_mul += 1;
3872 
3873 	/*
3874 	 * Set host parameters.
3875 	 */
3876 	max_clk = host->max_clk;
3877 
3878 	if (host->ops->get_min_clock)
3879 		mmc->f_min = host->ops->get_min_clock(host);
3880 	else if (host->version >= SDHCI_SPEC_300) {
3881 		if (host->clk_mul) {
3882 			mmc->f_min = (host->max_clk * host->clk_mul) / 1024;
3883 			max_clk = host->max_clk * host->clk_mul;
3884 		} else
3885 			mmc->f_min = host->max_clk / SDHCI_MAX_DIV_SPEC_300;
3886 	} else
3887 		mmc->f_min = host->max_clk / SDHCI_MAX_DIV_SPEC_200;
3888 
3889 	if (!mmc->f_max || mmc->f_max > max_clk)
3890 		mmc->f_max = max_clk;
3891 
3892 	if (!(host->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK)) {
3893 		host->timeout_clk = (host->caps & SDHCI_TIMEOUT_CLK_MASK) >>
3894 					SDHCI_TIMEOUT_CLK_SHIFT;
3895 
3896 		if (host->caps & SDHCI_TIMEOUT_CLK_UNIT)
3897 			host->timeout_clk *= 1000;
3898 
3899 		if (host->timeout_clk == 0) {
3900 			if (!host->ops->get_timeout_clock) {
3901 				pr_err("%s: Hardware doesn't specify timeout clock frequency.\n",
3902 					mmc_hostname(mmc));
3903 				ret = -ENODEV;
3904 				goto undma;
3905 			}
3906 
3907 			host->timeout_clk =
3908 				DIV_ROUND_UP(host->ops->get_timeout_clock(host),
3909 					     1000);
3910 		}
3911 
3912 		if (override_timeout_clk)
3913 			host->timeout_clk = override_timeout_clk;
3914 
3915 		mmc->max_busy_timeout = host->ops->get_max_timeout_count ?
3916 			host->ops->get_max_timeout_count(host) : 1 << 27;
3917 		mmc->max_busy_timeout /= host->timeout_clk;
3918 	}
3919 
3920 	if (host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT &&
3921 	    !host->ops->get_max_timeout_count)
3922 		mmc->max_busy_timeout = 0;
3923 
3924 	mmc->caps |= MMC_CAP_SDIO_IRQ | MMC_CAP_ERASE | MMC_CAP_CMD23;
3925 	mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD;
3926 
3927 	if (host->quirks & SDHCI_QUIRK_MULTIBLOCK_READ_ACMD12)
3928 		host->flags |= SDHCI_AUTO_CMD12;
3929 
3930 	/*
3931 	 * For v3 mode, Auto-CMD23 stuff only works in ADMA or PIO.
3932 	 * For v4 mode, SDMA may use Auto-CMD23 as well.
3933 	 */
3934 	if ((host->version >= SDHCI_SPEC_300) &&
3935 	    ((host->flags & SDHCI_USE_ADMA) ||
3936 	     !(host->flags & SDHCI_USE_SDMA) || host->v4_mode) &&
3937 	     !(host->quirks2 & SDHCI_QUIRK2_ACMD23_BROKEN)) {
3938 		host->flags |= SDHCI_AUTO_CMD23;
3939 		DBG("Auto-CMD23 available\n");
3940 	} else {
3941 		DBG("Auto-CMD23 unavailable\n");
3942 	}
3943 
3944 	/*
3945 	 * A controller may support 8-bit width, but the board itself
3946 	 * might not have the pins brought out.  Boards that support
3947 	 * 8-bit width must set "mmc->caps |= MMC_CAP_8_BIT_DATA;" in
3948 	 * their platform code before calling sdhci_add_host(), and we
3949 	 * won't assume 8-bit width for hosts without that CAP.
3950 	 */
3951 	if (!(host->quirks & SDHCI_QUIRK_FORCE_1_BIT_DATA))
3952 		mmc->caps |= MMC_CAP_4_BIT_DATA;
3953 
3954 	if (host->quirks2 & SDHCI_QUIRK2_HOST_NO_CMD23)
3955 		mmc->caps &= ~MMC_CAP_CMD23;
3956 
3957 	if (host->caps & SDHCI_CAN_DO_HISPD)
3958 		mmc->caps |= MMC_CAP_SD_HIGHSPEED | MMC_CAP_MMC_HIGHSPEED;
3959 
3960 	if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) &&
3961 	    mmc_card_is_removable(mmc) &&
3962 	    mmc_gpio_get_cd(host->mmc) < 0)
3963 		mmc->caps |= MMC_CAP_NEEDS_POLL;
3964 
3965 	if (!IS_ERR(mmc->supply.vqmmc)) {
3966 		ret = regulator_enable(mmc->supply.vqmmc);
3967 
3968 		/* If vqmmc provides no 1.8V signalling, then there's no UHS */
3969 		if (!regulator_is_supported_voltage(mmc->supply.vqmmc, 1700000,
3970 						    1950000))
3971 			host->caps1 &= ~(SDHCI_SUPPORT_SDR104 |
3972 					 SDHCI_SUPPORT_SDR50 |
3973 					 SDHCI_SUPPORT_DDR50);
3974 
3975 		/* In eMMC case vqmmc might be a fixed 1.8V regulator */
3976 		if (!regulator_is_supported_voltage(mmc->supply.vqmmc, 2700000,
3977 						    3600000))
3978 			host->flags &= ~SDHCI_SIGNALING_330;
3979 
3980 		if (ret) {
3981 			pr_warn("%s: Failed to enable vqmmc regulator: %d\n",
3982 				mmc_hostname(mmc), ret);
3983 			mmc->supply.vqmmc = ERR_PTR(-EINVAL);
3984 		}
3985 	}
3986 
3987 	if (host->quirks2 & SDHCI_QUIRK2_NO_1_8_V) {
3988 		host->caps1 &= ~(SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 |
3989 				 SDHCI_SUPPORT_DDR50);
3990 		/*
3991 		 * The SDHCI controller in a SoC might support HS200/HS400
3992 		 * (indicated using mmc-hs200-1_8v/mmc-hs400-1_8v dt property),
3993 		 * but if the board is modeled such that the IO lines are not
3994 		 * connected to 1.8v then HS200/HS400 cannot be supported.
3995 		 * Disable HS200/HS400 if the board does not have 1.8v connected
3996 		 * to the IO lines. (Applicable for other modes in 1.8v)
3997 		 */
3998 		mmc->caps2 &= ~(MMC_CAP2_HSX00_1_8V | MMC_CAP2_HS400_ES);
3999 		mmc->caps &= ~(MMC_CAP_1_8V_DDR | MMC_CAP_UHS);
4000 	}
4001 
4002 	/* Any UHS-I mode in caps implies SDR12 and SDR25 support. */
4003 	if (host->caps1 & (SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 |
4004 			   SDHCI_SUPPORT_DDR50))
4005 		mmc->caps |= MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25;
4006 
4007 	/* SDR104 supports also implies SDR50 support */
4008 	if (host->caps1 & SDHCI_SUPPORT_SDR104) {
4009 		mmc->caps |= MMC_CAP_UHS_SDR104 | MMC_CAP_UHS_SDR50;
4010 		/* SD3.0: SDR104 is supported so (for eMMC) the caps2
4011 		 * field can be promoted to support HS200.
4012 		 */
4013 		if (!(host->quirks2 & SDHCI_QUIRK2_BROKEN_HS200))
4014 			mmc->caps2 |= MMC_CAP2_HS200;
4015 	} else if (host->caps1 & SDHCI_SUPPORT_SDR50) {
4016 		mmc->caps |= MMC_CAP_UHS_SDR50;
4017 	}
4018 
4019 	if (host->quirks2 & SDHCI_QUIRK2_CAPS_BIT63_FOR_HS400 &&
4020 	    (host->caps1 & SDHCI_SUPPORT_HS400))
4021 		mmc->caps2 |= MMC_CAP2_HS400;
4022 
4023 	if ((mmc->caps2 & MMC_CAP2_HSX00_1_2V) &&
4024 	    (IS_ERR(mmc->supply.vqmmc) ||
4025 	     !regulator_is_supported_voltage(mmc->supply.vqmmc, 1100000,
4026 					     1300000)))
4027 		mmc->caps2 &= ~MMC_CAP2_HSX00_1_2V;
4028 
4029 	if ((host->caps1 & SDHCI_SUPPORT_DDR50) &&
4030 	    !(host->quirks2 & SDHCI_QUIRK2_BROKEN_DDR50))
4031 		mmc->caps |= MMC_CAP_UHS_DDR50;
4032 
4033 	/* Does the host need tuning for SDR50? */
4034 	if (host->caps1 & SDHCI_USE_SDR50_TUNING)
4035 		host->flags |= SDHCI_SDR50_NEEDS_TUNING;
4036 
4037 	/* Driver Type(s) (A, C, D) supported by the host */
4038 	if (host->caps1 & SDHCI_DRIVER_TYPE_A)
4039 		mmc->caps |= MMC_CAP_DRIVER_TYPE_A;
4040 	if (host->caps1 & SDHCI_DRIVER_TYPE_C)
4041 		mmc->caps |= MMC_CAP_DRIVER_TYPE_C;
4042 	if (host->caps1 & SDHCI_DRIVER_TYPE_D)
4043 		mmc->caps |= MMC_CAP_DRIVER_TYPE_D;
4044 
4045 	/* Initial value for re-tuning timer count */
4046 	host->tuning_count = (host->caps1 & SDHCI_RETUNING_TIMER_COUNT_MASK) >>
4047 			     SDHCI_RETUNING_TIMER_COUNT_SHIFT;
4048 
4049 	/*
4050 	 * In case Re-tuning Timer is not disabled, the actual value of
4051 	 * re-tuning timer will be 2 ^ (n - 1).
4052 	 */
4053 	if (host->tuning_count)
4054 		host->tuning_count = 1 << (host->tuning_count - 1);
4055 
4056 	/* Re-tuning mode supported by the Host Controller */
4057 	host->tuning_mode = (host->caps1 & SDHCI_RETUNING_MODE_MASK) >>
4058 			     SDHCI_RETUNING_MODE_SHIFT;
4059 
4060 	ocr_avail = 0;
4061 
4062 	/*
4063 	 * According to SD Host Controller spec v3.00, if the Host System
4064 	 * can afford more than 150mA, Host Driver should set XPC to 1. Also
4065 	 * the value is meaningful only if Voltage Support in the Capabilities
4066 	 * register is set. The actual current value is 4 times the register
4067 	 * value.
4068 	 */
4069 	max_current_caps = sdhci_readl(host, SDHCI_MAX_CURRENT);
4070 	if (!max_current_caps && !IS_ERR(mmc->supply.vmmc)) {
4071 		int curr = regulator_get_current_limit(mmc->supply.vmmc);
4072 		if (curr > 0) {
4073 
4074 			/* convert to SDHCI_MAX_CURRENT format */
4075 			curr = curr/1000;  /* convert to mA */
4076 			curr = curr/SDHCI_MAX_CURRENT_MULTIPLIER;
4077 
4078 			curr = min_t(u32, curr, SDHCI_MAX_CURRENT_LIMIT);
4079 			max_current_caps =
4080 				(curr << SDHCI_MAX_CURRENT_330_SHIFT) |
4081 				(curr << SDHCI_MAX_CURRENT_300_SHIFT) |
4082 				(curr << SDHCI_MAX_CURRENT_180_SHIFT);
4083 		}
4084 	}
4085 
4086 	if (host->caps & SDHCI_CAN_VDD_330) {
4087 		ocr_avail |= MMC_VDD_32_33 | MMC_VDD_33_34;
4088 
4089 		mmc->max_current_330 = ((max_current_caps &
4090 				   SDHCI_MAX_CURRENT_330_MASK) >>
4091 				   SDHCI_MAX_CURRENT_330_SHIFT) *
4092 				   SDHCI_MAX_CURRENT_MULTIPLIER;
4093 	}
4094 	if (host->caps & SDHCI_CAN_VDD_300) {
4095 		ocr_avail |= MMC_VDD_29_30 | MMC_VDD_30_31;
4096 
4097 		mmc->max_current_300 = ((max_current_caps &
4098 				   SDHCI_MAX_CURRENT_300_MASK) >>
4099 				   SDHCI_MAX_CURRENT_300_SHIFT) *
4100 				   SDHCI_MAX_CURRENT_MULTIPLIER;
4101 	}
4102 	if (host->caps & SDHCI_CAN_VDD_180) {
4103 		ocr_avail |= MMC_VDD_165_195;
4104 
4105 		mmc->max_current_180 = ((max_current_caps &
4106 				   SDHCI_MAX_CURRENT_180_MASK) >>
4107 				   SDHCI_MAX_CURRENT_180_SHIFT) *
4108 				   SDHCI_MAX_CURRENT_MULTIPLIER;
4109 	}
4110 
4111 	/* If OCR set by host, use it instead. */
4112 	if (host->ocr_mask)
4113 		ocr_avail = host->ocr_mask;
4114 
4115 	/* If OCR set by external regulators, give it highest prio. */
4116 	if (mmc->ocr_avail)
4117 		ocr_avail = mmc->ocr_avail;
4118 
4119 	mmc->ocr_avail = ocr_avail;
4120 	mmc->ocr_avail_sdio = ocr_avail;
4121 	if (host->ocr_avail_sdio)
4122 		mmc->ocr_avail_sdio &= host->ocr_avail_sdio;
4123 	mmc->ocr_avail_sd = ocr_avail;
4124 	if (host->ocr_avail_sd)
4125 		mmc->ocr_avail_sd &= host->ocr_avail_sd;
4126 	else /* normal SD controllers don't support 1.8V */
4127 		mmc->ocr_avail_sd &= ~MMC_VDD_165_195;
4128 	mmc->ocr_avail_mmc = ocr_avail;
4129 	if (host->ocr_avail_mmc)
4130 		mmc->ocr_avail_mmc &= host->ocr_avail_mmc;
4131 
4132 	if (mmc->ocr_avail == 0) {
4133 		pr_err("%s: Hardware doesn't report any support voltages.\n",
4134 		       mmc_hostname(mmc));
4135 		ret = -ENODEV;
4136 		goto unreg;
4137 	}
4138 
4139 	if ((mmc->caps & (MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25 |
4140 			  MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_SDR104 |
4141 			  MMC_CAP_UHS_DDR50 | MMC_CAP_1_8V_DDR)) ||
4142 	    (mmc->caps2 & (MMC_CAP2_HS200_1_8V_SDR | MMC_CAP2_HS400_1_8V)))
4143 		host->flags |= SDHCI_SIGNALING_180;
4144 
4145 	if (mmc->caps2 & MMC_CAP2_HSX00_1_2V)
4146 		host->flags |= SDHCI_SIGNALING_120;
4147 
4148 	spin_lock_init(&host->lock);
4149 
4150 	/*
4151 	 * Maximum number of sectors in one transfer. Limited by SDMA boundary
4152 	 * size (512KiB). Note some tuning modes impose a 4MiB limit, but this
4153 	 * is less anyway.
4154 	 */
4155 	mmc->max_req_size = 524288;
4156 
4157 	/*
4158 	 * Maximum number of segments. Depends on if the hardware
4159 	 * can do scatter/gather or not.
4160 	 */
4161 	if (host->flags & SDHCI_USE_ADMA) {
4162 		mmc->max_segs = SDHCI_MAX_SEGS;
4163 	} else if (host->flags & SDHCI_USE_SDMA) {
4164 		mmc->max_segs = 1;
4165 		if (swiotlb_max_segment()) {
4166 			unsigned int max_req_size = (1 << IO_TLB_SHIFT) *
4167 						IO_TLB_SEGSIZE;
4168 			mmc->max_req_size = min(mmc->max_req_size,
4169 						max_req_size);
4170 		}
4171 	} else { /* PIO */
4172 		mmc->max_segs = SDHCI_MAX_SEGS;
4173 	}
4174 
4175 	/*
4176 	 * Maximum segment size. Could be one segment with the maximum number
4177 	 * of bytes. When doing hardware scatter/gather, each entry cannot
4178 	 * be larger than 64 KiB though.
4179 	 */
4180 	if (host->flags & SDHCI_USE_ADMA) {
4181 		if (host->quirks & SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC)
4182 			mmc->max_seg_size = 65535;
4183 		else
4184 			mmc->max_seg_size = 65536;
4185 	} else {
4186 		mmc->max_seg_size = mmc->max_req_size;
4187 	}
4188 
4189 	/*
4190 	 * Maximum block size. This varies from controller to controller and
4191 	 * is specified in the capabilities register.
4192 	 */
4193 	if (host->quirks & SDHCI_QUIRK_FORCE_BLK_SZ_2048) {
4194 		mmc->max_blk_size = 2;
4195 	} else {
4196 		mmc->max_blk_size = (host->caps & SDHCI_MAX_BLOCK_MASK) >>
4197 				SDHCI_MAX_BLOCK_SHIFT;
4198 		if (mmc->max_blk_size >= 3) {
4199 			pr_warn("%s: Invalid maximum block size, assuming 512 bytes\n",
4200 				mmc_hostname(mmc));
4201 			mmc->max_blk_size = 0;
4202 		}
4203 	}
4204 
4205 	mmc->max_blk_size = 512 << mmc->max_blk_size;
4206 
4207 	/*
4208 	 * Maximum block count.
4209 	 */
4210 	mmc->max_blk_count = (host->quirks & SDHCI_QUIRK_NO_MULTIBLOCK) ? 1 : 65535;
4211 
4212 	if (mmc->max_segs == 1)
4213 		/* This may alter mmc->*_blk_* parameters */
4214 		sdhci_allocate_bounce_buffer(host);
4215 
4216 	return 0;
4217 
4218 unreg:
4219 	if (!IS_ERR(mmc->supply.vqmmc))
4220 		regulator_disable(mmc->supply.vqmmc);
4221 undma:
4222 	if (host->align_buffer)
4223 		dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
4224 				  host->adma_table_sz, host->align_buffer,
4225 				  host->align_addr);
4226 	host->adma_table = NULL;
4227 	host->align_buffer = NULL;
4228 
4229 	return ret;
4230 }
4231 EXPORT_SYMBOL_GPL(sdhci_setup_host);
4232 
4233 void sdhci_cleanup_host(struct sdhci_host *host)
4234 {
4235 	struct mmc_host *mmc = host->mmc;
4236 
4237 	if (!IS_ERR(mmc->supply.vqmmc))
4238 		regulator_disable(mmc->supply.vqmmc);
4239 
4240 	if (host->align_buffer)
4241 		dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
4242 				  host->adma_table_sz, host->align_buffer,
4243 				  host->align_addr);
4244 	host->adma_table = NULL;
4245 	host->align_buffer = NULL;
4246 }
4247 EXPORT_SYMBOL_GPL(sdhci_cleanup_host);
4248 
4249 int __sdhci_add_host(struct sdhci_host *host)
4250 {
4251 	unsigned int flags = WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_HIGHPRI;
4252 	struct mmc_host *mmc = host->mmc;
4253 	int ret;
4254 
4255 	host->complete_wq = alloc_workqueue("sdhci", flags, 0);
4256 	if (!host->complete_wq)
4257 		return -ENOMEM;
4258 
4259 	INIT_WORK(&host->complete_work, sdhci_complete_work);
4260 
4261 	timer_setup(&host->timer, sdhci_timeout_timer, 0);
4262 	timer_setup(&host->data_timer, sdhci_timeout_data_timer, 0);
4263 
4264 	init_waitqueue_head(&host->buf_ready_int);
4265 
4266 	sdhci_init(host, 0);
4267 
4268 	ret = request_threaded_irq(host->irq, sdhci_irq, sdhci_thread_irq,
4269 				   IRQF_SHARED,	mmc_hostname(mmc), host);
4270 	if (ret) {
4271 		pr_err("%s: Failed to request IRQ %d: %d\n",
4272 		       mmc_hostname(mmc), host->irq, ret);
4273 		goto unwq;
4274 	}
4275 
4276 	ret = sdhci_led_register(host);
4277 	if (ret) {
4278 		pr_err("%s: Failed to register LED device: %d\n",
4279 		       mmc_hostname(mmc), ret);
4280 		goto unirq;
4281 	}
4282 
4283 	ret = mmc_add_host(mmc);
4284 	if (ret)
4285 		goto unled;
4286 
4287 	pr_info("%s: SDHCI controller on %s [%s] using %s\n",
4288 		mmc_hostname(mmc), host->hw_name, dev_name(mmc_dev(mmc)),
4289 		(host->flags & SDHCI_USE_ADMA) ?
4290 		(host->flags & SDHCI_USE_64_BIT_DMA) ? "ADMA 64-bit" : "ADMA" :
4291 		(host->flags & SDHCI_USE_SDMA) ? "DMA" : "PIO");
4292 
4293 	sdhci_enable_card_detection(host);
4294 
4295 	return 0;
4296 
4297 unled:
4298 	sdhci_led_unregister(host);
4299 unirq:
4300 	sdhci_do_reset(host, SDHCI_RESET_ALL);
4301 	sdhci_writel(host, 0, SDHCI_INT_ENABLE);
4302 	sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
4303 	free_irq(host->irq, host);
4304 unwq:
4305 	destroy_workqueue(host->complete_wq);
4306 
4307 	return ret;
4308 }
4309 EXPORT_SYMBOL_GPL(__sdhci_add_host);
4310 
4311 int sdhci_add_host(struct sdhci_host *host)
4312 {
4313 	int ret;
4314 
4315 	ret = sdhci_setup_host(host);
4316 	if (ret)
4317 		return ret;
4318 
4319 	ret = __sdhci_add_host(host);
4320 	if (ret)
4321 		goto cleanup;
4322 
4323 	return 0;
4324 
4325 cleanup:
4326 	sdhci_cleanup_host(host);
4327 
4328 	return ret;
4329 }
4330 EXPORT_SYMBOL_GPL(sdhci_add_host);
4331 
4332 void sdhci_remove_host(struct sdhci_host *host, int dead)
4333 {
4334 	struct mmc_host *mmc = host->mmc;
4335 	unsigned long flags;
4336 
4337 	if (dead) {
4338 		spin_lock_irqsave(&host->lock, flags);
4339 
4340 		host->flags |= SDHCI_DEVICE_DEAD;
4341 
4342 		if (sdhci_has_requests(host)) {
4343 			pr_err("%s: Controller removed during "
4344 				" transfer!\n", mmc_hostname(mmc));
4345 			sdhci_error_out_mrqs(host, -ENOMEDIUM);
4346 		}
4347 
4348 		spin_unlock_irqrestore(&host->lock, flags);
4349 	}
4350 
4351 	sdhci_disable_card_detection(host);
4352 
4353 	mmc_remove_host(mmc);
4354 
4355 	sdhci_led_unregister(host);
4356 
4357 	if (!dead)
4358 		sdhci_do_reset(host, SDHCI_RESET_ALL);
4359 
4360 	sdhci_writel(host, 0, SDHCI_INT_ENABLE);
4361 	sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
4362 	free_irq(host->irq, host);
4363 
4364 	del_timer_sync(&host->timer);
4365 	del_timer_sync(&host->data_timer);
4366 
4367 	destroy_workqueue(host->complete_wq);
4368 
4369 	if (!IS_ERR(mmc->supply.vqmmc))
4370 		regulator_disable(mmc->supply.vqmmc);
4371 
4372 	if (host->align_buffer)
4373 		dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
4374 				  host->adma_table_sz, host->align_buffer,
4375 				  host->align_addr);
4376 
4377 	host->adma_table = NULL;
4378 	host->align_buffer = NULL;
4379 }
4380 
4381 EXPORT_SYMBOL_GPL(sdhci_remove_host);
4382 
4383 void sdhci_free_host(struct sdhci_host *host)
4384 {
4385 	mmc_free_host(host->mmc);
4386 }
4387 
4388 EXPORT_SYMBOL_GPL(sdhci_free_host);
4389 
4390 /*****************************************************************************\
4391  *                                                                           *
4392  * Driver init/exit                                                          *
4393  *                                                                           *
4394 \*****************************************************************************/
4395 
4396 static int __init sdhci_drv_init(void)
4397 {
4398 	pr_info(DRIVER_NAME
4399 		": Secure Digital Host Controller Interface driver\n");
4400 	pr_info(DRIVER_NAME ": Copyright(c) Pierre Ossman\n");
4401 
4402 	return 0;
4403 }
4404 
4405 static void __exit sdhci_drv_exit(void)
4406 {
4407 }
4408 
4409 module_init(sdhci_drv_init);
4410 module_exit(sdhci_drv_exit);
4411 
4412 module_param(debug_quirks, uint, 0444);
4413 module_param(debug_quirks2, uint, 0444);
4414 
4415 MODULE_AUTHOR("Pierre Ossman <pierre@ossman.eu>");
4416 MODULE_DESCRIPTION("Secure Digital Host Controller Interface core driver");
4417 MODULE_LICENSE("GPL");
4418 
4419 MODULE_PARM_DESC(debug_quirks, "Force certain quirks.");
4420 MODULE_PARM_DESC(debug_quirks2, "Force certain other quirks.");
4421