1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * 8250-core based driver for the OMAP internal UART
4  *
5  * based on omap-serial.c, Copyright (C) 2010 Texas Instruments.
6  *
7  * Copyright (C) 2014 Sebastian Andrzej Siewior
8  *
9  */
10 
11 #include <linux/clk.h>
12 #include <linux/device.h>
13 #include <linux/io.h>
14 #include <linux/module.h>
15 #include <linux/serial_8250.h>
16 #include <linux/serial_reg.h>
17 #include <linux/tty_flip.h>
18 #include <linux/platform_device.h>
19 #include <linux/slab.h>
20 #include <linux/of.h>
21 #include <linux/of_device.h>
22 #include <linux/of_gpio.h>
23 #include <linux/of_irq.h>
24 #include <linux/delay.h>
25 #include <linux/pm_runtime.h>
26 #include <linux/console.h>
27 #include <linux/pm_qos.h>
28 #include <linux/pm_wakeirq.h>
29 #include <linux/dma-mapping.h>
30 #include <linux/sys_soc.h>
31 
32 #include "8250.h"
33 
34 #define DEFAULT_CLK_SPEED	48000000
35 
36 #define UART_ERRATA_i202_MDR1_ACCESS	(1 << 0)
37 #define OMAP_UART_WER_HAS_TX_WAKEUP	(1 << 1)
38 #define OMAP_DMA_TX_KICK		(1 << 2)
39 /*
40  * See Advisory 21 in AM437x errata SPRZ408B, updated April 2015.
41  * The same errata is applicable to AM335x and DRA7x processors too.
42  */
43 #define UART_ERRATA_CLOCK_DISABLE	(1 << 3)
44 #define	UART_HAS_EFR2			BIT(4)
45 #define UART_HAS_RHR_IT_DIS		BIT(5)
46 
47 #define OMAP_UART_FCR_RX_TRIG		6
48 #define OMAP_UART_FCR_TX_TRIG		4
49 
50 /* SCR register bitmasks */
51 #define OMAP_UART_SCR_RX_TRIG_GRANU1_MASK	(1 << 7)
52 #define OMAP_UART_SCR_TX_TRIG_GRANU1_MASK	(1 << 6)
53 #define OMAP_UART_SCR_TX_EMPTY			(1 << 3)
54 #define OMAP_UART_SCR_DMAMODE_MASK		(3 << 1)
55 #define OMAP_UART_SCR_DMAMODE_1			(1 << 1)
56 #define OMAP_UART_SCR_DMAMODE_CTL		(1 << 0)
57 
58 /* MVR register bitmasks */
59 #define OMAP_UART_MVR_SCHEME_SHIFT	30
60 #define OMAP_UART_LEGACY_MVR_MAJ_MASK	0xf0
61 #define OMAP_UART_LEGACY_MVR_MAJ_SHIFT	4
62 #define OMAP_UART_LEGACY_MVR_MIN_MASK	0x0f
63 #define OMAP_UART_MVR_MAJ_MASK		0x700
64 #define OMAP_UART_MVR_MAJ_SHIFT		8
65 #define OMAP_UART_MVR_MIN_MASK		0x3f
66 
67 /* SYSC register bitmasks */
68 #define OMAP_UART_SYSC_SOFTRESET	(1 << 1)
69 
70 /* SYSS register bitmasks */
71 #define OMAP_UART_SYSS_RESETDONE	(1 << 0)
72 
73 #define UART_TI752_TLR_TX	0
74 #define UART_TI752_TLR_RX	4
75 
76 #define TRIGGER_TLR_MASK(x)	((x & 0x3c) >> 2)
77 #define TRIGGER_FCR_MASK(x)	(x & 3)
78 
79 /* Enable XON/XOFF flow control on output */
80 #define OMAP_UART_SW_TX		0x08
81 /* Enable XON/XOFF flow control on input */
82 #define OMAP_UART_SW_RX		0x02
83 
84 #define OMAP_UART_WER_MOD_WKUP	0x7f
85 #define OMAP_UART_TX_WAKEUP_EN	(1 << 7)
86 
87 #define TX_TRIGGER	1
88 #define RX_TRIGGER	48
89 
90 #define OMAP_UART_TCR_RESTORE(x)	((x / 4) << 4)
91 #define OMAP_UART_TCR_HALT(x)		((x / 4) << 0)
92 
93 #define UART_BUILD_REVISION(x, y)	(((x) << 8) | (y))
94 
95 #define OMAP_UART_REV_46 0x0406
96 #define OMAP_UART_REV_52 0x0502
97 #define OMAP_UART_REV_63 0x0603
98 
99 /* Interrupt Enable Register 2 */
100 #define UART_OMAP_IER2			0x1B
101 #define UART_OMAP_IER2_RHR_IT_DIS	BIT(2)
102 
103 /* Enhanced features register 2 */
104 #define UART_OMAP_EFR2			0x23
105 #define UART_OMAP_EFR2_TIMEOUT_BEHAVE	BIT(6)
106 
107 struct omap8250_priv {
108 	int line;
109 	u8 habit;
110 	u8 mdr1;
111 	u8 efr;
112 	u8 scr;
113 	u8 wer;
114 	u8 xon;
115 	u8 xoff;
116 	u8 delayed_restore;
117 	u16 quot;
118 
119 	u8 tx_trigger;
120 	u8 rx_trigger;
121 	bool is_suspending;
122 	int wakeirq;
123 	int wakeups_enabled;
124 	u32 latency;
125 	u32 calc_latency;
126 	struct pm_qos_request pm_qos_request;
127 	struct work_struct qos_work;
128 	struct uart_8250_dma omap8250_dma;
129 	spinlock_t rx_dma_lock;
130 	bool rx_dma_broken;
131 	bool throttled;
132 };
133 
134 struct omap8250_dma_params {
135 	u32 rx_size;
136 	u8 rx_trigger;
137 	u8 tx_trigger;
138 };
139 
140 struct omap8250_platdata {
141 	struct omap8250_dma_params *dma_params;
142 	u8 habit;
143 };
144 
145 #ifdef CONFIG_SERIAL_8250_DMA
146 static void omap_8250_rx_dma_flush(struct uart_8250_port *p);
147 #else
148 static inline void omap_8250_rx_dma_flush(struct uart_8250_port *p) { }
149 #endif
150 
151 static u32 uart_read(struct uart_8250_port *up, u32 reg)
152 {
153 	return readl(up->port.membase + (reg << up->port.regshift));
154 }
155 
156 static void omap8250_set_mctrl(struct uart_port *port, unsigned int mctrl)
157 {
158 	struct uart_8250_port *up = up_to_u8250p(port);
159 	struct omap8250_priv *priv = up->port.private_data;
160 	u8 lcr;
161 
162 	serial8250_do_set_mctrl(port, mctrl);
163 
164 	if (!mctrl_gpio_to_gpiod(up->gpios, UART_GPIO_RTS)) {
165 		/*
166 		 * Turn off autoRTS if RTS is lowered and restore autoRTS
167 		 * setting if RTS is raised
168 		 */
169 		lcr = serial_in(up, UART_LCR);
170 		serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
171 		if ((mctrl & TIOCM_RTS) && (port->status & UPSTAT_AUTORTS))
172 			priv->efr |= UART_EFR_RTS;
173 		else
174 			priv->efr &= ~UART_EFR_RTS;
175 		serial_out(up, UART_EFR, priv->efr);
176 		serial_out(up, UART_LCR, lcr);
177 	}
178 }
179 
180 /*
181  * Work Around for Errata i202 (2430, 3430, 3630, 4430 and 4460)
182  * The access to uart register after MDR1 Access
183  * causes UART to corrupt data.
184  *
185  * Need a delay =
186  * 5 L4 clock cycles + 5 UART functional clock cycle (@48MHz = ~0.2uS)
187  * give 10 times as much
188  */
189 static void omap_8250_mdr1_errataset(struct uart_8250_port *up,
190 				     struct omap8250_priv *priv)
191 {
192 	u8 timeout = 255;
193 
194 	serial_out(up, UART_OMAP_MDR1, priv->mdr1);
195 	udelay(2);
196 	serial_out(up, UART_FCR, up->fcr | UART_FCR_CLEAR_XMIT |
197 			UART_FCR_CLEAR_RCVR);
198 	/*
199 	 * Wait for FIFO to empty: when empty, RX_FIFO_E bit is 0 and
200 	 * TX_FIFO_E bit is 1.
201 	 */
202 	while (UART_LSR_THRE != (serial_in(up, UART_LSR) &
203 				(UART_LSR_THRE | UART_LSR_DR))) {
204 		timeout--;
205 		if (!timeout) {
206 			/* Should *never* happen. we warn and carry on */
207 			dev_crit(up->port.dev, "Errata i202: timedout %x\n",
208 				 serial_in(up, UART_LSR));
209 			break;
210 		}
211 		udelay(1);
212 	}
213 }
214 
215 static void omap_8250_get_divisor(struct uart_port *port, unsigned int baud,
216 				  struct omap8250_priv *priv)
217 {
218 	unsigned int uartclk = port->uartclk;
219 	unsigned int div_13, div_16;
220 	unsigned int abs_d13, abs_d16;
221 
222 	/*
223 	 * Old custom speed handling.
224 	 */
225 	if (baud == 38400 && (port->flags & UPF_SPD_MASK) == UPF_SPD_CUST) {
226 		priv->quot = port->custom_divisor & UART_DIV_MAX;
227 		/*
228 		 * I assume that nobody is using this. But hey, if somebody
229 		 * would like to specify the divisor _and_ the mode then the
230 		 * driver is ready and waiting for it.
231 		 */
232 		if (port->custom_divisor & (1 << 16))
233 			priv->mdr1 = UART_OMAP_MDR1_13X_MODE;
234 		else
235 			priv->mdr1 = UART_OMAP_MDR1_16X_MODE;
236 		return;
237 	}
238 	div_13 = DIV_ROUND_CLOSEST(uartclk, 13 * baud);
239 	div_16 = DIV_ROUND_CLOSEST(uartclk, 16 * baud);
240 
241 	if (!div_13)
242 		div_13 = 1;
243 	if (!div_16)
244 		div_16 = 1;
245 
246 	abs_d13 = abs(baud - uartclk / 13 / div_13);
247 	abs_d16 = abs(baud - uartclk / 16 / div_16);
248 
249 	if (abs_d13 >= abs_d16) {
250 		priv->mdr1 = UART_OMAP_MDR1_16X_MODE;
251 		priv->quot = div_16;
252 	} else {
253 		priv->mdr1 = UART_OMAP_MDR1_13X_MODE;
254 		priv->quot = div_13;
255 	}
256 }
257 
258 static void omap8250_update_scr(struct uart_8250_port *up,
259 				struct omap8250_priv *priv)
260 {
261 	u8 old_scr;
262 
263 	old_scr = serial_in(up, UART_OMAP_SCR);
264 	if (old_scr == priv->scr)
265 		return;
266 
267 	/*
268 	 * The manual recommends not to enable the DMA mode selector in the SCR
269 	 * (instead of the FCR) register _and_ selecting the DMA mode as one
270 	 * register write because this may lead to malfunction.
271 	 */
272 	if (priv->scr & OMAP_UART_SCR_DMAMODE_MASK)
273 		serial_out(up, UART_OMAP_SCR,
274 			   priv->scr & ~OMAP_UART_SCR_DMAMODE_MASK);
275 	serial_out(up, UART_OMAP_SCR, priv->scr);
276 }
277 
278 static void omap8250_update_mdr1(struct uart_8250_port *up,
279 				 struct omap8250_priv *priv)
280 {
281 	if (priv->habit & UART_ERRATA_i202_MDR1_ACCESS)
282 		omap_8250_mdr1_errataset(up, priv);
283 	else
284 		serial_out(up, UART_OMAP_MDR1, priv->mdr1);
285 }
286 
287 static void omap8250_restore_regs(struct uart_8250_port *up)
288 {
289 	struct omap8250_priv *priv = up->port.private_data;
290 	struct uart_8250_dma	*dma = up->dma;
291 
292 	if (dma && dma->tx_running) {
293 		/*
294 		 * TCSANOW requests the change to occur immediately however if
295 		 * we have a TX-DMA operation in progress then it has been
296 		 * observed that it might stall and never complete. Therefore we
297 		 * delay DMA completes to prevent this hang from happen.
298 		 */
299 		priv->delayed_restore = 1;
300 		return;
301 	}
302 
303 	serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
304 	serial_out(up, UART_EFR, UART_EFR_ECB);
305 
306 	serial_out(up, UART_LCR, UART_LCR_CONF_MODE_A);
307 	serial8250_out_MCR(up, UART_MCR_TCRTLR);
308 	serial_out(up, UART_FCR, up->fcr);
309 
310 	omap8250_update_scr(up, priv);
311 
312 	serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
313 
314 	serial_out(up, UART_TI752_TCR, OMAP_UART_TCR_RESTORE(16) |
315 			OMAP_UART_TCR_HALT(52));
316 	serial_out(up, UART_TI752_TLR,
317 		   TRIGGER_TLR_MASK(priv->tx_trigger) << UART_TI752_TLR_TX |
318 		   TRIGGER_TLR_MASK(priv->rx_trigger) << UART_TI752_TLR_RX);
319 
320 	serial_out(up, UART_LCR, 0);
321 
322 	/* drop TCR + TLR access, we setup XON/XOFF later */
323 	serial8250_out_MCR(up, up->mcr);
324 	serial_out(up, UART_IER, up->ier);
325 
326 	serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
327 	serial_dl_write(up, priv->quot);
328 
329 	serial_out(up, UART_EFR, priv->efr);
330 
331 	/* Configure flow control */
332 	serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
333 	serial_out(up, UART_XON1, priv->xon);
334 	serial_out(up, UART_XOFF1, priv->xoff);
335 
336 	serial_out(up, UART_LCR, up->lcr);
337 
338 	omap8250_update_mdr1(up, priv);
339 
340 	up->port.ops->set_mctrl(&up->port, up->port.mctrl);
341 }
342 
343 /*
344  * OMAP can use "CLK / (16 or 13) / div" for baud rate. And then we have have
345  * some differences in how we want to handle flow control.
346  */
347 static void omap_8250_set_termios(struct uart_port *port,
348 				  struct ktermios *termios,
349 				  struct ktermios *old)
350 {
351 	struct uart_8250_port *up = up_to_u8250p(port);
352 	struct omap8250_priv *priv = up->port.private_data;
353 	unsigned char cval = 0;
354 	unsigned int baud;
355 
356 	switch (termios->c_cflag & CSIZE) {
357 	case CS5:
358 		cval = UART_LCR_WLEN5;
359 		break;
360 	case CS6:
361 		cval = UART_LCR_WLEN6;
362 		break;
363 	case CS7:
364 		cval = UART_LCR_WLEN7;
365 		break;
366 	default:
367 	case CS8:
368 		cval = UART_LCR_WLEN8;
369 		break;
370 	}
371 
372 	if (termios->c_cflag & CSTOPB)
373 		cval |= UART_LCR_STOP;
374 	if (termios->c_cflag & PARENB)
375 		cval |= UART_LCR_PARITY;
376 	if (!(termios->c_cflag & PARODD))
377 		cval |= UART_LCR_EPAR;
378 	if (termios->c_cflag & CMSPAR)
379 		cval |= UART_LCR_SPAR;
380 
381 	/*
382 	 * Ask the core to calculate the divisor for us.
383 	 */
384 	baud = uart_get_baud_rate(port, termios, old,
385 				  port->uartclk / 16 / UART_DIV_MAX,
386 				  port->uartclk / 13);
387 	omap_8250_get_divisor(port, baud, priv);
388 
389 	/*
390 	 * Ok, we're now changing the port state. Do it with
391 	 * interrupts disabled.
392 	 */
393 	pm_runtime_get_sync(port->dev);
394 	spin_lock_irq(&port->lock);
395 
396 	/*
397 	 * Update the per-port timeout.
398 	 */
399 	uart_update_timeout(port, termios->c_cflag, baud);
400 
401 	up->port.read_status_mask = UART_LSR_OE | UART_LSR_THRE | UART_LSR_DR;
402 	if (termios->c_iflag & INPCK)
403 		up->port.read_status_mask |= UART_LSR_FE | UART_LSR_PE;
404 	if (termios->c_iflag & (IGNBRK | PARMRK))
405 		up->port.read_status_mask |= UART_LSR_BI;
406 
407 	/*
408 	 * Characters to ignore
409 	 */
410 	up->port.ignore_status_mask = 0;
411 	if (termios->c_iflag & IGNPAR)
412 		up->port.ignore_status_mask |= UART_LSR_PE | UART_LSR_FE;
413 	if (termios->c_iflag & IGNBRK) {
414 		up->port.ignore_status_mask |= UART_LSR_BI;
415 		/*
416 		 * If we're ignoring parity and break indicators,
417 		 * ignore overruns too (for real raw support).
418 		 */
419 		if (termios->c_iflag & IGNPAR)
420 			up->port.ignore_status_mask |= UART_LSR_OE;
421 	}
422 
423 	/*
424 	 * ignore all characters if CREAD is not set
425 	 */
426 	if ((termios->c_cflag & CREAD) == 0)
427 		up->port.ignore_status_mask |= UART_LSR_DR;
428 
429 	/*
430 	 * Modem status interrupts
431 	 */
432 	up->ier &= ~UART_IER_MSI;
433 	if (UART_ENABLE_MS(&up->port, termios->c_cflag))
434 		up->ier |= UART_IER_MSI;
435 
436 	up->lcr = cval;
437 	/* Up to here it was mostly serial8250_do_set_termios() */
438 
439 	/*
440 	 * We enable TRIG_GRANU for RX and TX and additionally we set
441 	 * SCR_TX_EMPTY bit. The result is the following:
442 	 * - RX_TRIGGER amount of bytes in the FIFO will cause an interrupt.
443 	 * - less than RX_TRIGGER number of bytes will also cause an interrupt
444 	 *   once the UART decides that there no new bytes arriving.
445 	 * - Once THRE is enabled, the interrupt will be fired once the FIFO is
446 	 *   empty - the trigger level is ignored here.
447 	 *
448 	 * Once DMA is enabled:
449 	 * - UART will assert the TX DMA line once there is room for TX_TRIGGER
450 	 *   bytes in the TX FIFO. On each assert the DMA engine will move
451 	 *   TX_TRIGGER bytes into the FIFO.
452 	 * - UART will assert the RX DMA line once there are RX_TRIGGER bytes in
453 	 *   the FIFO and move RX_TRIGGER bytes.
454 	 * This is because threshold and trigger values are the same.
455 	 */
456 	up->fcr = UART_FCR_ENABLE_FIFO;
457 	up->fcr |= TRIGGER_FCR_MASK(priv->tx_trigger) << OMAP_UART_FCR_TX_TRIG;
458 	up->fcr |= TRIGGER_FCR_MASK(priv->rx_trigger) << OMAP_UART_FCR_RX_TRIG;
459 
460 	priv->scr = OMAP_UART_SCR_RX_TRIG_GRANU1_MASK | OMAP_UART_SCR_TX_EMPTY |
461 		OMAP_UART_SCR_TX_TRIG_GRANU1_MASK;
462 
463 	if (up->dma)
464 		priv->scr |= OMAP_UART_SCR_DMAMODE_1 |
465 			OMAP_UART_SCR_DMAMODE_CTL;
466 
467 	priv->xon = termios->c_cc[VSTART];
468 	priv->xoff = termios->c_cc[VSTOP];
469 
470 	priv->efr = 0;
471 	up->port.status &= ~(UPSTAT_AUTOCTS | UPSTAT_AUTORTS | UPSTAT_AUTOXOFF);
472 
473 	if (termios->c_cflag & CRTSCTS && up->port.flags & UPF_HARD_FLOW &&
474 	    !mctrl_gpio_to_gpiod(up->gpios, UART_GPIO_RTS) &&
475 	    !mctrl_gpio_to_gpiod(up->gpios, UART_GPIO_CTS)) {
476 		/* Enable AUTOCTS (autoRTS is enabled when RTS is raised) */
477 		up->port.status |= UPSTAT_AUTOCTS | UPSTAT_AUTORTS;
478 		priv->efr |= UART_EFR_CTS;
479 	} else	if (up->port.flags & UPF_SOFT_FLOW) {
480 		/*
481 		 * OMAP rx s/w flow control is borked; the transmitter remains
482 		 * stuck off even if rx flow control is subsequently disabled
483 		 */
484 
485 		/*
486 		 * IXOFF Flag:
487 		 * Enable XON/XOFF flow control on output.
488 		 * Transmit XON1, XOFF1
489 		 */
490 		if (termios->c_iflag & IXOFF) {
491 			up->port.status |= UPSTAT_AUTOXOFF;
492 			priv->efr |= OMAP_UART_SW_TX;
493 		}
494 	}
495 	omap8250_restore_regs(up);
496 
497 	spin_unlock_irq(&up->port.lock);
498 	pm_runtime_mark_last_busy(port->dev);
499 	pm_runtime_put_autosuspend(port->dev);
500 
501 	/* calculate wakeup latency constraint */
502 	priv->calc_latency = USEC_PER_SEC * 64 * 8 / baud;
503 	priv->latency = priv->calc_latency;
504 
505 	schedule_work(&priv->qos_work);
506 
507 	/* Don't rewrite B0 */
508 	if (tty_termios_baud_rate(termios))
509 		tty_termios_encode_baud_rate(termios, baud, baud);
510 }
511 
512 /* same as 8250 except that we may have extra flow bits set in EFR */
513 static void omap_8250_pm(struct uart_port *port, unsigned int state,
514 			 unsigned int oldstate)
515 {
516 	struct uart_8250_port *up = up_to_u8250p(port);
517 	u8 efr;
518 
519 	pm_runtime_get_sync(port->dev);
520 	serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
521 	efr = serial_in(up, UART_EFR);
522 	serial_out(up, UART_EFR, efr | UART_EFR_ECB);
523 	serial_out(up, UART_LCR, 0);
524 
525 	serial_out(up, UART_IER, (state != 0) ? UART_IERX_SLEEP : 0);
526 	serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
527 	serial_out(up, UART_EFR, efr);
528 	serial_out(up, UART_LCR, 0);
529 
530 	pm_runtime_mark_last_busy(port->dev);
531 	pm_runtime_put_autosuspend(port->dev);
532 }
533 
534 static void omap_serial_fill_features_erratas(struct uart_8250_port *up,
535 					      struct omap8250_priv *priv)
536 {
537 	const struct soc_device_attribute k3_soc_devices[] = {
538 		{ .family = "AM65X",  },
539 		{ .family = "J721E", .revision = "SR1.0" },
540 		{ /* sentinel */ }
541 	};
542 	u32 mvr, scheme;
543 	u16 revision, major, minor;
544 
545 	mvr = uart_read(up, UART_OMAP_MVER);
546 
547 	/* Check revision register scheme */
548 	scheme = mvr >> OMAP_UART_MVR_SCHEME_SHIFT;
549 
550 	switch (scheme) {
551 	case 0: /* Legacy Scheme: OMAP2/3 */
552 		/* MINOR_REV[0:4], MAJOR_REV[4:7] */
553 		major = (mvr & OMAP_UART_LEGACY_MVR_MAJ_MASK) >>
554 			OMAP_UART_LEGACY_MVR_MAJ_SHIFT;
555 		minor = (mvr & OMAP_UART_LEGACY_MVR_MIN_MASK);
556 		break;
557 	case 1:
558 		/* New Scheme: OMAP4+ */
559 		/* MINOR_REV[0:5], MAJOR_REV[8:10] */
560 		major = (mvr & OMAP_UART_MVR_MAJ_MASK) >>
561 			OMAP_UART_MVR_MAJ_SHIFT;
562 		minor = (mvr & OMAP_UART_MVR_MIN_MASK);
563 		break;
564 	default:
565 		dev_warn(up->port.dev,
566 			 "Unknown revision, defaulting to highest\n");
567 		/* highest possible revision */
568 		major = 0xff;
569 		minor = 0xff;
570 	}
571 	/* normalize revision for the driver */
572 	revision = UART_BUILD_REVISION(major, minor);
573 
574 	switch (revision) {
575 	case OMAP_UART_REV_46:
576 		priv->habit |= UART_ERRATA_i202_MDR1_ACCESS;
577 		break;
578 	case OMAP_UART_REV_52:
579 		priv->habit |= UART_ERRATA_i202_MDR1_ACCESS |
580 				OMAP_UART_WER_HAS_TX_WAKEUP;
581 		break;
582 	case OMAP_UART_REV_63:
583 		priv->habit |= UART_ERRATA_i202_MDR1_ACCESS |
584 			OMAP_UART_WER_HAS_TX_WAKEUP;
585 		break;
586 	default:
587 		break;
588 	}
589 
590 	/*
591 	 * AM65x SR1.0, AM65x SR2.0 and J721e SR1.0 don't
592 	 * don't have RHR_IT_DIS bit in IER2 register. So drop to flag
593 	 * to enable errata workaround.
594 	 */
595 	if (soc_device_match(k3_soc_devices))
596 		priv->habit &= ~UART_HAS_RHR_IT_DIS;
597 }
598 
599 static void omap8250_uart_qos_work(struct work_struct *work)
600 {
601 	struct omap8250_priv *priv;
602 
603 	priv = container_of(work, struct omap8250_priv, qos_work);
604 	cpu_latency_qos_update_request(&priv->pm_qos_request, priv->latency);
605 }
606 
607 #ifdef CONFIG_SERIAL_8250_DMA
608 static int omap_8250_dma_handle_irq(struct uart_port *port);
609 #endif
610 
611 static irqreturn_t omap8250_irq(int irq, void *dev_id)
612 {
613 	struct uart_port *port = dev_id;
614 	struct uart_8250_port *up = up_to_u8250p(port);
615 	unsigned int iir;
616 	int ret;
617 
618 #ifdef CONFIG_SERIAL_8250_DMA
619 	if (up->dma) {
620 		ret = omap_8250_dma_handle_irq(port);
621 		return IRQ_RETVAL(ret);
622 	}
623 #endif
624 
625 	serial8250_rpm_get(up);
626 	iir = serial_port_in(port, UART_IIR);
627 	ret = serial8250_handle_irq(port, iir);
628 	serial8250_rpm_put(up);
629 
630 	return IRQ_RETVAL(ret);
631 }
632 
633 static int omap_8250_startup(struct uart_port *port)
634 {
635 	struct uart_8250_port *up = up_to_u8250p(port);
636 	struct omap8250_priv *priv = port->private_data;
637 	int ret;
638 
639 	if (priv->wakeirq) {
640 		ret = dev_pm_set_dedicated_wake_irq(port->dev, priv->wakeirq);
641 		if (ret)
642 			return ret;
643 	}
644 
645 	pm_runtime_get_sync(port->dev);
646 
647 	up->mcr = 0;
648 	serial_out(up, UART_FCR, UART_FCR_CLEAR_RCVR | UART_FCR_CLEAR_XMIT);
649 
650 	serial_out(up, UART_LCR, UART_LCR_WLEN8);
651 
652 	up->lsr_saved_flags = 0;
653 	up->msr_saved_flags = 0;
654 
655 	/* Disable DMA for console UART */
656 	if (uart_console(port))
657 		up->dma = NULL;
658 
659 	if (up->dma) {
660 		ret = serial8250_request_dma(up);
661 		if (ret) {
662 			dev_warn_ratelimited(port->dev,
663 					     "failed to request DMA\n");
664 			up->dma = NULL;
665 		}
666 	}
667 
668 	ret = request_irq(port->irq, omap8250_irq, IRQF_SHARED,
669 			  dev_name(port->dev), port);
670 	if (ret < 0)
671 		goto err;
672 
673 	up->ier = UART_IER_RLSI | UART_IER_RDI;
674 	serial_out(up, UART_IER, up->ier);
675 
676 #ifdef CONFIG_PM
677 	up->capabilities |= UART_CAP_RPM;
678 #endif
679 
680 	/* Enable module level wake up */
681 	priv->wer = OMAP_UART_WER_MOD_WKUP;
682 	if (priv->habit & OMAP_UART_WER_HAS_TX_WAKEUP)
683 		priv->wer |= OMAP_UART_TX_WAKEUP_EN;
684 	serial_out(up, UART_OMAP_WER, priv->wer);
685 
686 	if (up->dma && !(priv->habit & UART_HAS_EFR2))
687 		up->dma->rx_dma(up);
688 
689 	pm_runtime_mark_last_busy(port->dev);
690 	pm_runtime_put_autosuspend(port->dev);
691 	return 0;
692 err:
693 	pm_runtime_mark_last_busy(port->dev);
694 	pm_runtime_put_autosuspend(port->dev);
695 	dev_pm_clear_wake_irq(port->dev);
696 	return ret;
697 }
698 
699 static void omap_8250_shutdown(struct uart_port *port)
700 {
701 	struct uart_8250_port *up = up_to_u8250p(port);
702 	struct omap8250_priv *priv = port->private_data;
703 
704 	flush_work(&priv->qos_work);
705 	if (up->dma)
706 		omap_8250_rx_dma_flush(up);
707 
708 	pm_runtime_get_sync(port->dev);
709 
710 	serial_out(up, UART_OMAP_WER, 0);
711 	if (priv->habit & UART_HAS_EFR2)
712 		serial_out(up, UART_OMAP_EFR2, 0x0);
713 
714 	up->ier = 0;
715 	serial_out(up, UART_IER, 0);
716 
717 	if (up->dma)
718 		serial8250_release_dma(up);
719 
720 	/*
721 	 * Disable break condition and FIFOs
722 	 */
723 	if (up->lcr & UART_LCR_SBC)
724 		serial_out(up, UART_LCR, up->lcr & ~UART_LCR_SBC);
725 	serial_out(up, UART_FCR, UART_FCR_CLEAR_RCVR | UART_FCR_CLEAR_XMIT);
726 
727 	pm_runtime_mark_last_busy(port->dev);
728 	pm_runtime_put_autosuspend(port->dev);
729 	free_irq(port->irq, port);
730 	dev_pm_clear_wake_irq(port->dev);
731 }
732 
733 static void omap_8250_throttle(struct uart_port *port)
734 {
735 	struct omap8250_priv *priv = port->private_data;
736 	unsigned long flags;
737 
738 	pm_runtime_get_sync(port->dev);
739 
740 	spin_lock_irqsave(&port->lock, flags);
741 	port->ops->stop_rx(port);
742 	priv->throttled = true;
743 	spin_unlock_irqrestore(&port->lock, flags);
744 
745 	pm_runtime_mark_last_busy(port->dev);
746 	pm_runtime_put_autosuspend(port->dev);
747 }
748 
749 static void omap_8250_unthrottle(struct uart_port *port)
750 {
751 	struct omap8250_priv *priv = port->private_data;
752 	struct uart_8250_port *up = up_to_u8250p(port);
753 	unsigned long flags;
754 
755 	pm_runtime_get_sync(port->dev);
756 
757 	spin_lock_irqsave(&port->lock, flags);
758 	priv->throttled = false;
759 	if (up->dma)
760 		up->dma->rx_dma(up);
761 	up->ier |= UART_IER_RLSI | UART_IER_RDI;
762 	port->read_status_mask |= UART_LSR_DR;
763 	serial_out(up, UART_IER, up->ier);
764 	spin_unlock_irqrestore(&port->lock, flags);
765 
766 	pm_runtime_mark_last_busy(port->dev);
767 	pm_runtime_put_autosuspend(port->dev);
768 }
769 
770 #ifdef CONFIG_SERIAL_8250_DMA
771 static int omap_8250_rx_dma(struct uart_8250_port *p);
772 
773 /* Must be called while priv->rx_dma_lock is held */
774 static void __dma_rx_do_complete(struct uart_8250_port *p)
775 {
776 	struct uart_8250_dma    *dma = p->dma;
777 	struct tty_port         *tty_port = &p->port.state->port;
778 	struct omap8250_priv	*priv = p->port.private_data;
779 	struct dma_chan		*rxchan = dma->rxchan;
780 	dma_cookie_t		cookie;
781 	struct dma_tx_state     state;
782 	int                     count;
783 	int			ret;
784 	u32			reg;
785 
786 	if (!dma->rx_running)
787 		goto out;
788 
789 	cookie = dma->rx_cookie;
790 	dma->rx_running = 0;
791 
792 	/* Re-enable RX FIFO interrupt now that transfer is complete */
793 	if (priv->habit & UART_HAS_RHR_IT_DIS) {
794 		reg = serial_in(p, UART_OMAP_IER2);
795 		reg &= ~UART_OMAP_IER2_RHR_IT_DIS;
796 		serial_out(p, UART_OMAP_IER2, UART_OMAP_IER2_RHR_IT_DIS);
797 	}
798 
799 	dmaengine_tx_status(rxchan, cookie, &state);
800 
801 	count = dma->rx_size - state.residue + state.in_flight_bytes;
802 	if (count < dma->rx_size) {
803 		dmaengine_terminate_async(rxchan);
804 
805 		/*
806 		 * Poll for teardown to complete which guarantees in
807 		 * flight data is drained.
808 		 */
809 		if (state.in_flight_bytes) {
810 			int poll_count = 25;
811 
812 			while (dmaengine_tx_status(rxchan, cookie, NULL) &&
813 			       poll_count--)
814 				cpu_relax();
815 
816 			if (!poll_count)
817 				dev_err(p->port.dev, "teardown incomplete\n");
818 		}
819 	}
820 	if (!count)
821 		goto out;
822 	ret = tty_insert_flip_string(tty_port, dma->rx_buf, count);
823 
824 	p->port.icount.rx += ret;
825 	p->port.icount.buf_overrun += count - ret;
826 out:
827 
828 	tty_flip_buffer_push(tty_port);
829 }
830 
831 static void __dma_rx_complete(void *param)
832 {
833 	struct uart_8250_port *p = param;
834 	struct omap8250_priv *priv = p->port.private_data;
835 	struct uart_8250_dma *dma = p->dma;
836 	struct dma_tx_state     state;
837 	unsigned long flags;
838 
839 	spin_lock_irqsave(&p->port.lock, flags);
840 
841 	/*
842 	 * If the tx status is not DMA_COMPLETE, then this is a delayed
843 	 * completion callback. A previous RX timeout flush would have
844 	 * already pushed the data, so exit.
845 	 */
846 	if (dmaengine_tx_status(dma->rxchan, dma->rx_cookie, &state) !=
847 			DMA_COMPLETE) {
848 		spin_unlock_irqrestore(&p->port.lock, flags);
849 		return;
850 	}
851 	__dma_rx_do_complete(p);
852 	if (!priv->throttled) {
853 		p->ier |= UART_IER_RLSI | UART_IER_RDI;
854 		serial_out(p, UART_IER, p->ier);
855 		if (!(priv->habit & UART_HAS_EFR2))
856 			omap_8250_rx_dma(p);
857 	}
858 
859 	spin_unlock_irqrestore(&p->port.lock, flags);
860 }
861 
862 static void omap_8250_rx_dma_flush(struct uart_8250_port *p)
863 {
864 	struct omap8250_priv	*priv = p->port.private_data;
865 	struct uart_8250_dma	*dma = p->dma;
866 	struct dma_tx_state     state;
867 	unsigned long		flags;
868 	int ret;
869 
870 	spin_lock_irqsave(&priv->rx_dma_lock, flags);
871 
872 	if (!dma->rx_running) {
873 		spin_unlock_irqrestore(&priv->rx_dma_lock, flags);
874 		return;
875 	}
876 
877 	ret = dmaengine_tx_status(dma->rxchan, dma->rx_cookie, &state);
878 	if (ret == DMA_IN_PROGRESS) {
879 		ret = dmaengine_pause(dma->rxchan);
880 		if (WARN_ON_ONCE(ret))
881 			priv->rx_dma_broken = true;
882 	}
883 	__dma_rx_do_complete(p);
884 	spin_unlock_irqrestore(&priv->rx_dma_lock, flags);
885 }
886 
887 static int omap_8250_rx_dma(struct uart_8250_port *p)
888 {
889 	struct omap8250_priv		*priv = p->port.private_data;
890 	struct uart_8250_dma            *dma = p->dma;
891 	int				err = 0;
892 	struct dma_async_tx_descriptor  *desc;
893 	unsigned long			flags;
894 	u32				reg;
895 
896 	if (priv->rx_dma_broken)
897 		return -EINVAL;
898 
899 	spin_lock_irqsave(&priv->rx_dma_lock, flags);
900 
901 	if (dma->rx_running) {
902 		enum dma_status state;
903 
904 		state = dmaengine_tx_status(dma->rxchan, dma->rx_cookie, NULL);
905 		if (state == DMA_COMPLETE) {
906 			/*
907 			 * Disable RX interrupts to allow RX DMA completion
908 			 * callback to run.
909 			 */
910 			p->ier &= ~(UART_IER_RLSI | UART_IER_RDI);
911 			serial_out(p, UART_IER, p->ier);
912 		}
913 		goto out;
914 	}
915 
916 	desc = dmaengine_prep_slave_single(dma->rxchan, dma->rx_addr,
917 					   dma->rx_size, DMA_DEV_TO_MEM,
918 					   DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
919 	if (!desc) {
920 		err = -EBUSY;
921 		goto out;
922 	}
923 
924 	dma->rx_running = 1;
925 	desc->callback = __dma_rx_complete;
926 	desc->callback_param = p;
927 
928 	dma->rx_cookie = dmaengine_submit(desc);
929 
930 	/*
931 	 * Disable RX FIFO interrupt while RX DMA is enabled, else
932 	 * spurious interrupt may be raised when data is in the RX FIFO
933 	 * but is yet to be drained by DMA.
934 	 */
935 	if (priv->habit & UART_HAS_RHR_IT_DIS) {
936 		reg = serial_in(p, UART_OMAP_IER2);
937 		reg |= UART_OMAP_IER2_RHR_IT_DIS;
938 		serial_out(p, UART_OMAP_IER2, UART_OMAP_IER2_RHR_IT_DIS);
939 	}
940 
941 	dma_async_issue_pending(dma->rxchan);
942 out:
943 	spin_unlock_irqrestore(&priv->rx_dma_lock, flags);
944 	return err;
945 }
946 
947 static int omap_8250_tx_dma(struct uart_8250_port *p);
948 
949 static void omap_8250_dma_tx_complete(void *param)
950 {
951 	struct uart_8250_port	*p = param;
952 	struct uart_8250_dma	*dma = p->dma;
953 	struct circ_buf		*xmit = &p->port.state->xmit;
954 	unsigned long		flags;
955 	bool			en_thri = false;
956 	struct omap8250_priv	*priv = p->port.private_data;
957 
958 	dma_sync_single_for_cpu(dma->txchan->device->dev, dma->tx_addr,
959 				UART_XMIT_SIZE, DMA_TO_DEVICE);
960 
961 	spin_lock_irqsave(&p->port.lock, flags);
962 
963 	dma->tx_running = 0;
964 
965 	xmit->tail += dma->tx_size;
966 	xmit->tail &= UART_XMIT_SIZE - 1;
967 	p->port.icount.tx += dma->tx_size;
968 
969 	if (priv->delayed_restore) {
970 		priv->delayed_restore = 0;
971 		omap8250_restore_regs(p);
972 	}
973 
974 	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
975 		uart_write_wakeup(&p->port);
976 
977 	if (!uart_circ_empty(xmit) && !uart_tx_stopped(&p->port)) {
978 		int ret;
979 
980 		ret = omap_8250_tx_dma(p);
981 		if (ret)
982 			en_thri = true;
983 	} else if (p->capabilities & UART_CAP_RPM) {
984 		en_thri = true;
985 	}
986 
987 	if (en_thri) {
988 		dma->tx_err = 1;
989 		serial8250_set_THRI(p);
990 	}
991 
992 	spin_unlock_irqrestore(&p->port.lock, flags);
993 }
994 
995 static int omap_8250_tx_dma(struct uart_8250_port *p)
996 {
997 	struct uart_8250_dma		*dma = p->dma;
998 	struct omap8250_priv		*priv = p->port.private_data;
999 	struct circ_buf			*xmit = &p->port.state->xmit;
1000 	struct dma_async_tx_descriptor	*desc;
1001 	unsigned int	skip_byte = 0;
1002 	int ret;
1003 
1004 	if (dma->tx_running)
1005 		return 0;
1006 	if (uart_tx_stopped(&p->port) || uart_circ_empty(xmit)) {
1007 
1008 		/*
1009 		 * Even if no data, we need to return an error for the two cases
1010 		 * below so serial8250_tx_chars() is invoked and properly clears
1011 		 * THRI and/or runtime suspend.
1012 		 */
1013 		if (dma->tx_err || p->capabilities & UART_CAP_RPM) {
1014 			ret = -EBUSY;
1015 			goto err;
1016 		}
1017 		serial8250_clear_THRI(p);
1018 		return 0;
1019 	}
1020 
1021 	dma->tx_size = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
1022 	if (priv->habit & OMAP_DMA_TX_KICK) {
1023 		u8 tx_lvl;
1024 
1025 		/*
1026 		 * We need to put the first byte into the FIFO in order to start
1027 		 * the DMA transfer. For transfers smaller than four bytes we
1028 		 * don't bother doing DMA at all. It seem not matter if there
1029 		 * are still bytes in the FIFO from the last transfer (in case
1030 		 * we got here directly from omap_8250_dma_tx_complete()). Bytes
1031 		 * leaving the FIFO seem not to trigger the DMA transfer. It is
1032 		 * really the byte that we put into the FIFO.
1033 		 * If the FIFO is already full then we most likely got here from
1034 		 * omap_8250_dma_tx_complete(). And this means the DMA engine
1035 		 * just completed its work. We don't have to wait the complete
1036 		 * 86us at 115200,8n1 but around 60us (not to mention lower
1037 		 * baudrates). So in that case we take the interrupt and try
1038 		 * again with an empty FIFO.
1039 		 */
1040 		tx_lvl = serial_in(p, UART_OMAP_TX_LVL);
1041 		if (tx_lvl == p->tx_loadsz) {
1042 			ret = -EBUSY;
1043 			goto err;
1044 		}
1045 		if (dma->tx_size < 4) {
1046 			ret = -EINVAL;
1047 			goto err;
1048 		}
1049 		skip_byte = 1;
1050 	}
1051 
1052 	desc = dmaengine_prep_slave_single(dma->txchan,
1053 			dma->tx_addr + xmit->tail + skip_byte,
1054 			dma->tx_size - skip_byte, DMA_MEM_TO_DEV,
1055 			DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1056 	if (!desc) {
1057 		ret = -EBUSY;
1058 		goto err;
1059 	}
1060 
1061 	dma->tx_running = 1;
1062 
1063 	desc->callback = omap_8250_dma_tx_complete;
1064 	desc->callback_param = p;
1065 
1066 	dma->tx_cookie = dmaengine_submit(desc);
1067 
1068 	dma_sync_single_for_device(dma->txchan->device->dev, dma->tx_addr,
1069 				   UART_XMIT_SIZE, DMA_TO_DEVICE);
1070 
1071 	dma_async_issue_pending(dma->txchan);
1072 	if (dma->tx_err)
1073 		dma->tx_err = 0;
1074 
1075 	serial8250_clear_THRI(p);
1076 	if (skip_byte)
1077 		serial_out(p, UART_TX, xmit->buf[xmit->tail]);
1078 	return 0;
1079 err:
1080 	dma->tx_err = 1;
1081 	return ret;
1082 }
1083 
1084 static bool handle_rx_dma(struct uart_8250_port *up, unsigned int iir)
1085 {
1086 	switch (iir & 0x3f) {
1087 	case UART_IIR_RLSI:
1088 	case UART_IIR_RX_TIMEOUT:
1089 	case UART_IIR_RDI:
1090 		omap_8250_rx_dma_flush(up);
1091 		return true;
1092 	}
1093 	return omap_8250_rx_dma(up);
1094 }
1095 
1096 static unsigned char omap_8250_handle_rx_dma(struct uart_8250_port *up,
1097 					     u8 iir, unsigned char status)
1098 {
1099 	if ((status & (UART_LSR_DR | UART_LSR_BI)) &&
1100 	    (iir & UART_IIR_RDI)) {
1101 		if (handle_rx_dma(up, iir)) {
1102 			status = serial8250_rx_chars(up, status);
1103 			omap_8250_rx_dma(up);
1104 		}
1105 	}
1106 
1107 	return status;
1108 }
1109 
1110 static void am654_8250_handle_rx_dma(struct uart_8250_port *up, u8 iir,
1111 				     unsigned char status)
1112 {
1113 	/*
1114 	 * Queue a new transfer if FIFO has data.
1115 	 */
1116 	if ((status & (UART_LSR_DR | UART_LSR_BI)) &&
1117 	    (up->ier & UART_IER_RDI)) {
1118 		omap_8250_rx_dma(up);
1119 		serial_out(up, UART_OMAP_EFR2, UART_OMAP_EFR2_TIMEOUT_BEHAVE);
1120 	} else if ((iir & 0x3f) == UART_IIR_RX_TIMEOUT) {
1121 		/*
1122 		 * Disable RX timeout, read IIR to clear
1123 		 * current timeout condition, clear EFR2 to
1124 		 * periodic timeouts, re-enable interrupts.
1125 		 */
1126 		up->ier &= ~(UART_IER_RLSI | UART_IER_RDI);
1127 		serial_out(up, UART_IER, up->ier);
1128 		omap_8250_rx_dma_flush(up);
1129 		serial_in(up, UART_IIR);
1130 		serial_out(up, UART_OMAP_EFR2, 0x0);
1131 		up->ier |= UART_IER_RLSI | UART_IER_RDI;
1132 		serial_out(up, UART_IER, up->ier);
1133 	}
1134 }
1135 
1136 /*
1137  * This is mostly serial8250_handle_irq(). We have a slightly different DMA
1138  * hoook for RX/TX and need different logic for them in the ISR. Therefore we
1139  * use the default routine in the non-DMA case and this one for with DMA.
1140  */
1141 static int omap_8250_dma_handle_irq(struct uart_port *port)
1142 {
1143 	struct uart_8250_port *up = up_to_u8250p(port);
1144 	struct omap8250_priv *priv = up->port.private_data;
1145 	unsigned char status;
1146 	u8 iir;
1147 
1148 	serial8250_rpm_get(up);
1149 
1150 	iir = serial_port_in(port, UART_IIR);
1151 	if (iir & UART_IIR_NO_INT) {
1152 		serial8250_rpm_put(up);
1153 		return IRQ_HANDLED;
1154 	}
1155 
1156 	spin_lock(&port->lock);
1157 
1158 	status = serial_port_in(port, UART_LSR);
1159 
1160 	if (priv->habit & UART_HAS_EFR2)
1161 		am654_8250_handle_rx_dma(up, iir, status);
1162 	else
1163 		status = omap_8250_handle_rx_dma(up, iir, status);
1164 
1165 	serial8250_modem_status(up);
1166 	if (status & UART_LSR_THRE && up->dma->tx_err) {
1167 		if (uart_tx_stopped(&up->port) ||
1168 		    uart_circ_empty(&up->port.state->xmit)) {
1169 			up->dma->tx_err = 0;
1170 			serial8250_tx_chars(up);
1171 		} else  {
1172 			/*
1173 			 * try again due to an earlier failer which
1174 			 * might have been resolved by now.
1175 			 */
1176 			if (omap_8250_tx_dma(up))
1177 				serial8250_tx_chars(up);
1178 		}
1179 	}
1180 
1181 	uart_unlock_and_check_sysrq(port);
1182 
1183 	serial8250_rpm_put(up);
1184 	return 1;
1185 }
1186 
1187 static bool the_no_dma_filter_fn(struct dma_chan *chan, void *param)
1188 {
1189 	return false;
1190 }
1191 
1192 #else
1193 
1194 static inline int omap_8250_rx_dma(struct uart_8250_port *p)
1195 {
1196 	return -EINVAL;
1197 }
1198 #endif
1199 
1200 static int omap8250_no_handle_irq(struct uart_port *port)
1201 {
1202 	/* IRQ has not been requested but handling irq? */
1203 	WARN_ONCE(1, "Unexpected irq handling before port startup\n");
1204 	return 0;
1205 }
1206 
1207 static struct omap8250_dma_params am654_dma = {
1208 	.rx_size = SZ_2K,
1209 	.rx_trigger = 1,
1210 	.tx_trigger = TX_TRIGGER,
1211 };
1212 
1213 static struct omap8250_dma_params am33xx_dma = {
1214 	.rx_size = RX_TRIGGER,
1215 	.rx_trigger = RX_TRIGGER,
1216 	.tx_trigger = TX_TRIGGER,
1217 };
1218 
1219 static struct omap8250_platdata am654_platdata = {
1220 	.dma_params	= &am654_dma,
1221 	.habit		= UART_HAS_EFR2 | UART_HAS_RHR_IT_DIS,
1222 };
1223 
1224 static struct omap8250_platdata am33xx_platdata = {
1225 	.dma_params	= &am33xx_dma,
1226 	.habit		= OMAP_DMA_TX_KICK | UART_ERRATA_CLOCK_DISABLE,
1227 };
1228 
1229 static struct omap8250_platdata omap4_platdata = {
1230 	.dma_params	= &am33xx_dma,
1231 	.habit		= UART_ERRATA_CLOCK_DISABLE,
1232 };
1233 
1234 static const struct of_device_id omap8250_dt_ids[] = {
1235 	{ .compatible = "ti,am654-uart", .data = &am654_platdata, },
1236 	{ .compatible = "ti,omap2-uart" },
1237 	{ .compatible = "ti,omap3-uart" },
1238 	{ .compatible = "ti,omap4-uart", .data = &omap4_platdata, },
1239 	{ .compatible = "ti,am3352-uart", .data = &am33xx_platdata, },
1240 	{ .compatible = "ti,am4372-uart", .data = &am33xx_platdata, },
1241 	{ .compatible = "ti,dra742-uart", .data = &omap4_platdata, },
1242 	{},
1243 };
1244 MODULE_DEVICE_TABLE(of, omap8250_dt_ids);
1245 
1246 static int omap8250_probe(struct platform_device *pdev)
1247 {
1248 	struct device_node *np = pdev->dev.of_node;
1249 	struct omap8250_priv *priv;
1250 	const struct omap8250_platdata *pdata;
1251 	struct uart_8250_port up;
1252 	struct resource *regs;
1253 	void __iomem *membase;
1254 	int irq, ret;
1255 
1256 	irq = platform_get_irq(pdev, 0);
1257 	if (irq < 0)
1258 		return irq;
1259 
1260 	regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1261 	if (!regs) {
1262 		dev_err(&pdev->dev, "missing registers\n");
1263 		return -EINVAL;
1264 	}
1265 
1266 	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
1267 	if (!priv)
1268 		return -ENOMEM;
1269 
1270 	membase = devm_ioremap(&pdev->dev, regs->start,
1271 				       resource_size(regs));
1272 	if (!membase)
1273 		return -ENODEV;
1274 
1275 	memset(&up, 0, sizeof(up));
1276 	up.port.dev = &pdev->dev;
1277 	up.port.mapbase = regs->start;
1278 	up.port.membase = membase;
1279 	up.port.irq = irq;
1280 	/*
1281 	 * It claims to be 16C750 compatible however it is a little different.
1282 	 * It has EFR and has no FCR7_64byte bit. The AFE (which it claims to
1283 	 * have) is enabled via EFR instead of MCR. The type is set here 8250
1284 	 * just to get things going. UNKNOWN does not work for a few reasons and
1285 	 * we don't need our own type since we don't use 8250's set_termios()
1286 	 * or pm callback.
1287 	 */
1288 	up.port.type = PORT_8250;
1289 	up.port.iotype = UPIO_MEM;
1290 	up.port.flags = UPF_FIXED_PORT | UPF_FIXED_TYPE | UPF_SOFT_FLOW |
1291 		UPF_HARD_FLOW;
1292 	up.port.private_data = priv;
1293 
1294 	up.port.regshift = 2;
1295 	up.port.fifosize = 64;
1296 	up.tx_loadsz = 64;
1297 	up.capabilities = UART_CAP_FIFO;
1298 #ifdef CONFIG_PM
1299 	/*
1300 	 * Runtime PM is mostly transparent. However to do it right we need to a
1301 	 * TX empty interrupt before we can put the device to auto idle. So if
1302 	 * PM is not enabled we don't add that flag and can spare that one extra
1303 	 * interrupt in the TX path.
1304 	 */
1305 	up.capabilities |= UART_CAP_RPM;
1306 #endif
1307 	up.port.set_termios = omap_8250_set_termios;
1308 	up.port.set_mctrl = omap8250_set_mctrl;
1309 	up.port.pm = omap_8250_pm;
1310 	up.port.startup = omap_8250_startup;
1311 	up.port.shutdown = omap_8250_shutdown;
1312 	up.port.throttle = omap_8250_throttle;
1313 	up.port.unthrottle = omap_8250_unthrottle;
1314 	up.port.rs485_config = serial8250_em485_config;
1315 	up.rs485_start_tx = serial8250_em485_start_tx;
1316 	up.rs485_stop_tx = serial8250_em485_stop_tx;
1317 	up.port.has_sysrq = IS_ENABLED(CONFIG_SERIAL_8250_CONSOLE);
1318 
1319 	ret = of_alias_get_id(np, "serial");
1320 	if (ret < 0) {
1321 		dev_err(&pdev->dev, "failed to get alias\n");
1322 		return ret;
1323 	}
1324 	up.port.line = ret;
1325 
1326 	if (of_property_read_u32(np, "clock-frequency", &up.port.uartclk)) {
1327 		struct clk *clk;
1328 
1329 		clk = devm_clk_get(&pdev->dev, NULL);
1330 		if (IS_ERR(clk)) {
1331 			if (PTR_ERR(clk) == -EPROBE_DEFER)
1332 				return -EPROBE_DEFER;
1333 		} else {
1334 			up.port.uartclk = clk_get_rate(clk);
1335 		}
1336 	}
1337 
1338 	priv->wakeirq = irq_of_parse_and_map(np, 1);
1339 
1340 	pdata = of_device_get_match_data(&pdev->dev);
1341 	if (pdata)
1342 		priv->habit |= pdata->habit;
1343 
1344 	if (!up.port.uartclk) {
1345 		up.port.uartclk = DEFAULT_CLK_SPEED;
1346 		dev_warn(&pdev->dev,
1347 			 "No clock speed specified: using default: %d\n",
1348 			 DEFAULT_CLK_SPEED);
1349 	}
1350 
1351 	priv->latency = PM_QOS_CPU_LATENCY_DEFAULT_VALUE;
1352 	priv->calc_latency = PM_QOS_CPU_LATENCY_DEFAULT_VALUE;
1353 	cpu_latency_qos_add_request(&priv->pm_qos_request, priv->latency);
1354 	INIT_WORK(&priv->qos_work, omap8250_uart_qos_work);
1355 
1356 	spin_lock_init(&priv->rx_dma_lock);
1357 
1358 	device_init_wakeup(&pdev->dev, true);
1359 	pm_runtime_enable(&pdev->dev);
1360 	pm_runtime_use_autosuspend(&pdev->dev);
1361 
1362 	/*
1363 	 * Disable runtime PM until autosuspend delay unless specifically
1364 	 * enabled by the user via sysfs. This is the historic way to
1365 	 * prevent an unsafe default policy with lossy characters on wake-up.
1366 	 * For serdev devices this is not needed, the policy can be managed by
1367 	 * the serdev driver.
1368 	 */
1369 	if (!of_get_available_child_count(pdev->dev.of_node))
1370 		pm_runtime_set_autosuspend_delay(&pdev->dev, -1);
1371 
1372 	pm_runtime_irq_safe(&pdev->dev);
1373 
1374 	pm_runtime_get_sync(&pdev->dev);
1375 
1376 	omap_serial_fill_features_erratas(&up, priv);
1377 	up.port.handle_irq = omap8250_no_handle_irq;
1378 	priv->rx_trigger = RX_TRIGGER;
1379 	priv->tx_trigger = TX_TRIGGER;
1380 #ifdef CONFIG_SERIAL_8250_DMA
1381 	/*
1382 	 * Oh DMA support. If there are no DMA properties in the DT then
1383 	 * we will fall back to a generic DMA channel which does not
1384 	 * really work here. To ensure that we do not get a generic DMA
1385 	 * channel assigned, we have the the_no_dma_filter_fn() here.
1386 	 * To avoid "failed to request DMA" messages we check for DMA
1387 	 * properties in DT.
1388 	 */
1389 	ret = of_property_count_strings(np, "dma-names");
1390 	if (ret == 2) {
1391 		struct omap8250_dma_params *dma_params = NULL;
1392 
1393 		up.dma = &priv->omap8250_dma;
1394 		up.dma->fn = the_no_dma_filter_fn;
1395 		up.dma->tx_dma = omap_8250_tx_dma;
1396 		up.dma->rx_dma = omap_8250_rx_dma;
1397 		if (pdata)
1398 			dma_params = pdata->dma_params;
1399 
1400 		if (dma_params) {
1401 			up.dma->rx_size = dma_params->rx_size;
1402 			up.dma->rxconf.src_maxburst = dma_params->rx_trigger;
1403 			up.dma->txconf.dst_maxburst = dma_params->tx_trigger;
1404 			priv->rx_trigger = dma_params->rx_trigger;
1405 			priv->tx_trigger = dma_params->tx_trigger;
1406 		} else {
1407 			up.dma->rx_size = RX_TRIGGER;
1408 			up.dma->rxconf.src_maxburst = RX_TRIGGER;
1409 			up.dma->txconf.dst_maxburst = TX_TRIGGER;
1410 		}
1411 	}
1412 #endif
1413 	ret = serial8250_register_8250_port(&up);
1414 	if (ret < 0) {
1415 		dev_err(&pdev->dev, "unable to register 8250 port\n");
1416 		goto err;
1417 	}
1418 	priv->line = ret;
1419 	platform_set_drvdata(pdev, priv);
1420 	pm_runtime_mark_last_busy(&pdev->dev);
1421 	pm_runtime_put_autosuspend(&pdev->dev);
1422 	return 0;
1423 err:
1424 	pm_runtime_dont_use_autosuspend(&pdev->dev);
1425 	pm_runtime_put_sync(&pdev->dev);
1426 	pm_runtime_disable(&pdev->dev);
1427 	return ret;
1428 }
1429 
1430 static int omap8250_remove(struct platform_device *pdev)
1431 {
1432 	struct omap8250_priv *priv = platform_get_drvdata(pdev);
1433 
1434 	pm_runtime_dont_use_autosuspend(&pdev->dev);
1435 	pm_runtime_put_sync(&pdev->dev);
1436 	pm_runtime_disable(&pdev->dev);
1437 	serial8250_unregister_port(priv->line);
1438 	cpu_latency_qos_remove_request(&priv->pm_qos_request);
1439 	device_init_wakeup(&pdev->dev, false);
1440 	return 0;
1441 }
1442 
1443 #ifdef CONFIG_PM_SLEEP
1444 static int omap8250_prepare(struct device *dev)
1445 {
1446 	struct omap8250_priv *priv = dev_get_drvdata(dev);
1447 
1448 	if (!priv)
1449 		return 0;
1450 	priv->is_suspending = true;
1451 	return 0;
1452 }
1453 
1454 static void omap8250_complete(struct device *dev)
1455 {
1456 	struct omap8250_priv *priv = dev_get_drvdata(dev);
1457 
1458 	if (!priv)
1459 		return;
1460 	priv->is_suspending = false;
1461 }
1462 
1463 static int omap8250_suspend(struct device *dev)
1464 {
1465 	struct omap8250_priv *priv = dev_get_drvdata(dev);
1466 	struct uart_8250_port *up = serial8250_get_port(priv->line);
1467 
1468 	serial8250_suspend_port(priv->line);
1469 
1470 	pm_runtime_get_sync(dev);
1471 	if (!device_may_wakeup(dev))
1472 		priv->wer = 0;
1473 	serial_out(up, UART_OMAP_WER, priv->wer);
1474 	pm_runtime_mark_last_busy(dev);
1475 	pm_runtime_put_autosuspend(dev);
1476 
1477 	flush_work(&priv->qos_work);
1478 	return 0;
1479 }
1480 
1481 static int omap8250_resume(struct device *dev)
1482 {
1483 	struct omap8250_priv *priv = dev_get_drvdata(dev);
1484 
1485 	serial8250_resume_port(priv->line);
1486 	return 0;
1487 }
1488 #else
1489 #define omap8250_prepare NULL
1490 #define omap8250_complete NULL
1491 #endif
1492 
1493 #ifdef CONFIG_PM
1494 static int omap8250_lost_context(struct uart_8250_port *up)
1495 {
1496 	u32 val;
1497 
1498 	val = serial_in(up, UART_OMAP_SCR);
1499 	/*
1500 	 * If we lose context, then SCR is set to its reset value of zero.
1501 	 * After set_termios() we set bit 3 of SCR (TX_EMPTY_CTL_IT) to 1,
1502 	 * among other bits, to never set the register back to zero again.
1503 	 */
1504 	if (!val)
1505 		return 1;
1506 	return 0;
1507 }
1508 
1509 /* TODO: in future, this should happen via API in drivers/reset/ */
1510 static int omap8250_soft_reset(struct device *dev)
1511 {
1512 	struct omap8250_priv *priv = dev_get_drvdata(dev);
1513 	struct uart_8250_port *up = serial8250_get_port(priv->line);
1514 	int timeout = 100;
1515 	int sysc;
1516 	int syss;
1517 
1518 	/*
1519 	 * At least on omap4, unused uarts may not idle after reset without
1520 	 * a basic scr dma configuration even with no dma in use. The
1521 	 * module clkctrl status bits will be 1 instead of 3 blocking idle
1522 	 * for the whole clockdomain. The softreset below will clear scr,
1523 	 * and we restore it on resume so this is safe to do on all SoCs
1524 	 * needing omap8250_soft_reset() quirk. Do it in two writes as
1525 	 * recommended in the comment for omap8250_update_scr().
1526 	 */
1527 	serial_out(up, UART_OMAP_SCR, OMAP_UART_SCR_DMAMODE_1);
1528 	serial_out(up, UART_OMAP_SCR,
1529 		   OMAP_UART_SCR_DMAMODE_1 | OMAP_UART_SCR_DMAMODE_CTL);
1530 
1531 	sysc = serial_in(up, UART_OMAP_SYSC);
1532 
1533 	/* softreset the UART */
1534 	sysc |= OMAP_UART_SYSC_SOFTRESET;
1535 	serial_out(up, UART_OMAP_SYSC, sysc);
1536 
1537 	/* By experiments, 1us enough for reset complete on AM335x */
1538 	do {
1539 		udelay(1);
1540 		syss = serial_in(up, UART_OMAP_SYSS);
1541 	} while (--timeout && !(syss & OMAP_UART_SYSS_RESETDONE));
1542 
1543 	if (!timeout) {
1544 		dev_err(dev, "timed out waiting for reset done\n");
1545 		return -ETIMEDOUT;
1546 	}
1547 
1548 	return 0;
1549 }
1550 
1551 static int omap8250_runtime_suspend(struct device *dev)
1552 {
1553 	struct omap8250_priv *priv = dev_get_drvdata(dev);
1554 	struct uart_8250_port *up;
1555 
1556 	/* In case runtime-pm tries this before we are setup */
1557 	if (!priv)
1558 		return 0;
1559 
1560 	up = serial8250_get_port(priv->line);
1561 	/*
1562 	 * When using 'no_console_suspend', the console UART must not be
1563 	 * suspended. Since driver suspend is managed by runtime suspend,
1564 	 * preventing runtime suspend (by returning error) will keep device
1565 	 * active during suspend.
1566 	 */
1567 	if (priv->is_suspending && !console_suspend_enabled) {
1568 		if (uart_console(&up->port))
1569 			return -EBUSY;
1570 	}
1571 
1572 	if (priv->habit & UART_ERRATA_CLOCK_DISABLE) {
1573 		int ret;
1574 
1575 		ret = omap8250_soft_reset(dev);
1576 		if (ret)
1577 			return ret;
1578 
1579 		/* Restore to UART mode after reset (for wakeup) */
1580 		omap8250_update_mdr1(up, priv);
1581 		/* Restore wakeup enable register */
1582 		serial_out(up, UART_OMAP_WER, priv->wer);
1583 	}
1584 
1585 	if (up->dma && up->dma->rxchan)
1586 		omap_8250_rx_dma_flush(up);
1587 
1588 	priv->latency = PM_QOS_CPU_LATENCY_DEFAULT_VALUE;
1589 	schedule_work(&priv->qos_work);
1590 
1591 	return 0;
1592 }
1593 
1594 static int omap8250_runtime_resume(struct device *dev)
1595 {
1596 	struct omap8250_priv *priv = dev_get_drvdata(dev);
1597 	struct uart_8250_port *up;
1598 
1599 	/* In case runtime-pm tries this before we are setup */
1600 	if (!priv)
1601 		return 0;
1602 
1603 	up = serial8250_get_port(priv->line);
1604 
1605 	if (omap8250_lost_context(up))
1606 		omap8250_restore_regs(up);
1607 
1608 	if (up->dma && up->dma->rxchan && !(priv->habit & UART_HAS_EFR2))
1609 		omap_8250_rx_dma(up);
1610 
1611 	priv->latency = priv->calc_latency;
1612 	schedule_work(&priv->qos_work);
1613 	return 0;
1614 }
1615 #endif
1616 
1617 #ifdef CONFIG_SERIAL_8250_OMAP_TTYO_FIXUP
1618 static int __init omap8250_console_fixup(void)
1619 {
1620 	char *omap_str;
1621 	char *options;
1622 	u8 idx;
1623 
1624 	if (strstr(boot_command_line, "console=ttyS"))
1625 		/* user set a ttyS based name for the console */
1626 		return 0;
1627 
1628 	omap_str = strstr(boot_command_line, "console=ttyO");
1629 	if (!omap_str)
1630 		/* user did not set ttyO based console, so we don't care */
1631 		return 0;
1632 
1633 	omap_str += 12;
1634 	if ('0' <= *omap_str && *omap_str <= '9')
1635 		idx = *omap_str - '0';
1636 	else
1637 		return 0;
1638 
1639 	omap_str++;
1640 	if (omap_str[0] == ',') {
1641 		omap_str++;
1642 		options = omap_str;
1643 	} else {
1644 		options = NULL;
1645 	}
1646 
1647 	add_preferred_console("ttyS", idx, options);
1648 	pr_err("WARNING: Your 'console=ttyO%d' has been replaced by 'ttyS%d'\n",
1649 	       idx, idx);
1650 	pr_err("This ensures that you still see kernel messages. Please\n");
1651 	pr_err("update your kernel commandline.\n");
1652 	return 0;
1653 }
1654 console_initcall(omap8250_console_fixup);
1655 #endif
1656 
1657 static const struct dev_pm_ops omap8250_dev_pm_ops = {
1658 	SET_SYSTEM_SLEEP_PM_OPS(omap8250_suspend, omap8250_resume)
1659 	SET_RUNTIME_PM_OPS(omap8250_runtime_suspend,
1660 			   omap8250_runtime_resume, NULL)
1661 	.prepare        = omap8250_prepare,
1662 	.complete       = omap8250_complete,
1663 };
1664 
1665 static struct platform_driver omap8250_platform_driver = {
1666 	.driver = {
1667 		.name		= "omap8250",
1668 		.pm		= &omap8250_dev_pm_ops,
1669 		.of_match_table = omap8250_dt_ids,
1670 	},
1671 	.probe			= omap8250_probe,
1672 	.remove			= omap8250_remove,
1673 };
1674 module_platform_driver(omap8250_platform_driver);
1675 
1676 MODULE_AUTHOR("Sebastian Andrzej Siewior");
1677 MODULE_DESCRIPTION("OMAP 8250 Driver");
1678 MODULE_LICENSE("GPL v2");
1679