xref: /openbmc/linux/drivers/tty/serial/msm_serial.c (revision c7b03852)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Driver for msm7k serial device and console
4  *
5  * Copyright (C) 2007 Google, Inc.
6  * Author: Robert Love <rlove@google.com>
7  * Copyright (c) 2011, Code Aurora Forum. All rights reserved.
8  */
9 
10 #if defined(CONFIG_SERIAL_MSM_CONSOLE) && defined(CONFIG_MAGIC_SYSRQ)
11 # define SUPPORT_SYSRQ
12 #endif
13 
14 #include <linux/kernel.h>
15 #include <linux/atomic.h>
16 #include <linux/dma-mapping.h>
17 #include <linux/dmaengine.h>
18 #include <linux/module.h>
19 #include <linux/io.h>
20 #include <linux/ioport.h>
21 #include <linux/interrupt.h>
22 #include <linux/init.h>
23 #include <linux/console.h>
24 #include <linux/tty.h>
25 #include <linux/tty_flip.h>
26 #include <linux/serial_core.h>
27 #include <linux/slab.h>
28 #include <linux/clk.h>
29 #include <linux/platform_device.h>
30 #include <linux/delay.h>
31 #include <linux/of.h>
32 #include <linux/of_device.h>
33 #include <linux/wait.h>
34 
35 #define UART_MR1			0x0000
36 
37 #define UART_MR1_AUTO_RFR_LEVEL0	0x3F
38 #define UART_MR1_AUTO_RFR_LEVEL1	0x3FF00
39 #define UART_DM_MR1_AUTO_RFR_LEVEL1	0xFFFFFF00
40 #define UART_MR1_RX_RDY_CTL		BIT(7)
41 #define UART_MR1_CTS_CTL		BIT(6)
42 
43 #define UART_MR2			0x0004
44 #define UART_MR2_ERROR_MODE		BIT(6)
45 #define UART_MR2_BITS_PER_CHAR		0x30
46 #define UART_MR2_BITS_PER_CHAR_5	(0x0 << 4)
47 #define UART_MR2_BITS_PER_CHAR_6	(0x1 << 4)
48 #define UART_MR2_BITS_PER_CHAR_7	(0x2 << 4)
49 #define UART_MR2_BITS_PER_CHAR_8	(0x3 << 4)
50 #define UART_MR2_STOP_BIT_LEN_ONE	(0x1 << 2)
51 #define UART_MR2_STOP_BIT_LEN_TWO	(0x3 << 2)
52 #define UART_MR2_PARITY_MODE_NONE	0x0
53 #define UART_MR2_PARITY_MODE_ODD	0x1
54 #define UART_MR2_PARITY_MODE_EVEN	0x2
55 #define UART_MR2_PARITY_MODE_SPACE	0x3
56 #define UART_MR2_PARITY_MODE		0x3
57 
58 #define UART_CSR			0x0008
59 
60 #define UART_TF				0x000C
61 #define UARTDM_TF			0x0070
62 
63 #define UART_CR				0x0010
64 #define UART_CR_CMD_NULL		(0 << 4)
65 #define UART_CR_CMD_RESET_RX		(1 << 4)
66 #define UART_CR_CMD_RESET_TX		(2 << 4)
67 #define UART_CR_CMD_RESET_ERR		(3 << 4)
68 #define UART_CR_CMD_RESET_BREAK_INT	(4 << 4)
69 #define UART_CR_CMD_START_BREAK		(5 << 4)
70 #define UART_CR_CMD_STOP_BREAK		(6 << 4)
71 #define UART_CR_CMD_RESET_CTS		(7 << 4)
72 #define UART_CR_CMD_RESET_STALE_INT	(8 << 4)
73 #define UART_CR_CMD_PACKET_MODE		(9 << 4)
74 #define UART_CR_CMD_MODE_RESET		(12 << 4)
75 #define UART_CR_CMD_SET_RFR		(13 << 4)
76 #define UART_CR_CMD_RESET_RFR		(14 << 4)
77 #define UART_CR_CMD_PROTECTION_EN	(16 << 4)
78 #define UART_CR_CMD_STALE_EVENT_DISABLE	(6 << 8)
79 #define UART_CR_CMD_STALE_EVENT_ENABLE	(80 << 4)
80 #define UART_CR_CMD_FORCE_STALE		(4 << 8)
81 #define UART_CR_CMD_RESET_TX_READY	(3 << 8)
82 #define UART_CR_TX_DISABLE		BIT(3)
83 #define UART_CR_TX_ENABLE		BIT(2)
84 #define UART_CR_RX_DISABLE		BIT(1)
85 #define UART_CR_RX_ENABLE		BIT(0)
86 #define UART_CR_CMD_RESET_RXBREAK_START	((1 << 11) | (2 << 4))
87 
88 #define UART_IMR			0x0014
89 #define UART_IMR_TXLEV			BIT(0)
90 #define UART_IMR_RXSTALE		BIT(3)
91 #define UART_IMR_RXLEV			BIT(4)
92 #define UART_IMR_DELTA_CTS		BIT(5)
93 #define UART_IMR_CURRENT_CTS		BIT(6)
94 #define UART_IMR_RXBREAK_START		BIT(10)
95 
96 #define UART_IPR_RXSTALE_LAST		0x20
97 #define UART_IPR_STALE_LSB		0x1F
98 #define UART_IPR_STALE_TIMEOUT_MSB	0x3FF80
99 #define UART_DM_IPR_STALE_TIMEOUT_MSB	0xFFFFFF80
100 
101 #define UART_IPR			0x0018
102 #define UART_TFWR			0x001C
103 #define UART_RFWR			0x0020
104 #define UART_HCR			0x0024
105 
106 #define UART_MREG			0x0028
107 #define UART_NREG			0x002C
108 #define UART_DREG			0x0030
109 #define UART_MNDREG			0x0034
110 #define UART_IRDA			0x0038
111 #define UART_MISR_MODE			0x0040
112 #define UART_MISR_RESET			0x0044
113 #define UART_MISR_EXPORT		0x0048
114 #define UART_MISR_VAL			0x004C
115 #define UART_TEST_CTRL			0x0050
116 
117 #define UART_SR				0x0008
118 #define UART_SR_HUNT_CHAR		BIT(7)
119 #define UART_SR_RX_BREAK		BIT(6)
120 #define UART_SR_PAR_FRAME_ERR		BIT(5)
121 #define UART_SR_OVERRUN			BIT(4)
122 #define UART_SR_TX_EMPTY		BIT(3)
123 #define UART_SR_TX_READY		BIT(2)
124 #define UART_SR_RX_FULL			BIT(1)
125 #define UART_SR_RX_READY		BIT(0)
126 
127 #define UART_RF				0x000C
128 #define UARTDM_RF			0x0070
129 #define UART_MISR			0x0010
130 #define UART_ISR			0x0014
131 #define UART_ISR_TX_READY		BIT(7)
132 
133 #define UARTDM_RXFS			0x50
134 #define UARTDM_RXFS_BUF_SHIFT		0x7
135 #define UARTDM_RXFS_BUF_MASK		0x7
136 
137 #define UARTDM_DMEN			0x3C
138 #define UARTDM_DMEN_RX_SC_ENABLE	BIT(5)
139 #define UARTDM_DMEN_TX_SC_ENABLE	BIT(4)
140 
141 #define UARTDM_DMEN_TX_BAM_ENABLE	BIT(2)	/* UARTDM_1P4 */
142 #define UARTDM_DMEN_TX_DM_ENABLE	BIT(0)	/* < UARTDM_1P4 */
143 
144 #define UARTDM_DMEN_RX_BAM_ENABLE	BIT(3)	/* UARTDM_1P4 */
145 #define UARTDM_DMEN_RX_DM_ENABLE	BIT(1)	/* < UARTDM_1P4 */
146 
147 #define UARTDM_DMRX			0x34
148 #define UARTDM_NCF_TX			0x40
149 #define UARTDM_RX_TOTAL_SNAP		0x38
150 
151 #define UARTDM_BURST_SIZE		16   /* in bytes */
152 #define UARTDM_TX_AIGN(x)		((x) & ~0x3) /* valid for > 1p3 */
153 #define UARTDM_TX_MAX			256   /* in bytes, valid for <= 1p3 */
154 #define UARTDM_RX_SIZE			(UART_XMIT_SIZE / 4)
155 
156 enum {
157 	UARTDM_1P1 = 1,
158 	UARTDM_1P2,
159 	UARTDM_1P3,
160 	UARTDM_1P4,
161 };
162 
163 struct msm_dma {
164 	struct dma_chan		*chan;
165 	enum dma_data_direction dir;
166 	dma_addr_t		phys;
167 	unsigned char		*virt;
168 	dma_cookie_t		cookie;
169 	u32			enable_bit;
170 	unsigned int		count;
171 	struct dma_async_tx_descriptor	*desc;
172 };
173 
174 struct msm_port {
175 	struct uart_port	uart;
176 	char			name[16];
177 	struct clk		*clk;
178 	struct clk		*pclk;
179 	unsigned int		imr;
180 	int			is_uartdm;
181 	unsigned int		old_snap_state;
182 	bool			break_detected;
183 	struct msm_dma		tx_dma;
184 	struct msm_dma		rx_dma;
185 };
186 
187 #define UART_TO_MSM(uart_port)	container_of(uart_port, struct msm_port, uart)
188 
189 static
190 void msm_write(struct uart_port *port, unsigned int val, unsigned int off)
191 {
192 	writel_relaxed(val, port->membase + off);
193 }
194 
195 static
196 unsigned int msm_read(struct uart_port *port, unsigned int off)
197 {
198 	return readl_relaxed(port->membase + off);
199 }
200 
201 /*
202  * Setup the MND registers to use the TCXO clock.
203  */
204 static void msm_serial_set_mnd_regs_tcxo(struct uart_port *port)
205 {
206 	msm_write(port, 0x06, UART_MREG);
207 	msm_write(port, 0xF1, UART_NREG);
208 	msm_write(port, 0x0F, UART_DREG);
209 	msm_write(port, 0x1A, UART_MNDREG);
210 	port->uartclk = 1843200;
211 }
212 
213 /*
214  * Setup the MND registers to use the TCXO clock divided by 4.
215  */
216 static void msm_serial_set_mnd_regs_tcxoby4(struct uart_port *port)
217 {
218 	msm_write(port, 0x18, UART_MREG);
219 	msm_write(port, 0xF6, UART_NREG);
220 	msm_write(port, 0x0F, UART_DREG);
221 	msm_write(port, 0x0A, UART_MNDREG);
222 	port->uartclk = 1843200;
223 }
224 
225 static void msm_serial_set_mnd_regs(struct uart_port *port)
226 {
227 	struct msm_port *msm_port = UART_TO_MSM(port);
228 
229 	/*
230 	 * These registers don't exist so we change the clk input rate
231 	 * on uartdm hardware instead
232 	 */
233 	if (msm_port->is_uartdm)
234 		return;
235 
236 	if (port->uartclk == 19200000)
237 		msm_serial_set_mnd_regs_tcxo(port);
238 	else if (port->uartclk == 4800000)
239 		msm_serial_set_mnd_regs_tcxoby4(port);
240 }
241 
242 static void msm_handle_tx(struct uart_port *port);
243 static void msm_start_rx_dma(struct msm_port *msm_port);
244 
245 static void msm_stop_dma(struct uart_port *port, struct msm_dma *dma)
246 {
247 	struct device *dev = port->dev;
248 	unsigned int mapped;
249 	u32 val;
250 
251 	mapped = dma->count;
252 	dma->count = 0;
253 
254 	dmaengine_terminate_all(dma->chan);
255 
256 	/*
257 	 * DMA Stall happens if enqueue and flush command happens concurrently.
258 	 * For example before changing the baud rate/protocol configuration and
259 	 * sending flush command to ADM, disable the channel of UARTDM.
260 	 * Note: should not reset the receiver here immediately as it is not
261 	 * suggested to do disable/reset or reset/disable at the same time.
262 	 */
263 	val = msm_read(port, UARTDM_DMEN);
264 	val &= ~dma->enable_bit;
265 	msm_write(port, val, UARTDM_DMEN);
266 
267 	if (mapped)
268 		dma_unmap_single(dev, dma->phys, mapped, dma->dir);
269 }
270 
271 static void msm_release_dma(struct msm_port *msm_port)
272 {
273 	struct msm_dma *dma;
274 
275 	dma = &msm_port->tx_dma;
276 	if (dma->chan) {
277 		msm_stop_dma(&msm_port->uart, dma);
278 		dma_release_channel(dma->chan);
279 	}
280 
281 	memset(dma, 0, sizeof(*dma));
282 
283 	dma = &msm_port->rx_dma;
284 	if (dma->chan) {
285 		msm_stop_dma(&msm_port->uart, dma);
286 		dma_release_channel(dma->chan);
287 		kfree(dma->virt);
288 	}
289 
290 	memset(dma, 0, sizeof(*dma));
291 }
292 
293 static void msm_request_tx_dma(struct msm_port *msm_port, resource_size_t base)
294 {
295 	struct device *dev = msm_port->uart.dev;
296 	struct dma_slave_config conf;
297 	struct msm_dma *dma;
298 	u32 crci = 0;
299 	int ret;
300 
301 	dma = &msm_port->tx_dma;
302 
303 	/* allocate DMA resources, if available */
304 	dma->chan = dma_request_slave_channel_reason(dev, "tx");
305 	if (IS_ERR(dma->chan))
306 		goto no_tx;
307 
308 	of_property_read_u32(dev->of_node, "qcom,tx-crci", &crci);
309 
310 	memset(&conf, 0, sizeof(conf));
311 	conf.direction = DMA_MEM_TO_DEV;
312 	conf.device_fc = true;
313 	conf.dst_addr = base + UARTDM_TF;
314 	conf.dst_maxburst = UARTDM_BURST_SIZE;
315 	conf.slave_id = crci;
316 
317 	ret = dmaengine_slave_config(dma->chan, &conf);
318 	if (ret)
319 		goto rel_tx;
320 
321 	dma->dir = DMA_TO_DEVICE;
322 
323 	if (msm_port->is_uartdm < UARTDM_1P4)
324 		dma->enable_bit = UARTDM_DMEN_TX_DM_ENABLE;
325 	else
326 		dma->enable_bit = UARTDM_DMEN_TX_BAM_ENABLE;
327 
328 	return;
329 
330 rel_tx:
331 	dma_release_channel(dma->chan);
332 no_tx:
333 	memset(dma, 0, sizeof(*dma));
334 }
335 
336 static void msm_request_rx_dma(struct msm_port *msm_port, resource_size_t base)
337 {
338 	struct device *dev = msm_port->uart.dev;
339 	struct dma_slave_config conf;
340 	struct msm_dma *dma;
341 	u32 crci = 0;
342 	int ret;
343 
344 	dma = &msm_port->rx_dma;
345 
346 	/* allocate DMA resources, if available */
347 	dma->chan = dma_request_slave_channel_reason(dev, "rx");
348 	if (IS_ERR(dma->chan))
349 		goto no_rx;
350 
351 	of_property_read_u32(dev->of_node, "qcom,rx-crci", &crci);
352 
353 	dma->virt = kzalloc(UARTDM_RX_SIZE, GFP_KERNEL);
354 	if (!dma->virt)
355 		goto rel_rx;
356 
357 	memset(&conf, 0, sizeof(conf));
358 	conf.direction = DMA_DEV_TO_MEM;
359 	conf.device_fc = true;
360 	conf.src_addr = base + UARTDM_RF;
361 	conf.src_maxburst = UARTDM_BURST_SIZE;
362 	conf.slave_id = crci;
363 
364 	ret = dmaengine_slave_config(dma->chan, &conf);
365 	if (ret)
366 		goto err;
367 
368 	dma->dir = DMA_FROM_DEVICE;
369 
370 	if (msm_port->is_uartdm < UARTDM_1P4)
371 		dma->enable_bit = UARTDM_DMEN_RX_DM_ENABLE;
372 	else
373 		dma->enable_bit = UARTDM_DMEN_RX_BAM_ENABLE;
374 
375 	return;
376 err:
377 	kfree(dma->virt);
378 rel_rx:
379 	dma_release_channel(dma->chan);
380 no_rx:
381 	memset(dma, 0, sizeof(*dma));
382 }
383 
384 static inline void msm_wait_for_xmitr(struct uart_port *port)
385 {
386 	unsigned int timeout = 500000;
387 
388 	while (!(msm_read(port, UART_SR) & UART_SR_TX_EMPTY)) {
389 		if (msm_read(port, UART_ISR) & UART_ISR_TX_READY)
390 			break;
391 		udelay(1);
392 		if (!timeout--)
393 			break;
394 	}
395 	msm_write(port, UART_CR_CMD_RESET_TX_READY, UART_CR);
396 }
397 
398 static void msm_stop_tx(struct uart_port *port)
399 {
400 	struct msm_port *msm_port = UART_TO_MSM(port);
401 
402 	msm_port->imr &= ~UART_IMR_TXLEV;
403 	msm_write(port, msm_port->imr, UART_IMR);
404 }
405 
406 static void msm_start_tx(struct uart_port *port)
407 {
408 	struct msm_port *msm_port = UART_TO_MSM(port);
409 	struct msm_dma *dma = &msm_port->tx_dma;
410 
411 	/* Already started in DMA mode */
412 	if (dma->count)
413 		return;
414 
415 	msm_port->imr |= UART_IMR_TXLEV;
416 	msm_write(port, msm_port->imr, UART_IMR);
417 }
418 
419 static void msm_reset_dm_count(struct uart_port *port, int count)
420 {
421 	msm_wait_for_xmitr(port);
422 	msm_write(port, count, UARTDM_NCF_TX);
423 	msm_read(port, UARTDM_NCF_TX);
424 }
425 
426 static void msm_complete_tx_dma(void *args)
427 {
428 	struct msm_port *msm_port = args;
429 	struct uart_port *port = &msm_port->uart;
430 	struct circ_buf *xmit = &port->state->xmit;
431 	struct msm_dma *dma = &msm_port->tx_dma;
432 	struct dma_tx_state state;
433 	enum dma_status status;
434 	unsigned long flags;
435 	unsigned int count;
436 	u32 val;
437 
438 	spin_lock_irqsave(&port->lock, flags);
439 
440 	/* Already stopped */
441 	if (!dma->count)
442 		goto done;
443 
444 	status = dmaengine_tx_status(dma->chan, dma->cookie, &state);
445 
446 	dma_unmap_single(port->dev, dma->phys, dma->count, dma->dir);
447 
448 	val = msm_read(port, UARTDM_DMEN);
449 	val &= ~dma->enable_bit;
450 	msm_write(port, val, UARTDM_DMEN);
451 
452 	if (msm_port->is_uartdm > UARTDM_1P3) {
453 		msm_write(port, UART_CR_CMD_RESET_TX, UART_CR);
454 		msm_write(port, UART_CR_TX_ENABLE, UART_CR);
455 	}
456 
457 	count = dma->count - state.residue;
458 	port->icount.tx += count;
459 	dma->count = 0;
460 
461 	xmit->tail += count;
462 	xmit->tail &= UART_XMIT_SIZE - 1;
463 
464 	/* Restore "Tx FIFO below watermark" interrupt */
465 	msm_port->imr |= UART_IMR_TXLEV;
466 	msm_write(port, msm_port->imr, UART_IMR);
467 
468 	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
469 		uart_write_wakeup(port);
470 
471 	msm_handle_tx(port);
472 done:
473 	spin_unlock_irqrestore(&port->lock, flags);
474 }
475 
476 static int msm_handle_tx_dma(struct msm_port *msm_port, unsigned int count)
477 {
478 	struct circ_buf *xmit = &msm_port->uart.state->xmit;
479 	struct uart_port *port = &msm_port->uart;
480 	struct msm_dma *dma = &msm_port->tx_dma;
481 	void *cpu_addr;
482 	int ret;
483 	u32 val;
484 
485 	cpu_addr = &xmit->buf[xmit->tail];
486 
487 	dma->phys = dma_map_single(port->dev, cpu_addr, count, dma->dir);
488 	ret = dma_mapping_error(port->dev, dma->phys);
489 	if (ret)
490 		return ret;
491 
492 	dma->desc = dmaengine_prep_slave_single(dma->chan, dma->phys,
493 						count, DMA_MEM_TO_DEV,
494 						DMA_PREP_INTERRUPT |
495 						DMA_PREP_FENCE);
496 	if (!dma->desc) {
497 		ret = -EIO;
498 		goto unmap;
499 	}
500 
501 	dma->desc->callback = msm_complete_tx_dma;
502 	dma->desc->callback_param = msm_port;
503 
504 	dma->cookie = dmaengine_submit(dma->desc);
505 	ret = dma_submit_error(dma->cookie);
506 	if (ret)
507 		goto unmap;
508 
509 	/*
510 	 * Using DMA complete for Tx FIFO reload, no need for
511 	 * "Tx FIFO below watermark" one, disable it
512 	 */
513 	msm_port->imr &= ~UART_IMR_TXLEV;
514 	msm_write(port, msm_port->imr, UART_IMR);
515 
516 	dma->count = count;
517 
518 	val = msm_read(port, UARTDM_DMEN);
519 	val |= dma->enable_bit;
520 
521 	if (msm_port->is_uartdm < UARTDM_1P4)
522 		msm_write(port, val, UARTDM_DMEN);
523 
524 	msm_reset_dm_count(port, count);
525 
526 	if (msm_port->is_uartdm > UARTDM_1P3)
527 		msm_write(port, val, UARTDM_DMEN);
528 
529 	dma_async_issue_pending(dma->chan);
530 	return 0;
531 unmap:
532 	dma_unmap_single(port->dev, dma->phys, count, dma->dir);
533 	return ret;
534 }
535 
536 static void msm_complete_rx_dma(void *args)
537 {
538 	struct msm_port *msm_port = args;
539 	struct uart_port *port = &msm_port->uart;
540 	struct tty_port *tport = &port->state->port;
541 	struct msm_dma *dma = &msm_port->rx_dma;
542 	int count = 0, i, sysrq;
543 	unsigned long flags;
544 	u32 val;
545 
546 	spin_lock_irqsave(&port->lock, flags);
547 
548 	/* Already stopped */
549 	if (!dma->count)
550 		goto done;
551 
552 	val = msm_read(port, UARTDM_DMEN);
553 	val &= ~dma->enable_bit;
554 	msm_write(port, val, UARTDM_DMEN);
555 
556 	if (msm_read(port, UART_SR) & UART_SR_OVERRUN) {
557 		port->icount.overrun++;
558 		tty_insert_flip_char(tport, 0, TTY_OVERRUN);
559 		msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
560 	}
561 
562 	count = msm_read(port, UARTDM_RX_TOTAL_SNAP);
563 
564 	port->icount.rx += count;
565 
566 	dma->count = 0;
567 
568 	dma_unmap_single(port->dev, dma->phys, UARTDM_RX_SIZE, dma->dir);
569 
570 	for (i = 0; i < count; i++) {
571 		char flag = TTY_NORMAL;
572 
573 		if (msm_port->break_detected && dma->virt[i] == 0) {
574 			port->icount.brk++;
575 			flag = TTY_BREAK;
576 			msm_port->break_detected = false;
577 			if (uart_handle_break(port))
578 				continue;
579 		}
580 
581 		if (!(port->read_status_mask & UART_SR_RX_BREAK))
582 			flag = TTY_NORMAL;
583 
584 		spin_unlock_irqrestore(&port->lock, flags);
585 		sysrq = uart_handle_sysrq_char(port, dma->virt[i]);
586 		spin_lock_irqsave(&port->lock, flags);
587 		if (!sysrq)
588 			tty_insert_flip_char(tport, dma->virt[i], flag);
589 	}
590 
591 	msm_start_rx_dma(msm_port);
592 done:
593 	spin_unlock_irqrestore(&port->lock, flags);
594 
595 	if (count)
596 		tty_flip_buffer_push(tport);
597 }
598 
599 static void msm_start_rx_dma(struct msm_port *msm_port)
600 {
601 	struct msm_dma *dma = &msm_port->rx_dma;
602 	struct uart_port *uart = &msm_port->uart;
603 	u32 val;
604 	int ret;
605 
606 	if (!dma->chan)
607 		return;
608 
609 	dma->phys = dma_map_single(uart->dev, dma->virt,
610 				   UARTDM_RX_SIZE, dma->dir);
611 	ret = dma_mapping_error(uart->dev, dma->phys);
612 	if (ret)
613 		return;
614 
615 	dma->desc = dmaengine_prep_slave_single(dma->chan, dma->phys,
616 						UARTDM_RX_SIZE, DMA_DEV_TO_MEM,
617 						DMA_PREP_INTERRUPT);
618 	if (!dma->desc)
619 		goto unmap;
620 
621 	dma->desc->callback = msm_complete_rx_dma;
622 	dma->desc->callback_param = msm_port;
623 
624 	dma->cookie = dmaengine_submit(dma->desc);
625 	ret = dma_submit_error(dma->cookie);
626 	if (ret)
627 		goto unmap;
628 	/*
629 	 * Using DMA for FIFO off-load, no need for "Rx FIFO over
630 	 * watermark" or "stale" interrupts, disable them
631 	 */
632 	msm_port->imr &= ~(UART_IMR_RXLEV | UART_IMR_RXSTALE);
633 
634 	/*
635 	 * Well, when DMA is ADM3 engine(implied by <= UARTDM v1.3),
636 	 * we need RXSTALE to flush input DMA fifo to memory
637 	 */
638 	if (msm_port->is_uartdm < UARTDM_1P4)
639 		msm_port->imr |= UART_IMR_RXSTALE;
640 
641 	msm_write(uart, msm_port->imr, UART_IMR);
642 
643 	dma->count = UARTDM_RX_SIZE;
644 
645 	dma_async_issue_pending(dma->chan);
646 
647 	msm_write(uart, UART_CR_CMD_RESET_STALE_INT, UART_CR);
648 	msm_write(uart, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
649 
650 	val = msm_read(uart, UARTDM_DMEN);
651 	val |= dma->enable_bit;
652 
653 	if (msm_port->is_uartdm < UARTDM_1P4)
654 		msm_write(uart, val, UARTDM_DMEN);
655 
656 	msm_write(uart, UARTDM_RX_SIZE, UARTDM_DMRX);
657 
658 	if (msm_port->is_uartdm > UARTDM_1P3)
659 		msm_write(uart, val, UARTDM_DMEN);
660 
661 	return;
662 unmap:
663 	dma_unmap_single(uart->dev, dma->phys, UARTDM_RX_SIZE, dma->dir);
664 }
665 
666 static void msm_stop_rx(struct uart_port *port)
667 {
668 	struct msm_port *msm_port = UART_TO_MSM(port);
669 	struct msm_dma *dma = &msm_port->rx_dma;
670 
671 	msm_port->imr &= ~(UART_IMR_RXLEV | UART_IMR_RXSTALE);
672 	msm_write(port, msm_port->imr, UART_IMR);
673 
674 	if (dma->chan)
675 		msm_stop_dma(port, dma);
676 }
677 
678 static void msm_enable_ms(struct uart_port *port)
679 {
680 	struct msm_port *msm_port = UART_TO_MSM(port);
681 
682 	msm_port->imr |= UART_IMR_DELTA_CTS;
683 	msm_write(port, msm_port->imr, UART_IMR);
684 }
685 
686 static void msm_handle_rx_dm(struct uart_port *port, unsigned int misr)
687 {
688 	struct tty_port *tport = &port->state->port;
689 	unsigned int sr;
690 	int count = 0;
691 	struct msm_port *msm_port = UART_TO_MSM(port);
692 
693 	if ((msm_read(port, UART_SR) & UART_SR_OVERRUN)) {
694 		port->icount.overrun++;
695 		tty_insert_flip_char(tport, 0, TTY_OVERRUN);
696 		msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
697 	}
698 
699 	if (misr & UART_IMR_RXSTALE) {
700 		count = msm_read(port, UARTDM_RX_TOTAL_SNAP) -
701 			msm_port->old_snap_state;
702 		msm_port->old_snap_state = 0;
703 	} else {
704 		count = 4 * (msm_read(port, UART_RFWR));
705 		msm_port->old_snap_state += count;
706 	}
707 
708 	/* TODO: Precise error reporting */
709 
710 	port->icount.rx += count;
711 
712 	while (count > 0) {
713 		unsigned char buf[4];
714 		int sysrq, r_count, i;
715 
716 		sr = msm_read(port, UART_SR);
717 		if ((sr & UART_SR_RX_READY) == 0) {
718 			msm_port->old_snap_state -= count;
719 			break;
720 		}
721 
722 		ioread32_rep(port->membase + UARTDM_RF, buf, 1);
723 		r_count = min_t(int, count, sizeof(buf));
724 
725 		for (i = 0; i < r_count; i++) {
726 			char flag = TTY_NORMAL;
727 
728 			if (msm_port->break_detected && buf[i] == 0) {
729 				port->icount.brk++;
730 				flag = TTY_BREAK;
731 				msm_port->break_detected = false;
732 				if (uart_handle_break(port))
733 					continue;
734 			}
735 
736 			if (!(port->read_status_mask & UART_SR_RX_BREAK))
737 				flag = TTY_NORMAL;
738 
739 			spin_unlock(&port->lock);
740 			sysrq = uart_handle_sysrq_char(port, buf[i]);
741 			spin_lock(&port->lock);
742 			if (!sysrq)
743 				tty_insert_flip_char(tport, buf[i], flag);
744 		}
745 		count -= r_count;
746 	}
747 
748 	spin_unlock(&port->lock);
749 	tty_flip_buffer_push(tport);
750 	spin_lock(&port->lock);
751 
752 	if (misr & (UART_IMR_RXSTALE))
753 		msm_write(port, UART_CR_CMD_RESET_STALE_INT, UART_CR);
754 	msm_write(port, 0xFFFFFF, UARTDM_DMRX);
755 	msm_write(port, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
756 
757 	/* Try to use DMA */
758 	msm_start_rx_dma(msm_port);
759 }
760 
761 static void msm_handle_rx(struct uart_port *port)
762 {
763 	struct tty_port *tport = &port->state->port;
764 	unsigned int sr;
765 
766 	/*
767 	 * Handle overrun. My understanding of the hardware is that overrun
768 	 * is not tied to the RX buffer, so we handle the case out of band.
769 	 */
770 	if ((msm_read(port, UART_SR) & UART_SR_OVERRUN)) {
771 		port->icount.overrun++;
772 		tty_insert_flip_char(tport, 0, TTY_OVERRUN);
773 		msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
774 	}
775 
776 	/* and now the main RX loop */
777 	while ((sr = msm_read(port, UART_SR)) & UART_SR_RX_READY) {
778 		unsigned int c;
779 		char flag = TTY_NORMAL;
780 		int sysrq;
781 
782 		c = msm_read(port, UART_RF);
783 
784 		if (sr & UART_SR_RX_BREAK) {
785 			port->icount.brk++;
786 			if (uart_handle_break(port))
787 				continue;
788 		} else if (sr & UART_SR_PAR_FRAME_ERR) {
789 			port->icount.frame++;
790 		} else {
791 			port->icount.rx++;
792 		}
793 
794 		/* Mask conditions we're ignorning. */
795 		sr &= port->read_status_mask;
796 
797 		if (sr & UART_SR_RX_BREAK)
798 			flag = TTY_BREAK;
799 		else if (sr & UART_SR_PAR_FRAME_ERR)
800 			flag = TTY_FRAME;
801 
802 		spin_unlock(&port->lock);
803 		sysrq = uart_handle_sysrq_char(port, c);
804 		spin_lock(&port->lock);
805 		if (!sysrq)
806 			tty_insert_flip_char(tport, c, flag);
807 	}
808 
809 	spin_unlock(&port->lock);
810 	tty_flip_buffer_push(tport);
811 	spin_lock(&port->lock);
812 }
813 
814 static void msm_handle_tx_pio(struct uart_port *port, unsigned int tx_count)
815 {
816 	struct circ_buf *xmit = &port->state->xmit;
817 	struct msm_port *msm_port = UART_TO_MSM(port);
818 	unsigned int num_chars;
819 	unsigned int tf_pointer = 0;
820 	void __iomem *tf;
821 
822 	if (msm_port->is_uartdm)
823 		tf = port->membase + UARTDM_TF;
824 	else
825 		tf = port->membase + UART_TF;
826 
827 	if (tx_count && msm_port->is_uartdm)
828 		msm_reset_dm_count(port, tx_count);
829 
830 	while (tf_pointer < tx_count) {
831 		int i;
832 		char buf[4] = { 0 };
833 
834 		if (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
835 			break;
836 
837 		if (msm_port->is_uartdm)
838 			num_chars = min(tx_count - tf_pointer,
839 					(unsigned int)sizeof(buf));
840 		else
841 			num_chars = 1;
842 
843 		for (i = 0; i < num_chars; i++) {
844 			buf[i] = xmit->buf[xmit->tail + i];
845 			port->icount.tx++;
846 		}
847 
848 		iowrite32_rep(tf, buf, 1);
849 		xmit->tail = (xmit->tail + num_chars) & (UART_XMIT_SIZE - 1);
850 		tf_pointer += num_chars;
851 	}
852 
853 	/* disable tx interrupts if nothing more to send */
854 	if (uart_circ_empty(xmit))
855 		msm_stop_tx(port);
856 
857 	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
858 		uart_write_wakeup(port);
859 }
860 
861 static void msm_handle_tx(struct uart_port *port)
862 {
863 	struct msm_port *msm_port = UART_TO_MSM(port);
864 	struct circ_buf *xmit = &msm_port->uart.state->xmit;
865 	struct msm_dma *dma = &msm_port->tx_dma;
866 	unsigned int pio_count, dma_count, dma_min;
867 	char buf[4] = { 0 };
868 	void __iomem *tf;
869 	int err = 0;
870 
871 	if (port->x_char) {
872 		if (msm_port->is_uartdm)
873 			tf = port->membase + UARTDM_TF;
874 		else
875 			tf = port->membase + UART_TF;
876 
877 		buf[0] = port->x_char;
878 
879 		if (msm_port->is_uartdm)
880 			msm_reset_dm_count(port, 1);
881 
882 		iowrite32_rep(tf, buf, 1);
883 		port->icount.tx++;
884 		port->x_char = 0;
885 		return;
886 	}
887 
888 	if (uart_circ_empty(xmit) || uart_tx_stopped(port)) {
889 		msm_stop_tx(port);
890 		return;
891 	}
892 
893 	pio_count = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
894 	dma_count = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
895 
896 	dma_min = 1;	/* Always DMA */
897 	if (msm_port->is_uartdm > UARTDM_1P3) {
898 		dma_count = UARTDM_TX_AIGN(dma_count);
899 		dma_min = UARTDM_BURST_SIZE;
900 	} else {
901 		if (dma_count > UARTDM_TX_MAX)
902 			dma_count = UARTDM_TX_MAX;
903 	}
904 
905 	if (pio_count > port->fifosize)
906 		pio_count = port->fifosize;
907 
908 	if (!dma->chan || dma_count < dma_min)
909 		msm_handle_tx_pio(port, pio_count);
910 	else
911 		err = msm_handle_tx_dma(msm_port, dma_count);
912 
913 	if (err)	/* fall back to PIO mode */
914 		msm_handle_tx_pio(port, pio_count);
915 }
916 
917 static void msm_handle_delta_cts(struct uart_port *port)
918 {
919 	msm_write(port, UART_CR_CMD_RESET_CTS, UART_CR);
920 	port->icount.cts++;
921 	wake_up_interruptible(&port->state->port.delta_msr_wait);
922 }
923 
924 static irqreturn_t msm_uart_irq(int irq, void *dev_id)
925 {
926 	struct uart_port *port = dev_id;
927 	struct msm_port *msm_port = UART_TO_MSM(port);
928 	struct msm_dma *dma = &msm_port->rx_dma;
929 	unsigned long flags;
930 	unsigned int misr;
931 	u32 val;
932 
933 	spin_lock_irqsave(&port->lock, flags);
934 	misr = msm_read(port, UART_MISR);
935 	msm_write(port, 0, UART_IMR); /* disable interrupt */
936 
937 	if (misr & UART_IMR_RXBREAK_START) {
938 		msm_port->break_detected = true;
939 		msm_write(port, UART_CR_CMD_RESET_RXBREAK_START, UART_CR);
940 	}
941 
942 	if (misr & (UART_IMR_RXLEV | UART_IMR_RXSTALE)) {
943 		if (dma->count) {
944 			val = UART_CR_CMD_STALE_EVENT_DISABLE;
945 			msm_write(port, val, UART_CR);
946 			val = UART_CR_CMD_RESET_STALE_INT;
947 			msm_write(port, val, UART_CR);
948 			/*
949 			 * Flush DMA input fifo to memory, this will also
950 			 * trigger DMA RX completion
951 			 */
952 			dmaengine_terminate_all(dma->chan);
953 		} else if (msm_port->is_uartdm) {
954 			msm_handle_rx_dm(port, misr);
955 		} else {
956 			msm_handle_rx(port);
957 		}
958 	}
959 	if (misr & UART_IMR_TXLEV)
960 		msm_handle_tx(port);
961 	if (misr & UART_IMR_DELTA_CTS)
962 		msm_handle_delta_cts(port);
963 
964 	msm_write(port, msm_port->imr, UART_IMR); /* restore interrupt */
965 	spin_unlock_irqrestore(&port->lock, flags);
966 
967 	return IRQ_HANDLED;
968 }
969 
970 static unsigned int msm_tx_empty(struct uart_port *port)
971 {
972 	return (msm_read(port, UART_SR) & UART_SR_TX_EMPTY) ? TIOCSER_TEMT : 0;
973 }
974 
975 static unsigned int msm_get_mctrl(struct uart_port *port)
976 {
977 	return TIOCM_CAR | TIOCM_CTS | TIOCM_DSR | TIOCM_RTS;
978 }
979 
980 static void msm_reset(struct uart_port *port)
981 {
982 	struct msm_port *msm_port = UART_TO_MSM(port);
983 
984 	/* reset everything */
985 	msm_write(port, UART_CR_CMD_RESET_RX, UART_CR);
986 	msm_write(port, UART_CR_CMD_RESET_TX, UART_CR);
987 	msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
988 	msm_write(port, UART_CR_CMD_RESET_BREAK_INT, UART_CR);
989 	msm_write(port, UART_CR_CMD_RESET_CTS, UART_CR);
990 	msm_write(port, UART_CR_CMD_SET_RFR, UART_CR);
991 
992 	/* Disable DM modes */
993 	if (msm_port->is_uartdm)
994 		msm_write(port, 0, UARTDM_DMEN);
995 }
996 
997 static void msm_set_mctrl(struct uart_port *port, unsigned int mctrl)
998 {
999 	unsigned int mr;
1000 
1001 	mr = msm_read(port, UART_MR1);
1002 
1003 	if (!(mctrl & TIOCM_RTS)) {
1004 		mr &= ~UART_MR1_RX_RDY_CTL;
1005 		msm_write(port, mr, UART_MR1);
1006 		msm_write(port, UART_CR_CMD_RESET_RFR, UART_CR);
1007 	} else {
1008 		mr |= UART_MR1_RX_RDY_CTL;
1009 		msm_write(port, mr, UART_MR1);
1010 	}
1011 }
1012 
1013 static void msm_break_ctl(struct uart_port *port, int break_ctl)
1014 {
1015 	if (break_ctl)
1016 		msm_write(port, UART_CR_CMD_START_BREAK, UART_CR);
1017 	else
1018 		msm_write(port, UART_CR_CMD_STOP_BREAK, UART_CR);
1019 }
1020 
1021 struct msm_baud_map {
1022 	u16	divisor;
1023 	u8	code;
1024 	u8	rxstale;
1025 };
1026 
1027 static const struct msm_baud_map *
1028 msm_find_best_baud(struct uart_port *port, unsigned int baud,
1029 		   unsigned long *rate)
1030 {
1031 	struct msm_port *msm_port = UART_TO_MSM(port);
1032 	unsigned int divisor, result;
1033 	unsigned long target, old, best_rate = 0, diff, best_diff = ULONG_MAX;
1034 	const struct msm_baud_map *entry, *end, *best;
1035 	static const struct msm_baud_map table[] = {
1036 		{    1, 0xff, 31 },
1037 		{    2, 0xee, 16 },
1038 		{    3, 0xdd,  8 },
1039 		{    4, 0xcc,  6 },
1040 		{    6, 0xbb,  6 },
1041 		{    8, 0xaa,  6 },
1042 		{   12, 0x99,  6 },
1043 		{   16, 0x88,  1 },
1044 		{   24, 0x77,  1 },
1045 		{   32, 0x66,  1 },
1046 		{   48, 0x55,  1 },
1047 		{   96, 0x44,  1 },
1048 		{  192, 0x33,  1 },
1049 		{  384, 0x22,  1 },
1050 		{  768, 0x11,  1 },
1051 		{ 1536, 0x00,  1 },
1052 	};
1053 
1054 	best = table; /* Default to smallest divider */
1055 	target = clk_round_rate(msm_port->clk, 16 * baud);
1056 	divisor = DIV_ROUND_CLOSEST(target, 16 * baud);
1057 
1058 	end = table + ARRAY_SIZE(table);
1059 	entry = table;
1060 	while (entry < end) {
1061 		if (entry->divisor <= divisor) {
1062 			result = target / entry->divisor / 16;
1063 			diff = abs(result - baud);
1064 
1065 			/* Keep track of best entry */
1066 			if (diff < best_diff) {
1067 				best_diff = diff;
1068 				best = entry;
1069 				best_rate = target;
1070 			}
1071 
1072 			if (result == baud)
1073 				break;
1074 		} else if (entry->divisor > divisor) {
1075 			old = target;
1076 			target = clk_round_rate(msm_port->clk, old + 1);
1077 			/*
1078 			 * The rate didn't get any faster so we can't do
1079 			 * better at dividing it down
1080 			 */
1081 			if (target == old)
1082 				break;
1083 
1084 			/* Start the divisor search over at this new rate */
1085 			entry = table;
1086 			divisor = DIV_ROUND_CLOSEST(target, 16 * baud);
1087 			continue;
1088 		}
1089 		entry++;
1090 	}
1091 
1092 	*rate = best_rate;
1093 	return best;
1094 }
1095 
1096 static int msm_set_baud_rate(struct uart_port *port, unsigned int baud,
1097 			     unsigned long *saved_flags)
1098 {
1099 	unsigned int rxstale, watermark, mask;
1100 	struct msm_port *msm_port = UART_TO_MSM(port);
1101 	const struct msm_baud_map *entry;
1102 	unsigned long flags, rate;
1103 
1104 	flags = *saved_flags;
1105 	spin_unlock_irqrestore(&port->lock, flags);
1106 
1107 	entry = msm_find_best_baud(port, baud, &rate);
1108 	clk_set_rate(msm_port->clk, rate);
1109 	baud = rate / 16 / entry->divisor;
1110 
1111 	spin_lock_irqsave(&port->lock, flags);
1112 	*saved_flags = flags;
1113 	port->uartclk = rate;
1114 
1115 	msm_write(port, entry->code, UART_CSR);
1116 
1117 	/* RX stale watermark */
1118 	rxstale = entry->rxstale;
1119 	watermark = UART_IPR_STALE_LSB & rxstale;
1120 	if (msm_port->is_uartdm) {
1121 		mask = UART_DM_IPR_STALE_TIMEOUT_MSB;
1122 	} else {
1123 		watermark |= UART_IPR_RXSTALE_LAST;
1124 		mask = UART_IPR_STALE_TIMEOUT_MSB;
1125 	}
1126 
1127 	watermark |= mask & (rxstale << 2);
1128 
1129 	msm_write(port, watermark, UART_IPR);
1130 
1131 	/* set RX watermark */
1132 	watermark = (port->fifosize * 3) / 4;
1133 	msm_write(port, watermark, UART_RFWR);
1134 
1135 	/* set TX watermark */
1136 	msm_write(port, 10, UART_TFWR);
1137 
1138 	msm_write(port, UART_CR_CMD_PROTECTION_EN, UART_CR);
1139 	msm_reset(port);
1140 
1141 	/* Enable RX and TX */
1142 	msm_write(port, UART_CR_TX_ENABLE | UART_CR_RX_ENABLE, UART_CR);
1143 
1144 	/* turn on RX and CTS interrupts */
1145 	msm_port->imr = UART_IMR_RXLEV | UART_IMR_RXSTALE |
1146 			UART_IMR_CURRENT_CTS | UART_IMR_RXBREAK_START;
1147 
1148 	msm_write(port, msm_port->imr, UART_IMR);
1149 
1150 	if (msm_port->is_uartdm) {
1151 		msm_write(port, UART_CR_CMD_RESET_STALE_INT, UART_CR);
1152 		msm_write(port, 0xFFFFFF, UARTDM_DMRX);
1153 		msm_write(port, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
1154 	}
1155 
1156 	return baud;
1157 }
1158 
1159 static void msm_init_clock(struct uart_port *port)
1160 {
1161 	struct msm_port *msm_port = UART_TO_MSM(port);
1162 
1163 	clk_prepare_enable(msm_port->clk);
1164 	clk_prepare_enable(msm_port->pclk);
1165 	msm_serial_set_mnd_regs(port);
1166 }
1167 
1168 static int msm_startup(struct uart_port *port)
1169 {
1170 	struct msm_port *msm_port = UART_TO_MSM(port);
1171 	unsigned int data, rfr_level, mask;
1172 	int ret;
1173 
1174 	snprintf(msm_port->name, sizeof(msm_port->name),
1175 		 "msm_serial%d", port->line);
1176 
1177 	msm_init_clock(port);
1178 
1179 	if (likely(port->fifosize > 12))
1180 		rfr_level = port->fifosize - 12;
1181 	else
1182 		rfr_level = port->fifosize;
1183 
1184 	/* set automatic RFR level */
1185 	data = msm_read(port, UART_MR1);
1186 
1187 	if (msm_port->is_uartdm)
1188 		mask = UART_DM_MR1_AUTO_RFR_LEVEL1;
1189 	else
1190 		mask = UART_MR1_AUTO_RFR_LEVEL1;
1191 
1192 	data &= ~mask;
1193 	data &= ~UART_MR1_AUTO_RFR_LEVEL0;
1194 	data |= mask & (rfr_level << 2);
1195 	data |= UART_MR1_AUTO_RFR_LEVEL0 & rfr_level;
1196 	msm_write(port, data, UART_MR1);
1197 
1198 	if (msm_port->is_uartdm) {
1199 		msm_request_tx_dma(msm_port, msm_port->uart.mapbase);
1200 		msm_request_rx_dma(msm_port, msm_port->uart.mapbase);
1201 	}
1202 
1203 	ret = request_irq(port->irq, msm_uart_irq, IRQF_TRIGGER_HIGH,
1204 			  msm_port->name, port);
1205 	if (unlikely(ret))
1206 		goto err_irq;
1207 
1208 	return 0;
1209 
1210 err_irq:
1211 	if (msm_port->is_uartdm)
1212 		msm_release_dma(msm_port);
1213 
1214 	clk_disable_unprepare(msm_port->pclk);
1215 	clk_disable_unprepare(msm_port->clk);
1216 
1217 	return ret;
1218 }
1219 
1220 static void msm_shutdown(struct uart_port *port)
1221 {
1222 	struct msm_port *msm_port = UART_TO_MSM(port);
1223 
1224 	msm_port->imr = 0;
1225 	msm_write(port, 0, UART_IMR); /* disable interrupts */
1226 
1227 	if (msm_port->is_uartdm)
1228 		msm_release_dma(msm_port);
1229 
1230 	clk_disable_unprepare(msm_port->clk);
1231 
1232 	free_irq(port->irq, port);
1233 }
1234 
1235 static void msm_set_termios(struct uart_port *port, struct ktermios *termios,
1236 			    struct ktermios *old)
1237 {
1238 	struct msm_port *msm_port = UART_TO_MSM(port);
1239 	struct msm_dma *dma = &msm_port->rx_dma;
1240 	unsigned long flags;
1241 	unsigned int baud, mr;
1242 
1243 	spin_lock_irqsave(&port->lock, flags);
1244 
1245 	if (dma->chan) /* Terminate if any */
1246 		msm_stop_dma(port, dma);
1247 
1248 	/* calculate and set baud rate */
1249 	baud = uart_get_baud_rate(port, termios, old, 300, 4000000);
1250 	baud = msm_set_baud_rate(port, baud, &flags);
1251 	if (tty_termios_baud_rate(termios))
1252 		tty_termios_encode_baud_rate(termios, baud, baud);
1253 
1254 	/* calculate parity */
1255 	mr = msm_read(port, UART_MR2);
1256 	mr &= ~UART_MR2_PARITY_MODE;
1257 	if (termios->c_cflag & PARENB) {
1258 		if (termios->c_cflag & PARODD)
1259 			mr |= UART_MR2_PARITY_MODE_ODD;
1260 		else if (termios->c_cflag & CMSPAR)
1261 			mr |= UART_MR2_PARITY_MODE_SPACE;
1262 		else
1263 			mr |= UART_MR2_PARITY_MODE_EVEN;
1264 	}
1265 
1266 	/* calculate bits per char */
1267 	mr &= ~UART_MR2_BITS_PER_CHAR;
1268 	switch (termios->c_cflag & CSIZE) {
1269 	case CS5:
1270 		mr |= UART_MR2_BITS_PER_CHAR_5;
1271 		break;
1272 	case CS6:
1273 		mr |= UART_MR2_BITS_PER_CHAR_6;
1274 		break;
1275 	case CS7:
1276 		mr |= UART_MR2_BITS_PER_CHAR_7;
1277 		break;
1278 	case CS8:
1279 	default:
1280 		mr |= UART_MR2_BITS_PER_CHAR_8;
1281 		break;
1282 	}
1283 
1284 	/* calculate stop bits */
1285 	mr &= ~(UART_MR2_STOP_BIT_LEN_ONE | UART_MR2_STOP_BIT_LEN_TWO);
1286 	if (termios->c_cflag & CSTOPB)
1287 		mr |= UART_MR2_STOP_BIT_LEN_TWO;
1288 	else
1289 		mr |= UART_MR2_STOP_BIT_LEN_ONE;
1290 
1291 	/* set parity, bits per char, and stop bit */
1292 	msm_write(port, mr, UART_MR2);
1293 
1294 	/* calculate and set hardware flow control */
1295 	mr = msm_read(port, UART_MR1);
1296 	mr &= ~(UART_MR1_CTS_CTL | UART_MR1_RX_RDY_CTL);
1297 	if (termios->c_cflag & CRTSCTS) {
1298 		mr |= UART_MR1_CTS_CTL;
1299 		mr |= UART_MR1_RX_RDY_CTL;
1300 	}
1301 	msm_write(port, mr, UART_MR1);
1302 
1303 	/* Configure status bits to ignore based on termio flags. */
1304 	port->read_status_mask = 0;
1305 	if (termios->c_iflag & INPCK)
1306 		port->read_status_mask |= UART_SR_PAR_FRAME_ERR;
1307 	if (termios->c_iflag & (IGNBRK | BRKINT | PARMRK))
1308 		port->read_status_mask |= UART_SR_RX_BREAK;
1309 
1310 	uart_update_timeout(port, termios->c_cflag, baud);
1311 
1312 	/* Try to use DMA */
1313 	msm_start_rx_dma(msm_port);
1314 
1315 	spin_unlock_irqrestore(&port->lock, flags);
1316 }
1317 
1318 static const char *msm_type(struct uart_port *port)
1319 {
1320 	return "MSM";
1321 }
1322 
1323 static void msm_release_port(struct uart_port *port)
1324 {
1325 	struct platform_device *pdev = to_platform_device(port->dev);
1326 	struct resource *uart_resource;
1327 	resource_size_t size;
1328 
1329 	uart_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1330 	if (unlikely(!uart_resource))
1331 		return;
1332 	size = resource_size(uart_resource);
1333 
1334 	release_mem_region(port->mapbase, size);
1335 	iounmap(port->membase);
1336 	port->membase = NULL;
1337 }
1338 
1339 static int msm_request_port(struct uart_port *port)
1340 {
1341 	struct platform_device *pdev = to_platform_device(port->dev);
1342 	struct resource *uart_resource;
1343 	resource_size_t size;
1344 	int ret;
1345 
1346 	uart_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1347 	if (unlikely(!uart_resource))
1348 		return -ENXIO;
1349 
1350 	size = resource_size(uart_resource);
1351 
1352 	if (!request_mem_region(port->mapbase, size, "msm_serial"))
1353 		return -EBUSY;
1354 
1355 	port->membase = ioremap(port->mapbase, size);
1356 	if (!port->membase) {
1357 		ret = -EBUSY;
1358 		goto fail_release_port;
1359 	}
1360 
1361 	return 0;
1362 
1363 fail_release_port:
1364 	release_mem_region(port->mapbase, size);
1365 	return ret;
1366 }
1367 
1368 static void msm_config_port(struct uart_port *port, int flags)
1369 {
1370 	int ret;
1371 
1372 	if (flags & UART_CONFIG_TYPE) {
1373 		port->type = PORT_MSM;
1374 		ret = msm_request_port(port);
1375 		if (ret)
1376 			return;
1377 	}
1378 }
1379 
1380 static int msm_verify_port(struct uart_port *port, struct serial_struct *ser)
1381 {
1382 	if (unlikely(ser->type != PORT_UNKNOWN && ser->type != PORT_MSM))
1383 		return -EINVAL;
1384 	if (unlikely(port->irq != ser->irq))
1385 		return -EINVAL;
1386 	return 0;
1387 }
1388 
1389 static void msm_power(struct uart_port *port, unsigned int state,
1390 		      unsigned int oldstate)
1391 {
1392 	struct msm_port *msm_port = UART_TO_MSM(port);
1393 
1394 	switch (state) {
1395 	case 0:
1396 		clk_prepare_enable(msm_port->clk);
1397 		clk_prepare_enable(msm_port->pclk);
1398 		break;
1399 	case 3:
1400 		clk_disable_unprepare(msm_port->clk);
1401 		clk_disable_unprepare(msm_port->pclk);
1402 		break;
1403 	default:
1404 		pr_err("msm_serial: Unknown PM state %d\n", state);
1405 	}
1406 }
1407 
1408 #ifdef CONFIG_CONSOLE_POLL
1409 static int msm_poll_get_char_single(struct uart_port *port)
1410 {
1411 	struct msm_port *msm_port = UART_TO_MSM(port);
1412 	unsigned int rf_reg = msm_port->is_uartdm ? UARTDM_RF : UART_RF;
1413 
1414 	if (!(msm_read(port, UART_SR) & UART_SR_RX_READY))
1415 		return NO_POLL_CHAR;
1416 
1417 	return msm_read(port, rf_reg) & 0xff;
1418 }
1419 
1420 static int msm_poll_get_char_dm(struct uart_port *port)
1421 {
1422 	int c;
1423 	static u32 slop;
1424 	static int count;
1425 	unsigned char *sp = (unsigned char *)&slop;
1426 
1427 	/* Check if a previous read had more than one char */
1428 	if (count) {
1429 		c = sp[sizeof(slop) - count];
1430 		count--;
1431 	/* Or if FIFO is empty */
1432 	} else if (!(msm_read(port, UART_SR) & UART_SR_RX_READY)) {
1433 		/*
1434 		 * If RX packing buffer has less than a word, force stale to
1435 		 * push contents into RX FIFO
1436 		 */
1437 		count = msm_read(port, UARTDM_RXFS);
1438 		count = (count >> UARTDM_RXFS_BUF_SHIFT) & UARTDM_RXFS_BUF_MASK;
1439 		if (count) {
1440 			msm_write(port, UART_CR_CMD_FORCE_STALE, UART_CR);
1441 			slop = msm_read(port, UARTDM_RF);
1442 			c = sp[0];
1443 			count--;
1444 			msm_write(port, UART_CR_CMD_RESET_STALE_INT, UART_CR);
1445 			msm_write(port, 0xFFFFFF, UARTDM_DMRX);
1446 			msm_write(port, UART_CR_CMD_STALE_EVENT_ENABLE,
1447 				  UART_CR);
1448 		} else {
1449 			c = NO_POLL_CHAR;
1450 		}
1451 	/* FIFO has a word */
1452 	} else {
1453 		slop = msm_read(port, UARTDM_RF);
1454 		c = sp[0];
1455 		count = sizeof(slop) - 1;
1456 	}
1457 
1458 	return c;
1459 }
1460 
1461 static int msm_poll_get_char(struct uart_port *port)
1462 {
1463 	u32 imr;
1464 	int c;
1465 	struct msm_port *msm_port = UART_TO_MSM(port);
1466 
1467 	/* Disable all interrupts */
1468 	imr = msm_read(port, UART_IMR);
1469 	msm_write(port, 0, UART_IMR);
1470 
1471 	if (msm_port->is_uartdm)
1472 		c = msm_poll_get_char_dm(port);
1473 	else
1474 		c = msm_poll_get_char_single(port);
1475 
1476 	/* Enable interrupts */
1477 	msm_write(port, imr, UART_IMR);
1478 
1479 	return c;
1480 }
1481 
1482 static void msm_poll_put_char(struct uart_port *port, unsigned char c)
1483 {
1484 	u32 imr;
1485 	struct msm_port *msm_port = UART_TO_MSM(port);
1486 
1487 	/* Disable all interrupts */
1488 	imr = msm_read(port, UART_IMR);
1489 	msm_write(port, 0, UART_IMR);
1490 
1491 	if (msm_port->is_uartdm)
1492 		msm_reset_dm_count(port, 1);
1493 
1494 	/* Wait until FIFO is empty */
1495 	while (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
1496 		cpu_relax();
1497 
1498 	/* Write a character */
1499 	msm_write(port, c, msm_port->is_uartdm ? UARTDM_TF : UART_TF);
1500 
1501 	/* Wait until FIFO is empty */
1502 	while (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
1503 		cpu_relax();
1504 
1505 	/* Enable interrupts */
1506 	msm_write(port, imr, UART_IMR);
1507 }
1508 #endif
1509 
1510 static struct uart_ops msm_uart_pops = {
1511 	.tx_empty = msm_tx_empty,
1512 	.set_mctrl = msm_set_mctrl,
1513 	.get_mctrl = msm_get_mctrl,
1514 	.stop_tx = msm_stop_tx,
1515 	.start_tx = msm_start_tx,
1516 	.stop_rx = msm_stop_rx,
1517 	.enable_ms = msm_enable_ms,
1518 	.break_ctl = msm_break_ctl,
1519 	.startup = msm_startup,
1520 	.shutdown = msm_shutdown,
1521 	.set_termios = msm_set_termios,
1522 	.type = msm_type,
1523 	.release_port = msm_release_port,
1524 	.request_port = msm_request_port,
1525 	.config_port = msm_config_port,
1526 	.verify_port = msm_verify_port,
1527 	.pm = msm_power,
1528 #ifdef CONFIG_CONSOLE_POLL
1529 	.poll_get_char	= msm_poll_get_char,
1530 	.poll_put_char	= msm_poll_put_char,
1531 #endif
1532 };
1533 
1534 static struct msm_port msm_uart_ports[] = {
1535 	{
1536 		.uart = {
1537 			.iotype = UPIO_MEM,
1538 			.ops = &msm_uart_pops,
1539 			.flags = UPF_BOOT_AUTOCONF,
1540 			.fifosize = 64,
1541 			.line = 0,
1542 		},
1543 	},
1544 	{
1545 		.uart = {
1546 			.iotype = UPIO_MEM,
1547 			.ops = &msm_uart_pops,
1548 			.flags = UPF_BOOT_AUTOCONF,
1549 			.fifosize = 64,
1550 			.line = 1,
1551 		},
1552 	},
1553 	{
1554 		.uart = {
1555 			.iotype = UPIO_MEM,
1556 			.ops = &msm_uart_pops,
1557 			.flags = UPF_BOOT_AUTOCONF,
1558 			.fifosize = 64,
1559 			.line = 2,
1560 		},
1561 	},
1562 };
1563 
1564 #define UART_NR	ARRAY_SIZE(msm_uart_ports)
1565 
1566 static inline struct uart_port *msm_get_port_from_line(unsigned int line)
1567 {
1568 	return &msm_uart_ports[line].uart;
1569 }
1570 
1571 #ifdef CONFIG_SERIAL_MSM_CONSOLE
1572 static void __msm_console_write(struct uart_port *port, const char *s,
1573 				unsigned int count, bool is_uartdm)
1574 {
1575 	int i;
1576 	int num_newlines = 0;
1577 	bool replaced = false;
1578 	void __iomem *tf;
1579 
1580 	if (is_uartdm)
1581 		tf = port->membase + UARTDM_TF;
1582 	else
1583 		tf = port->membase + UART_TF;
1584 
1585 	/* Account for newlines that will get a carriage return added */
1586 	for (i = 0; i < count; i++)
1587 		if (s[i] == '\n')
1588 			num_newlines++;
1589 	count += num_newlines;
1590 
1591 	spin_lock(&port->lock);
1592 	if (is_uartdm)
1593 		msm_reset_dm_count(port, count);
1594 
1595 	i = 0;
1596 	while (i < count) {
1597 		int j;
1598 		unsigned int num_chars;
1599 		char buf[4] = { 0 };
1600 
1601 		if (is_uartdm)
1602 			num_chars = min(count - i, (unsigned int)sizeof(buf));
1603 		else
1604 			num_chars = 1;
1605 
1606 		for (j = 0; j < num_chars; j++) {
1607 			char c = *s;
1608 
1609 			if (c == '\n' && !replaced) {
1610 				buf[j] = '\r';
1611 				j++;
1612 				replaced = true;
1613 			}
1614 			if (j < num_chars) {
1615 				buf[j] = c;
1616 				s++;
1617 				replaced = false;
1618 			}
1619 		}
1620 
1621 		while (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
1622 			cpu_relax();
1623 
1624 		iowrite32_rep(tf, buf, 1);
1625 		i += num_chars;
1626 	}
1627 	spin_unlock(&port->lock);
1628 }
1629 
1630 static void msm_console_write(struct console *co, const char *s,
1631 			      unsigned int count)
1632 {
1633 	struct uart_port *port;
1634 	struct msm_port *msm_port;
1635 
1636 	BUG_ON(co->index < 0 || co->index >= UART_NR);
1637 
1638 	port = msm_get_port_from_line(co->index);
1639 	msm_port = UART_TO_MSM(port);
1640 
1641 	__msm_console_write(port, s, count, msm_port->is_uartdm);
1642 }
1643 
1644 static int msm_console_setup(struct console *co, char *options)
1645 {
1646 	struct uart_port *port;
1647 	int baud = 115200;
1648 	int bits = 8;
1649 	int parity = 'n';
1650 	int flow = 'n';
1651 
1652 	if (unlikely(co->index >= UART_NR || co->index < 0))
1653 		return -ENXIO;
1654 
1655 	port = msm_get_port_from_line(co->index);
1656 
1657 	if (unlikely(!port->membase))
1658 		return -ENXIO;
1659 
1660 	msm_init_clock(port);
1661 
1662 	if (options)
1663 		uart_parse_options(options, &baud, &parity, &bits, &flow);
1664 
1665 	pr_info("msm_serial: console setup on port #%d\n", port->line);
1666 
1667 	return uart_set_options(port, co, baud, parity, bits, flow);
1668 }
1669 
1670 static void
1671 msm_serial_early_write(struct console *con, const char *s, unsigned n)
1672 {
1673 	struct earlycon_device *dev = con->data;
1674 
1675 	__msm_console_write(&dev->port, s, n, false);
1676 }
1677 
1678 static int __init
1679 msm_serial_early_console_setup(struct earlycon_device *device, const char *opt)
1680 {
1681 	if (!device->port.membase)
1682 		return -ENODEV;
1683 
1684 	device->con->write = msm_serial_early_write;
1685 	return 0;
1686 }
1687 OF_EARLYCON_DECLARE(msm_serial, "qcom,msm-uart",
1688 		    msm_serial_early_console_setup);
1689 
1690 static void
1691 msm_serial_early_write_dm(struct console *con, const char *s, unsigned n)
1692 {
1693 	struct earlycon_device *dev = con->data;
1694 
1695 	__msm_console_write(&dev->port, s, n, true);
1696 }
1697 
1698 static int __init
1699 msm_serial_early_console_setup_dm(struct earlycon_device *device,
1700 				  const char *opt)
1701 {
1702 	if (!device->port.membase)
1703 		return -ENODEV;
1704 
1705 	device->con->write = msm_serial_early_write_dm;
1706 	return 0;
1707 }
1708 OF_EARLYCON_DECLARE(msm_serial_dm, "qcom,msm-uartdm",
1709 		    msm_serial_early_console_setup_dm);
1710 
1711 static struct uart_driver msm_uart_driver;
1712 
1713 static struct console msm_console = {
1714 	.name = "ttyMSM",
1715 	.write = msm_console_write,
1716 	.device = uart_console_device,
1717 	.setup = msm_console_setup,
1718 	.flags = CON_PRINTBUFFER,
1719 	.index = -1,
1720 	.data = &msm_uart_driver,
1721 };
1722 
1723 #define MSM_CONSOLE	(&msm_console)
1724 
1725 #else
1726 #define MSM_CONSOLE	NULL
1727 #endif
1728 
1729 static struct uart_driver msm_uart_driver = {
1730 	.owner = THIS_MODULE,
1731 	.driver_name = "msm_serial",
1732 	.dev_name = "ttyMSM",
1733 	.nr = UART_NR,
1734 	.cons = MSM_CONSOLE,
1735 };
1736 
1737 static atomic_t msm_uart_next_id = ATOMIC_INIT(0);
1738 
1739 static const struct of_device_id msm_uartdm_table[] = {
1740 	{ .compatible = "qcom,msm-uartdm-v1.1", .data = (void *)UARTDM_1P1 },
1741 	{ .compatible = "qcom,msm-uartdm-v1.2", .data = (void *)UARTDM_1P2 },
1742 	{ .compatible = "qcom,msm-uartdm-v1.3", .data = (void *)UARTDM_1P3 },
1743 	{ .compatible = "qcom,msm-uartdm-v1.4", .data = (void *)UARTDM_1P4 },
1744 	{ }
1745 };
1746 
1747 static int msm_serial_probe(struct platform_device *pdev)
1748 {
1749 	struct msm_port *msm_port;
1750 	struct resource *resource;
1751 	struct uart_port *port;
1752 	const struct of_device_id *id;
1753 	int irq, line;
1754 
1755 	if (pdev->dev.of_node)
1756 		line = of_alias_get_id(pdev->dev.of_node, "serial");
1757 	else
1758 		line = pdev->id;
1759 
1760 	if (line < 0)
1761 		line = atomic_inc_return(&msm_uart_next_id) - 1;
1762 
1763 	if (unlikely(line < 0 || line >= UART_NR))
1764 		return -ENXIO;
1765 
1766 	dev_info(&pdev->dev, "msm_serial: detected port #%d\n", line);
1767 
1768 	port = msm_get_port_from_line(line);
1769 	port->dev = &pdev->dev;
1770 	msm_port = UART_TO_MSM(port);
1771 
1772 	id = of_match_device(msm_uartdm_table, &pdev->dev);
1773 	if (id)
1774 		msm_port->is_uartdm = (unsigned long)id->data;
1775 	else
1776 		msm_port->is_uartdm = 0;
1777 
1778 	msm_port->clk = devm_clk_get(&pdev->dev, "core");
1779 	if (IS_ERR(msm_port->clk))
1780 		return PTR_ERR(msm_port->clk);
1781 
1782 	if (msm_port->is_uartdm) {
1783 		msm_port->pclk = devm_clk_get(&pdev->dev, "iface");
1784 		if (IS_ERR(msm_port->pclk))
1785 			return PTR_ERR(msm_port->pclk);
1786 	}
1787 
1788 	port->uartclk = clk_get_rate(msm_port->clk);
1789 	dev_info(&pdev->dev, "uartclk = %d\n", port->uartclk);
1790 
1791 	resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1792 	if (unlikely(!resource))
1793 		return -ENXIO;
1794 	port->mapbase = resource->start;
1795 
1796 	irq = platform_get_irq(pdev, 0);
1797 	if (unlikely(irq < 0))
1798 		return -ENXIO;
1799 	port->irq = irq;
1800 
1801 	platform_set_drvdata(pdev, port);
1802 
1803 	return uart_add_one_port(&msm_uart_driver, port);
1804 }
1805 
1806 static int msm_serial_remove(struct platform_device *pdev)
1807 {
1808 	struct uart_port *port = platform_get_drvdata(pdev);
1809 
1810 	uart_remove_one_port(&msm_uart_driver, port);
1811 
1812 	return 0;
1813 }
1814 
1815 static const struct of_device_id msm_match_table[] = {
1816 	{ .compatible = "qcom,msm-uart" },
1817 	{ .compatible = "qcom,msm-uartdm" },
1818 	{}
1819 };
1820 MODULE_DEVICE_TABLE(of, msm_match_table);
1821 
1822 static int __maybe_unused msm_serial_suspend(struct device *dev)
1823 {
1824 	struct msm_port *port = dev_get_drvdata(dev);
1825 
1826 	uart_suspend_port(&msm_uart_driver, &port->uart);
1827 
1828 	return 0;
1829 }
1830 
1831 static int __maybe_unused msm_serial_resume(struct device *dev)
1832 {
1833 	struct msm_port *port = dev_get_drvdata(dev);
1834 
1835 	uart_resume_port(&msm_uart_driver, &port->uart);
1836 
1837 	return 0;
1838 }
1839 
1840 static const struct dev_pm_ops msm_serial_dev_pm_ops = {
1841 	SET_SYSTEM_SLEEP_PM_OPS(msm_serial_suspend, msm_serial_resume)
1842 };
1843 
1844 static struct platform_driver msm_platform_driver = {
1845 	.remove = msm_serial_remove,
1846 	.probe = msm_serial_probe,
1847 	.driver = {
1848 		.name = "msm_serial",
1849 		.pm = &msm_serial_dev_pm_ops,
1850 		.of_match_table = msm_match_table,
1851 	},
1852 };
1853 
1854 static int __init msm_serial_init(void)
1855 {
1856 	int ret;
1857 
1858 	ret = uart_register_driver(&msm_uart_driver);
1859 	if (unlikely(ret))
1860 		return ret;
1861 
1862 	ret = platform_driver_register(&msm_platform_driver);
1863 	if (unlikely(ret))
1864 		uart_unregister_driver(&msm_uart_driver);
1865 
1866 	pr_info("msm_serial: driver initialized\n");
1867 
1868 	return ret;
1869 }
1870 
1871 static void __exit msm_serial_exit(void)
1872 {
1873 	platform_driver_unregister(&msm_platform_driver);
1874 	uart_unregister_driver(&msm_uart_driver);
1875 }
1876 
1877 module_init(msm_serial_init);
1878 module_exit(msm_serial_exit);
1879 
1880 MODULE_AUTHOR("Robert Love <rlove@google.com>");
1881 MODULE_DESCRIPTION("Driver for msm7x serial device");
1882 MODULE_LICENSE("GPL");
1883