xref: /openbmc/linux/drivers/tty/serial/sprd_serial.c (revision a1aef488) 
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2012-2015 Spreadtrum Communications Inc.
4  */
5 
6 #if defined(CONFIG_SERIAL_SPRD_CONSOLE) && defined(CONFIG_MAGIC_SYSRQ)
7 #define SUPPORT_SYSRQ
8 #endif
9 
10 #include <linux/clk.h>
11 #include <linux/console.h>
12 #include <linux/delay.h>
13 #include <linux/dmaengine.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/dma/sprd-dma.h>
16 #include <linux/io.h>
17 #include <linux/ioport.h>
18 #include <linux/kernel.h>
19 #include <linux/module.h>
20 #include <linux/of.h>
21 #include <linux/platform_device.h>
22 #include <linux/serial_core.h>
23 #include <linux/serial.h>
24 #include <linux/slab.h>
25 #include <linux/tty.h>
26 #include <linux/tty_flip.h>
27 
28 /* device name */
29 #define UART_NR_MAX		8
30 #define SPRD_TTY_NAME		"ttyS"
31 #define SPRD_FIFO_SIZE		128
32 #define SPRD_DEF_RATE		26000000
33 #define SPRD_BAUD_IO_LIMIT	3000000
34 #define SPRD_TIMEOUT		256000
35 
36 /* the offset of serial registers and BITs for them */
37 /* data registers */
38 #define SPRD_TXD		0x0000
39 #define SPRD_RXD		0x0004
40 
41 /* line status register and its BITs  */
42 #define SPRD_LSR		0x0008
43 #define SPRD_LSR_OE		BIT(4)
44 #define SPRD_LSR_FE		BIT(3)
45 #define SPRD_LSR_PE		BIT(2)
46 #define SPRD_LSR_BI		BIT(7)
47 #define SPRD_LSR_TX_OVER	BIT(15)
48 
49 /* data number in TX and RX fifo */
50 #define SPRD_STS1		0x000C
51 #define SPRD_RX_FIFO_CNT_MASK	GENMASK(7, 0)
52 #define SPRD_TX_FIFO_CNT_MASK	GENMASK(15, 8)
53 
54 /* interrupt enable register and its BITs */
55 #define SPRD_IEN		0x0010
56 #define SPRD_IEN_RX_FULL	BIT(0)
57 #define SPRD_IEN_TX_EMPTY	BIT(1)
58 #define SPRD_IEN_BREAK_DETECT	BIT(7)
59 #define SPRD_IEN_TIMEOUT	BIT(13)
60 
61 /* interrupt clear register */
62 #define SPRD_ICLR		0x0014
63 #define SPRD_ICLR_TIMEOUT	BIT(13)
64 
65 /* line control register */
66 #define SPRD_LCR		0x0018
67 #define SPRD_LCR_STOP_1BIT	0x10
68 #define SPRD_LCR_STOP_2BIT	0x30
69 #define SPRD_LCR_DATA_LEN	(BIT(2) | BIT(3))
70 #define SPRD_LCR_DATA_LEN5	0x0
71 #define SPRD_LCR_DATA_LEN6	0x4
72 #define SPRD_LCR_DATA_LEN7	0x8
73 #define SPRD_LCR_DATA_LEN8	0xc
74 #define SPRD_LCR_PARITY		(BIT(0) | BIT(1))
75 #define SPRD_LCR_PARITY_EN	0x2
76 #define SPRD_LCR_EVEN_PAR	0x0
77 #define SPRD_LCR_ODD_PAR	0x1
78 
79 /* control register 1 */
80 #define SPRD_CTL1		0x001C
81 #define SPRD_DMA_EN		BIT(15)
82 #define SPRD_LOOPBACK_EN	BIT(14)
83 #define RX_HW_FLOW_CTL_THLD	BIT(6)
84 #define RX_HW_FLOW_CTL_EN	BIT(7)
85 #define TX_HW_FLOW_CTL_EN	BIT(8)
86 #define RX_TOUT_THLD_DEF	0x3E00
87 #define RX_HFC_THLD_DEF		0x40
88 
89 /* fifo threshold register */
90 #define SPRD_CTL2		0x0020
91 #define THLD_TX_EMPTY		0x40
92 #define THLD_TX_EMPTY_SHIFT	8
93 #define THLD_RX_FULL		0x40
94 #define THLD_RX_FULL_MASK	GENMASK(6, 0)
95 
96 /* config baud rate register */
97 #define SPRD_CLKD0		0x0024
98 #define SPRD_CLKD0_MASK		GENMASK(15, 0)
99 #define SPRD_CLKD1		0x0028
100 #define SPRD_CLKD1_MASK		GENMASK(20, 16)
101 #define SPRD_CLKD1_SHIFT	16
102 
103 /* interrupt mask status register */
104 #define SPRD_IMSR		0x002C
105 #define SPRD_IMSR_RX_FIFO_FULL	BIT(0)
106 #define SPRD_IMSR_TX_FIFO_EMPTY	BIT(1)
107 #define SPRD_IMSR_BREAK_DETECT	BIT(7)
108 #define SPRD_IMSR_TIMEOUT	BIT(13)
109 #define SPRD_DEFAULT_SOURCE_CLK	26000000
110 
111 #define SPRD_RX_DMA_STEP	1
112 #define SPRD_RX_FIFO_FULL	1
113 #define SPRD_TX_FIFO_FULL	0x20
114 #define SPRD_UART_RX_SIZE	(UART_XMIT_SIZE / 4)
115 
116 struct sprd_uart_dma {
117 	struct dma_chan *chn;
118 	unsigned char *virt;
119 	dma_addr_t phys_addr;
120 	dma_cookie_t cookie;
121 	u32 trans_len;
122 	bool enable;
123 };
124 
125 struct sprd_uart_port {
126 	struct uart_port port;
127 	char name[16];
128 	struct clk *clk;
129 	struct sprd_uart_dma tx_dma;
130 	struct sprd_uart_dma rx_dma;
131 	dma_addr_t pos;
132 	unsigned char *rx_buf_tail;
133 };
134 
135 static struct sprd_uart_port *sprd_port[UART_NR_MAX];
136 static int sprd_ports_num;
137 
138 static int sprd_start_dma_rx(struct uart_port *port);
139 static int sprd_tx_dma_config(struct uart_port *port);
140 
141 static inline unsigned int serial_in(struct uart_port *port,
142 				     unsigned int offset)
143 {
144 	return readl_relaxed(port->membase + offset);
145 }
146 
147 static inline void serial_out(struct uart_port *port, unsigned int offset,
148 			      int value)
149 {
150 	writel_relaxed(value, port->membase + offset);
151 }
152 
153 static unsigned int sprd_tx_empty(struct uart_port *port)
154 {
155 	if (serial_in(port, SPRD_STS1) & SPRD_TX_FIFO_CNT_MASK)
156 		return 0;
157 	else
158 		return TIOCSER_TEMT;
159 }
160 
161 static unsigned int sprd_get_mctrl(struct uart_port *port)
162 {
163 	return TIOCM_DSR | TIOCM_CTS;
164 }
165 
166 static void sprd_set_mctrl(struct uart_port *port, unsigned int mctrl)
167 {
168 	u32 val = serial_in(port, SPRD_CTL1);
169 
170 	if (mctrl & TIOCM_LOOP)
171 		val |= SPRD_LOOPBACK_EN;
172 	else
173 		val &= ~SPRD_LOOPBACK_EN;
174 
175 	serial_out(port, SPRD_CTL1, val);
176 }
177 
178 static void sprd_stop_rx(struct uart_port *port)
179 {
180 	struct sprd_uart_port *sp =
181 		container_of(port, struct sprd_uart_port, port);
182 	unsigned int ien, iclr;
183 
184 	if (sp->rx_dma.enable)
185 		dmaengine_terminate_all(sp->rx_dma.chn);
186 
187 	iclr = serial_in(port, SPRD_ICLR);
188 	ien = serial_in(port, SPRD_IEN);
189 
190 	ien &= ~(SPRD_IEN_RX_FULL | SPRD_IEN_BREAK_DETECT);
191 	iclr |= SPRD_IEN_RX_FULL | SPRD_IEN_BREAK_DETECT;
192 
193 	serial_out(port, SPRD_IEN, ien);
194 	serial_out(port, SPRD_ICLR, iclr);
195 }
196 
197 static void sprd_uart_dma_enable(struct uart_port *port, bool enable)
198 {
199 	u32 val = serial_in(port, SPRD_CTL1);
200 
201 	if (enable)
202 		val |= SPRD_DMA_EN;
203 	else
204 		val &= ~SPRD_DMA_EN;
205 
206 	serial_out(port, SPRD_CTL1, val);
207 }
208 
209 static void sprd_stop_tx_dma(struct uart_port *port)
210 {
211 	struct sprd_uart_port *sp =
212 		container_of(port, struct sprd_uart_port, port);
213 	struct circ_buf *xmit = &port->state->xmit;
214 	struct dma_tx_state state;
215 	u32 trans_len;
216 
217 	dmaengine_pause(sp->tx_dma.chn);
218 
219 	dmaengine_tx_status(sp->tx_dma.chn, sp->tx_dma.cookie, &state);
220 	if (state.residue) {
221 		trans_len = state.residue - sp->tx_dma.phys_addr;
222 		xmit->tail = (xmit->tail + trans_len) & (UART_XMIT_SIZE - 1);
223 		port->icount.tx += trans_len;
224 		dma_unmap_single(port->dev, sp->tx_dma.phys_addr,
225 				 sp->tx_dma.trans_len, DMA_TO_DEVICE);
226 	}
227 
228 	dmaengine_terminate_all(sp->tx_dma.chn);
229 	sp->tx_dma.trans_len = 0;
230 }
231 
232 static int sprd_tx_buf_remap(struct uart_port *port)
233 {
234 	struct sprd_uart_port *sp =
235 		container_of(port, struct sprd_uart_port, port);
236 	struct circ_buf *xmit = &port->state->xmit;
237 
238 	sp->tx_dma.trans_len =
239 		CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
240 
241 	sp->tx_dma.phys_addr = dma_map_single(port->dev,
242 					      (void *)&(xmit->buf[xmit->tail]),
243 					      sp->tx_dma.trans_len,
244 					      DMA_TO_DEVICE);
245 	return dma_mapping_error(port->dev, sp->tx_dma.phys_addr);
246 }
247 
248 static void sprd_complete_tx_dma(void *data)
249 {
250 	struct uart_port *port = (struct uart_port *)data;
251 	struct sprd_uart_port *sp =
252 		container_of(port, struct sprd_uart_port, port);
253 	struct circ_buf *xmit = &port->state->xmit;
254 	unsigned long flags;
255 
256 	spin_lock_irqsave(&port->lock, flags);
257 	dma_unmap_single(port->dev, sp->tx_dma.phys_addr,
258 			 sp->tx_dma.trans_len, DMA_TO_DEVICE);
259 
260 	xmit->tail = (xmit->tail + sp->tx_dma.trans_len) & (UART_XMIT_SIZE - 1);
261 	port->icount.tx += sp->tx_dma.trans_len;
262 
263 	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
264 		uart_write_wakeup(port);
265 
266 	if (uart_circ_empty(xmit) || sprd_tx_buf_remap(port) ||
267 	    sprd_tx_dma_config(port))
268 		sp->tx_dma.trans_len = 0;
269 
270 	spin_unlock_irqrestore(&port->lock, flags);
271 }
272 
273 static int sprd_uart_dma_submit(struct uart_port *port,
274 				struct sprd_uart_dma *ud, u32 trans_len,
275 				enum dma_transfer_direction direction,
276 				dma_async_tx_callback callback)
277 {
278 	struct dma_async_tx_descriptor *dma_des;
279 	unsigned long flags;
280 
281 	flags = SPRD_DMA_FLAGS(SPRD_DMA_CHN_MODE_NONE,
282 			       SPRD_DMA_NO_TRG,
283 			       SPRD_DMA_FRAG_REQ,
284 			       SPRD_DMA_TRANS_INT);
285 
286 	dma_des = dmaengine_prep_slave_single(ud->chn, ud->phys_addr, trans_len,
287 					      direction, flags);
288 	if (!dma_des)
289 		return -ENODEV;
290 
291 	dma_des->callback = callback;
292 	dma_des->callback_param = port;
293 
294 	ud->cookie = dmaengine_submit(dma_des);
295 	if (dma_submit_error(ud->cookie))
296 		return dma_submit_error(ud->cookie);
297 
298 	dma_async_issue_pending(ud->chn);
299 
300 	return 0;
301 }
302 
303 static int sprd_tx_dma_config(struct uart_port *port)
304 {
305 	struct sprd_uart_port *sp =
306 		container_of(port, struct sprd_uart_port, port);
307 	u32 burst = sp->tx_dma.trans_len > SPRD_TX_FIFO_FULL ?
308 		SPRD_TX_FIFO_FULL : sp->tx_dma.trans_len;
309 	int ret;
310 	struct dma_slave_config cfg = {
311 		.dst_addr = port->mapbase + SPRD_TXD,
312 		.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
313 		.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
314 		.src_maxburst = burst,
315 	};
316 
317 	ret = dmaengine_slave_config(sp->tx_dma.chn, &cfg);
318 	if (ret < 0)
319 		return ret;
320 
321 	return sprd_uart_dma_submit(port, &sp->tx_dma, sp->tx_dma.trans_len,
322 				    DMA_MEM_TO_DEV, sprd_complete_tx_dma);
323 }
324 
325 static void sprd_start_tx_dma(struct uart_port *port)
326 {
327 	struct sprd_uart_port *sp =
328 		container_of(port, struct sprd_uart_port, port);
329 	struct circ_buf *xmit = &port->state->xmit;
330 
331 	if (port->x_char) {
332 		serial_out(port, SPRD_TXD, port->x_char);
333 		port->icount.tx++;
334 		port->x_char = 0;
335 		return;
336 	}
337 
338 	if (uart_circ_empty(xmit) || uart_tx_stopped(port)) {
339 		sprd_stop_tx_dma(port);
340 		return;
341 	}
342 
343 	if (sp->tx_dma.trans_len)
344 		return;
345 
346 	if (sprd_tx_buf_remap(port) || sprd_tx_dma_config(port))
347 		sp->tx_dma.trans_len = 0;
348 }
349 
350 static void sprd_rx_full_thld(struct uart_port *port, u32 thld)
351 {
352 	u32 val = serial_in(port, SPRD_CTL2);
353 
354 	val &= ~THLD_RX_FULL_MASK;
355 	val |= thld & THLD_RX_FULL_MASK;
356 	serial_out(port, SPRD_CTL2, val);
357 }
358 
359 static int sprd_rx_alloc_buf(struct sprd_uart_port *sp)
360 {
361 	sp->rx_dma.virt = dma_alloc_coherent(sp->port.dev, SPRD_UART_RX_SIZE,
362 					     &sp->rx_dma.phys_addr, GFP_KERNEL);
363 	if (!sp->rx_dma.virt)
364 		return -ENOMEM;
365 
366 	return 0;
367 }
368 
369 static void sprd_rx_free_buf(struct sprd_uart_port *sp)
370 {
371 	if (sp->rx_dma.virt)
372 		dma_free_coherent(sp->port.dev, SPRD_UART_RX_SIZE,
373 				  sp->rx_dma.virt, sp->rx_dma.phys_addr);
374 
375 }
376 
377 static int sprd_rx_dma_config(struct uart_port *port, u32 burst)
378 {
379 	struct sprd_uart_port *sp =
380 		container_of(port, struct sprd_uart_port, port);
381 	struct dma_slave_config cfg = {
382 		.src_addr = port->mapbase + SPRD_RXD,
383 		.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
384 		.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
385 		.src_maxburst = burst,
386 	};
387 
388 	return dmaengine_slave_config(sp->rx_dma.chn, &cfg);
389 }
390 
391 static void sprd_uart_dma_rx(struct uart_port *port)
392 {
393 	struct sprd_uart_port *sp =
394 		container_of(port, struct sprd_uart_port, port);
395 	struct tty_port *tty = &port->state->port;
396 
397 	port->icount.rx += sp->rx_dma.trans_len;
398 	tty_insert_flip_string(tty, sp->rx_buf_tail, sp->rx_dma.trans_len);
399 	tty_flip_buffer_push(tty);
400 }
401 
402 static void sprd_uart_dma_irq(struct uart_port *port)
403 {
404 	struct sprd_uart_port *sp =
405 		container_of(port, struct sprd_uart_port, port);
406 	struct dma_tx_state state;
407 	enum dma_status status;
408 
409 	status = dmaengine_tx_status(sp->rx_dma.chn,
410 				     sp->rx_dma.cookie, &state);
411 	if (status == DMA_ERROR)
412 		sprd_stop_rx(port);
413 
414 	if (!state.residue && sp->pos == sp->rx_dma.phys_addr)
415 		return;
416 
417 	if (!state.residue) {
418 		sp->rx_dma.trans_len = SPRD_UART_RX_SIZE +
419 			sp->rx_dma.phys_addr - sp->pos;
420 		sp->pos = sp->rx_dma.phys_addr;
421 	} else {
422 		sp->rx_dma.trans_len = state.residue - sp->pos;
423 		sp->pos = state.residue;
424 	}
425 
426 	sprd_uart_dma_rx(port);
427 	sp->rx_buf_tail += sp->rx_dma.trans_len;
428 }
429 
430 static void sprd_complete_rx_dma(void *data)
431 {
432 	struct uart_port *port = (struct uart_port *)data;
433 	struct sprd_uart_port *sp =
434 		container_of(port, struct sprd_uart_port, port);
435 	struct dma_tx_state state;
436 	enum dma_status status;
437 	unsigned long flags;
438 
439 	spin_lock_irqsave(&port->lock, flags);
440 
441 	status = dmaengine_tx_status(sp->rx_dma.chn,
442 				     sp->rx_dma.cookie, &state);
443 	if (status != DMA_COMPLETE) {
444 		sprd_stop_rx(port);
445 		spin_unlock_irqrestore(&port->lock, flags);
446 		return;
447 	}
448 
449 	if (sp->pos != sp->rx_dma.phys_addr) {
450 		sp->rx_dma.trans_len =  SPRD_UART_RX_SIZE +
451 			sp->rx_dma.phys_addr - sp->pos;
452 		sprd_uart_dma_rx(port);
453 		sp->rx_buf_tail += sp->rx_dma.trans_len;
454 	}
455 
456 	if (sprd_start_dma_rx(port))
457 		sprd_stop_rx(port);
458 
459 	spin_unlock_irqrestore(&port->lock, flags);
460 }
461 
462 static int sprd_start_dma_rx(struct uart_port *port)
463 {
464 	struct sprd_uart_port *sp =
465 		container_of(port, struct sprd_uart_port, port);
466 	int ret;
467 
468 	if (!sp->rx_dma.enable)
469 		return 0;
470 
471 	sp->pos = sp->rx_dma.phys_addr;
472 	sp->rx_buf_tail = sp->rx_dma.virt;
473 	sprd_rx_full_thld(port, SPRD_RX_FIFO_FULL);
474 	ret = sprd_rx_dma_config(port, SPRD_RX_DMA_STEP);
475 	if (ret)
476 		return ret;
477 
478 	return sprd_uart_dma_submit(port, &sp->rx_dma, SPRD_UART_RX_SIZE,
479 				    DMA_DEV_TO_MEM, sprd_complete_rx_dma);
480 }
481 
482 static void sprd_release_dma(struct uart_port *port)
483 {
484 	struct sprd_uart_port *sp =
485 		container_of(port, struct sprd_uart_port, port);
486 
487 	sprd_uart_dma_enable(port, false);
488 
489 	if (sp->rx_dma.enable)
490 		dma_release_channel(sp->rx_dma.chn);
491 
492 	if (sp->tx_dma.enable)
493 		dma_release_channel(sp->tx_dma.chn);
494 
495 	sp->tx_dma.enable = false;
496 	sp->rx_dma.enable = false;
497 }
498 
499 static void sprd_request_dma(struct uart_port *port)
500 {
501 	struct sprd_uart_port *sp =
502 		container_of(port, struct sprd_uart_port, port);
503 
504 	sp->tx_dma.enable = true;
505 	sp->rx_dma.enable = true;
506 
507 	sp->tx_dma.chn = dma_request_chan(port->dev, "tx");
508 	if (IS_ERR(sp->tx_dma.chn)) {
509 		dev_err(port->dev, "request TX DMA channel failed, ret = %ld\n",
510 			PTR_ERR(sp->tx_dma.chn));
511 		sp->tx_dma.enable = false;
512 	}
513 
514 	sp->rx_dma.chn = dma_request_chan(port->dev, "rx");
515 	if (IS_ERR(sp->rx_dma.chn)) {
516 		dev_err(port->dev, "request RX DMA channel failed, ret = %ld\n",
517 			PTR_ERR(sp->rx_dma.chn));
518 		sp->rx_dma.enable = false;
519 	}
520 }
521 
522 static void sprd_stop_tx(struct uart_port *port)
523 {
524 	struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,
525 						 port);
526 	unsigned int ien, iclr;
527 
528 	if (sp->tx_dma.enable) {
529 		sprd_stop_tx_dma(port);
530 		return;
531 	}
532 
533 	iclr = serial_in(port, SPRD_ICLR);
534 	ien = serial_in(port, SPRD_IEN);
535 
536 	iclr |= SPRD_IEN_TX_EMPTY;
537 	ien &= ~SPRD_IEN_TX_EMPTY;
538 
539 	serial_out(port, SPRD_IEN, ien);
540 	serial_out(port, SPRD_ICLR, iclr);
541 }
542 
543 static void sprd_start_tx(struct uart_port *port)
544 {
545 	struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,
546 						 port);
547 	unsigned int ien;
548 
549 	if (sp->tx_dma.enable) {
550 		sprd_start_tx_dma(port);
551 		return;
552 	}
553 
554 	ien = serial_in(port, SPRD_IEN);
555 	if (!(ien & SPRD_IEN_TX_EMPTY)) {
556 		ien |= SPRD_IEN_TX_EMPTY;
557 		serial_out(port, SPRD_IEN, ien);
558 	}
559 }
560 
561 /* The Sprd serial does not support this function. */
562 static void sprd_break_ctl(struct uart_port *port, int break_state)
563 {
564 	/* nothing to do */
565 }
566 
567 static int handle_lsr_errors(struct uart_port *port,
568 			     unsigned int *flag,
569 			     unsigned int *lsr)
570 {
571 	int ret = 0;
572 
573 	/* statistics */
574 	if (*lsr & SPRD_LSR_BI) {
575 		*lsr &= ~(SPRD_LSR_FE | SPRD_LSR_PE);
576 		port->icount.brk++;
577 		ret = uart_handle_break(port);
578 		if (ret)
579 			return ret;
580 	} else if (*lsr & SPRD_LSR_PE)
581 		port->icount.parity++;
582 	else if (*lsr & SPRD_LSR_FE)
583 		port->icount.frame++;
584 	if (*lsr & SPRD_LSR_OE)
585 		port->icount.overrun++;
586 
587 	/* mask off conditions which should be ignored */
588 	*lsr &= port->read_status_mask;
589 	if (*lsr & SPRD_LSR_BI)
590 		*flag = TTY_BREAK;
591 	else if (*lsr & SPRD_LSR_PE)
592 		*flag = TTY_PARITY;
593 	else if (*lsr & SPRD_LSR_FE)
594 		*flag = TTY_FRAME;
595 
596 	return ret;
597 }
598 
599 static inline void sprd_rx(struct uart_port *port)
600 {
601 	struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,
602 						 port);
603 	struct tty_port *tty = &port->state->port;
604 	unsigned int ch, flag, lsr, max_count = SPRD_TIMEOUT;
605 
606 	if (sp->rx_dma.enable) {
607 		sprd_uart_dma_irq(port);
608 		return;
609 	}
610 
611 	while ((serial_in(port, SPRD_STS1) & SPRD_RX_FIFO_CNT_MASK) &&
612 	       max_count--) {
613 		lsr = serial_in(port, SPRD_LSR);
614 		ch = serial_in(port, SPRD_RXD);
615 		flag = TTY_NORMAL;
616 		port->icount.rx++;
617 
618 		if (lsr & (SPRD_LSR_BI | SPRD_LSR_PE |
619 			   SPRD_LSR_FE | SPRD_LSR_OE))
620 			if (handle_lsr_errors(port, &flag, &lsr))
621 				continue;
622 		if (uart_handle_sysrq_char(port, ch))
623 			continue;
624 
625 		uart_insert_char(port, lsr, SPRD_LSR_OE, ch, flag);
626 	}
627 
628 	tty_flip_buffer_push(tty);
629 }
630 
631 static inline void sprd_tx(struct uart_port *port)
632 {
633 	struct circ_buf *xmit = &port->state->xmit;
634 	int count;
635 
636 	if (port->x_char) {
637 		serial_out(port, SPRD_TXD, port->x_char);
638 		port->icount.tx++;
639 		port->x_char = 0;
640 		return;
641 	}
642 
643 	if (uart_circ_empty(xmit) || uart_tx_stopped(port)) {
644 		sprd_stop_tx(port);
645 		return;
646 	}
647 
648 	count = THLD_TX_EMPTY;
649 	do {
650 		serial_out(port, SPRD_TXD, xmit->buf[xmit->tail]);
651 		xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1);
652 		port->icount.tx++;
653 		if (uart_circ_empty(xmit))
654 			break;
655 	} while (--count > 0);
656 
657 	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
658 		uart_write_wakeup(port);
659 
660 	if (uart_circ_empty(xmit))
661 		sprd_stop_tx(port);
662 }
663 
664 /* this handles the interrupt from one port */
665 static irqreturn_t sprd_handle_irq(int irq, void *dev_id)
666 {
667 	struct uart_port *port = dev_id;
668 	unsigned int ims;
669 
670 	spin_lock(&port->lock);
671 
672 	ims = serial_in(port, SPRD_IMSR);
673 
674 	if (!ims) {
675 		spin_unlock(&port->lock);
676 		return IRQ_NONE;
677 	}
678 
679 	if (ims & SPRD_IMSR_TIMEOUT)
680 		serial_out(port, SPRD_ICLR, SPRD_ICLR_TIMEOUT);
681 
682 	if (ims & (SPRD_IMSR_RX_FIFO_FULL | SPRD_IMSR_BREAK_DETECT |
683 		   SPRD_IMSR_TIMEOUT))
684 		sprd_rx(port);
685 
686 	if (ims & SPRD_IMSR_TX_FIFO_EMPTY)
687 		sprd_tx(port);
688 
689 	spin_unlock(&port->lock);
690 
691 	return IRQ_HANDLED;
692 }
693 
694 static void sprd_uart_dma_startup(struct uart_port *port,
695 				  struct sprd_uart_port *sp)
696 {
697 	int ret;
698 
699 	sprd_request_dma(port);
700 	if (!(sp->rx_dma.enable || sp->tx_dma.enable))
701 		return;
702 
703 	ret = sprd_start_dma_rx(port);
704 	if (ret) {
705 		sp->rx_dma.enable = false;
706 		dma_release_channel(sp->rx_dma.chn);
707 		dev_warn(port->dev, "fail to start RX dma mode\n");
708 	}
709 
710 	sprd_uart_dma_enable(port, true);
711 }
712 
713 static int sprd_startup(struct uart_port *port)
714 {
715 	int ret = 0;
716 	unsigned int ien, fc;
717 	unsigned int timeout;
718 	struct sprd_uart_port *sp;
719 	unsigned long flags;
720 
721 	serial_out(port, SPRD_CTL2,
722 		   THLD_TX_EMPTY << THLD_TX_EMPTY_SHIFT | THLD_RX_FULL);
723 
724 	/* clear rx fifo */
725 	timeout = SPRD_TIMEOUT;
726 	while (timeout-- && serial_in(port, SPRD_STS1) & SPRD_RX_FIFO_CNT_MASK)
727 		serial_in(port, SPRD_RXD);
728 
729 	/* clear tx fifo */
730 	timeout = SPRD_TIMEOUT;
731 	while (timeout-- && serial_in(port, SPRD_STS1) & SPRD_TX_FIFO_CNT_MASK)
732 		cpu_relax();
733 
734 	/* clear interrupt */
735 	serial_out(port, SPRD_IEN, 0);
736 	serial_out(port, SPRD_ICLR, ~0);
737 
738 	/* allocate irq */
739 	sp = container_of(port, struct sprd_uart_port, port);
740 	snprintf(sp->name, sizeof(sp->name), "sprd_serial%d", port->line);
741 
742 	sprd_uart_dma_startup(port, sp);
743 
744 	ret = devm_request_irq(port->dev, port->irq, sprd_handle_irq,
745 			       IRQF_SHARED, sp->name, port);
746 	if (ret) {
747 		dev_err(port->dev, "fail to request serial irq %d, ret=%d\n",
748 			port->irq, ret);
749 		return ret;
750 	}
751 	fc = serial_in(port, SPRD_CTL1);
752 	fc |= RX_TOUT_THLD_DEF | RX_HFC_THLD_DEF;
753 	serial_out(port, SPRD_CTL1, fc);
754 
755 	/* enable interrupt */
756 	spin_lock_irqsave(&port->lock, flags);
757 	ien = serial_in(port, SPRD_IEN);
758 	ien |= SPRD_IEN_BREAK_DETECT | SPRD_IEN_TIMEOUT;
759 	if (!sp->rx_dma.enable)
760 		ien |= SPRD_IEN_RX_FULL;
761 	serial_out(port, SPRD_IEN, ien);
762 	spin_unlock_irqrestore(&port->lock, flags);
763 
764 	return 0;
765 }
766 
767 static void sprd_shutdown(struct uart_port *port)
768 {
769 	sprd_release_dma(port);
770 	serial_out(port, SPRD_IEN, 0);
771 	serial_out(port, SPRD_ICLR, ~0);
772 	devm_free_irq(port->dev, port->irq, port);
773 }
774 
775 static void sprd_set_termios(struct uart_port *port,
776 			     struct ktermios *termios,
777 			     struct ktermios *old)
778 {
779 	unsigned int baud, quot;
780 	unsigned int lcr = 0, fc;
781 	unsigned long flags;
782 
783 	/* ask the core to calculate the divisor for us */
784 	baud = uart_get_baud_rate(port, termios, old, 0, SPRD_BAUD_IO_LIMIT);
785 
786 	quot = port->uartclk / baud;
787 
788 	/* set data length */
789 	switch (termios->c_cflag & CSIZE) {
790 	case CS5:
791 		lcr |= SPRD_LCR_DATA_LEN5;
792 		break;
793 	case CS6:
794 		lcr |= SPRD_LCR_DATA_LEN6;
795 		break;
796 	case CS7:
797 		lcr |= SPRD_LCR_DATA_LEN7;
798 		break;
799 	case CS8:
800 	default:
801 		lcr |= SPRD_LCR_DATA_LEN8;
802 		break;
803 	}
804 
805 	/* calculate stop bits */
806 	lcr &= ~(SPRD_LCR_STOP_1BIT | SPRD_LCR_STOP_2BIT);
807 	if (termios->c_cflag & CSTOPB)
808 		lcr |= SPRD_LCR_STOP_2BIT;
809 	else
810 		lcr |= SPRD_LCR_STOP_1BIT;
811 
812 	/* calculate parity */
813 	lcr &= ~SPRD_LCR_PARITY;
814 	termios->c_cflag &= ~CMSPAR;	/* no support mark/space */
815 	if (termios->c_cflag & PARENB) {
816 		lcr |= SPRD_LCR_PARITY_EN;
817 		if (termios->c_cflag & PARODD)
818 			lcr |= SPRD_LCR_ODD_PAR;
819 		else
820 			lcr |= SPRD_LCR_EVEN_PAR;
821 	}
822 
823 	spin_lock_irqsave(&port->lock, flags);
824 
825 	/* update the per-port timeout */
826 	uart_update_timeout(port, termios->c_cflag, baud);
827 
828 	port->read_status_mask = SPRD_LSR_OE;
829 	if (termios->c_iflag & INPCK)
830 		port->read_status_mask |= SPRD_LSR_FE | SPRD_LSR_PE;
831 	if (termios->c_iflag & (IGNBRK | BRKINT | PARMRK))
832 		port->read_status_mask |= SPRD_LSR_BI;
833 
834 	/* characters to ignore */
835 	port->ignore_status_mask = 0;
836 	if (termios->c_iflag & IGNPAR)
837 		port->ignore_status_mask |= SPRD_LSR_PE | SPRD_LSR_FE;
838 	if (termios->c_iflag & IGNBRK) {
839 		port->ignore_status_mask |= SPRD_LSR_BI;
840 		/*
841 		 * If we're ignoring parity and break indicators,
842 		 * ignore overruns too (for real raw support).
843 		 */
844 		if (termios->c_iflag & IGNPAR)
845 			port->ignore_status_mask |= SPRD_LSR_OE;
846 	}
847 
848 	/* flow control */
849 	fc = serial_in(port, SPRD_CTL1);
850 	fc &= ~(RX_HW_FLOW_CTL_THLD | RX_HW_FLOW_CTL_EN | TX_HW_FLOW_CTL_EN);
851 	if (termios->c_cflag & CRTSCTS) {
852 		fc |= RX_HW_FLOW_CTL_THLD;
853 		fc |= RX_HW_FLOW_CTL_EN;
854 		fc |= TX_HW_FLOW_CTL_EN;
855 	}
856 
857 	/* clock divider bit0~bit15 */
858 	serial_out(port, SPRD_CLKD0, quot & SPRD_CLKD0_MASK);
859 
860 	/* clock divider bit16~bit20 */
861 	serial_out(port, SPRD_CLKD1,
862 		   (quot & SPRD_CLKD1_MASK) >> SPRD_CLKD1_SHIFT);
863 	serial_out(port, SPRD_LCR, lcr);
864 	fc |= RX_TOUT_THLD_DEF | RX_HFC_THLD_DEF;
865 	serial_out(port, SPRD_CTL1, fc);
866 
867 	spin_unlock_irqrestore(&port->lock, flags);
868 
869 	/* Don't rewrite B0 */
870 	if (tty_termios_baud_rate(termios))
871 		tty_termios_encode_baud_rate(termios, baud, baud);
872 }
873 
874 static const char *sprd_type(struct uart_port *port)
875 {
876 	return "SPX";
877 }
878 
879 static void sprd_release_port(struct uart_port *port)
880 {
881 	/* nothing to do */
882 }
883 
884 static int sprd_request_port(struct uart_port *port)
885 {
886 	return 0;
887 }
888 
889 static void sprd_config_port(struct uart_port *port, int flags)
890 {
891 	if (flags & UART_CONFIG_TYPE)
892 		port->type = PORT_SPRD;
893 }
894 
895 static int sprd_verify_port(struct uart_port *port, struct serial_struct *ser)
896 {
897 	if (ser->type != PORT_SPRD)
898 		return -EINVAL;
899 	if (port->irq != ser->irq)
900 		return -EINVAL;
901 	if (port->iotype != ser->io_type)
902 		return -EINVAL;
903 	return 0;
904 }
905 
906 static void sprd_pm(struct uart_port *port, unsigned int state,
907 		unsigned int oldstate)
908 {
909 	struct sprd_uart_port *sup =
910 		container_of(port, struct sprd_uart_port, port);
911 
912 	switch (state) {
913 	case UART_PM_STATE_ON:
914 		clk_prepare_enable(sup->clk);
915 		break;
916 	case UART_PM_STATE_OFF:
917 		clk_disable_unprepare(sup->clk);
918 		break;
919 	}
920 }
921 
922 static const struct uart_ops serial_sprd_ops = {
923 	.tx_empty = sprd_tx_empty,
924 	.get_mctrl = sprd_get_mctrl,
925 	.set_mctrl = sprd_set_mctrl,
926 	.stop_tx = sprd_stop_tx,
927 	.start_tx = sprd_start_tx,
928 	.stop_rx = sprd_stop_rx,
929 	.break_ctl = sprd_break_ctl,
930 	.startup = sprd_startup,
931 	.shutdown = sprd_shutdown,
932 	.set_termios = sprd_set_termios,
933 	.type = sprd_type,
934 	.release_port = sprd_release_port,
935 	.request_port = sprd_request_port,
936 	.config_port = sprd_config_port,
937 	.verify_port = sprd_verify_port,
938 	.pm = sprd_pm,
939 };
940 
941 #ifdef CONFIG_SERIAL_SPRD_CONSOLE
942 static void wait_for_xmitr(struct uart_port *port)
943 {
944 	unsigned int status, tmout = 10000;
945 
946 	/* wait up to 10ms for the character(s) to be sent */
947 	do {
948 		status = serial_in(port, SPRD_STS1);
949 		if (--tmout == 0)
950 			break;
951 		udelay(1);
952 	} while (status & SPRD_TX_FIFO_CNT_MASK);
953 }
954 
955 static void sprd_console_putchar(struct uart_port *port, int ch)
956 {
957 	wait_for_xmitr(port);
958 	serial_out(port, SPRD_TXD, ch);
959 }
960 
961 static void sprd_console_write(struct console *co, const char *s,
962 			       unsigned int count)
963 {
964 	struct uart_port *port = &sprd_port[co->index]->port;
965 	int locked = 1;
966 	unsigned long flags;
967 
968 	if (port->sysrq)
969 		locked = 0;
970 	else if (oops_in_progress)
971 		locked = spin_trylock_irqsave(&port->lock, flags);
972 	else
973 		spin_lock_irqsave(&port->lock, flags);
974 
975 	uart_console_write(port, s, count, sprd_console_putchar);
976 
977 	/* wait for transmitter to become empty */
978 	wait_for_xmitr(port);
979 
980 	if (locked)
981 		spin_unlock_irqrestore(&port->lock, flags);
982 }
983 
984 static int __init sprd_console_setup(struct console *co, char *options)
985 {
986 	struct sprd_uart_port *sprd_uart_port;
987 	int baud = 115200;
988 	int bits = 8;
989 	int parity = 'n';
990 	int flow = 'n';
991 
992 	if (co->index >= UART_NR_MAX || co->index < 0)
993 		co->index = 0;
994 
995 	sprd_uart_port = sprd_port[co->index];
996 	if (!sprd_uart_port || !sprd_uart_port->port.membase) {
997 		pr_info("serial port %d not yet initialized\n", co->index);
998 		return -ENODEV;
999 	}
1000 
1001 	if (options)
1002 		uart_parse_options(options, &baud, &parity, &bits, &flow);
1003 
1004 	return uart_set_options(&sprd_uart_port->port, co, baud,
1005 				parity, bits, flow);
1006 }
1007 
1008 static struct uart_driver sprd_uart_driver;
1009 static struct console sprd_console = {
1010 	.name = SPRD_TTY_NAME,
1011 	.write = sprd_console_write,
1012 	.device = uart_console_device,
1013 	.setup = sprd_console_setup,
1014 	.flags = CON_PRINTBUFFER,
1015 	.index = -1,
1016 	.data = &sprd_uart_driver,
1017 };
1018 
1019 static int __init sprd_serial_console_init(void)
1020 {
1021 	register_console(&sprd_console);
1022 	return 0;
1023 }
1024 console_initcall(sprd_serial_console_init);
1025 
1026 #define SPRD_CONSOLE	(&sprd_console)
1027 
1028 /* Support for earlycon */
1029 static void sprd_putc(struct uart_port *port, int c)
1030 {
1031 	unsigned int timeout = SPRD_TIMEOUT;
1032 
1033 	while (timeout-- &&
1034 	       !(readl(port->membase + SPRD_LSR) & SPRD_LSR_TX_OVER))
1035 		cpu_relax();
1036 
1037 	writeb(c, port->membase + SPRD_TXD);
1038 }
1039 
1040 static void sprd_early_write(struct console *con, const char *s, unsigned int n)
1041 {
1042 	struct earlycon_device *dev = con->data;
1043 
1044 	uart_console_write(&dev->port, s, n, sprd_putc);
1045 }
1046 
1047 static int __init sprd_early_console_setup(struct earlycon_device *device,
1048 					   const char *opt)
1049 {
1050 	if (!device->port.membase)
1051 		return -ENODEV;
1052 
1053 	device->con->write = sprd_early_write;
1054 	return 0;
1055 }
1056 OF_EARLYCON_DECLARE(sprd_serial, "sprd,sc9836-uart",
1057 		    sprd_early_console_setup);
1058 
1059 #else /* !CONFIG_SERIAL_SPRD_CONSOLE */
1060 #define SPRD_CONSOLE		NULL
1061 #endif
1062 
1063 static struct uart_driver sprd_uart_driver = {
1064 	.owner = THIS_MODULE,
1065 	.driver_name = "sprd_serial",
1066 	.dev_name = SPRD_TTY_NAME,
1067 	.major = 0,
1068 	.minor = 0,
1069 	.nr = UART_NR_MAX,
1070 	.cons = SPRD_CONSOLE,
1071 };
1072 
1073 static int sprd_probe_dt_alias(int index, struct device *dev)
1074 {
1075 	struct device_node *np;
1076 	int ret = index;
1077 
1078 	if (!IS_ENABLED(CONFIG_OF))
1079 		return ret;
1080 
1081 	np = dev->of_node;
1082 	if (!np)
1083 		return ret;
1084 
1085 	ret = of_alias_get_id(np, "serial");
1086 	if (ret < 0)
1087 		ret = index;
1088 	else if (ret >= ARRAY_SIZE(sprd_port) || sprd_port[ret] != NULL) {
1089 		dev_warn(dev, "requested serial port %d not available.\n", ret);
1090 		ret = index;
1091 	}
1092 
1093 	return ret;
1094 }
1095 
1096 static int sprd_remove(struct platform_device *dev)
1097 {
1098 	struct sprd_uart_port *sup = platform_get_drvdata(dev);
1099 
1100 	if (sup) {
1101 		uart_remove_one_port(&sprd_uart_driver, &sup->port);
1102 		sprd_port[sup->port.line] = NULL;
1103 		sprd_ports_num--;
1104 	}
1105 
1106 	if (!sprd_ports_num)
1107 		uart_unregister_driver(&sprd_uart_driver);
1108 
1109 	sprd_rx_free_buf(sup);
1110 
1111 	return 0;
1112 }
1113 
1114 static bool sprd_uart_is_console(struct uart_port *uport)
1115 {
1116 	struct console *cons = sprd_uart_driver.cons;
1117 
1118 	if (cons && cons->index >= 0 && cons->index == uport->line)
1119 		return true;
1120 
1121 	return false;
1122 }
1123 
1124 static int sprd_clk_init(struct uart_port *uport)
1125 {
1126 	struct clk *clk_uart, *clk_parent;
1127 	struct sprd_uart_port *u = sprd_port[uport->line];
1128 
1129 	clk_uart = devm_clk_get(uport->dev, "uart");
1130 	if (IS_ERR(clk_uart)) {
1131 		dev_warn(uport->dev, "uart%d can't get uart clock\n",
1132 			 uport->line);
1133 		clk_uart = NULL;
1134 	}
1135 
1136 	clk_parent = devm_clk_get(uport->dev, "source");
1137 	if (IS_ERR(clk_parent)) {
1138 		dev_warn(uport->dev, "uart%d can't get source clock\n",
1139 			 uport->line);
1140 		clk_parent = NULL;
1141 	}
1142 
1143 	if (!clk_uart || clk_set_parent(clk_uart, clk_parent))
1144 		uport->uartclk = SPRD_DEFAULT_SOURCE_CLK;
1145 	else
1146 		uport->uartclk = clk_get_rate(clk_uart);
1147 
1148 	u->clk = devm_clk_get(uport->dev, "enable");
1149 	if (IS_ERR(u->clk)) {
1150 		if (PTR_ERR(u->clk) == -EPROBE_DEFER)
1151 			return -EPROBE_DEFER;
1152 
1153 		dev_warn(uport->dev, "uart%d can't get enable clock\n",
1154 			uport->line);
1155 
1156 		/* To keep console alive even if the error occurred */
1157 		if (!sprd_uart_is_console(uport))
1158 			return PTR_ERR(u->clk);
1159 
1160 		u->clk = NULL;
1161 	}
1162 
1163 	return 0;
1164 }
1165 
1166 static int sprd_probe(struct platform_device *pdev)
1167 {
1168 	struct resource *res;
1169 	struct uart_port *up;
1170 	int irq;
1171 	int index;
1172 	int ret;
1173 
1174 	for (index = 0; index < ARRAY_SIZE(sprd_port); index++)
1175 		if (sprd_port[index] == NULL)
1176 			break;
1177 
1178 	if (index == ARRAY_SIZE(sprd_port))
1179 		return -EBUSY;
1180 
1181 	index = sprd_probe_dt_alias(index, &pdev->dev);
1182 
1183 	sprd_port[index] = devm_kzalloc(&pdev->dev, sizeof(*sprd_port[index]),
1184 					GFP_KERNEL);
1185 	if (!sprd_port[index])
1186 		return -ENOMEM;
1187 
1188 	up = &sprd_port[index]->port;
1189 	up->dev = &pdev->dev;
1190 	up->line = index;
1191 	up->type = PORT_SPRD;
1192 	up->iotype = UPIO_MEM;
1193 	up->uartclk = SPRD_DEF_RATE;
1194 	up->fifosize = SPRD_FIFO_SIZE;
1195 	up->ops = &serial_sprd_ops;
1196 	up->flags = UPF_BOOT_AUTOCONF;
1197 
1198 	ret = sprd_clk_init(up);
1199 	if (ret)
1200 		return ret;
1201 
1202 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1203 	up->membase = devm_ioremap_resource(&pdev->dev, res);
1204 	if (IS_ERR(up->membase))
1205 		return PTR_ERR(up->membase);
1206 
1207 	up->mapbase = res->start;
1208 
1209 	irq = platform_get_irq(pdev, 0);
1210 	if (irq < 0)
1211 		return irq;
1212 	up->irq = irq;
1213 
1214 	/*
1215 	 * Allocate one dma buffer to prepare for receive transfer, in case
1216 	 * memory allocation failure at runtime.
1217 	 */
1218 	ret = sprd_rx_alloc_buf(sprd_port[index]);
1219 	if (ret)
1220 		return ret;
1221 
1222 	if (!sprd_ports_num) {
1223 		ret = uart_register_driver(&sprd_uart_driver);
1224 		if (ret < 0) {
1225 			pr_err("Failed to register SPRD-UART driver\n");
1226 			return ret;
1227 		}
1228 	}
1229 	sprd_ports_num++;
1230 
1231 	ret = uart_add_one_port(&sprd_uart_driver, up);
1232 	if (ret) {
1233 		sprd_port[index] = NULL;
1234 		sprd_remove(pdev);
1235 	}
1236 
1237 	platform_set_drvdata(pdev, up);
1238 
1239 	return ret;
1240 }
1241 
1242 #ifdef CONFIG_PM_SLEEP
1243 static int sprd_suspend(struct device *dev)
1244 {
1245 	struct sprd_uart_port *sup = dev_get_drvdata(dev);
1246 
1247 	uart_suspend_port(&sprd_uart_driver, &sup->port);
1248 
1249 	return 0;
1250 }
1251 
1252 static int sprd_resume(struct device *dev)
1253 {
1254 	struct sprd_uart_port *sup = dev_get_drvdata(dev);
1255 
1256 	uart_resume_port(&sprd_uart_driver, &sup->port);
1257 
1258 	return 0;
1259 }
1260 #endif
1261 
1262 static SIMPLE_DEV_PM_OPS(sprd_pm_ops, sprd_suspend, sprd_resume);
1263 
1264 static const struct of_device_id serial_ids[] = {
1265 	{.compatible = "sprd,sc9836-uart",},
1266 	{}
1267 };
1268 MODULE_DEVICE_TABLE(of, serial_ids);
1269 
1270 static struct platform_driver sprd_platform_driver = {
1271 	.probe		= sprd_probe,
1272 	.remove		= sprd_remove,
1273 	.driver		= {
1274 		.name	= "sprd_serial",
1275 		.of_match_table = of_match_ptr(serial_ids),
1276 		.pm	= &sprd_pm_ops,
1277 	},
1278 };
1279 
1280 module_platform_driver(sprd_platform_driver);
1281 
1282 MODULE_LICENSE("GPL v2");
1283 MODULE_DESCRIPTION("Spreadtrum SoC serial driver series");
1284