xref: /openbmc/linux/drivers/tty/serial/sprd_serial.c (revision b830f94f) 
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2012-2015 Spreadtrum Communications Inc.
4  */
5 
6 #if defined(CONFIG_SERIAL_SPRD_CONSOLE) && defined(CONFIG_MAGIC_SYSRQ)
7 #define SUPPORT_SYSRQ
8 #endif
9 
10 #include <linux/clk.h>
11 #include <linux/console.h>
12 #include <linux/delay.h>
13 #include <linux/dmaengine.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/dma/sprd-dma.h>
16 #include <linux/io.h>
17 #include <linux/ioport.h>
18 #include <linux/kernel.h>
19 #include <linux/module.h>
20 #include <linux/of.h>
21 #include <linux/platform_device.h>
22 #include <linux/serial_core.h>
23 #include <linux/serial.h>
24 #include <linux/slab.h>
25 #include <linux/tty.h>
26 #include <linux/tty_flip.h>
27 
28 /* device name */
29 #define UART_NR_MAX		8
30 #define SPRD_TTY_NAME		"ttyS"
31 #define SPRD_FIFO_SIZE		128
32 #define SPRD_DEF_RATE		26000000
33 #define SPRD_BAUD_IO_LIMIT	3000000
34 #define SPRD_TIMEOUT		256000
35 
36 /* the offset of serial registers and BITs for them */
37 /* data registers */
38 #define SPRD_TXD		0x0000
39 #define SPRD_RXD		0x0004
40 
41 /* line status register and its BITs  */
42 #define SPRD_LSR		0x0008
43 #define SPRD_LSR_OE		BIT(4)
44 #define SPRD_LSR_FE		BIT(3)
45 #define SPRD_LSR_PE		BIT(2)
46 #define SPRD_LSR_BI		BIT(7)
47 #define SPRD_LSR_TX_OVER	BIT(15)
48 
49 /* data number in TX and RX fifo */
50 #define SPRD_STS1		0x000C
51 #define SPRD_RX_FIFO_CNT_MASK	GENMASK(7, 0)
52 #define SPRD_TX_FIFO_CNT_MASK	GENMASK(15, 8)
53 
54 /* interrupt enable register and its BITs */
55 #define SPRD_IEN		0x0010
56 #define SPRD_IEN_RX_FULL	BIT(0)
57 #define SPRD_IEN_TX_EMPTY	BIT(1)
58 #define SPRD_IEN_BREAK_DETECT	BIT(7)
59 #define SPRD_IEN_TIMEOUT	BIT(13)
60 
61 /* interrupt clear register */
62 #define SPRD_ICLR		0x0014
63 #define SPRD_ICLR_TIMEOUT	BIT(13)
64 
65 /* line control register */
66 #define SPRD_LCR		0x0018
67 #define SPRD_LCR_STOP_1BIT	0x10
68 #define SPRD_LCR_STOP_2BIT	0x30
69 #define SPRD_LCR_DATA_LEN	(BIT(2) | BIT(3))
70 #define SPRD_LCR_DATA_LEN5	0x0
71 #define SPRD_LCR_DATA_LEN6	0x4
72 #define SPRD_LCR_DATA_LEN7	0x8
73 #define SPRD_LCR_DATA_LEN8	0xc
74 #define SPRD_LCR_PARITY		(BIT(0) | BIT(1))
75 #define SPRD_LCR_PARITY_EN	0x2
76 #define SPRD_LCR_EVEN_PAR	0x0
77 #define SPRD_LCR_ODD_PAR	0x1
78 
79 /* control register 1 */
80 #define SPRD_CTL1		0x001C
81 #define SPRD_DMA_EN		BIT(15)
82 #define RX_HW_FLOW_CTL_THLD	BIT(6)
83 #define RX_HW_FLOW_CTL_EN	BIT(7)
84 #define TX_HW_FLOW_CTL_EN	BIT(8)
85 #define RX_TOUT_THLD_DEF	0x3E00
86 #define RX_HFC_THLD_DEF		0x40
87 
88 /* fifo threshold register */
89 #define SPRD_CTL2		0x0020
90 #define THLD_TX_EMPTY		0x40
91 #define THLD_TX_EMPTY_SHIFT	8
92 #define THLD_RX_FULL		0x40
93 #define THLD_RX_FULL_MASK	GENMASK(6, 0)
94 
95 /* config baud rate register */
96 #define SPRD_CLKD0		0x0024
97 #define SPRD_CLKD0_MASK		GENMASK(15, 0)
98 #define SPRD_CLKD1		0x0028
99 #define SPRD_CLKD1_MASK		GENMASK(20, 16)
100 #define SPRD_CLKD1_SHIFT	16
101 
102 /* interrupt mask status register */
103 #define SPRD_IMSR		0x002C
104 #define SPRD_IMSR_RX_FIFO_FULL	BIT(0)
105 #define SPRD_IMSR_TX_FIFO_EMPTY	BIT(1)
106 #define SPRD_IMSR_BREAK_DETECT	BIT(7)
107 #define SPRD_IMSR_TIMEOUT	BIT(13)
108 #define SPRD_DEFAULT_SOURCE_CLK	26000000
109 
110 #define SPRD_RX_DMA_STEP	1
111 #define SPRD_RX_FIFO_FULL	1
112 #define SPRD_TX_FIFO_FULL	0x20
113 #define SPRD_UART_RX_SIZE	(UART_XMIT_SIZE / 4)
114 
115 struct sprd_uart_dma {
116 	struct dma_chan *chn;
117 	unsigned char *virt;
118 	dma_addr_t phys_addr;
119 	dma_cookie_t cookie;
120 	u32 trans_len;
121 	bool enable;
122 };
123 
124 struct sprd_uart_port {
125 	struct uart_port port;
126 	char name[16];
127 	struct clk *clk;
128 	struct sprd_uart_dma tx_dma;
129 	struct sprd_uart_dma rx_dma;
130 	dma_addr_t pos;
131 	unsigned char *rx_buf_tail;
132 };
133 
134 static struct sprd_uart_port *sprd_port[UART_NR_MAX];
135 static int sprd_ports_num;
136 
137 static int sprd_start_dma_rx(struct uart_port *port);
138 static int sprd_tx_dma_config(struct uart_port *port);
139 
140 static inline unsigned int serial_in(struct uart_port *port,
141 				     unsigned int offset)
142 {
143 	return readl_relaxed(port->membase + offset);
144 }
145 
146 static inline void serial_out(struct uart_port *port, unsigned int offset,
147 			      int value)
148 {
149 	writel_relaxed(value, port->membase + offset);
150 }
151 
152 static unsigned int sprd_tx_empty(struct uart_port *port)
153 {
154 	if (serial_in(port, SPRD_STS1) & SPRD_TX_FIFO_CNT_MASK)
155 		return 0;
156 	else
157 		return TIOCSER_TEMT;
158 }
159 
160 static unsigned int sprd_get_mctrl(struct uart_port *port)
161 {
162 	return TIOCM_DSR | TIOCM_CTS;
163 }
164 
165 static void sprd_set_mctrl(struct uart_port *port, unsigned int mctrl)
166 {
167 	/* nothing to do */
168 }
169 
170 static void sprd_stop_rx(struct uart_port *port)
171 {
172 	struct sprd_uart_port *sp =
173 		container_of(port, struct sprd_uart_port, port);
174 	unsigned int ien, iclr;
175 
176 	if (sp->rx_dma.enable)
177 		dmaengine_terminate_all(sp->rx_dma.chn);
178 
179 	iclr = serial_in(port, SPRD_ICLR);
180 	ien = serial_in(port, SPRD_IEN);
181 
182 	ien &= ~(SPRD_IEN_RX_FULL | SPRD_IEN_BREAK_DETECT);
183 	iclr |= SPRD_IEN_RX_FULL | SPRD_IEN_BREAK_DETECT;
184 
185 	serial_out(port, SPRD_IEN, ien);
186 	serial_out(port, SPRD_ICLR, iclr);
187 }
188 
189 static void sprd_uart_dma_enable(struct uart_port *port, bool enable)
190 {
191 	u32 val = serial_in(port, SPRD_CTL1);
192 
193 	if (enable)
194 		val |= SPRD_DMA_EN;
195 	else
196 		val &= ~SPRD_DMA_EN;
197 
198 	serial_out(port, SPRD_CTL1, val);
199 }
200 
201 static void sprd_stop_tx_dma(struct uart_port *port)
202 {
203 	struct sprd_uart_port *sp =
204 		container_of(port, struct sprd_uart_port, port);
205 	struct circ_buf *xmit = &port->state->xmit;
206 	struct dma_tx_state state;
207 	u32 trans_len;
208 
209 	dmaengine_pause(sp->tx_dma.chn);
210 
211 	dmaengine_tx_status(sp->tx_dma.chn, sp->tx_dma.cookie, &state);
212 	if (state.residue) {
213 		trans_len = state.residue - sp->tx_dma.phys_addr;
214 		xmit->tail = (xmit->tail + trans_len) & (UART_XMIT_SIZE - 1);
215 		port->icount.tx += trans_len;
216 		dma_unmap_single(port->dev, sp->tx_dma.phys_addr,
217 				 sp->tx_dma.trans_len, DMA_TO_DEVICE);
218 	}
219 
220 	dmaengine_terminate_all(sp->tx_dma.chn);
221 	sp->tx_dma.trans_len = 0;
222 }
223 
224 static int sprd_tx_buf_remap(struct uart_port *port)
225 {
226 	struct sprd_uart_port *sp =
227 		container_of(port, struct sprd_uart_port, port);
228 	struct circ_buf *xmit = &port->state->xmit;
229 
230 	sp->tx_dma.trans_len =
231 		CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
232 
233 	sp->tx_dma.phys_addr = dma_map_single(port->dev,
234 					      (void *)&(xmit->buf[xmit->tail]),
235 					      sp->tx_dma.trans_len,
236 					      DMA_TO_DEVICE);
237 	return dma_mapping_error(port->dev, sp->tx_dma.phys_addr);
238 }
239 
240 static void sprd_complete_tx_dma(void *data)
241 {
242 	struct uart_port *port = (struct uart_port *)data;
243 	struct sprd_uart_port *sp =
244 		container_of(port, struct sprd_uart_port, port);
245 	struct circ_buf *xmit = &port->state->xmit;
246 	unsigned long flags;
247 
248 	spin_lock_irqsave(&port->lock, flags);
249 	dma_unmap_single(port->dev, sp->tx_dma.phys_addr,
250 			 sp->tx_dma.trans_len, DMA_TO_DEVICE);
251 
252 	xmit->tail = (xmit->tail + sp->tx_dma.trans_len) & (UART_XMIT_SIZE - 1);
253 	port->icount.tx += sp->tx_dma.trans_len;
254 
255 	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
256 		uart_write_wakeup(port);
257 
258 	if (uart_circ_empty(xmit) || sprd_tx_buf_remap(port) ||
259 	    sprd_tx_dma_config(port))
260 		sp->tx_dma.trans_len = 0;
261 
262 	spin_unlock_irqrestore(&port->lock, flags);
263 }
264 
265 static int sprd_uart_dma_submit(struct uart_port *port,
266 				struct sprd_uart_dma *ud, u32 trans_len,
267 				enum dma_transfer_direction direction,
268 				dma_async_tx_callback callback)
269 {
270 	struct dma_async_tx_descriptor *dma_des;
271 	unsigned long flags;
272 
273 	flags = SPRD_DMA_FLAGS(SPRD_DMA_CHN_MODE_NONE,
274 			       SPRD_DMA_NO_TRG,
275 			       SPRD_DMA_FRAG_REQ,
276 			       SPRD_DMA_TRANS_INT);
277 
278 	dma_des = dmaengine_prep_slave_single(ud->chn, ud->phys_addr, trans_len,
279 					      direction, flags);
280 	if (!dma_des)
281 		return -ENODEV;
282 
283 	dma_des->callback = callback;
284 	dma_des->callback_param = port;
285 
286 	ud->cookie = dmaengine_submit(dma_des);
287 	if (dma_submit_error(ud->cookie))
288 		return dma_submit_error(ud->cookie);
289 
290 	dma_async_issue_pending(ud->chn);
291 
292 	return 0;
293 }
294 
295 static int sprd_tx_dma_config(struct uart_port *port)
296 {
297 	struct sprd_uart_port *sp =
298 		container_of(port, struct sprd_uart_port, port);
299 	u32 burst = sp->tx_dma.trans_len > SPRD_TX_FIFO_FULL ?
300 		SPRD_TX_FIFO_FULL : sp->tx_dma.trans_len;
301 	int ret;
302 	struct dma_slave_config cfg = {
303 		.dst_addr = port->mapbase + SPRD_TXD,
304 		.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
305 		.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
306 		.src_maxburst = burst,
307 	};
308 
309 	ret = dmaengine_slave_config(sp->tx_dma.chn, &cfg);
310 	if (ret < 0)
311 		return ret;
312 
313 	return sprd_uart_dma_submit(port, &sp->tx_dma, sp->tx_dma.trans_len,
314 				    DMA_MEM_TO_DEV, sprd_complete_tx_dma);
315 }
316 
317 static void sprd_start_tx_dma(struct uart_port *port)
318 {
319 	struct sprd_uart_port *sp =
320 		container_of(port, struct sprd_uart_port, port);
321 	struct circ_buf *xmit = &port->state->xmit;
322 
323 	if (port->x_char) {
324 		serial_out(port, SPRD_TXD, port->x_char);
325 		port->icount.tx++;
326 		port->x_char = 0;
327 		return;
328 	}
329 
330 	if (uart_circ_empty(xmit) || uart_tx_stopped(port)) {
331 		sprd_stop_tx_dma(port);
332 		return;
333 	}
334 
335 	if (sp->tx_dma.trans_len)
336 		return;
337 
338 	if (sprd_tx_buf_remap(port) || sprd_tx_dma_config(port))
339 		sp->tx_dma.trans_len = 0;
340 }
341 
342 static void sprd_rx_full_thld(struct uart_port *port, u32 thld)
343 {
344 	u32 val = serial_in(port, SPRD_CTL2);
345 
346 	val &= ~THLD_RX_FULL_MASK;
347 	val |= thld & THLD_RX_FULL_MASK;
348 	serial_out(port, SPRD_CTL2, val);
349 }
350 
351 static int sprd_rx_alloc_buf(struct sprd_uart_port *sp)
352 {
353 	sp->rx_dma.virt = dma_alloc_coherent(sp->port.dev, SPRD_UART_RX_SIZE,
354 					     &sp->rx_dma.phys_addr, GFP_KERNEL);
355 	if (!sp->rx_dma.virt)
356 		return -ENOMEM;
357 
358 	return 0;
359 }
360 
361 static void sprd_rx_free_buf(struct sprd_uart_port *sp)
362 {
363 	if (sp->rx_dma.virt)
364 		dma_free_coherent(sp->port.dev, SPRD_UART_RX_SIZE,
365 				  sp->rx_dma.virt, sp->rx_dma.phys_addr);
366 
367 }
368 
369 static int sprd_rx_dma_config(struct uart_port *port, u32 burst)
370 {
371 	struct sprd_uart_port *sp =
372 		container_of(port, struct sprd_uart_port, port);
373 	struct dma_slave_config cfg = {
374 		.src_addr = port->mapbase + SPRD_RXD,
375 		.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
376 		.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
377 		.src_maxburst = burst,
378 	};
379 
380 	return dmaengine_slave_config(sp->rx_dma.chn, &cfg);
381 }
382 
383 static void sprd_uart_dma_rx(struct uart_port *port)
384 {
385 	struct sprd_uart_port *sp =
386 		container_of(port, struct sprd_uart_port, port);
387 	struct tty_port *tty = &port->state->port;
388 
389 	port->icount.rx += sp->rx_dma.trans_len;
390 	tty_insert_flip_string(tty, sp->rx_buf_tail, sp->rx_dma.trans_len);
391 	tty_flip_buffer_push(tty);
392 }
393 
394 static void sprd_uart_dma_irq(struct uart_port *port)
395 {
396 	struct sprd_uart_port *sp =
397 		container_of(port, struct sprd_uart_port, port);
398 	struct dma_tx_state state;
399 	enum dma_status status;
400 
401 	status = dmaengine_tx_status(sp->rx_dma.chn,
402 				     sp->rx_dma.cookie, &state);
403 	if (status == DMA_ERROR)
404 		sprd_stop_rx(port);
405 
406 	if (!state.residue && sp->pos == sp->rx_dma.phys_addr)
407 		return;
408 
409 	if (!state.residue) {
410 		sp->rx_dma.trans_len = SPRD_UART_RX_SIZE +
411 			sp->rx_dma.phys_addr - sp->pos;
412 		sp->pos = sp->rx_dma.phys_addr;
413 	} else {
414 		sp->rx_dma.trans_len = state.residue - sp->pos;
415 		sp->pos = state.residue;
416 	}
417 
418 	sprd_uart_dma_rx(port);
419 	sp->rx_buf_tail += sp->rx_dma.trans_len;
420 }
421 
422 static void sprd_complete_rx_dma(void *data)
423 {
424 	struct uart_port *port = (struct uart_port *)data;
425 	struct sprd_uart_port *sp =
426 		container_of(port, struct sprd_uart_port, port);
427 	struct dma_tx_state state;
428 	enum dma_status status;
429 	unsigned long flags;
430 
431 	spin_lock_irqsave(&port->lock, flags);
432 
433 	status = dmaengine_tx_status(sp->rx_dma.chn,
434 				     sp->rx_dma.cookie, &state);
435 	if (status != DMA_COMPLETE) {
436 		sprd_stop_rx(port);
437 		spin_unlock_irqrestore(&port->lock, flags);
438 		return;
439 	}
440 
441 	if (sp->pos != sp->rx_dma.phys_addr) {
442 		sp->rx_dma.trans_len =  SPRD_UART_RX_SIZE +
443 			sp->rx_dma.phys_addr - sp->pos;
444 		sprd_uart_dma_rx(port);
445 		sp->rx_buf_tail += sp->rx_dma.trans_len;
446 	}
447 
448 	if (sprd_start_dma_rx(port))
449 		sprd_stop_rx(port);
450 
451 	spin_unlock_irqrestore(&port->lock, flags);
452 }
453 
454 static int sprd_start_dma_rx(struct uart_port *port)
455 {
456 	struct sprd_uart_port *sp =
457 		container_of(port, struct sprd_uart_port, port);
458 	int ret;
459 
460 	if (!sp->rx_dma.enable)
461 		return 0;
462 
463 	sp->pos = sp->rx_dma.phys_addr;
464 	sp->rx_buf_tail = sp->rx_dma.virt;
465 	sprd_rx_full_thld(port, SPRD_RX_FIFO_FULL);
466 	ret = sprd_rx_dma_config(port, SPRD_RX_DMA_STEP);
467 	if (ret)
468 		return ret;
469 
470 	return sprd_uart_dma_submit(port, &sp->rx_dma, SPRD_UART_RX_SIZE,
471 				    DMA_DEV_TO_MEM, sprd_complete_rx_dma);
472 }
473 
474 static void sprd_release_dma(struct uart_port *port)
475 {
476 	struct sprd_uart_port *sp =
477 		container_of(port, struct sprd_uart_port, port);
478 
479 	sprd_uart_dma_enable(port, false);
480 
481 	if (sp->rx_dma.enable)
482 		dma_release_channel(sp->rx_dma.chn);
483 
484 	if (sp->tx_dma.enable)
485 		dma_release_channel(sp->tx_dma.chn);
486 
487 	sp->tx_dma.enable = false;
488 	sp->rx_dma.enable = false;
489 }
490 
491 static void sprd_request_dma(struct uart_port *port)
492 {
493 	struct sprd_uart_port *sp =
494 		container_of(port, struct sprd_uart_port, port);
495 
496 	sp->tx_dma.enable = true;
497 	sp->rx_dma.enable = true;
498 
499 	sp->tx_dma.chn = dma_request_chan(port->dev, "tx");
500 	if (IS_ERR(sp->tx_dma.chn)) {
501 		dev_err(port->dev, "request TX DMA channel failed, ret = %ld\n",
502 			PTR_ERR(sp->tx_dma.chn));
503 		sp->tx_dma.enable = false;
504 	}
505 
506 	sp->rx_dma.chn = dma_request_chan(port->dev, "rx");
507 	if (IS_ERR(sp->rx_dma.chn)) {
508 		dev_err(port->dev, "request RX DMA channel failed, ret = %ld\n",
509 			PTR_ERR(sp->rx_dma.chn));
510 		sp->rx_dma.enable = false;
511 	}
512 }
513 
514 static void sprd_stop_tx(struct uart_port *port)
515 {
516 	struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,
517 						 port);
518 	unsigned int ien, iclr;
519 
520 	if (sp->tx_dma.enable) {
521 		sprd_stop_tx_dma(port);
522 		return;
523 	}
524 
525 	iclr = serial_in(port, SPRD_ICLR);
526 	ien = serial_in(port, SPRD_IEN);
527 
528 	iclr |= SPRD_IEN_TX_EMPTY;
529 	ien &= ~SPRD_IEN_TX_EMPTY;
530 
531 	serial_out(port, SPRD_IEN, ien);
532 	serial_out(port, SPRD_ICLR, iclr);
533 }
534 
535 static void sprd_start_tx(struct uart_port *port)
536 {
537 	struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,
538 						 port);
539 	unsigned int ien;
540 
541 	if (sp->tx_dma.enable) {
542 		sprd_start_tx_dma(port);
543 		return;
544 	}
545 
546 	ien = serial_in(port, SPRD_IEN);
547 	if (!(ien & SPRD_IEN_TX_EMPTY)) {
548 		ien |= SPRD_IEN_TX_EMPTY;
549 		serial_out(port, SPRD_IEN, ien);
550 	}
551 }
552 
553 /* The Sprd serial does not support this function. */
554 static void sprd_break_ctl(struct uart_port *port, int break_state)
555 {
556 	/* nothing to do */
557 }
558 
559 static int handle_lsr_errors(struct uart_port *port,
560 			     unsigned int *flag,
561 			     unsigned int *lsr)
562 {
563 	int ret = 0;
564 
565 	/* statistics */
566 	if (*lsr & SPRD_LSR_BI) {
567 		*lsr &= ~(SPRD_LSR_FE | SPRD_LSR_PE);
568 		port->icount.brk++;
569 		ret = uart_handle_break(port);
570 		if (ret)
571 			return ret;
572 	} else if (*lsr & SPRD_LSR_PE)
573 		port->icount.parity++;
574 	else if (*lsr & SPRD_LSR_FE)
575 		port->icount.frame++;
576 	if (*lsr & SPRD_LSR_OE)
577 		port->icount.overrun++;
578 
579 	/* mask off conditions which should be ignored */
580 	*lsr &= port->read_status_mask;
581 	if (*lsr & SPRD_LSR_BI)
582 		*flag = TTY_BREAK;
583 	else if (*lsr & SPRD_LSR_PE)
584 		*flag = TTY_PARITY;
585 	else if (*lsr & SPRD_LSR_FE)
586 		*flag = TTY_FRAME;
587 
588 	return ret;
589 }
590 
591 static inline void sprd_rx(struct uart_port *port)
592 {
593 	struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,
594 						 port);
595 	struct tty_port *tty = &port->state->port;
596 	unsigned int ch, flag, lsr, max_count = SPRD_TIMEOUT;
597 
598 	if (sp->rx_dma.enable) {
599 		sprd_uart_dma_irq(port);
600 		return;
601 	}
602 
603 	while ((serial_in(port, SPRD_STS1) & SPRD_RX_FIFO_CNT_MASK) &&
604 	       max_count--) {
605 		lsr = serial_in(port, SPRD_LSR);
606 		ch = serial_in(port, SPRD_RXD);
607 		flag = TTY_NORMAL;
608 		port->icount.rx++;
609 
610 		if (lsr & (SPRD_LSR_BI | SPRD_LSR_PE |
611 			   SPRD_LSR_FE | SPRD_LSR_OE))
612 			if (handle_lsr_errors(port, &lsr, &flag))
613 				continue;
614 		if (uart_handle_sysrq_char(port, ch))
615 			continue;
616 
617 		uart_insert_char(port, lsr, SPRD_LSR_OE, ch, flag);
618 	}
619 
620 	tty_flip_buffer_push(tty);
621 }
622 
623 static inline void sprd_tx(struct uart_port *port)
624 {
625 	struct circ_buf *xmit = &port->state->xmit;
626 	int count;
627 
628 	if (port->x_char) {
629 		serial_out(port, SPRD_TXD, port->x_char);
630 		port->icount.tx++;
631 		port->x_char = 0;
632 		return;
633 	}
634 
635 	if (uart_circ_empty(xmit) || uart_tx_stopped(port)) {
636 		sprd_stop_tx(port);
637 		return;
638 	}
639 
640 	count = THLD_TX_EMPTY;
641 	do {
642 		serial_out(port, SPRD_TXD, xmit->buf[xmit->tail]);
643 		xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1);
644 		port->icount.tx++;
645 		if (uart_circ_empty(xmit))
646 			break;
647 	} while (--count > 0);
648 
649 	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
650 		uart_write_wakeup(port);
651 
652 	if (uart_circ_empty(xmit))
653 		sprd_stop_tx(port);
654 }
655 
656 /* this handles the interrupt from one port */
657 static irqreturn_t sprd_handle_irq(int irq, void *dev_id)
658 {
659 	struct uart_port *port = dev_id;
660 	unsigned int ims;
661 
662 	spin_lock(&port->lock);
663 
664 	ims = serial_in(port, SPRD_IMSR);
665 
666 	if (!ims) {
667 		spin_unlock(&port->lock);
668 		return IRQ_NONE;
669 	}
670 
671 	if (ims & SPRD_IMSR_TIMEOUT)
672 		serial_out(port, SPRD_ICLR, SPRD_ICLR_TIMEOUT);
673 
674 	if (ims & (SPRD_IMSR_RX_FIFO_FULL | SPRD_IMSR_BREAK_DETECT |
675 		   SPRD_IMSR_TIMEOUT))
676 		sprd_rx(port);
677 
678 	if (ims & SPRD_IMSR_TX_FIFO_EMPTY)
679 		sprd_tx(port);
680 
681 	spin_unlock(&port->lock);
682 
683 	return IRQ_HANDLED;
684 }
685 
686 static void sprd_uart_dma_startup(struct uart_port *port,
687 				  struct sprd_uart_port *sp)
688 {
689 	int ret;
690 
691 	sprd_request_dma(port);
692 	if (!(sp->rx_dma.enable || sp->tx_dma.enable))
693 		return;
694 
695 	ret = sprd_start_dma_rx(port);
696 	if (ret) {
697 		sp->rx_dma.enable = false;
698 		dma_release_channel(sp->rx_dma.chn);
699 		dev_warn(port->dev, "fail to start RX dma mode\n");
700 	}
701 
702 	sprd_uart_dma_enable(port, true);
703 }
704 
705 static int sprd_startup(struct uart_port *port)
706 {
707 	int ret = 0;
708 	unsigned int ien, fc;
709 	unsigned int timeout;
710 	struct sprd_uart_port *sp;
711 	unsigned long flags;
712 
713 	serial_out(port, SPRD_CTL2,
714 		   THLD_TX_EMPTY << THLD_TX_EMPTY_SHIFT | THLD_RX_FULL);
715 
716 	/* clear rx fifo */
717 	timeout = SPRD_TIMEOUT;
718 	while (timeout-- && serial_in(port, SPRD_STS1) & SPRD_RX_FIFO_CNT_MASK)
719 		serial_in(port, SPRD_RXD);
720 
721 	/* clear tx fifo */
722 	timeout = SPRD_TIMEOUT;
723 	while (timeout-- && serial_in(port, SPRD_STS1) & SPRD_TX_FIFO_CNT_MASK)
724 		cpu_relax();
725 
726 	/* clear interrupt */
727 	serial_out(port, SPRD_IEN, 0);
728 	serial_out(port, SPRD_ICLR, ~0);
729 
730 	/* allocate irq */
731 	sp = container_of(port, struct sprd_uart_port, port);
732 	snprintf(sp->name, sizeof(sp->name), "sprd_serial%d", port->line);
733 
734 	sprd_uart_dma_startup(port, sp);
735 
736 	ret = devm_request_irq(port->dev, port->irq, sprd_handle_irq,
737 			       IRQF_SHARED, sp->name, port);
738 	if (ret) {
739 		dev_err(port->dev, "fail to request serial irq %d, ret=%d\n",
740 			port->irq, ret);
741 		return ret;
742 	}
743 	fc = serial_in(port, SPRD_CTL1);
744 	fc |= RX_TOUT_THLD_DEF | RX_HFC_THLD_DEF;
745 	serial_out(port, SPRD_CTL1, fc);
746 
747 	/* enable interrupt */
748 	spin_lock_irqsave(&port->lock, flags);
749 	ien = serial_in(port, SPRD_IEN);
750 	ien |= SPRD_IEN_BREAK_DETECT | SPRD_IEN_TIMEOUT;
751 	if (!sp->rx_dma.enable)
752 		ien |= SPRD_IEN_RX_FULL;
753 	serial_out(port, SPRD_IEN, ien);
754 	spin_unlock_irqrestore(&port->lock, flags);
755 
756 	return 0;
757 }
758 
759 static void sprd_shutdown(struct uart_port *port)
760 {
761 	sprd_release_dma(port);
762 	serial_out(port, SPRD_IEN, 0);
763 	serial_out(port, SPRD_ICLR, ~0);
764 	devm_free_irq(port->dev, port->irq, port);
765 }
766 
767 static void sprd_set_termios(struct uart_port *port,
768 			     struct ktermios *termios,
769 			     struct ktermios *old)
770 {
771 	unsigned int baud, quot;
772 	unsigned int lcr = 0, fc;
773 	unsigned long flags;
774 
775 	/* ask the core to calculate the divisor for us */
776 	baud = uart_get_baud_rate(port, termios, old, 0, SPRD_BAUD_IO_LIMIT);
777 
778 	quot = port->uartclk / baud;
779 
780 	/* set data length */
781 	switch (termios->c_cflag & CSIZE) {
782 	case CS5:
783 		lcr |= SPRD_LCR_DATA_LEN5;
784 		break;
785 	case CS6:
786 		lcr |= SPRD_LCR_DATA_LEN6;
787 		break;
788 	case CS7:
789 		lcr |= SPRD_LCR_DATA_LEN7;
790 		break;
791 	case CS8:
792 	default:
793 		lcr |= SPRD_LCR_DATA_LEN8;
794 		break;
795 	}
796 
797 	/* calculate stop bits */
798 	lcr &= ~(SPRD_LCR_STOP_1BIT | SPRD_LCR_STOP_2BIT);
799 	if (termios->c_cflag & CSTOPB)
800 		lcr |= SPRD_LCR_STOP_2BIT;
801 	else
802 		lcr |= SPRD_LCR_STOP_1BIT;
803 
804 	/* calculate parity */
805 	lcr &= ~SPRD_LCR_PARITY;
806 	termios->c_cflag &= ~CMSPAR;	/* no support mark/space */
807 	if (termios->c_cflag & PARENB) {
808 		lcr |= SPRD_LCR_PARITY_EN;
809 		if (termios->c_cflag & PARODD)
810 			lcr |= SPRD_LCR_ODD_PAR;
811 		else
812 			lcr |= SPRD_LCR_EVEN_PAR;
813 	}
814 
815 	spin_lock_irqsave(&port->lock, flags);
816 
817 	/* update the per-port timeout */
818 	uart_update_timeout(port, termios->c_cflag, baud);
819 
820 	port->read_status_mask = SPRD_LSR_OE;
821 	if (termios->c_iflag & INPCK)
822 		port->read_status_mask |= SPRD_LSR_FE | SPRD_LSR_PE;
823 	if (termios->c_iflag & (IGNBRK | BRKINT | PARMRK))
824 		port->read_status_mask |= SPRD_LSR_BI;
825 
826 	/* characters to ignore */
827 	port->ignore_status_mask = 0;
828 	if (termios->c_iflag & IGNPAR)
829 		port->ignore_status_mask |= SPRD_LSR_PE | SPRD_LSR_FE;
830 	if (termios->c_iflag & IGNBRK) {
831 		port->ignore_status_mask |= SPRD_LSR_BI;
832 		/*
833 		 * If we're ignoring parity and break indicators,
834 		 * ignore overruns too (for real raw support).
835 		 */
836 		if (termios->c_iflag & IGNPAR)
837 			port->ignore_status_mask |= SPRD_LSR_OE;
838 	}
839 
840 	/* flow control */
841 	fc = serial_in(port, SPRD_CTL1);
842 	fc &= ~(RX_HW_FLOW_CTL_THLD | RX_HW_FLOW_CTL_EN | TX_HW_FLOW_CTL_EN);
843 	if (termios->c_cflag & CRTSCTS) {
844 		fc |= RX_HW_FLOW_CTL_THLD;
845 		fc |= RX_HW_FLOW_CTL_EN;
846 		fc |= TX_HW_FLOW_CTL_EN;
847 	}
848 
849 	/* clock divider bit0~bit15 */
850 	serial_out(port, SPRD_CLKD0, quot & SPRD_CLKD0_MASK);
851 
852 	/* clock divider bit16~bit20 */
853 	serial_out(port, SPRD_CLKD1,
854 		   (quot & SPRD_CLKD1_MASK) >> SPRD_CLKD1_SHIFT);
855 	serial_out(port, SPRD_LCR, lcr);
856 	fc |= RX_TOUT_THLD_DEF | RX_HFC_THLD_DEF;
857 	serial_out(port, SPRD_CTL1, fc);
858 
859 	spin_unlock_irqrestore(&port->lock, flags);
860 
861 	/* Don't rewrite B0 */
862 	if (tty_termios_baud_rate(termios))
863 		tty_termios_encode_baud_rate(termios, baud, baud);
864 }
865 
866 static const char *sprd_type(struct uart_port *port)
867 {
868 	return "SPX";
869 }
870 
871 static void sprd_release_port(struct uart_port *port)
872 {
873 	/* nothing to do */
874 }
875 
876 static int sprd_request_port(struct uart_port *port)
877 {
878 	return 0;
879 }
880 
881 static void sprd_config_port(struct uart_port *port, int flags)
882 {
883 	if (flags & UART_CONFIG_TYPE)
884 		port->type = PORT_SPRD;
885 }
886 
887 static int sprd_verify_port(struct uart_port *port, struct serial_struct *ser)
888 {
889 	if (ser->type != PORT_SPRD)
890 		return -EINVAL;
891 	if (port->irq != ser->irq)
892 		return -EINVAL;
893 	if (port->iotype != ser->io_type)
894 		return -EINVAL;
895 	return 0;
896 }
897 
898 static void sprd_pm(struct uart_port *port, unsigned int state,
899 		unsigned int oldstate)
900 {
901 	struct sprd_uart_port *sup =
902 		container_of(port, struct sprd_uart_port, port);
903 
904 	switch (state) {
905 	case UART_PM_STATE_ON:
906 		clk_prepare_enable(sup->clk);
907 		break;
908 	case UART_PM_STATE_OFF:
909 		clk_disable_unprepare(sup->clk);
910 		break;
911 	}
912 }
913 
914 static const struct uart_ops serial_sprd_ops = {
915 	.tx_empty = sprd_tx_empty,
916 	.get_mctrl = sprd_get_mctrl,
917 	.set_mctrl = sprd_set_mctrl,
918 	.stop_tx = sprd_stop_tx,
919 	.start_tx = sprd_start_tx,
920 	.stop_rx = sprd_stop_rx,
921 	.break_ctl = sprd_break_ctl,
922 	.startup = sprd_startup,
923 	.shutdown = sprd_shutdown,
924 	.set_termios = sprd_set_termios,
925 	.type = sprd_type,
926 	.release_port = sprd_release_port,
927 	.request_port = sprd_request_port,
928 	.config_port = sprd_config_port,
929 	.verify_port = sprd_verify_port,
930 	.pm = sprd_pm,
931 };
932 
933 #ifdef CONFIG_SERIAL_SPRD_CONSOLE
934 static void wait_for_xmitr(struct uart_port *port)
935 {
936 	unsigned int status, tmout = 10000;
937 
938 	/* wait up to 10ms for the character(s) to be sent */
939 	do {
940 		status = serial_in(port, SPRD_STS1);
941 		if (--tmout == 0)
942 			break;
943 		udelay(1);
944 	} while (status & SPRD_TX_FIFO_CNT_MASK);
945 }
946 
947 static void sprd_console_putchar(struct uart_port *port, int ch)
948 {
949 	wait_for_xmitr(port);
950 	serial_out(port, SPRD_TXD, ch);
951 }
952 
953 static void sprd_console_write(struct console *co, const char *s,
954 			       unsigned int count)
955 {
956 	struct uart_port *port = &sprd_port[co->index]->port;
957 	int locked = 1;
958 	unsigned long flags;
959 
960 	if (port->sysrq)
961 		locked = 0;
962 	else if (oops_in_progress)
963 		locked = spin_trylock_irqsave(&port->lock, flags);
964 	else
965 		spin_lock_irqsave(&port->lock, flags);
966 
967 	uart_console_write(port, s, count, sprd_console_putchar);
968 
969 	/* wait for transmitter to become empty */
970 	wait_for_xmitr(port);
971 
972 	if (locked)
973 		spin_unlock_irqrestore(&port->lock, flags);
974 }
975 
976 static int __init sprd_console_setup(struct console *co, char *options)
977 {
978 	struct uart_port *port;
979 	int baud = 115200;
980 	int bits = 8;
981 	int parity = 'n';
982 	int flow = 'n';
983 
984 	if (co->index >= UART_NR_MAX || co->index < 0)
985 		co->index = 0;
986 
987 	port = &sprd_port[co->index]->port;
988 	if (port == NULL) {
989 		pr_info("serial port %d not yet initialized\n", co->index);
990 		return -ENODEV;
991 	}
992 	if (options)
993 		uart_parse_options(options, &baud, &parity, &bits, &flow);
994 
995 	return uart_set_options(port, co, baud, parity, bits, flow);
996 }
997 
998 static struct uart_driver sprd_uart_driver;
999 static struct console sprd_console = {
1000 	.name = SPRD_TTY_NAME,
1001 	.write = sprd_console_write,
1002 	.device = uart_console_device,
1003 	.setup = sprd_console_setup,
1004 	.flags = CON_PRINTBUFFER,
1005 	.index = -1,
1006 	.data = &sprd_uart_driver,
1007 };
1008 
1009 #define SPRD_CONSOLE	(&sprd_console)
1010 
1011 /* Support for earlycon */
1012 static void sprd_putc(struct uart_port *port, int c)
1013 {
1014 	unsigned int timeout = SPRD_TIMEOUT;
1015 
1016 	while (timeout-- &&
1017 	       !(readl(port->membase + SPRD_LSR) & SPRD_LSR_TX_OVER))
1018 		cpu_relax();
1019 
1020 	writeb(c, port->membase + SPRD_TXD);
1021 }
1022 
1023 static void sprd_early_write(struct console *con, const char *s, unsigned int n)
1024 {
1025 	struct earlycon_device *dev = con->data;
1026 
1027 	uart_console_write(&dev->port, s, n, sprd_putc);
1028 }
1029 
1030 static int __init sprd_early_console_setup(struct earlycon_device *device,
1031 					   const char *opt)
1032 {
1033 	if (!device->port.membase)
1034 		return -ENODEV;
1035 
1036 	device->con->write = sprd_early_write;
1037 	return 0;
1038 }
1039 OF_EARLYCON_DECLARE(sprd_serial, "sprd,sc9836-uart",
1040 		    sprd_early_console_setup);
1041 
1042 #else /* !CONFIG_SERIAL_SPRD_CONSOLE */
1043 #define SPRD_CONSOLE		NULL
1044 #endif
1045 
1046 static struct uart_driver sprd_uart_driver = {
1047 	.owner = THIS_MODULE,
1048 	.driver_name = "sprd_serial",
1049 	.dev_name = SPRD_TTY_NAME,
1050 	.major = 0,
1051 	.minor = 0,
1052 	.nr = UART_NR_MAX,
1053 	.cons = SPRD_CONSOLE,
1054 };
1055 
1056 static int sprd_probe_dt_alias(int index, struct device *dev)
1057 {
1058 	struct device_node *np;
1059 	int ret = index;
1060 
1061 	if (!IS_ENABLED(CONFIG_OF))
1062 		return ret;
1063 
1064 	np = dev->of_node;
1065 	if (!np)
1066 		return ret;
1067 
1068 	ret = of_alias_get_id(np, "serial");
1069 	if (ret < 0)
1070 		ret = index;
1071 	else if (ret >= ARRAY_SIZE(sprd_port) || sprd_port[ret] != NULL) {
1072 		dev_warn(dev, "requested serial port %d not available.\n", ret);
1073 		ret = index;
1074 	}
1075 
1076 	return ret;
1077 }
1078 
1079 static int sprd_remove(struct platform_device *dev)
1080 {
1081 	struct sprd_uart_port *sup = platform_get_drvdata(dev);
1082 
1083 	if (sup) {
1084 		uart_remove_one_port(&sprd_uart_driver, &sup->port);
1085 		sprd_port[sup->port.line] = NULL;
1086 		sprd_ports_num--;
1087 	}
1088 
1089 	if (!sprd_ports_num)
1090 		uart_unregister_driver(&sprd_uart_driver);
1091 
1092 	sprd_rx_free_buf(sup);
1093 
1094 	return 0;
1095 }
1096 
1097 static int sprd_clk_init(struct uart_port *uport)
1098 {
1099 	struct clk *clk_uart, *clk_parent;
1100 	struct sprd_uart_port *u = sprd_port[uport->line];
1101 
1102 	clk_uart = devm_clk_get(uport->dev, "uart");
1103 	if (IS_ERR(clk_uart)) {
1104 		dev_warn(uport->dev, "uart%d can't get uart clock\n",
1105 			 uport->line);
1106 		clk_uart = NULL;
1107 	}
1108 
1109 	clk_parent = devm_clk_get(uport->dev, "source");
1110 	if (IS_ERR(clk_parent)) {
1111 		dev_warn(uport->dev, "uart%d can't get source clock\n",
1112 			 uport->line);
1113 		clk_parent = NULL;
1114 	}
1115 
1116 	if (!clk_uart || clk_set_parent(clk_uart, clk_parent))
1117 		uport->uartclk = SPRD_DEFAULT_SOURCE_CLK;
1118 	else
1119 		uport->uartclk = clk_get_rate(clk_uart);
1120 
1121 	u->clk = devm_clk_get(uport->dev, "enable");
1122 	if (IS_ERR(u->clk)) {
1123 		if (PTR_ERR(u->clk) != -EPROBE_DEFER)
1124 			dev_err(uport->dev, "uart%d can't get enable clock\n",
1125 				uport->line);
1126 		return PTR_ERR(u->clk);
1127 	}
1128 
1129 	return 0;
1130 }
1131 
1132 static int sprd_probe(struct platform_device *pdev)
1133 {
1134 	struct resource *res;
1135 	struct uart_port *up;
1136 	int irq;
1137 	int index;
1138 	int ret;
1139 
1140 	for (index = 0; index < ARRAY_SIZE(sprd_port); index++)
1141 		if (sprd_port[index] == NULL)
1142 			break;
1143 
1144 	if (index == ARRAY_SIZE(sprd_port))
1145 		return -EBUSY;
1146 
1147 	index = sprd_probe_dt_alias(index, &pdev->dev);
1148 
1149 	sprd_port[index] = devm_kzalloc(&pdev->dev, sizeof(*sprd_port[index]),
1150 					GFP_KERNEL);
1151 	if (!sprd_port[index])
1152 		return -ENOMEM;
1153 
1154 	up = &sprd_port[index]->port;
1155 	up->dev = &pdev->dev;
1156 	up->line = index;
1157 	up->type = PORT_SPRD;
1158 	up->iotype = UPIO_MEM;
1159 	up->uartclk = SPRD_DEF_RATE;
1160 	up->fifosize = SPRD_FIFO_SIZE;
1161 	up->ops = &serial_sprd_ops;
1162 	up->flags = UPF_BOOT_AUTOCONF;
1163 
1164 	ret = sprd_clk_init(up);
1165 	if (ret)
1166 		return ret;
1167 
1168 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1169 	up->membase = devm_ioremap_resource(&pdev->dev, res);
1170 	if (IS_ERR(up->membase))
1171 		return PTR_ERR(up->membase);
1172 
1173 	up->mapbase = res->start;
1174 
1175 	irq = platform_get_irq(pdev, 0);
1176 	if (irq < 0) {
1177 		dev_err(&pdev->dev, "not provide irq resource: %d\n", irq);
1178 		return irq;
1179 	}
1180 	up->irq = irq;
1181 
1182 	/*
1183 	 * Allocate one dma buffer to prepare for receive transfer, in case
1184 	 * memory allocation failure at runtime.
1185 	 */
1186 	ret = sprd_rx_alloc_buf(sprd_port[index]);
1187 	if (ret)
1188 		return ret;
1189 
1190 	if (!sprd_ports_num) {
1191 		ret = uart_register_driver(&sprd_uart_driver);
1192 		if (ret < 0) {
1193 			pr_err("Failed to register SPRD-UART driver\n");
1194 			return ret;
1195 		}
1196 	}
1197 	sprd_ports_num++;
1198 
1199 	ret = uart_add_one_port(&sprd_uart_driver, up);
1200 	if (ret) {
1201 		sprd_port[index] = NULL;
1202 		sprd_remove(pdev);
1203 	}
1204 
1205 	platform_set_drvdata(pdev, up);
1206 
1207 	return ret;
1208 }
1209 
1210 #ifdef CONFIG_PM_SLEEP
1211 static int sprd_suspend(struct device *dev)
1212 {
1213 	struct sprd_uart_port *sup = dev_get_drvdata(dev);
1214 
1215 	uart_suspend_port(&sprd_uart_driver, &sup->port);
1216 
1217 	return 0;
1218 }
1219 
1220 static int sprd_resume(struct device *dev)
1221 {
1222 	struct sprd_uart_port *sup = dev_get_drvdata(dev);
1223 
1224 	uart_resume_port(&sprd_uart_driver, &sup->port);
1225 
1226 	return 0;
1227 }
1228 #endif
1229 
1230 static SIMPLE_DEV_PM_OPS(sprd_pm_ops, sprd_suspend, sprd_resume);
1231 
1232 static const struct of_device_id serial_ids[] = {
1233 	{.compatible = "sprd,sc9836-uart",},
1234 	{}
1235 };
1236 MODULE_DEVICE_TABLE(of, serial_ids);
1237 
1238 static struct platform_driver sprd_platform_driver = {
1239 	.probe		= sprd_probe,
1240 	.remove		= sprd_remove,
1241 	.driver		= {
1242 		.name	= "sprd_serial",
1243 		.of_match_table = of_match_ptr(serial_ids),
1244 		.pm	= &sprd_pm_ops,
1245 	},
1246 };
1247 
1248 module_platform_driver(sprd_platform_driver);
1249 
1250 MODULE_LICENSE("GPL v2");
1251 MODULE_DESCRIPTION("Spreadtrum SoC serial driver series");
1252