xref: /openbmc/linux/drivers/tty/serial/amba-pl011.c (revision 09de5cd2)
1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  *  Driver for AMBA serial ports
4  *
5  *  Based on drivers/char/serial.c, by Linus Torvalds, Theodore Ts'o.
6  *
7  *  Copyright 1999 ARM Limited
8  *  Copyright (C) 2000 Deep Blue Solutions Ltd.
9  *  Copyright (C) 2010 ST-Ericsson SA
10  *
11  * This is a generic driver for ARM AMBA-type serial ports.  They
12  * have a lot of 16550-like features, but are not register compatible.
13  * Note that although they do have CTS, DCD and DSR inputs, they do
14  * not have an RI input, nor do they have DTR or RTS outputs.  If
15  * required, these have to be supplied via some other means (eg, GPIO)
16  * and hooked into this driver.
17  */
18 
19 #include <linux/module.h>
20 #include <linux/ioport.h>
21 #include <linux/init.h>
22 #include <linux/console.h>
23 #include <linux/sysrq.h>
24 #include <linux/device.h>
25 #include <linux/tty.h>
26 #include <linux/tty_flip.h>
27 #include <linux/serial_core.h>
28 #include <linux/serial.h>
29 #include <linux/amba/bus.h>
30 #include <linux/amba/serial.h>
31 #include <linux/clk.h>
32 #include <linux/slab.h>
33 #include <linux/dmaengine.h>
34 #include <linux/dma-mapping.h>
35 #include <linux/scatterlist.h>
36 #include <linux/delay.h>
37 #include <linux/types.h>
38 #include <linux/of.h>
39 #include <linux/of_device.h>
40 #include <linux/pinctrl/consumer.h>
41 #include <linux/sizes.h>
42 #include <linux/io.h>
43 #include <linux/acpi.h>
44 
45 #include "amba-pl011.h"
46 
47 #define UART_NR			14
48 
49 #define SERIAL_AMBA_MAJOR	204
50 #define SERIAL_AMBA_MINOR	64
51 #define SERIAL_AMBA_NR		UART_NR
52 
53 #define AMBA_ISR_PASS_LIMIT	256
54 
55 #define UART_DR_ERROR		(UART011_DR_OE|UART011_DR_BE|UART011_DR_PE|UART011_DR_FE)
56 #define UART_DUMMY_DR_RX	(1 << 16)
57 
58 static u16 pl011_std_offsets[REG_ARRAY_SIZE] = {
59 	[REG_DR] = UART01x_DR,
60 	[REG_FR] = UART01x_FR,
61 	[REG_LCRH_RX] = UART011_LCRH,
62 	[REG_LCRH_TX] = UART011_LCRH,
63 	[REG_IBRD] = UART011_IBRD,
64 	[REG_FBRD] = UART011_FBRD,
65 	[REG_CR] = UART011_CR,
66 	[REG_IFLS] = UART011_IFLS,
67 	[REG_IMSC] = UART011_IMSC,
68 	[REG_RIS] = UART011_RIS,
69 	[REG_MIS] = UART011_MIS,
70 	[REG_ICR] = UART011_ICR,
71 	[REG_DMACR] = UART011_DMACR,
72 };
73 
74 /* There is by now at least one vendor with differing details, so handle it */
75 struct vendor_data {
76 	const u16		*reg_offset;
77 	unsigned int		ifls;
78 	unsigned int		fr_busy;
79 	unsigned int		fr_dsr;
80 	unsigned int		fr_cts;
81 	unsigned int		fr_ri;
82 	unsigned int		inv_fr;
83 	bool			access_32b;
84 	bool			oversampling;
85 	bool			dma_threshold;
86 	bool			cts_event_workaround;
87 	bool			always_enabled;
88 	bool			fixed_options;
89 
90 	unsigned int (*get_fifosize)(struct amba_device *dev);
91 };
92 
93 static unsigned int get_fifosize_arm(struct amba_device *dev)
94 {
95 	return amba_rev(dev) < 3 ? 16 : 32;
96 }
97 
98 static struct vendor_data vendor_arm = {
99 	.reg_offset		= pl011_std_offsets,
100 	.ifls			= UART011_IFLS_RX4_8|UART011_IFLS_TX4_8,
101 	.fr_busy		= UART01x_FR_BUSY,
102 	.fr_dsr			= UART01x_FR_DSR,
103 	.fr_cts			= UART01x_FR_CTS,
104 	.fr_ri			= UART011_FR_RI,
105 	.oversampling		= false,
106 	.dma_threshold		= false,
107 	.cts_event_workaround	= false,
108 	.always_enabled		= false,
109 	.fixed_options		= false,
110 	.get_fifosize		= get_fifosize_arm,
111 };
112 
113 static const struct vendor_data vendor_sbsa = {
114 	.reg_offset		= pl011_std_offsets,
115 	.fr_busy		= UART01x_FR_BUSY,
116 	.fr_dsr			= UART01x_FR_DSR,
117 	.fr_cts			= UART01x_FR_CTS,
118 	.fr_ri			= UART011_FR_RI,
119 	.access_32b		= true,
120 	.oversampling		= false,
121 	.dma_threshold		= false,
122 	.cts_event_workaround	= false,
123 	.always_enabled		= true,
124 	.fixed_options		= true,
125 };
126 
127 #ifdef CONFIG_ACPI_SPCR_TABLE
128 static const struct vendor_data vendor_qdt_qdf2400_e44 = {
129 	.reg_offset		= pl011_std_offsets,
130 	.fr_busy		= UART011_FR_TXFE,
131 	.fr_dsr			= UART01x_FR_DSR,
132 	.fr_cts			= UART01x_FR_CTS,
133 	.fr_ri			= UART011_FR_RI,
134 	.inv_fr			= UART011_FR_TXFE,
135 	.access_32b		= true,
136 	.oversampling		= false,
137 	.dma_threshold		= false,
138 	.cts_event_workaround	= false,
139 	.always_enabled		= true,
140 	.fixed_options		= true,
141 };
142 #endif
143 
144 static u16 pl011_st_offsets[REG_ARRAY_SIZE] = {
145 	[REG_DR] = UART01x_DR,
146 	[REG_ST_DMAWM] = ST_UART011_DMAWM,
147 	[REG_ST_TIMEOUT] = ST_UART011_TIMEOUT,
148 	[REG_FR] = UART01x_FR,
149 	[REG_LCRH_RX] = ST_UART011_LCRH_RX,
150 	[REG_LCRH_TX] = ST_UART011_LCRH_TX,
151 	[REG_IBRD] = UART011_IBRD,
152 	[REG_FBRD] = UART011_FBRD,
153 	[REG_CR] = UART011_CR,
154 	[REG_IFLS] = UART011_IFLS,
155 	[REG_IMSC] = UART011_IMSC,
156 	[REG_RIS] = UART011_RIS,
157 	[REG_MIS] = UART011_MIS,
158 	[REG_ICR] = UART011_ICR,
159 	[REG_DMACR] = UART011_DMACR,
160 	[REG_ST_XFCR] = ST_UART011_XFCR,
161 	[REG_ST_XON1] = ST_UART011_XON1,
162 	[REG_ST_XON2] = ST_UART011_XON2,
163 	[REG_ST_XOFF1] = ST_UART011_XOFF1,
164 	[REG_ST_XOFF2] = ST_UART011_XOFF2,
165 	[REG_ST_ITCR] = ST_UART011_ITCR,
166 	[REG_ST_ITIP] = ST_UART011_ITIP,
167 	[REG_ST_ABCR] = ST_UART011_ABCR,
168 	[REG_ST_ABIMSC] = ST_UART011_ABIMSC,
169 };
170 
171 static unsigned int get_fifosize_st(struct amba_device *dev)
172 {
173 	return 64;
174 }
175 
176 static struct vendor_data vendor_st = {
177 	.reg_offset		= pl011_st_offsets,
178 	.ifls			= UART011_IFLS_RX_HALF|UART011_IFLS_TX_HALF,
179 	.fr_busy		= UART01x_FR_BUSY,
180 	.fr_dsr			= UART01x_FR_DSR,
181 	.fr_cts			= UART01x_FR_CTS,
182 	.fr_ri			= UART011_FR_RI,
183 	.oversampling		= true,
184 	.dma_threshold		= true,
185 	.cts_event_workaround	= true,
186 	.always_enabled		= false,
187 	.fixed_options		= false,
188 	.get_fifosize		= get_fifosize_st,
189 };
190 
191 /* Deals with DMA transactions */
192 
193 struct pl011_sgbuf {
194 	struct scatterlist sg;
195 	char *buf;
196 };
197 
198 struct pl011_dmarx_data {
199 	struct dma_chan		*chan;
200 	struct completion	complete;
201 	bool			use_buf_b;
202 	struct pl011_sgbuf	sgbuf_a;
203 	struct pl011_sgbuf	sgbuf_b;
204 	dma_cookie_t		cookie;
205 	bool			running;
206 	struct timer_list	timer;
207 	unsigned int last_residue;
208 	unsigned long last_jiffies;
209 	bool auto_poll_rate;
210 	unsigned int poll_rate;
211 	unsigned int poll_timeout;
212 };
213 
214 struct pl011_dmatx_data {
215 	struct dma_chan		*chan;
216 	struct scatterlist	sg;
217 	char			*buf;
218 	bool			queued;
219 };
220 
221 /*
222  * We wrap our port structure around the generic uart_port.
223  */
224 struct uart_amba_port {
225 	struct uart_port	port;
226 	const u16		*reg_offset;
227 	struct clk		*clk;
228 	const struct vendor_data *vendor;
229 	unsigned int		dmacr;		/* dma control reg */
230 	unsigned int		im;		/* interrupt mask */
231 	unsigned int		old_status;
232 	unsigned int		fifosize;	/* vendor-specific */
233 	unsigned int		fixed_baud;	/* vendor-set fixed baud rate */
234 	char			type[12];
235 	bool			rs485_tx_started;
236 	unsigned int		rs485_tx_drain_interval; /* usecs */
237 #ifdef CONFIG_DMA_ENGINE
238 	/* DMA stuff */
239 	bool			using_tx_dma;
240 	bool			using_rx_dma;
241 	struct pl011_dmarx_data dmarx;
242 	struct pl011_dmatx_data	dmatx;
243 	bool			dma_probed;
244 #endif
245 };
246 
247 static unsigned int pl011_tx_empty(struct uart_port *port);
248 
249 static unsigned int pl011_reg_to_offset(const struct uart_amba_port *uap,
250 	unsigned int reg)
251 {
252 	return uap->reg_offset[reg];
253 }
254 
255 static unsigned int pl011_read(const struct uart_amba_port *uap,
256 	unsigned int reg)
257 {
258 	void __iomem *addr = uap->port.membase + pl011_reg_to_offset(uap, reg);
259 
260 	return (uap->port.iotype == UPIO_MEM32) ?
261 		readl_relaxed(addr) : readw_relaxed(addr);
262 }
263 
264 static void pl011_write(unsigned int val, const struct uart_amba_port *uap,
265 	unsigned int reg)
266 {
267 	void __iomem *addr = uap->port.membase + pl011_reg_to_offset(uap, reg);
268 
269 	if (uap->port.iotype == UPIO_MEM32)
270 		writel_relaxed(val, addr);
271 	else
272 		writew_relaxed(val, addr);
273 }
274 
275 /*
276  * Reads up to 256 characters from the FIFO or until it's empty and
277  * inserts them into the TTY layer. Returns the number of characters
278  * read from the FIFO.
279  */
280 static int pl011_fifo_to_tty(struct uart_amba_port *uap)
281 {
282 	unsigned int ch, flag, fifotaken;
283 	int sysrq;
284 	u16 status;
285 
286 	for (fifotaken = 0; fifotaken != 256; fifotaken++) {
287 		status = pl011_read(uap, REG_FR);
288 		if (status & UART01x_FR_RXFE)
289 			break;
290 
291 		/* Take chars from the FIFO and update status */
292 		ch = pl011_read(uap, REG_DR) | UART_DUMMY_DR_RX;
293 		flag = TTY_NORMAL;
294 		uap->port.icount.rx++;
295 
296 		if (unlikely(ch & UART_DR_ERROR)) {
297 			if (ch & UART011_DR_BE) {
298 				ch &= ~(UART011_DR_FE | UART011_DR_PE);
299 				uap->port.icount.brk++;
300 				if (uart_handle_break(&uap->port))
301 					continue;
302 			} else if (ch & UART011_DR_PE)
303 				uap->port.icount.parity++;
304 			else if (ch & UART011_DR_FE)
305 				uap->port.icount.frame++;
306 			if (ch & UART011_DR_OE)
307 				uap->port.icount.overrun++;
308 
309 			ch &= uap->port.read_status_mask;
310 
311 			if (ch & UART011_DR_BE)
312 				flag = TTY_BREAK;
313 			else if (ch & UART011_DR_PE)
314 				flag = TTY_PARITY;
315 			else if (ch & UART011_DR_FE)
316 				flag = TTY_FRAME;
317 		}
318 
319 		spin_unlock(&uap->port.lock);
320 		sysrq = uart_handle_sysrq_char(&uap->port, ch & 255);
321 		spin_lock(&uap->port.lock);
322 
323 		if (!sysrq)
324 			uart_insert_char(&uap->port, ch, UART011_DR_OE, ch, flag);
325 	}
326 
327 	return fifotaken;
328 }
329 
330 
331 /*
332  * All the DMA operation mode stuff goes inside this ifdef.
333  * This assumes that you have a generic DMA device interface,
334  * no custom DMA interfaces are supported.
335  */
336 #ifdef CONFIG_DMA_ENGINE
337 
338 #define PL011_DMA_BUFFER_SIZE PAGE_SIZE
339 
340 static int pl011_sgbuf_init(struct dma_chan *chan, struct pl011_sgbuf *sg,
341 	enum dma_data_direction dir)
342 {
343 	dma_addr_t dma_addr;
344 
345 	sg->buf = dma_alloc_coherent(chan->device->dev,
346 		PL011_DMA_BUFFER_SIZE, &dma_addr, GFP_KERNEL);
347 	if (!sg->buf)
348 		return -ENOMEM;
349 
350 	sg_init_table(&sg->sg, 1);
351 	sg_set_page(&sg->sg, phys_to_page(dma_addr),
352 		PL011_DMA_BUFFER_SIZE, offset_in_page(dma_addr));
353 	sg_dma_address(&sg->sg) = dma_addr;
354 	sg_dma_len(&sg->sg) = PL011_DMA_BUFFER_SIZE;
355 
356 	return 0;
357 }
358 
359 static void pl011_sgbuf_free(struct dma_chan *chan, struct pl011_sgbuf *sg,
360 	enum dma_data_direction dir)
361 {
362 	if (sg->buf) {
363 		dma_free_coherent(chan->device->dev,
364 			PL011_DMA_BUFFER_SIZE, sg->buf,
365 			sg_dma_address(&sg->sg));
366 	}
367 }
368 
369 static void pl011_dma_probe(struct uart_amba_port *uap)
370 {
371 	/* DMA is the sole user of the platform data right now */
372 	struct amba_pl011_data *plat = dev_get_platdata(uap->port.dev);
373 	struct device *dev = uap->port.dev;
374 	struct dma_slave_config tx_conf = {
375 		.dst_addr = uap->port.mapbase +
376 				 pl011_reg_to_offset(uap, REG_DR),
377 		.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
378 		.direction = DMA_MEM_TO_DEV,
379 		.dst_maxburst = uap->fifosize >> 1,
380 		.device_fc = false,
381 	};
382 	struct dma_chan *chan;
383 	dma_cap_mask_t mask;
384 
385 	uap->dma_probed = true;
386 	chan = dma_request_chan(dev, "tx");
387 	if (IS_ERR(chan)) {
388 		if (PTR_ERR(chan) == -EPROBE_DEFER) {
389 			uap->dma_probed = false;
390 			return;
391 		}
392 
393 		/* We need platform data */
394 		if (!plat || !plat->dma_filter) {
395 			dev_info(uap->port.dev, "no DMA platform data\n");
396 			return;
397 		}
398 
399 		/* Try to acquire a generic DMA engine slave TX channel */
400 		dma_cap_zero(mask);
401 		dma_cap_set(DMA_SLAVE, mask);
402 
403 		chan = dma_request_channel(mask, plat->dma_filter,
404 						plat->dma_tx_param);
405 		if (!chan) {
406 			dev_err(uap->port.dev, "no TX DMA channel!\n");
407 			return;
408 		}
409 	}
410 
411 	dmaengine_slave_config(chan, &tx_conf);
412 	uap->dmatx.chan = chan;
413 
414 	dev_info(uap->port.dev, "DMA channel TX %s\n",
415 		 dma_chan_name(uap->dmatx.chan));
416 
417 	/* Optionally make use of an RX channel as well */
418 	chan = dma_request_slave_channel(dev, "rx");
419 
420 	if (!chan && plat && plat->dma_rx_param) {
421 		chan = dma_request_channel(mask, plat->dma_filter, plat->dma_rx_param);
422 
423 		if (!chan) {
424 			dev_err(uap->port.dev, "no RX DMA channel!\n");
425 			return;
426 		}
427 	}
428 
429 	if (chan) {
430 		struct dma_slave_config rx_conf = {
431 			.src_addr = uap->port.mapbase +
432 				pl011_reg_to_offset(uap, REG_DR),
433 			.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
434 			.direction = DMA_DEV_TO_MEM,
435 			.src_maxburst = uap->fifosize >> 2,
436 			.device_fc = false,
437 		};
438 		struct dma_slave_caps caps;
439 
440 		/*
441 		 * Some DMA controllers provide information on their capabilities.
442 		 * If the controller does, check for suitable residue processing
443 		 * otherwise assime all is well.
444 		 */
445 		if (0 == dma_get_slave_caps(chan, &caps)) {
446 			if (caps.residue_granularity ==
447 					DMA_RESIDUE_GRANULARITY_DESCRIPTOR) {
448 				dma_release_channel(chan);
449 				dev_info(uap->port.dev,
450 					"RX DMA disabled - no residue processing\n");
451 				return;
452 			}
453 		}
454 		dmaengine_slave_config(chan, &rx_conf);
455 		uap->dmarx.chan = chan;
456 
457 		uap->dmarx.auto_poll_rate = false;
458 		if (plat && plat->dma_rx_poll_enable) {
459 			/* Set poll rate if specified. */
460 			if (plat->dma_rx_poll_rate) {
461 				uap->dmarx.auto_poll_rate = false;
462 				uap->dmarx.poll_rate = plat->dma_rx_poll_rate;
463 			} else {
464 				/*
465 				 * 100 ms defaults to poll rate if not
466 				 * specified. This will be adjusted with
467 				 * the baud rate at set_termios.
468 				 */
469 				uap->dmarx.auto_poll_rate = true;
470 				uap->dmarx.poll_rate =  100;
471 			}
472 			/* 3 secs defaults poll_timeout if not specified. */
473 			if (plat->dma_rx_poll_timeout)
474 				uap->dmarx.poll_timeout =
475 					plat->dma_rx_poll_timeout;
476 			else
477 				uap->dmarx.poll_timeout = 3000;
478 		} else if (!plat && dev->of_node) {
479 			uap->dmarx.auto_poll_rate = of_property_read_bool(
480 						dev->of_node, "auto-poll");
481 			if (uap->dmarx.auto_poll_rate) {
482 				u32 x;
483 
484 				if (0 == of_property_read_u32(dev->of_node,
485 						"poll-rate-ms", &x))
486 					uap->dmarx.poll_rate = x;
487 				else
488 					uap->dmarx.poll_rate = 100;
489 				if (0 == of_property_read_u32(dev->of_node,
490 						"poll-timeout-ms", &x))
491 					uap->dmarx.poll_timeout = x;
492 				else
493 					uap->dmarx.poll_timeout = 3000;
494 			}
495 		}
496 		dev_info(uap->port.dev, "DMA channel RX %s\n",
497 			 dma_chan_name(uap->dmarx.chan));
498 	}
499 }
500 
501 static void pl011_dma_remove(struct uart_amba_port *uap)
502 {
503 	if (uap->dmatx.chan)
504 		dma_release_channel(uap->dmatx.chan);
505 	if (uap->dmarx.chan)
506 		dma_release_channel(uap->dmarx.chan);
507 }
508 
509 /* Forward declare these for the refill routine */
510 static int pl011_dma_tx_refill(struct uart_amba_port *uap);
511 static void pl011_start_tx_pio(struct uart_amba_port *uap);
512 
513 /*
514  * The current DMA TX buffer has been sent.
515  * Try to queue up another DMA buffer.
516  */
517 static void pl011_dma_tx_callback(void *data)
518 {
519 	struct uart_amba_port *uap = data;
520 	struct pl011_dmatx_data *dmatx = &uap->dmatx;
521 	unsigned long flags;
522 	u16 dmacr;
523 
524 	spin_lock_irqsave(&uap->port.lock, flags);
525 	if (uap->dmatx.queued)
526 		dma_unmap_sg(dmatx->chan->device->dev, &dmatx->sg, 1,
527 			     DMA_TO_DEVICE);
528 
529 	dmacr = uap->dmacr;
530 	uap->dmacr = dmacr & ~UART011_TXDMAE;
531 	pl011_write(uap->dmacr, uap, REG_DMACR);
532 
533 	/*
534 	 * If TX DMA was disabled, it means that we've stopped the DMA for
535 	 * some reason (eg, XOFF received, or we want to send an X-char.)
536 	 *
537 	 * Note: we need to be careful here of a potential race between DMA
538 	 * and the rest of the driver - if the driver disables TX DMA while
539 	 * a TX buffer completing, we must update the tx queued status to
540 	 * get further refills (hence we check dmacr).
541 	 */
542 	if (!(dmacr & UART011_TXDMAE) || uart_tx_stopped(&uap->port) ||
543 	    uart_circ_empty(&uap->port.state->xmit)) {
544 		uap->dmatx.queued = false;
545 		spin_unlock_irqrestore(&uap->port.lock, flags);
546 		return;
547 	}
548 
549 	if (pl011_dma_tx_refill(uap) <= 0)
550 		/*
551 		 * We didn't queue a DMA buffer for some reason, but we
552 		 * have data pending to be sent.  Re-enable the TX IRQ.
553 		 */
554 		pl011_start_tx_pio(uap);
555 
556 	spin_unlock_irqrestore(&uap->port.lock, flags);
557 }
558 
559 /*
560  * Try to refill the TX DMA buffer.
561  * Locking: called with port lock held and IRQs disabled.
562  * Returns:
563  *   1 if we queued up a TX DMA buffer.
564  *   0 if we didn't want to handle this by DMA
565  *  <0 on error
566  */
567 static int pl011_dma_tx_refill(struct uart_amba_port *uap)
568 {
569 	struct pl011_dmatx_data *dmatx = &uap->dmatx;
570 	struct dma_chan *chan = dmatx->chan;
571 	struct dma_device *dma_dev = chan->device;
572 	struct dma_async_tx_descriptor *desc;
573 	struct circ_buf *xmit = &uap->port.state->xmit;
574 	unsigned int count;
575 
576 	/*
577 	 * Try to avoid the overhead involved in using DMA if the
578 	 * transaction fits in the first half of the FIFO, by using
579 	 * the standard interrupt handling.  This ensures that we
580 	 * issue a uart_write_wakeup() at the appropriate time.
581 	 */
582 	count = uart_circ_chars_pending(xmit);
583 	if (count < (uap->fifosize >> 1)) {
584 		uap->dmatx.queued = false;
585 		return 0;
586 	}
587 
588 	/*
589 	 * Bodge: don't send the last character by DMA, as this
590 	 * will prevent XON from notifying us to restart DMA.
591 	 */
592 	count -= 1;
593 
594 	/* Else proceed to copy the TX chars to the DMA buffer and fire DMA */
595 	if (count > PL011_DMA_BUFFER_SIZE)
596 		count = PL011_DMA_BUFFER_SIZE;
597 
598 	if (xmit->tail < xmit->head)
599 		memcpy(&dmatx->buf[0], &xmit->buf[xmit->tail], count);
600 	else {
601 		size_t first = UART_XMIT_SIZE - xmit->tail;
602 		size_t second;
603 
604 		if (first > count)
605 			first = count;
606 		second = count - first;
607 
608 		memcpy(&dmatx->buf[0], &xmit->buf[xmit->tail], first);
609 		if (second)
610 			memcpy(&dmatx->buf[first], &xmit->buf[0], second);
611 	}
612 
613 	dmatx->sg.length = count;
614 
615 	if (dma_map_sg(dma_dev->dev, &dmatx->sg, 1, DMA_TO_DEVICE) != 1) {
616 		uap->dmatx.queued = false;
617 		dev_dbg(uap->port.dev, "unable to map TX DMA\n");
618 		return -EBUSY;
619 	}
620 
621 	desc = dmaengine_prep_slave_sg(chan, &dmatx->sg, 1, DMA_MEM_TO_DEV,
622 					     DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
623 	if (!desc) {
624 		dma_unmap_sg(dma_dev->dev, &dmatx->sg, 1, DMA_TO_DEVICE);
625 		uap->dmatx.queued = false;
626 		/*
627 		 * If DMA cannot be used right now, we complete this
628 		 * transaction via IRQ and let the TTY layer retry.
629 		 */
630 		dev_dbg(uap->port.dev, "TX DMA busy\n");
631 		return -EBUSY;
632 	}
633 
634 	/* Some data to go along to the callback */
635 	desc->callback = pl011_dma_tx_callback;
636 	desc->callback_param = uap;
637 
638 	/* All errors should happen at prepare time */
639 	dmaengine_submit(desc);
640 
641 	/* Fire the DMA transaction */
642 	dma_dev->device_issue_pending(chan);
643 
644 	uap->dmacr |= UART011_TXDMAE;
645 	pl011_write(uap->dmacr, uap, REG_DMACR);
646 	uap->dmatx.queued = true;
647 
648 	/*
649 	 * Now we know that DMA will fire, so advance the ring buffer
650 	 * with the stuff we just dispatched.
651 	 */
652 	xmit->tail = (xmit->tail + count) & (UART_XMIT_SIZE - 1);
653 	uap->port.icount.tx += count;
654 
655 	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
656 		uart_write_wakeup(&uap->port);
657 
658 	return 1;
659 }
660 
661 /*
662  * We received a transmit interrupt without a pending X-char but with
663  * pending characters.
664  * Locking: called with port lock held and IRQs disabled.
665  * Returns:
666  *   false if we want to use PIO to transmit
667  *   true if we queued a DMA buffer
668  */
669 static bool pl011_dma_tx_irq(struct uart_amba_port *uap)
670 {
671 	if (!uap->using_tx_dma)
672 		return false;
673 
674 	/*
675 	 * If we already have a TX buffer queued, but received a
676 	 * TX interrupt, it will be because we've just sent an X-char.
677 	 * Ensure the TX DMA is enabled and the TX IRQ is disabled.
678 	 */
679 	if (uap->dmatx.queued) {
680 		uap->dmacr |= UART011_TXDMAE;
681 		pl011_write(uap->dmacr, uap, REG_DMACR);
682 		uap->im &= ~UART011_TXIM;
683 		pl011_write(uap->im, uap, REG_IMSC);
684 		return true;
685 	}
686 
687 	/*
688 	 * We don't have a TX buffer queued, so try to queue one.
689 	 * If we successfully queued a buffer, mask the TX IRQ.
690 	 */
691 	if (pl011_dma_tx_refill(uap) > 0) {
692 		uap->im &= ~UART011_TXIM;
693 		pl011_write(uap->im, uap, REG_IMSC);
694 		return true;
695 	}
696 	return false;
697 }
698 
699 /*
700  * Stop the DMA transmit (eg, due to received XOFF).
701  * Locking: called with port lock held and IRQs disabled.
702  */
703 static inline void pl011_dma_tx_stop(struct uart_amba_port *uap)
704 {
705 	if (uap->dmatx.queued) {
706 		uap->dmacr &= ~UART011_TXDMAE;
707 		pl011_write(uap->dmacr, uap, REG_DMACR);
708 	}
709 }
710 
711 /*
712  * Try to start a DMA transmit, or in the case of an XON/OFF
713  * character queued for send, try to get that character out ASAP.
714  * Locking: called with port lock held and IRQs disabled.
715  * Returns:
716  *   false if we want the TX IRQ to be enabled
717  *   true if we have a buffer queued
718  */
719 static inline bool pl011_dma_tx_start(struct uart_amba_port *uap)
720 {
721 	u16 dmacr;
722 
723 	if (!uap->using_tx_dma)
724 		return false;
725 
726 	if (!uap->port.x_char) {
727 		/* no X-char, try to push chars out in DMA mode */
728 		bool ret = true;
729 
730 		if (!uap->dmatx.queued) {
731 			if (pl011_dma_tx_refill(uap) > 0) {
732 				uap->im &= ~UART011_TXIM;
733 				pl011_write(uap->im, uap, REG_IMSC);
734 			} else
735 				ret = false;
736 		} else if (!(uap->dmacr & UART011_TXDMAE)) {
737 			uap->dmacr |= UART011_TXDMAE;
738 			pl011_write(uap->dmacr, uap, REG_DMACR);
739 		}
740 		return ret;
741 	}
742 
743 	/*
744 	 * We have an X-char to send.  Disable DMA to prevent it loading
745 	 * the TX fifo, and then see if we can stuff it into the FIFO.
746 	 */
747 	dmacr = uap->dmacr;
748 	uap->dmacr &= ~UART011_TXDMAE;
749 	pl011_write(uap->dmacr, uap, REG_DMACR);
750 
751 	if (pl011_read(uap, REG_FR) & UART01x_FR_TXFF) {
752 		/*
753 		 * No space in the FIFO, so enable the transmit interrupt
754 		 * so we know when there is space.  Note that once we've
755 		 * loaded the character, we should just re-enable DMA.
756 		 */
757 		return false;
758 	}
759 
760 	pl011_write(uap->port.x_char, uap, REG_DR);
761 	uap->port.icount.tx++;
762 	uap->port.x_char = 0;
763 
764 	/* Success - restore the DMA state */
765 	uap->dmacr = dmacr;
766 	pl011_write(dmacr, uap, REG_DMACR);
767 
768 	return true;
769 }
770 
771 /*
772  * Flush the transmit buffer.
773  * Locking: called with port lock held and IRQs disabled.
774  */
775 static void pl011_dma_flush_buffer(struct uart_port *port)
776 __releases(&uap->port.lock)
777 __acquires(&uap->port.lock)
778 {
779 	struct uart_amba_port *uap =
780 	    container_of(port, struct uart_amba_port, port);
781 
782 	if (!uap->using_tx_dma)
783 		return;
784 
785 	dmaengine_terminate_async(uap->dmatx.chan);
786 
787 	if (uap->dmatx.queued) {
788 		dma_unmap_sg(uap->dmatx.chan->device->dev, &uap->dmatx.sg, 1,
789 			     DMA_TO_DEVICE);
790 		uap->dmatx.queued = false;
791 		uap->dmacr &= ~UART011_TXDMAE;
792 		pl011_write(uap->dmacr, uap, REG_DMACR);
793 	}
794 }
795 
796 static void pl011_dma_rx_callback(void *data);
797 
798 static int pl011_dma_rx_trigger_dma(struct uart_amba_port *uap)
799 {
800 	struct dma_chan *rxchan = uap->dmarx.chan;
801 	struct pl011_dmarx_data *dmarx = &uap->dmarx;
802 	struct dma_async_tx_descriptor *desc;
803 	struct pl011_sgbuf *sgbuf;
804 
805 	if (!rxchan)
806 		return -EIO;
807 
808 	/* Start the RX DMA job */
809 	sgbuf = uap->dmarx.use_buf_b ?
810 		&uap->dmarx.sgbuf_b : &uap->dmarx.sgbuf_a;
811 	desc = dmaengine_prep_slave_sg(rxchan, &sgbuf->sg, 1,
812 					DMA_DEV_TO_MEM,
813 					DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
814 	/*
815 	 * If the DMA engine is busy and cannot prepare a
816 	 * channel, no big deal, the driver will fall back
817 	 * to interrupt mode as a result of this error code.
818 	 */
819 	if (!desc) {
820 		uap->dmarx.running = false;
821 		dmaengine_terminate_all(rxchan);
822 		return -EBUSY;
823 	}
824 
825 	/* Some data to go along to the callback */
826 	desc->callback = pl011_dma_rx_callback;
827 	desc->callback_param = uap;
828 	dmarx->cookie = dmaengine_submit(desc);
829 	dma_async_issue_pending(rxchan);
830 
831 	uap->dmacr |= UART011_RXDMAE;
832 	pl011_write(uap->dmacr, uap, REG_DMACR);
833 	uap->dmarx.running = true;
834 
835 	uap->im &= ~UART011_RXIM;
836 	pl011_write(uap->im, uap, REG_IMSC);
837 
838 	return 0;
839 }
840 
841 /*
842  * This is called when either the DMA job is complete, or
843  * the FIFO timeout interrupt occurred. This must be called
844  * with the port spinlock uap->port.lock held.
845  */
846 static void pl011_dma_rx_chars(struct uart_amba_port *uap,
847 			       u32 pending, bool use_buf_b,
848 			       bool readfifo)
849 {
850 	struct tty_port *port = &uap->port.state->port;
851 	struct pl011_sgbuf *sgbuf = use_buf_b ?
852 		&uap->dmarx.sgbuf_b : &uap->dmarx.sgbuf_a;
853 	int dma_count = 0;
854 	u32 fifotaken = 0; /* only used for vdbg() */
855 
856 	struct pl011_dmarx_data *dmarx = &uap->dmarx;
857 	int dmataken = 0;
858 
859 	if (uap->dmarx.poll_rate) {
860 		/* The data can be taken by polling */
861 		dmataken = sgbuf->sg.length - dmarx->last_residue;
862 		/* Recalculate the pending size */
863 		if (pending >= dmataken)
864 			pending -= dmataken;
865 	}
866 
867 	/* Pick the remain data from the DMA */
868 	if (pending) {
869 
870 		/*
871 		 * First take all chars in the DMA pipe, then look in the FIFO.
872 		 * Note that tty_insert_flip_buf() tries to take as many chars
873 		 * as it can.
874 		 */
875 		dma_count = tty_insert_flip_string(port, sgbuf->buf + dmataken,
876 				pending);
877 
878 		uap->port.icount.rx += dma_count;
879 		if (dma_count < pending)
880 			dev_warn(uap->port.dev,
881 				 "couldn't insert all characters (TTY is full?)\n");
882 	}
883 
884 	/* Reset the last_residue for Rx DMA poll */
885 	if (uap->dmarx.poll_rate)
886 		dmarx->last_residue = sgbuf->sg.length;
887 
888 	/*
889 	 * Only continue with trying to read the FIFO if all DMA chars have
890 	 * been taken first.
891 	 */
892 	if (dma_count == pending && readfifo) {
893 		/* Clear any error flags */
894 		pl011_write(UART011_OEIS | UART011_BEIS | UART011_PEIS |
895 			    UART011_FEIS, uap, REG_ICR);
896 
897 		/*
898 		 * If we read all the DMA'd characters, and we had an
899 		 * incomplete buffer, that could be due to an rx error, or
900 		 * maybe we just timed out. Read any pending chars and check
901 		 * the error status.
902 		 *
903 		 * Error conditions will only occur in the FIFO, these will
904 		 * trigger an immediate interrupt and stop the DMA job, so we
905 		 * will always find the error in the FIFO, never in the DMA
906 		 * buffer.
907 		 */
908 		fifotaken = pl011_fifo_to_tty(uap);
909 	}
910 
911 	dev_vdbg(uap->port.dev,
912 		 "Took %d chars from DMA buffer and %d chars from the FIFO\n",
913 		 dma_count, fifotaken);
914 	tty_flip_buffer_push(port);
915 }
916 
917 static void pl011_dma_rx_irq(struct uart_amba_port *uap)
918 {
919 	struct pl011_dmarx_data *dmarx = &uap->dmarx;
920 	struct dma_chan *rxchan = dmarx->chan;
921 	struct pl011_sgbuf *sgbuf = dmarx->use_buf_b ?
922 		&dmarx->sgbuf_b : &dmarx->sgbuf_a;
923 	size_t pending;
924 	struct dma_tx_state state;
925 	enum dma_status dmastat;
926 
927 	/*
928 	 * Pause the transfer so we can trust the current counter,
929 	 * do this before we pause the PL011 block, else we may
930 	 * overflow the FIFO.
931 	 */
932 	if (dmaengine_pause(rxchan))
933 		dev_err(uap->port.dev, "unable to pause DMA transfer\n");
934 	dmastat = rxchan->device->device_tx_status(rxchan,
935 						   dmarx->cookie, &state);
936 	if (dmastat != DMA_PAUSED)
937 		dev_err(uap->port.dev, "unable to pause DMA transfer\n");
938 
939 	/* Disable RX DMA - incoming data will wait in the FIFO */
940 	uap->dmacr &= ~UART011_RXDMAE;
941 	pl011_write(uap->dmacr, uap, REG_DMACR);
942 	uap->dmarx.running = false;
943 
944 	pending = sgbuf->sg.length - state.residue;
945 	BUG_ON(pending > PL011_DMA_BUFFER_SIZE);
946 	/* Then we terminate the transfer - we now know our residue */
947 	dmaengine_terminate_all(rxchan);
948 
949 	/*
950 	 * This will take the chars we have so far and insert
951 	 * into the framework.
952 	 */
953 	pl011_dma_rx_chars(uap, pending, dmarx->use_buf_b, true);
954 
955 	/* Switch buffer & re-trigger DMA job */
956 	dmarx->use_buf_b = !dmarx->use_buf_b;
957 	if (pl011_dma_rx_trigger_dma(uap)) {
958 		dev_dbg(uap->port.dev, "could not retrigger RX DMA job "
959 			"fall back to interrupt mode\n");
960 		uap->im |= UART011_RXIM;
961 		pl011_write(uap->im, uap, REG_IMSC);
962 	}
963 }
964 
965 static void pl011_dma_rx_callback(void *data)
966 {
967 	struct uart_amba_port *uap = data;
968 	struct pl011_dmarx_data *dmarx = &uap->dmarx;
969 	struct dma_chan *rxchan = dmarx->chan;
970 	bool lastbuf = dmarx->use_buf_b;
971 	struct pl011_sgbuf *sgbuf = dmarx->use_buf_b ?
972 		&dmarx->sgbuf_b : &dmarx->sgbuf_a;
973 	size_t pending;
974 	struct dma_tx_state state;
975 	int ret;
976 
977 	/*
978 	 * This completion interrupt occurs typically when the
979 	 * RX buffer is totally stuffed but no timeout has yet
980 	 * occurred. When that happens, we just want the RX
981 	 * routine to flush out the secondary DMA buffer while
982 	 * we immediately trigger the next DMA job.
983 	 */
984 	spin_lock_irq(&uap->port.lock);
985 	/*
986 	 * Rx data can be taken by the UART interrupts during
987 	 * the DMA irq handler. So we check the residue here.
988 	 */
989 	rxchan->device->device_tx_status(rxchan, dmarx->cookie, &state);
990 	pending = sgbuf->sg.length - state.residue;
991 	BUG_ON(pending > PL011_DMA_BUFFER_SIZE);
992 	/* Then we terminate the transfer - we now know our residue */
993 	dmaengine_terminate_all(rxchan);
994 
995 	uap->dmarx.running = false;
996 	dmarx->use_buf_b = !lastbuf;
997 	ret = pl011_dma_rx_trigger_dma(uap);
998 
999 	pl011_dma_rx_chars(uap, pending, lastbuf, false);
1000 	spin_unlock_irq(&uap->port.lock);
1001 	/*
1002 	 * Do this check after we picked the DMA chars so we don't
1003 	 * get some IRQ immediately from RX.
1004 	 */
1005 	if (ret) {
1006 		dev_dbg(uap->port.dev, "could not retrigger RX DMA job "
1007 			"fall back to interrupt mode\n");
1008 		uap->im |= UART011_RXIM;
1009 		pl011_write(uap->im, uap, REG_IMSC);
1010 	}
1011 }
1012 
1013 /*
1014  * Stop accepting received characters, when we're shutting down or
1015  * suspending this port.
1016  * Locking: called with port lock held and IRQs disabled.
1017  */
1018 static inline void pl011_dma_rx_stop(struct uart_amba_port *uap)
1019 {
1020 	/* FIXME.  Just disable the DMA enable */
1021 	uap->dmacr &= ~UART011_RXDMAE;
1022 	pl011_write(uap->dmacr, uap, REG_DMACR);
1023 }
1024 
1025 /*
1026  * Timer handler for Rx DMA polling.
1027  * Every polling, It checks the residue in the dma buffer and transfer
1028  * data to the tty. Also, last_residue is updated for the next polling.
1029  */
1030 static void pl011_dma_rx_poll(struct timer_list *t)
1031 {
1032 	struct uart_amba_port *uap = from_timer(uap, t, dmarx.timer);
1033 	struct tty_port *port = &uap->port.state->port;
1034 	struct pl011_dmarx_data *dmarx = &uap->dmarx;
1035 	struct dma_chan *rxchan = uap->dmarx.chan;
1036 	unsigned long flags;
1037 	unsigned int dmataken = 0;
1038 	unsigned int size = 0;
1039 	struct pl011_sgbuf *sgbuf;
1040 	int dma_count;
1041 	struct dma_tx_state state;
1042 
1043 	sgbuf = dmarx->use_buf_b ? &uap->dmarx.sgbuf_b : &uap->dmarx.sgbuf_a;
1044 	rxchan->device->device_tx_status(rxchan, dmarx->cookie, &state);
1045 	if (likely(state.residue < dmarx->last_residue)) {
1046 		dmataken = sgbuf->sg.length - dmarx->last_residue;
1047 		size = dmarx->last_residue - state.residue;
1048 		dma_count = tty_insert_flip_string(port, sgbuf->buf + dmataken,
1049 				size);
1050 		if (dma_count == size)
1051 			dmarx->last_residue =  state.residue;
1052 		dmarx->last_jiffies = jiffies;
1053 	}
1054 	tty_flip_buffer_push(port);
1055 
1056 	/*
1057 	 * If no data is received in poll_timeout, the driver will fall back
1058 	 * to interrupt mode. We will retrigger DMA at the first interrupt.
1059 	 */
1060 	if (jiffies_to_msecs(jiffies - dmarx->last_jiffies)
1061 			> uap->dmarx.poll_timeout) {
1062 
1063 		spin_lock_irqsave(&uap->port.lock, flags);
1064 		pl011_dma_rx_stop(uap);
1065 		uap->im |= UART011_RXIM;
1066 		pl011_write(uap->im, uap, REG_IMSC);
1067 		spin_unlock_irqrestore(&uap->port.lock, flags);
1068 
1069 		uap->dmarx.running = false;
1070 		dmaengine_terminate_all(rxchan);
1071 		del_timer(&uap->dmarx.timer);
1072 	} else {
1073 		mod_timer(&uap->dmarx.timer,
1074 			jiffies + msecs_to_jiffies(uap->dmarx.poll_rate));
1075 	}
1076 }
1077 
1078 static void pl011_dma_startup(struct uart_amba_port *uap)
1079 {
1080 	int ret;
1081 
1082 	if (!uap->dma_probed)
1083 		pl011_dma_probe(uap);
1084 
1085 	if (!uap->dmatx.chan)
1086 		return;
1087 
1088 	uap->dmatx.buf = kmalloc(PL011_DMA_BUFFER_SIZE, GFP_KERNEL | __GFP_DMA);
1089 	if (!uap->dmatx.buf) {
1090 		dev_err(uap->port.dev, "no memory for DMA TX buffer\n");
1091 		uap->port.fifosize = uap->fifosize;
1092 		return;
1093 	}
1094 
1095 	sg_init_one(&uap->dmatx.sg, uap->dmatx.buf, PL011_DMA_BUFFER_SIZE);
1096 
1097 	/* The DMA buffer is now the FIFO the TTY subsystem can use */
1098 	uap->port.fifosize = PL011_DMA_BUFFER_SIZE;
1099 	uap->using_tx_dma = true;
1100 
1101 	if (!uap->dmarx.chan)
1102 		goto skip_rx;
1103 
1104 	/* Allocate and map DMA RX buffers */
1105 	ret = pl011_sgbuf_init(uap->dmarx.chan, &uap->dmarx.sgbuf_a,
1106 			       DMA_FROM_DEVICE);
1107 	if (ret) {
1108 		dev_err(uap->port.dev, "failed to init DMA %s: %d\n",
1109 			"RX buffer A", ret);
1110 		goto skip_rx;
1111 	}
1112 
1113 	ret = pl011_sgbuf_init(uap->dmarx.chan, &uap->dmarx.sgbuf_b,
1114 			       DMA_FROM_DEVICE);
1115 	if (ret) {
1116 		dev_err(uap->port.dev, "failed to init DMA %s: %d\n",
1117 			"RX buffer B", ret);
1118 		pl011_sgbuf_free(uap->dmarx.chan, &uap->dmarx.sgbuf_a,
1119 				 DMA_FROM_DEVICE);
1120 		goto skip_rx;
1121 	}
1122 
1123 	uap->using_rx_dma = true;
1124 
1125 skip_rx:
1126 	/* Turn on DMA error (RX/TX will be enabled on demand) */
1127 	uap->dmacr |= UART011_DMAONERR;
1128 	pl011_write(uap->dmacr, uap, REG_DMACR);
1129 
1130 	/*
1131 	 * ST Micro variants has some specific dma burst threshold
1132 	 * compensation. Set this to 16 bytes, so burst will only
1133 	 * be issued above/below 16 bytes.
1134 	 */
1135 	if (uap->vendor->dma_threshold)
1136 		pl011_write(ST_UART011_DMAWM_RX_16 | ST_UART011_DMAWM_TX_16,
1137 			    uap, REG_ST_DMAWM);
1138 
1139 	if (uap->using_rx_dma) {
1140 		if (pl011_dma_rx_trigger_dma(uap))
1141 			dev_dbg(uap->port.dev, "could not trigger initial "
1142 				"RX DMA job, fall back to interrupt mode\n");
1143 		if (uap->dmarx.poll_rate) {
1144 			timer_setup(&uap->dmarx.timer, pl011_dma_rx_poll, 0);
1145 			mod_timer(&uap->dmarx.timer,
1146 				jiffies +
1147 				msecs_to_jiffies(uap->dmarx.poll_rate));
1148 			uap->dmarx.last_residue = PL011_DMA_BUFFER_SIZE;
1149 			uap->dmarx.last_jiffies = jiffies;
1150 		}
1151 	}
1152 }
1153 
1154 static void pl011_dma_shutdown(struct uart_amba_port *uap)
1155 {
1156 	if (!(uap->using_tx_dma || uap->using_rx_dma))
1157 		return;
1158 
1159 	/* Disable RX and TX DMA */
1160 	while (pl011_read(uap, REG_FR) & uap->vendor->fr_busy)
1161 		cpu_relax();
1162 
1163 	spin_lock_irq(&uap->port.lock);
1164 	uap->dmacr &= ~(UART011_DMAONERR | UART011_RXDMAE | UART011_TXDMAE);
1165 	pl011_write(uap->dmacr, uap, REG_DMACR);
1166 	spin_unlock_irq(&uap->port.lock);
1167 
1168 	if (uap->using_tx_dma) {
1169 		/* In theory, this should already be done by pl011_dma_flush_buffer */
1170 		dmaengine_terminate_all(uap->dmatx.chan);
1171 		if (uap->dmatx.queued) {
1172 			dma_unmap_sg(uap->dmatx.chan->device->dev, &uap->dmatx.sg, 1,
1173 				     DMA_TO_DEVICE);
1174 			uap->dmatx.queued = false;
1175 		}
1176 
1177 		kfree(uap->dmatx.buf);
1178 		uap->using_tx_dma = false;
1179 	}
1180 
1181 	if (uap->using_rx_dma) {
1182 		dmaengine_terminate_all(uap->dmarx.chan);
1183 		/* Clean up the RX DMA */
1184 		pl011_sgbuf_free(uap->dmarx.chan, &uap->dmarx.sgbuf_a, DMA_FROM_DEVICE);
1185 		pl011_sgbuf_free(uap->dmarx.chan, &uap->dmarx.sgbuf_b, DMA_FROM_DEVICE);
1186 		if (uap->dmarx.poll_rate)
1187 			del_timer_sync(&uap->dmarx.timer);
1188 		uap->using_rx_dma = false;
1189 	}
1190 }
1191 
1192 static inline bool pl011_dma_rx_available(struct uart_amba_port *uap)
1193 {
1194 	return uap->using_rx_dma;
1195 }
1196 
1197 static inline bool pl011_dma_rx_running(struct uart_amba_port *uap)
1198 {
1199 	return uap->using_rx_dma && uap->dmarx.running;
1200 }
1201 
1202 #else
1203 /* Blank functions if the DMA engine is not available */
1204 static inline void pl011_dma_remove(struct uart_amba_port *uap)
1205 {
1206 }
1207 
1208 static inline void pl011_dma_startup(struct uart_amba_port *uap)
1209 {
1210 }
1211 
1212 static inline void pl011_dma_shutdown(struct uart_amba_port *uap)
1213 {
1214 }
1215 
1216 static inline bool pl011_dma_tx_irq(struct uart_amba_port *uap)
1217 {
1218 	return false;
1219 }
1220 
1221 static inline void pl011_dma_tx_stop(struct uart_amba_port *uap)
1222 {
1223 }
1224 
1225 static inline bool pl011_dma_tx_start(struct uart_amba_port *uap)
1226 {
1227 	return false;
1228 }
1229 
1230 static inline void pl011_dma_rx_irq(struct uart_amba_port *uap)
1231 {
1232 }
1233 
1234 static inline void pl011_dma_rx_stop(struct uart_amba_port *uap)
1235 {
1236 }
1237 
1238 static inline int pl011_dma_rx_trigger_dma(struct uart_amba_port *uap)
1239 {
1240 	return -EIO;
1241 }
1242 
1243 static inline bool pl011_dma_rx_available(struct uart_amba_port *uap)
1244 {
1245 	return false;
1246 }
1247 
1248 static inline bool pl011_dma_rx_running(struct uart_amba_port *uap)
1249 {
1250 	return false;
1251 }
1252 
1253 #define pl011_dma_flush_buffer	NULL
1254 #endif
1255 
1256 static void pl011_rs485_tx_stop(struct uart_amba_port *uap)
1257 {
1258 	/*
1259 	 * To be on the safe side only time out after twice as many iterations
1260 	 * as fifo size.
1261 	 */
1262 	const int MAX_TX_DRAIN_ITERS = uap->port.fifosize * 2;
1263 	struct uart_port *port = &uap->port;
1264 	int i = 0;
1265 	u32 cr;
1266 
1267 	/* Wait until hardware tx queue is empty */
1268 	while (!pl011_tx_empty(port)) {
1269 		if (i > MAX_TX_DRAIN_ITERS) {
1270 			dev_warn(port->dev,
1271 				 "timeout while draining hardware tx queue\n");
1272 			break;
1273 		}
1274 
1275 		udelay(uap->rs485_tx_drain_interval);
1276 		i++;
1277 	}
1278 
1279 	if (port->rs485.delay_rts_after_send)
1280 		mdelay(port->rs485.delay_rts_after_send);
1281 
1282 	cr = pl011_read(uap, REG_CR);
1283 
1284 	if (port->rs485.flags & SER_RS485_RTS_AFTER_SEND)
1285 		cr &= ~UART011_CR_RTS;
1286 	else
1287 		cr |= UART011_CR_RTS;
1288 
1289 	/* Disable the transmitter and reenable the transceiver */
1290 	cr &= ~UART011_CR_TXE;
1291 	cr |= UART011_CR_RXE;
1292 	pl011_write(cr, uap, REG_CR);
1293 
1294 	uap->rs485_tx_started = false;
1295 }
1296 
1297 static void pl011_stop_tx(struct uart_port *port)
1298 {
1299 	struct uart_amba_port *uap =
1300 	    container_of(port, struct uart_amba_port, port);
1301 
1302 	uap->im &= ~UART011_TXIM;
1303 	pl011_write(uap->im, uap, REG_IMSC);
1304 	pl011_dma_tx_stop(uap);
1305 
1306 	if ((port->rs485.flags & SER_RS485_ENABLED) && uap->rs485_tx_started)
1307 		pl011_rs485_tx_stop(uap);
1308 }
1309 
1310 static bool pl011_tx_chars(struct uart_amba_port *uap, bool from_irq);
1311 
1312 /* Start TX with programmed I/O only (no DMA) */
1313 static void pl011_start_tx_pio(struct uart_amba_port *uap)
1314 {
1315 	if (pl011_tx_chars(uap, false)) {
1316 		uap->im |= UART011_TXIM;
1317 		pl011_write(uap->im, uap, REG_IMSC);
1318 	}
1319 }
1320 
1321 static void pl011_start_tx(struct uart_port *port)
1322 {
1323 	struct uart_amba_port *uap =
1324 	    container_of(port, struct uart_amba_port, port);
1325 
1326 	if (!pl011_dma_tx_start(uap))
1327 		pl011_start_tx_pio(uap);
1328 }
1329 
1330 static void pl011_stop_rx(struct uart_port *port)
1331 {
1332 	struct uart_amba_port *uap =
1333 	    container_of(port, struct uart_amba_port, port);
1334 
1335 	uap->im &= ~(UART011_RXIM|UART011_RTIM|UART011_FEIM|
1336 		     UART011_PEIM|UART011_BEIM|UART011_OEIM);
1337 	pl011_write(uap->im, uap, REG_IMSC);
1338 
1339 	pl011_dma_rx_stop(uap);
1340 }
1341 
1342 static void pl011_enable_ms(struct uart_port *port)
1343 {
1344 	struct uart_amba_port *uap =
1345 	    container_of(port, struct uart_amba_port, port);
1346 
1347 	uap->im |= UART011_RIMIM|UART011_CTSMIM|UART011_DCDMIM|UART011_DSRMIM;
1348 	pl011_write(uap->im, uap, REG_IMSC);
1349 }
1350 
1351 static void pl011_rx_chars(struct uart_amba_port *uap)
1352 __releases(&uap->port.lock)
1353 __acquires(&uap->port.lock)
1354 {
1355 	pl011_fifo_to_tty(uap);
1356 
1357 	spin_unlock(&uap->port.lock);
1358 	tty_flip_buffer_push(&uap->port.state->port);
1359 	/*
1360 	 * If we were temporarily out of DMA mode for a while,
1361 	 * attempt to switch back to DMA mode again.
1362 	 */
1363 	if (pl011_dma_rx_available(uap)) {
1364 		if (pl011_dma_rx_trigger_dma(uap)) {
1365 			dev_dbg(uap->port.dev, "could not trigger RX DMA job "
1366 				"fall back to interrupt mode again\n");
1367 			uap->im |= UART011_RXIM;
1368 			pl011_write(uap->im, uap, REG_IMSC);
1369 		} else {
1370 #ifdef CONFIG_DMA_ENGINE
1371 			/* Start Rx DMA poll */
1372 			if (uap->dmarx.poll_rate) {
1373 				uap->dmarx.last_jiffies = jiffies;
1374 				uap->dmarx.last_residue	= PL011_DMA_BUFFER_SIZE;
1375 				mod_timer(&uap->dmarx.timer,
1376 					jiffies +
1377 					msecs_to_jiffies(uap->dmarx.poll_rate));
1378 			}
1379 #endif
1380 		}
1381 	}
1382 	spin_lock(&uap->port.lock);
1383 }
1384 
1385 static bool pl011_tx_char(struct uart_amba_port *uap, unsigned char c,
1386 			  bool from_irq)
1387 {
1388 	if (unlikely(!from_irq) &&
1389 	    pl011_read(uap, REG_FR) & UART01x_FR_TXFF)
1390 		return false; /* unable to transmit character */
1391 
1392 	pl011_write(c, uap, REG_DR);
1393 	uap->port.icount.tx++;
1394 
1395 	return true;
1396 }
1397 
1398 static void pl011_rs485_tx_start(struct uart_amba_port *uap)
1399 {
1400 	struct uart_port *port = &uap->port;
1401 	u32 cr;
1402 
1403 	/* Enable transmitter */
1404 	cr = pl011_read(uap, REG_CR);
1405 	cr |= UART011_CR_TXE;
1406 
1407 	/* Disable receiver if half-duplex */
1408 	if (!(port->rs485.flags & SER_RS485_RX_DURING_TX))
1409 		cr &= ~UART011_CR_RXE;
1410 
1411 	if (port->rs485.flags & SER_RS485_RTS_ON_SEND)
1412 		cr &= ~UART011_CR_RTS;
1413 	else
1414 		cr |= UART011_CR_RTS;
1415 
1416 	pl011_write(cr, uap, REG_CR);
1417 
1418 	if (port->rs485.delay_rts_before_send)
1419 		mdelay(port->rs485.delay_rts_before_send);
1420 
1421 	uap->rs485_tx_started = true;
1422 }
1423 
1424 /* Returns true if tx interrupts have to be (kept) enabled  */
1425 static bool pl011_tx_chars(struct uart_amba_port *uap, bool from_irq)
1426 {
1427 	struct circ_buf *xmit = &uap->port.state->xmit;
1428 	int count = uap->fifosize >> 1;
1429 
1430 	if (uap->port.x_char) {
1431 		if (!pl011_tx_char(uap, uap->port.x_char, from_irq))
1432 			return true;
1433 		uap->port.x_char = 0;
1434 		--count;
1435 	}
1436 	if (uart_circ_empty(xmit) || uart_tx_stopped(&uap->port)) {
1437 		pl011_stop_tx(&uap->port);
1438 		return false;
1439 	}
1440 
1441 	if ((uap->port.rs485.flags & SER_RS485_ENABLED) &&
1442 	    !uap->rs485_tx_started)
1443 		pl011_rs485_tx_start(uap);
1444 
1445 	/* If we are using DMA mode, try to send some characters. */
1446 	if (pl011_dma_tx_irq(uap))
1447 		return true;
1448 
1449 	do {
1450 		if (likely(from_irq) && count-- == 0)
1451 			break;
1452 
1453 		if (!pl011_tx_char(uap, xmit->buf[xmit->tail], from_irq))
1454 			break;
1455 
1456 		xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1);
1457 	} while (!uart_circ_empty(xmit));
1458 
1459 	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
1460 		uart_write_wakeup(&uap->port);
1461 
1462 	if (uart_circ_empty(xmit)) {
1463 		pl011_stop_tx(&uap->port);
1464 		return false;
1465 	}
1466 	return true;
1467 }
1468 
1469 static void pl011_modem_status(struct uart_amba_port *uap)
1470 {
1471 	unsigned int status, delta;
1472 
1473 	status = pl011_read(uap, REG_FR) & UART01x_FR_MODEM_ANY;
1474 
1475 	delta = status ^ uap->old_status;
1476 	uap->old_status = status;
1477 
1478 	if (!delta)
1479 		return;
1480 
1481 	if (delta & UART01x_FR_DCD)
1482 		uart_handle_dcd_change(&uap->port, status & UART01x_FR_DCD);
1483 
1484 	if (delta & uap->vendor->fr_dsr)
1485 		uap->port.icount.dsr++;
1486 
1487 	if (delta & uap->vendor->fr_cts)
1488 		uart_handle_cts_change(&uap->port,
1489 				       status & uap->vendor->fr_cts);
1490 
1491 	wake_up_interruptible(&uap->port.state->port.delta_msr_wait);
1492 }
1493 
1494 static void check_apply_cts_event_workaround(struct uart_amba_port *uap)
1495 {
1496 	if (!uap->vendor->cts_event_workaround)
1497 		return;
1498 
1499 	/* workaround to make sure that all bits are unlocked.. */
1500 	pl011_write(0x00, uap, REG_ICR);
1501 
1502 	/*
1503 	 * WA: introduce 26ns(1 uart clk) delay before W1C;
1504 	 * single apb access will incur 2 pclk(133.12Mhz) delay,
1505 	 * so add 2 dummy reads
1506 	 */
1507 	pl011_read(uap, REG_ICR);
1508 	pl011_read(uap, REG_ICR);
1509 }
1510 
1511 static irqreturn_t pl011_int(int irq, void *dev_id)
1512 {
1513 	struct uart_amba_port *uap = dev_id;
1514 	unsigned long flags;
1515 	unsigned int status, pass_counter = AMBA_ISR_PASS_LIMIT;
1516 	int handled = 0;
1517 
1518 	spin_lock_irqsave(&uap->port.lock, flags);
1519 	status = pl011_read(uap, REG_RIS) & uap->im;
1520 	if (status) {
1521 		do {
1522 			check_apply_cts_event_workaround(uap);
1523 
1524 			pl011_write(status & ~(UART011_TXIS|UART011_RTIS|
1525 					       UART011_RXIS),
1526 				    uap, REG_ICR);
1527 
1528 			if (status & (UART011_RTIS|UART011_RXIS)) {
1529 				if (pl011_dma_rx_running(uap))
1530 					pl011_dma_rx_irq(uap);
1531 				else
1532 					pl011_rx_chars(uap);
1533 			}
1534 			if (status & (UART011_DSRMIS|UART011_DCDMIS|
1535 				      UART011_CTSMIS|UART011_RIMIS))
1536 				pl011_modem_status(uap);
1537 			if (status & UART011_TXIS)
1538 				pl011_tx_chars(uap, true);
1539 
1540 			if (pass_counter-- == 0)
1541 				break;
1542 
1543 			status = pl011_read(uap, REG_RIS) & uap->im;
1544 		} while (status != 0);
1545 		handled = 1;
1546 	}
1547 
1548 	spin_unlock_irqrestore(&uap->port.lock, flags);
1549 
1550 	return IRQ_RETVAL(handled);
1551 }
1552 
1553 static unsigned int pl011_tx_empty(struct uart_port *port)
1554 {
1555 	struct uart_amba_port *uap =
1556 	    container_of(port, struct uart_amba_port, port);
1557 
1558 	/* Allow feature register bits to be inverted to work around errata */
1559 	unsigned int status = pl011_read(uap, REG_FR) ^ uap->vendor->inv_fr;
1560 
1561 	return status & (uap->vendor->fr_busy | UART01x_FR_TXFF) ?
1562 							0 : TIOCSER_TEMT;
1563 }
1564 
1565 static unsigned int pl011_get_mctrl(struct uart_port *port)
1566 {
1567 	struct uart_amba_port *uap =
1568 	    container_of(port, struct uart_amba_port, port);
1569 	unsigned int result = 0;
1570 	unsigned int status = pl011_read(uap, REG_FR);
1571 
1572 #define TIOCMBIT(uartbit, tiocmbit)	\
1573 	if (status & uartbit)		\
1574 		result |= tiocmbit
1575 
1576 	TIOCMBIT(UART01x_FR_DCD, TIOCM_CAR);
1577 	TIOCMBIT(uap->vendor->fr_dsr, TIOCM_DSR);
1578 	TIOCMBIT(uap->vendor->fr_cts, TIOCM_CTS);
1579 	TIOCMBIT(uap->vendor->fr_ri, TIOCM_RNG);
1580 #undef TIOCMBIT
1581 	return result;
1582 }
1583 
1584 static void pl011_set_mctrl(struct uart_port *port, unsigned int mctrl)
1585 {
1586 	struct uart_amba_port *uap =
1587 	    container_of(port, struct uart_amba_port, port);
1588 	unsigned int cr;
1589 
1590 	cr = pl011_read(uap, REG_CR);
1591 
1592 #define	TIOCMBIT(tiocmbit, uartbit)		\
1593 	if (mctrl & tiocmbit)		\
1594 		cr |= uartbit;		\
1595 	else				\
1596 		cr &= ~uartbit
1597 
1598 	TIOCMBIT(TIOCM_RTS, UART011_CR_RTS);
1599 	TIOCMBIT(TIOCM_DTR, UART011_CR_DTR);
1600 	TIOCMBIT(TIOCM_OUT1, UART011_CR_OUT1);
1601 	TIOCMBIT(TIOCM_OUT2, UART011_CR_OUT2);
1602 	TIOCMBIT(TIOCM_LOOP, UART011_CR_LBE);
1603 
1604 	if (port->status & UPSTAT_AUTORTS) {
1605 		/* We need to disable auto-RTS if we want to turn RTS off */
1606 		TIOCMBIT(TIOCM_RTS, UART011_CR_RTSEN);
1607 	}
1608 #undef TIOCMBIT
1609 
1610 	pl011_write(cr, uap, REG_CR);
1611 }
1612 
1613 static void pl011_break_ctl(struct uart_port *port, int break_state)
1614 {
1615 	struct uart_amba_port *uap =
1616 	    container_of(port, struct uart_amba_port, port);
1617 	unsigned long flags;
1618 	unsigned int lcr_h;
1619 
1620 	spin_lock_irqsave(&uap->port.lock, flags);
1621 	lcr_h = pl011_read(uap, REG_LCRH_TX);
1622 	if (break_state == -1)
1623 		lcr_h |= UART01x_LCRH_BRK;
1624 	else
1625 		lcr_h &= ~UART01x_LCRH_BRK;
1626 	pl011_write(lcr_h, uap, REG_LCRH_TX);
1627 	spin_unlock_irqrestore(&uap->port.lock, flags);
1628 }
1629 
1630 #ifdef CONFIG_CONSOLE_POLL
1631 
1632 static void pl011_quiesce_irqs(struct uart_port *port)
1633 {
1634 	struct uart_amba_port *uap =
1635 	    container_of(port, struct uart_amba_port, port);
1636 
1637 	pl011_write(pl011_read(uap, REG_MIS), uap, REG_ICR);
1638 	/*
1639 	 * There is no way to clear TXIM as this is "ready to transmit IRQ", so
1640 	 * we simply mask it. start_tx() will unmask it.
1641 	 *
1642 	 * Note we can race with start_tx(), and if the race happens, the
1643 	 * polling user might get another interrupt just after we clear it.
1644 	 * But it should be OK and can happen even w/o the race, e.g.
1645 	 * controller immediately got some new data and raised the IRQ.
1646 	 *
1647 	 * And whoever uses polling routines assumes that it manages the device
1648 	 * (including tx queue), so we're also fine with start_tx()'s caller
1649 	 * side.
1650 	 */
1651 	pl011_write(pl011_read(uap, REG_IMSC) & ~UART011_TXIM, uap,
1652 		    REG_IMSC);
1653 }
1654 
1655 static int pl011_get_poll_char(struct uart_port *port)
1656 {
1657 	struct uart_amba_port *uap =
1658 	    container_of(port, struct uart_amba_port, port);
1659 	unsigned int status;
1660 
1661 	/*
1662 	 * The caller might need IRQs lowered, e.g. if used with KDB NMI
1663 	 * debugger.
1664 	 */
1665 	pl011_quiesce_irqs(port);
1666 
1667 	status = pl011_read(uap, REG_FR);
1668 	if (status & UART01x_FR_RXFE)
1669 		return NO_POLL_CHAR;
1670 
1671 	return pl011_read(uap, REG_DR);
1672 }
1673 
1674 static void pl011_put_poll_char(struct uart_port *port,
1675 			 unsigned char ch)
1676 {
1677 	struct uart_amba_port *uap =
1678 	    container_of(port, struct uart_amba_port, port);
1679 
1680 	while (pl011_read(uap, REG_FR) & UART01x_FR_TXFF)
1681 		cpu_relax();
1682 
1683 	pl011_write(ch, uap, REG_DR);
1684 }
1685 
1686 #endif /* CONFIG_CONSOLE_POLL */
1687 
1688 static int pl011_hwinit(struct uart_port *port)
1689 {
1690 	struct uart_amba_port *uap =
1691 	    container_of(port, struct uart_amba_port, port);
1692 	int retval;
1693 
1694 	/* Optionaly enable pins to be muxed in and configured */
1695 	pinctrl_pm_select_default_state(port->dev);
1696 
1697 	/*
1698 	 * Try to enable the clock producer.
1699 	 */
1700 	retval = clk_prepare_enable(uap->clk);
1701 	if (retval)
1702 		return retval;
1703 
1704 	uap->port.uartclk = clk_get_rate(uap->clk);
1705 
1706 	/* Clear pending error and receive interrupts */
1707 	pl011_write(UART011_OEIS | UART011_BEIS | UART011_PEIS |
1708 		    UART011_FEIS | UART011_RTIS | UART011_RXIS,
1709 		    uap, REG_ICR);
1710 
1711 	/*
1712 	 * Save interrupts enable mask, and enable RX interrupts in case if
1713 	 * the interrupt is used for NMI entry.
1714 	 */
1715 	uap->im = pl011_read(uap, REG_IMSC);
1716 	pl011_write(UART011_RTIM | UART011_RXIM, uap, REG_IMSC);
1717 
1718 	if (dev_get_platdata(uap->port.dev)) {
1719 		struct amba_pl011_data *plat;
1720 
1721 		plat = dev_get_platdata(uap->port.dev);
1722 		if (plat->init)
1723 			plat->init();
1724 	}
1725 	return 0;
1726 }
1727 
1728 static bool pl011_split_lcrh(const struct uart_amba_port *uap)
1729 {
1730 	return pl011_reg_to_offset(uap, REG_LCRH_RX) !=
1731 	       pl011_reg_to_offset(uap, REG_LCRH_TX);
1732 }
1733 
1734 static void pl011_write_lcr_h(struct uart_amba_port *uap, unsigned int lcr_h)
1735 {
1736 	pl011_write(lcr_h, uap, REG_LCRH_RX);
1737 	if (pl011_split_lcrh(uap)) {
1738 		int i;
1739 		/*
1740 		 * Wait 10 PCLKs before writing LCRH_TX register,
1741 		 * to get this delay write read only register 10 times
1742 		 */
1743 		for (i = 0; i < 10; ++i)
1744 			pl011_write(0xff, uap, REG_MIS);
1745 		pl011_write(lcr_h, uap, REG_LCRH_TX);
1746 	}
1747 }
1748 
1749 static int pl011_allocate_irq(struct uart_amba_port *uap)
1750 {
1751 	pl011_write(uap->im, uap, REG_IMSC);
1752 
1753 	return request_irq(uap->port.irq, pl011_int, IRQF_SHARED, "uart-pl011", uap);
1754 }
1755 
1756 /*
1757  * Enable interrupts, only timeouts when using DMA
1758  * if initial RX DMA job failed, start in interrupt mode
1759  * as well.
1760  */
1761 static void pl011_enable_interrupts(struct uart_amba_port *uap)
1762 {
1763 	unsigned int i;
1764 
1765 	spin_lock_irq(&uap->port.lock);
1766 
1767 	/* Clear out any spuriously appearing RX interrupts */
1768 	pl011_write(UART011_RTIS | UART011_RXIS, uap, REG_ICR);
1769 
1770 	/*
1771 	 * RXIS is asserted only when the RX FIFO transitions from below
1772 	 * to above the trigger threshold.  If the RX FIFO is already
1773 	 * full to the threshold this can't happen and RXIS will now be
1774 	 * stuck off.  Drain the RX FIFO explicitly to fix this:
1775 	 */
1776 	for (i = 0; i < uap->fifosize * 2; ++i) {
1777 		if (pl011_read(uap, REG_FR) & UART01x_FR_RXFE)
1778 			break;
1779 
1780 		pl011_read(uap, REG_DR);
1781 	}
1782 
1783 	uap->im = UART011_RTIM;
1784 	if (!pl011_dma_rx_running(uap))
1785 		uap->im |= UART011_RXIM;
1786 	pl011_write(uap->im, uap, REG_IMSC);
1787 	spin_unlock_irq(&uap->port.lock);
1788 }
1789 
1790 static int pl011_startup(struct uart_port *port)
1791 {
1792 	struct uart_amba_port *uap =
1793 	    container_of(port, struct uart_amba_port, port);
1794 	unsigned int cr;
1795 	int retval;
1796 
1797 	retval = pl011_hwinit(port);
1798 	if (retval)
1799 		goto clk_dis;
1800 
1801 	retval = pl011_allocate_irq(uap);
1802 	if (retval)
1803 		goto clk_dis;
1804 
1805 	pl011_write(uap->vendor->ifls, uap, REG_IFLS);
1806 
1807 	spin_lock_irq(&uap->port.lock);
1808 
1809 	cr = pl011_read(uap, REG_CR);
1810 	cr &= UART011_CR_RTS | UART011_CR_DTR;
1811 	cr |= UART01x_CR_UARTEN | UART011_CR_RXE;
1812 
1813 	if (!(port->rs485.flags & SER_RS485_ENABLED))
1814 		cr |= UART011_CR_TXE;
1815 
1816 	pl011_write(cr, uap, REG_CR);
1817 
1818 	spin_unlock_irq(&uap->port.lock);
1819 
1820 	/*
1821 	 * initialise the old status of the modem signals
1822 	 */
1823 	uap->old_status = pl011_read(uap, REG_FR) & UART01x_FR_MODEM_ANY;
1824 
1825 	/* Startup DMA */
1826 	pl011_dma_startup(uap);
1827 
1828 	pl011_enable_interrupts(uap);
1829 
1830 	return 0;
1831 
1832  clk_dis:
1833 	clk_disable_unprepare(uap->clk);
1834 	return retval;
1835 }
1836 
1837 static int sbsa_uart_startup(struct uart_port *port)
1838 {
1839 	struct uart_amba_port *uap =
1840 		container_of(port, struct uart_amba_port, port);
1841 	int retval;
1842 
1843 	retval = pl011_hwinit(port);
1844 	if (retval)
1845 		return retval;
1846 
1847 	retval = pl011_allocate_irq(uap);
1848 	if (retval)
1849 		return retval;
1850 
1851 	/* The SBSA UART does not support any modem status lines. */
1852 	uap->old_status = 0;
1853 
1854 	pl011_enable_interrupts(uap);
1855 
1856 	return 0;
1857 }
1858 
1859 static void pl011_shutdown_channel(struct uart_amba_port *uap,
1860 					unsigned int lcrh)
1861 {
1862       unsigned long val;
1863 
1864       val = pl011_read(uap, lcrh);
1865       val &= ~(UART01x_LCRH_BRK | UART01x_LCRH_FEN);
1866       pl011_write(val, uap, lcrh);
1867 }
1868 
1869 /*
1870  * disable the port. It should not disable RTS and DTR.
1871  * Also RTS and DTR state should be preserved to restore
1872  * it during startup().
1873  */
1874 static void pl011_disable_uart(struct uart_amba_port *uap)
1875 {
1876 	unsigned int cr;
1877 
1878 	uap->port.status &= ~(UPSTAT_AUTOCTS | UPSTAT_AUTORTS);
1879 	spin_lock_irq(&uap->port.lock);
1880 	cr = pl011_read(uap, REG_CR);
1881 	cr &= UART011_CR_RTS | UART011_CR_DTR;
1882 	cr |= UART01x_CR_UARTEN | UART011_CR_TXE;
1883 	pl011_write(cr, uap, REG_CR);
1884 	spin_unlock_irq(&uap->port.lock);
1885 
1886 	/*
1887 	 * disable break condition and fifos
1888 	 */
1889 	pl011_shutdown_channel(uap, REG_LCRH_RX);
1890 	if (pl011_split_lcrh(uap))
1891 		pl011_shutdown_channel(uap, REG_LCRH_TX);
1892 }
1893 
1894 static void pl011_disable_interrupts(struct uart_amba_port *uap)
1895 {
1896 	spin_lock_irq(&uap->port.lock);
1897 
1898 	/* mask all interrupts and clear all pending ones */
1899 	uap->im = 0;
1900 	pl011_write(uap->im, uap, REG_IMSC);
1901 	pl011_write(0xffff, uap, REG_ICR);
1902 
1903 	spin_unlock_irq(&uap->port.lock);
1904 }
1905 
1906 static void pl011_shutdown(struct uart_port *port)
1907 {
1908 	struct uart_amba_port *uap =
1909 		container_of(port, struct uart_amba_port, port);
1910 
1911 	pl011_disable_interrupts(uap);
1912 
1913 	pl011_dma_shutdown(uap);
1914 
1915 	if ((port->rs485.flags & SER_RS485_ENABLED) && uap->rs485_tx_started)
1916 		pl011_rs485_tx_stop(uap);
1917 
1918 	free_irq(uap->port.irq, uap);
1919 
1920 	pl011_disable_uart(uap);
1921 
1922 	/*
1923 	 * Shut down the clock producer
1924 	 */
1925 	clk_disable_unprepare(uap->clk);
1926 	/* Optionally let pins go into sleep states */
1927 	pinctrl_pm_select_sleep_state(port->dev);
1928 
1929 	if (dev_get_platdata(uap->port.dev)) {
1930 		struct amba_pl011_data *plat;
1931 
1932 		plat = dev_get_platdata(uap->port.dev);
1933 		if (plat->exit)
1934 			plat->exit();
1935 	}
1936 
1937 	if (uap->port.ops->flush_buffer)
1938 		uap->port.ops->flush_buffer(port);
1939 }
1940 
1941 static void sbsa_uart_shutdown(struct uart_port *port)
1942 {
1943 	struct uart_amba_port *uap =
1944 		container_of(port, struct uart_amba_port, port);
1945 
1946 	pl011_disable_interrupts(uap);
1947 
1948 	free_irq(uap->port.irq, uap);
1949 
1950 	if (uap->port.ops->flush_buffer)
1951 		uap->port.ops->flush_buffer(port);
1952 }
1953 
1954 static void
1955 pl011_setup_status_masks(struct uart_port *port, struct ktermios *termios)
1956 {
1957 	port->read_status_mask = UART011_DR_OE | 255;
1958 	if (termios->c_iflag & INPCK)
1959 		port->read_status_mask |= UART011_DR_FE | UART011_DR_PE;
1960 	if (termios->c_iflag & (IGNBRK | BRKINT | PARMRK))
1961 		port->read_status_mask |= UART011_DR_BE;
1962 
1963 	/*
1964 	 * Characters to ignore
1965 	 */
1966 	port->ignore_status_mask = 0;
1967 	if (termios->c_iflag & IGNPAR)
1968 		port->ignore_status_mask |= UART011_DR_FE | UART011_DR_PE;
1969 	if (termios->c_iflag & IGNBRK) {
1970 		port->ignore_status_mask |= UART011_DR_BE;
1971 		/*
1972 		 * If we're ignoring parity and break indicators,
1973 		 * ignore overruns too (for real raw support).
1974 		 */
1975 		if (termios->c_iflag & IGNPAR)
1976 			port->ignore_status_mask |= UART011_DR_OE;
1977 	}
1978 
1979 	/*
1980 	 * Ignore all characters if CREAD is not set.
1981 	 */
1982 	if ((termios->c_cflag & CREAD) == 0)
1983 		port->ignore_status_mask |= UART_DUMMY_DR_RX;
1984 }
1985 
1986 static void
1987 pl011_set_termios(struct uart_port *port, struct ktermios *termios,
1988 		     struct ktermios *old)
1989 {
1990 	struct uart_amba_port *uap =
1991 	    container_of(port, struct uart_amba_port, port);
1992 	unsigned int lcr_h, old_cr;
1993 	unsigned long flags;
1994 	unsigned int baud, quot, clkdiv;
1995 	unsigned int bits;
1996 
1997 	if (uap->vendor->oversampling)
1998 		clkdiv = 8;
1999 	else
2000 		clkdiv = 16;
2001 
2002 	/*
2003 	 * Ask the core to calculate the divisor for us.
2004 	 */
2005 	baud = uart_get_baud_rate(port, termios, old, 0,
2006 				  port->uartclk / clkdiv);
2007 #ifdef CONFIG_DMA_ENGINE
2008 	/*
2009 	 * Adjust RX DMA polling rate with baud rate if not specified.
2010 	 */
2011 	if (uap->dmarx.auto_poll_rate)
2012 		uap->dmarx.poll_rate = DIV_ROUND_UP(10000000, baud);
2013 #endif
2014 
2015 	if (baud > port->uartclk/16)
2016 		quot = DIV_ROUND_CLOSEST(port->uartclk * 8, baud);
2017 	else
2018 		quot = DIV_ROUND_CLOSEST(port->uartclk * 4, baud);
2019 
2020 	switch (termios->c_cflag & CSIZE) {
2021 	case CS5:
2022 		lcr_h = UART01x_LCRH_WLEN_5;
2023 		break;
2024 	case CS6:
2025 		lcr_h = UART01x_LCRH_WLEN_6;
2026 		break;
2027 	case CS7:
2028 		lcr_h = UART01x_LCRH_WLEN_7;
2029 		break;
2030 	default: // CS8
2031 		lcr_h = UART01x_LCRH_WLEN_8;
2032 		break;
2033 	}
2034 	if (termios->c_cflag & CSTOPB)
2035 		lcr_h |= UART01x_LCRH_STP2;
2036 	if (termios->c_cflag & PARENB) {
2037 		lcr_h |= UART01x_LCRH_PEN;
2038 		if (!(termios->c_cflag & PARODD))
2039 			lcr_h |= UART01x_LCRH_EPS;
2040 		if (termios->c_cflag & CMSPAR)
2041 			lcr_h |= UART011_LCRH_SPS;
2042 	}
2043 	if (uap->fifosize > 1)
2044 		lcr_h |= UART01x_LCRH_FEN;
2045 
2046 	bits = tty_get_frame_size(termios->c_cflag);
2047 
2048 	spin_lock_irqsave(&port->lock, flags);
2049 
2050 	/*
2051 	 * Update the per-port timeout.
2052 	 */
2053 	uart_update_timeout(port, termios->c_cflag, baud);
2054 
2055 	/*
2056 	 * Calculate the approximated time it takes to transmit one character
2057 	 * with the given baud rate. We use this as the poll interval when we
2058 	 * wait for the tx queue to empty.
2059 	 */
2060 	uap->rs485_tx_drain_interval = DIV_ROUND_UP(bits * 1000 * 1000, baud);
2061 
2062 	pl011_setup_status_masks(port, termios);
2063 
2064 	if (UART_ENABLE_MS(port, termios->c_cflag))
2065 		pl011_enable_ms(port);
2066 
2067 	if (port->rs485.flags & SER_RS485_ENABLED)
2068 		termios->c_cflag &= ~CRTSCTS;
2069 
2070 	old_cr = pl011_read(uap, REG_CR);
2071 
2072 	if (termios->c_cflag & CRTSCTS) {
2073 		if (old_cr & UART011_CR_RTS)
2074 			old_cr |= UART011_CR_RTSEN;
2075 
2076 		old_cr |= UART011_CR_CTSEN;
2077 		port->status |= UPSTAT_AUTOCTS | UPSTAT_AUTORTS;
2078 	} else {
2079 		old_cr &= ~(UART011_CR_CTSEN | UART011_CR_RTSEN);
2080 		port->status &= ~(UPSTAT_AUTOCTS | UPSTAT_AUTORTS);
2081 	}
2082 
2083 	if (uap->vendor->oversampling) {
2084 		if (baud > port->uartclk / 16)
2085 			old_cr |= ST_UART011_CR_OVSFACT;
2086 		else
2087 			old_cr &= ~ST_UART011_CR_OVSFACT;
2088 	}
2089 
2090 	/*
2091 	 * Workaround for the ST Micro oversampling variants to
2092 	 * increase the bitrate slightly, by lowering the divisor,
2093 	 * to avoid delayed sampling of start bit at high speeds,
2094 	 * else we see data corruption.
2095 	 */
2096 	if (uap->vendor->oversampling) {
2097 		if ((baud >= 3000000) && (baud < 3250000) && (quot > 1))
2098 			quot -= 1;
2099 		else if ((baud > 3250000) && (quot > 2))
2100 			quot -= 2;
2101 	}
2102 	/* Set baud rate */
2103 	pl011_write(quot & 0x3f, uap, REG_FBRD);
2104 	pl011_write(quot >> 6, uap, REG_IBRD);
2105 
2106 	/*
2107 	 * ----------v----------v----------v----------v-----
2108 	 * NOTE: REG_LCRH_TX and REG_LCRH_RX MUST BE WRITTEN AFTER
2109 	 * REG_FBRD & REG_IBRD.
2110 	 * ----------^----------^----------^----------^-----
2111 	 */
2112 	pl011_write_lcr_h(uap, lcr_h);
2113 	pl011_write(old_cr, uap, REG_CR);
2114 
2115 	spin_unlock_irqrestore(&port->lock, flags);
2116 }
2117 
2118 static void
2119 sbsa_uart_set_termios(struct uart_port *port, struct ktermios *termios,
2120 		      struct ktermios *old)
2121 {
2122 	struct uart_amba_port *uap =
2123 	    container_of(port, struct uart_amba_port, port);
2124 	unsigned long flags;
2125 
2126 	tty_termios_encode_baud_rate(termios, uap->fixed_baud, uap->fixed_baud);
2127 
2128 	/* The SBSA UART only supports 8n1 without hardware flow control. */
2129 	termios->c_cflag &= ~(CSIZE | CSTOPB | PARENB | PARODD);
2130 	termios->c_cflag &= ~(CMSPAR | CRTSCTS);
2131 	termios->c_cflag |= CS8 | CLOCAL;
2132 
2133 	spin_lock_irqsave(&port->lock, flags);
2134 	uart_update_timeout(port, CS8, uap->fixed_baud);
2135 	pl011_setup_status_masks(port, termios);
2136 	spin_unlock_irqrestore(&port->lock, flags);
2137 }
2138 
2139 static const char *pl011_type(struct uart_port *port)
2140 {
2141 	struct uart_amba_port *uap =
2142 	    container_of(port, struct uart_amba_port, port);
2143 	return uap->port.type == PORT_AMBA ? uap->type : NULL;
2144 }
2145 
2146 /*
2147  * Configure/autoconfigure the port.
2148  */
2149 static void pl011_config_port(struct uart_port *port, int flags)
2150 {
2151 	if (flags & UART_CONFIG_TYPE)
2152 		port->type = PORT_AMBA;
2153 }
2154 
2155 /*
2156  * verify the new serial_struct (for TIOCSSERIAL).
2157  */
2158 static int pl011_verify_port(struct uart_port *port, struct serial_struct *ser)
2159 {
2160 	int ret = 0;
2161 	if (ser->type != PORT_UNKNOWN && ser->type != PORT_AMBA)
2162 		ret = -EINVAL;
2163 	if (ser->irq < 0 || ser->irq >= nr_irqs)
2164 		ret = -EINVAL;
2165 	if (ser->baud_base < 9600)
2166 		ret = -EINVAL;
2167 	if (port->mapbase != (unsigned long) ser->iomem_base)
2168 		ret = -EINVAL;
2169 	return ret;
2170 }
2171 
2172 static int pl011_rs485_config(struct uart_port *port,
2173 			      struct serial_rs485 *rs485)
2174 {
2175 	struct uart_amba_port *uap =
2176 		container_of(port, struct uart_amba_port, port);
2177 
2178 	/* pick sane settings if the user hasn't */
2179 	if (!(rs485->flags & SER_RS485_RTS_ON_SEND) ==
2180 	    !(rs485->flags & SER_RS485_RTS_AFTER_SEND)) {
2181 		rs485->flags |= SER_RS485_RTS_ON_SEND;
2182 		rs485->flags &= ~SER_RS485_RTS_AFTER_SEND;
2183 	}
2184 	/* clamp the delays to [0, 100ms] */
2185 	rs485->delay_rts_before_send = min(rs485->delay_rts_before_send, 100U);
2186 	rs485->delay_rts_after_send = min(rs485->delay_rts_after_send, 100U);
2187 	memset(rs485->padding, 0, sizeof(rs485->padding));
2188 
2189 	if (port->rs485.flags & SER_RS485_ENABLED)
2190 		pl011_rs485_tx_stop(uap);
2191 
2192 	/* Set new configuration */
2193 	port->rs485 = *rs485;
2194 
2195 	/* Make sure auto RTS is disabled */
2196 	if (port->rs485.flags & SER_RS485_ENABLED) {
2197 		u32 cr = pl011_read(uap, REG_CR);
2198 
2199 		cr &= ~UART011_CR_RTSEN;
2200 		pl011_write(cr, uap, REG_CR);
2201 		port->status &= ~UPSTAT_AUTORTS;
2202 	}
2203 
2204 	return 0;
2205 }
2206 
2207 static const struct uart_ops amba_pl011_pops = {
2208 	.tx_empty	= pl011_tx_empty,
2209 	.set_mctrl	= pl011_set_mctrl,
2210 	.get_mctrl	= pl011_get_mctrl,
2211 	.stop_tx	= pl011_stop_tx,
2212 	.start_tx	= pl011_start_tx,
2213 	.stop_rx	= pl011_stop_rx,
2214 	.enable_ms	= pl011_enable_ms,
2215 	.break_ctl	= pl011_break_ctl,
2216 	.startup	= pl011_startup,
2217 	.shutdown	= pl011_shutdown,
2218 	.flush_buffer	= pl011_dma_flush_buffer,
2219 	.set_termios	= pl011_set_termios,
2220 	.type		= pl011_type,
2221 	.config_port	= pl011_config_port,
2222 	.verify_port	= pl011_verify_port,
2223 #ifdef CONFIG_CONSOLE_POLL
2224 	.poll_init     = pl011_hwinit,
2225 	.poll_get_char = pl011_get_poll_char,
2226 	.poll_put_char = pl011_put_poll_char,
2227 #endif
2228 };
2229 
2230 static void sbsa_uart_set_mctrl(struct uart_port *port, unsigned int mctrl)
2231 {
2232 }
2233 
2234 static unsigned int sbsa_uart_get_mctrl(struct uart_port *port)
2235 {
2236 	return 0;
2237 }
2238 
2239 static const struct uart_ops sbsa_uart_pops = {
2240 	.tx_empty	= pl011_tx_empty,
2241 	.set_mctrl	= sbsa_uart_set_mctrl,
2242 	.get_mctrl	= sbsa_uart_get_mctrl,
2243 	.stop_tx	= pl011_stop_tx,
2244 	.start_tx	= pl011_start_tx,
2245 	.stop_rx	= pl011_stop_rx,
2246 	.startup	= sbsa_uart_startup,
2247 	.shutdown	= sbsa_uart_shutdown,
2248 	.set_termios	= sbsa_uart_set_termios,
2249 	.type		= pl011_type,
2250 	.config_port	= pl011_config_port,
2251 	.verify_port	= pl011_verify_port,
2252 #ifdef CONFIG_CONSOLE_POLL
2253 	.poll_init     = pl011_hwinit,
2254 	.poll_get_char = pl011_get_poll_char,
2255 	.poll_put_char = pl011_put_poll_char,
2256 #endif
2257 };
2258 
2259 static struct uart_amba_port *amba_ports[UART_NR];
2260 
2261 #ifdef CONFIG_SERIAL_AMBA_PL011_CONSOLE
2262 
2263 static void pl011_console_putchar(struct uart_port *port, unsigned char ch)
2264 {
2265 	struct uart_amba_port *uap =
2266 	    container_of(port, struct uart_amba_port, port);
2267 
2268 	while (pl011_read(uap, REG_FR) & UART01x_FR_TXFF)
2269 		cpu_relax();
2270 	pl011_write(ch, uap, REG_DR);
2271 }
2272 
2273 static void
2274 pl011_console_write(struct console *co, const char *s, unsigned int count)
2275 {
2276 	struct uart_amba_port *uap = amba_ports[co->index];
2277 	unsigned int old_cr = 0, new_cr;
2278 	unsigned long flags;
2279 	int locked = 1;
2280 
2281 	clk_enable(uap->clk);
2282 
2283 	local_irq_save(flags);
2284 	if (uap->port.sysrq)
2285 		locked = 0;
2286 	else if (oops_in_progress)
2287 		locked = spin_trylock(&uap->port.lock);
2288 	else
2289 		spin_lock(&uap->port.lock);
2290 
2291 	/*
2292 	 *	First save the CR then disable the interrupts
2293 	 */
2294 	if (!uap->vendor->always_enabled) {
2295 		old_cr = pl011_read(uap, REG_CR);
2296 		new_cr = old_cr & ~UART011_CR_CTSEN;
2297 		new_cr |= UART01x_CR_UARTEN | UART011_CR_TXE;
2298 		pl011_write(new_cr, uap, REG_CR);
2299 	}
2300 
2301 	uart_console_write(&uap->port, s, count, pl011_console_putchar);
2302 
2303 	/*
2304 	 *	Finally, wait for transmitter to become empty and restore the
2305 	 *	TCR. Allow feature register bits to be inverted to work around
2306 	 *	errata.
2307 	 */
2308 	while ((pl011_read(uap, REG_FR) ^ uap->vendor->inv_fr)
2309 						& uap->vendor->fr_busy)
2310 		cpu_relax();
2311 	if (!uap->vendor->always_enabled)
2312 		pl011_write(old_cr, uap, REG_CR);
2313 
2314 	if (locked)
2315 		spin_unlock(&uap->port.lock);
2316 	local_irq_restore(flags);
2317 
2318 	clk_disable(uap->clk);
2319 }
2320 
2321 static void pl011_console_get_options(struct uart_amba_port *uap, int *baud,
2322 				      int *parity, int *bits)
2323 {
2324 	if (pl011_read(uap, REG_CR) & UART01x_CR_UARTEN) {
2325 		unsigned int lcr_h, ibrd, fbrd;
2326 
2327 		lcr_h = pl011_read(uap, REG_LCRH_TX);
2328 
2329 		*parity = 'n';
2330 		if (lcr_h & UART01x_LCRH_PEN) {
2331 			if (lcr_h & UART01x_LCRH_EPS)
2332 				*parity = 'e';
2333 			else
2334 				*parity = 'o';
2335 		}
2336 
2337 		if ((lcr_h & 0x60) == UART01x_LCRH_WLEN_7)
2338 			*bits = 7;
2339 		else
2340 			*bits = 8;
2341 
2342 		ibrd = pl011_read(uap, REG_IBRD);
2343 		fbrd = pl011_read(uap, REG_FBRD);
2344 
2345 		*baud = uap->port.uartclk * 4 / (64 * ibrd + fbrd);
2346 
2347 		if (uap->vendor->oversampling) {
2348 			if (pl011_read(uap, REG_CR)
2349 				  & ST_UART011_CR_OVSFACT)
2350 				*baud *= 2;
2351 		}
2352 	}
2353 }
2354 
2355 static int pl011_console_setup(struct console *co, char *options)
2356 {
2357 	struct uart_amba_port *uap;
2358 	int baud = 38400;
2359 	int bits = 8;
2360 	int parity = 'n';
2361 	int flow = 'n';
2362 	int ret;
2363 
2364 	/*
2365 	 * Check whether an invalid uart number has been specified, and
2366 	 * if so, search for the first available port that does have
2367 	 * console support.
2368 	 */
2369 	if (co->index >= UART_NR)
2370 		co->index = 0;
2371 	uap = amba_ports[co->index];
2372 	if (!uap)
2373 		return -ENODEV;
2374 
2375 	/* Allow pins to be muxed in and configured */
2376 	pinctrl_pm_select_default_state(uap->port.dev);
2377 
2378 	ret = clk_prepare(uap->clk);
2379 	if (ret)
2380 		return ret;
2381 
2382 	if (dev_get_platdata(uap->port.dev)) {
2383 		struct amba_pl011_data *plat;
2384 
2385 		plat = dev_get_platdata(uap->port.dev);
2386 		if (plat->init)
2387 			plat->init();
2388 	}
2389 
2390 	uap->port.uartclk = clk_get_rate(uap->clk);
2391 
2392 	if (uap->vendor->fixed_options) {
2393 		baud = uap->fixed_baud;
2394 	} else {
2395 		if (options)
2396 			uart_parse_options(options,
2397 					   &baud, &parity, &bits, &flow);
2398 		else
2399 			pl011_console_get_options(uap, &baud, &parity, &bits);
2400 	}
2401 
2402 	return uart_set_options(&uap->port, co, baud, parity, bits, flow);
2403 }
2404 
2405 /**
2406  *	pl011_console_match - non-standard console matching
2407  *	@co:	  registering console
2408  *	@name:	  name from console command line
2409  *	@idx:	  index from console command line
2410  *	@options: ptr to option string from console command line
2411  *
2412  *	Only attempts to match console command lines of the form:
2413  *	    console=pl011,mmio|mmio32,<addr>[,<options>]
2414  *	    console=pl011,0x<addr>[,<options>]
2415  *	This form is used to register an initial earlycon boot console and
2416  *	replace it with the amba_console at pl011 driver init.
2417  *
2418  *	Performs console setup for a match (as required by interface)
2419  *	If no <options> are specified, then assume the h/w is already setup.
2420  *
2421  *	Returns 0 if console matches; otherwise non-zero to use default matching
2422  */
2423 static int pl011_console_match(struct console *co, char *name, int idx,
2424 			       char *options)
2425 {
2426 	unsigned char iotype;
2427 	resource_size_t addr;
2428 	int i;
2429 
2430 	/*
2431 	 * Systems affected by the Qualcomm Technologies QDF2400 E44 erratum
2432 	 * have a distinct console name, so make sure we check for that.
2433 	 * The actual implementation of the erratum occurs in the probe
2434 	 * function.
2435 	 */
2436 	if ((strcmp(name, "qdf2400_e44") != 0) && (strcmp(name, "pl011") != 0))
2437 		return -ENODEV;
2438 
2439 	if (uart_parse_earlycon(options, &iotype, &addr, &options))
2440 		return -ENODEV;
2441 
2442 	if (iotype != UPIO_MEM && iotype != UPIO_MEM32)
2443 		return -ENODEV;
2444 
2445 	/* try to match the port specified on the command line */
2446 	for (i = 0; i < ARRAY_SIZE(amba_ports); i++) {
2447 		struct uart_port *port;
2448 
2449 		if (!amba_ports[i])
2450 			continue;
2451 
2452 		port = &amba_ports[i]->port;
2453 
2454 		if (port->mapbase != addr)
2455 			continue;
2456 
2457 		co->index = i;
2458 		port->cons = co;
2459 		return pl011_console_setup(co, options);
2460 	}
2461 
2462 	return -ENODEV;
2463 }
2464 
2465 static struct uart_driver amba_reg;
2466 static struct console amba_console = {
2467 	.name		= "ttyAMA",
2468 	.write		= pl011_console_write,
2469 	.device		= uart_console_device,
2470 	.setup		= pl011_console_setup,
2471 	.match		= pl011_console_match,
2472 	.flags		= CON_PRINTBUFFER | CON_ANYTIME,
2473 	.index		= -1,
2474 	.data		= &amba_reg,
2475 };
2476 
2477 #define AMBA_CONSOLE	(&amba_console)
2478 
2479 static void qdf2400_e44_putc(struct uart_port *port, unsigned char c)
2480 {
2481 	while (readl(port->membase + UART01x_FR) & UART01x_FR_TXFF)
2482 		cpu_relax();
2483 	writel(c, port->membase + UART01x_DR);
2484 	while (!(readl(port->membase + UART01x_FR) & UART011_FR_TXFE))
2485 		cpu_relax();
2486 }
2487 
2488 static void qdf2400_e44_early_write(struct console *con, const char *s, unsigned n)
2489 {
2490 	struct earlycon_device *dev = con->data;
2491 
2492 	uart_console_write(&dev->port, s, n, qdf2400_e44_putc);
2493 }
2494 
2495 static void pl011_putc(struct uart_port *port, unsigned char c)
2496 {
2497 	while (readl(port->membase + UART01x_FR) & UART01x_FR_TXFF)
2498 		cpu_relax();
2499 	if (port->iotype == UPIO_MEM32)
2500 		writel(c, port->membase + UART01x_DR);
2501 	else
2502 		writeb(c, port->membase + UART01x_DR);
2503 	while (readl(port->membase + UART01x_FR) & UART01x_FR_BUSY)
2504 		cpu_relax();
2505 }
2506 
2507 static void pl011_early_write(struct console *con, const char *s, unsigned n)
2508 {
2509 	struct earlycon_device *dev = con->data;
2510 
2511 	uart_console_write(&dev->port, s, n, pl011_putc);
2512 }
2513 
2514 #ifdef CONFIG_CONSOLE_POLL
2515 static int pl011_getc(struct uart_port *port)
2516 {
2517 	if (readl(port->membase + UART01x_FR) & UART01x_FR_RXFE)
2518 		return NO_POLL_CHAR;
2519 
2520 	if (port->iotype == UPIO_MEM32)
2521 		return readl(port->membase + UART01x_DR);
2522 	else
2523 		return readb(port->membase + UART01x_DR);
2524 }
2525 
2526 static int pl011_early_read(struct console *con, char *s, unsigned int n)
2527 {
2528 	struct earlycon_device *dev = con->data;
2529 	int ch, num_read = 0;
2530 
2531 	while (num_read < n) {
2532 		ch = pl011_getc(&dev->port);
2533 		if (ch == NO_POLL_CHAR)
2534 			break;
2535 
2536 		s[num_read++] = ch;
2537 	}
2538 
2539 	return num_read;
2540 }
2541 #else
2542 #define pl011_early_read NULL
2543 #endif
2544 
2545 /*
2546  * On non-ACPI systems, earlycon is enabled by specifying
2547  * "earlycon=pl011,<address>" on the kernel command line.
2548  *
2549  * On ACPI ARM64 systems, an "early" console is enabled via the SPCR table,
2550  * by specifying only "earlycon" on the command line.  Because it requires
2551  * SPCR, the console starts after ACPI is parsed, which is later than a
2552  * traditional early console.
2553  *
2554  * To get the traditional early console that starts before ACPI is parsed,
2555  * specify the full "earlycon=pl011,<address>" option.
2556  */
2557 static int __init pl011_early_console_setup(struct earlycon_device *device,
2558 					    const char *opt)
2559 {
2560 	if (!device->port.membase)
2561 		return -ENODEV;
2562 
2563 	device->con->write = pl011_early_write;
2564 	device->con->read = pl011_early_read;
2565 
2566 	return 0;
2567 }
2568 OF_EARLYCON_DECLARE(pl011, "arm,pl011", pl011_early_console_setup);
2569 OF_EARLYCON_DECLARE(pl011, "arm,sbsa-uart", pl011_early_console_setup);
2570 
2571 /*
2572  * On Qualcomm Datacenter Technologies QDF2400 SOCs affected by
2573  * Erratum 44, traditional earlycon can be enabled by specifying
2574  * "earlycon=qdf2400_e44,<address>".  Any options are ignored.
2575  *
2576  * Alternatively, you can just specify "earlycon", and the early console
2577  * will be enabled with the information from the SPCR table.  In this
2578  * case, the SPCR code will detect the need for the E44 work-around,
2579  * and set the console name to "qdf2400_e44".
2580  */
2581 static int __init
2582 qdf2400_e44_early_console_setup(struct earlycon_device *device,
2583 				const char *opt)
2584 {
2585 	if (!device->port.membase)
2586 		return -ENODEV;
2587 
2588 	device->con->write = qdf2400_e44_early_write;
2589 	return 0;
2590 }
2591 EARLYCON_DECLARE(qdf2400_e44, qdf2400_e44_early_console_setup);
2592 
2593 #else
2594 #define AMBA_CONSOLE	NULL
2595 #endif
2596 
2597 static struct uart_driver amba_reg = {
2598 	.owner			= THIS_MODULE,
2599 	.driver_name		= "ttyAMA",
2600 	.dev_name		= "ttyAMA",
2601 	.major			= SERIAL_AMBA_MAJOR,
2602 	.minor			= SERIAL_AMBA_MINOR,
2603 	.nr			= UART_NR,
2604 	.cons			= AMBA_CONSOLE,
2605 };
2606 
2607 static int pl011_probe_dt_alias(int index, struct device *dev)
2608 {
2609 	struct device_node *np;
2610 	static bool seen_dev_with_alias = false;
2611 	static bool seen_dev_without_alias = false;
2612 	int ret = index;
2613 
2614 	if (!IS_ENABLED(CONFIG_OF))
2615 		return ret;
2616 
2617 	np = dev->of_node;
2618 	if (!np)
2619 		return ret;
2620 
2621 	ret = of_alias_get_id(np, "serial");
2622 	if (ret < 0) {
2623 		seen_dev_without_alias = true;
2624 		ret = index;
2625 	} else {
2626 		seen_dev_with_alias = true;
2627 		if (ret >= ARRAY_SIZE(amba_ports) || amba_ports[ret] != NULL) {
2628 			dev_warn(dev, "requested serial port %d  not available.\n", ret);
2629 			ret = index;
2630 		}
2631 	}
2632 
2633 	if (seen_dev_with_alias && seen_dev_without_alias)
2634 		dev_warn(dev, "aliased and non-aliased serial devices found in device tree. Serial port enumeration may be unpredictable.\n");
2635 
2636 	return ret;
2637 }
2638 
2639 /* unregisters the driver also if no more ports are left */
2640 static void pl011_unregister_port(struct uart_amba_port *uap)
2641 {
2642 	int i;
2643 	bool busy = false;
2644 
2645 	for (i = 0; i < ARRAY_SIZE(amba_ports); i++) {
2646 		if (amba_ports[i] == uap)
2647 			amba_ports[i] = NULL;
2648 		else if (amba_ports[i])
2649 			busy = true;
2650 	}
2651 	pl011_dma_remove(uap);
2652 	if (!busy)
2653 		uart_unregister_driver(&amba_reg);
2654 }
2655 
2656 static int pl011_find_free_port(void)
2657 {
2658 	int i;
2659 
2660 	for (i = 0; i < ARRAY_SIZE(amba_ports); i++)
2661 		if (amba_ports[i] == NULL)
2662 			return i;
2663 
2664 	return -EBUSY;
2665 }
2666 
2667 static int pl011_get_rs485_mode(struct uart_amba_port *uap)
2668 {
2669 	struct uart_port *port = &uap->port;
2670 	struct serial_rs485 *rs485 = &port->rs485;
2671 	int ret;
2672 
2673 	ret = uart_get_rs485_mode(port);
2674 	if (ret)
2675 		return ret;
2676 
2677 	/* clamp the delays to [0, 100ms] */
2678 	rs485->delay_rts_before_send = min(rs485->delay_rts_before_send, 100U);
2679 	rs485->delay_rts_after_send = min(rs485->delay_rts_after_send, 100U);
2680 
2681 	return 0;
2682 }
2683 
2684 static int pl011_setup_port(struct device *dev, struct uart_amba_port *uap,
2685 			    struct resource *mmiobase, int index)
2686 {
2687 	void __iomem *base;
2688 	int ret;
2689 
2690 	base = devm_ioremap_resource(dev, mmiobase);
2691 	if (IS_ERR(base))
2692 		return PTR_ERR(base);
2693 
2694 	index = pl011_probe_dt_alias(index, dev);
2695 
2696 	uap->port.dev = dev;
2697 	uap->port.mapbase = mmiobase->start;
2698 	uap->port.membase = base;
2699 	uap->port.fifosize = uap->fifosize;
2700 	uap->port.has_sysrq = IS_ENABLED(CONFIG_SERIAL_AMBA_PL011_CONSOLE);
2701 	uap->port.flags = UPF_BOOT_AUTOCONF;
2702 	uap->port.line = index;
2703 
2704 	ret = pl011_get_rs485_mode(uap);
2705 	if (ret)
2706 		return ret;
2707 
2708 	amba_ports[index] = uap;
2709 
2710 	return 0;
2711 }
2712 
2713 static int pl011_register_port(struct uart_amba_port *uap)
2714 {
2715 	int ret, i;
2716 
2717 	/* Ensure interrupts from this UART are masked and cleared */
2718 	pl011_write(0, uap, REG_IMSC);
2719 	pl011_write(0xffff, uap, REG_ICR);
2720 
2721 	if (!amba_reg.state) {
2722 		ret = uart_register_driver(&amba_reg);
2723 		if (ret < 0) {
2724 			dev_err(uap->port.dev,
2725 				"Failed to register AMBA-PL011 driver\n");
2726 			for (i = 0; i < ARRAY_SIZE(amba_ports); i++)
2727 				if (amba_ports[i] == uap)
2728 					amba_ports[i] = NULL;
2729 			return ret;
2730 		}
2731 	}
2732 
2733 	ret = uart_add_one_port(&amba_reg, &uap->port);
2734 	if (ret)
2735 		pl011_unregister_port(uap);
2736 
2737 	return ret;
2738 }
2739 
2740 static int pl011_probe(struct amba_device *dev, const struct amba_id *id)
2741 {
2742 	struct uart_amba_port *uap;
2743 	struct vendor_data *vendor = id->data;
2744 	int portnr, ret;
2745 
2746 	portnr = pl011_find_free_port();
2747 	if (portnr < 0)
2748 		return portnr;
2749 
2750 	uap = devm_kzalloc(&dev->dev, sizeof(struct uart_amba_port),
2751 			   GFP_KERNEL);
2752 	if (!uap)
2753 		return -ENOMEM;
2754 
2755 	uap->clk = devm_clk_get(&dev->dev, NULL);
2756 	if (IS_ERR(uap->clk))
2757 		return PTR_ERR(uap->clk);
2758 
2759 	uap->reg_offset = vendor->reg_offset;
2760 	uap->vendor = vendor;
2761 	uap->fifosize = vendor->get_fifosize(dev);
2762 	uap->port.iotype = vendor->access_32b ? UPIO_MEM32 : UPIO_MEM;
2763 	uap->port.irq = dev->irq[0];
2764 	uap->port.ops = &amba_pl011_pops;
2765 	uap->port.rs485_config = pl011_rs485_config;
2766 	snprintf(uap->type, sizeof(uap->type), "PL011 rev%u", amba_rev(dev));
2767 
2768 	ret = pl011_setup_port(&dev->dev, uap, &dev->res, portnr);
2769 	if (ret)
2770 		return ret;
2771 
2772 	amba_set_drvdata(dev, uap);
2773 
2774 	return pl011_register_port(uap);
2775 }
2776 
2777 static void pl011_remove(struct amba_device *dev)
2778 {
2779 	struct uart_amba_port *uap = amba_get_drvdata(dev);
2780 
2781 	uart_remove_one_port(&amba_reg, &uap->port);
2782 	pl011_unregister_port(uap);
2783 }
2784 
2785 #ifdef CONFIG_PM_SLEEP
2786 static int pl011_suspend(struct device *dev)
2787 {
2788 	struct uart_amba_port *uap = dev_get_drvdata(dev);
2789 
2790 	if (!uap)
2791 		return -EINVAL;
2792 
2793 	return uart_suspend_port(&amba_reg, &uap->port);
2794 }
2795 
2796 static int pl011_resume(struct device *dev)
2797 {
2798 	struct uart_amba_port *uap = dev_get_drvdata(dev);
2799 
2800 	if (!uap)
2801 		return -EINVAL;
2802 
2803 	return uart_resume_port(&amba_reg, &uap->port);
2804 }
2805 #endif
2806 
2807 static SIMPLE_DEV_PM_OPS(pl011_dev_pm_ops, pl011_suspend, pl011_resume);
2808 
2809 static int sbsa_uart_probe(struct platform_device *pdev)
2810 {
2811 	struct uart_amba_port *uap;
2812 	struct resource *r;
2813 	int portnr, ret;
2814 	int baudrate;
2815 
2816 	/*
2817 	 * Check the mandatory baud rate parameter in the DT node early
2818 	 * so that we can easily exit with the error.
2819 	 */
2820 	if (pdev->dev.of_node) {
2821 		struct device_node *np = pdev->dev.of_node;
2822 
2823 		ret = of_property_read_u32(np, "current-speed", &baudrate);
2824 		if (ret)
2825 			return ret;
2826 	} else {
2827 		baudrate = 115200;
2828 	}
2829 
2830 	portnr = pl011_find_free_port();
2831 	if (portnr < 0)
2832 		return portnr;
2833 
2834 	uap = devm_kzalloc(&pdev->dev, sizeof(struct uart_amba_port),
2835 			   GFP_KERNEL);
2836 	if (!uap)
2837 		return -ENOMEM;
2838 
2839 	ret = platform_get_irq(pdev, 0);
2840 	if (ret < 0)
2841 		return ret;
2842 	uap->port.irq	= ret;
2843 
2844 #ifdef CONFIG_ACPI_SPCR_TABLE
2845 	if (qdf2400_e44_present) {
2846 		dev_info(&pdev->dev, "working around QDF2400 SoC erratum 44\n");
2847 		uap->vendor = &vendor_qdt_qdf2400_e44;
2848 	} else
2849 #endif
2850 		uap->vendor = &vendor_sbsa;
2851 
2852 	uap->reg_offset	= uap->vendor->reg_offset;
2853 	uap->fifosize	= 32;
2854 	uap->port.iotype = uap->vendor->access_32b ? UPIO_MEM32 : UPIO_MEM;
2855 	uap->port.ops	= &sbsa_uart_pops;
2856 	uap->fixed_baud = baudrate;
2857 
2858 	snprintf(uap->type, sizeof(uap->type), "SBSA");
2859 
2860 	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2861 
2862 	ret = pl011_setup_port(&pdev->dev, uap, r, portnr);
2863 	if (ret)
2864 		return ret;
2865 
2866 	platform_set_drvdata(pdev, uap);
2867 
2868 	return pl011_register_port(uap);
2869 }
2870 
2871 static int sbsa_uart_remove(struct platform_device *pdev)
2872 {
2873 	struct uart_amba_port *uap = platform_get_drvdata(pdev);
2874 
2875 	uart_remove_one_port(&amba_reg, &uap->port);
2876 	pl011_unregister_port(uap);
2877 	return 0;
2878 }
2879 
2880 static const struct of_device_id sbsa_uart_of_match[] = {
2881 	{ .compatible = "arm,sbsa-uart", },
2882 	{},
2883 };
2884 MODULE_DEVICE_TABLE(of, sbsa_uart_of_match);
2885 
2886 static const struct acpi_device_id __maybe_unused sbsa_uart_acpi_match[] = {
2887 	{ "ARMH0011", 0 },
2888 	{ "ARMHB000", 0 },
2889 	{},
2890 };
2891 MODULE_DEVICE_TABLE(acpi, sbsa_uart_acpi_match);
2892 
2893 static struct platform_driver arm_sbsa_uart_platform_driver = {
2894 	.probe		= sbsa_uart_probe,
2895 	.remove		= sbsa_uart_remove,
2896 	.driver	= {
2897 		.name	= "sbsa-uart",
2898 		.pm	= &pl011_dev_pm_ops,
2899 		.of_match_table = of_match_ptr(sbsa_uart_of_match),
2900 		.acpi_match_table = ACPI_PTR(sbsa_uart_acpi_match),
2901 		.suppress_bind_attrs = IS_BUILTIN(CONFIG_SERIAL_AMBA_PL011),
2902 	},
2903 };
2904 
2905 static const struct amba_id pl011_ids[] = {
2906 	{
2907 		.id	= 0x00041011,
2908 		.mask	= 0x000fffff,
2909 		.data	= &vendor_arm,
2910 	},
2911 	{
2912 		.id	= 0x00380802,
2913 		.mask	= 0x00ffffff,
2914 		.data	= &vendor_st,
2915 	},
2916 	{ 0, 0 },
2917 };
2918 
2919 MODULE_DEVICE_TABLE(amba, pl011_ids);
2920 
2921 static struct amba_driver pl011_driver = {
2922 	.drv = {
2923 		.name	= "uart-pl011",
2924 		.pm	= &pl011_dev_pm_ops,
2925 		.suppress_bind_attrs = IS_BUILTIN(CONFIG_SERIAL_AMBA_PL011),
2926 	},
2927 	.id_table	= pl011_ids,
2928 	.probe		= pl011_probe,
2929 	.remove		= pl011_remove,
2930 };
2931 
2932 static int __init pl011_init(void)
2933 {
2934 	printk(KERN_INFO "Serial: AMBA PL011 UART driver\n");
2935 
2936 	if (platform_driver_register(&arm_sbsa_uart_platform_driver))
2937 		pr_warn("could not register SBSA UART platform driver\n");
2938 	return amba_driver_register(&pl011_driver);
2939 }
2940 
2941 static void __exit pl011_exit(void)
2942 {
2943 	platform_driver_unregister(&arm_sbsa_uart_platform_driver);
2944 	amba_driver_unregister(&pl011_driver);
2945 }
2946 
2947 /*
2948  * While this can be a module, if builtin it's most likely the console
2949  * So let's leave module_exit but move module_init to an earlier place
2950  */
2951 arch_initcall(pl011_init);
2952 module_exit(pl011_exit);
2953 
2954 MODULE_AUTHOR("ARM Ltd/Deep Blue Solutions Ltd");
2955 MODULE_DESCRIPTION("ARM AMBA serial port driver");
2956 MODULE_LICENSE("GPL");
2957