1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2020, Broadcom */
3 /*
4  * 8250-core based driver for Broadcom ns16550a UARTs
5  *
6  * This driver uses the standard 8250 driver core but adds additional
7  * optional features including the ability to use a baud rate clock
8  * mux for more accurate high speed baud rate selection and also
9  * an optional DMA engine.
10  *
11  */
12 
13 #include <linux/module.h>
14 #include <linux/types.h>
15 #include <linux/tty.h>
16 #include <linux/errno.h>
17 #include <linux/device.h>
18 #include <linux/io.h>
19 #include <linux/of.h>
20 #include <linux/dma-mapping.h>
21 #include <linux/tty_flip.h>
22 #include <linux/delay.h>
23 #include <linux/clk.h>
24 #include <linux/debugfs.h>
25 
26 #include "8250.h"
27 
28 /* Register definitions for UART DMA block. Version 1.1 or later. */
29 #define UDMA_ARB_RX		0x00
30 #define UDMA_ARB_TX		0x04
31 #define		UDMA_ARB_REQ				0x00000001
32 #define		UDMA_ARB_GRANT				0x00000002
33 
34 #define UDMA_RX_REVISION	0x00
35 #define UDMA_RX_REVISION_REQUIRED			0x00000101
36 #define UDMA_RX_CTRL		0x04
37 #define		UDMA_RX_CTRL_BUF_CLOSE_MODE		0x00010000
38 #define		UDMA_RX_CTRL_MASK_WR_DONE		0x00008000
39 #define		UDMA_RX_CTRL_ENDIAN_OVERRIDE		0x00004000
40 #define		UDMA_RX_CTRL_ENDIAN			0x00002000
41 #define		UDMA_RX_CTRL_OE_IS_ERR			0x00001000
42 #define		UDMA_RX_CTRL_PE_IS_ERR			0x00000800
43 #define		UDMA_RX_CTRL_FE_IS_ERR			0x00000400
44 #define		UDMA_RX_CTRL_NUM_BUF_USED_MASK		0x000003c0
45 #define		UDMA_RX_CTRL_NUM_BUF_USED_SHIFT	6
46 #define		UDMA_RX_CTRL_BUF_CLOSE_CLK_SEL_SYS	0x00000020
47 #define		UDMA_RX_CTRL_BUF_CLOSE_ENA		0x00000010
48 #define		UDMA_RX_CTRL_TIMEOUT_CLK_SEL_SYS	0x00000008
49 #define		UDMA_RX_CTRL_TIMEOUT_ENA		0x00000004
50 #define		UDMA_RX_CTRL_ABORT			0x00000002
51 #define		UDMA_RX_CTRL_ENA			0x00000001
52 #define UDMA_RX_STATUS		0x08
53 #define		UDMA_RX_STATUS_ACTIVE_BUF_MASK		0x0000000f
54 #define UDMA_RX_TRANSFER_LEN	0x0c
55 #define UDMA_RX_TRANSFER_TOTAL	0x10
56 #define UDMA_RX_BUFFER_SIZE	0x14
57 #define UDMA_RX_SRC_ADDR	0x18
58 #define UDMA_RX_TIMEOUT		0x1c
59 #define UDMA_RX_BUFFER_CLOSE	0x20
60 #define UDMA_RX_BLOCKOUT_COUNTER 0x24
61 #define UDMA_RX_BUF0_PTR_LO	0x28
62 #define UDMA_RX_BUF0_PTR_HI	0x2c
63 #define UDMA_RX_BUF0_STATUS	0x30
64 #define		UDMA_RX_BUFX_STATUS_OVERRUN_ERR		0x00000010
65 #define		UDMA_RX_BUFX_STATUS_FRAME_ERR		0x00000008
66 #define		UDMA_RX_BUFX_STATUS_PARITY_ERR		0x00000004
67 #define		UDMA_RX_BUFX_STATUS_CLOSE_EXPIRED	0x00000002
68 #define		UDMA_RX_BUFX_STATUS_DATA_RDY		0x00000001
69 #define UDMA_RX_BUF0_DATA_LEN	0x34
70 #define UDMA_RX_BUF1_PTR_LO	0x38
71 #define UDMA_RX_BUF1_PTR_HI	0x3c
72 #define UDMA_RX_BUF1_STATUS	0x40
73 #define UDMA_RX_BUF1_DATA_LEN	0x44
74 
75 #define UDMA_TX_REVISION	0x00
76 #define UDMA_TX_REVISION_REQUIRED			0x00000101
77 #define UDMA_TX_CTRL		0x04
78 #define		UDMA_TX_CTRL_ENDIAN_OVERRIDE		0x00000080
79 #define		UDMA_TX_CTRL_ENDIAN			0x00000040
80 #define		UDMA_TX_CTRL_NUM_BUF_USED_MASK		0x00000030
81 #define		UDMA_TX_CTRL_NUM_BUF_USED_1		0x00000010
82 #define		UDMA_TX_CTRL_ABORT			0x00000002
83 #define		UDMA_TX_CTRL_ENA			0x00000001
84 #define UDMA_TX_DST_ADDR	0x08
85 #define UDMA_TX_BLOCKOUT_COUNTER 0x10
86 #define UDMA_TX_TRANSFER_LEN	0x14
87 #define UDMA_TX_TRANSFER_TOTAL	0x18
88 #define UDMA_TX_STATUS		0x20
89 #define UDMA_TX_BUF0_PTR_LO	0x24
90 #define UDMA_TX_BUF0_PTR_HI	0x28
91 #define UDMA_TX_BUF0_STATUS	0x2c
92 #define		UDMA_TX_BUFX_LAST			0x00000002
93 #define		UDMA_TX_BUFX_EMPTY			0x00000001
94 #define UDMA_TX_BUF0_DATA_LEN	0x30
95 #define UDMA_TX_BUF0_DATA_SENT	0x34
96 #define UDMA_TX_BUF1_PTR_LO	0x38
97 
98 #define UDMA_INTR_STATUS	0x00
99 #define		UDMA_INTR_ARB_TX_GRANT			0x00040000
100 #define		UDMA_INTR_ARB_RX_GRANT			0x00020000
101 #define		UDMA_INTR_TX_ALL_EMPTY			0x00010000
102 #define		UDMA_INTR_TX_EMPTY_BUF1			0x00008000
103 #define		UDMA_INTR_TX_EMPTY_BUF0			0x00004000
104 #define		UDMA_INTR_TX_ABORT			0x00002000
105 #define		UDMA_INTR_TX_DONE			0x00001000
106 #define		UDMA_INTR_RX_ERROR			0x00000800
107 #define		UDMA_INTR_RX_TIMEOUT			0x00000400
108 #define		UDMA_INTR_RX_READY_BUF7			0x00000200
109 #define		UDMA_INTR_RX_READY_BUF6			0x00000100
110 #define		UDMA_INTR_RX_READY_BUF5			0x00000080
111 #define		UDMA_INTR_RX_READY_BUF4			0x00000040
112 #define		UDMA_INTR_RX_READY_BUF3			0x00000020
113 #define		UDMA_INTR_RX_READY_BUF2			0x00000010
114 #define		UDMA_INTR_RX_READY_BUF1			0x00000008
115 #define		UDMA_INTR_RX_READY_BUF0			0x00000004
116 #define		UDMA_INTR_RX_READY_MASK			0x000003fc
117 #define		UDMA_INTR_RX_READY_SHIFT		2
118 #define		UDMA_INTR_RX_ABORT			0x00000002
119 #define		UDMA_INTR_RX_DONE			0x00000001
120 #define UDMA_INTR_SET		0x04
121 #define UDMA_INTR_CLEAR		0x08
122 #define UDMA_INTR_MASK_STATUS	0x0c
123 #define UDMA_INTR_MASK_SET	0x10
124 #define UDMA_INTR_MASK_CLEAR	0x14
125 
126 
127 #define UDMA_RX_INTERRUPTS ( \
128 	UDMA_INTR_RX_ERROR | \
129 	UDMA_INTR_RX_TIMEOUT | \
130 	UDMA_INTR_RX_READY_BUF0 | \
131 	UDMA_INTR_RX_READY_BUF1 | \
132 	UDMA_INTR_RX_READY_BUF2 | \
133 	UDMA_INTR_RX_READY_BUF3 | \
134 	UDMA_INTR_RX_READY_BUF4 | \
135 	UDMA_INTR_RX_READY_BUF5 | \
136 	UDMA_INTR_RX_READY_BUF6 | \
137 	UDMA_INTR_RX_READY_BUF7 | \
138 	UDMA_INTR_RX_ABORT | \
139 	UDMA_INTR_RX_DONE)
140 
141 #define UDMA_RX_ERR_INTERRUPTS ( \
142 	UDMA_INTR_RX_ERROR | \
143 	UDMA_INTR_RX_TIMEOUT | \
144 	UDMA_INTR_RX_ABORT | \
145 	UDMA_INTR_RX_DONE)
146 
147 #define UDMA_TX_INTERRUPTS ( \
148 	UDMA_INTR_TX_ABORT | \
149 	UDMA_INTR_TX_DONE)
150 
151 #define UDMA_IS_RX_INTERRUPT(status) ((status) & UDMA_RX_INTERRUPTS)
152 #define UDMA_IS_TX_INTERRUPT(status) ((status) & UDMA_TX_INTERRUPTS)
153 
154 
155 /* Current devices have 8 sets of RX buffer registers */
156 #define UDMA_RX_BUFS_COUNT	8
157 #define UDMA_RX_BUFS_REG_OFFSET (UDMA_RX_BUF1_PTR_LO - UDMA_RX_BUF0_PTR_LO)
158 #define UDMA_RX_BUFx_PTR_LO(x)	(UDMA_RX_BUF0_PTR_LO + \
159 				 ((x) * UDMA_RX_BUFS_REG_OFFSET))
160 #define UDMA_RX_BUFx_PTR_HI(x)	(UDMA_RX_BUF0_PTR_HI + \
161 				 ((x) * UDMA_RX_BUFS_REG_OFFSET))
162 #define UDMA_RX_BUFx_STATUS(x)	(UDMA_RX_BUF0_STATUS + \
163 				 ((x) * UDMA_RX_BUFS_REG_OFFSET))
164 #define UDMA_RX_BUFx_DATA_LEN(x) (UDMA_RX_BUF0_DATA_LEN + \
165 				  ((x) * UDMA_RX_BUFS_REG_OFFSET))
166 
167 /* Current devices have 2 sets of TX buffer registers */
168 #define UDMA_TX_BUFS_COUNT	2
169 #define UDMA_TX_BUFS_REG_OFFSET (UDMA_TX_BUF1_PTR_LO - UDMA_TX_BUF0_PTR_LO)
170 #define UDMA_TX_BUFx_PTR_LO(x)	(UDMA_TX_BUF0_PTR_LO + \
171 				 ((x) * UDMA_TX_BUFS_REG_OFFSET))
172 #define UDMA_TX_BUFx_PTR_HI(x)	(UDMA_TX_BUF0_PTR_HI + \
173 				 ((x) * UDMA_TX_BUFS_REG_OFFSET))
174 #define UDMA_TX_BUFx_STATUS(x)	(UDMA_TX_BUF0_STATUS + \
175 				 ((x) * UDMA_TX_BUFS_REG_OFFSET))
176 #define UDMA_TX_BUFx_DATA_LEN(x) (UDMA_TX_BUF0_DATA_LEN + \
177 				  ((x) * UDMA_TX_BUFS_REG_OFFSET))
178 #define UDMA_TX_BUFx_DATA_SENT(x) (UDMA_TX_BUF0_DATA_SENT + \
179 				   ((x) * UDMA_TX_BUFS_REG_OFFSET))
180 #define REGS_8250 0
181 #define REGS_DMA_RX 1
182 #define REGS_DMA_TX 2
183 #define REGS_DMA_ISR 3
184 #define REGS_DMA_ARB 4
185 #define REGS_MAX 5
186 
187 #define TX_BUF_SIZE 4096
188 #define RX_BUF_SIZE 4096
189 #define RX_BUFS_COUNT 2
190 #define KHZ    1000
191 #define MHZ(x) ((x) * KHZ * KHZ)
192 
193 static const u32 brcmstb_rate_table[] = {
194 	MHZ(81),
195 	MHZ(108),
196 	MHZ(64),		/* Actually 64285715 for some chips */
197 	MHZ(48),
198 };
199 
200 static const u32 brcmstb_rate_table_7278[] = {
201 	MHZ(81),
202 	MHZ(108),
203 	0,
204 	MHZ(48),
205 };
206 
207 struct brcmuart_priv {
208 	int		line;
209 	struct clk	*baud_mux_clk;
210 	unsigned long	default_mux_rate;
211 	u32		real_rates[ARRAY_SIZE(brcmstb_rate_table)];
212 	const u32	*rate_table;
213 	ktime_t		char_wait;
214 	struct uart_port *up;
215 	struct hrtimer	hrt;
216 	bool		shutdown;
217 	bool		dma_enabled;
218 	struct uart_8250_dma dma;
219 	void __iomem	*regs[REGS_MAX];
220 	dma_addr_t	rx_addr;
221 	void		*rx_bufs;
222 	size_t		rx_size;
223 	int		rx_next_buf;
224 	dma_addr_t	tx_addr;
225 	void		*tx_buf;
226 	size_t		tx_size;
227 	bool		tx_running;
228 	bool		rx_running;
229 	struct dentry	*debugfs_dir;
230 
231 	/* stats exposed through debugfs */
232 	u64		dma_rx_partial_buf;
233 	u64		dma_rx_full_buf;
234 	u32		rx_bad_timeout_late_char;
235 	u32		rx_bad_timeout_no_char;
236 	u32		rx_missing_close_timeout;
237 	u32		rx_err;
238 	u32		rx_timeout;
239 	u32		rx_abort;
240 	u32		saved_mctrl;
241 };
242 
243 static struct dentry *brcmuart_debugfs_root;
244 
245 /*
246  * Register access routines
247  */
248 static u32 udma_readl(struct brcmuart_priv *priv,
249 		int reg_type, int offset)
250 {
251 	return readl(priv->regs[reg_type] + offset);
252 }
253 
254 static void udma_writel(struct brcmuart_priv *priv,
255 			int reg_type, int offset, u32 value)
256 {
257 	writel(value, priv->regs[reg_type] + offset);
258 }
259 
260 static void udma_set(struct brcmuart_priv *priv,
261 		int reg_type, int offset, u32 bits)
262 {
263 	void __iomem *reg = priv->regs[reg_type] + offset;
264 	u32 value;
265 
266 	value = readl(reg);
267 	value |= bits;
268 	writel(value, reg);
269 }
270 
271 static void udma_unset(struct brcmuart_priv *priv,
272 		int reg_type, int offset, u32 bits)
273 {
274 	void __iomem *reg = priv->regs[reg_type] + offset;
275 	u32 value;
276 
277 	value = readl(reg);
278 	value &= ~bits;
279 	writel(value, reg);
280 }
281 
282 /*
283  * The UART DMA engine hardware can be used by multiple UARTS, but
284  * only one at a time. Sharing is not currently supported so
285  * the first UART to request the DMA engine will get it and any
286  * subsequent requests by other UARTS will fail.
287  */
288 static int brcmuart_arbitration(struct brcmuart_priv *priv, bool acquire)
289 {
290 	u32 rx_grant;
291 	u32 tx_grant;
292 	int waits;
293 	int ret = 0;
294 
295 	if (acquire) {
296 		udma_set(priv, REGS_DMA_ARB, UDMA_ARB_RX, UDMA_ARB_REQ);
297 		udma_set(priv, REGS_DMA_ARB, UDMA_ARB_TX, UDMA_ARB_REQ);
298 
299 		waits = 1;
300 		while (1) {
301 			rx_grant = udma_readl(priv, REGS_DMA_ARB, UDMA_ARB_RX);
302 			tx_grant = udma_readl(priv, REGS_DMA_ARB, UDMA_ARB_TX);
303 			if (rx_grant & tx_grant & UDMA_ARB_GRANT)
304 				return 0;
305 			if (waits-- == 0)
306 				break;
307 			msleep(1);
308 		}
309 		ret = 1;
310 	}
311 
312 	udma_unset(priv, REGS_DMA_ARB, UDMA_ARB_RX, UDMA_ARB_REQ);
313 	udma_unset(priv, REGS_DMA_ARB, UDMA_ARB_TX, UDMA_ARB_REQ);
314 	return ret;
315 }
316 
317 static void brcmuart_init_dma_hardware(struct brcmuart_priv *priv)
318 {
319 	u32 daddr;
320 	u32 value;
321 	int x;
322 
323 	/* Start with all interrupts disabled */
324 	udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_SET, 0xffffffff);
325 
326 	udma_writel(priv, REGS_DMA_RX, UDMA_RX_BUFFER_SIZE, RX_BUF_SIZE);
327 
328 	/*
329 	 * Setup buffer close to happen when 32 character times have
330 	 * elapsed since the last character was received.
331 	 */
332 	udma_writel(priv, REGS_DMA_RX, UDMA_RX_BUFFER_CLOSE, 16*10*32);
333 	value = (RX_BUFS_COUNT << UDMA_RX_CTRL_NUM_BUF_USED_SHIFT)
334 		| UDMA_RX_CTRL_BUF_CLOSE_MODE
335 		| UDMA_RX_CTRL_BUF_CLOSE_ENA;
336 	udma_writel(priv, REGS_DMA_RX, UDMA_RX_CTRL, value);
337 
338 	udma_writel(priv, REGS_DMA_RX, UDMA_RX_BLOCKOUT_COUNTER, 0);
339 	daddr = priv->rx_addr;
340 	for (x = 0; x < RX_BUFS_COUNT; x++) {
341 
342 		/* Set RX transfer length to 0 for unknown */
343 		udma_writel(priv, REGS_DMA_RX, UDMA_RX_TRANSFER_LEN, 0);
344 
345 		udma_writel(priv, REGS_DMA_RX, UDMA_RX_BUFx_PTR_LO(x),
346 			    lower_32_bits(daddr));
347 		udma_writel(priv, REGS_DMA_RX, UDMA_RX_BUFx_PTR_HI(x),
348 			    upper_32_bits(daddr));
349 		daddr += RX_BUF_SIZE;
350 	}
351 
352 	daddr = priv->tx_addr;
353 	udma_writel(priv, REGS_DMA_TX, UDMA_TX_BUFx_PTR_LO(0),
354 		    lower_32_bits(daddr));
355 	udma_writel(priv, REGS_DMA_TX, UDMA_TX_BUFx_PTR_HI(0),
356 		    upper_32_bits(daddr));
357 	udma_writel(priv, REGS_DMA_TX, UDMA_TX_CTRL,
358 		    UDMA_TX_CTRL_NUM_BUF_USED_1);
359 
360 	/* clear all interrupts then enable them */
361 	udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_CLEAR, 0xffffffff);
362 	udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_CLEAR,
363 		UDMA_RX_INTERRUPTS | UDMA_TX_INTERRUPTS);
364 
365 }
366 
367 static void start_rx_dma(struct uart_8250_port *p)
368 {
369 	struct brcmuart_priv *priv = p->port.private_data;
370 	int x;
371 
372 	udma_unset(priv, REGS_DMA_RX, UDMA_RX_CTRL, UDMA_RX_CTRL_ENA);
373 
374 	/* Clear the RX ready bit for all buffers */
375 	for (x = 0; x < RX_BUFS_COUNT; x++)
376 		udma_unset(priv, REGS_DMA_RX, UDMA_RX_BUFx_STATUS(x),
377 			UDMA_RX_BUFX_STATUS_DATA_RDY);
378 
379 	/* always start with buffer 0 */
380 	udma_unset(priv, REGS_DMA_RX, UDMA_RX_STATUS,
381 		   UDMA_RX_STATUS_ACTIVE_BUF_MASK);
382 	priv->rx_next_buf = 0;
383 
384 	udma_set(priv, REGS_DMA_RX, UDMA_RX_CTRL, UDMA_RX_CTRL_ENA);
385 	priv->rx_running = true;
386 }
387 
388 static void stop_rx_dma(struct uart_8250_port *p)
389 {
390 	struct brcmuart_priv *priv = p->port.private_data;
391 
392 	/* If RX is running, set the RX ABORT */
393 	if (priv->rx_running)
394 		udma_set(priv, REGS_DMA_RX, UDMA_RX_CTRL, UDMA_RX_CTRL_ABORT);
395 }
396 
397 static int stop_tx_dma(struct uart_8250_port *p)
398 {
399 	struct brcmuart_priv *priv = p->port.private_data;
400 	u32 value;
401 
402 	/* If TX is running, set the TX ABORT */
403 	value = udma_readl(priv, REGS_DMA_TX, UDMA_TX_CTRL);
404 	if (value & UDMA_TX_CTRL_ENA)
405 		udma_set(priv, REGS_DMA_TX, UDMA_TX_CTRL, UDMA_TX_CTRL_ABORT);
406 	priv->tx_running = false;
407 	return 0;
408 }
409 
410 /*
411  * NOTE: printk's in this routine will hang the system if this is
412  * the console tty
413  */
414 static int brcmuart_tx_dma(struct uart_8250_port *p)
415 {
416 	struct brcmuart_priv *priv = p->port.private_data;
417 	struct circ_buf *xmit = &p->port.state->xmit;
418 	u32 tx_size;
419 
420 	if (uart_tx_stopped(&p->port) || priv->tx_running ||
421 		uart_circ_empty(xmit)) {
422 		return 0;
423 	}
424 	tx_size = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
425 
426 	priv->dma.tx_err = 0;
427 	memcpy(priv->tx_buf, &xmit->buf[xmit->tail], tx_size);
428 	uart_xmit_advance(&p->port, tx_size);
429 
430 	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
431 		uart_write_wakeup(&p->port);
432 
433 	udma_writel(priv, REGS_DMA_TX, UDMA_TX_TRANSFER_LEN, tx_size);
434 	udma_writel(priv, REGS_DMA_TX, UDMA_TX_BUF0_DATA_LEN, tx_size);
435 	udma_unset(priv, REGS_DMA_TX, UDMA_TX_BUF0_STATUS, UDMA_TX_BUFX_EMPTY);
436 	udma_set(priv, REGS_DMA_TX, UDMA_TX_CTRL, UDMA_TX_CTRL_ENA);
437 	priv->tx_running = true;
438 
439 	return 0;
440 }
441 
442 static void brcmuart_rx_buf_done_isr(struct uart_port *up, int index)
443 {
444 	struct brcmuart_priv *priv = up->private_data;
445 	struct tty_port *tty_port = &up->state->port;
446 	u32 status;
447 	u32 length;
448 	u32 copied;
449 
450 	/* Make sure we're still in sync with the hardware */
451 	status = udma_readl(priv, REGS_DMA_RX, UDMA_RX_BUFx_STATUS(index));
452 	length = udma_readl(priv, REGS_DMA_RX, UDMA_RX_BUFx_DATA_LEN(index));
453 
454 	if ((status & UDMA_RX_BUFX_STATUS_DATA_RDY) == 0) {
455 		dev_err(up->dev, "RX done interrupt but DATA_RDY not found\n");
456 		return;
457 	}
458 	if (status & (UDMA_RX_BUFX_STATUS_OVERRUN_ERR |
459 		      UDMA_RX_BUFX_STATUS_FRAME_ERR |
460 		      UDMA_RX_BUFX_STATUS_PARITY_ERR)) {
461 		if (status & UDMA_RX_BUFX_STATUS_OVERRUN_ERR) {
462 			up->icount.overrun++;
463 			dev_warn(up->dev, "RX OVERRUN Error\n");
464 		}
465 		if (status & UDMA_RX_BUFX_STATUS_FRAME_ERR) {
466 			up->icount.frame++;
467 			dev_warn(up->dev, "RX FRAMING Error\n");
468 		}
469 		if (status & UDMA_RX_BUFX_STATUS_PARITY_ERR) {
470 			up->icount.parity++;
471 			dev_warn(up->dev, "RX PARITY Error\n");
472 		}
473 	}
474 	copied = (u32)tty_insert_flip_string(
475 		tty_port,
476 		priv->rx_bufs + (index * RX_BUF_SIZE),
477 		length);
478 	if (copied != length) {
479 		dev_warn(up->dev, "Flip buffer overrun of %d bytes\n",
480 			 length - copied);
481 		up->icount.overrun += length - copied;
482 	}
483 	up->icount.rx += length;
484 	if (status & UDMA_RX_BUFX_STATUS_CLOSE_EXPIRED)
485 		priv->dma_rx_partial_buf++;
486 	else if (length != RX_BUF_SIZE)
487 		/*
488 		 * This is a bug in the controller that doesn't cause
489 		 * any problems but will be fixed in the future.
490 		 */
491 		priv->rx_missing_close_timeout++;
492 	else
493 		priv->dma_rx_full_buf++;
494 
495 	tty_flip_buffer_push(tty_port);
496 }
497 
498 static void brcmuart_rx_isr(struct uart_port *up, u32 rx_isr)
499 {
500 	struct brcmuart_priv *priv = up->private_data;
501 	struct device *dev = up->dev;
502 	u32 rx_done_isr;
503 	u32 check_isr;
504 
505 	rx_done_isr = (rx_isr & UDMA_INTR_RX_READY_MASK);
506 	while (rx_done_isr) {
507 		check_isr = UDMA_INTR_RX_READY_BUF0 << priv->rx_next_buf;
508 		if (check_isr & rx_done_isr) {
509 			brcmuart_rx_buf_done_isr(up, priv->rx_next_buf);
510 		} else {
511 			dev_err(dev,
512 				"RX buffer ready out of sequence, restarting RX DMA\n");
513 			start_rx_dma(up_to_u8250p(up));
514 			break;
515 		}
516 		if (rx_isr & UDMA_RX_ERR_INTERRUPTS) {
517 			if (rx_isr & UDMA_INTR_RX_ERROR)
518 				priv->rx_err++;
519 			if (rx_isr & UDMA_INTR_RX_TIMEOUT) {
520 				priv->rx_timeout++;
521 				dev_err(dev, "RX TIMEOUT Error\n");
522 			}
523 			if (rx_isr & UDMA_INTR_RX_ABORT)
524 				priv->rx_abort++;
525 			priv->rx_running = false;
526 		}
527 		/* If not ABORT, re-enable RX buffer */
528 		if (!(rx_isr & UDMA_INTR_RX_ABORT))
529 			udma_unset(priv, REGS_DMA_RX,
530 				   UDMA_RX_BUFx_STATUS(priv->rx_next_buf),
531 				   UDMA_RX_BUFX_STATUS_DATA_RDY);
532 		rx_done_isr &= ~check_isr;
533 		priv->rx_next_buf++;
534 		if (priv->rx_next_buf == RX_BUFS_COUNT)
535 			priv->rx_next_buf = 0;
536 	}
537 }
538 
539 static void brcmuart_tx_isr(struct uart_port *up, u32 isr)
540 {
541 	struct brcmuart_priv *priv = up->private_data;
542 	struct device *dev = up->dev;
543 	struct uart_8250_port *port_8250 = up_to_u8250p(up);
544 	struct circ_buf	*xmit = &port_8250->port.state->xmit;
545 
546 	if (isr & UDMA_INTR_TX_ABORT) {
547 		if (priv->tx_running)
548 			dev_err(dev, "Unexpected TX_ABORT interrupt\n");
549 		return;
550 	}
551 	priv->tx_running = false;
552 	if (!uart_circ_empty(xmit) && !uart_tx_stopped(up))
553 		brcmuart_tx_dma(port_8250);
554 }
555 
556 static irqreturn_t brcmuart_isr(int irq, void *dev_id)
557 {
558 	struct uart_port *up = dev_id;
559 	struct device *dev = up->dev;
560 	struct brcmuart_priv *priv = up->private_data;
561 	unsigned long flags;
562 	u32 interrupts;
563 	u32 rval;
564 	u32 tval;
565 
566 	interrupts = udma_readl(priv, REGS_DMA_ISR, UDMA_INTR_STATUS);
567 	if (interrupts == 0)
568 		return IRQ_NONE;
569 
570 	spin_lock_irqsave(&up->lock, flags);
571 
572 	/* Clear all interrupts */
573 	udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_CLEAR, interrupts);
574 
575 	rval = UDMA_IS_RX_INTERRUPT(interrupts);
576 	if (rval)
577 		brcmuart_rx_isr(up, rval);
578 	tval = UDMA_IS_TX_INTERRUPT(interrupts);
579 	if (tval)
580 		brcmuart_tx_isr(up, tval);
581 	if ((rval | tval) == 0)
582 		dev_warn(dev, "Spurious interrupt: 0x%x\n", interrupts);
583 
584 	spin_unlock_irqrestore(&up->lock, flags);
585 	return IRQ_HANDLED;
586 }
587 
588 static int brcmuart_startup(struct uart_port *port)
589 {
590 	int res;
591 	struct uart_8250_port *up = up_to_u8250p(port);
592 	struct brcmuart_priv *priv = up->port.private_data;
593 
594 	priv->shutdown = false;
595 
596 	/*
597 	 * prevent serial8250_do_startup() from allocating non-existent
598 	 * DMA resources
599 	 */
600 	up->dma = NULL;
601 
602 	res = serial8250_do_startup(port);
603 	if (!priv->dma_enabled)
604 		return res;
605 	/*
606 	 * Disable the Receive Data Interrupt because the DMA engine
607 	 * will handle this.
608 	 */
609 	up->ier &= ~UART_IER_RDI;
610 	serial_port_out(port, UART_IER, up->ier);
611 
612 	priv->tx_running = false;
613 	priv->dma.rx_dma = NULL;
614 	priv->dma.tx_dma = brcmuart_tx_dma;
615 	up->dma = &priv->dma;
616 
617 	brcmuart_init_dma_hardware(priv);
618 	start_rx_dma(up);
619 	return res;
620 }
621 
622 static void brcmuart_shutdown(struct uart_port *port)
623 {
624 	struct uart_8250_port *up = up_to_u8250p(port);
625 	struct brcmuart_priv *priv = up->port.private_data;
626 	unsigned long flags;
627 
628 	spin_lock_irqsave(&port->lock, flags);
629 	priv->shutdown = true;
630 	if (priv->dma_enabled) {
631 		stop_rx_dma(up);
632 		stop_tx_dma(up);
633 		/* disable all interrupts */
634 		udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_SET,
635 			UDMA_RX_INTERRUPTS | UDMA_TX_INTERRUPTS);
636 	}
637 
638 	/*
639 	 * prevent serial8250_do_shutdown() from trying to free
640 	 * DMA resources that we never alloc'd for this driver.
641 	 */
642 	up->dma = NULL;
643 
644 	spin_unlock_irqrestore(&port->lock, flags);
645 	serial8250_do_shutdown(port);
646 }
647 
648 /*
649  * Not all clocks run at the exact specified rate, so set each requested
650  * rate and then get the actual rate.
651  */
652 static void init_real_clk_rates(struct device *dev, struct brcmuart_priv *priv)
653 {
654 	int x;
655 	int rc;
656 
657 	priv->default_mux_rate = clk_get_rate(priv->baud_mux_clk);
658 	for (x = 0; x < ARRAY_SIZE(priv->real_rates); x++) {
659 		if (priv->rate_table[x] == 0) {
660 			priv->real_rates[x] = 0;
661 			continue;
662 		}
663 		rc = clk_set_rate(priv->baud_mux_clk, priv->rate_table[x]);
664 		if (rc) {
665 			dev_err(dev, "Error selecting BAUD MUX clock for %u\n",
666 				priv->rate_table[x]);
667 			priv->real_rates[x] = priv->rate_table[x];
668 		} else {
669 			priv->real_rates[x] = clk_get_rate(priv->baud_mux_clk);
670 		}
671 	}
672 	clk_set_rate(priv->baud_mux_clk, priv->default_mux_rate);
673 }
674 
675 static void set_clock_mux(struct uart_port *up, struct brcmuart_priv *priv,
676 			u32 baud)
677 {
678 	u32 percent;
679 	u32 best_percent = UINT_MAX;
680 	u32 quot;
681 	u32 best_quot = 1;
682 	u32 rate;
683 	int best_index = -1;
684 	u64 hires_rate;
685 	u64 hires_baud;
686 	u64 hires_err;
687 	int rc;
688 	int i;
689 	int real_baud;
690 
691 	/* If the Baud Mux Clock was not specified, just return */
692 	if (priv->baud_mux_clk == NULL)
693 		return;
694 
695 	/* Find the closest match for specified baud */
696 	for (i = 0; i < ARRAY_SIZE(priv->real_rates); i++) {
697 		if (priv->real_rates[i] == 0)
698 			continue;
699 		rate = priv->real_rates[i] / 16;
700 		quot = DIV_ROUND_CLOSEST(rate, baud);
701 		if (!quot)
702 			continue;
703 
704 		/* increase resolution to get xx.xx percent */
705 		hires_rate = (u64)rate * 10000;
706 		hires_baud = (u64)baud * 10000;
707 
708 		hires_err = div_u64(hires_rate, (u64)quot);
709 
710 		/* get the delta */
711 		if (hires_err > hires_baud)
712 			hires_err = (hires_err - hires_baud);
713 		else
714 			hires_err = (hires_baud - hires_err);
715 
716 		percent = (unsigned long)DIV_ROUND_CLOSEST_ULL(hires_err, baud);
717 		dev_dbg(up->dev,
718 			"Baud rate: %u, MUX Clk: %u, Error: %u.%u%%\n",
719 			baud, priv->real_rates[i], percent / 100,
720 			percent % 100);
721 		if (percent < best_percent) {
722 			best_percent = percent;
723 			best_index = i;
724 			best_quot = quot;
725 		}
726 	}
727 	if (best_index == -1) {
728 		dev_err(up->dev, "Error, %d BAUD rate is too fast.\n", baud);
729 		return;
730 	}
731 	rate = priv->real_rates[best_index];
732 	rc = clk_set_rate(priv->baud_mux_clk, rate);
733 	if (rc)
734 		dev_err(up->dev, "Error selecting BAUD MUX clock\n");
735 
736 	/* Error over 3 percent will cause data errors */
737 	if (best_percent > 300)
738 		dev_err(up->dev, "Error, baud: %d has %u.%u%% error\n",
739 			baud, percent / 100, percent % 100);
740 
741 	real_baud = rate / 16 / best_quot;
742 	dev_dbg(up->dev, "Selecting BAUD MUX rate: %u\n", rate);
743 	dev_dbg(up->dev, "Requested baud: %u, Actual baud: %u\n",
744 		baud, real_baud);
745 
746 	/* calc nanoseconds for 1.5 characters time at the given baud rate */
747 	i = NSEC_PER_SEC / real_baud / 10;
748 	i += (i / 2);
749 	priv->char_wait = ns_to_ktime(i);
750 
751 	up->uartclk = rate;
752 }
753 
754 static void brcmstb_set_termios(struct uart_port *up,
755 				struct ktermios *termios,
756 				const struct ktermios *old)
757 {
758 	struct uart_8250_port *p8250 = up_to_u8250p(up);
759 	struct brcmuart_priv *priv = up->private_data;
760 
761 	if (priv->dma_enabled)
762 		stop_rx_dma(p8250);
763 	set_clock_mux(up, priv, tty_termios_baud_rate(termios));
764 	serial8250_do_set_termios(up, termios, old);
765 	if (p8250->mcr & UART_MCR_AFE)
766 		p8250->port.status |= UPSTAT_AUTOCTS;
767 	if (priv->dma_enabled)
768 		start_rx_dma(p8250);
769 }
770 
771 static int brcmuart_handle_irq(struct uart_port *p)
772 {
773 	unsigned int iir = serial_port_in(p, UART_IIR);
774 	struct brcmuart_priv *priv = p->private_data;
775 	struct uart_8250_port *up = up_to_u8250p(p);
776 	unsigned int status;
777 	unsigned long flags;
778 	unsigned int ier;
779 	unsigned int mcr;
780 	int handled = 0;
781 
782 	/*
783 	 * There's a bug in some 8250 cores where we get a timeout
784 	 * interrupt but there is no data ready.
785 	 */
786 	if (((iir & UART_IIR_ID) == UART_IIR_RX_TIMEOUT) && !(priv->shutdown)) {
787 		spin_lock_irqsave(&p->lock, flags);
788 		status = serial_port_in(p, UART_LSR);
789 		if ((status & UART_LSR_DR) == 0) {
790 
791 			ier = serial_port_in(p, UART_IER);
792 			/*
793 			 * if Receive Data Interrupt is enabled and
794 			 * we're uing hardware flow control, deassert
795 			 * RTS and wait for any chars in the pipline to
796 			 * arrive and then check for DR again.
797 			 */
798 			if ((ier & UART_IER_RDI) && (up->mcr & UART_MCR_AFE)) {
799 				ier &= ~(UART_IER_RLSI | UART_IER_RDI);
800 				serial_port_out(p, UART_IER, ier);
801 				mcr = serial_port_in(p, UART_MCR);
802 				mcr &= ~UART_MCR_RTS;
803 				serial_port_out(p, UART_MCR, mcr);
804 				hrtimer_start(&priv->hrt, priv->char_wait,
805 					      HRTIMER_MODE_REL);
806 			} else {
807 				serial_port_in(p, UART_RX);
808 			}
809 
810 			handled = 1;
811 		}
812 		spin_unlock_irqrestore(&p->lock, flags);
813 		if (handled)
814 			return 1;
815 	}
816 	return serial8250_handle_irq(p, iir);
817 }
818 
819 static enum hrtimer_restart brcmuart_hrtimer_func(struct hrtimer *t)
820 {
821 	struct brcmuart_priv *priv = container_of(t, struct brcmuart_priv, hrt);
822 	struct uart_port *p = priv->up;
823 	struct uart_8250_port *up = up_to_u8250p(p);
824 	unsigned int status;
825 	unsigned long flags;
826 
827 	if (priv->shutdown)
828 		return HRTIMER_NORESTART;
829 
830 	spin_lock_irqsave(&p->lock, flags);
831 	status = serial_port_in(p, UART_LSR);
832 
833 	/*
834 	 * If a character did not arrive after the timeout, clear the false
835 	 * receive timeout.
836 	 */
837 	if ((status & UART_LSR_DR) == 0) {
838 		serial_port_in(p, UART_RX);
839 		priv->rx_bad_timeout_no_char++;
840 	} else {
841 		priv->rx_bad_timeout_late_char++;
842 	}
843 
844 	/* re-enable receive unless upper layer has disabled it */
845 	if ((up->ier & (UART_IER_RLSI | UART_IER_RDI)) ==
846 	    (UART_IER_RLSI | UART_IER_RDI)) {
847 		status = serial_port_in(p, UART_IER);
848 		status |= (UART_IER_RLSI | UART_IER_RDI);
849 		serial_port_out(p, UART_IER, status);
850 		status = serial_port_in(p, UART_MCR);
851 		status |= UART_MCR_RTS;
852 		serial_port_out(p, UART_MCR, status);
853 	}
854 	spin_unlock_irqrestore(&p->lock, flags);
855 	return HRTIMER_NORESTART;
856 }
857 
858 static const struct of_device_id brcmuart_dt_ids[] = {
859 	{
860 		.compatible = "brcm,bcm7278-uart",
861 		.data = brcmstb_rate_table_7278,
862 	},
863 	{
864 		.compatible = "brcm,bcm7271-uart",
865 		.data = brcmstb_rate_table,
866 	},
867 	{},
868 };
869 
870 MODULE_DEVICE_TABLE(of, brcmuart_dt_ids);
871 
872 static void brcmuart_free_bufs(struct device *dev, struct brcmuart_priv *priv)
873 {
874 	if (priv->rx_bufs)
875 		dma_free_coherent(dev, priv->rx_size, priv->rx_bufs,
876 				  priv->rx_addr);
877 	if (priv->tx_buf)
878 		dma_free_coherent(dev, priv->tx_size, priv->tx_buf,
879 				  priv->tx_addr);
880 }
881 
882 static void brcmuart_throttle(struct uart_port *port)
883 {
884 	struct brcmuart_priv *priv = port->private_data;
885 
886 	udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_SET, UDMA_RX_INTERRUPTS);
887 }
888 
889 static void brcmuart_unthrottle(struct uart_port *port)
890 {
891 	struct brcmuart_priv *priv = port->private_data;
892 
893 	udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_CLEAR,
894 		    UDMA_RX_INTERRUPTS);
895 }
896 
897 static int debugfs_stats_show(struct seq_file *s, void *unused)
898 {
899 	struct brcmuart_priv *priv = s->private;
900 
901 	seq_printf(s, "rx_err:\t\t\t\t%u\n",
902 		   priv->rx_err);
903 	seq_printf(s, "rx_timeout:\t\t\t%u\n",
904 		   priv->rx_timeout);
905 	seq_printf(s, "rx_abort:\t\t\t%u\n",
906 		   priv->rx_abort);
907 	seq_printf(s, "rx_bad_timeout_late_char:\t%u\n",
908 		   priv->rx_bad_timeout_late_char);
909 	seq_printf(s, "rx_bad_timeout_no_char:\t\t%u\n",
910 		   priv->rx_bad_timeout_no_char);
911 	seq_printf(s, "rx_missing_close_timeout:\t%u\n",
912 		   priv->rx_missing_close_timeout);
913 	if (priv->dma_enabled) {
914 		seq_printf(s, "dma_rx_partial_buf:\t\t%llu\n",
915 			   priv->dma_rx_partial_buf);
916 		seq_printf(s, "dma_rx_full_buf:\t\t%llu\n",
917 			   priv->dma_rx_full_buf);
918 	}
919 	return 0;
920 }
921 DEFINE_SHOW_ATTRIBUTE(debugfs_stats);
922 
923 static void brcmuart_init_debugfs(struct brcmuart_priv *priv,
924 				  const char *device)
925 {
926 	priv->debugfs_dir = debugfs_create_dir(device, brcmuart_debugfs_root);
927 	debugfs_create_file("stats", 0444, priv->debugfs_dir, priv,
928 			    &debugfs_stats_fops);
929 }
930 
931 
932 static int brcmuart_probe(struct platform_device *pdev)
933 {
934 	struct resource *regs;
935 	struct device_node *np = pdev->dev.of_node;
936 	const struct of_device_id *of_id = NULL;
937 	struct uart_8250_port *new_port;
938 	struct device *dev = &pdev->dev;
939 	struct brcmuart_priv *priv;
940 	struct clk *baud_mux_clk;
941 	struct uart_8250_port up;
942 	int irq;
943 	void __iomem *membase = NULL;
944 	resource_size_t mapbase = 0;
945 	u32 clk_rate = 0;
946 	int ret;
947 	int x;
948 	int dma_irq;
949 	static const char * const reg_names[REGS_MAX] = {
950 		"uart", "dma_rx", "dma_tx", "dma_intr2", "dma_arb"
951 	};
952 
953 	irq = platform_get_irq(pdev, 0);
954 	if (irq < 0)
955 		return irq;
956 	priv = devm_kzalloc(dev, sizeof(struct brcmuart_priv),
957 			GFP_KERNEL);
958 	if (!priv)
959 		return -ENOMEM;
960 
961 	of_id = of_match_node(brcmuart_dt_ids, np);
962 	if (!of_id || !of_id->data)
963 		priv->rate_table = brcmstb_rate_table;
964 	else
965 		priv->rate_table = of_id->data;
966 
967 	for (x = 0; x < REGS_MAX; x++) {
968 		regs = platform_get_resource_byname(pdev, IORESOURCE_MEM,
969 						reg_names[x]);
970 		if (!regs)
971 			break;
972 		priv->regs[x] =	devm_ioremap(dev, regs->start,
973 					     resource_size(regs));
974 		if (!priv->regs[x])
975 			return -ENOMEM;
976 		if (x == REGS_8250) {
977 			mapbase = regs->start;
978 			membase = priv->regs[x];
979 		}
980 	}
981 
982 	/* We should have just the uart base registers or all the registers */
983 	if (x != 1 && x != REGS_MAX) {
984 		dev_warn(dev, "%s registers not specified\n", reg_names[x]);
985 		return -EINVAL;
986 	}
987 
988 	/* if the DMA registers were specified, try to enable DMA */
989 	if (x > REGS_DMA_RX) {
990 		if (brcmuart_arbitration(priv, 1) == 0) {
991 			u32 txrev = 0;
992 			u32 rxrev = 0;
993 
994 			txrev = udma_readl(priv, REGS_DMA_RX, UDMA_RX_REVISION);
995 			rxrev = udma_readl(priv, REGS_DMA_TX, UDMA_TX_REVISION);
996 			if ((txrev >= UDMA_TX_REVISION_REQUIRED) &&
997 				(rxrev >= UDMA_RX_REVISION_REQUIRED)) {
998 
999 				/* Enable the use of the DMA hardware */
1000 				priv->dma_enabled = true;
1001 			} else {
1002 				brcmuart_arbitration(priv, 0);
1003 				dev_err(dev,
1004 					"Unsupported DMA Hardware Revision\n");
1005 			}
1006 		} else {
1007 			dev_err(dev,
1008 				"Timeout arbitrating for UART DMA hardware\n");
1009 		}
1010 	}
1011 
1012 	of_property_read_u32(np, "clock-frequency", &clk_rate);
1013 
1014 	/* See if a Baud clock has been specified */
1015 	baud_mux_clk = of_clk_get_by_name(np, "sw_baud");
1016 	if (IS_ERR(baud_mux_clk)) {
1017 		if (PTR_ERR(baud_mux_clk) == -EPROBE_DEFER)
1018 			return -EPROBE_DEFER;
1019 		dev_dbg(dev, "BAUD MUX clock not specified\n");
1020 	} else {
1021 		dev_dbg(dev, "BAUD MUX clock found\n");
1022 		ret = clk_prepare_enable(baud_mux_clk);
1023 		if (ret)
1024 			return ret;
1025 		priv->baud_mux_clk = baud_mux_clk;
1026 		init_real_clk_rates(dev, priv);
1027 		clk_rate = priv->default_mux_rate;
1028 	}
1029 
1030 	if (clk_rate == 0) {
1031 		dev_err(dev, "clock-frequency or clk not defined\n");
1032 		return -EINVAL;
1033 	}
1034 
1035 	dev_dbg(dev, "DMA is %senabled\n", priv->dma_enabled ? "" : "not ");
1036 
1037 	memset(&up, 0, sizeof(up));
1038 	up.port.type = PORT_16550A;
1039 	up.port.uartclk = clk_rate;
1040 	up.port.dev = dev;
1041 	up.port.mapbase = mapbase;
1042 	up.port.membase = membase;
1043 	up.port.irq = irq;
1044 	up.port.handle_irq = brcmuart_handle_irq;
1045 	up.port.regshift = 2;
1046 	up.port.iotype = of_device_is_big_endian(np) ?
1047 		UPIO_MEM32BE : UPIO_MEM32;
1048 	up.port.flags = UPF_SHARE_IRQ | UPF_BOOT_AUTOCONF
1049 		| UPF_FIXED_PORT | UPF_FIXED_TYPE;
1050 	up.port.dev = dev;
1051 	up.port.private_data = priv;
1052 	up.capabilities = UART_CAP_FIFO | UART_CAP_AFE;
1053 	up.port.fifosize = 32;
1054 
1055 	/* Check for a fixed line number */
1056 	ret = of_alias_get_id(np, "serial");
1057 	if (ret >= 0)
1058 		up.port.line = ret;
1059 
1060 	/* setup HR timer */
1061 	hrtimer_init(&priv->hrt, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1062 	priv->hrt.function = brcmuart_hrtimer_func;
1063 
1064 	up.port.shutdown = brcmuart_shutdown;
1065 	up.port.startup = brcmuart_startup;
1066 	up.port.throttle = brcmuart_throttle;
1067 	up.port.unthrottle = brcmuart_unthrottle;
1068 	up.port.set_termios = brcmstb_set_termios;
1069 
1070 	if (priv->dma_enabled) {
1071 		priv->rx_size = RX_BUF_SIZE * RX_BUFS_COUNT;
1072 		priv->rx_bufs = dma_alloc_coherent(dev,
1073 						   priv->rx_size,
1074 						   &priv->rx_addr, GFP_KERNEL);
1075 		if (!priv->rx_bufs) {
1076 			ret = -ENOMEM;
1077 			goto err;
1078 		}
1079 		priv->tx_size = UART_XMIT_SIZE;
1080 		priv->tx_buf = dma_alloc_coherent(dev,
1081 						  priv->tx_size,
1082 						  &priv->tx_addr, GFP_KERNEL);
1083 		if (!priv->tx_buf) {
1084 			ret = -ENOMEM;
1085 			goto err;
1086 		}
1087 	}
1088 
1089 	ret = serial8250_register_8250_port(&up);
1090 	if (ret < 0) {
1091 		dev_err(dev, "unable to register 8250 port\n");
1092 		goto err;
1093 	}
1094 	priv->line = ret;
1095 	new_port = serial8250_get_port(ret);
1096 	priv->up = &new_port->port;
1097 	if (priv->dma_enabled) {
1098 		dma_irq = platform_get_irq_byname(pdev,  "dma");
1099 		if (dma_irq < 0) {
1100 			ret = dma_irq;
1101 			dev_err(dev, "no IRQ resource info\n");
1102 			goto err1;
1103 		}
1104 		ret = devm_request_irq(dev, dma_irq, brcmuart_isr,
1105 				IRQF_SHARED, "uart DMA irq", &new_port->port);
1106 		if (ret) {
1107 			dev_err(dev, "unable to register IRQ handler\n");
1108 			goto err1;
1109 		}
1110 	}
1111 	platform_set_drvdata(pdev, priv);
1112 	brcmuart_init_debugfs(priv, dev_name(&pdev->dev));
1113 	return 0;
1114 
1115 err1:
1116 	serial8250_unregister_port(priv->line);
1117 err:
1118 	brcmuart_free_bufs(dev, priv);
1119 	brcmuart_arbitration(priv, 0);
1120 	return ret;
1121 }
1122 
1123 static int brcmuart_remove(struct platform_device *pdev)
1124 {
1125 	struct brcmuart_priv *priv = platform_get_drvdata(pdev);
1126 
1127 	debugfs_remove_recursive(priv->debugfs_dir);
1128 	hrtimer_cancel(&priv->hrt);
1129 	serial8250_unregister_port(priv->line);
1130 	brcmuart_free_bufs(&pdev->dev, priv);
1131 	brcmuart_arbitration(priv, 0);
1132 	return 0;
1133 }
1134 
1135 static int __maybe_unused brcmuart_suspend(struct device *dev)
1136 {
1137 	struct brcmuart_priv *priv = dev_get_drvdata(dev);
1138 	struct uart_8250_port *up = serial8250_get_port(priv->line);
1139 	struct uart_port *port = &up->port;
1140 	unsigned long flags;
1141 
1142 	/*
1143 	 * This will prevent resume from enabling RTS before the
1144 	 *  baud rate has been restored.
1145 	 */
1146 	spin_lock_irqsave(&port->lock, flags);
1147 	priv->saved_mctrl = port->mctrl;
1148 	port->mctrl &= ~TIOCM_RTS;
1149 	spin_unlock_irqrestore(&port->lock, flags);
1150 
1151 	serial8250_suspend_port(priv->line);
1152 	clk_disable_unprepare(priv->baud_mux_clk);
1153 
1154 	return 0;
1155 }
1156 
1157 static int __maybe_unused brcmuart_resume(struct device *dev)
1158 {
1159 	struct brcmuart_priv *priv = dev_get_drvdata(dev);
1160 	struct uart_8250_port *up = serial8250_get_port(priv->line);
1161 	struct uart_port *port = &up->port;
1162 	unsigned long flags;
1163 	int ret;
1164 
1165 	ret = clk_prepare_enable(priv->baud_mux_clk);
1166 	if (ret)
1167 		dev_err(dev, "Error enabling BAUD MUX clock\n");
1168 
1169 	/*
1170 	 * The hardware goes back to it's default after suspend
1171 	 * so get the "clk" back in sync.
1172 	 */
1173 	ret = clk_set_rate(priv->baud_mux_clk, priv->default_mux_rate);
1174 	if (ret)
1175 		dev_err(dev, "Error restoring default BAUD MUX clock\n");
1176 	if (priv->dma_enabled) {
1177 		if (brcmuart_arbitration(priv, 1)) {
1178 			dev_err(dev, "Timeout arbitrating for DMA hardware on resume\n");
1179 			return(-EBUSY);
1180 		}
1181 		brcmuart_init_dma_hardware(priv);
1182 		start_rx_dma(serial8250_get_port(priv->line));
1183 	}
1184 	serial8250_resume_port(priv->line);
1185 
1186 	if (priv->saved_mctrl & TIOCM_RTS) {
1187 		/* Restore RTS */
1188 		spin_lock_irqsave(&port->lock, flags);
1189 		port->mctrl |= TIOCM_RTS;
1190 		port->ops->set_mctrl(port, port->mctrl);
1191 		spin_unlock_irqrestore(&port->lock, flags);
1192 	}
1193 
1194 	return 0;
1195 }
1196 
1197 static const struct dev_pm_ops brcmuart_dev_pm_ops = {
1198 	SET_SYSTEM_SLEEP_PM_OPS(brcmuart_suspend, brcmuart_resume)
1199 };
1200 
1201 static struct platform_driver brcmuart_platform_driver = {
1202 	.driver = {
1203 		.name	= "bcm7271-uart",
1204 		.pm		= &brcmuart_dev_pm_ops,
1205 		.of_match_table = brcmuart_dt_ids,
1206 	},
1207 	.probe		= brcmuart_probe,
1208 	.remove		= brcmuart_remove,
1209 };
1210 
1211 static int __init brcmuart_init(void)
1212 {
1213 	int ret;
1214 
1215 	brcmuart_debugfs_root = debugfs_create_dir(
1216 		brcmuart_platform_driver.driver.name, NULL);
1217 	ret = platform_driver_register(&brcmuart_platform_driver);
1218 	if (ret) {
1219 		debugfs_remove_recursive(brcmuart_debugfs_root);
1220 		return ret;
1221 	}
1222 
1223 	return 0;
1224 }
1225 module_init(brcmuart_init);
1226 
1227 static void __exit brcmuart_deinit(void)
1228 {
1229 	platform_driver_unregister(&brcmuart_platform_driver);
1230 	debugfs_remove_recursive(brcmuart_debugfs_root);
1231 }
1232 module_exit(brcmuart_deinit);
1233 
1234 MODULE_AUTHOR("Al Cooper");
1235 MODULE_DESCRIPTION("Broadcom NS16550A compatible serial port driver");
1236 MODULE_LICENSE("GPL v2");
1237