xref: /openbmc/linux/drivers/spi/spi-img-spfi.c (revision 979ac5ef) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * IMG SPFI controller driver
4  *
5  * Copyright (C) 2007,2008,2013 Imagination Technologies Ltd.
6  * Copyright (C) 2014 Google, Inc.
7  */
8 
9 #include <linux/clk.h>
10 #include <linux/delay.h>
11 #include <linux/dmaengine.h>
12 #include <linux/interrupt.h>
13 #include <linux/io.h>
14 #include <linux/irq.h>
15 #include <linux/module.h>
16 #include <linux/of.h>
17 #include <linux/platform_device.h>
18 #include <linux/pm_runtime.h>
19 #include <linux/scatterlist.h>
20 #include <linux/slab.h>
21 #include <linux/spi/spi.h>
22 #include <linux/spinlock.h>
23 
24 #define SPFI_DEVICE_PARAMETER(x)		(0x00 + 0x4 * (x))
25 #define SPFI_DEVICE_PARAMETER_BITCLK_SHIFT	24
26 #define SPFI_DEVICE_PARAMETER_BITCLK_MASK	0xff
27 #define SPFI_DEVICE_PARAMETER_CSSETUP_SHIFT	16
28 #define SPFI_DEVICE_PARAMETER_CSSETUP_MASK	0xff
29 #define SPFI_DEVICE_PARAMETER_CSHOLD_SHIFT	8
30 #define SPFI_DEVICE_PARAMETER_CSHOLD_MASK	0xff
31 #define SPFI_DEVICE_PARAMETER_CSDELAY_SHIFT	0
32 #define SPFI_DEVICE_PARAMETER_CSDELAY_MASK	0xff
33 
34 #define SPFI_CONTROL				0x14
35 #define SPFI_CONTROL_CONTINUE			BIT(12)
36 #define SPFI_CONTROL_SOFT_RESET			BIT(11)
37 #define SPFI_CONTROL_SEND_DMA			BIT(10)
38 #define SPFI_CONTROL_GET_DMA			BIT(9)
39 #define SPFI_CONTROL_SE			BIT(8)
40 #define SPFI_CONTROL_TMODE_SHIFT		5
41 #define SPFI_CONTROL_TMODE_MASK			0x7
42 #define SPFI_CONTROL_TMODE_SINGLE		0
43 #define SPFI_CONTROL_TMODE_DUAL			1
44 #define SPFI_CONTROL_TMODE_QUAD			2
45 #define SPFI_CONTROL_SPFI_EN			BIT(0)
46 
47 #define SPFI_TRANSACTION			0x18
48 #define SPFI_TRANSACTION_TSIZE_SHIFT		16
49 #define SPFI_TRANSACTION_TSIZE_MASK		0xffff
50 
51 #define SPFI_PORT_STATE				0x1c
52 #define SPFI_PORT_STATE_DEV_SEL_SHIFT		20
53 #define SPFI_PORT_STATE_DEV_SEL_MASK		0x7
54 #define SPFI_PORT_STATE_CK_POL(x)		BIT(19 - (x))
55 #define SPFI_PORT_STATE_CK_PHASE(x)		BIT(14 - (x))
56 
57 #define SPFI_TX_32BIT_VALID_DATA		0x20
58 #define SPFI_TX_8BIT_VALID_DATA			0x24
59 #define SPFI_RX_32BIT_VALID_DATA		0x28
60 #define SPFI_RX_8BIT_VALID_DATA			0x2c
61 
62 #define SPFI_INTERRUPT_STATUS			0x30
63 #define SPFI_INTERRUPT_ENABLE			0x34
64 #define SPFI_INTERRUPT_CLEAR			0x38
65 #define SPFI_INTERRUPT_IACCESS			BIT(12)
66 #define SPFI_INTERRUPT_GDEX8BIT			BIT(11)
67 #define SPFI_INTERRUPT_ALLDONETRIG		BIT(9)
68 #define SPFI_INTERRUPT_GDFUL			BIT(8)
69 #define SPFI_INTERRUPT_GDHF			BIT(7)
70 #define SPFI_INTERRUPT_GDEX32BIT		BIT(6)
71 #define SPFI_INTERRUPT_GDTRIG			BIT(5)
72 #define SPFI_INTERRUPT_SDFUL			BIT(3)
73 #define SPFI_INTERRUPT_SDHF			BIT(2)
74 #define SPFI_INTERRUPT_SDE			BIT(1)
75 #define SPFI_INTERRUPT_SDTRIG			BIT(0)
76 
77 /*
78  * There are four parallel FIFOs of 16 bytes each.  The word buffer
79  * (*_32BIT_VALID_DATA) accesses all four FIFOs at once, resulting in an
80  * effective FIFO size of 64 bytes.  The byte buffer (*_8BIT_VALID_DATA)
81  * accesses only a single FIFO, resulting in an effective FIFO size of
82  * 16 bytes.
83  */
84 #define SPFI_32BIT_FIFO_SIZE			64
85 #define SPFI_8BIT_FIFO_SIZE			16
86 
87 struct img_spfi {
88 	struct device *dev;
89 	struct spi_master *master;
90 	spinlock_t lock;
91 
92 	void __iomem *regs;
93 	phys_addr_t phys;
94 	int irq;
95 	struct clk *spfi_clk;
96 	struct clk *sys_clk;
97 
98 	struct dma_chan *rx_ch;
99 	struct dma_chan *tx_ch;
100 	bool tx_dma_busy;
101 	bool rx_dma_busy;
102 };
103 
104 static inline u32 spfi_readl(struct img_spfi *spfi, u32 reg)
105 {
106 	return readl(spfi->regs + reg);
107 }
108 
109 static inline void spfi_writel(struct img_spfi *spfi, u32 val, u32 reg)
110 {
111 	writel(val, spfi->regs + reg);
112 }
113 
114 static inline void spfi_start(struct img_spfi *spfi)
115 {
116 	u32 val;
117 
118 	val = spfi_readl(spfi, SPFI_CONTROL);
119 	val |= SPFI_CONTROL_SPFI_EN;
120 	spfi_writel(spfi, val, SPFI_CONTROL);
121 }
122 
123 static inline void spfi_reset(struct img_spfi *spfi)
124 {
125 	spfi_writel(spfi, SPFI_CONTROL_SOFT_RESET, SPFI_CONTROL);
126 	spfi_writel(spfi, 0, SPFI_CONTROL);
127 }
128 
129 static int spfi_wait_all_done(struct img_spfi *spfi)
130 {
131 	unsigned long timeout = jiffies + msecs_to_jiffies(50);
132 
133 	while (time_before(jiffies, timeout)) {
134 		u32 status = spfi_readl(spfi, SPFI_INTERRUPT_STATUS);
135 
136 		if (status & SPFI_INTERRUPT_ALLDONETRIG) {
137 			spfi_writel(spfi, SPFI_INTERRUPT_ALLDONETRIG,
138 				    SPFI_INTERRUPT_CLEAR);
139 			return 0;
140 		}
141 		cpu_relax();
142 	}
143 
144 	dev_err(spfi->dev, "Timed out waiting for transaction to complete\n");
145 	spfi_reset(spfi);
146 
147 	return -ETIMEDOUT;
148 }
149 
150 static unsigned int spfi_pio_write32(struct img_spfi *spfi, const u32 *buf,
151 				     unsigned int max)
152 {
153 	unsigned int count = 0;
154 	u32 status;
155 
156 	while (count < max / 4) {
157 		spfi_writel(spfi, SPFI_INTERRUPT_SDFUL, SPFI_INTERRUPT_CLEAR);
158 		status = spfi_readl(spfi, SPFI_INTERRUPT_STATUS);
159 		if (status & SPFI_INTERRUPT_SDFUL)
160 			break;
161 		spfi_writel(spfi, buf[count], SPFI_TX_32BIT_VALID_DATA);
162 		count++;
163 	}
164 
165 	return count * 4;
166 }
167 
168 static unsigned int spfi_pio_write8(struct img_spfi *spfi, const u8 *buf,
169 				    unsigned int max)
170 {
171 	unsigned int count = 0;
172 	u32 status;
173 
174 	while (count < max) {
175 		spfi_writel(spfi, SPFI_INTERRUPT_SDFUL, SPFI_INTERRUPT_CLEAR);
176 		status = spfi_readl(spfi, SPFI_INTERRUPT_STATUS);
177 		if (status & SPFI_INTERRUPT_SDFUL)
178 			break;
179 		spfi_writel(spfi, buf[count], SPFI_TX_8BIT_VALID_DATA);
180 		count++;
181 	}
182 
183 	return count;
184 }
185 
186 static unsigned int spfi_pio_read32(struct img_spfi *spfi, u32 *buf,
187 				    unsigned int max)
188 {
189 	unsigned int count = 0;
190 	u32 status;
191 
192 	while (count < max / 4) {
193 		spfi_writel(spfi, SPFI_INTERRUPT_GDEX32BIT,
194 			    SPFI_INTERRUPT_CLEAR);
195 		status = spfi_readl(spfi, SPFI_INTERRUPT_STATUS);
196 		if (!(status & SPFI_INTERRUPT_GDEX32BIT))
197 			break;
198 		buf[count] = spfi_readl(spfi, SPFI_RX_32BIT_VALID_DATA);
199 		count++;
200 	}
201 
202 	return count * 4;
203 }
204 
205 static unsigned int spfi_pio_read8(struct img_spfi *spfi, u8 *buf,
206 				   unsigned int max)
207 {
208 	unsigned int count = 0;
209 	u32 status;
210 
211 	while (count < max) {
212 		spfi_writel(spfi, SPFI_INTERRUPT_GDEX8BIT,
213 			    SPFI_INTERRUPT_CLEAR);
214 		status = spfi_readl(spfi, SPFI_INTERRUPT_STATUS);
215 		if (!(status & SPFI_INTERRUPT_GDEX8BIT))
216 			break;
217 		buf[count] = spfi_readl(spfi, SPFI_RX_8BIT_VALID_DATA);
218 		count++;
219 	}
220 
221 	return count;
222 }
223 
224 static int img_spfi_start_pio(struct spi_master *master,
225 			       struct spi_device *spi,
226 			       struct spi_transfer *xfer)
227 {
228 	struct img_spfi *spfi = spi_master_get_devdata(spi->master);
229 	unsigned int tx_bytes = 0, rx_bytes = 0;
230 	const void *tx_buf = xfer->tx_buf;
231 	void *rx_buf = xfer->rx_buf;
232 	unsigned long timeout;
233 	int ret;
234 
235 	if (tx_buf)
236 		tx_bytes = xfer->len;
237 	if (rx_buf)
238 		rx_bytes = xfer->len;
239 
240 	spfi_start(spfi);
241 
242 	timeout = jiffies +
243 		msecs_to_jiffies(xfer->len * 8 * 1000 / xfer->speed_hz + 100);
244 	while ((tx_bytes > 0 || rx_bytes > 0) &&
245 	       time_before(jiffies, timeout)) {
246 		unsigned int tx_count, rx_count;
247 
248 		if (tx_bytes >= 4)
249 			tx_count = spfi_pio_write32(spfi, tx_buf, tx_bytes);
250 		else
251 			tx_count = spfi_pio_write8(spfi, tx_buf, tx_bytes);
252 
253 		if (rx_bytes >= 4)
254 			rx_count = spfi_pio_read32(spfi, rx_buf, rx_bytes);
255 		else
256 			rx_count = spfi_pio_read8(spfi, rx_buf, rx_bytes);
257 
258 		tx_buf += tx_count;
259 		rx_buf += rx_count;
260 		tx_bytes -= tx_count;
261 		rx_bytes -= rx_count;
262 
263 		cpu_relax();
264 	}
265 
266 	if (rx_bytes > 0 || tx_bytes > 0) {
267 		dev_err(spfi->dev, "PIO transfer timed out\n");
268 		return -ETIMEDOUT;
269 	}
270 
271 	ret = spfi_wait_all_done(spfi);
272 	if (ret < 0)
273 		return ret;
274 
275 	return 0;
276 }
277 
278 static void img_spfi_dma_rx_cb(void *data)
279 {
280 	struct img_spfi *spfi = data;
281 	unsigned long flags;
282 
283 	spfi_wait_all_done(spfi);
284 
285 	spin_lock_irqsave(&spfi->lock, flags);
286 	spfi->rx_dma_busy = false;
287 	if (!spfi->tx_dma_busy)
288 		spi_finalize_current_transfer(spfi->master);
289 	spin_unlock_irqrestore(&spfi->lock, flags);
290 }
291 
292 static void img_spfi_dma_tx_cb(void *data)
293 {
294 	struct img_spfi *spfi = data;
295 	unsigned long flags;
296 
297 	spfi_wait_all_done(spfi);
298 
299 	spin_lock_irqsave(&spfi->lock, flags);
300 	spfi->tx_dma_busy = false;
301 	if (!spfi->rx_dma_busy)
302 		spi_finalize_current_transfer(spfi->master);
303 	spin_unlock_irqrestore(&spfi->lock, flags);
304 }
305 
306 static int img_spfi_start_dma(struct spi_master *master,
307 			      struct spi_device *spi,
308 			      struct spi_transfer *xfer)
309 {
310 	struct img_spfi *spfi = spi_master_get_devdata(spi->master);
311 	struct dma_async_tx_descriptor *rxdesc = NULL, *txdesc = NULL;
312 	struct dma_slave_config rxconf, txconf;
313 
314 	spfi->rx_dma_busy = false;
315 	spfi->tx_dma_busy = false;
316 
317 	if (xfer->rx_buf) {
318 		rxconf.direction = DMA_DEV_TO_MEM;
319 		if (xfer->len % 4 == 0) {
320 			rxconf.src_addr = spfi->phys + SPFI_RX_32BIT_VALID_DATA;
321 			rxconf.src_addr_width = 4;
322 			rxconf.src_maxburst = 4;
323 		} else {
324 			rxconf.src_addr = spfi->phys + SPFI_RX_8BIT_VALID_DATA;
325 			rxconf.src_addr_width = 1;
326 			rxconf.src_maxburst = 4;
327 		}
328 		dmaengine_slave_config(spfi->rx_ch, &rxconf);
329 
330 		rxdesc = dmaengine_prep_slave_sg(spfi->rx_ch, xfer->rx_sg.sgl,
331 						 xfer->rx_sg.nents,
332 						 DMA_DEV_TO_MEM,
333 						 DMA_PREP_INTERRUPT);
334 		if (!rxdesc)
335 			goto stop_dma;
336 
337 		rxdesc->callback = img_spfi_dma_rx_cb;
338 		rxdesc->callback_param = spfi;
339 	}
340 
341 	if (xfer->tx_buf) {
342 		txconf.direction = DMA_MEM_TO_DEV;
343 		if (xfer->len % 4 == 0) {
344 			txconf.dst_addr = spfi->phys + SPFI_TX_32BIT_VALID_DATA;
345 			txconf.dst_addr_width = 4;
346 			txconf.dst_maxburst = 4;
347 		} else {
348 			txconf.dst_addr = spfi->phys + SPFI_TX_8BIT_VALID_DATA;
349 			txconf.dst_addr_width = 1;
350 			txconf.dst_maxburst = 4;
351 		}
352 		dmaengine_slave_config(spfi->tx_ch, &txconf);
353 
354 		txdesc = dmaengine_prep_slave_sg(spfi->tx_ch, xfer->tx_sg.sgl,
355 						 xfer->tx_sg.nents,
356 						 DMA_MEM_TO_DEV,
357 						 DMA_PREP_INTERRUPT);
358 		if (!txdesc)
359 			goto stop_dma;
360 
361 		txdesc->callback = img_spfi_dma_tx_cb;
362 		txdesc->callback_param = spfi;
363 	}
364 
365 	if (xfer->rx_buf) {
366 		spfi->rx_dma_busy = true;
367 		dmaengine_submit(rxdesc);
368 		dma_async_issue_pending(spfi->rx_ch);
369 	}
370 
371 	spfi_start(spfi);
372 
373 	if (xfer->tx_buf) {
374 		spfi->tx_dma_busy = true;
375 		dmaengine_submit(txdesc);
376 		dma_async_issue_pending(spfi->tx_ch);
377 	}
378 
379 	return 1;
380 
381 stop_dma:
382 	dmaengine_terminate_all(spfi->rx_ch);
383 	dmaengine_terminate_all(spfi->tx_ch);
384 	return -EIO;
385 }
386 
387 static void img_spfi_handle_err(struct spi_master *master,
388 				struct spi_message *msg)
389 {
390 	struct img_spfi *spfi = spi_master_get_devdata(master);
391 	unsigned long flags;
392 
393 	/*
394 	 * Stop all DMA and reset the controller if the previous transaction
395 	 * timed-out and never completed it's DMA.
396 	 */
397 	spin_lock_irqsave(&spfi->lock, flags);
398 	if (spfi->tx_dma_busy || spfi->rx_dma_busy) {
399 		spfi->tx_dma_busy = false;
400 		spfi->rx_dma_busy = false;
401 
402 		dmaengine_terminate_all(spfi->tx_ch);
403 		dmaengine_terminate_all(spfi->rx_ch);
404 	}
405 	spin_unlock_irqrestore(&spfi->lock, flags);
406 }
407 
408 static int img_spfi_prepare(struct spi_master *master, struct spi_message *msg)
409 {
410 	struct img_spfi *spfi = spi_master_get_devdata(master);
411 	u32 val;
412 
413 	val = spfi_readl(spfi, SPFI_PORT_STATE);
414 	val &= ~(SPFI_PORT_STATE_DEV_SEL_MASK <<
415 		 SPFI_PORT_STATE_DEV_SEL_SHIFT);
416 	val |= msg->spi->chip_select << SPFI_PORT_STATE_DEV_SEL_SHIFT;
417 	if (msg->spi->mode & SPI_CPHA)
418 		val |= SPFI_PORT_STATE_CK_PHASE(msg->spi->chip_select);
419 	else
420 		val &= ~SPFI_PORT_STATE_CK_PHASE(msg->spi->chip_select);
421 	if (msg->spi->mode & SPI_CPOL)
422 		val |= SPFI_PORT_STATE_CK_POL(msg->spi->chip_select);
423 	else
424 		val &= ~SPFI_PORT_STATE_CK_POL(msg->spi->chip_select);
425 	spfi_writel(spfi, val, SPFI_PORT_STATE);
426 
427 	return 0;
428 }
429 
430 static int img_spfi_unprepare(struct spi_master *master,
431 			      struct spi_message *msg)
432 {
433 	struct img_spfi *spfi = spi_master_get_devdata(master);
434 
435 	spfi_reset(spfi);
436 
437 	return 0;
438 }
439 
440 static void img_spfi_config(struct spi_master *master, struct spi_device *spi,
441 			    struct spi_transfer *xfer)
442 {
443 	struct img_spfi *spfi = spi_master_get_devdata(spi->master);
444 	u32 val, div;
445 
446 	/*
447 	 * output = spfi_clk * (BITCLK / 512), where BITCLK must be a
448 	 * power of 2 up to 128
449 	 */
450 	div = DIV_ROUND_UP(clk_get_rate(spfi->spfi_clk), xfer->speed_hz);
451 	div = clamp(512 / (1 << get_count_order(div)), 1, 128);
452 
453 	val = spfi_readl(spfi, SPFI_DEVICE_PARAMETER(spi->chip_select));
454 	val &= ~(SPFI_DEVICE_PARAMETER_BITCLK_MASK <<
455 		 SPFI_DEVICE_PARAMETER_BITCLK_SHIFT);
456 	val |= div << SPFI_DEVICE_PARAMETER_BITCLK_SHIFT;
457 	spfi_writel(spfi, val, SPFI_DEVICE_PARAMETER(spi->chip_select));
458 
459 	spfi_writel(spfi, xfer->len << SPFI_TRANSACTION_TSIZE_SHIFT,
460 		    SPFI_TRANSACTION);
461 
462 	val = spfi_readl(spfi, SPFI_CONTROL);
463 	val &= ~(SPFI_CONTROL_SEND_DMA | SPFI_CONTROL_GET_DMA);
464 	if (xfer->tx_buf)
465 		val |= SPFI_CONTROL_SEND_DMA;
466 	if (xfer->rx_buf)
467 		val |= SPFI_CONTROL_GET_DMA;
468 	val &= ~(SPFI_CONTROL_TMODE_MASK << SPFI_CONTROL_TMODE_SHIFT);
469 	if (xfer->tx_nbits == SPI_NBITS_DUAL &&
470 	    xfer->rx_nbits == SPI_NBITS_DUAL)
471 		val |= SPFI_CONTROL_TMODE_DUAL << SPFI_CONTROL_TMODE_SHIFT;
472 	else if (xfer->tx_nbits == SPI_NBITS_QUAD &&
473 		 xfer->rx_nbits == SPI_NBITS_QUAD)
474 		val |= SPFI_CONTROL_TMODE_QUAD << SPFI_CONTROL_TMODE_SHIFT;
475 	val |= SPFI_CONTROL_SE;
476 	spfi_writel(spfi, val, SPFI_CONTROL);
477 }
478 
479 static int img_spfi_transfer_one(struct spi_master *master,
480 				 struct spi_device *spi,
481 				 struct spi_transfer *xfer)
482 {
483 	struct img_spfi *spfi = spi_master_get_devdata(spi->master);
484 	int ret;
485 
486 	if (xfer->len > SPFI_TRANSACTION_TSIZE_MASK) {
487 		dev_err(spfi->dev,
488 			"Transfer length (%d) is greater than the max supported (%d)",
489 			xfer->len, SPFI_TRANSACTION_TSIZE_MASK);
490 		return -EINVAL;
491 	}
492 
493 	img_spfi_config(master, spi, xfer);
494 	if (master->can_dma && master->can_dma(master, spi, xfer))
495 		ret = img_spfi_start_dma(master, spi, xfer);
496 	else
497 		ret = img_spfi_start_pio(master, spi, xfer);
498 
499 	return ret;
500 }
501 
502 static bool img_spfi_can_dma(struct spi_master *master, struct spi_device *spi,
503 			     struct spi_transfer *xfer)
504 {
505 	if (xfer->len > SPFI_32BIT_FIFO_SIZE)
506 		return true;
507 	return false;
508 }
509 
510 static irqreturn_t img_spfi_irq(int irq, void *dev_id)
511 {
512 	struct img_spfi *spfi = (struct img_spfi *)dev_id;
513 	u32 status;
514 
515 	status = spfi_readl(spfi, SPFI_INTERRUPT_STATUS);
516 	if (status & SPFI_INTERRUPT_IACCESS) {
517 		spfi_writel(spfi, SPFI_INTERRUPT_IACCESS, SPFI_INTERRUPT_CLEAR);
518 		dev_err(spfi->dev, "Illegal access interrupt");
519 		return IRQ_HANDLED;
520 	}
521 
522 	return IRQ_NONE;
523 }
524 
525 static int img_spfi_probe(struct platform_device *pdev)
526 {
527 	struct spi_master *master;
528 	struct img_spfi *spfi;
529 	struct resource *res;
530 	int ret;
531 	u32 max_speed_hz;
532 
533 	master = spi_alloc_master(&pdev->dev, sizeof(*spfi));
534 	if (!master)
535 		return -ENOMEM;
536 	platform_set_drvdata(pdev, master);
537 
538 	spfi = spi_master_get_devdata(master);
539 	spfi->dev = &pdev->dev;
540 	spfi->master = master;
541 	spin_lock_init(&spfi->lock);
542 
543 	spfi->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
544 	if (IS_ERR(spfi->regs)) {
545 		ret = PTR_ERR(spfi->regs);
546 		goto put_spi;
547 	}
548 	spfi->phys = res->start;
549 
550 	spfi->irq = platform_get_irq(pdev, 0);
551 	if (spfi->irq < 0) {
552 		ret = spfi->irq;
553 		goto put_spi;
554 	}
555 	ret = devm_request_irq(spfi->dev, spfi->irq, img_spfi_irq,
556 			       IRQ_TYPE_LEVEL_HIGH, dev_name(spfi->dev), spfi);
557 	if (ret)
558 		goto put_spi;
559 
560 	spfi->sys_clk = devm_clk_get(spfi->dev, "sys");
561 	if (IS_ERR(spfi->sys_clk)) {
562 		ret = PTR_ERR(spfi->sys_clk);
563 		goto put_spi;
564 	}
565 	spfi->spfi_clk = devm_clk_get(spfi->dev, "spfi");
566 	if (IS_ERR(spfi->spfi_clk)) {
567 		ret = PTR_ERR(spfi->spfi_clk);
568 		goto put_spi;
569 	}
570 
571 	ret = clk_prepare_enable(spfi->sys_clk);
572 	if (ret)
573 		goto put_spi;
574 	ret = clk_prepare_enable(spfi->spfi_clk);
575 	if (ret)
576 		goto disable_pclk;
577 
578 	spfi_reset(spfi);
579 	/*
580 	 * Only enable the error (IACCESS) interrupt.  In PIO mode we'll
581 	 * poll the status of the FIFOs.
582 	 */
583 	spfi_writel(spfi, SPFI_INTERRUPT_IACCESS, SPFI_INTERRUPT_ENABLE);
584 
585 	master->auto_runtime_pm = true;
586 	master->bus_num = pdev->id;
587 	master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_TX_DUAL | SPI_RX_DUAL;
588 	if (of_property_read_bool(spfi->dev->of_node, "img,supports-quad-mode"))
589 		master->mode_bits |= SPI_TX_QUAD | SPI_RX_QUAD;
590 	master->dev.of_node = pdev->dev.of_node;
591 	master->bits_per_word_mask = SPI_BPW_MASK(32) | SPI_BPW_MASK(8);
592 	master->max_speed_hz = clk_get_rate(spfi->spfi_clk) / 4;
593 	master->min_speed_hz = clk_get_rate(spfi->spfi_clk) / 512;
594 
595 	/*
596 	 * Maximum speed supported by spfi is limited to the lower value
597 	 * between 1/4 of the SPFI clock or to "spfi-max-frequency"
598 	 * defined in the device tree.
599 	 * If no value is defined in the device tree assume the maximum
600 	 * speed supported to be 1/4 of the SPFI clock.
601 	 */
602 	if (!of_property_read_u32(spfi->dev->of_node, "spfi-max-frequency",
603 				  &max_speed_hz)) {
604 		if (master->max_speed_hz > max_speed_hz)
605 			master->max_speed_hz = max_speed_hz;
606 	}
607 
608 	master->transfer_one = img_spfi_transfer_one;
609 	master->prepare_message = img_spfi_prepare;
610 	master->unprepare_message = img_spfi_unprepare;
611 	master->handle_err = img_spfi_handle_err;
612 	master->use_gpio_descriptors = true;
613 
614 	spfi->tx_ch = dma_request_chan(spfi->dev, "tx");
615 	if (IS_ERR(spfi->tx_ch)) {
616 		ret = PTR_ERR(spfi->tx_ch);
617 		spfi->tx_ch = NULL;
618 		if (ret == -EPROBE_DEFER)
619 			goto disable_pm;
620 	}
621 
622 	spfi->rx_ch = dma_request_chan(spfi->dev, "rx");
623 	if (IS_ERR(spfi->rx_ch)) {
624 		ret = PTR_ERR(spfi->rx_ch);
625 		spfi->rx_ch = NULL;
626 		if (ret == -EPROBE_DEFER)
627 			goto disable_pm;
628 	}
629 
630 	if (!spfi->tx_ch || !spfi->rx_ch) {
631 		if (spfi->tx_ch)
632 			dma_release_channel(spfi->tx_ch);
633 		if (spfi->rx_ch)
634 			dma_release_channel(spfi->rx_ch);
635 		spfi->tx_ch = NULL;
636 		spfi->rx_ch = NULL;
637 		dev_warn(spfi->dev, "Failed to get DMA channels, falling back to PIO mode\n");
638 	} else {
639 		master->dma_tx = spfi->tx_ch;
640 		master->dma_rx = spfi->rx_ch;
641 		master->can_dma = img_spfi_can_dma;
642 	}
643 
644 	pm_runtime_set_active(spfi->dev);
645 	pm_runtime_enable(spfi->dev);
646 
647 	ret = devm_spi_register_master(spfi->dev, master);
648 	if (ret)
649 		goto disable_pm;
650 
651 	return 0;
652 
653 disable_pm:
654 	pm_runtime_disable(spfi->dev);
655 	if (spfi->rx_ch)
656 		dma_release_channel(spfi->rx_ch);
657 	if (spfi->tx_ch)
658 		dma_release_channel(spfi->tx_ch);
659 	clk_disable_unprepare(spfi->spfi_clk);
660 disable_pclk:
661 	clk_disable_unprepare(spfi->sys_clk);
662 put_spi:
663 	spi_master_put(master);
664 
665 	return ret;
666 }
667 
668 static int img_spfi_remove(struct platform_device *pdev)
669 {
670 	struct spi_master *master = platform_get_drvdata(pdev);
671 	struct img_spfi *spfi = spi_master_get_devdata(master);
672 
673 	if (spfi->tx_ch)
674 		dma_release_channel(spfi->tx_ch);
675 	if (spfi->rx_ch)
676 		dma_release_channel(spfi->rx_ch);
677 
678 	pm_runtime_disable(spfi->dev);
679 	if (!pm_runtime_status_suspended(spfi->dev)) {
680 		clk_disable_unprepare(spfi->spfi_clk);
681 		clk_disable_unprepare(spfi->sys_clk);
682 	}
683 
684 	return 0;
685 }
686 
687 #ifdef CONFIG_PM
688 static int img_spfi_runtime_suspend(struct device *dev)
689 {
690 	struct spi_master *master = dev_get_drvdata(dev);
691 	struct img_spfi *spfi = spi_master_get_devdata(master);
692 
693 	clk_disable_unprepare(spfi->spfi_clk);
694 	clk_disable_unprepare(spfi->sys_clk);
695 
696 	return 0;
697 }
698 
699 static int img_spfi_runtime_resume(struct device *dev)
700 {
701 	struct spi_master *master = dev_get_drvdata(dev);
702 	struct img_spfi *spfi = spi_master_get_devdata(master);
703 	int ret;
704 
705 	ret = clk_prepare_enable(spfi->sys_clk);
706 	if (ret)
707 		return ret;
708 	ret = clk_prepare_enable(spfi->spfi_clk);
709 	if (ret) {
710 		clk_disable_unprepare(spfi->sys_clk);
711 		return ret;
712 	}
713 
714 	return 0;
715 }
716 #endif /* CONFIG_PM */
717 
718 #ifdef CONFIG_PM_SLEEP
719 static int img_spfi_suspend(struct device *dev)
720 {
721 	struct spi_master *master = dev_get_drvdata(dev);
722 
723 	return spi_master_suspend(master);
724 }
725 
726 static int img_spfi_resume(struct device *dev)
727 {
728 	struct spi_master *master = dev_get_drvdata(dev);
729 	struct img_spfi *spfi = spi_master_get_devdata(master);
730 	int ret;
731 
732 	ret = pm_runtime_resume_and_get(dev);
733 	if (ret < 0)
734 		return ret;
735 	spfi_reset(spfi);
736 	pm_runtime_put(dev);
737 
738 	return spi_master_resume(master);
739 }
740 #endif /* CONFIG_PM_SLEEP */
741 
742 static const struct dev_pm_ops img_spfi_pm_ops = {
743 	SET_RUNTIME_PM_OPS(img_spfi_runtime_suspend, img_spfi_runtime_resume,
744 			   NULL)
745 	SET_SYSTEM_SLEEP_PM_OPS(img_spfi_suspend, img_spfi_resume)
746 };
747 
748 static const struct of_device_id img_spfi_of_match[] = {
749 	{ .compatible = "img,spfi", },
750 	{ },
751 };
752 MODULE_DEVICE_TABLE(of, img_spfi_of_match);
753 
754 static struct platform_driver img_spfi_driver = {
755 	.driver = {
756 		.name = "img-spfi",
757 		.pm = &img_spfi_pm_ops,
758 		.of_match_table = of_match_ptr(img_spfi_of_match),
759 	},
760 	.probe = img_spfi_probe,
761 	.remove = img_spfi_remove,
762 };
763 module_platform_driver(img_spfi_driver);
764 
765 MODULE_DESCRIPTION("IMG SPFI controller driver");
766 MODULE_AUTHOR("Andrew Bresticker <abrestic@chromium.org>");
767 MODULE_LICENSE("GPL v2");
768