xref: /openbmc/linux/drivers/spi/spi-omap2-mcspi.c (revision 6c33a6f4)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * OMAP2 McSPI controller driver
4  *
5  * Copyright (C) 2005, 2006 Nokia Corporation
6  * Author:	Samuel Ortiz <samuel.ortiz@nokia.com> and
7  *		Juha Yrj�l� <juha.yrjola@nokia.com>
8  */
9 
10 #include <linux/kernel.h>
11 #include <linux/interrupt.h>
12 #include <linux/module.h>
13 #include <linux/device.h>
14 #include <linux/delay.h>
15 #include <linux/dma-mapping.h>
16 #include <linux/dmaengine.h>
17 #include <linux/pinctrl/consumer.h>
18 #include <linux/platform_device.h>
19 #include <linux/err.h>
20 #include <linux/clk.h>
21 #include <linux/io.h>
22 #include <linux/slab.h>
23 #include <linux/pm_runtime.h>
24 #include <linux/of.h>
25 #include <linux/of_device.h>
26 #include <linux/gcd.h>
27 #include <linux/iopoll.h>
28 
29 #include <linux/spi/spi.h>
30 #include <linux/gpio.h>
31 
32 #include <linux/platform_data/spi-omap2-mcspi.h>
33 
34 #define OMAP2_MCSPI_MAX_FREQ		48000000
35 #define OMAP2_MCSPI_MAX_DIVIDER		4096
36 #define OMAP2_MCSPI_MAX_FIFODEPTH	64
37 #define OMAP2_MCSPI_MAX_FIFOWCNT	0xFFFF
38 #define SPI_AUTOSUSPEND_TIMEOUT		2000
39 
40 #define OMAP2_MCSPI_REVISION		0x00
41 #define OMAP2_MCSPI_SYSSTATUS		0x14
42 #define OMAP2_MCSPI_IRQSTATUS		0x18
43 #define OMAP2_MCSPI_IRQENABLE		0x1c
44 #define OMAP2_MCSPI_WAKEUPENABLE	0x20
45 #define OMAP2_MCSPI_SYST		0x24
46 #define OMAP2_MCSPI_MODULCTRL		0x28
47 #define OMAP2_MCSPI_XFERLEVEL		0x7c
48 
49 /* per-channel banks, 0x14 bytes each, first is: */
50 #define OMAP2_MCSPI_CHCONF0		0x2c
51 #define OMAP2_MCSPI_CHSTAT0		0x30
52 #define OMAP2_MCSPI_CHCTRL0		0x34
53 #define OMAP2_MCSPI_TX0			0x38
54 #define OMAP2_MCSPI_RX0			0x3c
55 
56 /* per-register bitmasks: */
57 #define OMAP2_MCSPI_IRQSTATUS_EOW	BIT(17)
58 
59 #define OMAP2_MCSPI_MODULCTRL_SINGLE	BIT(0)
60 #define OMAP2_MCSPI_MODULCTRL_MS	BIT(2)
61 #define OMAP2_MCSPI_MODULCTRL_STEST	BIT(3)
62 
63 #define OMAP2_MCSPI_CHCONF_PHA		BIT(0)
64 #define OMAP2_MCSPI_CHCONF_POL		BIT(1)
65 #define OMAP2_MCSPI_CHCONF_CLKD_MASK	(0x0f << 2)
66 #define OMAP2_MCSPI_CHCONF_EPOL		BIT(6)
67 #define OMAP2_MCSPI_CHCONF_WL_MASK	(0x1f << 7)
68 #define OMAP2_MCSPI_CHCONF_TRM_RX_ONLY	BIT(12)
69 #define OMAP2_MCSPI_CHCONF_TRM_TX_ONLY	BIT(13)
70 #define OMAP2_MCSPI_CHCONF_TRM_MASK	(0x03 << 12)
71 #define OMAP2_MCSPI_CHCONF_DMAW		BIT(14)
72 #define OMAP2_MCSPI_CHCONF_DMAR		BIT(15)
73 #define OMAP2_MCSPI_CHCONF_DPE0		BIT(16)
74 #define OMAP2_MCSPI_CHCONF_DPE1		BIT(17)
75 #define OMAP2_MCSPI_CHCONF_IS		BIT(18)
76 #define OMAP2_MCSPI_CHCONF_TURBO	BIT(19)
77 #define OMAP2_MCSPI_CHCONF_FORCE	BIT(20)
78 #define OMAP2_MCSPI_CHCONF_FFET		BIT(27)
79 #define OMAP2_MCSPI_CHCONF_FFER		BIT(28)
80 #define OMAP2_MCSPI_CHCONF_CLKG		BIT(29)
81 
82 #define OMAP2_MCSPI_CHSTAT_RXS		BIT(0)
83 #define OMAP2_MCSPI_CHSTAT_TXS		BIT(1)
84 #define OMAP2_MCSPI_CHSTAT_EOT		BIT(2)
85 #define OMAP2_MCSPI_CHSTAT_TXFFE	BIT(3)
86 
87 #define OMAP2_MCSPI_CHCTRL_EN		BIT(0)
88 #define OMAP2_MCSPI_CHCTRL_EXTCLK_MASK	(0xff << 8)
89 
90 #define OMAP2_MCSPI_WAKEUPENABLE_WKEN	BIT(0)
91 
92 /* We have 2 DMA channels per CS, one for RX and one for TX */
93 struct omap2_mcspi_dma {
94 	struct dma_chan *dma_tx;
95 	struct dma_chan *dma_rx;
96 
97 	struct completion dma_tx_completion;
98 	struct completion dma_rx_completion;
99 
100 	char dma_rx_ch_name[14];
101 	char dma_tx_ch_name[14];
102 };
103 
104 /* use PIO for small transfers, avoiding DMA setup/teardown overhead and
105  * cache operations; better heuristics consider wordsize and bitrate.
106  */
107 #define DMA_MIN_BYTES			160
108 
109 
110 /*
111  * Used for context save and restore, structure members to be updated whenever
112  * corresponding registers are modified.
113  */
114 struct omap2_mcspi_regs {
115 	u32 modulctrl;
116 	u32 wakeupenable;
117 	struct list_head cs;
118 };
119 
120 struct omap2_mcspi {
121 	struct completion	txdone;
122 	struct spi_master	*master;
123 	/* Virtual base address of the controller */
124 	void __iomem		*base;
125 	unsigned long		phys;
126 	/* SPI1 has 4 channels, while SPI2 has 2 */
127 	struct omap2_mcspi_dma	*dma_channels;
128 	struct device		*dev;
129 	struct omap2_mcspi_regs ctx;
130 	int			fifo_depth;
131 	bool			slave_aborted;
132 	unsigned int		pin_dir:1;
133 	size_t			max_xfer_len;
134 };
135 
136 struct omap2_mcspi_cs {
137 	void __iomem		*base;
138 	unsigned long		phys;
139 	int			word_len;
140 	u16			mode;
141 	struct list_head	node;
142 	/* Context save and restore shadow register */
143 	u32			chconf0, chctrl0;
144 };
145 
146 static inline void mcspi_write_reg(struct spi_master *master,
147 		int idx, u32 val)
148 {
149 	struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
150 
151 	writel_relaxed(val, mcspi->base + idx);
152 }
153 
154 static inline u32 mcspi_read_reg(struct spi_master *master, int idx)
155 {
156 	struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
157 
158 	return readl_relaxed(mcspi->base + idx);
159 }
160 
161 static inline void mcspi_write_cs_reg(const struct spi_device *spi,
162 		int idx, u32 val)
163 {
164 	struct omap2_mcspi_cs	*cs = spi->controller_state;
165 
166 	writel_relaxed(val, cs->base +  idx);
167 }
168 
169 static inline u32 mcspi_read_cs_reg(const struct spi_device *spi, int idx)
170 {
171 	struct omap2_mcspi_cs	*cs = spi->controller_state;
172 
173 	return readl_relaxed(cs->base + idx);
174 }
175 
176 static inline u32 mcspi_cached_chconf0(const struct spi_device *spi)
177 {
178 	struct omap2_mcspi_cs *cs = spi->controller_state;
179 
180 	return cs->chconf0;
181 }
182 
183 static inline void mcspi_write_chconf0(const struct spi_device *spi, u32 val)
184 {
185 	struct omap2_mcspi_cs *cs = spi->controller_state;
186 
187 	cs->chconf0 = val;
188 	mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCONF0, val);
189 	mcspi_read_cs_reg(spi, OMAP2_MCSPI_CHCONF0);
190 }
191 
192 static inline int mcspi_bytes_per_word(int word_len)
193 {
194 	if (word_len <= 8)
195 		return 1;
196 	else if (word_len <= 16)
197 		return 2;
198 	else /* word_len <= 32 */
199 		return 4;
200 }
201 
202 static void omap2_mcspi_set_dma_req(const struct spi_device *spi,
203 		int is_read, int enable)
204 {
205 	u32 l, rw;
206 
207 	l = mcspi_cached_chconf0(spi);
208 
209 	if (is_read) /* 1 is read, 0 write */
210 		rw = OMAP2_MCSPI_CHCONF_DMAR;
211 	else
212 		rw = OMAP2_MCSPI_CHCONF_DMAW;
213 
214 	if (enable)
215 		l |= rw;
216 	else
217 		l &= ~rw;
218 
219 	mcspi_write_chconf0(spi, l);
220 }
221 
222 static void omap2_mcspi_set_enable(const struct spi_device *spi, int enable)
223 {
224 	struct omap2_mcspi_cs *cs = spi->controller_state;
225 	u32 l;
226 
227 	l = cs->chctrl0;
228 	if (enable)
229 		l |= OMAP2_MCSPI_CHCTRL_EN;
230 	else
231 		l &= ~OMAP2_MCSPI_CHCTRL_EN;
232 	cs->chctrl0 = l;
233 	mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCTRL0, cs->chctrl0);
234 	/* Flash post-writes */
235 	mcspi_read_cs_reg(spi, OMAP2_MCSPI_CHCTRL0);
236 }
237 
238 static void omap2_mcspi_set_cs(struct spi_device *spi, bool enable)
239 {
240 	struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
241 	u32 l;
242 
243 	/* The controller handles the inverted chip selects
244 	 * using the OMAP2_MCSPI_CHCONF_EPOL bit so revert
245 	 * the inversion from the core spi_set_cs function.
246 	 */
247 	if (spi->mode & SPI_CS_HIGH)
248 		enable = !enable;
249 
250 	if (spi->controller_state) {
251 		int err = pm_runtime_get_sync(mcspi->dev);
252 		if (err < 0) {
253 			pm_runtime_put_noidle(mcspi->dev);
254 			dev_err(mcspi->dev, "failed to get sync: %d\n", err);
255 			return;
256 		}
257 
258 		l = mcspi_cached_chconf0(spi);
259 
260 		if (enable)
261 			l &= ~OMAP2_MCSPI_CHCONF_FORCE;
262 		else
263 			l |= OMAP2_MCSPI_CHCONF_FORCE;
264 
265 		mcspi_write_chconf0(spi, l);
266 
267 		pm_runtime_mark_last_busy(mcspi->dev);
268 		pm_runtime_put_autosuspend(mcspi->dev);
269 	}
270 }
271 
272 static void omap2_mcspi_set_mode(struct spi_master *master)
273 {
274 	struct omap2_mcspi	*mcspi = spi_master_get_devdata(master);
275 	struct omap2_mcspi_regs	*ctx = &mcspi->ctx;
276 	u32 l;
277 
278 	/*
279 	 * Choose master or slave mode
280 	 */
281 	l = mcspi_read_reg(master, OMAP2_MCSPI_MODULCTRL);
282 	l &= ~(OMAP2_MCSPI_MODULCTRL_STEST);
283 	if (spi_controller_is_slave(master)) {
284 		l |= (OMAP2_MCSPI_MODULCTRL_MS);
285 	} else {
286 		l &= ~(OMAP2_MCSPI_MODULCTRL_MS);
287 		l |= OMAP2_MCSPI_MODULCTRL_SINGLE;
288 	}
289 	mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, l);
290 
291 	ctx->modulctrl = l;
292 }
293 
294 static void omap2_mcspi_set_fifo(const struct spi_device *spi,
295 				struct spi_transfer *t, int enable)
296 {
297 	struct spi_master *master = spi->master;
298 	struct omap2_mcspi_cs *cs = spi->controller_state;
299 	struct omap2_mcspi *mcspi;
300 	unsigned int wcnt;
301 	int max_fifo_depth, bytes_per_word;
302 	u32 chconf, xferlevel;
303 
304 	mcspi = spi_master_get_devdata(master);
305 
306 	chconf = mcspi_cached_chconf0(spi);
307 	if (enable) {
308 		bytes_per_word = mcspi_bytes_per_word(cs->word_len);
309 		if (t->len % bytes_per_word != 0)
310 			goto disable_fifo;
311 
312 		if (t->rx_buf != NULL && t->tx_buf != NULL)
313 			max_fifo_depth = OMAP2_MCSPI_MAX_FIFODEPTH / 2;
314 		else
315 			max_fifo_depth = OMAP2_MCSPI_MAX_FIFODEPTH;
316 
317 		wcnt = t->len / bytes_per_word;
318 		if (wcnt > OMAP2_MCSPI_MAX_FIFOWCNT)
319 			goto disable_fifo;
320 
321 		xferlevel = wcnt << 16;
322 		if (t->rx_buf != NULL) {
323 			chconf |= OMAP2_MCSPI_CHCONF_FFER;
324 			xferlevel |= (bytes_per_word - 1) << 8;
325 		}
326 
327 		if (t->tx_buf != NULL) {
328 			chconf |= OMAP2_MCSPI_CHCONF_FFET;
329 			xferlevel |= bytes_per_word - 1;
330 		}
331 
332 		mcspi_write_reg(master, OMAP2_MCSPI_XFERLEVEL, xferlevel);
333 		mcspi_write_chconf0(spi, chconf);
334 		mcspi->fifo_depth = max_fifo_depth;
335 
336 		return;
337 	}
338 
339 disable_fifo:
340 	if (t->rx_buf != NULL)
341 		chconf &= ~OMAP2_MCSPI_CHCONF_FFER;
342 
343 	if (t->tx_buf != NULL)
344 		chconf &= ~OMAP2_MCSPI_CHCONF_FFET;
345 
346 	mcspi_write_chconf0(spi, chconf);
347 	mcspi->fifo_depth = 0;
348 }
349 
350 static int mcspi_wait_for_reg_bit(void __iomem *reg, unsigned long bit)
351 {
352 	u32 val;
353 
354 	return readl_poll_timeout(reg, val, val & bit, 1, MSEC_PER_SEC);
355 }
356 
357 static int mcspi_wait_for_completion(struct  omap2_mcspi *mcspi,
358 				     struct completion *x)
359 {
360 	if (spi_controller_is_slave(mcspi->master)) {
361 		if (wait_for_completion_interruptible(x) ||
362 		    mcspi->slave_aborted)
363 			return -EINTR;
364 	} else {
365 		wait_for_completion(x);
366 	}
367 
368 	return 0;
369 }
370 
371 static void omap2_mcspi_rx_callback(void *data)
372 {
373 	struct spi_device *spi = data;
374 	struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
375 	struct omap2_mcspi_dma *mcspi_dma = &mcspi->dma_channels[spi->chip_select];
376 
377 	/* We must disable the DMA RX request */
378 	omap2_mcspi_set_dma_req(spi, 1, 0);
379 
380 	complete(&mcspi_dma->dma_rx_completion);
381 }
382 
383 static void omap2_mcspi_tx_callback(void *data)
384 {
385 	struct spi_device *spi = data;
386 	struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
387 	struct omap2_mcspi_dma *mcspi_dma = &mcspi->dma_channels[spi->chip_select];
388 
389 	/* We must disable the DMA TX request */
390 	omap2_mcspi_set_dma_req(spi, 0, 0);
391 
392 	complete(&mcspi_dma->dma_tx_completion);
393 }
394 
395 static void omap2_mcspi_tx_dma(struct spi_device *spi,
396 				struct spi_transfer *xfer,
397 				struct dma_slave_config cfg)
398 {
399 	struct omap2_mcspi	*mcspi;
400 	struct omap2_mcspi_dma  *mcspi_dma;
401 	struct dma_async_tx_descriptor *tx;
402 
403 	mcspi = spi_master_get_devdata(spi->master);
404 	mcspi_dma = &mcspi->dma_channels[spi->chip_select];
405 
406 	dmaengine_slave_config(mcspi_dma->dma_tx, &cfg);
407 
408 	tx = dmaengine_prep_slave_sg(mcspi_dma->dma_tx, xfer->tx_sg.sgl,
409 				     xfer->tx_sg.nents,
410 				     DMA_MEM_TO_DEV,
411 				     DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
412 	if (tx) {
413 		tx->callback = omap2_mcspi_tx_callback;
414 		tx->callback_param = spi;
415 		dmaengine_submit(tx);
416 	} else {
417 		/* FIXME: fall back to PIO? */
418 	}
419 	dma_async_issue_pending(mcspi_dma->dma_tx);
420 	omap2_mcspi_set_dma_req(spi, 0, 1);
421 }
422 
423 static unsigned
424 omap2_mcspi_rx_dma(struct spi_device *spi, struct spi_transfer *xfer,
425 				struct dma_slave_config cfg,
426 				unsigned es)
427 {
428 	struct omap2_mcspi	*mcspi;
429 	struct omap2_mcspi_dma  *mcspi_dma;
430 	unsigned int		count, transfer_reduction = 0;
431 	struct scatterlist	*sg_out[2];
432 	int			nb_sizes = 0, out_mapped_nents[2], ret, x;
433 	size_t			sizes[2];
434 	u32			l;
435 	int			elements = 0;
436 	int			word_len, element_count;
437 	struct omap2_mcspi_cs	*cs = spi->controller_state;
438 	void __iomem		*chstat_reg = cs->base + OMAP2_MCSPI_CHSTAT0;
439 	struct dma_async_tx_descriptor *tx;
440 
441 	mcspi = spi_master_get_devdata(spi->master);
442 	mcspi_dma = &mcspi->dma_channels[spi->chip_select];
443 	count = xfer->len;
444 
445 	/*
446 	 *  In the "End-of-Transfer Procedure" section for DMA RX in OMAP35x TRM
447 	 *  it mentions reducing DMA transfer length by one element in master
448 	 *  normal mode.
449 	 */
450 	if (mcspi->fifo_depth == 0)
451 		transfer_reduction = es;
452 
453 	word_len = cs->word_len;
454 	l = mcspi_cached_chconf0(spi);
455 
456 	if (word_len <= 8)
457 		element_count = count;
458 	else if (word_len <= 16)
459 		element_count = count >> 1;
460 	else /* word_len <= 32 */
461 		element_count = count >> 2;
462 
463 
464 	dmaengine_slave_config(mcspi_dma->dma_rx, &cfg);
465 
466 	/*
467 	 *  Reduce DMA transfer length by one more if McSPI is
468 	 *  configured in turbo mode.
469 	 */
470 	if ((l & OMAP2_MCSPI_CHCONF_TURBO) && mcspi->fifo_depth == 0)
471 		transfer_reduction += es;
472 
473 	if (transfer_reduction) {
474 		/* Split sgl into two. The second sgl won't be used. */
475 		sizes[0] = count - transfer_reduction;
476 		sizes[1] = transfer_reduction;
477 		nb_sizes = 2;
478 	} else {
479 		/*
480 		 * Don't bother splitting the sgl. This essentially
481 		 * clones the original sgl.
482 		 */
483 		sizes[0] = count;
484 		nb_sizes = 1;
485 	}
486 
487 	ret = sg_split(xfer->rx_sg.sgl, xfer->rx_sg.nents, 0, nb_sizes,
488 		       sizes, sg_out, out_mapped_nents, GFP_KERNEL);
489 
490 	if (ret < 0) {
491 		dev_err(&spi->dev, "sg_split failed\n");
492 		return 0;
493 	}
494 
495 	tx = dmaengine_prep_slave_sg(mcspi_dma->dma_rx, sg_out[0],
496 				     out_mapped_nents[0], DMA_DEV_TO_MEM,
497 				     DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
498 	if (tx) {
499 		tx->callback = omap2_mcspi_rx_callback;
500 		tx->callback_param = spi;
501 		dmaengine_submit(tx);
502 	} else {
503 		/* FIXME: fall back to PIO? */
504 	}
505 
506 	dma_async_issue_pending(mcspi_dma->dma_rx);
507 	omap2_mcspi_set_dma_req(spi, 1, 1);
508 
509 	ret = mcspi_wait_for_completion(mcspi, &mcspi_dma->dma_rx_completion);
510 	if (ret || mcspi->slave_aborted) {
511 		dmaengine_terminate_sync(mcspi_dma->dma_rx);
512 		omap2_mcspi_set_dma_req(spi, 1, 0);
513 		return 0;
514 	}
515 
516 	for (x = 0; x < nb_sizes; x++)
517 		kfree(sg_out[x]);
518 
519 	if (mcspi->fifo_depth > 0)
520 		return count;
521 
522 	/*
523 	 *  Due to the DMA transfer length reduction the missing bytes must
524 	 *  be read manually to receive all of the expected data.
525 	 */
526 	omap2_mcspi_set_enable(spi, 0);
527 
528 	elements = element_count - 1;
529 
530 	if (l & OMAP2_MCSPI_CHCONF_TURBO) {
531 		elements--;
532 
533 		if (!mcspi_wait_for_reg_bit(chstat_reg,
534 					    OMAP2_MCSPI_CHSTAT_RXS)) {
535 			u32 w;
536 
537 			w = mcspi_read_cs_reg(spi, OMAP2_MCSPI_RX0);
538 			if (word_len <= 8)
539 				((u8 *)xfer->rx_buf)[elements++] = w;
540 			else if (word_len <= 16)
541 				((u16 *)xfer->rx_buf)[elements++] = w;
542 			else /* word_len <= 32 */
543 				((u32 *)xfer->rx_buf)[elements++] = w;
544 		} else {
545 			int bytes_per_word = mcspi_bytes_per_word(word_len);
546 			dev_err(&spi->dev, "DMA RX penultimate word empty\n");
547 			count -= (bytes_per_word << 1);
548 			omap2_mcspi_set_enable(spi, 1);
549 			return count;
550 		}
551 	}
552 	if (!mcspi_wait_for_reg_bit(chstat_reg, OMAP2_MCSPI_CHSTAT_RXS)) {
553 		u32 w;
554 
555 		w = mcspi_read_cs_reg(spi, OMAP2_MCSPI_RX0);
556 		if (word_len <= 8)
557 			((u8 *)xfer->rx_buf)[elements] = w;
558 		else if (word_len <= 16)
559 			((u16 *)xfer->rx_buf)[elements] = w;
560 		else /* word_len <= 32 */
561 			((u32 *)xfer->rx_buf)[elements] = w;
562 	} else {
563 		dev_err(&spi->dev, "DMA RX last word empty\n");
564 		count -= mcspi_bytes_per_word(word_len);
565 	}
566 	omap2_mcspi_set_enable(spi, 1);
567 	return count;
568 }
569 
570 static unsigned
571 omap2_mcspi_txrx_dma(struct spi_device *spi, struct spi_transfer *xfer)
572 {
573 	struct omap2_mcspi	*mcspi;
574 	struct omap2_mcspi_cs	*cs = spi->controller_state;
575 	struct omap2_mcspi_dma  *mcspi_dma;
576 	unsigned int		count;
577 	u8			*rx;
578 	const u8		*tx;
579 	struct dma_slave_config	cfg;
580 	enum dma_slave_buswidth width;
581 	unsigned es;
582 	void __iomem		*chstat_reg;
583 	void __iomem            *irqstat_reg;
584 	int			wait_res;
585 
586 	mcspi = spi_master_get_devdata(spi->master);
587 	mcspi_dma = &mcspi->dma_channels[spi->chip_select];
588 
589 	if (cs->word_len <= 8) {
590 		width = DMA_SLAVE_BUSWIDTH_1_BYTE;
591 		es = 1;
592 	} else if (cs->word_len <= 16) {
593 		width = DMA_SLAVE_BUSWIDTH_2_BYTES;
594 		es = 2;
595 	} else {
596 		width = DMA_SLAVE_BUSWIDTH_4_BYTES;
597 		es = 4;
598 	}
599 
600 	count = xfer->len;
601 
602 	memset(&cfg, 0, sizeof(cfg));
603 	cfg.src_addr = cs->phys + OMAP2_MCSPI_RX0;
604 	cfg.dst_addr = cs->phys + OMAP2_MCSPI_TX0;
605 	cfg.src_addr_width = width;
606 	cfg.dst_addr_width = width;
607 	cfg.src_maxburst = 1;
608 	cfg.dst_maxburst = 1;
609 
610 	rx = xfer->rx_buf;
611 	tx = xfer->tx_buf;
612 
613 	mcspi->slave_aborted = false;
614 	reinit_completion(&mcspi_dma->dma_tx_completion);
615 	reinit_completion(&mcspi_dma->dma_rx_completion);
616 	reinit_completion(&mcspi->txdone);
617 	if (tx) {
618 		/* Enable EOW IRQ to know end of tx in slave mode */
619 		if (spi_controller_is_slave(spi->master))
620 			mcspi_write_reg(spi->master,
621 					OMAP2_MCSPI_IRQENABLE,
622 					OMAP2_MCSPI_IRQSTATUS_EOW);
623 		omap2_mcspi_tx_dma(spi, xfer, cfg);
624 	}
625 
626 	if (rx != NULL)
627 		count = omap2_mcspi_rx_dma(spi, xfer, cfg, es);
628 
629 	if (tx != NULL) {
630 		int ret;
631 
632 		ret = mcspi_wait_for_completion(mcspi, &mcspi_dma->dma_tx_completion);
633 		if (ret || mcspi->slave_aborted) {
634 			dmaengine_terminate_sync(mcspi_dma->dma_tx);
635 			omap2_mcspi_set_dma_req(spi, 0, 0);
636 			return 0;
637 		}
638 
639 		if (spi_controller_is_slave(mcspi->master)) {
640 			ret = mcspi_wait_for_completion(mcspi, &mcspi->txdone);
641 			if (ret || mcspi->slave_aborted)
642 				return 0;
643 		}
644 
645 		if (mcspi->fifo_depth > 0) {
646 			irqstat_reg = mcspi->base + OMAP2_MCSPI_IRQSTATUS;
647 
648 			if (mcspi_wait_for_reg_bit(irqstat_reg,
649 						OMAP2_MCSPI_IRQSTATUS_EOW) < 0)
650 				dev_err(&spi->dev, "EOW timed out\n");
651 
652 			mcspi_write_reg(mcspi->master, OMAP2_MCSPI_IRQSTATUS,
653 					OMAP2_MCSPI_IRQSTATUS_EOW);
654 		}
655 
656 		/* for TX_ONLY mode, be sure all words have shifted out */
657 		if (rx == NULL) {
658 			chstat_reg = cs->base + OMAP2_MCSPI_CHSTAT0;
659 			if (mcspi->fifo_depth > 0) {
660 				wait_res = mcspi_wait_for_reg_bit(chstat_reg,
661 						OMAP2_MCSPI_CHSTAT_TXFFE);
662 				if (wait_res < 0)
663 					dev_err(&spi->dev, "TXFFE timed out\n");
664 			} else {
665 				wait_res = mcspi_wait_for_reg_bit(chstat_reg,
666 						OMAP2_MCSPI_CHSTAT_TXS);
667 				if (wait_res < 0)
668 					dev_err(&spi->dev, "TXS timed out\n");
669 			}
670 			if (wait_res >= 0 &&
671 				(mcspi_wait_for_reg_bit(chstat_reg,
672 					OMAP2_MCSPI_CHSTAT_EOT) < 0))
673 				dev_err(&spi->dev, "EOT timed out\n");
674 		}
675 	}
676 	return count;
677 }
678 
679 static unsigned
680 omap2_mcspi_txrx_pio(struct spi_device *spi, struct spi_transfer *xfer)
681 {
682 	struct omap2_mcspi_cs	*cs = spi->controller_state;
683 	unsigned int		count, c;
684 	u32			l;
685 	void __iomem		*base = cs->base;
686 	void __iomem		*tx_reg;
687 	void __iomem		*rx_reg;
688 	void __iomem		*chstat_reg;
689 	int			word_len;
690 
691 	count = xfer->len;
692 	c = count;
693 	word_len = cs->word_len;
694 
695 	l = mcspi_cached_chconf0(spi);
696 
697 	/* We store the pre-calculated register addresses on stack to speed
698 	 * up the transfer loop. */
699 	tx_reg		= base + OMAP2_MCSPI_TX0;
700 	rx_reg		= base + OMAP2_MCSPI_RX0;
701 	chstat_reg	= base + OMAP2_MCSPI_CHSTAT0;
702 
703 	if (c < (word_len>>3))
704 		return 0;
705 
706 	if (word_len <= 8) {
707 		u8		*rx;
708 		const u8	*tx;
709 
710 		rx = xfer->rx_buf;
711 		tx = xfer->tx_buf;
712 
713 		do {
714 			c -= 1;
715 			if (tx != NULL) {
716 				if (mcspi_wait_for_reg_bit(chstat_reg,
717 						OMAP2_MCSPI_CHSTAT_TXS) < 0) {
718 					dev_err(&spi->dev, "TXS timed out\n");
719 					goto out;
720 				}
721 				dev_vdbg(&spi->dev, "write-%d %02x\n",
722 						word_len, *tx);
723 				writel_relaxed(*tx++, tx_reg);
724 			}
725 			if (rx != NULL) {
726 				if (mcspi_wait_for_reg_bit(chstat_reg,
727 						OMAP2_MCSPI_CHSTAT_RXS) < 0) {
728 					dev_err(&spi->dev, "RXS timed out\n");
729 					goto out;
730 				}
731 
732 				if (c == 1 && tx == NULL &&
733 				    (l & OMAP2_MCSPI_CHCONF_TURBO)) {
734 					omap2_mcspi_set_enable(spi, 0);
735 					*rx++ = readl_relaxed(rx_reg);
736 					dev_vdbg(&spi->dev, "read-%d %02x\n",
737 						    word_len, *(rx - 1));
738 					if (mcspi_wait_for_reg_bit(chstat_reg,
739 						OMAP2_MCSPI_CHSTAT_RXS) < 0) {
740 						dev_err(&spi->dev,
741 							"RXS timed out\n");
742 						goto out;
743 					}
744 					c = 0;
745 				} else if (c == 0 && tx == NULL) {
746 					omap2_mcspi_set_enable(spi, 0);
747 				}
748 
749 				*rx++ = readl_relaxed(rx_reg);
750 				dev_vdbg(&spi->dev, "read-%d %02x\n",
751 						word_len, *(rx - 1));
752 			}
753 		} while (c);
754 	} else if (word_len <= 16) {
755 		u16		*rx;
756 		const u16	*tx;
757 
758 		rx = xfer->rx_buf;
759 		tx = xfer->tx_buf;
760 		do {
761 			c -= 2;
762 			if (tx != NULL) {
763 				if (mcspi_wait_for_reg_bit(chstat_reg,
764 						OMAP2_MCSPI_CHSTAT_TXS) < 0) {
765 					dev_err(&spi->dev, "TXS timed out\n");
766 					goto out;
767 				}
768 				dev_vdbg(&spi->dev, "write-%d %04x\n",
769 						word_len, *tx);
770 				writel_relaxed(*tx++, tx_reg);
771 			}
772 			if (rx != NULL) {
773 				if (mcspi_wait_for_reg_bit(chstat_reg,
774 						OMAP2_MCSPI_CHSTAT_RXS) < 0) {
775 					dev_err(&spi->dev, "RXS timed out\n");
776 					goto out;
777 				}
778 
779 				if (c == 2 && tx == NULL &&
780 				    (l & OMAP2_MCSPI_CHCONF_TURBO)) {
781 					omap2_mcspi_set_enable(spi, 0);
782 					*rx++ = readl_relaxed(rx_reg);
783 					dev_vdbg(&spi->dev, "read-%d %04x\n",
784 						    word_len, *(rx - 1));
785 					if (mcspi_wait_for_reg_bit(chstat_reg,
786 						OMAP2_MCSPI_CHSTAT_RXS) < 0) {
787 						dev_err(&spi->dev,
788 							"RXS timed out\n");
789 						goto out;
790 					}
791 					c = 0;
792 				} else if (c == 0 && tx == NULL) {
793 					omap2_mcspi_set_enable(spi, 0);
794 				}
795 
796 				*rx++ = readl_relaxed(rx_reg);
797 				dev_vdbg(&spi->dev, "read-%d %04x\n",
798 						word_len, *(rx - 1));
799 			}
800 		} while (c >= 2);
801 	} else if (word_len <= 32) {
802 		u32		*rx;
803 		const u32	*tx;
804 
805 		rx = xfer->rx_buf;
806 		tx = xfer->tx_buf;
807 		do {
808 			c -= 4;
809 			if (tx != NULL) {
810 				if (mcspi_wait_for_reg_bit(chstat_reg,
811 						OMAP2_MCSPI_CHSTAT_TXS) < 0) {
812 					dev_err(&spi->dev, "TXS timed out\n");
813 					goto out;
814 				}
815 				dev_vdbg(&spi->dev, "write-%d %08x\n",
816 						word_len, *tx);
817 				writel_relaxed(*tx++, tx_reg);
818 			}
819 			if (rx != NULL) {
820 				if (mcspi_wait_for_reg_bit(chstat_reg,
821 						OMAP2_MCSPI_CHSTAT_RXS) < 0) {
822 					dev_err(&spi->dev, "RXS timed out\n");
823 					goto out;
824 				}
825 
826 				if (c == 4 && tx == NULL &&
827 				    (l & OMAP2_MCSPI_CHCONF_TURBO)) {
828 					omap2_mcspi_set_enable(spi, 0);
829 					*rx++ = readl_relaxed(rx_reg);
830 					dev_vdbg(&spi->dev, "read-%d %08x\n",
831 						    word_len, *(rx - 1));
832 					if (mcspi_wait_for_reg_bit(chstat_reg,
833 						OMAP2_MCSPI_CHSTAT_RXS) < 0) {
834 						dev_err(&spi->dev,
835 							"RXS timed out\n");
836 						goto out;
837 					}
838 					c = 0;
839 				} else if (c == 0 && tx == NULL) {
840 					omap2_mcspi_set_enable(spi, 0);
841 				}
842 
843 				*rx++ = readl_relaxed(rx_reg);
844 				dev_vdbg(&spi->dev, "read-%d %08x\n",
845 						word_len, *(rx - 1));
846 			}
847 		} while (c >= 4);
848 	}
849 
850 	/* for TX_ONLY mode, be sure all words have shifted out */
851 	if (xfer->rx_buf == NULL) {
852 		if (mcspi_wait_for_reg_bit(chstat_reg,
853 				OMAP2_MCSPI_CHSTAT_TXS) < 0) {
854 			dev_err(&spi->dev, "TXS timed out\n");
855 		} else if (mcspi_wait_for_reg_bit(chstat_reg,
856 				OMAP2_MCSPI_CHSTAT_EOT) < 0)
857 			dev_err(&spi->dev, "EOT timed out\n");
858 
859 		/* disable chan to purge rx datas received in TX_ONLY transfer,
860 		 * otherwise these rx datas will affect the direct following
861 		 * RX_ONLY transfer.
862 		 */
863 		omap2_mcspi_set_enable(spi, 0);
864 	}
865 out:
866 	omap2_mcspi_set_enable(spi, 1);
867 	return count - c;
868 }
869 
870 static u32 omap2_mcspi_calc_divisor(u32 speed_hz)
871 {
872 	u32 div;
873 
874 	for (div = 0; div < 15; div++)
875 		if (speed_hz >= (OMAP2_MCSPI_MAX_FREQ >> div))
876 			return div;
877 
878 	return 15;
879 }
880 
881 /* called only when no transfer is active to this device */
882 static int omap2_mcspi_setup_transfer(struct spi_device *spi,
883 		struct spi_transfer *t)
884 {
885 	struct omap2_mcspi_cs *cs = spi->controller_state;
886 	struct omap2_mcspi *mcspi;
887 	u32 l = 0, clkd = 0, div, extclk = 0, clkg = 0;
888 	u8 word_len = spi->bits_per_word;
889 	u32 speed_hz = spi->max_speed_hz;
890 
891 	mcspi = spi_master_get_devdata(spi->master);
892 
893 	if (t != NULL && t->bits_per_word)
894 		word_len = t->bits_per_word;
895 
896 	cs->word_len = word_len;
897 
898 	if (t && t->speed_hz)
899 		speed_hz = t->speed_hz;
900 
901 	speed_hz = min_t(u32, speed_hz, OMAP2_MCSPI_MAX_FREQ);
902 	if (speed_hz < (OMAP2_MCSPI_MAX_FREQ / OMAP2_MCSPI_MAX_DIVIDER)) {
903 		clkd = omap2_mcspi_calc_divisor(speed_hz);
904 		speed_hz = OMAP2_MCSPI_MAX_FREQ >> clkd;
905 		clkg = 0;
906 	} else {
907 		div = (OMAP2_MCSPI_MAX_FREQ + speed_hz - 1) / speed_hz;
908 		speed_hz = OMAP2_MCSPI_MAX_FREQ / div;
909 		clkd = (div - 1) & 0xf;
910 		extclk = (div - 1) >> 4;
911 		clkg = OMAP2_MCSPI_CHCONF_CLKG;
912 	}
913 
914 	l = mcspi_cached_chconf0(spi);
915 
916 	/* standard 4-wire master mode:  SCK, MOSI/out, MISO/in, nCS
917 	 * REVISIT: this controller could support SPI_3WIRE mode.
918 	 */
919 	if (mcspi->pin_dir == MCSPI_PINDIR_D0_IN_D1_OUT) {
920 		l &= ~OMAP2_MCSPI_CHCONF_IS;
921 		l &= ~OMAP2_MCSPI_CHCONF_DPE1;
922 		l |= OMAP2_MCSPI_CHCONF_DPE0;
923 	} else {
924 		l |= OMAP2_MCSPI_CHCONF_IS;
925 		l |= OMAP2_MCSPI_CHCONF_DPE1;
926 		l &= ~OMAP2_MCSPI_CHCONF_DPE0;
927 	}
928 
929 	/* wordlength */
930 	l &= ~OMAP2_MCSPI_CHCONF_WL_MASK;
931 	l |= (word_len - 1) << 7;
932 
933 	/* set chipselect polarity; manage with FORCE */
934 	if (!(spi->mode & SPI_CS_HIGH))
935 		l |= OMAP2_MCSPI_CHCONF_EPOL;	/* active-low; normal */
936 	else
937 		l &= ~OMAP2_MCSPI_CHCONF_EPOL;
938 
939 	/* set clock divisor */
940 	l &= ~OMAP2_MCSPI_CHCONF_CLKD_MASK;
941 	l |= clkd << 2;
942 
943 	/* set clock granularity */
944 	l &= ~OMAP2_MCSPI_CHCONF_CLKG;
945 	l |= clkg;
946 	if (clkg) {
947 		cs->chctrl0 &= ~OMAP2_MCSPI_CHCTRL_EXTCLK_MASK;
948 		cs->chctrl0 |= extclk << 8;
949 		mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCTRL0, cs->chctrl0);
950 	}
951 
952 	/* set SPI mode 0..3 */
953 	if (spi->mode & SPI_CPOL)
954 		l |= OMAP2_MCSPI_CHCONF_POL;
955 	else
956 		l &= ~OMAP2_MCSPI_CHCONF_POL;
957 	if (spi->mode & SPI_CPHA)
958 		l |= OMAP2_MCSPI_CHCONF_PHA;
959 	else
960 		l &= ~OMAP2_MCSPI_CHCONF_PHA;
961 
962 	mcspi_write_chconf0(spi, l);
963 
964 	cs->mode = spi->mode;
965 
966 	dev_dbg(&spi->dev, "setup: speed %d, sample %s edge, clk %s\n",
967 			speed_hz,
968 			(spi->mode & SPI_CPHA) ? "trailing" : "leading",
969 			(spi->mode & SPI_CPOL) ? "inverted" : "normal");
970 
971 	return 0;
972 }
973 
974 /*
975  * Note that we currently allow DMA only if we get a channel
976  * for both rx and tx. Otherwise we'll do PIO for both rx and tx.
977  */
978 static int omap2_mcspi_request_dma(struct omap2_mcspi *mcspi,
979 				   struct omap2_mcspi_dma *mcspi_dma)
980 {
981 	int ret = 0;
982 
983 	mcspi_dma->dma_rx = dma_request_chan(mcspi->dev,
984 					     mcspi_dma->dma_rx_ch_name);
985 	if (IS_ERR(mcspi_dma->dma_rx)) {
986 		ret = PTR_ERR(mcspi_dma->dma_rx);
987 		mcspi_dma->dma_rx = NULL;
988 		goto no_dma;
989 	}
990 
991 	mcspi_dma->dma_tx = dma_request_chan(mcspi->dev,
992 					     mcspi_dma->dma_tx_ch_name);
993 	if (IS_ERR(mcspi_dma->dma_tx)) {
994 		ret = PTR_ERR(mcspi_dma->dma_tx);
995 		mcspi_dma->dma_tx = NULL;
996 		dma_release_channel(mcspi_dma->dma_rx);
997 		mcspi_dma->dma_rx = NULL;
998 	}
999 
1000 	init_completion(&mcspi_dma->dma_rx_completion);
1001 	init_completion(&mcspi_dma->dma_tx_completion);
1002 
1003 no_dma:
1004 	return ret;
1005 }
1006 
1007 static void omap2_mcspi_release_dma(struct spi_master *master)
1008 {
1009 	struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1010 	struct omap2_mcspi_dma	*mcspi_dma;
1011 	int i;
1012 
1013 	for (i = 0; i < master->num_chipselect; i++) {
1014 		mcspi_dma = &mcspi->dma_channels[i];
1015 
1016 		if (mcspi_dma->dma_rx) {
1017 			dma_release_channel(mcspi_dma->dma_rx);
1018 			mcspi_dma->dma_rx = NULL;
1019 		}
1020 		if (mcspi_dma->dma_tx) {
1021 			dma_release_channel(mcspi_dma->dma_tx);
1022 			mcspi_dma->dma_tx = NULL;
1023 		}
1024 	}
1025 }
1026 
1027 static int omap2_mcspi_setup(struct spi_device *spi)
1028 {
1029 	int			ret;
1030 	struct omap2_mcspi	*mcspi = spi_master_get_devdata(spi->master);
1031 	struct omap2_mcspi_regs	*ctx = &mcspi->ctx;
1032 	struct omap2_mcspi_cs	*cs = spi->controller_state;
1033 
1034 	if (!cs) {
1035 		cs = kzalloc(sizeof *cs, GFP_KERNEL);
1036 		if (!cs)
1037 			return -ENOMEM;
1038 		cs->base = mcspi->base + spi->chip_select * 0x14;
1039 		cs->phys = mcspi->phys + spi->chip_select * 0x14;
1040 		cs->mode = 0;
1041 		cs->chconf0 = 0;
1042 		cs->chctrl0 = 0;
1043 		spi->controller_state = cs;
1044 		/* Link this to context save list */
1045 		list_add_tail(&cs->node, &ctx->cs);
1046 
1047 		if (gpio_is_valid(spi->cs_gpio)) {
1048 			ret = gpio_request(spi->cs_gpio, dev_name(&spi->dev));
1049 			if (ret) {
1050 				dev_err(&spi->dev, "failed to request gpio\n");
1051 				return ret;
1052 			}
1053 			gpio_direction_output(spi->cs_gpio,
1054 					 !(spi->mode & SPI_CS_HIGH));
1055 		}
1056 	}
1057 
1058 	ret = pm_runtime_get_sync(mcspi->dev);
1059 	if (ret < 0) {
1060 		pm_runtime_put_noidle(mcspi->dev);
1061 
1062 		return ret;
1063 	}
1064 
1065 	ret = omap2_mcspi_setup_transfer(spi, NULL);
1066 	pm_runtime_mark_last_busy(mcspi->dev);
1067 	pm_runtime_put_autosuspend(mcspi->dev);
1068 
1069 	return ret;
1070 }
1071 
1072 static void omap2_mcspi_cleanup(struct spi_device *spi)
1073 {
1074 	struct omap2_mcspi_cs	*cs;
1075 
1076 	if (spi->controller_state) {
1077 		/* Unlink controller state from context save list */
1078 		cs = spi->controller_state;
1079 		list_del(&cs->node);
1080 
1081 		kfree(cs);
1082 	}
1083 
1084 	if (gpio_is_valid(spi->cs_gpio))
1085 		gpio_free(spi->cs_gpio);
1086 }
1087 
1088 static irqreturn_t omap2_mcspi_irq_handler(int irq, void *data)
1089 {
1090 	struct omap2_mcspi *mcspi = data;
1091 	u32 irqstat;
1092 
1093 	irqstat	= mcspi_read_reg(mcspi->master, OMAP2_MCSPI_IRQSTATUS);
1094 	if (!irqstat)
1095 		return IRQ_NONE;
1096 
1097 	/* Disable IRQ and wakeup slave xfer task */
1098 	mcspi_write_reg(mcspi->master, OMAP2_MCSPI_IRQENABLE, 0);
1099 	if (irqstat & OMAP2_MCSPI_IRQSTATUS_EOW)
1100 		complete(&mcspi->txdone);
1101 
1102 	return IRQ_HANDLED;
1103 }
1104 
1105 static int omap2_mcspi_slave_abort(struct spi_master *master)
1106 {
1107 	struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1108 	struct omap2_mcspi_dma *mcspi_dma = mcspi->dma_channels;
1109 
1110 	mcspi->slave_aborted = true;
1111 	complete(&mcspi_dma->dma_rx_completion);
1112 	complete(&mcspi_dma->dma_tx_completion);
1113 	complete(&mcspi->txdone);
1114 
1115 	return 0;
1116 }
1117 
1118 static int omap2_mcspi_transfer_one(struct spi_master *master,
1119 				    struct spi_device *spi,
1120 				    struct spi_transfer *t)
1121 {
1122 
1123 	/* We only enable one channel at a time -- the one whose message is
1124 	 * -- although this controller would gladly
1125 	 * arbitrate among multiple channels.  This corresponds to "single
1126 	 * channel" master mode.  As a side effect, we need to manage the
1127 	 * chipselect with the FORCE bit ... CS != channel enable.
1128 	 */
1129 
1130 	struct omap2_mcspi		*mcspi;
1131 	struct omap2_mcspi_dma		*mcspi_dma;
1132 	struct omap2_mcspi_cs		*cs;
1133 	struct omap2_mcspi_device_config *cd;
1134 	int				par_override = 0;
1135 	int				status = 0;
1136 	u32				chconf;
1137 
1138 	mcspi = spi_master_get_devdata(master);
1139 	mcspi_dma = mcspi->dma_channels + spi->chip_select;
1140 	cs = spi->controller_state;
1141 	cd = spi->controller_data;
1142 
1143 	/*
1144 	 * The slave driver could have changed spi->mode in which case
1145 	 * it will be different from cs->mode (the current hardware setup).
1146 	 * If so, set par_override (even though its not a parity issue) so
1147 	 * omap2_mcspi_setup_transfer will be called to configure the hardware
1148 	 * with the correct mode on the first iteration of the loop below.
1149 	 */
1150 	if (spi->mode != cs->mode)
1151 		par_override = 1;
1152 
1153 	omap2_mcspi_set_enable(spi, 0);
1154 
1155 	if (gpio_is_valid(spi->cs_gpio))
1156 		omap2_mcspi_set_cs(spi, spi->mode & SPI_CS_HIGH);
1157 
1158 	if (par_override ||
1159 	    (t->speed_hz != spi->max_speed_hz) ||
1160 	    (t->bits_per_word != spi->bits_per_word)) {
1161 		par_override = 1;
1162 		status = omap2_mcspi_setup_transfer(spi, t);
1163 		if (status < 0)
1164 			goto out;
1165 		if (t->speed_hz == spi->max_speed_hz &&
1166 		    t->bits_per_word == spi->bits_per_word)
1167 			par_override = 0;
1168 	}
1169 	if (cd && cd->cs_per_word) {
1170 		chconf = mcspi->ctx.modulctrl;
1171 		chconf &= ~OMAP2_MCSPI_MODULCTRL_SINGLE;
1172 		mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, chconf);
1173 		mcspi->ctx.modulctrl =
1174 			mcspi_read_cs_reg(spi, OMAP2_MCSPI_MODULCTRL);
1175 	}
1176 
1177 	chconf = mcspi_cached_chconf0(spi);
1178 	chconf &= ~OMAP2_MCSPI_CHCONF_TRM_MASK;
1179 	chconf &= ~OMAP2_MCSPI_CHCONF_TURBO;
1180 
1181 	if (t->tx_buf == NULL)
1182 		chconf |= OMAP2_MCSPI_CHCONF_TRM_RX_ONLY;
1183 	else if (t->rx_buf == NULL)
1184 		chconf |= OMAP2_MCSPI_CHCONF_TRM_TX_ONLY;
1185 
1186 	if (cd && cd->turbo_mode && t->tx_buf == NULL) {
1187 		/* Turbo mode is for more than one word */
1188 		if (t->len > ((cs->word_len + 7) >> 3))
1189 			chconf |= OMAP2_MCSPI_CHCONF_TURBO;
1190 	}
1191 
1192 	mcspi_write_chconf0(spi, chconf);
1193 
1194 	if (t->len) {
1195 		unsigned	count;
1196 
1197 		if ((mcspi_dma->dma_rx && mcspi_dma->dma_tx) &&
1198 		    master->cur_msg_mapped &&
1199 		    master->can_dma(master, spi, t))
1200 			omap2_mcspi_set_fifo(spi, t, 1);
1201 
1202 		omap2_mcspi_set_enable(spi, 1);
1203 
1204 		/* RX_ONLY mode needs dummy data in TX reg */
1205 		if (t->tx_buf == NULL)
1206 			writel_relaxed(0, cs->base
1207 					+ OMAP2_MCSPI_TX0);
1208 
1209 		if ((mcspi_dma->dma_rx && mcspi_dma->dma_tx) &&
1210 		    master->cur_msg_mapped &&
1211 		    master->can_dma(master, spi, t))
1212 			count = omap2_mcspi_txrx_dma(spi, t);
1213 		else
1214 			count = omap2_mcspi_txrx_pio(spi, t);
1215 
1216 		if (count != t->len) {
1217 			status = -EIO;
1218 			goto out;
1219 		}
1220 	}
1221 
1222 	omap2_mcspi_set_enable(spi, 0);
1223 
1224 	if (mcspi->fifo_depth > 0)
1225 		omap2_mcspi_set_fifo(spi, t, 0);
1226 
1227 out:
1228 	/* Restore defaults if they were overriden */
1229 	if (par_override) {
1230 		par_override = 0;
1231 		status = omap2_mcspi_setup_transfer(spi, NULL);
1232 	}
1233 
1234 	if (cd && cd->cs_per_word) {
1235 		chconf = mcspi->ctx.modulctrl;
1236 		chconf |= OMAP2_MCSPI_MODULCTRL_SINGLE;
1237 		mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, chconf);
1238 		mcspi->ctx.modulctrl =
1239 			mcspi_read_cs_reg(spi, OMAP2_MCSPI_MODULCTRL);
1240 	}
1241 
1242 	omap2_mcspi_set_enable(spi, 0);
1243 
1244 	if (gpio_is_valid(spi->cs_gpio))
1245 		omap2_mcspi_set_cs(spi, !(spi->mode & SPI_CS_HIGH));
1246 
1247 	if (mcspi->fifo_depth > 0 && t)
1248 		omap2_mcspi_set_fifo(spi, t, 0);
1249 
1250 	return status;
1251 }
1252 
1253 static int omap2_mcspi_prepare_message(struct spi_master *master,
1254 				       struct spi_message *msg)
1255 {
1256 	struct omap2_mcspi	*mcspi = spi_master_get_devdata(master);
1257 	struct omap2_mcspi_regs	*ctx = &mcspi->ctx;
1258 	struct omap2_mcspi_cs	*cs;
1259 
1260 	/* Only a single channel can have the FORCE bit enabled
1261 	 * in its chconf0 register.
1262 	 * Scan all channels and disable them except the current one.
1263 	 * A FORCE can remain from a last transfer having cs_change enabled
1264 	 */
1265 	list_for_each_entry(cs, &ctx->cs, node) {
1266 		if (msg->spi->controller_state == cs)
1267 			continue;
1268 
1269 		if ((cs->chconf0 & OMAP2_MCSPI_CHCONF_FORCE)) {
1270 			cs->chconf0 &= ~OMAP2_MCSPI_CHCONF_FORCE;
1271 			writel_relaxed(cs->chconf0,
1272 					cs->base + OMAP2_MCSPI_CHCONF0);
1273 			readl_relaxed(cs->base + OMAP2_MCSPI_CHCONF0);
1274 		}
1275 	}
1276 
1277 	return 0;
1278 }
1279 
1280 static bool omap2_mcspi_can_dma(struct spi_master *master,
1281 				struct spi_device *spi,
1282 				struct spi_transfer *xfer)
1283 {
1284 	struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
1285 	struct omap2_mcspi_dma *mcspi_dma =
1286 		&mcspi->dma_channels[spi->chip_select];
1287 
1288 	if (!mcspi_dma->dma_rx || !mcspi_dma->dma_tx)
1289 		return false;
1290 
1291 	if (spi_controller_is_slave(master))
1292 		return true;
1293 
1294 	master->dma_rx = mcspi_dma->dma_rx;
1295 	master->dma_tx = mcspi_dma->dma_tx;
1296 
1297 	return (xfer->len >= DMA_MIN_BYTES);
1298 }
1299 
1300 static size_t omap2_mcspi_max_xfer_size(struct spi_device *spi)
1301 {
1302 	struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
1303 	struct omap2_mcspi_dma *mcspi_dma =
1304 		&mcspi->dma_channels[spi->chip_select];
1305 
1306 	if (mcspi->max_xfer_len && mcspi_dma->dma_rx)
1307 		return mcspi->max_xfer_len;
1308 
1309 	return SIZE_MAX;
1310 }
1311 
1312 static int omap2_mcspi_controller_setup(struct omap2_mcspi *mcspi)
1313 {
1314 	struct spi_master	*master = mcspi->master;
1315 	struct omap2_mcspi_regs	*ctx = &mcspi->ctx;
1316 	int			ret = 0;
1317 
1318 	ret = pm_runtime_get_sync(mcspi->dev);
1319 	if (ret < 0) {
1320 		pm_runtime_put_noidle(mcspi->dev);
1321 
1322 		return ret;
1323 	}
1324 
1325 	mcspi_write_reg(master, OMAP2_MCSPI_WAKEUPENABLE,
1326 			OMAP2_MCSPI_WAKEUPENABLE_WKEN);
1327 	ctx->wakeupenable = OMAP2_MCSPI_WAKEUPENABLE_WKEN;
1328 
1329 	omap2_mcspi_set_mode(master);
1330 	pm_runtime_mark_last_busy(mcspi->dev);
1331 	pm_runtime_put_autosuspend(mcspi->dev);
1332 	return 0;
1333 }
1334 
1335 /*
1336  * When SPI wake up from off-mode, CS is in activate state. If it was in
1337  * inactive state when driver was suspend, then force it to inactive state at
1338  * wake up.
1339  */
1340 static int omap_mcspi_runtime_resume(struct device *dev)
1341 {
1342 	struct spi_master *master = dev_get_drvdata(dev);
1343 	struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1344 	struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1345 	struct omap2_mcspi_cs *cs;
1346 
1347 	/* McSPI: context restore */
1348 	mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, ctx->modulctrl);
1349 	mcspi_write_reg(master, OMAP2_MCSPI_WAKEUPENABLE, ctx->wakeupenable);
1350 
1351 	list_for_each_entry(cs, &ctx->cs, node) {
1352 		/*
1353 		 * We need to toggle CS state for OMAP take this
1354 		 * change in account.
1355 		 */
1356 		if ((cs->chconf0 & OMAP2_MCSPI_CHCONF_FORCE) == 0) {
1357 			cs->chconf0 |= OMAP2_MCSPI_CHCONF_FORCE;
1358 			writel_relaxed(cs->chconf0,
1359 				       cs->base + OMAP2_MCSPI_CHCONF0);
1360 			cs->chconf0 &= ~OMAP2_MCSPI_CHCONF_FORCE;
1361 			writel_relaxed(cs->chconf0,
1362 				       cs->base + OMAP2_MCSPI_CHCONF0);
1363 		} else {
1364 			writel_relaxed(cs->chconf0,
1365 				       cs->base + OMAP2_MCSPI_CHCONF0);
1366 		}
1367 	}
1368 
1369 	return 0;
1370 }
1371 
1372 static struct omap2_mcspi_platform_config omap2_pdata = {
1373 	.regs_offset = 0,
1374 };
1375 
1376 static struct omap2_mcspi_platform_config omap4_pdata = {
1377 	.regs_offset = OMAP4_MCSPI_REG_OFFSET,
1378 };
1379 
1380 static struct omap2_mcspi_platform_config am654_pdata = {
1381 	.regs_offset = OMAP4_MCSPI_REG_OFFSET,
1382 	.max_xfer_len = SZ_4K - 1,
1383 };
1384 
1385 static const struct of_device_id omap_mcspi_of_match[] = {
1386 	{
1387 		.compatible = "ti,omap2-mcspi",
1388 		.data = &omap2_pdata,
1389 	},
1390 	{
1391 		.compatible = "ti,omap4-mcspi",
1392 		.data = &omap4_pdata,
1393 	},
1394 	{
1395 		.compatible = "ti,am654-mcspi",
1396 		.data = &am654_pdata,
1397 	},
1398 	{ },
1399 };
1400 MODULE_DEVICE_TABLE(of, omap_mcspi_of_match);
1401 
1402 static int omap2_mcspi_probe(struct platform_device *pdev)
1403 {
1404 	struct spi_master	*master;
1405 	const struct omap2_mcspi_platform_config *pdata;
1406 	struct omap2_mcspi	*mcspi;
1407 	struct resource		*r;
1408 	int			status = 0, i;
1409 	u32			regs_offset = 0;
1410 	struct device_node	*node = pdev->dev.of_node;
1411 	const struct of_device_id *match;
1412 
1413 	if (of_property_read_bool(node, "spi-slave"))
1414 		master = spi_alloc_slave(&pdev->dev, sizeof(*mcspi));
1415 	else
1416 		master = spi_alloc_master(&pdev->dev, sizeof(*mcspi));
1417 	if (!master)
1418 		return -ENOMEM;
1419 
1420 	/* the spi->mode bits understood by this driver: */
1421 	master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;
1422 	master->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 32);
1423 	master->setup = omap2_mcspi_setup;
1424 	master->auto_runtime_pm = true;
1425 	master->prepare_message = omap2_mcspi_prepare_message;
1426 	master->can_dma = omap2_mcspi_can_dma;
1427 	master->transfer_one = omap2_mcspi_transfer_one;
1428 	master->set_cs = omap2_mcspi_set_cs;
1429 	master->cleanup = omap2_mcspi_cleanup;
1430 	master->slave_abort = omap2_mcspi_slave_abort;
1431 	master->dev.of_node = node;
1432 	master->max_speed_hz = OMAP2_MCSPI_MAX_FREQ;
1433 	master->min_speed_hz = OMAP2_MCSPI_MAX_FREQ >> 15;
1434 
1435 	platform_set_drvdata(pdev, master);
1436 
1437 	mcspi = spi_master_get_devdata(master);
1438 	mcspi->master = master;
1439 
1440 	match = of_match_device(omap_mcspi_of_match, &pdev->dev);
1441 	if (match) {
1442 		u32 num_cs = 1; /* default number of chipselect */
1443 		pdata = match->data;
1444 
1445 		of_property_read_u32(node, "ti,spi-num-cs", &num_cs);
1446 		master->num_chipselect = num_cs;
1447 		if (of_get_property(node, "ti,pindir-d0-out-d1-in", NULL))
1448 			mcspi->pin_dir = MCSPI_PINDIR_D0_OUT_D1_IN;
1449 	} else {
1450 		pdata = dev_get_platdata(&pdev->dev);
1451 		master->num_chipselect = pdata->num_cs;
1452 		mcspi->pin_dir = pdata->pin_dir;
1453 	}
1454 	regs_offset = pdata->regs_offset;
1455 	if (pdata->max_xfer_len) {
1456 		mcspi->max_xfer_len = pdata->max_xfer_len;
1457 		master->max_transfer_size = omap2_mcspi_max_xfer_size;
1458 	}
1459 
1460 	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1461 	mcspi->base = devm_ioremap_resource(&pdev->dev, r);
1462 	if (IS_ERR(mcspi->base)) {
1463 		status = PTR_ERR(mcspi->base);
1464 		goto free_master;
1465 	}
1466 	mcspi->phys = r->start + regs_offset;
1467 	mcspi->base += regs_offset;
1468 
1469 	mcspi->dev = &pdev->dev;
1470 
1471 	INIT_LIST_HEAD(&mcspi->ctx.cs);
1472 
1473 	mcspi->dma_channels = devm_kcalloc(&pdev->dev, master->num_chipselect,
1474 					   sizeof(struct omap2_mcspi_dma),
1475 					   GFP_KERNEL);
1476 	if (mcspi->dma_channels == NULL) {
1477 		status = -ENOMEM;
1478 		goto free_master;
1479 	}
1480 
1481 	for (i = 0; i < master->num_chipselect; i++) {
1482 		sprintf(mcspi->dma_channels[i].dma_rx_ch_name, "rx%d", i);
1483 		sprintf(mcspi->dma_channels[i].dma_tx_ch_name, "tx%d", i);
1484 
1485 		status = omap2_mcspi_request_dma(mcspi,
1486 						 &mcspi->dma_channels[i]);
1487 		if (status == -EPROBE_DEFER)
1488 			goto free_master;
1489 	}
1490 
1491 	status = platform_get_irq(pdev, 0);
1492 	if (status == -EPROBE_DEFER)
1493 		goto free_master;
1494 	if (status < 0) {
1495 		dev_err(&pdev->dev, "no irq resource found\n");
1496 		goto free_master;
1497 	}
1498 	init_completion(&mcspi->txdone);
1499 	status = devm_request_irq(&pdev->dev, status,
1500 				  omap2_mcspi_irq_handler, 0, pdev->name,
1501 				  mcspi);
1502 	if (status) {
1503 		dev_err(&pdev->dev, "Cannot request IRQ");
1504 		goto free_master;
1505 	}
1506 
1507 	pm_runtime_use_autosuspend(&pdev->dev);
1508 	pm_runtime_set_autosuspend_delay(&pdev->dev, SPI_AUTOSUSPEND_TIMEOUT);
1509 	pm_runtime_enable(&pdev->dev);
1510 
1511 	status = omap2_mcspi_controller_setup(mcspi);
1512 	if (status < 0)
1513 		goto disable_pm;
1514 
1515 	status = devm_spi_register_controller(&pdev->dev, master);
1516 	if (status < 0)
1517 		goto disable_pm;
1518 
1519 	return status;
1520 
1521 disable_pm:
1522 	pm_runtime_dont_use_autosuspend(&pdev->dev);
1523 	pm_runtime_put_sync(&pdev->dev);
1524 	pm_runtime_disable(&pdev->dev);
1525 free_master:
1526 	omap2_mcspi_release_dma(master);
1527 	spi_master_put(master);
1528 	return status;
1529 }
1530 
1531 static int omap2_mcspi_remove(struct platform_device *pdev)
1532 {
1533 	struct spi_master *master = platform_get_drvdata(pdev);
1534 	struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1535 
1536 	omap2_mcspi_release_dma(master);
1537 
1538 	pm_runtime_dont_use_autosuspend(mcspi->dev);
1539 	pm_runtime_put_sync(mcspi->dev);
1540 	pm_runtime_disable(&pdev->dev);
1541 
1542 	return 0;
1543 }
1544 
1545 /* work with hotplug and coldplug */
1546 MODULE_ALIAS("platform:omap2_mcspi");
1547 
1548 static int __maybe_unused omap2_mcspi_suspend(struct device *dev)
1549 {
1550 	struct spi_master *master = dev_get_drvdata(dev);
1551 	struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1552 	int error;
1553 
1554 	error = pinctrl_pm_select_sleep_state(dev);
1555 	if (error)
1556 		dev_warn(mcspi->dev, "%s: failed to set pins: %i\n",
1557 			 __func__, error);
1558 
1559 	error = spi_master_suspend(master);
1560 	if (error)
1561 		dev_warn(mcspi->dev, "%s: master suspend failed: %i\n",
1562 			 __func__, error);
1563 
1564 	return pm_runtime_force_suspend(dev);
1565 }
1566 
1567 static int __maybe_unused omap2_mcspi_resume(struct device *dev)
1568 {
1569 	struct spi_master *master = dev_get_drvdata(dev);
1570 	struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1571 	int error;
1572 
1573 	error = pinctrl_pm_select_default_state(dev);
1574 	if (error)
1575 		dev_warn(mcspi->dev, "%s: failed to set pins: %i\n",
1576 			 __func__, error);
1577 
1578 	error = spi_master_resume(master);
1579 	if (error)
1580 		dev_warn(mcspi->dev, "%s: master resume failed: %i\n",
1581 			 __func__, error);
1582 
1583 	return pm_runtime_force_resume(dev);
1584 }
1585 
1586 static const struct dev_pm_ops omap2_mcspi_pm_ops = {
1587 	SET_SYSTEM_SLEEP_PM_OPS(omap2_mcspi_suspend,
1588 				omap2_mcspi_resume)
1589 	.runtime_resume	= omap_mcspi_runtime_resume,
1590 };
1591 
1592 static struct platform_driver omap2_mcspi_driver = {
1593 	.driver = {
1594 		.name =		"omap2_mcspi",
1595 		.pm =		&omap2_mcspi_pm_ops,
1596 		.of_match_table = omap_mcspi_of_match,
1597 	},
1598 	.probe =	omap2_mcspi_probe,
1599 	.remove =	omap2_mcspi_remove,
1600 };
1601 
1602 module_platform_driver(omap2_mcspi_driver);
1603 MODULE_LICENSE("GPL");
1604