1 // SPDX-License-Identifier: GPL-2.0-only
2 //
3 // Driver for Cadence QSPI Controller
4 //
5 // Copyright Altera Corporation (C) 2012-2014. All rights reserved.
6 // Copyright Intel Corporation (C) 2019-2020. All rights reserved.
7 // Copyright (C) 2020 Texas Instruments Incorporated - http://www.ti.com
8 
9 #include <linux/clk.h>
10 #include <linux/completion.h>
11 #include <linux/delay.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/dmaengine.h>
14 #include <linux/err.h>
15 #include <linux/errno.h>
16 #include <linux/firmware/xlnx-zynqmp.h>
17 #include <linux/interrupt.h>
18 #include <linux/io.h>
19 #include <linux/iopoll.h>
20 #include <linux/jiffies.h>
21 #include <linux/kernel.h>
22 #include <linux/log2.h>
23 #include <linux/module.h>
24 #include <linux/of.h>
25 #include <linux/platform_device.h>
26 #include <linux/pm_runtime.h>
27 #include <linux/reset.h>
28 #include <linux/sched.h>
29 #include <linux/spi/spi.h>
30 #include <linux/spi/spi-mem.h>
31 #include <linux/timer.h>
32 
33 #define CQSPI_NAME			"cadence-qspi"
34 #define CQSPI_MAX_CHIPSELECT		16
35 
36 /* Quirks */
37 #define CQSPI_NEEDS_WR_DELAY		BIT(0)
38 #define CQSPI_DISABLE_DAC_MODE		BIT(1)
39 #define CQSPI_SUPPORT_EXTERNAL_DMA	BIT(2)
40 #define CQSPI_NO_SUPPORT_WR_COMPLETION	BIT(3)
41 #define CQSPI_SLOW_SRAM		BIT(4)
42 #define CQSPI_NEEDS_APB_AHB_HAZARD_WAR	BIT(5)
43 
44 /* Capabilities */
45 #define CQSPI_SUPPORTS_OCTAL		BIT(0)
46 
47 #define CQSPI_OP_WIDTH(part) ((part).nbytes ? ilog2((part).buswidth) : 0)
48 
49 enum {
50 	CLK_QSPI_APB = 0,
51 	CLK_QSPI_AHB,
52 	CLK_QSPI_NUM,
53 };
54 
55 struct cqspi_st;
56 
57 struct cqspi_flash_pdata {
58 	struct cqspi_st	*cqspi;
59 	u32		clk_rate;
60 	u32		read_delay;
61 	u32		tshsl_ns;
62 	u32		tsd2d_ns;
63 	u32		tchsh_ns;
64 	u32		tslch_ns;
65 	u8		cs;
66 };
67 
68 struct cqspi_st {
69 	struct platform_device	*pdev;
70 	struct spi_controller	*host;
71 	struct clk		*clk;
72 	struct clk		*clks[CLK_QSPI_NUM];
73 	unsigned int		sclk;
74 
75 	void __iomem		*iobase;
76 	void __iomem		*ahb_base;
77 	resource_size_t		ahb_size;
78 	struct completion	transfer_complete;
79 
80 	struct dma_chan		*rx_chan;
81 	struct completion	rx_dma_complete;
82 	dma_addr_t		mmap_phys_base;
83 
84 	int			current_cs;
85 	unsigned long		master_ref_clk_hz;
86 	bool			is_decoded_cs;
87 	u32			fifo_depth;
88 	u32			fifo_width;
89 	u32			num_chipselect;
90 	bool			rclk_en;
91 	u32			trigger_address;
92 	u32			wr_delay;
93 	bool			use_direct_mode;
94 	bool			use_direct_mode_wr;
95 	struct cqspi_flash_pdata f_pdata[CQSPI_MAX_CHIPSELECT];
96 	bool			use_dma_read;
97 	u32			pd_dev_id;
98 	bool			wr_completion;
99 	bool			slow_sram;
100 	bool			apb_ahb_hazard;
101 
102 	bool			is_jh7110; /* Flag for StarFive JH7110 SoC */
103 };
104 
105 struct cqspi_driver_platdata {
106 	u32 hwcaps_mask;
107 	u8 quirks;
108 	int (*indirect_read_dma)(struct cqspi_flash_pdata *f_pdata,
109 				 u_char *rxbuf, loff_t from_addr, size_t n_rx);
110 	u32 (*get_dma_status)(struct cqspi_st *cqspi);
111 	int (*jh7110_clk_init)(struct platform_device *pdev,
112 			       struct cqspi_st *cqspi);
113 };
114 
115 /* Operation timeout value */
116 #define CQSPI_TIMEOUT_MS			500
117 #define CQSPI_READ_TIMEOUT_MS			10
118 
119 #define CQSPI_DUMMY_CLKS_PER_BYTE		8
120 #define CQSPI_DUMMY_BYTES_MAX			4
121 #define CQSPI_DUMMY_CLKS_MAX			31
122 
123 #define CQSPI_STIG_DATA_LEN_MAX			8
124 
125 /* Register map */
126 #define CQSPI_REG_CONFIG			0x00
127 #define CQSPI_REG_CONFIG_ENABLE_MASK		BIT(0)
128 #define CQSPI_REG_CONFIG_ENB_DIR_ACC_CTRL	BIT(7)
129 #define CQSPI_REG_CONFIG_DECODE_MASK		BIT(9)
130 #define CQSPI_REG_CONFIG_CHIPSELECT_LSB		10
131 #define CQSPI_REG_CONFIG_DMA_MASK		BIT(15)
132 #define CQSPI_REG_CONFIG_BAUD_LSB		19
133 #define CQSPI_REG_CONFIG_DTR_PROTO		BIT(24)
134 #define CQSPI_REG_CONFIG_DUAL_OPCODE		BIT(30)
135 #define CQSPI_REG_CONFIG_IDLE_LSB		31
136 #define CQSPI_REG_CONFIG_CHIPSELECT_MASK	0xF
137 #define CQSPI_REG_CONFIG_BAUD_MASK		0xF
138 
139 #define CQSPI_REG_RD_INSTR			0x04
140 #define CQSPI_REG_RD_INSTR_OPCODE_LSB		0
141 #define CQSPI_REG_RD_INSTR_TYPE_INSTR_LSB	8
142 #define CQSPI_REG_RD_INSTR_TYPE_ADDR_LSB	12
143 #define CQSPI_REG_RD_INSTR_TYPE_DATA_LSB	16
144 #define CQSPI_REG_RD_INSTR_MODE_EN_LSB		20
145 #define CQSPI_REG_RD_INSTR_DUMMY_LSB		24
146 #define CQSPI_REG_RD_INSTR_TYPE_INSTR_MASK	0x3
147 #define CQSPI_REG_RD_INSTR_TYPE_ADDR_MASK	0x3
148 #define CQSPI_REG_RD_INSTR_TYPE_DATA_MASK	0x3
149 #define CQSPI_REG_RD_INSTR_DUMMY_MASK		0x1F
150 
151 #define CQSPI_REG_WR_INSTR			0x08
152 #define CQSPI_REG_WR_INSTR_OPCODE_LSB		0
153 #define CQSPI_REG_WR_INSTR_TYPE_ADDR_LSB	12
154 #define CQSPI_REG_WR_INSTR_TYPE_DATA_LSB	16
155 
156 #define CQSPI_REG_DELAY				0x0C
157 #define CQSPI_REG_DELAY_TSLCH_LSB		0
158 #define CQSPI_REG_DELAY_TCHSH_LSB		8
159 #define CQSPI_REG_DELAY_TSD2D_LSB		16
160 #define CQSPI_REG_DELAY_TSHSL_LSB		24
161 #define CQSPI_REG_DELAY_TSLCH_MASK		0xFF
162 #define CQSPI_REG_DELAY_TCHSH_MASK		0xFF
163 #define CQSPI_REG_DELAY_TSD2D_MASK		0xFF
164 #define CQSPI_REG_DELAY_TSHSL_MASK		0xFF
165 
166 #define CQSPI_REG_READCAPTURE			0x10
167 #define CQSPI_REG_READCAPTURE_BYPASS_LSB	0
168 #define CQSPI_REG_READCAPTURE_DELAY_LSB		1
169 #define CQSPI_REG_READCAPTURE_DELAY_MASK	0xF
170 
171 #define CQSPI_REG_SIZE				0x14
172 #define CQSPI_REG_SIZE_ADDRESS_LSB		0
173 #define CQSPI_REG_SIZE_PAGE_LSB			4
174 #define CQSPI_REG_SIZE_BLOCK_LSB		16
175 #define CQSPI_REG_SIZE_ADDRESS_MASK		0xF
176 #define CQSPI_REG_SIZE_PAGE_MASK		0xFFF
177 #define CQSPI_REG_SIZE_BLOCK_MASK		0x3F
178 
179 #define CQSPI_REG_SRAMPARTITION			0x18
180 #define CQSPI_REG_INDIRECTTRIGGER		0x1C
181 
182 #define CQSPI_REG_DMA				0x20
183 #define CQSPI_REG_DMA_SINGLE_LSB		0
184 #define CQSPI_REG_DMA_BURST_LSB			8
185 #define CQSPI_REG_DMA_SINGLE_MASK		0xFF
186 #define CQSPI_REG_DMA_BURST_MASK		0xFF
187 
188 #define CQSPI_REG_REMAP				0x24
189 #define CQSPI_REG_MODE_BIT			0x28
190 
191 #define CQSPI_REG_SDRAMLEVEL			0x2C
192 #define CQSPI_REG_SDRAMLEVEL_RD_LSB		0
193 #define CQSPI_REG_SDRAMLEVEL_WR_LSB		16
194 #define CQSPI_REG_SDRAMLEVEL_RD_MASK		0xFFFF
195 #define CQSPI_REG_SDRAMLEVEL_WR_MASK		0xFFFF
196 
197 #define CQSPI_REG_WR_COMPLETION_CTRL		0x38
198 #define CQSPI_REG_WR_DISABLE_AUTO_POLL		BIT(14)
199 
200 #define CQSPI_REG_IRQSTATUS			0x40
201 #define CQSPI_REG_IRQMASK			0x44
202 
203 #define CQSPI_REG_INDIRECTRD			0x60
204 #define CQSPI_REG_INDIRECTRD_START_MASK		BIT(0)
205 #define CQSPI_REG_INDIRECTRD_CANCEL_MASK	BIT(1)
206 #define CQSPI_REG_INDIRECTRD_DONE_MASK		BIT(5)
207 
208 #define CQSPI_REG_INDIRECTRDWATERMARK		0x64
209 #define CQSPI_REG_INDIRECTRDSTARTADDR		0x68
210 #define CQSPI_REG_INDIRECTRDBYTES		0x6C
211 
212 #define CQSPI_REG_CMDCTRL			0x90
213 #define CQSPI_REG_CMDCTRL_EXECUTE_MASK		BIT(0)
214 #define CQSPI_REG_CMDCTRL_INPROGRESS_MASK	BIT(1)
215 #define CQSPI_REG_CMDCTRL_DUMMY_LSB		7
216 #define CQSPI_REG_CMDCTRL_WR_BYTES_LSB		12
217 #define CQSPI_REG_CMDCTRL_WR_EN_LSB		15
218 #define CQSPI_REG_CMDCTRL_ADD_BYTES_LSB		16
219 #define CQSPI_REG_CMDCTRL_ADDR_EN_LSB		19
220 #define CQSPI_REG_CMDCTRL_RD_BYTES_LSB		20
221 #define CQSPI_REG_CMDCTRL_RD_EN_LSB		23
222 #define CQSPI_REG_CMDCTRL_OPCODE_LSB		24
223 #define CQSPI_REG_CMDCTRL_WR_BYTES_MASK		0x7
224 #define CQSPI_REG_CMDCTRL_ADD_BYTES_MASK	0x3
225 #define CQSPI_REG_CMDCTRL_RD_BYTES_MASK		0x7
226 #define CQSPI_REG_CMDCTRL_DUMMY_MASK		0x1F
227 
228 #define CQSPI_REG_INDIRECTWR			0x70
229 #define CQSPI_REG_INDIRECTWR_START_MASK		BIT(0)
230 #define CQSPI_REG_INDIRECTWR_CANCEL_MASK	BIT(1)
231 #define CQSPI_REG_INDIRECTWR_DONE_MASK		BIT(5)
232 
233 #define CQSPI_REG_INDIRECTWRWATERMARK		0x74
234 #define CQSPI_REG_INDIRECTWRSTARTADDR		0x78
235 #define CQSPI_REG_INDIRECTWRBYTES		0x7C
236 
237 #define CQSPI_REG_INDTRIG_ADDRRANGE		0x80
238 
239 #define CQSPI_REG_CMDADDRESS			0x94
240 #define CQSPI_REG_CMDREADDATALOWER		0xA0
241 #define CQSPI_REG_CMDREADDATAUPPER		0xA4
242 #define CQSPI_REG_CMDWRITEDATALOWER		0xA8
243 #define CQSPI_REG_CMDWRITEDATAUPPER		0xAC
244 
245 #define CQSPI_REG_POLLING_STATUS		0xB0
246 #define CQSPI_REG_POLLING_STATUS_DUMMY_LSB	16
247 
248 #define CQSPI_REG_OP_EXT_LOWER			0xE0
249 #define CQSPI_REG_OP_EXT_READ_LSB		24
250 #define CQSPI_REG_OP_EXT_WRITE_LSB		16
251 #define CQSPI_REG_OP_EXT_STIG_LSB		0
252 
253 #define CQSPI_REG_VERSAL_DMA_SRC_ADDR		0x1000
254 
255 #define CQSPI_REG_VERSAL_DMA_DST_ADDR		0x1800
256 #define CQSPI_REG_VERSAL_DMA_DST_SIZE		0x1804
257 
258 #define CQSPI_REG_VERSAL_DMA_DST_CTRL		0x180C
259 
260 #define CQSPI_REG_VERSAL_DMA_DST_I_STS		0x1814
261 #define CQSPI_REG_VERSAL_DMA_DST_I_EN		0x1818
262 #define CQSPI_REG_VERSAL_DMA_DST_I_DIS		0x181C
263 #define CQSPI_REG_VERSAL_DMA_DST_DONE_MASK	BIT(1)
264 
265 #define CQSPI_REG_VERSAL_DMA_DST_ADDR_MSB	0x1828
266 
267 #define CQSPI_REG_VERSAL_DMA_DST_CTRL_VAL	0xF43FFA00
268 #define CQSPI_REG_VERSAL_ADDRRANGE_WIDTH_VAL	0x6
269 
270 /* Interrupt status bits */
271 #define CQSPI_REG_IRQ_MODE_ERR			BIT(0)
272 #define CQSPI_REG_IRQ_UNDERFLOW			BIT(1)
273 #define CQSPI_REG_IRQ_IND_COMP			BIT(2)
274 #define CQSPI_REG_IRQ_IND_RD_REJECT		BIT(3)
275 #define CQSPI_REG_IRQ_WR_PROTECTED_ERR		BIT(4)
276 #define CQSPI_REG_IRQ_ILLEGAL_AHB_ERR		BIT(5)
277 #define CQSPI_REG_IRQ_WATERMARK			BIT(6)
278 #define CQSPI_REG_IRQ_IND_SRAM_FULL		BIT(12)
279 
280 #define CQSPI_IRQ_MASK_RD		(CQSPI_REG_IRQ_WATERMARK	| \
281 					 CQSPI_REG_IRQ_IND_SRAM_FULL	| \
282 					 CQSPI_REG_IRQ_IND_COMP)
283 
284 #define CQSPI_IRQ_MASK_WR		(CQSPI_REG_IRQ_IND_COMP		| \
285 					 CQSPI_REG_IRQ_WATERMARK	| \
286 					 CQSPI_REG_IRQ_UNDERFLOW)
287 
288 #define CQSPI_IRQ_STATUS_MASK		0x1FFFF
289 #define CQSPI_DMA_UNALIGN		0x3
290 
291 #define CQSPI_REG_VERSAL_DMA_VAL		0x602
292 
293 static int cqspi_wait_for_bit(void __iomem *reg, const u32 mask, bool clr)
294 {
295 	u32 val;
296 
297 	return readl_relaxed_poll_timeout(reg, val,
298 					  (((clr ? ~val : val) & mask) == mask),
299 					  10, CQSPI_TIMEOUT_MS * 1000);
300 }
301 
302 static bool cqspi_is_idle(struct cqspi_st *cqspi)
303 {
304 	u32 reg = readl(cqspi->iobase + CQSPI_REG_CONFIG);
305 
306 	return reg & (1UL << CQSPI_REG_CONFIG_IDLE_LSB);
307 }
308 
309 static u32 cqspi_get_rd_sram_level(struct cqspi_st *cqspi)
310 {
311 	u32 reg = readl(cqspi->iobase + CQSPI_REG_SDRAMLEVEL);
312 
313 	reg >>= CQSPI_REG_SDRAMLEVEL_RD_LSB;
314 	return reg & CQSPI_REG_SDRAMLEVEL_RD_MASK;
315 }
316 
317 static u32 cqspi_get_versal_dma_status(struct cqspi_st *cqspi)
318 {
319 	u32 dma_status;
320 
321 	dma_status = readl(cqspi->iobase +
322 					   CQSPI_REG_VERSAL_DMA_DST_I_STS);
323 	writel(dma_status, cqspi->iobase +
324 		   CQSPI_REG_VERSAL_DMA_DST_I_STS);
325 
326 	return dma_status & CQSPI_REG_VERSAL_DMA_DST_DONE_MASK;
327 }
328 
329 static irqreturn_t cqspi_irq_handler(int this_irq, void *dev)
330 {
331 	struct cqspi_st *cqspi = dev;
332 	unsigned int irq_status;
333 	struct device *device = &cqspi->pdev->dev;
334 	const struct cqspi_driver_platdata *ddata;
335 
336 	ddata = of_device_get_match_data(device);
337 
338 	/* Read interrupt status */
339 	irq_status = readl(cqspi->iobase + CQSPI_REG_IRQSTATUS);
340 
341 	/* Clear interrupt */
342 	writel(irq_status, cqspi->iobase + CQSPI_REG_IRQSTATUS);
343 
344 	if (cqspi->use_dma_read && ddata && ddata->get_dma_status) {
345 		if (ddata->get_dma_status(cqspi)) {
346 			complete(&cqspi->transfer_complete);
347 			return IRQ_HANDLED;
348 		}
349 	}
350 
351 	else if (!cqspi->slow_sram)
352 		irq_status &= CQSPI_IRQ_MASK_RD | CQSPI_IRQ_MASK_WR;
353 	else
354 		irq_status &= CQSPI_REG_IRQ_WATERMARK | CQSPI_IRQ_MASK_WR;
355 
356 	if (irq_status)
357 		complete(&cqspi->transfer_complete);
358 
359 	return IRQ_HANDLED;
360 }
361 
362 static unsigned int cqspi_calc_rdreg(const struct spi_mem_op *op)
363 {
364 	u32 rdreg = 0;
365 
366 	rdreg |= CQSPI_OP_WIDTH(op->cmd) << CQSPI_REG_RD_INSTR_TYPE_INSTR_LSB;
367 	rdreg |= CQSPI_OP_WIDTH(op->addr) << CQSPI_REG_RD_INSTR_TYPE_ADDR_LSB;
368 	rdreg |= CQSPI_OP_WIDTH(op->data) << CQSPI_REG_RD_INSTR_TYPE_DATA_LSB;
369 
370 	return rdreg;
371 }
372 
373 static unsigned int cqspi_calc_dummy(const struct spi_mem_op *op)
374 {
375 	unsigned int dummy_clk;
376 
377 	if (!op->dummy.nbytes)
378 		return 0;
379 
380 	dummy_clk = op->dummy.nbytes * (8 / op->dummy.buswidth);
381 	if (op->cmd.dtr)
382 		dummy_clk /= 2;
383 
384 	return dummy_clk;
385 }
386 
387 static int cqspi_wait_idle(struct cqspi_st *cqspi)
388 {
389 	const unsigned int poll_idle_retry = 3;
390 	unsigned int count = 0;
391 	unsigned long timeout;
392 
393 	timeout = jiffies + msecs_to_jiffies(CQSPI_TIMEOUT_MS);
394 	while (1) {
395 		/*
396 		 * Read few times in succession to ensure the controller
397 		 * is indeed idle, that is, the bit does not transition
398 		 * low again.
399 		 */
400 		if (cqspi_is_idle(cqspi))
401 			count++;
402 		else
403 			count = 0;
404 
405 		if (count >= poll_idle_retry)
406 			return 0;
407 
408 		if (time_after(jiffies, timeout)) {
409 			/* Timeout, in busy mode. */
410 			dev_err(&cqspi->pdev->dev,
411 				"QSPI is still busy after %dms timeout.\n",
412 				CQSPI_TIMEOUT_MS);
413 			return -ETIMEDOUT;
414 		}
415 
416 		cpu_relax();
417 	}
418 }
419 
420 static int cqspi_exec_flash_cmd(struct cqspi_st *cqspi, unsigned int reg)
421 {
422 	void __iomem *reg_base = cqspi->iobase;
423 	int ret;
424 
425 	/* Write the CMDCTRL without start execution. */
426 	writel(reg, reg_base + CQSPI_REG_CMDCTRL);
427 	/* Start execute */
428 	reg |= CQSPI_REG_CMDCTRL_EXECUTE_MASK;
429 	writel(reg, reg_base + CQSPI_REG_CMDCTRL);
430 
431 	/* Polling for completion. */
432 	ret = cqspi_wait_for_bit(reg_base + CQSPI_REG_CMDCTRL,
433 				 CQSPI_REG_CMDCTRL_INPROGRESS_MASK, 1);
434 	if (ret) {
435 		dev_err(&cqspi->pdev->dev,
436 			"Flash command execution timed out.\n");
437 		return ret;
438 	}
439 
440 	/* Polling QSPI idle status. */
441 	return cqspi_wait_idle(cqspi);
442 }
443 
444 static int cqspi_setup_opcode_ext(struct cqspi_flash_pdata *f_pdata,
445 				  const struct spi_mem_op *op,
446 				  unsigned int shift)
447 {
448 	struct cqspi_st *cqspi = f_pdata->cqspi;
449 	void __iomem *reg_base = cqspi->iobase;
450 	unsigned int reg;
451 	u8 ext;
452 
453 	if (op->cmd.nbytes != 2)
454 		return -EINVAL;
455 
456 	/* Opcode extension is the LSB. */
457 	ext = op->cmd.opcode & 0xff;
458 
459 	reg = readl(reg_base + CQSPI_REG_OP_EXT_LOWER);
460 	reg &= ~(0xff << shift);
461 	reg |= ext << shift;
462 	writel(reg, reg_base + CQSPI_REG_OP_EXT_LOWER);
463 
464 	return 0;
465 }
466 
467 static int cqspi_enable_dtr(struct cqspi_flash_pdata *f_pdata,
468 			    const struct spi_mem_op *op, unsigned int shift)
469 {
470 	struct cqspi_st *cqspi = f_pdata->cqspi;
471 	void __iomem *reg_base = cqspi->iobase;
472 	unsigned int reg;
473 	int ret;
474 
475 	reg = readl(reg_base + CQSPI_REG_CONFIG);
476 
477 	/*
478 	 * We enable dual byte opcode here. The callers have to set up the
479 	 * extension opcode based on which type of operation it is.
480 	 */
481 	if (op->cmd.dtr) {
482 		reg |= CQSPI_REG_CONFIG_DTR_PROTO;
483 		reg |= CQSPI_REG_CONFIG_DUAL_OPCODE;
484 
485 		/* Set up command opcode extension. */
486 		ret = cqspi_setup_opcode_ext(f_pdata, op, shift);
487 		if (ret)
488 			return ret;
489 	} else {
490 		reg &= ~CQSPI_REG_CONFIG_DTR_PROTO;
491 		reg &= ~CQSPI_REG_CONFIG_DUAL_OPCODE;
492 	}
493 
494 	writel(reg, reg_base + CQSPI_REG_CONFIG);
495 
496 	return cqspi_wait_idle(cqspi);
497 }
498 
499 static int cqspi_command_read(struct cqspi_flash_pdata *f_pdata,
500 			      const struct spi_mem_op *op)
501 {
502 	struct cqspi_st *cqspi = f_pdata->cqspi;
503 	void __iomem *reg_base = cqspi->iobase;
504 	u8 *rxbuf = op->data.buf.in;
505 	u8 opcode;
506 	size_t n_rx = op->data.nbytes;
507 	unsigned int rdreg;
508 	unsigned int reg;
509 	unsigned int dummy_clk;
510 	size_t read_len;
511 	int status;
512 
513 	status = cqspi_enable_dtr(f_pdata, op, CQSPI_REG_OP_EXT_STIG_LSB);
514 	if (status)
515 		return status;
516 
517 	if (!n_rx || n_rx > CQSPI_STIG_DATA_LEN_MAX || !rxbuf) {
518 		dev_err(&cqspi->pdev->dev,
519 			"Invalid input argument, len %zu rxbuf 0x%p\n",
520 			n_rx, rxbuf);
521 		return -EINVAL;
522 	}
523 
524 	if (op->cmd.dtr)
525 		opcode = op->cmd.opcode >> 8;
526 	else
527 		opcode = op->cmd.opcode;
528 
529 	reg = opcode << CQSPI_REG_CMDCTRL_OPCODE_LSB;
530 
531 	rdreg = cqspi_calc_rdreg(op);
532 	writel(rdreg, reg_base + CQSPI_REG_RD_INSTR);
533 
534 	dummy_clk = cqspi_calc_dummy(op);
535 	if (dummy_clk > CQSPI_DUMMY_CLKS_MAX)
536 		return -EOPNOTSUPP;
537 
538 	if (dummy_clk)
539 		reg |= (dummy_clk & CQSPI_REG_CMDCTRL_DUMMY_MASK)
540 		     << CQSPI_REG_CMDCTRL_DUMMY_LSB;
541 
542 	reg |= (0x1 << CQSPI_REG_CMDCTRL_RD_EN_LSB);
543 
544 	/* 0 means 1 byte. */
545 	reg |= (((n_rx - 1) & CQSPI_REG_CMDCTRL_RD_BYTES_MASK)
546 		<< CQSPI_REG_CMDCTRL_RD_BYTES_LSB);
547 
548 	/* setup ADDR BIT field */
549 	if (op->addr.nbytes) {
550 		reg |= (0x1 << CQSPI_REG_CMDCTRL_ADDR_EN_LSB);
551 		reg |= ((op->addr.nbytes - 1) &
552 			CQSPI_REG_CMDCTRL_ADD_BYTES_MASK)
553 			<< CQSPI_REG_CMDCTRL_ADD_BYTES_LSB;
554 
555 		writel(op->addr.val, reg_base + CQSPI_REG_CMDADDRESS);
556 	}
557 
558 	status = cqspi_exec_flash_cmd(cqspi, reg);
559 	if (status)
560 		return status;
561 
562 	reg = readl(reg_base + CQSPI_REG_CMDREADDATALOWER);
563 
564 	/* Put the read value into rx_buf */
565 	read_len = (n_rx > 4) ? 4 : n_rx;
566 	memcpy(rxbuf, &reg, read_len);
567 	rxbuf += read_len;
568 
569 	if (n_rx > 4) {
570 		reg = readl(reg_base + CQSPI_REG_CMDREADDATAUPPER);
571 
572 		read_len = n_rx - read_len;
573 		memcpy(rxbuf, &reg, read_len);
574 	}
575 
576 	/* Reset CMD_CTRL Reg once command read completes */
577 	writel(0, reg_base + CQSPI_REG_CMDCTRL);
578 
579 	return 0;
580 }
581 
582 static int cqspi_command_write(struct cqspi_flash_pdata *f_pdata,
583 			       const struct spi_mem_op *op)
584 {
585 	struct cqspi_st *cqspi = f_pdata->cqspi;
586 	void __iomem *reg_base = cqspi->iobase;
587 	u8 opcode;
588 	const u8 *txbuf = op->data.buf.out;
589 	size_t n_tx = op->data.nbytes;
590 	unsigned int reg;
591 	unsigned int data;
592 	size_t write_len;
593 	int ret;
594 
595 	ret = cqspi_enable_dtr(f_pdata, op, CQSPI_REG_OP_EXT_STIG_LSB);
596 	if (ret)
597 		return ret;
598 
599 	if (n_tx > CQSPI_STIG_DATA_LEN_MAX || (n_tx && !txbuf)) {
600 		dev_err(&cqspi->pdev->dev,
601 			"Invalid input argument, cmdlen %zu txbuf 0x%p\n",
602 			n_tx, txbuf);
603 		return -EINVAL;
604 	}
605 
606 	reg = cqspi_calc_rdreg(op);
607 	writel(reg, reg_base + CQSPI_REG_RD_INSTR);
608 
609 	if (op->cmd.dtr)
610 		opcode = op->cmd.opcode >> 8;
611 	else
612 		opcode = op->cmd.opcode;
613 
614 	reg = opcode << CQSPI_REG_CMDCTRL_OPCODE_LSB;
615 
616 	if (op->addr.nbytes) {
617 		reg |= (0x1 << CQSPI_REG_CMDCTRL_ADDR_EN_LSB);
618 		reg |= ((op->addr.nbytes - 1) &
619 			CQSPI_REG_CMDCTRL_ADD_BYTES_MASK)
620 			<< CQSPI_REG_CMDCTRL_ADD_BYTES_LSB;
621 
622 		writel(op->addr.val, reg_base + CQSPI_REG_CMDADDRESS);
623 	}
624 
625 	if (n_tx) {
626 		reg |= (0x1 << CQSPI_REG_CMDCTRL_WR_EN_LSB);
627 		reg |= ((n_tx - 1) & CQSPI_REG_CMDCTRL_WR_BYTES_MASK)
628 			<< CQSPI_REG_CMDCTRL_WR_BYTES_LSB;
629 		data = 0;
630 		write_len = (n_tx > 4) ? 4 : n_tx;
631 		memcpy(&data, txbuf, write_len);
632 		txbuf += write_len;
633 		writel(data, reg_base + CQSPI_REG_CMDWRITEDATALOWER);
634 
635 		if (n_tx > 4) {
636 			data = 0;
637 			write_len = n_tx - 4;
638 			memcpy(&data, txbuf, write_len);
639 			writel(data, reg_base + CQSPI_REG_CMDWRITEDATAUPPER);
640 		}
641 	}
642 
643 	ret = cqspi_exec_flash_cmd(cqspi, reg);
644 
645 	/* Reset CMD_CTRL Reg once command write completes */
646 	writel(0, reg_base + CQSPI_REG_CMDCTRL);
647 
648 	return ret;
649 }
650 
651 static int cqspi_read_setup(struct cqspi_flash_pdata *f_pdata,
652 			    const struct spi_mem_op *op)
653 {
654 	struct cqspi_st *cqspi = f_pdata->cqspi;
655 	void __iomem *reg_base = cqspi->iobase;
656 	unsigned int dummy_clk = 0;
657 	unsigned int reg;
658 	int ret;
659 	u8 opcode;
660 
661 	ret = cqspi_enable_dtr(f_pdata, op, CQSPI_REG_OP_EXT_READ_LSB);
662 	if (ret)
663 		return ret;
664 
665 	if (op->cmd.dtr)
666 		opcode = op->cmd.opcode >> 8;
667 	else
668 		opcode = op->cmd.opcode;
669 
670 	reg = opcode << CQSPI_REG_RD_INSTR_OPCODE_LSB;
671 	reg |= cqspi_calc_rdreg(op);
672 
673 	/* Setup dummy clock cycles */
674 	dummy_clk = cqspi_calc_dummy(op);
675 
676 	if (dummy_clk > CQSPI_DUMMY_CLKS_MAX)
677 		return -EOPNOTSUPP;
678 
679 	if (dummy_clk)
680 		reg |= (dummy_clk & CQSPI_REG_RD_INSTR_DUMMY_MASK)
681 		       << CQSPI_REG_RD_INSTR_DUMMY_LSB;
682 
683 	writel(reg, reg_base + CQSPI_REG_RD_INSTR);
684 
685 	/* Set address width */
686 	reg = readl(reg_base + CQSPI_REG_SIZE);
687 	reg &= ~CQSPI_REG_SIZE_ADDRESS_MASK;
688 	reg |= (op->addr.nbytes - 1);
689 	writel(reg, reg_base + CQSPI_REG_SIZE);
690 	return 0;
691 }
692 
693 static int cqspi_indirect_read_execute(struct cqspi_flash_pdata *f_pdata,
694 				       u8 *rxbuf, loff_t from_addr,
695 				       const size_t n_rx)
696 {
697 	struct cqspi_st *cqspi = f_pdata->cqspi;
698 	struct device *dev = &cqspi->pdev->dev;
699 	void __iomem *reg_base = cqspi->iobase;
700 	void __iomem *ahb_base = cqspi->ahb_base;
701 	unsigned int remaining = n_rx;
702 	unsigned int mod_bytes = n_rx % 4;
703 	unsigned int bytes_to_read = 0;
704 	u8 *rxbuf_end = rxbuf + n_rx;
705 	int ret = 0;
706 
707 	writel(from_addr, reg_base + CQSPI_REG_INDIRECTRDSTARTADDR);
708 	writel(remaining, reg_base + CQSPI_REG_INDIRECTRDBYTES);
709 
710 	/* Clear all interrupts. */
711 	writel(CQSPI_IRQ_STATUS_MASK, reg_base + CQSPI_REG_IRQSTATUS);
712 
713 	/*
714 	 * On SoCFPGA platform reading the SRAM is slow due to
715 	 * hardware limitation and causing read interrupt storm to CPU,
716 	 * so enabling only watermark interrupt to disable all read
717 	 * interrupts later as we want to run "bytes to read" loop with
718 	 * all the read interrupts disabled for max performance.
719 	 */
720 
721 	if (!cqspi->slow_sram)
722 		writel(CQSPI_IRQ_MASK_RD, reg_base + CQSPI_REG_IRQMASK);
723 	else
724 		writel(CQSPI_REG_IRQ_WATERMARK, reg_base + CQSPI_REG_IRQMASK);
725 
726 	reinit_completion(&cqspi->transfer_complete);
727 	writel(CQSPI_REG_INDIRECTRD_START_MASK,
728 	       reg_base + CQSPI_REG_INDIRECTRD);
729 
730 	while (remaining > 0) {
731 		if (!wait_for_completion_timeout(&cqspi->transfer_complete,
732 						 msecs_to_jiffies(CQSPI_READ_TIMEOUT_MS)))
733 			ret = -ETIMEDOUT;
734 
735 		/*
736 		 * Disable all read interrupts until
737 		 * we are out of "bytes to read"
738 		 */
739 		if (cqspi->slow_sram)
740 			writel(0x0, reg_base + CQSPI_REG_IRQMASK);
741 
742 		bytes_to_read = cqspi_get_rd_sram_level(cqspi);
743 
744 		if (ret && bytes_to_read == 0) {
745 			dev_err(dev, "Indirect read timeout, no bytes\n");
746 			goto failrd;
747 		}
748 
749 		while (bytes_to_read != 0) {
750 			unsigned int word_remain = round_down(remaining, 4);
751 
752 			bytes_to_read *= cqspi->fifo_width;
753 			bytes_to_read = bytes_to_read > remaining ?
754 					remaining : bytes_to_read;
755 			bytes_to_read = round_down(bytes_to_read, 4);
756 			/* Read 4 byte word chunks then single bytes */
757 			if (bytes_to_read) {
758 				ioread32_rep(ahb_base, rxbuf,
759 					     (bytes_to_read / 4));
760 			} else if (!word_remain && mod_bytes) {
761 				unsigned int temp = ioread32(ahb_base);
762 
763 				bytes_to_read = mod_bytes;
764 				memcpy(rxbuf, &temp, min((unsigned int)
765 							 (rxbuf_end - rxbuf),
766 							 bytes_to_read));
767 			}
768 			rxbuf += bytes_to_read;
769 			remaining -= bytes_to_read;
770 			bytes_to_read = cqspi_get_rd_sram_level(cqspi);
771 		}
772 
773 		if (remaining > 0) {
774 			reinit_completion(&cqspi->transfer_complete);
775 			if (cqspi->slow_sram)
776 				writel(CQSPI_REG_IRQ_WATERMARK, reg_base + CQSPI_REG_IRQMASK);
777 		}
778 	}
779 
780 	/* Check indirect done status */
781 	ret = cqspi_wait_for_bit(reg_base + CQSPI_REG_INDIRECTRD,
782 				 CQSPI_REG_INDIRECTRD_DONE_MASK, 0);
783 	if (ret) {
784 		dev_err(dev, "Indirect read completion error (%i)\n", ret);
785 		goto failrd;
786 	}
787 
788 	/* Disable interrupt */
789 	writel(0, reg_base + CQSPI_REG_IRQMASK);
790 
791 	/* Clear indirect completion status */
792 	writel(CQSPI_REG_INDIRECTRD_DONE_MASK, reg_base + CQSPI_REG_INDIRECTRD);
793 
794 	return 0;
795 
796 failrd:
797 	/* Disable interrupt */
798 	writel(0, reg_base + CQSPI_REG_IRQMASK);
799 
800 	/* Cancel the indirect read */
801 	writel(CQSPI_REG_INDIRECTRD_CANCEL_MASK,
802 	       reg_base + CQSPI_REG_INDIRECTRD);
803 	return ret;
804 }
805 
806 static void cqspi_controller_enable(struct cqspi_st *cqspi, bool enable)
807 {
808 	void __iomem *reg_base = cqspi->iobase;
809 	unsigned int reg;
810 
811 	reg = readl(reg_base + CQSPI_REG_CONFIG);
812 
813 	if (enable)
814 		reg |= CQSPI_REG_CONFIG_ENABLE_MASK;
815 	else
816 		reg &= ~CQSPI_REG_CONFIG_ENABLE_MASK;
817 
818 	writel(reg, reg_base + CQSPI_REG_CONFIG);
819 }
820 
821 static int cqspi_versal_indirect_read_dma(struct cqspi_flash_pdata *f_pdata,
822 					  u_char *rxbuf, loff_t from_addr,
823 					  size_t n_rx)
824 {
825 	struct cqspi_st *cqspi = f_pdata->cqspi;
826 	struct device *dev = &cqspi->pdev->dev;
827 	void __iomem *reg_base = cqspi->iobase;
828 	u32 reg, bytes_to_dma;
829 	loff_t addr = from_addr;
830 	void *buf = rxbuf;
831 	dma_addr_t dma_addr;
832 	u8 bytes_rem;
833 	int ret = 0;
834 
835 	bytes_rem = n_rx % 4;
836 	bytes_to_dma = (n_rx - bytes_rem);
837 
838 	if (!bytes_to_dma)
839 		goto nondmard;
840 
841 	ret = zynqmp_pm_ospi_mux_select(cqspi->pd_dev_id, PM_OSPI_MUX_SEL_DMA);
842 	if (ret)
843 		return ret;
844 
845 	cqspi_controller_enable(cqspi, 0);
846 
847 	reg = readl(cqspi->iobase + CQSPI_REG_CONFIG);
848 	reg |= CQSPI_REG_CONFIG_DMA_MASK;
849 	writel(reg, cqspi->iobase + CQSPI_REG_CONFIG);
850 
851 	cqspi_controller_enable(cqspi, 1);
852 
853 	dma_addr = dma_map_single(dev, rxbuf, bytes_to_dma, DMA_FROM_DEVICE);
854 	if (dma_mapping_error(dev, dma_addr)) {
855 		dev_err(dev, "dma mapping failed\n");
856 		return -ENOMEM;
857 	}
858 
859 	writel(from_addr, reg_base + CQSPI_REG_INDIRECTRDSTARTADDR);
860 	writel(bytes_to_dma, reg_base + CQSPI_REG_INDIRECTRDBYTES);
861 	writel(CQSPI_REG_VERSAL_ADDRRANGE_WIDTH_VAL,
862 	       reg_base + CQSPI_REG_INDTRIG_ADDRRANGE);
863 
864 	/* Clear all interrupts. */
865 	writel(CQSPI_IRQ_STATUS_MASK, reg_base + CQSPI_REG_IRQSTATUS);
866 
867 	/* Enable DMA done interrupt */
868 	writel(CQSPI_REG_VERSAL_DMA_DST_DONE_MASK,
869 	       reg_base + CQSPI_REG_VERSAL_DMA_DST_I_EN);
870 
871 	/* Default DMA periph configuration */
872 	writel(CQSPI_REG_VERSAL_DMA_VAL, reg_base + CQSPI_REG_DMA);
873 
874 	/* Configure DMA Dst address */
875 	writel(lower_32_bits(dma_addr),
876 	       reg_base + CQSPI_REG_VERSAL_DMA_DST_ADDR);
877 	writel(upper_32_bits(dma_addr),
878 	       reg_base + CQSPI_REG_VERSAL_DMA_DST_ADDR_MSB);
879 
880 	/* Configure DMA Src address */
881 	writel(cqspi->trigger_address, reg_base +
882 	       CQSPI_REG_VERSAL_DMA_SRC_ADDR);
883 
884 	/* Set DMA destination size */
885 	writel(bytes_to_dma, reg_base + CQSPI_REG_VERSAL_DMA_DST_SIZE);
886 
887 	/* Set DMA destination control */
888 	writel(CQSPI_REG_VERSAL_DMA_DST_CTRL_VAL,
889 	       reg_base + CQSPI_REG_VERSAL_DMA_DST_CTRL);
890 
891 	writel(CQSPI_REG_INDIRECTRD_START_MASK,
892 	       reg_base + CQSPI_REG_INDIRECTRD);
893 
894 	reinit_completion(&cqspi->transfer_complete);
895 
896 	if (!wait_for_completion_timeout(&cqspi->transfer_complete,
897 					 msecs_to_jiffies(max_t(size_t, bytes_to_dma, 500)))) {
898 		ret = -ETIMEDOUT;
899 		goto failrd;
900 	}
901 
902 	/* Disable DMA interrupt */
903 	writel(0x0, cqspi->iobase + CQSPI_REG_VERSAL_DMA_DST_I_DIS);
904 
905 	/* Clear indirect completion status */
906 	writel(CQSPI_REG_INDIRECTRD_DONE_MASK,
907 	       cqspi->iobase + CQSPI_REG_INDIRECTRD);
908 	dma_unmap_single(dev, dma_addr, bytes_to_dma, DMA_FROM_DEVICE);
909 
910 	cqspi_controller_enable(cqspi, 0);
911 
912 	reg = readl(cqspi->iobase + CQSPI_REG_CONFIG);
913 	reg &= ~CQSPI_REG_CONFIG_DMA_MASK;
914 	writel(reg, cqspi->iobase + CQSPI_REG_CONFIG);
915 
916 	cqspi_controller_enable(cqspi, 1);
917 
918 	ret = zynqmp_pm_ospi_mux_select(cqspi->pd_dev_id,
919 					PM_OSPI_MUX_SEL_LINEAR);
920 	if (ret)
921 		return ret;
922 
923 nondmard:
924 	if (bytes_rem) {
925 		addr += bytes_to_dma;
926 		buf += bytes_to_dma;
927 		ret = cqspi_indirect_read_execute(f_pdata, buf, addr,
928 						  bytes_rem);
929 		if (ret)
930 			return ret;
931 	}
932 
933 	return 0;
934 
935 failrd:
936 	/* Disable DMA interrupt */
937 	writel(0x0, reg_base + CQSPI_REG_VERSAL_DMA_DST_I_DIS);
938 
939 	/* Cancel the indirect read */
940 	writel(CQSPI_REG_INDIRECTWR_CANCEL_MASK,
941 	       reg_base + CQSPI_REG_INDIRECTRD);
942 
943 	dma_unmap_single(dev, dma_addr, bytes_to_dma, DMA_FROM_DEVICE);
944 
945 	reg = readl(cqspi->iobase + CQSPI_REG_CONFIG);
946 	reg &= ~CQSPI_REG_CONFIG_DMA_MASK;
947 	writel(reg, cqspi->iobase + CQSPI_REG_CONFIG);
948 
949 	zynqmp_pm_ospi_mux_select(cqspi->pd_dev_id, PM_OSPI_MUX_SEL_LINEAR);
950 
951 	return ret;
952 }
953 
954 static int cqspi_write_setup(struct cqspi_flash_pdata *f_pdata,
955 			     const struct spi_mem_op *op)
956 {
957 	unsigned int reg;
958 	int ret;
959 	struct cqspi_st *cqspi = f_pdata->cqspi;
960 	void __iomem *reg_base = cqspi->iobase;
961 	u8 opcode;
962 
963 	ret = cqspi_enable_dtr(f_pdata, op, CQSPI_REG_OP_EXT_WRITE_LSB);
964 	if (ret)
965 		return ret;
966 
967 	if (op->cmd.dtr)
968 		opcode = op->cmd.opcode >> 8;
969 	else
970 		opcode = op->cmd.opcode;
971 
972 	/* Set opcode. */
973 	reg = opcode << CQSPI_REG_WR_INSTR_OPCODE_LSB;
974 	reg |= CQSPI_OP_WIDTH(op->data) << CQSPI_REG_WR_INSTR_TYPE_DATA_LSB;
975 	reg |= CQSPI_OP_WIDTH(op->addr) << CQSPI_REG_WR_INSTR_TYPE_ADDR_LSB;
976 	writel(reg, reg_base + CQSPI_REG_WR_INSTR);
977 	reg = cqspi_calc_rdreg(op);
978 	writel(reg, reg_base + CQSPI_REG_RD_INSTR);
979 
980 	/*
981 	 * SPI NAND flashes require the address of the status register to be
982 	 * passed in the Read SR command. Also, some SPI NOR flashes like the
983 	 * cypress Semper flash expect a 4-byte dummy address in the Read SR
984 	 * command in DTR mode.
985 	 *
986 	 * But this controller does not support address phase in the Read SR
987 	 * command when doing auto-HW polling. So, disable write completion
988 	 * polling on the controller's side. spinand and spi-nor will take
989 	 * care of polling the status register.
990 	 */
991 	if (cqspi->wr_completion) {
992 		reg = readl(reg_base + CQSPI_REG_WR_COMPLETION_CTRL);
993 		reg |= CQSPI_REG_WR_DISABLE_AUTO_POLL;
994 		writel(reg, reg_base + CQSPI_REG_WR_COMPLETION_CTRL);
995 		/*
996 		 * DAC mode require auto polling as flash needs to be polled
997 		 * for write completion in case of bubble in SPI transaction
998 		 * due to slow CPU/DMA master.
999 		 */
1000 		cqspi->use_direct_mode_wr = false;
1001 	}
1002 
1003 	reg = readl(reg_base + CQSPI_REG_SIZE);
1004 	reg &= ~CQSPI_REG_SIZE_ADDRESS_MASK;
1005 	reg |= (op->addr.nbytes - 1);
1006 	writel(reg, reg_base + CQSPI_REG_SIZE);
1007 	return 0;
1008 }
1009 
1010 static int cqspi_indirect_write_execute(struct cqspi_flash_pdata *f_pdata,
1011 					loff_t to_addr, const u8 *txbuf,
1012 					const size_t n_tx)
1013 {
1014 	struct cqspi_st *cqspi = f_pdata->cqspi;
1015 	struct device *dev = &cqspi->pdev->dev;
1016 	void __iomem *reg_base = cqspi->iobase;
1017 	unsigned int remaining = n_tx;
1018 	unsigned int write_bytes;
1019 	int ret;
1020 
1021 	writel(to_addr, reg_base + CQSPI_REG_INDIRECTWRSTARTADDR);
1022 	writel(remaining, reg_base + CQSPI_REG_INDIRECTWRBYTES);
1023 
1024 	/* Clear all interrupts. */
1025 	writel(CQSPI_IRQ_STATUS_MASK, reg_base + CQSPI_REG_IRQSTATUS);
1026 
1027 	writel(CQSPI_IRQ_MASK_WR, reg_base + CQSPI_REG_IRQMASK);
1028 
1029 	reinit_completion(&cqspi->transfer_complete);
1030 	writel(CQSPI_REG_INDIRECTWR_START_MASK,
1031 	       reg_base + CQSPI_REG_INDIRECTWR);
1032 	/*
1033 	 * As per 66AK2G02 TRM SPRUHY8F section 11.15.5.3 Indirect Access
1034 	 * Controller programming sequence, couple of cycles of
1035 	 * QSPI_REF_CLK delay is required for the above bit to
1036 	 * be internally synchronized by the QSPI module. Provide 5
1037 	 * cycles of delay.
1038 	 */
1039 	if (cqspi->wr_delay)
1040 		ndelay(cqspi->wr_delay);
1041 
1042 	/*
1043 	 * If a hazard exists between the APB and AHB interfaces, perform a
1044 	 * dummy readback from the controller to ensure synchronization.
1045 	 */
1046 	if (cqspi->apb_ahb_hazard)
1047 		readl(reg_base + CQSPI_REG_INDIRECTWR);
1048 
1049 	while (remaining > 0) {
1050 		size_t write_words, mod_bytes;
1051 
1052 		write_bytes = remaining;
1053 		write_words = write_bytes / 4;
1054 		mod_bytes = write_bytes % 4;
1055 		/* Write 4 bytes at a time then single bytes. */
1056 		if (write_words) {
1057 			iowrite32_rep(cqspi->ahb_base, txbuf, write_words);
1058 			txbuf += (write_words * 4);
1059 		}
1060 		if (mod_bytes) {
1061 			unsigned int temp = 0xFFFFFFFF;
1062 
1063 			memcpy(&temp, txbuf, mod_bytes);
1064 			iowrite32(temp, cqspi->ahb_base);
1065 			txbuf += mod_bytes;
1066 		}
1067 
1068 		if (!wait_for_completion_timeout(&cqspi->transfer_complete,
1069 						 msecs_to_jiffies(CQSPI_TIMEOUT_MS))) {
1070 			dev_err(dev, "Indirect write timeout\n");
1071 			ret = -ETIMEDOUT;
1072 			goto failwr;
1073 		}
1074 
1075 		remaining -= write_bytes;
1076 
1077 		if (remaining > 0)
1078 			reinit_completion(&cqspi->transfer_complete);
1079 	}
1080 
1081 	/* Check indirect done status */
1082 	ret = cqspi_wait_for_bit(reg_base + CQSPI_REG_INDIRECTWR,
1083 				 CQSPI_REG_INDIRECTWR_DONE_MASK, 0);
1084 	if (ret) {
1085 		dev_err(dev, "Indirect write completion error (%i)\n", ret);
1086 		goto failwr;
1087 	}
1088 
1089 	/* Disable interrupt. */
1090 	writel(0, reg_base + CQSPI_REG_IRQMASK);
1091 
1092 	/* Clear indirect completion status */
1093 	writel(CQSPI_REG_INDIRECTWR_DONE_MASK, reg_base + CQSPI_REG_INDIRECTWR);
1094 
1095 	cqspi_wait_idle(cqspi);
1096 
1097 	return 0;
1098 
1099 failwr:
1100 	/* Disable interrupt. */
1101 	writel(0, reg_base + CQSPI_REG_IRQMASK);
1102 
1103 	/* Cancel the indirect write */
1104 	writel(CQSPI_REG_INDIRECTWR_CANCEL_MASK,
1105 	       reg_base + CQSPI_REG_INDIRECTWR);
1106 	return ret;
1107 }
1108 
1109 static void cqspi_chipselect(struct cqspi_flash_pdata *f_pdata)
1110 {
1111 	struct cqspi_st *cqspi = f_pdata->cqspi;
1112 	void __iomem *reg_base = cqspi->iobase;
1113 	unsigned int chip_select = f_pdata->cs;
1114 	unsigned int reg;
1115 
1116 	reg = readl(reg_base + CQSPI_REG_CONFIG);
1117 	if (cqspi->is_decoded_cs) {
1118 		reg |= CQSPI_REG_CONFIG_DECODE_MASK;
1119 	} else {
1120 		reg &= ~CQSPI_REG_CONFIG_DECODE_MASK;
1121 
1122 		/* Convert CS if without decoder.
1123 		 * CS0 to 4b'1110
1124 		 * CS1 to 4b'1101
1125 		 * CS2 to 4b'1011
1126 		 * CS3 to 4b'0111
1127 		 */
1128 		chip_select = 0xF & ~(1 << chip_select);
1129 	}
1130 
1131 	reg &= ~(CQSPI_REG_CONFIG_CHIPSELECT_MASK
1132 		 << CQSPI_REG_CONFIG_CHIPSELECT_LSB);
1133 	reg |= (chip_select & CQSPI_REG_CONFIG_CHIPSELECT_MASK)
1134 	    << CQSPI_REG_CONFIG_CHIPSELECT_LSB;
1135 	writel(reg, reg_base + CQSPI_REG_CONFIG);
1136 }
1137 
1138 static unsigned int calculate_ticks_for_ns(const unsigned int ref_clk_hz,
1139 					   const unsigned int ns_val)
1140 {
1141 	unsigned int ticks;
1142 
1143 	ticks = ref_clk_hz / 1000;	/* kHz */
1144 	ticks = DIV_ROUND_UP(ticks * ns_val, 1000000);
1145 
1146 	return ticks;
1147 }
1148 
1149 static void cqspi_delay(struct cqspi_flash_pdata *f_pdata)
1150 {
1151 	struct cqspi_st *cqspi = f_pdata->cqspi;
1152 	void __iomem *iobase = cqspi->iobase;
1153 	const unsigned int ref_clk_hz = cqspi->master_ref_clk_hz;
1154 	unsigned int tshsl, tchsh, tslch, tsd2d;
1155 	unsigned int reg;
1156 	unsigned int tsclk;
1157 
1158 	/* calculate the number of ref ticks for one sclk tick */
1159 	tsclk = DIV_ROUND_UP(ref_clk_hz, cqspi->sclk);
1160 
1161 	tshsl = calculate_ticks_for_ns(ref_clk_hz, f_pdata->tshsl_ns);
1162 	/* this particular value must be at least one sclk */
1163 	if (tshsl < tsclk)
1164 		tshsl = tsclk;
1165 
1166 	tchsh = calculate_ticks_for_ns(ref_clk_hz, f_pdata->tchsh_ns);
1167 	tslch = calculate_ticks_for_ns(ref_clk_hz, f_pdata->tslch_ns);
1168 	tsd2d = calculate_ticks_for_ns(ref_clk_hz, f_pdata->tsd2d_ns);
1169 
1170 	reg = (tshsl & CQSPI_REG_DELAY_TSHSL_MASK)
1171 	       << CQSPI_REG_DELAY_TSHSL_LSB;
1172 	reg |= (tchsh & CQSPI_REG_DELAY_TCHSH_MASK)
1173 		<< CQSPI_REG_DELAY_TCHSH_LSB;
1174 	reg |= (tslch & CQSPI_REG_DELAY_TSLCH_MASK)
1175 		<< CQSPI_REG_DELAY_TSLCH_LSB;
1176 	reg |= (tsd2d & CQSPI_REG_DELAY_TSD2D_MASK)
1177 		<< CQSPI_REG_DELAY_TSD2D_LSB;
1178 	writel(reg, iobase + CQSPI_REG_DELAY);
1179 }
1180 
1181 static void cqspi_config_baudrate_div(struct cqspi_st *cqspi)
1182 {
1183 	const unsigned int ref_clk_hz = cqspi->master_ref_clk_hz;
1184 	void __iomem *reg_base = cqspi->iobase;
1185 	u32 reg, div;
1186 
1187 	/* Recalculate the baudrate divisor based on QSPI specification. */
1188 	div = DIV_ROUND_UP(ref_clk_hz, 2 * cqspi->sclk) - 1;
1189 
1190 	/* Maximum baud divisor */
1191 	if (div > CQSPI_REG_CONFIG_BAUD_MASK) {
1192 		div = CQSPI_REG_CONFIG_BAUD_MASK;
1193 		dev_warn(&cqspi->pdev->dev,
1194 			"Unable to adjust clock <= %d hz. Reduced to %d hz\n",
1195 			cqspi->sclk, ref_clk_hz/((div+1)*2));
1196 	}
1197 
1198 	reg = readl(reg_base + CQSPI_REG_CONFIG);
1199 	reg &= ~(CQSPI_REG_CONFIG_BAUD_MASK << CQSPI_REG_CONFIG_BAUD_LSB);
1200 	reg |= (div & CQSPI_REG_CONFIG_BAUD_MASK) << CQSPI_REG_CONFIG_BAUD_LSB;
1201 	writel(reg, reg_base + CQSPI_REG_CONFIG);
1202 }
1203 
1204 static void cqspi_readdata_capture(struct cqspi_st *cqspi,
1205 				   const bool bypass,
1206 				   const unsigned int delay)
1207 {
1208 	void __iomem *reg_base = cqspi->iobase;
1209 	unsigned int reg;
1210 
1211 	reg = readl(reg_base + CQSPI_REG_READCAPTURE);
1212 
1213 	if (bypass)
1214 		reg |= (1 << CQSPI_REG_READCAPTURE_BYPASS_LSB);
1215 	else
1216 		reg &= ~(1 << CQSPI_REG_READCAPTURE_BYPASS_LSB);
1217 
1218 	reg &= ~(CQSPI_REG_READCAPTURE_DELAY_MASK
1219 		 << CQSPI_REG_READCAPTURE_DELAY_LSB);
1220 
1221 	reg |= (delay & CQSPI_REG_READCAPTURE_DELAY_MASK)
1222 		<< CQSPI_REG_READCAPTURE_DELAY_LSB;
1223 
1224 	writel(reg, reg_base + CQSPI_REG_READCAPTURE);
1225 }
1226 
1227 static void cqspi_configure(struct cqspi_flash_pdata *f_pdata,
1228 			    unsigned long sclk)
1229 {
1230 	struct cqspi_st *cqspi = f_pdata->cqspi;
1231 	int switch_cs = (cqspi->current_cs != f_pdata->cs);
1232 	int switch_ck = (cqspi->sclk != sclk);
1233 
1234 	if (switch_cs || switch_ck)
1235 		cqspi_controller_enable(cqspi, 0);
1236 
1237 	/* Switch chip select. */
1238 	if (switch_cs) {
1239 		cqspi->current_cs = f_pdata->cs;
1240 		cqspi_chipselect(f_pdata);
1241 	}
1242 
1243 	/* Setup baudrate divisor and delays */
1244 	if (switch_ck) {
1245 		cqspi->sclk = sclk;
1246 		cqspi_config_baudrate_div(cqspi);
1247 		cqspi_delay(f_pdata);
1248 		cqspi_readdata_capture(cqspi, !cqspi->rclk_en,
1249 				       f_pdata->read_delay);
1250 	}
1251 
1252 	if (switch_cs || switch_ck)
1253 		cqspi_controller_enable(cqspi, 1);
1254 }
1255 
1256 static ssize_t cqspi_write(struct cqspi_flash_pdata *f_pdata,
1257 			   const struct spi_mem_op *op)
1258 {
1259 	struct cqspi_st *cqspi = f_pdata->cqspi;
1260 	loff_t to = op->addr.val;
1261 	size_t len = op->data.nbytes;
1262 	const u_char *buf = op->data.buf.out;
1263 	int ret;
1264 
1265 	ret = cqspi_write_setup(f_pdata, op);
1266 	if (ret)
1267 		return ret;
1268 
1269 	/*
1270 	 * Some flashes like the Cypress Semper flash expect a dummy 4-byte
1271 	 * address (all 0s) with the read status register command in DTR mode.
1272 	 * But this controller does not support sending dummy address bytes to
1273 	 * the flash when it is polling the write completion register in DTR
1274 	 * mode. So, we can not use direct mode when in DTR mode for writing
1275 	 * data.
1276 	 */
1277 	if (!op->cmd.dtr && cqspi->use_direct_mode &&
1278 	    cqspi->use_direct_mode_wr && ((to + len) <= cqspi->ahb_size)) {
1279 		memcpy_toio(cqspi->ahb_base + to, buf, len);
1280 		return cqspi_wait_idle(cqspi);
1281 	}
1282 
1283 	return cqspi_indirect_write_execute(f_pdata, to, buf, len);
1284 }
1285 
1286 static void cqspi_rx_dma_callback(void *param)
1287 {
1288 	struct cqspi_st *cqspi = param;
1289 
1290 	complete(&cqspi->rx_dma_complete);
1291 }
1292 
1293 static int cqspi_direct_read_execute(struct cqspi_flash_pdata *f_pdata,
1294 				     u_char *buf, loff_t from, size_t len)
1295 {
1296 	struct cqspi_st *cqspi = f_pdata->cqspi;
1297 	struct device *dev = &cqspi->pdev->dev;
1298 	enum dma_ctrl_flags flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
1299 	dma_addr_t dma_src = (dma_addr_t)cqspi->mmap_phys_base + from;
1300 	int ret = 0;
1301 	struct dma_async_tx_descriptor *tx;
1302 	dma_cookie_t cookie;
1303 	dma_addr_t dma_dst;
1304 	struct device *ddev;
1305 
1306 	if (!cqspi->rx_chan || !virt_addr_valid(buf)) {
1307 		memcpy_fromio(buf, cqspi->ahb_base + from, len);
1308 		return 0;
1309 	}
1310 
1311 	ddev = cqspi->rx_chan->device->dev;
1312 	dma_dst = dma_map_single(ddev, buf, len, DMA_FROM_DEVICE);
1313 	if (dma_mapping_error(ddev, dma_dst)) {
1314 		dev_err(dev, "dma mapping failed\n");
1315 		return -ENOMEM;
1316 	}
1317 	tx = dmaengine_prep_dma_memcpy(cqspi->rx_chan, dma_dst, dma_src,
1318 				       len, flags);
1319 	if (!tx) {
1320 		dev_err(dev, "device_prep_dma_memcpy error\n");
1321 		ret = -EIO;
1322 		goto err_unmap;
1323 	}
1324 
1325 	tx->callback = cqspi_rx_dma_callback;
1326 	tx->callback_param = cqspi;
1327 	cookie = tx->tx_submit(tx);
1328 	reinit_completion(&cqspi->rx_dma_complete);
1329 
1330 	ret = dma_submit_error(cookie);
1331 	if (ret) {
1332 		dev_err(dev, "dma_submit_error %d\n", cookie);
1333 		ret = -EIO;
1334 		goto err_unmap;
1335 	}
1336 
1337 	dma_async_issue_pending(cqspi->rx_chan);
1338 	if (!wait_for_completion_timeout(&cqspi->rx_dma_complete,
1339 					 msecs_to_jiffies(max_t(size_t, len, 500)))) {
1340 		dmaengine_terminate_sync(cqspi->rx_chan);
1341 		dev_err(dev, "DMA wait_for_completion_timeout\n");
1342 		ret = -ETIMEDOUT;
1343 		goto err_unmap;
1344 	}
1345 
1346 err_unmap:
1347 	dma_unmap_single(ddev, dma_dst, len, DMA_FROM_DEVICE);
1348 
1349 	return ret;
1350 }
1351 
1352 static ssize_t cqspi_read(struct cqspi_flash_pdata *f_pdata,
1353 			  const struct spi_mem_op *op)
1354 {
1355 	struct cqspi_st *cqspi = f_pdata->cqspi;
1356 	struct device *dev = &cqspi->pdev->dev;
1357 	const struct cqspi_driver_platdata *ddata;
1358 	loff_t from = op->addr.val;
1359 	size_t len = op->data.nbytes;
1360 	u_char *buf = op->data.buf.in;
1361 	u64 dma_align = (u64)(uintptr_t)buf;
1362 	int ret;
1363 
1364 	ddata = of_device_get_match_data(dev);
1365 
1366 	ret = cqspi_read_setup(f_pdata, op);
1367 	if (ret)
1368 		return ret;
1369 
1370 	if (cqspi->use_direct_mode && ((from + len) <= cqspi->ahb_size))
1371 		return cqspi_direct_read_execute(f_pdata, buf, from, len);
1372 
1373 	if (cqspi->use_dma_read && ddata && ddata->indirect_read_dma &&
1374 	    virt_addr_valid(buf) && ((dma_align & CQSPI_DMA_UNALIGN) == 0))
1375 		return ddata->indirect_read_dma(f_pdata, buf, from, len);
1376 
1377 	return cqspi_indirect_read_execute(f_pdata, buf, from, len);
1378 }
1379 
1380 static int cqspi_mem_process(struct spi_mem *mem, const struct spi_mem_op *op)
1381 {
1382 	struct cqspi_st *cqspi = spi_controller_get_devdata(mem->spi->controller);
1383 	struct cqspi_flash_pdata *f_pdata;
1384 
1385 	f_pdata = &cqspi->f_pdata[spi_get_chipselect(mem->spi, 0)];
1386 	cqspi_configure(f_pdata, mem->spi->max_speed_hz);
1387 
1388 	if (op->data.dir == SPI_MEM_DATA_IN && op->data.buf.in) {
1389 	/*
1390 	 * Performing reads in DAC mode forces to read minimum 4 bytes
1391 	 * which is unsupported on some flash devices during register
1392 	 * reads, prefer STIG mode for such small reads.
1393 	 */
1394 		if (!op->addr.nbytes ||
1395 		    op->data.nbytes <= CQSPI_STIG_DATA_LEN_MAX)
1396 			return cqspi_command_read(f_pdata, op);
1397 
1398 		return cqspi_read(f_pdata, op);
1399 	}
1400 
1401 	if (!op->addr.nbytes || !op->data.buf.out)
1402 		return cqspi_command_write(f_pdata, op);
1403 
1404 	return cqspi_write(f_pdata, op);
1405 }
1406 
1407 static int cqspi_exec_mem_op(struct spi_mem *mem, const struct spi_mem_op *op)
1408 {
1409 	int ret;
1410 
1411 	ret = cqspi_mem_process(mem, op);
1412 	if (ret)
1413 		dev_err(&mem->spi->dev, "operation failed with %d\n", ret);
1414 
1415 	return ret;
1416 }
1417 
1418 static bool cqspi_supports_mem_op(struct spi_mem *mem,
1419 				  const struct spi_mem_op *op)
1420 {
1421 	bool all_true, all_false;
1422 
1423 	/*
1424 	 * op->dummy.dtr is required for converting nbytes into ncycles.
1425 	 * Also, don't check the dtr field of the op phase having zero nbytes.
1426 	 */
1427 	all_true = op->cmd.dtr &&
1428 		   (!op->addr.nbytes || op->addr.dtr) &&
1429 		   (!op->dummy.nbytes || op->dummy.dtr) &&
1430 		   (!op->data.nbytes || op->data.dtr);
1431 
1432 	all_false = !op->cmd.dtr && !op->addr.dtr && !op->dummy.dtr &&
1433 		    !op->data.dtr;
1434 
1435 	if (all_true) {
1436 		/* Right now we only support 8-8-8 DTR mode. */
1437 		if (op->cmd.nbytes && op->cmd.buswidth != 8)
1438 			return false;
1439 		if (op->addr.nbytes && op->addr.buswidth != 8)
1440 			return false;
1441 		if (op->data.nbytes && op->data.buswidth != 8)
1442 			return false;
1443 	} else if (!all_false) {
1444 		/* Mixed DTR modes are not supported. */
1445 		return false;
1446 	}
1447 
1448 	return spi_mem_default_supports_op(mem, op);
1449 }
1450 
1451 static int cqspi_of_get_flash_pdata(struct platform_device *pdev,
1452 				    struct cqspi_flash_pdata *f_pdata,
1453 				    struct device_node *np)
1454 {
1455 	if (of_property_read_u32(np, "cdns,read-delay", &f_pdata->read_delay)) {
1456 		dev_err(&pdev->dev, "couldn't determine read-delay\n");
1457 		return -ENXIO;
1458 	}
1459 
1460 	if (of_property_read_u32(np, "cdns,tshsl-ns", &f_pdata->tshsl_ns)) {
1461 		dev_err(&pdev->dev, "couldn't determine tshsl-ns\n");
1462 		return -ENXIO;
1463 	}
1464 
1465 	if (of_property_read_u32(np, "cdns,tsd2d-ns", &f_pdata->tsd2d_ns)) {
1466 		dev_err(&pdev->dev, "couldn't determine tsd2d-ns\n");
1467 		return -ENXIO;
1468 	}
1469 
1470 	if (of_property_read_u32(np, "cdns,tchsh-ns", &f_pdata->tchsh_ns)) {
1471 		dev_err(&pdev->dev, "couldn't determine tchsh-ns\n");
1472 		return -ENXIO;
1473 	}
1474 
1475 	if (of_property_read_u32(np, "cdns,tslch-ns", &f_pdata->tslch_ns)) {
1476 		dev_err(&pdev->dev, "couldn't determine tslch-ns\n");
1477 		return -ENXIO;
1478 	}
1479 
1480 	if (of_property_read_u32(np, "spi-max-frequency", &f_pdata->clk_rate)) {
1481 		dev_err(&pdev->dev, "couldn't determine spi-max-frequency\n");
1482 		return -ENXIO;
1483 	}
1484 
1485 	return 0;
1486 }
1487 
1488 static int cqspi_of_get_pdata(struct cqspi_st *cqspi)
1489 {
1490 	struct device *dev = &cqspi->pdev->dev;
1491 	struct device_node *np = dev->of_node;
1492 	u32 id[2];
1493 
1494 	cqspi->is_decoded_cs = of_property_read_bool(np, "cdns,is-decoded-cs");
1495 
1496 	if (of_property_read_u32(np, "cdns,fifo-depth", &cqspi->fifo_depth)) {
1497 		dev_err(dev, "couldn't determine fifo-depth\n");
1498 		return -ENXIO;
1499 	}
1500 
1501 	if (of_property_read_u32(np, "cdns,fifo-width", &cqspi->fifo_width)) {
1502 		dev_err(dev, "couldn't determine fifo-width\n");
1503 		return -ENXIO;
1504 	}
1505 
1506 	if (of_property_read_u32(np, "cdns,trigger-address",
1507 				 &cqspi->trigger_address)) {
1508 		dev_err(dev, "couldn't determine trigger-address\n");
1509 		return -ENXIO;
1510 	}
1511 
1512 	if (of_property_read_u32(np, "num-cs", &cqspi->num_chipselect))
1513 		cqspi->num_chipselect = CQSPI_MAX_CHIPSELECT;
1514 
1515 	cqspi->rclk_en = of_property_read_bool(np, "cdns,rclk-en");
1516 
1517 	if (!of_property_read_u32_array(np, "power-domains", id,
1518 					ARRAY_SIZE(id)))
1519 		cqspi->pd_dev_id = id[1];
1520 
1521 	return 0;
1522 }
1523 
1524 static void cqspi_controller_init(struct cqspi_st *cqspi)
1525 {
1526 	u32 reg;
1527 
1528 	cqspi_controller_enable(cqspi, 0);
1529 
1530 	/* Configure the remap address register, no remap */
1531 	writel(0, cqspi->iobase + CQSPI_REG_REMAP);
1532 
1533 	/* Disable all interrupts. */
1534 	writel(0, cqspi->iobase + CQSPI_REG_IRQMASK);
1535 
1536 	/* Configure the SRAM split to 1:1 . */
1537 	writel(cqspi->fifo_depth / 2, cqspi->iobase + CQSPI_REG_SRAMPARTITION);
1538 
1539 	/* Load indirect trigger address. */
1540 	writel(cqspi->trigger_address,
1541 	       cqspi->iobase + CQSPI_REG_INDIRECTTRIGGER);
1542 
1543 	/* Program read watermark -- 1/2 of the FIFO. */
1544 	writel(cqspi->fifo_depth * cqspi->fifo_width / 2,
1545 	       cqspi->iobase + CQSPI_REG_INDIRECTRDWATERMARK);
1546 	/* Program write watermark -- 1/8 of the FIFO. */
1547 	writel(cqspi->fifo_depth * cqspi->fifo_width / 8,
1548 	       cqspi->iobase + CQSPI_REG_INDIRECTWRWATERMARK);
1549 
1550 	/* Disable direct access controller */
1551 	if (!cqspi->use_direct_mode) {
1552 		reg = readl(cqspi->iobase + CQSPI_REG_CONFIG);
1553 		reg &= ~CQSPI_REG_CONFIG_ENB_DIR_ACC_CTRL;
1554 		writel(reg, cqspi->iobase + CQSPI_REG_CONFIG);
1555 	}
1556 
1557 	/* Enable DMA interface */
1558 	if (cqspi->use_dma_read) {
1559 		reg = readl(cqspi->iobase + CQSPI_REG_CONFIG);
1560 		reg |= CQSPI_REG_CONFIG_DMA_MASK;
1561 		writel(reg, cqspi->iobase + CQSPI_REG_CONFIG);
1562 	}
1563 
1564 	cqspi_controller_enable(cqspi, 1);
1565 }
1566 
1567 static int cqspi_request_mmap_dma(struct cqspi_st *cqspi)
1568 {
1569 	dma_cap_mask_t mask;
1570 
1571 	dma_cap_zero(mask);
1572 	dma_cap_set(DMA_MEMCPY, mask);
1573 
1574 	cqspi->rx_chan = dma_request_chan_by_mask(&mask);
1575 	if (IS_ERR(cqspi->rx_chan)) {
1576 		int ret = PTR_ERR(cqspi->rx_chan);
1577 
1578 		cqspi->rx_chan = NULL;
1579 		return dev_err_probe(&cqspi->pdev->dev, ret, "No Rx DMA available\n");
1580 	}
1581 	init_completion(&cqspi->rx_dma_complete);
1582 
1583 	return 0;
1584 }
1585 
1586 static const char *cqspi_get_name(struct spi_mem *mem)
1587 {
1588 	struct cqspi_st *cqspi = spi_controller_get_devdata(mem->spi->controller);
1589 	struct device *dev = &cqspi->pdev->dev;
1590 
1591 	return devm_kasprintf(dev, GFP_KERNEL, "%s.%d", dev_name(dev),
1592 			      spi_get_chipselect(mem->spi, 0));
1593 }
1594 
1595 static const struct spi_controller_mem_ops cqspi_mem_ops = {
1596 	.exec_op = cqspi_exec_mem_op,
1597 	.get_name = cqspi_get_name,
1598 	.supports_op = cqspi_supports_mem_op,
1599 };
1600 
1601 static const struct spi_controller_mem_caps cqspi_mem_caps = {
1602 	.dtr = true,
1603 };
1604 
1605 static int cqspi_setup_flash(struct cqspi_st *cqspi)
1606 {
1607 	struct platform_device *pdev = cqspi->pdev;
1608 	struct device *dev = &pdev->dev;
1609 	struct device_node *np = dev->of_node;
1610 	struct cqspi_flash_pdata *f_pdata;
1611 	unsigned int cs;
1612 	int ret;
1613 
1614 	/* Get flash device data */
1615 	for_each_available_child_of_node(dev->of_node, np) {
1616 		ret = of_property_read_u32(np, "reg", &cs);
1617 		if (ret) {
1618 			dev_err(dev, "Couldn't determine chip select.\n");
1619 			of_node_put(np);
1620 			return ret;
1621 		}
1622 
1623 		if (cs >= CQSPI_MAX_CHIPSELECT) {
1624 			dev_err(dev, "Chip select %d out of range.\n", cs);
1625 			of_node_put(np);
1626 			return -EINVAL;
1627 		}
1628 
1629 		f_pdata = &cqspi->f_pdata[cs];
1630 		f_pdata->cqspi = cqspi;
1631 		f_pdata->cs = cs;
1632 
1633 		ret = cqspi_of_get_flash_pdata(pdev, f_pdata, np);
1634 		if (ret) {
1635 			of_node_put(np);
1636 			return ret;
1637 		}
1638 	}
1639 
1640 	return 0;
1641 }
1642 
1643 static int cqspi_jh7110_clk_init(struct platform_device *pdev, struct cqspi_st *cqspi)
1644 {
1645 	static struct clk_bulk_data qspiclk[] = {
1646 		{ .id = "apb" },
1647 		{ .id = "ahb" },
1648 	};
1649 
1650 	int ret = 0;
1651 
1652 	ret = devm_clk_bulk_get(&pdev->dev, ARRAY_SIZE(qspiclk), qspiclk);
1653 	if (ret) {
1654 		dev_err(&pdev->dev, "%s: failed to get qspi clocks\n", __func__);
1655 		return ret;
1656 	}
1657 
1658 	cqspi->clks[CLK_QSPI_APB] = qspiclk[0].clk;
1659 	cqspi->clks[CLK_QSPI_AHB] = qspiclk[1].clk;
1660 
1661 	ret = clk_prepare_enable(cqspi->clks[CLK_QSPI_APB]);
1662 	if (ret) {
1663 		dev_err(&pdev->dev, "%s: failed to enable CLK_QSPI_APB\n", __func__);
1664 		return ret;
1665 	}
1666 
1667 	ret = clk_prepare_enable(cqspi->clks[CLK_QSPI_AHB]);
1668 	if (ret) {
1669 		dev_err(&pdev->dev, "%s: failed to enable CLK_QSPI_AHB\n", __func__);
1670 		goto disable_apb_clk;
1671 	}
1672 
1673 	cqspi->is_jh7110 = true;
1674 
1675 	return 0;
1676 
1677 disable_apb_clk:
1678 	clk_disable_unprepare(cqspi->clks[CLK_QSPI_APB]);
1679 
1680 	return ret;
1681 }
1682 
1683 static void cqspi_jh7110_disable_clk(struct platform_device *pdev, struct cqspi_st *cqspi)
1684 {
1685 	clk_disable_unprepare(cqspi->clks[CLK_QSPI_AHB]);
1686 	clk_disable_unprepare(cqspi->clks[CLK_QSPI_APB]);
1687 }
1688 static int cqspi_probe(struct platform_device *pdev)
1689 {
1690 	const struct cqspi_driver_platdata *ddata;
1691 	struct reset_control *rstc, *rstc_ocp, *rstc_ref;
1692 	struct device *dev = &pdev->dev;
1693 	struct spi_controller *host;
1694 	struct resource *res_ahb;
1695 	struct cqspi_st *cqspi;
1696 	int ret;
1697 	int irq;
1698 
1699 	host = devm_spi_alloc_host(&pdev->dev, sizeof(*cqspi));
1700 	if (!host) {
1701 		dev_err(&pdev->dev, "devm_spi_alloc_host failed\n");
1702 		return -ENOMEM;
1703 	}
1704 	host->mode_bits = SPI_RX_QUAD | SPI_RX_DUAL;
1705 	host->mem_ops = &cqspi_mem_ops;
1706 	host->mem_caps = &cqspi_mem_caps;
1707 	host->dev.of_node = pdev->dev.of_node;
1708 
1709 	cqspi = spi_controller_get_devdata(host);
1710 
1711 	cqspi->pdev = pdev;
1712 	cqspi->host = host;
1713 	cqspi->is_jh7110 = false;
1714 	platform_set_drvdata(pdev, cqspi);
1715 
1716 	/* Obtain configuration from OF. */
1717 	ret = cqspi_of_get_pdata(cqspi);
1718 	if (ret) {
1719 		dev_err(dev, "Cannot get mandatory OF data.\n");
1720 		return -ENODEV;
1721 	}
1722 
1723 	/* Obtain QSPI clock. */
1724 	cqspi->clk = devm_clk_get(dev, NULL);
1725 	if (IS_ERR(cqspi->clk)) {
1726 		dev_err(dev, "Cannot claim QSPI clock.\n");
1727 		ret = PTR_ERR(cqspi->clk);
1728 		return ret;
1729 	}
1730 
1731 	/* Obtain and remap controller address. */
1732 	cqspi->iobase = devm_platform_ioremap_resource(pdev, 0);
1733 	if (IS_ERR(cqspi->iobase)) {
1734 		dev_err(dev, "Cannot remap controller address.\n");
1735 		ret = PTR_ERR(cqspi->iobase);
1736 		return ret;
1737 	}
1738 
1739 	/* Obtain and remap AHB address. */
1740 	cqspi->ahb_base = devm_platform_get_and_ioremap_resource(pdev, 1, &res_ahb);
1741 	if (IS_ERR(cqspi->ahb_base)) {
1742 		dev_err(dev, "Cannot remap AHB address.\n");
1743 		ret = PTR_ERR(cqspi->ahb_base);
1744 		return ret;
1745 	}
1746 	cqspi->mmap_phys_base = (dma_addr_t)res_ahb->start;
1747 	cqspi->ahb_size = resource_size(res_ahb);
1748 
1749 	init_completion(&cqspi->transfer_complete);
1750 
1751 	/* Obtain IRQ line. */
1752 	irq = platform_get_irq(pdev, 0);
1753 	if (irq < 0)
1754 		return -ENXIO;
1755 
1756 	pm_runtime_enable(dev);
1757 	ret = pm_runtime_resume_and_get(dev);
1758 	if (ret < 0)
1759 		goto probe_pm_failed;
1760 
1761 	ret = clk_prepare_enable(cqspi->clk);
1762 	if (ret) {
1763 		dev_err(dev, "Cannot enable QSPI clock.\n");
1764 		goto probe_clk_failed;
1765 	}
1766 
1767 	/* Obtain QSPI reset control */
1768 	rstc = devm_reset_control_get_optional_exclusive(dev, "qspi");
1769 	if (IS_ERR(rstc)) {
1770 		ret = PTR_ERR(rstc);
1771 		dev_err(dev, "Cannot get QSPI reset.\n");
1772 		goto probe_reset_failed;
1773 	}
1774 
1775 	rstc_ocp = devm_reset_control_get_optional_exclusive(dev, "qspi-ocp");
1776 	if (IS_ERR(rstc_ocp)) {
1777 		ret = PTR_ERR(rstc_ocp);
1778 		dev_err(dev, "Cannot get QSPI OCP reset.\n");
1779 		goto probe_reset_failed;
1780 	}
1781 
1782 	if (of_device_is_compatible(pdev->dev.of_node, "starfive,jh7110-qspi")) {
1783 		rstc_ref = devm_reset_control_get_optional_exclusive(dev, "rstc_ref");
1784 		if (IS_ERR(rstc_ref)) {
1785 			ret = PTR_ERR(rstc_ref);
1786 			dev_err(dev, "Cannot get QSPI REF reset.\n");
1787 			goto probe_reset_failed;
1788 		}
1789 		reset_control_assert(rstc_ref);
1790 		reset_control_deassert(rstc_ref);
1791 	}
1792 
1793 	reset_control_assert(rstc);
1794 	reset_control_deassert(rstc);
1795 
1796 	reset_control_assert(rstc_ocp);
1797 	reset_control_deassert(rstc_ocp);
1798 
1799 	cqspi->master_ref_clk_hz = clk_get_rate(cqspi->clk);
1800 	host->max_speed_hz = cqspi->master_ref_clk_hz;
1801 
1802 	/* write completion is supported by default */
1803 	cqspi->wr_completion = true;
1804 
1805 	ddata  = of_device_get_match_data(dev);
1806 	if (ddata) {
1807 		if (ddata->quirks & CQSPI_NEEDS_WR_DELAY)
1808 			cqspi->wr_delay = 50 * DIV_ROUND_UP(NSEC_PER_SEC,
1809 						cqspi->master_ref_clk_hz);
1810 		if (ddata->hwcaps_mask & CQSPI_SUPPORTS_OCTAL)
1811 			host->mode_bits |= SPI_RX_OCTAL | SPI_TX_OCTAL;
1812 		if (!(ddata->quirks & CQSPI_DISABLE_DAC_MODE)) {
1813 			cqspi->use_direct_mode = true;
1814 			cqspi->use_direct_mode_wr = true;
1815 		}
1816 		if (ddata->quirks & CQSPI_SUPPORT_EXTERNAL_DMA)
1817 			cqspi->use_dma_read = true;
1818 		if (ddata->quirks & CQSPI_NO_SUPPORT_WR_COMPLETION)
1819 			cqspi->wr_completion = false;
1820 		if (ddata->quirks & CQSPI_SLOW_SRAM)
1821 			cqspi->slow_sram = true;
1822 		if (ddata->quirks & CQSPI_NEEDS_APB_AHB_HAZARD_WAR)
1823 			cqspi->apb_ahb_hazard = true;
1824 
1825 		if (ddata->jh7110_clk_init) {
1826 			ret = cqspi_jh7110_clk_init(pdev, cqspi);
1827 			if (ret)
1828 				goto probe_clk_failed;
1829 		}
1830 
1831 		if (of_device_is_compatible(pdev->dev.of_node,
1832 					    "xlnx,versal-ospi-1.0")) {
1833 			ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
1834 			if (ret)
1835 				goto probe_reset_failed;
1836 		}
1837 	}
1838 
1839 	ret = devm_request_irq(dev, irq, cqspi_irq_handler, 0,
1840 			       pdev->name, cqspi);
1841 	if (ret) {
1842 		dev_err(dev, "Cannot request IRQ.\n");
1843 		goto probe_reset_failed;
1844 	}
1845 
1846 	cqspi_wait_idle(cqspi);
1847 	cqspi_controller_init(cqspi);
1848 	cqspi->current_cs = -1;
1849 	cqspi->sclk = 0;
1850 
1851 	host->num_chipselect = cqspi->num_chipselect;
1852 
1853 	ret = cqspi_setup_flash(cqspi);
1854 	if (ret) {
1855 		dev_err(dev, "failed to setup flash parameters %d\n", ret);
1856 		goto probe_setup_failed;
1857 	}
1858 
1859 	if (cqspi->use_direct_mode) {
1860 		ret = cqspi_request_mmap_dma(cqspi);
1861 		if (ret == -EPROBE_DEFER)
1862 			goto probe_setup_failed;
1863 	}
1864 
1865 	ret = spi_register_controller(host);
1866 	if (ret) {
1867 		dev_err(&pdev->dev, "failed to register SPI ctlr %d\n", ret);
1868 		goto probe_setup_failed;
1869 	}
1870 
1871 	return 0;
1872 probe_setup_failed:
1873 	cqspi_controller_enable(cqspi, 0);
1874 probe_reset_failed:
1875 	clk_disable_unprepare(cqspi->clk);
1876 probe_clk_failed:
1877 	pm_runtime_put_sync(dev);
1878 probe_pm_failed:
1879 	pm_runtime_disable(dev);
1880 	return ret;
1881 }
1882 
1883 static void cqspi_remove(struct platform_device *pdev)
1884 {
1885 	struct cqspi_st *cqspi = platform_get_drvdata(pdev);
1886 
1887 	spi_unregister_controller(cqspi->host);
1888 	cqspi_controller_enable(cqspi, 0);
1889 
1890 	if (cqspi->rx_chan)
1891 		dma_release_channel(cqspi->rx_chan);
1892 
1893 	clk_disable_unprepare(cqspi->clk);
1894 
1895 	if (cqspi->is_jh7110)
1896 		cqspi_jh7110_disable_clk(pdev, cqspi);
1897 
1898 	pm_runtime_put_sync(&pdev->dev);
1899 	pm_runtime_disable(&pdev->dev);
1900 }
1901 
1902 static int cqspi_suspend(struct device *dev)
1903 {
1904 	struct cqspi_st *cqspi = dev_get_drvdata(dev);
1905 	struct spi_controller *host = dev_get_drvdata(dev);
1906 	int ret;
1907 
1908 	ret = spi_controller_suspend(host);
1909 	cqspi_controller_enable(cqspi, 0);
1910 
1911 	clk_disable_unprepare(cqspi->clk);
1912 
1913 	return ret;
1914 }
1915 
1916 static int cqspi_resume(struct device *dev)
1917 {
1918 	struct cqspi_st *cqspi = dev_get_drvdata(dev);
1919 	struct spi_controller *host = dev_get_drvdata(dev);
1920 
1921 	clk_prepare_enable(cqspi->clk);
1922 	cqspi_wait_idle(cqspi);
1923 	cqspi_controller_init(cqspi);
1924 
1925 	cqspi->current_cs = -1;
1926 	cqspi->sclk = 0;
1927 
1928 	return spi_controller_resume(host);
1929 }
1930 
1931 static DEFINE_SIMPLE_DEV_PM_OPS(cqspi_dev_pm_ops, cqspi_suspend, cqspi_resume);
1932 
1933 static const struct cqspi_driver_platdata cdns_qspi = {
1934 	.quirks = CQSPI_DISABLE_DAC_MODE,
1935 };
1936 
1937 static const struct cqspi_driver_platdata k2g_qspi = {
1938 	.quirks = CQSPI_NEEDS_WR_DELAY,
1939 };
1940 
1941 static const struct cqspi_driver_platdata am654_ospi = {
1942 	.hwcaps_mask = CQSPI_SUPPORTS_OCTAL,
1943 	.quirks = CQSPI_NEEDS_WR_DELAY,
1944 };
1945 
1946 static const struct cqspi_driver_platdata intel_lgm_qspi = {
1947 	.quirks = CQSPI_DISABLE_DAC_MODE,
1948 };
1949 
1950 static const struct cqspi_driver_platdata socfpga_qspi = {
1951 	.quirks = CQSPI_DISABLE_DAC_MODE
1952 			| CQSPI_NO_SUPPORT_WR_COMPLETION
1953 			| CQSPI_SLOW_SRAM,
1954 };
1955 
1956 static const struct cqspi_driver_platdata versal_ospi = {
1957 	.hwcaps_mask = CQSPI_SUPPORTS_OCTAL,
1958 	.quirks = CQSPI_DISABLE_DAC_MODE | CQSPI_SUPPORT_EXTERNAL_DMA,
1959 	.indirect_read_dma = cqspi_versal_indirect_read_dma,
1960 	.get_dma_status = cqspi_get_versal_dma_status,
1961 };
1962 
1963 static const struct cqspi_driver_platdata jh7110_qspi = {
1964 	.quirks = CQSPI_DISABLE_DAC_MODE,
1965 	.jh7110_clk_init = cqspi_jh7110_clk_init,
1966 };
1967 
1968 static const struct cqspi_driver_platdata pensando_cdns_qspi = {
1969 	.quirks = CQSPI_NEEDS_APB_AHB_HAZARD_WAR | CQSPI_DISABLE_DAC_MODE,
1970 };
1971 
1972 static const struct of_device_id cqspi_dt_ids[] = {
1973 	{
1974 		.compatible = "cdns,qspi-nor",
1975 		.data = &cdns_qspi,
1976 	},
1977 	{
1978 		.compatible = "ti,k2g-qspi",
1979 		.data = &k2g_qspi,
1980 	},
1981 	{
1982 		.compatible = "ti,am654-ospi",
1983 		.data = &am654_ospi,
1984 	},
1985 	{
1986 		.compatible = "intel,lgm-qspi",
1987 		.data = &intel_lgm_qspi,
1988 	},
1989 	{
1990 		.compatible = "xlnx,versal-ospi-1.0",
1991 		.data = &versal_ospi,
1992 	},
1993 	{
1994 		.compatible = "intel,socfpga-qspi",
1995 		.data = &socfpga_qspi,
1996 	},
1997 	{
1998 		.compatible = "starfive,jh7110-qspi",
1999 		.data = &jh7110_qspi,
2000 	},
2001 	{
2002 		.compatible = "amd,pensando-elba-qspi",
2003 		.data = &pensando_cdns_qspi,
2004 	},
2005 	{ /* end of table */ }
2006 };
2007 
2008 MODULE_DEVICE_TABLE(of, cqspi_dt_ids);
2009 
2010 static struct platform_driver cqspi_platform_driver = {
2011 	.probe = cqspi_probe,
2012 	.remove_new = cqspi_remove,
2013 	.driver = {
2014 		.name = CQSPI_NAME,
2015 		.pm = &cqspi_dev_pm_ops,
2016 		.of_match_table = cqspi_dt_ids,
2017 	},
2018 };
2019 
2020 module_platform_driver(cqspi_platform_driver);
2021 
2022 MODULE_DESCRIPTION("Cadence QSPI Controller Driver");
2023 MODULE_LICENSE("GPL v2");
2024 MODULE_ALIAS("platform:" CQSPI_NAME);
2025 MODULE_AUTHOR("Ley Foon Tan <lftan@altera.com>");
2026 MODULE_AUTHOR("Graham Moore <grmoore@opensource.altera.com>");
2027 MODULE_AUTHOR("Vadivel Murugan R <vadivel.muruganx.ramuthevar@intel.com>");
2028 MODULE_AUTHOR("Vignesh Raghavendra <vigneshr@ti.com>");
2029 MODULE_AUTHOR("Pratyush Yadav <p.yadav@ti.com>");
2030