1 // SPDX-License-Identifier: GPL-2.0-only 2 // 3 // Copyright (C) 2020 NVIDIA CORPORATION. 4 5 #include <linux/clk.h> 6 #include <linux/completion.h> 7 #include <linux/delay.h> 8 #include <linux/dmaengine.h> 9 #include <linux/dma-mapping.h> 10 #include <linux/dmapool.h> 11 #include <linux/err.h> 12 #include <linux/interrupt.h> 13 #include <linux/io.h> 14 #include <linux/iopoll.h> 15 #include <linux/kernel.h> 16 #include <linux/kthread.h> 17 #include <linux/module.h> 18 #include <linux/platform_device.h> 19 #include <linux/pm_runtime.h> 20 #include <linux/of.h> 21 #include <linux/of_device.h> 22 #include <linux/reset.h> 23 #include <linux/spi/spi.h> 24 25 #define QSPI_COMMAND1 0x000 26 #define QSPI_BIT_LENGTH(x) (((x) & 0x1f) << 0) 27 #define QSPI_PACKED BIT(5) 28 #define QSPI_INTERFACE_WIDTH_MASK (0x03 << 7) 29 #define QSPI_INTERFACE_WIDTH(x) (((x) & 0x03) << 7) 30 #define QSPI_INTERFACE_WIDTH_SINGLE QSPI_INTERFACE_WIDTH(0) 31 #define QSPI_INTERFACE_WIDTH_DUAL QSPI_INTERFACE_WIDTH(1) 32 #define QSPI_INTERFACE_WIDTH_QUAD QSPI_INTERFACE_WIDTH(2) 33 #define QSPI_SDR_DDR_SEL BIT(9) 34 #define QSPI_TX_EN BIT(11) 35 #define QSPI_RX_EN BIT(12) 36 #define QSPI_CS_SW_VAL BIT(20) 37 #define QSPI_CS_SW_HW BIT(21) 38 #define QSPI_CONTROL_MODE_0 (0 << 28) 39 #define QSPI_CONTROL_MODE_3 (3 << 28) 40 #define QSPI_CONTROL_MODE_MASK (3 << 28) 41 #define QSPI_M_S BIT(30) 42 #define QSPI_PIO BIT(31) 43 44 #define QSPI_COMMAND2 0x004 45 #define QSPI_TX_TAP_DELAY(x) (((x) & 0x3f) << 10) 46 #define QSPI_RX_TAP_DELAY(x) (((x) & 0xff) << 0) 47 48 #define QSPI_CS_TIMING1 0x008 49 #define QSPI_SETUP_HOLD(setup, hold) (((setup) << 4) | (hold)) 50 51 #define QSPI_CS_TIMING2 0x00c 52 #define CYCLES_BETWEEN_PACKETS_0(x) (((x) & 0x1f) << 0) 53 #define CS_ACTIVE_BETWEEN_PACKETS_0 BIT(5) 54 55 #define QSPI_TRANS_STATUS 0x010 56 #define QSPI_BLK_CNT(val) (((val) >> 0) & 0xffff) 57 #define QSPI_RDY BIT(30) 58 59 #define QSPI_FIFO_STATUS 0x014 60 #define QSPI_RX_FIFO_EMPTY BIT(0) 61 #define QSPI_RX_FIFO_FULL BIT(1) 62 #define QSPI_TX_FIFO_EMPTY BIT(2) 63 #define QSPI_TX_FIFO_FULL BIT(3) 64 #define QSPI_RX_FIFO_UNF BIT(4) 65 #define QSPI_RX_FIFO_OVF BIT(5) 66 #define QSPI_TX_FIFO_UNF BIT(6) 67 #define QSPI_TX_FIFO_OVF BIT(7) 68 #define QSPI_ERR BIT(8) 69 #define QSPI_TX_FIFO_FLUSH BIT(14) 70 #define QSPI_RX_FIFO_FLUSH BIT(15) 71 #define QSPI_TX_FIFO_EMPTY_COUNT(val) (((val) >> 16) & 0x7f) 72 #define QSPI_RX_FIFO_FULL_COUNT(val) (((val) >> 23) & 0x7f) 73 74 #define QSPI_FIFO_ERROR (QSPI_RX_FIFO_UNF | \ 75 QSPI_RX_FIFO_OVF | \ 76 QSPI_TX_FIFO_UNF | \ 77 QSPI_TX_FIFO_OVF) 78 #define QSPI_FIFO_EMPTY (QSPI_RX_FIFO_EMPTY | \ 79 QSPI_TX_FIFO_EMPTY) 80 81 #define QSPI_TX_DATA 0x018 82 #define QSPI_RX_DATA 0x01c 83 84 #define QSPI_DMA_CTL 0x020 85 #define QSPI_TX_TRIG(n) (((n) & 0x3) << 15) 86 #define QSPI_TX_TRIG_1 QSPI_TX_TRIG(0) 87 #define QSPI_TX_TRIG_4 QSPI_TX_TRIG(1) 88 #define QSPI_TX_TRIG_8 QSPI_TX_TRIG(2) 89 #define QSPI_TX_TRIG_16 QSPI_TX_TRIG(3) 90 91 #define QSPI_RX_TRIG(n) (((n) & 0x3) << 19) 92 #define QSPI_RX_TRIG_1 QSPI_RX_TRIG(0) 93 #define QSPI_RX_TRIG_4 QSPI_RX_TRIG(1) 94 #define QSPI_RX_TRIG_8 QSPI_RX_TRIG(2) 95 #define QSPI_RX_TRIG_16 QSPI_RX_TRIG(3) 96 97 #define QSPI_DMA_EN BIT(31) 98 99 #define QSPI_DMA_BLK 0x024 100 #define QSPI_DMA_BLK_SET(x) (((x) & 0xffff) << 0) 101 102 #define QSPI_TX_FIFO 0x108 103 #define QSPI_RX_FIFO 0x188 104 105 #define QSPI_FIFO_DEPTH 64 106 107 #define QSPI_INTR_MASK 0x18c 108 #define QSPI_INTR_RX_FIFO_UNF_MASK BIT(25) 109 #define QSPI_INTR_RX_FIFO_OVF_MASK BIT(26) 110 #define QSPI_INTR_TX_FIFO_UNF_MASK BIT(27) 111 #define QSPI_INTR_TX_FIFO_OVF_MASK BIT(28) 112 #define QSPI_INTR_RDY_MASK BIT(29) 113 #define QSPI_INTR_RX_TX_FIFO_ERR (QSPI_INTR_RX_FIFO_UNF_MASK | \ 114 QSPI_INTR_RX_FIFO_OVF_MASK | \ 115 QSPI_INTR_TX_FIFO_UNF_MASK | \ 116 QSPI_INTR_TX_FIFO_OVF_MASK) 117 118 #define QSPI_MISC_REG 0x194 119 #define QSPI_NUM_DUMMY_CYCLE(x) (((x) & 0xff) << 0) 120 #define QSPI_DUMMY_CYCLES_MAX 0xff 121 122 #define DATA_DIR_TX BIT(0) 123 #define DATA_DIR_RX BIT(1) 124 125 #define QSPI_DMA_TIMEOUT (msecs_to_jiffies(1000)) 126 #define DEFAULT_QSPI_DMA_BUF_LEN (64 * 1024) 127 128 struct tegra_qspi_client_data { 129 int tx_clk_tap_delay; 130 int rx_clk_tap_delay; 131 }; 132 133 struct tegra_qspi { 134 struct device *dev; 135 struct spi_master *master; 136 /* lock to protect data accessed by irq */ 137 spinlock_t lock; 138 139 struct clk *clk; 140 struct reset_control *rst; 141 void __iomem *base; 142 phys_addr_t phys; 143 unsigned int irq; 144 145 u32 cur_speed; 146 unsigned int cur_pos; 147 unsigned int words_per_32bit; 148 unsigned int bytes_per_word; 149 unsigned int curr_dma_words; 150 unsigned int cur_direction; 151 152 unsigned int cur_rx_pos; 153 unsigned int cur_tx_pos; 154 155 unsigned int dma_buf_size; 156 unsigned int max_buf_size; 157 bool is_curr_dma_xfer; 158 159 struct completion rx_dma_complete; 160 struct completion tx_dma_complete; 161 162 u32 tx_status; 163 u32 rx_status; 164 u32 status_reg; 165 bool is_packed; 166 bool use_dma; 167 168 u32 command1_reg; 169 u32 dma_control_reg; 170 u32 def_command1_reg; 171 u32 def_command2_reg; 172 u32 spi_cs_timing1; 173 u32 spi_cs_timing2; 174 u8 dummy_cycles; 175 176 struct completion xfer_completion; 177 struct spi_transfer *curr_xfer; 178 179 struct dma_chan *rx_dma_chan; 180 u32 *rx_dma_buf; 181 dma_addr_t rx_dma_phys; 182 struct dma_async_tx_descriptor *rx_dma_desc; 183 184 struct dma_chan *tx_dma_chan; 185 u32 *tx_dma_buf; 186 dma_addr_t tx_dma_phys; 187 struct dma_async_tx_descriptor *tx_dma_desc; 188 }; 189 190 static inline u32 tegra_qspi_readl(struct tegra_qspi *tqspi, unsigned long offset) 191 { 192 return readl(tqspi->base + offset); 193 } 194 195 static inline void tegra_qspi_writel(struct tegra_qspi *tqspi, u32 value, unsigned long offset) 196 { 197 writel(value, tqspi->base + offset); 198 199 /* read back register to make sure that register writes completed */ 200 if (offset != QSPI_TX_FIFO) 201 readl(tqspi->base + QSPI_COMMAND1); 202 } 203 204 static void tegra_qspi_mask_clear_irq(struct tegra_qspi *tqspi) 205 { 206 u32 value; 207 208 /* write 1 to clear status register */ 209 value = tegra_qspi_readl(tqspi, QSPI_TRANS_STATUS); 210 tegra_qspi_writel(tqspi, value, QSPI_TRANS_STATUS); 211 212 value = tegra_qspi_readl(tqspi, QSPI_INTR_MASK); 213 if (!(value & QSPI_INTR_RDY_MASK)) { 214 value |= (QSPI_INTR_RDY_MASK | QSPI_INTR_RX_TX_FIFO_ERR); 215 tegra_qspi_writel(tqspi, value, QSPI_INTR_MASK); 216 } 217 218 /* clear fifo status error if any */ 219 value = tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS); 220 if (value & QSPI_ERR) 221 tegra_qspi_writel(tqspi, QSPI_ERR | QSPI_FIFO_ERROR, QSPI_FIFO_STATUS); 222 } 223 224 static unsigned int 225 tegra_qspi_calculate_curr_xfer_param(struct tegra_qspi *tqspi, struct spi_transfer *t) 226 { 227 unsigned int max_word, max_len, total_fifo_words; 228 unsigned int remain_len = t->len - tqspi->cur_pos; 229 unsigned int bits_per_word = t->bits_per_word; 230 231 tqspi->bytes_per_word = DIV_ROUND_UP(bits_per_word, 8); 232 233 /* 234 * Tegra QSPI controller supports packed or unpacked mode transfers. 235 * Packed mode is used for data transfers using 8, 16, or 32 bits per 236 * word with a minimum transfer of 1 word and for all other transfers 237 * unpacked mode will be used. 238 */ 239 240 if ((bits_per_word == 8 || bits_per_word == 16 || 241 bits_per_word == 32) && t->len > 3) { 242 tqspi->is_packed = true; 243 tqspi->words_per_32bit = 32 / bits_per_word; 244 } else { 245 tqspi->is_packed = false; 246 tqspi->words_per_32bit = 1; 247 } 248 249 if (tqspi->is_packed) { 250 max_len = min(remain_len, tqspi->max_buf_size); 251 tqspi->curr_dma_words = max_len / tqspi->bytes_per_word; 252 total_fifo_words = (max_len + 3) / 4; 253 } else { 254 max_word = (remain_len - 1) / tqspi->bytes_per_word + 1; 255 max_word = min(max_word, tqspi->max_buf_size / 4); 256 tqspi->curr_dma_words = max_word; 257 total_fifo_words = max_word; 258 } 259 260 return total_fifo_words; 261 } 262 263 static unsigned int 264 tegra_qspi_fill_tx_fifo_from_client_txbuf(struct tegra_qspi *tqspi, struct spi_transfer *t) 265 { 266 unsigned int written_words, fifo_words_left, count; 267 unsigned int len, tx_empty_count, max_n_32bit, i; 268 u8 *tx_buf = (u8 *)t->tx_buf + tqspi->cur_tx_pos; 269 u32 fifo_status; 270 271 fifo_status = tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS); 272 tx_empty_count = QSPI_TX_FIFO_EMPTY_COUNT(fifo_status); 273 274 if (tqspi->is_packed) { 275 fifo_words_left = tx_empty_count * tqspi->words_per_32bit; 276 written_words = min(fifo_words_left, tqspi->curr_dma_words); 277 len = written_words * tqspi->bytes_per_word; 278 max_n_32bit = DIV_ROUND_UP(len, 4); 279 for (count = 0; count < max_n_32bit; count++) { 280 u32 x = 0; 281 282 for (i = 0; (i < 4) && len; i++, len--) 283 x |= (u32)(*tx_buf++) << (i * 8); 284 tegra_qspi_writel(tqspi, x, QSPI_TX_FIFO); 285 } 286 287 tqspi->cur_tx_pos += written_words * tqspi->bytes_per_word; 288 } else { 289 unsigned int write_bytes; 290 u8 bytes_per_word = tqspi->bytes_per_word; 291 292 max_n_32bit = min(tqspi->curr_dma_words, tx_empty_count); 293 written_words = max_n_32bit; 294 len = written_words * tqspi->bytes_per_word; 295 if (len > t->len - tqspi->cur_pos) 296 len = t->len - tqspi->cur_pos; 297 write_bytes = len; 298 for (count = 0; count < max_n_32bit; count++) { 299 u32 x = 0; 300 301 for (i = 0; len && (i < bytes_per_word); i++, len--) 302 x |= (u32)(*tx_buf++) << (i * 8); 303 tegra_qspi_writel(tqspi, x, QSPI_TX_FIFO); 304 } 305 306 tqspi->cur_tx_pos += write_bytes; 307 } 308 309 return written_words; 310 } 311 312 static unsigned int 313 tegra_qspi_read_rx_fifo_to_client_rxbuf(struct tegra_qspi *tqspi, struct spi_transfer *t) 314 { 315 u8 *rx_buf = (u8 *)t->rx_buf + tqspi->cur_rx_pos; 316 unsigned int len, rx_full_count, count, i; 317 unsigned int read_words = 0; 318 u32 fifo_status, x; 319 320 fifo_status = tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS); 321 rx_full_count = QSPI_RX_FIFO_FULL_COUNT(fifo_status); 322 if (tqspi->is_packed) { 323 len = tqspi->curr_dma_words * tqspi->bytes_per_word; 324 for (count = 0; count < rx_full_count; count++) { 325 x = tegra_qspi_readl(tqspi, QSPI_RX_FIFO); 326 327 for (i = 0; len && (i < 4); i++, len--) 328 *rx_buf++ = (x >> i * 8) & 0xff; 329 } 330 331 read_words += tqspi->curr_dma_words; 332 tqspi->cur_rx_pos += tqspi->curr_dma_words * tqspi->bytes_per_word; 333 } else { 334 u32 rx_mask = ((u32)1 << t->bits_per_word) - 1; 335 u8 bytes_per_word = tqspi->bytes_per_word; 336 unsigned int read_bytes; 337 338 len = rx_full_count * bytes_per_word; 339 if (len > t->len - tqspi->cur_pos) 340 len = t->len - tqspi->cur_pos; 341 read_bytes = len; 342 for (count = 0; count < rx_full_count; count++) { 343 x = tegra_qspi_readl(tqspi, QSPI_RX_FIFO) & rx_mask; 344 345 for (i = 0; len && (i < bytes_per_word); i++, len--) 346 *rx_buf++ = (x >> (i * 8)) & 0xff; 347 } 348 349 read_words += rx_full_count; 350 tqspi->cur_rx_pos += read_bytes; 351 } 352 353 return read_words; 354 } 355 356 static void 357 tegra_qspi_copy_client_txbuf_to_qspi_txbuf(struct tegra_qspi *tqspi, struct spi_transfer *t) 358 { 359 dma_sync_single_for_cpu(tqspi->dev, tqspi->tx_dma_phys, 360 tqspi->dma_buf_size, DMA_TO_DEVICE); 361 362 /* 363 * In packed mode, each word in FIFO may contain multiple packets 364 * based on bits per word. So all bytes in each FIFO word are valid. 365 * 366 * In unpacked mode, each word in FIFO contains single packet and 367 * based on bits per word any remaining bits in FIFO word will be 368 * ignored by the hardware and are invalid bits. 369 */ 370 if (tqspi->is_packed) { 371 tqspi->cur_tx_pos += tqspi->curr_dma_words * tqspi->bytes_per_word; 372 } else { 373 u8 *tx_buf = (u8 *)t->tx_buf + tqspi->cur_tx_pos; 374 unsigned int i, count, consume, write_bytes; 375 376 /* 377 * Fill tx_dma_buf to contain single packet in each word based 378 * on bits per word from SPI core tx_buf. 379 */ 380 consume = tqspi->curr_dma_words * tqspi->bytes_per_word; 381 if (consume > t->len - tqspi->cur_pos) 382 consume = t->len - tqspi->cur_pos; 383 write_bytes = consume; 384 for (count = 0; count < tqspi->curr_dma_words; count++) { 385 u32 x = 0; 386 387 for (i = 0; consume && (i < tqspi->bytes_per_word); i++, consume--) 388 x |= (u32)(*tx_buf++) << (i * 8); 389 tqspi->tx_dma_buf[count] = x; 390 } 391 392 tqspi->cur_tx_pos += write_bytes; 393 } 394 395 dma_sync_single_for_device(tqspi->dev, tqspi->tx_dma_phys, 396 tqspi->dma_buf_size, DMA_TO_DEVICE); 397 } 398 399 static void 400 tegra_qspi_copy_qspi_rxbuf_to_client_rxbuf(struct tegra_qspi *tqspi, struct spi_transfer *t) 401 { 402 dma_sync_single_for_cpu(tqspi->dev, tqspi->rx_dma_phys, 403 tqspi->dma_buf_size, DMA_FROM_DEVICE); 404 405 if (tqspi->is_packed) { 406 tqspi->cur_rx_pos += tqspi->curr_dma_words * tqspi->bytes_per_word; 407 } else { 408 unsigned char *rx_buf = t->rx_buf + tqspi->cur_rx_pos; 409 u32 rx_mask = ((u32)1 << t->bits_per_word) - 1; 410 unsigned int i, count, consume, read_bytes; 411 412 /* 413 * Each FIFO word contains single data packet. 414 * Skip invalid bits in each FIFO word based on bits per word 415 * and align bytes while filling in SPI core rx_buf. 416 */ 417 consume = tqspi->curr_dma_words * tqspi->bytes_per_word; 418 if (consume > t->len - tqspi->cur_pos) 419 consume = t->len - tqspi->cur_pos; 420 read_bytes = consume; 421 for (count = 0; count < tqspi->curr_dma_words; count++) { 422 u32 x = tqspi->rx_dma_buf[count] & rx_mask; 423 424 for (i = 0; consume && (i < tqspi->bytes_per_word); i++, consume--) 425 *rx_buf++ = (x >> (i * 8)) & 0xff; 426 } 427 428 tqspi->cur_rx_pos += read_bytes; 429 } 430 431 dma_sync_single_for_device(tqspi->dev, tqspi->rx_dma_phys, 432 tqspi->dma_buf_size, DMA_FROM_DEVICE); 433 } 434 435 static void tegra_qspi_dma_complete(void *args) 436 { 437 struct completion *dma_complete = args; 438 439 complete(dma_complete); 440 } 441 442 static int tegra_qspi_start_tx_dma(struct tegra_qspi *tqspi, struct spi_transfer *t, int len) 443 { 444 dma_addr_t tx_dma_phys; 445 446 reinit_completion(&tqspi->tx_dma_complete); 447 448 if (tqspi->is_packed) 449 tx_dma_phys = t->tx_dma; 450 else 451 tx_dma_phys = tqspi->tx_dma_phys; 452 453 tqspi->tx_dma_desc = dmaengine_prep_slave_single(tqspi->tx_dma_chan, tx_dma_phys, 454 len, DMA_MEM_TO_DEV, 455 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 456 457 if (!tqspi->tx_dma_desc) { 458 dev_err(tqspi->dev, "Unable to get TX descriptor\n"); 459 return -EIO; 460 } 461 462 tqspi->tx_dma_desc->callback = tegra_qspi_dma_complete; 463 tqspi->tx_dma_desc->callback_param = &tqspi->tx_dma_complete; 464 dmaengine_submit(tqspi->tx_dma_desc); 465 dma_async_issue_pending(tqspi->tx_dma_chan); 466 467 return 0; 468 } 469 470 static int tegra_qspi_start_rx_dma(struct tegra_qspi *tqspi, struct spi_transfer *t, int len) 471 { 472 dma_addr_t rx_dma_phys; 473 474 reinit_completion(&tqspi->rx_dma_complete); 475 476 if (tqspi->is_packed) 477 rx_dma_phys = t->rx_dma; 478 else 479 rx_dma_phys = tqspi->rx_dma_phys; 480 481 tqspi->rx_dma_desc = dmaengine_prep_slave_single(tqspi->rx_dma_chan, rx_dma_phys, 482 len, DMA_DEV_TO_MEM, 483 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 484 485 if (!tqspi->rx_dma_desc) { 486 dev_err(tqspi->dev, "Unable to get RX descriptor\n"); 487 return -EIO; 488 } 489 490 tqspi->rx_dma_desc->callback = tegra_qspi_dma_complete; 491 tqspi->rx_dma_desc->callback_param = &tqspi->rx_dma_complete; 492 dmaengine_submit(tqspi->rx_dma_desc); 493 dma_async_issue_pending(tqspi->rx_dma_chan); 494 495 return 0; 496 } 497 498 static int tegra_qspi_flush_fifos(struct tegra_qspi *tqspi, bool atomic) 499 { 500 void __iomem *addr = tqspi->base + QSPI_FIFO_STATUS; 501 u32 val; 502 503 val = tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS); 504 if ((val & QSPI_FIFO_EMPTY) == QSPI_FIFO_EMPTY) 505 return 0; 506 507 val |= QSPI_RX_FIFO_FLUSH | QSPI_TX_FIFO_FLUSH; 508 tegra_qspi_writel(tqspi, val, QSPI_FIFO_STATUS); 509 510 if (!atomic) 511 return readl_relaxed_poll_timeout(addr, val, 512 (val & QSPI_FIFO_EMPTY) == QSPI_FIFO_EMPTY, 513 1000, 1000000); 514 515 return readl_relaxed_poll_timeout_atomic(addr, val, 516 (val & QSPI_FIFO_EMPTY) == QSPI_FIFO_EMPTY, 517 1000, 1000000); 518 } 519 520 static void tegra_qspi_unmask_irq(struct tegra_qspi *tqspi) 521 { 522 u32 intr_mask; 523 524 intr_mask = tegra_qspi_readl(tqspi, QSPI_INTR_MASK); 525 intr_mask &= ~(QSPI_INTR_RDY_MASK | QSPI_INTR_RX_TX_FIFO_ERR); 526 tegra_qspi_writel(tqspi, intr_mask, QSPI_INTR_MASK); 527 } 528 529 static int tegra_qspi_dma_map_xfer(struct tegra_qspi *tqspi, struct spi_transfer *t) 530 { 531 u8 *tx_buf = (u8 *)t->tx_buf + tqspi->cur_tx_pos; 532 u8 *rx_buf = (u8 *)t->rx_buf + tqspi->cur_rx_pos; 533 unsigned int len; 534 535 len = DIV_ROUND_UP(tqspi->curr_dma_words * tqspi->bytes_per_word, 4) * 4; 536 537 if (t->tx_buf) { 538 t->tx_dma = dma_map_single(tqspi->dev, (void *)tx_buf, len, DMA_TO_DEVICE); 539 if (dma_mapping_error(tqspi->dev, t->tx_dma)) 540 return -ENOMEM; 541 } 542 543 if (t->rx_buf) { 544 t->rx_dma = dma_map_single(tqspi->dev, (void *)rx_buf, len, DMA_FROM_DEVICE); 545 if (dma_mapping_error(tqspi->dev, t->rx_dma)) { 546 dma_unmap_single(tqspi->dev, t->tx_dma, len, DMA_TO_DEVICE); 547 return -ENOMEM; 548 } 549 } 550 551 return 0; 552 } 553 554 static void tegra_qspi_dma_unmap_xfer(struct tegra_qspi *tqspi, struct spi_transfer *t) 555 { 556 unsigned int len; 557 558 len = DIV_ROUND_UP(tqspi->curr_dma_words * tqspi->bytes_per_word, 4) * 4; 559 560 dma_unmap_single(tqspi->dev, t->tx_dma, len, DMA_TO_DEVICE); 561 dma_unmap_single(tqspi->dev, t->rx_dma, len, DMA_FROM_DEVICE); 562 } 563 564 static int tegra_qspi_start_dma_based_transfer(struct tegra_qspi *tqspi, struct spi_transfer *t) 565 { 566 struct dma_slave_config dma_sconfig = { 0 }; 567 unsigned int len; 568 u8 dma_burst; 569 int ret = 0; 570 u32 val; 571 572 if (tqspi->is_packed) { 573 ret = tegra_qspi_dma_map_xfer(tqspi, t); 574 if (ret < 0) 575 return ret; 576 } 577 578 val = QSPI_DMA_BLK_SET(tqspi->curr_dma_words - 1); 579 tegra_qspi_writel(tqspi, val, QSPI_DMA_BLK); 580 581 tegra_qspi_unmask_irq(tqspi); 582 583 if (tqspi->is_packed) 584 len = DIV_ROUND_UP(tqspi->curr_dma_words * tqspi->bytes_per_word, 4) * 4; 585 else 586 len = tqspi->curr_dma_words * 4; 587 588 /* set attention level based on length of transfer */ 589 val = 0; 590 if (len & 0xf) { 591 val |= QSPI_TX_TRIG_1 | QSPI_RX_TRIG_1; 592 dma_burst = 1; 593 } else if (((len) >> 4) & 0x1) { 594 val |= QSPI_TX_TRIG_4 | QSPI_RX_TRIG_4; 595 dma_burst = 4; 596 } else { 597 val |= QSPI_TX_TRIG_8 | QSPI_RX_TRIG_8; 598 dma_burst = 8; 599 } 600 601 tegra_qspi_writel(tqspi, val, QSPI_DMA_CTL); 602 tqspi->dma_control_reg = val; 603 604 dma_sconfig.device_fc = true; 605 if (tqspi->cur_direction & DATA_DIR_TX) { 606 dma_sconfig.dst_addr = tqspi->phys + QSPI_TX_FIFO; 607 dma_sconfig.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 608 dma_sconfig.dst_maxburst = dma_burst; 609 ret = dmaengine_slave_config(tqspi->tx_dma_chan, &dma_sconfig); 610 if (ret < 0) { 611 dev_err(tqspi->dev, "failed DMA slave config: %d\n", ret); 612 return ret; 613 } 614 615 tegra_qspi_copy_client_txbuf_to_qspi_txbuf(tqspi, t); 616 ret = tegra_qspi_start_tx_dma(tqspi, t, len); 617 if (ret < 0) { 618 dev_err(tqspi->dev, "failed to starting TX DMA: %d\n", ret); 619 return ret; 620 } 621 } 622 623 if (tqspi->cur_direction & DATA_DIR_RX) { 624 dma_sconfig.src_addr = tqspi->phys + QSPI_RX_FIFO; 625 dma_sconfig.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 626 dma_sconfig.src_maxburst = dma_burst; 627 ret = dmaengine_slave_config(tqspi->rx_dma_chan, &dma_sconfig); 628 if (ret < 0) { 629 dev_err(tqspi->dev, "failed DMA slave config: %d\n", ret); 630 return ret; 631 } 632 633 dma_sync_single_for_device(tqspi->dev, tqspi->rx_dma_phys, 634 tqspi->dma_buf_size, 635 DMA_FROM_DEVICE); 636 637 ret = tegra_qspi_start_rx_dma(tqspi, t, len); 638 if (ret < 0) { 639 dev_err(tqspi->dev, "failed to start RX DMA: %d\n", ret); 640 if (tqspi->cur_direction & DATA_DIR_TX) 641 dmaengine_terminate_all(tqspi->tx_dma_chan); 642 return ret; 643 } 644 } 645 646 tegra_qspi_writel(tqspi, tqspi->command1_reg, QSPI_COMMAND1); 647 648 tqspi->is_curr_dma_xfer = true; 649 tqspi->dma_control_reg = val; 650 val |= QSPI_DMA_EN; 651 tegra_qspi_writel(tqspi, val, QSPI_DMA_CTL); 652 653 return ret; 654 } 655 656 static int tegra_qspi_start_cpu_based_transfer(struct tegra_qspi *qspi, struct spi_transfer *t) 657 { 658 u32 val; 659 unsigned int cur_words; 660 661 if (qspi->cur_direction & DATA_DIR_TX) 662 cur_words = tegra_qspi_fill_tx_fifo_from_client_txbuf(qspi, t); 663 else 664 cur_words = qspi->curr_dma_words; 665 666 val = QSPI_DMA_BLK_SET(cur_words - 1); 667 tegra_qspi_writel(qspi, val, QSPI_DMA_BLK); 668 669 tegra_qspi_unmask_irq(qspi); 670 671 qspi->is_curr_dma_xfer = false; 672 val = qspi->command1_reg; 673 val |= QSPI_PIO; 674 tegra_qspi_writel(qspi, val, QSPI_COMMAND1); 675 676 return 0; 677 } 678 679 static void tegra_qspi_deinit_dma(struct tegra_qspi *tqspi) 680 { 681 if (tqspi->tx_dma_buf) { 682 dma_free_coherent(tqspi->dev, tqspi->dma_buf_size, 683 tqspi->tx_dma_buf, tqspi->tx_dma_phys); 684 tqspi->tx_dma_buf = NULL; 685 } 686 687 if (tqspi->tx_dma_chan) { 688 dma_release_channel(tqspi->tx_dma_chan); 689 tqspi->tx_dma_chan = NULL; 690 } 691 692 if (tqspi->rx_dma_buf) { 693 dma_free_coherent(tqspi->dev, tqspi->dma_buf_size, 694 tqspi->rx_dma_buf, tqspi->rx_dma_phys); 695 tqspi->rx_dma_buf = NULL; 696 } 697 698 if (tqspi->rx_dma_chan) { 699 dma_release_channel(tqspi->rx_dma_chan); 700 tqspi->rx_dma_chan = NULL; 701 } 702 } 703 704 static int tegra_qspi_init_dma(struct tegra_qspi *tqspi) 705 { 706 struct dma_chan *dma_chan; 707 dma_addr_t dma_phys; 708 u32 *dma_buf; 709 int err; 710 711 dma_chan = dma_request_chan(tqspi->dev, "rx"); 712 if (IS_ERR(dma_chan)) { 713 err = PTR_ERR(dma_chan); 714 goto err_out; 715 } 716 717 tqspi->rx_dma_chan = dma_chan; 718 719 dma_buf = dma_alloc_coherent(tqspi->dev, tqspi->dma_buf_size, &dma_phys, GFP_KERNEL); 720 if (!dma_buf) { 721 err = -ENOMEM; 722 goto err_out; 723 } 724 725 tqspi->rx_dma_buf = dma_buf; 726 tqspi->rx_dma_phys = dma_phys; 727 728 dma_chan = dma_request_chan(tqspi->dev, "tx"); 729 if (IS_ERR(dma_chan)) { 730 err = PTR_ERR(dma_chan); 731 goto err_out; 732 } 733 734 tqspi->tx_dma_chan = dma_chan; 735 736 dma_buf = dma_alloc_coherent(tqspi->dev, tqspi->dma_buf_size, &dma_phys, GFP_KERNEL); 737 if (!dma_buf) { 738 err = -ENOMEM; 739 goto err_out; 740 } 741 742 tqspi->tx_dma_buf = dma_buf; 743 tqspi->tx_dma_phys = dma_phys; 744 tqspi->use_dma = true; 745 746 return 0; 747 748 err_out: 749 tegra_qspi_deinit_dma(tqspi); 750 751 if (err != -EPROBE_DEFER) { 752 dev_err(tqspi->dev, "cannot use DMA: %d\n", err); 753 dev_err(tqspi->dev, "falling back to PIO\n"); 754 return 0; 755 } 756 757 return err; 758 } 759 760 static u32 tegra_qspi_setup_transfer_one(struct spi_device *spi, struct spi_transfer *t, 761 bool is_first_of_msg) 762 { 763 struct tegra_qspi *tqspi = spi_master_get_devdata(spi->master); 764 struct tegra_qspi_client_data *cdata = spi->controller_data; 765 u32 command1, command2, speed = t->speed_hz; 766 u8 bits_per_word = t->bits_per_word; 767 u32 tx_tap = 0, rx_tap = 0; 768 int req_mode; 769 770 if (speed != tqspi->cur_speed) { 771 clk_set_rate(tqspi->clk, speed); 772 tqspi->cur_speed = speed; 773 } 774 775 tqspi->cur_pos = 0; 776 tqspi->cur_rx_pos = 0; 777 tqspi->cur_tx_pos = 0; 778 tqspi->curr_xfer = t; 779 780 if (is_first_of_msg) { 781 tegra_qspi_mask_clear_irq(tqspi); 782 783 command1 = tqspi->def_command1_reg; 784 command1 |= QSPI_BIT_LENGTH(bits_per_word - 1); 785 786 command1 &= ~QSPI_CONTROL_MODE_MASK; 787 req_mode = spi->mode & 0x3; 788 if (req_mode == SPI_MODE_3) 789 command1 |= QSPI_CONTROL_MODE_3; 790 else 791 command1 |= QSPI_CONTROL_MODE_0; 792 793 if (spi->mode & SPI_CS_HIGH) 794 command1 |= QSPI_CS_SW_VAL; 795 else 796 command1 &= ~QSPI_CS_SW_VAL; 797 tegra_qspi_writel(tqspi, command1, QSPI_COMMAND1); 798 799 if (cdata && cdata->tx_clk_tap_delay) 800 tx_tap = cdata->tx_clk_tap_delay; 801 802 if (cdata && cdata->rx_clk_tap_delay) 803 rx_tap = cdata->rx_clk_tap_delay; 804 805 command2 = QSPI_TX_TAP_DELAY(tx_tap) | QSPI_RX_TAP_DELAY(rx_tap); 806 if (command2 != tqspi->def_command2_reg) 807 tegra_qspi_writel(tqspi, command2, QSPI_COMMAND2); 808 809 } else { 810 command1 = tqspi->command1_reg; 811 command1 &= ~QSPI_BIT_LENGTH(~0); 812 command1 |= QSPI_BIT_LENGTH(bits_per_word - 1); 813 } 814 815 command1 &= ~QSPI_SDR_DDR_SEL; 816 817 return command1; 818 } 819 820 static int tegra_qspi_start_transfer_one(struct spi_device *spi, 821 struct spi_transfer *t, u32 command1) 822 { 823 struct tegra_qspi *tqspi = spi_master_get_devdata(spi->master); 824 unsigned int total_fifo_words; 825 u8 bus_width = 0; 826 int ret; 827 828 total_fifo_words = tegra_qspi_calculate_curr_xfer_param(tqspi, t); 829 830 command1 &= ~QSPI_PACKED; 831 if (tqspi->is_packed) 832 command1 |= QSPI_PACKED; 833 tegra_qspi_writel(tqspi, command1, QSPI_COMMAND1); 834 835 tqspi->cur_direction = 0; 836 837 command1 &= ~(QSPI_TX_EN | QSPI_RX_EN); 838 if (t->rx_buf) { 839 command1 |= QSPI_RX_EN; 840 tqspi->cur_direction |= DATA_DIR_RX; 841 bus_width = t->rx_nbits; 842 } 843 844 if (t->tx_buf) { 845 command1 |= QSPI_TX_EN; 846 tqspi->cur_direction |= DATA_DIR_TX; 847 bus_width = t->tx_nbits; 848 } 849 850 command1 &= ~QSPI_INTERFACE_WIDTH_MASK; 851 852 if (bus_width == SPI_NBITS_QUAD) 853 command1 |= QSPI_INTERFACE_WIDTH_QUAD; 854 else if (bus_width == SPI_NBITS_DUAL) 855 command1 |= QSPI_INTERFACE_WIDTH_DUAL; 856 else 857 command1 |= QSPI_INTERFACE_WIDTH_SINGLE; 858 859 tqspi->command1_reg = command1; 860 861 tegra_qspi_writel(tqspi, QSPI_NUM_DUMMY_CYCLE(tqspi->dummy_cycles), QSPI_MISC_REG); 862 863 ret = tegra_qspi_flush_fifos(tqspi, false); 864 if (ret < 0) 865 return ret; 866 867 if (tqspi->use_dma && total_fifo_words > QSPI_FIFO_DEPTH) 868 ret = tegra_qspi_start_dma_based_transfer(tqspi, t); 869 else 870 ret = tegra_qspi_start_cpu_based_transfer(tqspi, t); 871 872 return ret; 873 } 874 875 static struct tegra_qspi_client_data *tegra_qspi_parse_cdata_dt(struct spi_device *spi) 876 { 877 struct tegra_qspi_client_data *cdata; 878 struct device_node *slave_np = spi->dev.of_node; 879 880 cdata = kzalloc(sizeof(*cdata), GFP_KERNEL); 881 if (!cdata) 882 return NULL; 883 884 of_property_read_u32(slave_np, "nvidia,tx-clk-tap-delay", 885 &cdata->tx_clk_tap_delay); 886 of_property_read_u32(slave_np, "nvidia,rx-clk-tap-delay", 887 &cdata->rx_clk_tap_delay); 888 return cdata; 889 } 890 891 static void tegra_qspi_cleanup(struct spi_device *spi) 892 { 893 struct tegra_qspi_client_data *cdata = spi->controller_data; 894 895 spi->controller_data = NULL; 896 kfree(cdata); 897 } 898 899 static int tegra_qspi_setup(struct spi_device *spi) 900 { 901 struct tegra_qspi *tqspi = spi_master_get_devdata(spi->master); 902 struct tegra_qspi_client_data *cdata = spi->controller_data; 903 unsigned long flags; 904 u32 val; 905 int ret; 906 907 ret = pm_runtime_resume_and_get(tqspi->dev); 908 if (ret < 0) { 909 dev_err(tqspi->dev, "failed to get runtime PM: %d\n", ret); 910 return ret; 911 } 912 913 if (!cdata) { 914 cdata = tegra_qspi_parse_cdata_dt(spi); 915 spi->controller_data = cdata; 916 } 917 918 spin_lock_irqsave(&tqspi->lock, flags); 919 920 /* keep default cs state to inactive */ 921 val = tqspi->def_command1_reg; 922 if (spi->mode & SPI_CS_HIGH) 923 val &= ~QSPI_CS_SW_VAL; 924 else 925 val |= QSPI_CS_SW_VAL; 926 927 tqspi->def_command1_reg = val; 928 tegra_qspi_writel(tqspi, tqspi->def_command1_reg, QSPI_COMMAND1); 929 930 spin_unlock_irqrestore(&tqspi->lock, flags); 931 932 pm_runtime_put(tqspi->dev); 933 934 return 0; 935 } 936 937 static void tegra_qspi_dump_regs(struct tegra_qspi *tqspi) 938 { 939 dev_dbg(tqspi->dev, "============ QSPI REGISTER DUMP ============\n"); 940 dev_dbg(tqspi->dev, "Command1: 0x%08x | Command2: 0x%08x\n", 941 tegra_qspi_readl(tqspi, QSPI_COMMAND1), 942 tegra_qspi_readl(tqspi, QSPI_COMMAND2)); 943 dev_dbg(tqspi->dev, "DMA_CTL: 0x%08x | DMA_BLK: 0x%08x\n", 944 tegra_qspi_readl(tqspi, QSPI_DMA_CTL), 945 tegra_qspi_readl(tqspi, QSPI_DMA_BLK)); 946 dev_dbg(tqspi->dev, "INTR_MASK: 0x%08x | MISC: 0x%08x\n", 947 tegra_qspi_readl(tqspi, QSPI_INTR_MASK), 948 tegra_qspi_readl(tqspi, QSPI_MISC_REG)); 949 dev_dbg(tqspi->dev, "TRANS_STAT: 0x%08x | FIFO_STATUS: 0x%08x\n", 950 tegra_qspi_readl(tqspi, QSPI_TRANS_STATUS), 951 tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS)); 952 } 953 954 static void tegra_qspi_handle_error(struct tegra_qspi *tqspi) 955 { 956 dev_err(tqspi->dev, "error in transfer, fifo status 0x%08x\n", tqspi->status_reg); 957 tegra_qspi_dump_regs(tqspi); 958 tegra_qspi_flush_fifos(tqspi, true); 959 reset_control_assert(tqspi->rst); 960 udelay(2); 961 reset_control_deassert(tqspi->rst); 962 } 963 964 static void tegra_qspi_transfer_end(struct spi_device *spi) 965 { 966 struct tegra_qspi *tqspi = spi_master_get_devdata(spi->master); 967 int cs_val = (spi->mode & SPI_CS_HIGH) ? 0 : 1; 968 969 if (cs_val) 970 tqspi->command1_reg |= QSPI_CS_SW_VAL; 971 else 972 tqspi->command1_reg &= ~QSPI_CS_SW_VAL; 973 tegra_qspi_writel(tqspi, tqspi->command1_reg, QSPI_COMMAND1); 974 tegra_qspi_writel(tqspi, tqspi->def_command1_reg, QSPI_COMMAND1); 975 } 976 977 static int tegra_qspi_transfer_one_message(struct spi_master *master, struct spi_message *msg) 978 { 979 struct tegra_qspi *tqspi = spi_master_get_devdata(master); 980 struct spi_device *spi = msg->spi; 981 struct spi_transfer *transfer; 982 bool is_first_msg = true; 983 int ret; 984 985 msg->status = 0; 986 msg->actual_length = 0; 987 tqspi->tx_status = 0; 988 tqspi->rx_status = 0; 989 990 list_for_each_entry(transfer, &msg->transfers, transfer_list) { 991 struct spi_transfer *xfer = transfer; 992 u8 dummy_bytes = 0; 993 u32 cmd1; 994 995 tqspi->dummy_cycles = 0; 996 /* 997 * Tegra QSPI hardware supports dummy bytes transfer after actual transfer 998 * bytes based on programmed dummy clock cycles in the QSPI_MISC register. 999 * So, check if the next transfer is dummy data transfer and program dummy 1000 * clock cycles along with the current transfer and skip next transfer. 1001 */ 1002 if (!list_is_last(&xfer->transfer_list, &msg->transfers)) { 1003 struct spi_transfer *next_xfer; 1004 1005 next_xfer = list_next_entry(xfer, transfer_list); 1006 if (next_xfer->dummy_data) { 1007 u32 dummy_cycles = next_xfer->len * 8 / next_xfer->tx_nbits; 1008 1009 if (dummy_cycles <= QSPI_DUMMY_CYCLES_MAX) { 1010 tqspi->dummy_cycles = dummy_cycles; 1011 dummy_bytes = next_xfer->len; 1012 transfer = next_xfer; 1013 } 1014 } 1015 } 1016 1017 reinit_completion(&tqspi->xfer_completion); 1018 1019 cmd1 = tegra_qspi_setup_transfer_one(spi, xfer, is_first_msg); 1020 1021 ret = tegra_qspi_start_transfer_one(spi, xfer, cmd1); 1022 if (ret < 0) { 1023 dev_err(tqspi->dev, "failed to start transfer: %d\n", ret); 1024 goto complete_xfer; 1025 } 1026 1027 is_first_msg = false; 1028 ret = wait_for_completion_timeout(&tqspi->xfer_completion, 1029 QSPI_DMA_TIMEOUT); 1030 if (WARN_ON(ret == 0)) { 1031 dev_err(tqspi->dev, "transfer timeout\n"); 1032 if (tqspi->is_curr_dma_xfer && (tqspi->cur_direction & DATA_DIR_TX)) 1033 dmaengine_terminate_all(tqspi->tx_dma_chan); 1034 if (tqspi->is_curr_dma_xfer && (tqspi->cur_direction & DATA_DIR_RX)) 1035 dmaengine_terminate_all(tqspi->rx_dma_chan); 1036 tegra_qspi_handle_error(tqspi); 1037 ret = -EIO; 1038 goto complete_xfer; 1039 } 1040 1041 if (tqspi->tx_status || tqspi->rx_status) { 1042 tegra_qspi_handle_error(tqspi); 1043 ret = -EIO; 1044 goto complete_xfer; 1045 } 1046 1047 msg->actual_length += xfer->len + dummy_bytes; 1048 1049 complete_xfer: 1050 if (ret < 0) { 1051 tegra_qspi_transfer_end(spi); 1052 spi_transfer_delay_exec(xfer); 1053 goto exit; 1054 } 1055 1056 if (list_is_last(&xfer->transfer_list, &msg->transfers)) { 1057 /* de-activate CS after last transfer only when cs_change is not set */ 1058 if (!xfer->cs_change) { 1059 tegra_qspi_transfer_end(spi); 1060 spi_transfer_delay_exec(xfer); 1061 } 1062 } else if (xfer->cs_change) { 1063 /* de-activated CS between the transfers only when cs_change is set */ 1064 tegra_qspi_transfer_end(spi); 1065 spi_transfer_delay_exec(xfer); 1066 } 1067 } 1068 1069 ret = 0; 1070 exit: 1071 msg->status = ret; 1072 spi_finalize_current_message(master); 1073 return ret; 1074 } 1075 1076 static irqreturn_t handle_cpu_based_xfer(struct tegra_qspi *tqspi) 1077 { 1078 struct spi_transfer *t = tqspi->curr_xfer; 1079 unsigned long flags; 1080 1081 spin_lock_irqsave(&tqspi->lock, flags); 1082 1083 if (tqspi->tx_status || tqspi->rx_status) { 1084 tegra_qspi_handle_error(tqspi); 1085 complete(&tqspi->xfer_completion); 1086 goto exit; 1087 } 1088 1089 if (tqspi->cur_direction & DATA_DIR_RX) 1090 tegra_qspi_read_rx_fifo_to_client_rxbuf(tqspi, t); 1091 1092 if (tqspi->cur_direction & DATA_DIR_TX) 1093 tqspi->cur_pos = tqspi->cur_tx_pos; 1094 else 1095 tqspi->cur_pos = tqspi->cur_rx_pos; 1096 1097 if (tqspi->cur_pos == t->len) { 1098 complete(&tqspi->xfer_completion); 1099 goto exit; 1100 } 1101 1102 tegra_qspi_calculate_curr_xfer_param(tqspi, t); 1103 tegra_qspi_start_cpu_based_transfer(tqspi, t); 1104 exit: 1105 spin_unlock_irqrestore(&tqspi->lock, flags); 1106 return IRQ_HANDLED; 1107 } 1108 1109 static irqreturn_t handle_dma_based_xfer(struct tegra_qspi *tqspi) 1110 { 1111 struct spi_transfer *t = tqspi->curr_xfer; 1112 unsigned int total_fifo_words; 1113 unsigned long flags; 1114 long wait_status; 1115 int err = 0; 1116 1117 if (tqspi->cur_direction & DATA_DIR_TX) { 1118 if (tqspi->tx_status) { 1119 dmaengine_terminate_all(tqspi->tx_dma_chan); 1120 err += 1; 1121 } else { 1122 wait_status = wait_for_completion_interruptible_timeout( 1123 &tqspi->tx_dma_complete, QSPI_DMA_TIMEOUT); 1124 if (wait_status <= 0) { 1125 dmaengine_terminate_all(tqspi->tx_dma_chan); 1126 dev_err(tqspi->dev, "failed TX DMA transfer\n"); 1127 err += 1; 1128 } 1129 } 1130 } 1131 1132 if (tqspi->cur_direction & DATA_DIR_RX) { 1133 if (tqspi->rx_status) { 1134 dmaengine_terminate_all(tqspi->rx_dma_chan); 1135 err += 2; 1136 } else { 1137 wait_status = wait_for_completion_interruptible_timeout( 1138 &tqspi->rx_dma_complete, QSPI_DMA_TIMEOUT); 1139 if (wait_status <= 0) { 1140 dmaengine_terminate_all(tqspi->rx_dma_chan); 1141 dev_err(tqspi->dev, "failed RX DMA transfer\n"); 1142 err += 2; 1143 } 1144 } 1145 } 1146 1147 spin_lock_irqsave(&tqspi->lock, flags); 1148 1149 if (err) { 1150 tegra_qspi_dma_unmap_xfer(tqspi, t); 1151 tegra_qspi_handle_error(tqspi); 1152 complete(&tqspi->xfer_completion); 1153 goto exit; 1154 } 1155 1156 if (tqspi->cur_direction & DATA_DIR_RX) 1157 tegra_qspi_copy_qspi_rxbuf_to_client_rxbuf(tqspi, t); 1158 1159 if (tqspi->cur_direction & DATA_DIR_TX) 1160 tqspi->cur_pos = tqspi->cur_tx_pos; 1161 else 1162 tqspi->cur_pos = tqspi->cur_rx_pos; 1163 1164 if (tqspi->cur_pos == t->len) { 1165 tegra_qspi_dma_unmap_xfer(tqspi, t); 1166 complete(&tqspi->xfer_completion); 1167 goto exit; 1168 } 1169 1170 tegra_qspi_dma_unmap_xfer(tqspi, t); 1171 1172 /* continue transfer in current message */ 1173 total_fifo_words = tegra_qspi_calculate_curr_xfer_param(tqspi, t); 1174 if (total_fifo_words > QSPI_FIFO_DEPTH) 1175 err = tegra_qspi_start_dma_based_transfer(tqspi, t); 1176 else 1177 err = tegra_qspi_start_cpu_based_transfer(tqspi, t); 1178 1179 exit: 1180 spin_unlock_irqrestore(&tqspi->lock, flags); 1181 return IRQ_HANDLED; 1182 } 1183 1184 static irqreturn_t tegra_qspi_isr_thread(int irq, void *context_data) 1185 { 1186 struct tegra_qspi *tqspi = context_data; 1187 1188 tqspi->status_reg = tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS); 1189 1190 if (tqspi->cur_direction & DATA_DIR_TX) 1191 tqspi->tx_status = tqspi->status_reg & (QSPI_TX_FIFO_UNF | QSPI_TX_FIFO_OVF); 1192 1193 if (tqspi->cur_direction & DATA_DIR_RX) 1194 tqspi->rx_status = tqspi->status_reg & (QSPI_RX_FIFO_OVF | QSPI_RX_FIFO_UNF); 1195 1196 tegra_qspi_mask_clear_irq(tqspi); 1197 1198 if (!tqspi->is_curr_dma_xfer) 1199 return handle_cpu_based_xfer(tqspi); 1200 1201 return handle_dma_based_xfer(tqspi); 1202 } 1203 1204 static const struct of_device_id tegra_qspi_of_match[] = { 1205 { .compatible = "nvidia,tegra210-qspi", }, 1206 { .compatible = "nvidia,tegra186-qspi", }, 1207 { .compatible = "nvidia,tegra194-qspi", }, 1208 {} 1209 }; 1210 1211 MODULE_DEVICE_TABLE(of, tegra_qspi_of_match); 1212 1213 static int tegra_qspi_probe(struct platform_device *pdev) 1214 { 1215 struct spi_master *master; 1216 struct tegra_qspi *tqspi; 1217 struct resource *r; 1218 int ret, qspi_irq; 1219 int bus_num; 1220 1221 master = devm_spi_alloc_master(&pdev->dev, sizeof(*tqspi)); 1222 if (!master) 1223 return -ENOMEM; 1224 1225 platform_set_drvdata(pdev, master); 1226 tqspi = spi_master_get_devdata(master); 1227 1228 master->mode_bits = SPI_MODE_0 | SPI_MODE_3 | SPI_CS_HIGH | 1229 SPI_TX_DUAL | SPI_RX_DUAL | SPI_TX_QUAD | SPI_RX_QUAD; 1230 master->bits_per_word_mask = SPI_BPW_MASK(32) | SPI_BPW_MASK(16) | SPI_BPW_MASK(8); 1231 master->setup = tegra_qspi_setup; 1232 master->cleanup = tegra_qspi_cleanup; 1233 master->transfer_one_message = tegra_qspi_transfer_one_message; 1234 master->num_chipselect = 1; 1235 master->auto_runtime_pm = true; 1236 1237 bus_num = of_alias_get_id(pdev->dev.of_node, "spi"); 1238 if (bus_num >= 0) 1239 master->bus_num = bus_num; 1240 1241 tqspi->master = master; 1242 tqspi->dev = &pdev->dev; 1243 spin_lock_init(&tqspi->lock); 1244 1245 r = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1246 tqspi->base = devm_ioremap_resource(&pdev->dev, r); 1247 if (IS_ERR(tqspi->base)) 1248 return PTR_ERR(tqspi->base); 1249 1250 tqspi->phys = r->start; 1251 qspi_irq = platform_get_irq(pdev, 0); 1252 tqspi->irq = qspi_irq; 1253 1254 tqspi->clk = devm_clk_get(&pdev->dev, "qspi"); 1255 if (IS_ERR(tqspi->clk)) { 1256 ret = PTR_ERR(tqspi->clk); 1257 dev_err(&pdev->dev, "failed to get clock: %d\n", ret); 1258 return ret; 1259 } 1260 1261 tqspi->rst = devm_reset_control_get_exclusive(&pdev->dev, NULL); 1262 if (IS_ERR(tqspi->rst)) { 1263 ret = PTR_ERR(tqspi->rst); 1264 dev_err(&pdev->dev, "failed to get reset control: %d\n", ret); 1265 return ret; 1266 } 1267 1268 tqspi->max_buf_size = QSPI_FIFO_DEPTH << 2; 1269 tqspi->dma_buf_size = DEFAULT_QSPI_DMA_BUF_LEN; 1270 1271 ret = tegra_qspi_init_dma(tqspi); 1272 if (ret < 0) 1273 return ret; 1274 1275 if (tqspi->use_dma) 1276 tqspi->max_buf_size = tqspi->dma_buf_size; 1277 1278 init_completion(&tqspi->tx_dma_complete); 1279 init_completion(&tqspi->rx_dma_complete); 1280 init_completion(&tqspi->xfer_completion); 1281 1282 pm_runtime_enable(&pdev->dev); 1283 ret = pm_runtime_resume_and_get(&pdev->dev); 1284 if (ret < 0) { 1285 dev_err(&pdev->dev, "failed to get runtime PM: %d\n", ret); 1286 goto exit_pm_disable; 1287 } 1288 1289 reset_control_assert(tqspi->rst); 1290 udelay(2); 1291 reset_control_deassert(tqspi->rst); 1292 1293 tqspi->def_command1_reg = QSPI_M_S | QSPI_CS_SW_HW | QSPI_CS_SW_VAL; 1294 tegra_qspi_writel(tqspi, tqspi->def_command1_reg, QSPI_COMMAND1); 1295 tqspi->spi_cs_timing1 = tegra_qspi_readl(tqspi, QSPI_CS_TIMING1); 1296 tqspi->spi_cs_timing2 = tegra_qspi_readl(tqspi, QSPI_CS_TIMING2); 1297 tqspi->def_command2_reg = tegra_qspi_readl(tqspi, QSPI_COMMAND2); 1298 1299 pm_runtime_put(&pdev->dev); 1300 1301 ret = request_threaded_irq(tqspi->irq, NULL, 1302 tegra_qspi_isr_thread, IRQF_ONESHOT, 1303 dev_name(&pdev->dev), tqspi); 1304 if (ret < 0) { 1305 dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", tqspi->irq, ret); 1306 goto exit_pm_disable; 1307 } 1308 1309 master->dev.of_node = pdev->dev.of_node; 1310 ret = spi_register_master(master); 1311 if (ret < 0) { 1312 dev_err(&pdev->dev, "failed to register master: %d\n", ret); 1313 goto exit_free_irq; 1314 } 1315 1316 return 0; 1317 1318 exit_free_irq: 1319 free_irq(qspi_irq, tqspi); 1320 exit_pm_disable: 1321 pm_runtime_disable(&pdev->dev); 1322 tegra_qspi_deinit_dma(tqspi); 1323 return ret; 1324 } 1325 1326 static int tegra_qspi_remove(struct platform_device *pdev) 1327 { 1328 struct spi_master *master = platform_get_drvdata(pdev); 1329 struct tegra_qspi *tqspi = spi_master_get_devdata(master); 1330 1331 spi_unregister_master(master); 1332 free_irq(tqspi->irq, tqspi); 1333 pm_runtime_disable(&pdev->dev); 1334 tegra_qspi_deinit_dma(tqspi); 1335 1336 return 0; 1337 } 1338 1339 static int __maybe_unused tegra_qspi_suspend(struct device *dev) 1340 { 1341 struct spi_master *master = dev_get_drvdata(dev); 1342 1343 return spi_master_suspend(master); 1344 } 1345 1346 static int __maybe_unused tegra_qspi_resume(struct device *dev) 1347 { 1348 struct spi_master *master = dev_get_drvdata(dev); 1349 struct tegra_qspi *tqspi = spi_master_get_devdata(master); 1350 int ret; 1351 1352 ret = pm_runtime_resume_and_get(dev); 1353 if (ret < 0) { 1354 dev_err(dev, "failed to get runtime PM: %d\n", ret); 1355 return ret; 1356 } 1357 1358 tegra_qspi_writel(tqspi, tqspi->command1_reg, QSPI_COMMAND1); 1359 tegra_qspi_writel(tqspi, tqspi->def_command2_reg, QSPI_COMMAND2); 1360 pm_runtime_put(dev); 1361 1362 return spi_master_resume(master); 1363 } 1364 1365 static int __maybe_unused tegra_qspi_runtime_suspend(struct device *dev) 1366 { 1367 struct spi_master *master = dev_get_drvdata(dev); 1368 struct tegra_qspi *tqspi = spi_master_get_devdata(master); 1369 1370 /* flush all write which are in PPSB queue by reading back */ 1371 tegra_qspi_readl(tqspi, QSPI_COMMAND1); 1372 1373 clk_disable_unprepare(tqspi->clk); 1374 1375 return 0; 1376 } 1377 1378 static int __maybe_unused tegra_qspi_runtime_resume(struct device *dev) 1379 { 1380 struct spi_master *master = dev_get_drvdata(dev); 1381 struct tegra_qspi *tqspi = spi_master_get_devdata(master); 1382 int ret; 1383 1384 ret = clk_prepare_enable(tqspi->clk); 1385 if (ret < 0) 1386 dev_err(tqspi->dev, "failed to enable clock: %d\n", ret); 1387 1388 return ret; 1389 } 1390 1391 static const struct dev_pm_ops tegra_qspi_pm_ops = { 1392 SET_RUNTIME_PM_OPS(tegra_qspi_runtime_suspend, tegra_qspi_runtime_resume, NULL) 1393 SET_SYSTEM_SLEEP_PM_OPS(tegra_qspi_suspend, tegra_qspi_resume) 1394 }; 1395 1396 static struct platform_driver tegra_qspi_driver = { 1397 .driver = { 1398 .name = "tegra-qspi", 1399 .pm = &tegra_qspi_pm_ops, 1400 .of_match_table = tegra_qspi_of_match, 1401 }, 1402 .probe = tegra_qspi_probe, 1403 .remove = tegra_qspi_remove, 1404 }; 1405 module_platform_driver(tegra_qspi_driver); 1406 1407 MODULE_ALIAS("platform:qspi-tegra"); 1408 MODULE_DESCRIPTION("NVIDIA Tegra QSPI Controller Driver"); 1409 MODULE_AUTHOR("Sowjanya Komatineni <skomatineni@nvidia.com>"); 1410 MODULE_LICENSE("GPL v2"); 1411