1 /* 2 * Driver for Atmel AT32 and AT91 SPI Controllers 3 * 4 * Copyright (C) 2006 Atmel Corporation 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <linux/kernel.h> 12 #include <linux/clk.h> 13 #include <linux/module.h> 14 #include <linux/platform_device.h> 15 #include <linux/delay.h> 16 #include <linux/dma-mapping.h> 17 #include <linux/dmaengine.h> 18 #include <linux/err.h> 19 #include <linux/interrupt.h> 20 #include <linux/spi/spi.h> 21 #include <linux/slab.h> 22 #include <linux/platform_data/dma-atmel.h> 23 #include <linux/of.h> 24 25 #include <linux/io.h> 26 #include <linux/gpio/consumer.h> 27 #include <linux/pinctrl/consumer.h> 28 #include <linux/pm_runtime.h> 29 30 /* SPI register offsets */ 31 #define SPI_CR 0x0000 32 #define SPI_MR 0x0004 33 #define SPI_RDR 0x0008 34 #define SPI_TDR 0x000c 35 #define SPI_SR 0x0010 36 #define SPI_IER 0x0014 37 #define SPI_IDR 0x0018 38 #define SPI_IMR 0x001c 39 #define SPI_CSR0 0x0030 40 #define SPI_CSR1 0x0034 41 #define SPI_CSR2 0x0038 42 #define SPI_CSR3 0x003c 43 #define SPI_FMR 0x0040 44 #define SPI_FLR 0x0044 45 #define SPI_VERSION 0x00fc 46 #define SPI_RPR 0x0100 47 #define SPI_RCR 0x0104 48 #define SPI_TPR 0x0108 49 #define SPI_TCR 0x010c 50 #define SPI_RNPR 0x0110 51 #define SPI_RNCR 0x0114 52 #define SPI_TNPR 0x0118 53 #define SPI_TNCR 0x011c 54 #define SPI_PTCR 0x0120 55 #define SPI_PTSR 0x0124 56 57 /* Bitfields in CR */ 58 #define SPI_SPIEN_OFFSET 0 59 #define SPI_SPIEN_SIZE 1 60 #define SPI_SPIDIS_OFFSET 1 61 #define SPI_SPIDIS_SIZE 1 62 #define SPI_SWRST_OFFSET 7 63 #define SPI_SWRST_SIZE 1 64 #define SPI_LASTXFER_OFFSET 24 65 #define SPI_LASTXFER_SIZE 1 66 #define SPI_TXFCLR_OFFSET 16 67 #define SPI_TXFCLR_SIZE 1 68 #define SPI_RXFCLR_OFFSET 17 69 #define SPI_RXFCLR_SIZE 1 70 #define SPI_FIFOEN_OFFSET 30 71 #define SPI_FIFOEN_SIZE 1 72 #define SPI_FIFODIS_OFFSET 31 73 #define SPI_FIFODIS_SIZE 1 74 75 /* Bitfields in MR */ 76 #define SPI_MSTR_OFFSET 0 77 #define SPI_MSTR_SIZE 1 78 #define SPI_PS_OFFSET 1 79 #define SPI_PS_SIZE 1 80 #define SPI_PCSDEC_OFFSET 2 81 #define SPI_PCSDEC_SIZE 1 82 #define SPI_FDIV_OFFSET 3 83 #define SPI_FDIV_SIZE 1 84 #define SPI_MODFDIS_OFFSET 4 85 #define SPI_MODFDIS_SIZE 1 86 #define SPI_WDRBT_OFFSET 5 87 #define SPI_WDRBT_SIZE 1 88 #define SPI_LLB_OFFSET 7 89 #define SPI_LLB_SIZE 1 90 #define SPI_PCS_OFFSET 16 91 #define SPI_PCS_SIZE 4 92 #define SPI_DLYBCS_OFFSET 24 93 #define SPI_DLYBCS_SIZE 8 94 95 /* Bitfields in RDR */ 96 #define SPI_RD_OFFSET 0 97 #define SPI_RD_SIZE 16 98 99 /* Bitfields in TDR */ 100 #define SPI_TD_OFFSET 0 101 #define SPI_TD_SIZE 16 102 103 /* Bitfields in SR */ 104 #define SPI_RDRF_OFFSET 0 105 #define SPI_RDRF_SIZE 1 106 #define SPI_TDRE_OFFSET 1 107 #define SPI_TDRE_SIZE 1 108 #define SPI_MODF_OFFSET 2 109 #define SPI_MODF_SIZE 1 110 #define SPI_OVRES_OFFSET 3 111 #define SPI_OVRES_SIZE 1 112 #define SPI_ENDRX_OFFSET 4 113 #define SPI_ENDRX_SIZE 1 114 #define SPI_ENDTX_OFFSET 5 115 #define SPI_ENDTX_SIZE 1 116 #define SPI_RXBUFF_OFFSET 6 117 #define SPI_RXBUFF_SIZE 1 118 #define SPI_TXBUFE_OFFSET 7 119 #define SPI_TXBUFE_SIZE 1 120 #define SPI_NSSR_OFFSET 8 121 #define SPI_NSSR_SIZE 1 122 #define SPI_TXEMPTY_OFFSET 9 123 #define SPI_TXEMPTY_SIZE 1 124 #define SPI_SPIENS_OFFSET 16 125 #define SPI_SPIENS_SIZE 1 126 #define SPI_TXFEF_OFFSET 24 127 #define SPI_TXFEF_SIZE 1 128 #define SPI_TXFFF_OFFSET 25 129 #define SPI_TXFFF_SIZE 1 130 #define SPI_TXFTHF_OFFSET 26 131 #define SPI_TXFTHF_SIZE 1 132 #define SPI_RXFEF_OFFSET 27 133 #define SPI_RXFEF_SIZE 1 134 #define SPI_RXFFF_OFFSET 28 135 #define SPI_RXFFF_SIZE 1 136 #define SPI_RXFTHF_OFFSET 29 137 #define SPI_RXFTHF_SIZE 1 138 #define SPI_TXFPTEF_OFFSET 30 139 #define SPI_TXFPTEF_SIZE 1 140 #define SPI_RXFPTEF_OFFSET 31 141 #define SPI_RXFPTEF_SIZE 1 142 143 /* Bitfields in CSR0 */ 144 #define SPI_CPOL_OFFSET 0 145 #define SPI_CPOL_SIZE 1 146 #define SPI_NCPHA_OFFSET 1 147 #define SPI_NCPHA_SIZE 1 148 #define SPI_CSAAT_OFFSET 3 149 #define SPI_CSAAT_SIZE 1 150 #define SPI_BITS_OFFSET 4 151 #define SPI_BITS_SIZE 4 152 #define SPI_SCBR_OFFSET 8 153 #define SPI_SCBR_SIZE 8 154 #define SPI_DLYBS_OFFSET 16 155 #define SPI_DLYBS_SIZE 8 156 #define SPI_DLYBCT_OFFSET 24 157 #define SPI_DLYBCT_SIZE 8 158 159 /* Bitfields in RCR */ 160 #define SPI_RXCTR_OFFSET 0 161 #define SPI_RXCTR_SIZE 16 162 163 /* Bitfields in TCR */ 164 #define SPI_TXCTR_OFFSET 0 165 #define SPI_TXCTR_SIZE 16 166 167 /* Bitfields in RNCR */ 168 #define SPI_RXNCR_OFFSET 0 169 #define SPI_RXNCR_SIZE 16 170 171 /* Bitfields in TNCR */ 172 #define SPI_TXNCR_OFFSET 0 173 #define SPI_TXNCR_SIZE 16 174 175 /* Bitfields in PTCR */ 176 #define SPI_RXTEN_OFFSET 0 177 #define SPI_RXTEN_SIZE 1 178 #define SPI_RXTDIS_OFFSET 1 179 #define SPI_RXTDIS_SIZE 1 180 #define SPI_TXTEN_OFFSET 8 181 #define SPI_TXTEN_SIZE 1 182 #define SPI_TXTDIS_OFFSET 9 183 #define SPI_TXTDIS_SIZE 1 184 185 /* Bitfields in FMR */ 186 #define SPI_TXRDYM_OFFSET 0 187 #define SPI_TXRDYM_SIZE 2 188 #define SPI_RXRDYM_OFFSET 4 189 #define SPI_RXRDYM_SIZE 2 190 #define SPI_TXFTHRES_OFFSET 16 191 #define SPI_TXFTHRES_SIZE 6 192 #define SPI_RXFTHRES_OFFSET 24 193 #define SPI_RXFTHRES_SIZE 6 194 195 /* Bitfields in FLR */ 196 #define SPI_TXFL_OFFSET 0 197 #define SPI_TXFL_SIZE 6 198 #define SPI_RXFL_OFFSET 16 199 #define SPI_RXFL_SIZE 6 200 201 /* Constants for BITS */ 202 #define SPI_BITS_8_BPT 0 203 #define SPI_BITS_9_BPT 1 204 #define SPI_BITS_10_BPT 2 205 #define SPI_BITS_11_BPT 3 206 #define SPI_BITS_12_BPT 4 207 #define SPI_BITS_13_BPT 5 208 #define SPI_BITS_14_BPT 6 209 #define SPI_BITS_15_BPT 7 210 #define SPI_BITS_16_BPT 8 211 #define SPI_ONE_DATA 0 212 #define SPI_TWO_DATA 1 213 #define SPI_FOUR_DATA 2 214 215 /* Bit manipulation macros */ 216 #define SPI_BIT(name) \ 217 (1 << SPI_##name##_OFFSET) 218 #define SPI_BF(name, value) \ 219 (((value) & ((1 << SPI_##name##_SIZE) - 1)) << SPI_##name##_OFFSET) 220 #define SPI_BFEXT(name, value) \ 221 (((value) >> SPI_##name##_OFFSET) & ((1 << SPI_##name##_SIZE) - 1)) 222 #define SPI_BFINS(name, value, old) \ 223 (((old) & ~(((1 << SPI_##name##_SIZE) - 1) << SPI_##name##_OFFSET)) \ 224 | SPI_BF(name, value)) 225 226 /* Register access macros */ 227 #ifdef CONFIG_AVR32 228 #define spi_readl(port, reg) \ 229 __raw_readl((port)->regs + SPI_##reg) 230 #define spi_writel(port, reg, value) \ 231 __raw_writel((value), (port)->regs + SPI_##reg) 232 233 #define spi_readw(port, reg) \ 234 __raw_readw((port)->regs + SPI_##reg) 235 #define spi_writew(port, reg, value) \ 236 __raw_writew((value), (port)->regs + SPI_##reg) 237 238 #define spi_readb(port, reg) \ 239 __raw_readb((port)->regs + SPI_##reg) 240 #define spi_writeb(port, reg, value) \ 241 __raw_writeb((value), (port)->regs + SPI_##reg) 242 #else 243 #define spi_readl(port, reg) \ 244 readl_relaxed((port)->regs + SPI_##reg) 245 #define spi_writel(port, reg, value) \ 246 writel_relaxed((value), (port)->regs + SPI_##reg) 247 248 #define spi_readw(port, reg) \ 249 readw_relaxed((port)->regs + SPI_##reg) 250 #define spi_writew(port, reg, value) \ 251 writew_relaxed((value), (port)->regs + SPI_##reg) 252 253 #define spi_readb(port, reg) \ 254 readb_relaxed((port)->regs + SPI_##reg) 255 #define spi_writeb(port, reg, value) \ 256 writeb_relaxed((value), (port)->regs + SPI_##reg) 257 #endif 258 /* use PIO for small transfers, avoiding DMA setup/teardown overhead and 259 * cache operations; better heuristics consider wordsize and bitrate. 260 */ 261 #define DMA_MIN_BYTES 16 262 263 #define SPI_DMA_TIMEOUT (msecs_to_jiffies(1000)) 264 265 #define AUTOSUSPEND_TIMEOUT 2000 266 267 struct atmel_spi_caps { 268 bool is_spi2; 269 bool has_wdrbt; 270 bool has_dma_support; 271 bool has_pdc_support; 272 }; 273 274 /* 275 * The core SPI transfer engine just talks to a register bank to set up 276 * DMA transfers; transfer queue progress is driven by IRQs. The clock 277 * framework provides the base clock, subdivided for each spi_device. 278 */ 279 struct atmel_spi { 280 spinlock_t lock; 281 unsigned long flags; 282 283 phys_addr_t phybase; 284 void __iomem *regs; 285 int irq; 286 struct clk *clk; 287 struct platform_device *pdev; 288 unsigned long spi_clk; 289 290 struct spi_transfer *current_transfer; 291 int current_remaining_bytes; 292 int done_status; 293 dma_addr_t dma_addr_rx_bbuf; 294 dma_addr_t dma_addr_tx_bbuf; 295 void *addr_rx_bbuf; 296 void *addr_tx_bbuf; 297 298 struct completion xfer_completion; 299 300 struct atmel_spi_caps caps; 301 302 bool use_dma; 303 bool use_pdc; 304 bool use_cs_gpios; 305 306 bool keep_cs; 307 bool cs_active; 308 309 u32 fifo_size; 310 }; 311 312 /* Controller-specific per-slave state */ 313 struct atmel_spi_device { 314 struct gpio_desc *npcs_pin; 315 u32 csr; 316 }; 317 318 #define SPI_MAX_DMA_XFER 65535 /* true for both PDC and DMA */ 319 #define INVALID_DMA_ADDRESS 0xffffffff 320 321 /* 322 * Version 2 of the SPI controller has 323 * - CR.LASTXFER 324 * - SPI_MR.DIV32 may become FDIV or must-be-zero (here: always zero) 325 * - SPI_SR.TXEMPTY, SPI_SR.NSSR (and corresponding irqs) 326 * - SPI_CSRx.CSAAT 327 * - SPI_CSRx.SBCR allows faster clocking 328 */ 329 static bool atmel_spi_is_v2(struct atmel_spi *as) 330 { 331 return as->caps.is_spi2; 332 } 333 334 /* 335 * Earlier SPI controllers (e.g. on at91rm9200) have a design bug whereby 336 * they assume that spi slave device state will not change on deselect, so 337 * that automagic deselection is OK. ("NPCSx rises if no data is to be 338 * transmitted") Not so! Workaround uses nCSx pins as GPIOs; or newer 339 * controllers have CSAAT and friends. 340 * 341 * Since the CSAAT functionality is a bit weird on newer controllers as 342 * well, we use GPIO to control nCSx pins on all controllers, updating 343 * MR.PCS to avoid confusing the controller. Using GPIOs also lets us 344 * support active-high chipselects despite the controller's belief that 345 * only active-low devices/systems exists. 346 * 347 * However, at91rm9200 has a second erratum whereby nCS0 doesn't work 348 * right when driven with GPIO. ("Mode Fault does not allow more than one 349 * Master on Chip Select 0.") No workaround exists for that ... so for 350 * nCS0 on that chip, we (a) don't use the GPIO, (b) can't support CS_HIGH, 351 * and (c) will trigger that first erratum in some cases. 352 */ 353 354 static void cs_activate(struct atmel_spi *as, struct spi_device *spi) 355 { 356 struct atmel_spi_device *asd = spi->controller_state; 357 u32 mr; 358 359 if (atmel_spi_is_v2(as)) { 360 spi_writel(as, CSR0 + 4 * spi->chip_select, asd->csr); 361 /* For the low SPI version, there is a issue that PDC transfer 362 * on CS1,2,3 needs SPI_CSR0.BITS config as SPI_CSR1,2,3.BITS 363 */ 364 spi_writel(as, CSR0, asd->csr); 365 if (as->caps.has_wdrbt) { 366 spi_writel(as, MR, 367 SPI_BF(PCS, ~(0x01 << spi->chip_select)) 368 | SPI_BIT(WDRBT) 369 | SPI_BIT(MODFDIS) 370 | SPI_BIT(MSTR)); 371 } else { 372 spi_writel(as, MR, 373 SPI_BF(PCS, ~(0x01 << spi->chip_select)) 374 | SPI_BIT(MODFDIS) 375 | SPI_BIT(MSTR)); 376 } 377 378 mr = spi_readl(as, MR); 379 if (as->use_cs_gpios) 380 gpiod_set_value(asd->npcs_pin, 1); 381 } else { 382 u32 cpol = (spi->mode & SPI_CPOL) ? SPI_BIT(CPOL) : 0; 383 int i; 384 u32 csr; 385 386 /* Make sure clock polarity is correct */ 387 for (i = 0; i < spi->master->num_chipselect; i++) { 388 csr = spi_readl(as, CSR0 + 4 * i); 389 if ((csr ^ cpol) & SPI_BIT(CPOL)) 390 spi_writel(as, CSR0 + 4 * i, 391 csr ^ SPI_BIT(CPOL)); 392 } 393 394 mr = spi_readl(as, MR); 395 mr = SPI_BFINS(PCS, ~(1 << spi->chip_select), mr); 396 if (as->use_cs_gpios && spi->chip_select != 0) 397 gpiod_set_value(asd->npcs_pin, 1); 398 spi_writel(as, MR, mr); 399 } 400 401 dev_dbg(&spi->dev, "activate NPCS, mr %08x\n", mr); 402 } 403 404 static void cs_deactivate(struct atmel_spi *as, struct spi_device *spi) 405 { 406 struct atmel_spi_device *asd = spi->controller_state; 407 u32 mr; 408 409 /* only deactivate *this* device; sometimes transfers to 410 * another device may be active when this routine is called. 411 */ 412 mr = spi_readl(as, MR); 413 if (~SPI_BFEXT(PCS, mr) & (1 << spi->chip_select)) { 414 mr = SPI_BFINS(PCS, 0xf, mr); 415 spi_writel(as, MR, mr); 416 } 417 418 dev_dbg(&spi->dev, "DEactivate NPCS, mr %08x\n", mr); 419 420 if (!as->use_cs_gpios) 421 spi_writel(as, CR, SPI_BIT(LASTXFER)); 422 else if (atmel_spi_is_v2(as) || spi->chip_select != 0) 423 gpiod_set_value(asd->npcs_pin, 0); 424 } 425 426 static void atmel_spi_lock(struct atmel_spi *as) __acquires(&as->lock) 427 { 428 spin_lock_irqsave(&as->lock, as->flags); 429 } 430 431 static void atmel_spi_unlock(struct atmel_spi *as) __releases(&as->lock) 432 { 433 spin_unlock_irqrestore(&as->lock, as->flags); 434 } 435 436 static inline bool atmel_spi_is_vmalloc_xfer(struct spi_transfer *xfer) 437 { 438 return is_vmalloc_addr(xfer->tx_buf) || is_vmalloc_addr(xfer->rx_buf); 439 } 440 441 static inline bool atmel_spi_use_dma(struct atmel_spi *as, 442 struct spi_transfer *xfer) 443 { 444 return as->use_dma && xfer->len >= DMA_MIN_BYTES; 445 } 446 447 static bool atmel_spi_can_dma(struct spi_master *master, 448 struct spi_device *spi, 449 struct spi_transfer *xfer) 450 { 451 struct atmel_spi *as = spi_master_get_devdata(master); 452 453 if (IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) 454 return atmel_spi_use_dma(as, xfer) && 455 !atmel_spi_is_vmalloc_xfer(xfer); 456 else 457 return atmel_spi_use_dma(as, xfer); 458 459 } 460 461 static int atmel_spi_dma_slave_config(struct atmel_spi *as, 462 struct dma_slave_config *slave_config, 463 u8 bits_per_word) 464 { 465 struct spi_master *master = platform_get_drvdata(as->pdev); 466 int err = 0; 467 468 if (bits_per_word > 8) { 469 slave_config->dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES; 470 slave_config->src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES; 471 } else { 472 slave_config->dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE; 473 slave_config->src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE; 474 } 475 476 slave_config->dst_addr = (dma_addr_t)as->phybase + SPI_TDR; 477 slave_config->src_addr = (dma_addr_t)as->phybase + SPI_RDR; 478 slave_config->src_maxburst = 1; 479 slave_config->dst_maxburst = 1; 480 slave_config->device_fc = false; 481 482 /* 483 * This driver uses fixed peripheral select mode (PS bit set to '0' in 484 * the Mode Register). 485 * So according to the datasheet, when FIFOs are available (and 486 * enabled), the Transmit FIFO operates in Multiple Data Mode. 487 * In this mode, up to 2 data, not 4, can be written into the Transmit 488 * Data Register in a single access. 489 * However, the first data has to be written into the lowest 16 bits and 490 * the second data into the highest 16 bits of the Transmit 491 * Data Register. For 8bit data (the most frequent case), it would 492 * require to rework tx_buf so each data would actualy fit 16 bits. 493 * So we'd rather write only one data at the time. Hence the transmit 494 * path works the same whether FIFOs are available (and enabled) or not. 495 */ 496 slave_config->direction = DMA_MEM_TO_DEV; 497 if (dmaengine_slave_config(master->dma_tx, slave_config)) { 498 dev_err(&as->pdev->dev, 499 "failed to configure tx dma channel\n"); 500 err = -EINVAL; 501 } 502 503 /* 504 * This driver configures the spi controller for master mode (MSTR bit 505 * set to '1' in the Mode Register). 506 * So according to the datasheet, when FIFOs are available (and 507 * enabled), the Receive FIFO operates in Single Data Mode. 508 * So the receive path works the same whether FIFOs are available (and 509 * enabled) or not. 510 */ 511 slave_config->direction = DMA_DEV_TO_MEM; 512 if (dmaengine_slave_config(master->dma_rx, slave_config)) { 513 dev_err(&as->pdev->dev, 514 "failed to configure rx dma channel\n"); 515 err = -EINVAL; 516 } 517 518 return err; 519 } 520 521 static int atmel_spi_configure_dma(struct spi_master *master, 522 struct atmel_spi *as) 523 { 524 struct dma_slave_config slave_config; 525 struct device *dev = &as->pdev->dev; 526 int err; 527 528 dma_cap_mask_t mask; 529 dma_cap_zero(mask); 530 dma_cap_set(DMA_SLAVE, mask); 531 532 master->dma_tx = dma_request_slave_channel_reason(dev, "tx"); 533 if (IS_ERR(master->dma_tx)) { 534 err = PTR_ERR(master->dma_tx); 535 if (err == -EPROBE_DEFER) { 536 dev_warn(dev, "no DMA channel available at the moment\n"); 537 goto error_clear; 538 } 539 dev_err(dev, 540 "DMA TX channel not available, SPI unable to use DMA\n"); 541 err = -EBUSY; 542 goto error_clear; 543 } 544 545 /* 546 * No reason to check EPROBE_DEFER here since we have already requested 547 * tx channel. If it fails here, it's for another reason. 548 */ 549 master->dma_rx = dma_request_slave_channel(dev, "rx"); 550 551 if (!master->dma_rx) { 552 dev_err(dev, 553 "DMA RX channel not available, SPI unable to use DMA\n"); 554 err = -EBUSY; 555 goto error; 556 } 557 558 err = atmel_spi_dma_slave_config(as, &slave_config, 8); 559 if (err) 560 goto error; 561 562 dev_info(&as->pdev->dev, 563 "Using %s (tx) and %s (rx) for DMA transfers\n", 564 dma_chan_name(master->dma_tx), 565 dma_chan_name(master->dma_rx)); 566 567 return 0; 568 error: 569 if (master->dma_rx) 570 dma_release_channel(master->dma_rx); 571 if (!IS_ERR(master->dma_tx)) 572 dma_release_channel(master->dma_tx); 573 error_clear: 574 master->dma_tx = master->dma_rx = NULL; 575 return err; 576 } 577 578 static void atmel_spi_stop_dma(struct spi_master *master) 579 { 580 if (master->dma_rx) 581 dmaengine_terminate_all(master->dma_rx); 582 if (master->dma_tx) 583 dmaengine_terminate_all(master->dma_tx); 584 } 585 586 static void atmel_spi_release_dma(struct spi_master *master) 587 { 588 if (master->dma_rx) { 589 dma_release_channel(master->dma_rx); 590 master->dma_rx = NULL; 591 } 592 if (master->dma_tx) { 593 dma_release_channel(master->dma_tx); 594 master->dma_tx = NULL; 595 } 596 } 597 598 /* This function is called by the DMA driver from tasklet context */ 599 static void dma_callback(void *data) 600 { 601 struct spi_master *master = data; 602 struct atmel_spi *as = spi_master_get_devdata(master); 603 604 if (is_vmalloc_addr(as->current_transfer->rx_buf) && 605 IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) { 606 memcpy(as->current_transfer->rx_buf, as->addr_rx_bbuf, 607 as->current_transfer->len); 608 } 609 complete(&as->xfer_completion); 610 } 611 612 /* 613 * Next transfer using PIO without FIFO. 614 */ 615 static void atmel_spi_next_xfer_single(struct spi_master *master, 616 struct spi_transfer *xfer) 617 { 618 struct atmel_spi *as = spi_master_get_devdata(master); 619 unsigned long xfer_pos = xfer->len - as->current_remaining_bytes; 620 621 dev_vdbg(master->dev.parent, "atmel_spi_next_xfer_pio\n"); 622 623 /* Make sure data is not remaining in RDR */ 624 spi_readl(as, RDR); 625 while (spi_readl(as, SR) & SPI_BIT(RDRF)) { 626 spi_readl(as, RDR); 627 cpu_relax(); 628 } 629 630 if (xfer->bits_per_word > 8) 631 spi_writel(as, TDR, *(u16 *)(xfer->tx_buf + xfer_pos)); 632 else 633 spi_writel(as, TDR, *(u8 *)(xfer->tx_buf + xfer_pos)); 634 635 dev_dbg(master->dev.parent, 636 " start pio xfer %p: len %u tx %p rx %p bitpw %d\n", 637 xfer, xfer->len, xfer->tx_buf, xfer->rx_buf, 638 xfer->bits_per_word); 639 640 /* Enable relevant interrupts */ 641 spi_writel(as, IER, SPI_BIT(RDRF) | SPI_BIT(OVRES)); 642 } 643 644 /* 645 * Next transfer using PIO with FIFO. 646 */ 647 static void atmel_spi_next_xfer_fifo(struct spi_master *master, 648 struct spi_transfer *xfer) 649 { 650 struct atmel_spi *as = spi_master_get_devdata(master); 651 u32 current_remaining_data, num_data; 652 u32 offset = xfer->len - as->current_remaining_bytes; 653 const u16 *words = (const u16 *)((u8 *)xfer->tx_buf + offset); 654 const u8 *bytes = (const u8 *)((u8 *)xfer->tx_buf + offset); 655 u16 td0, td1; 656 u32 fifomr; 657 658 dev_vdbg(master->dev.parent, "atmel_spi_next_xfer_fifo\n"); 659 660 /* Compute the number of data to transfer in the current iteration */ 661 current_remaining_data = ((xfer->bits_per_word > 8) ? 662 ((u32)as->current_remaining_bytes >> 1) : 663 (u32)as->current_remaining_bytes); 664 num_data = min(current_remaining_data, as->fifo_size); 665 666 /* Flush RX and TX FIFOs */ 667 spi_writel(as, CR, SPI_BIT(RXFCLR) | SPI_BIT(TXFCLR)); 668 while (spi_readl(as, FLR)) 669 cpu_relax(); 670 671 /* Set RX FIFO Threshold to the number of data to transfer */ 672 fifomr = spi_readl(as, FMR); 673 spi_writel(as, FMR, SPI_BFINS(RXFTHRES, num_data, fifomr)); 674 675 /* Clear FIFO flags in the Status Register, especially RXFTHF */ 676 (void)spi_readl(as, SR); 677 678 /* Fill TX FIFO */ 679 while (num_data >= 2) { 680 if (xfer->bits_per_word > 8) { 681 td0 = *words++; 682 td1 = *words++; 683 } else { 684 td0 = *bytes++; 685 td1 = *bytes++; 686 } 687 688 spi_writel(as, TDR, (td1 << 16) | td0); 689 num_data -= 2; 690 } 691 692 if (num_data) { 693 if (xfer->bits_per_word > 8) 694 td0 = *words++; 695 else 696 td0 = *bytes++; 697 698 spi_writew(as, TDR, td0); 699 num_data--; 700 } 701 702 dev_dbg(master->dev.parent, 703 " start fifo xfer %p: len %u tx %p rx %p bitpw %d\n", 704 xfer, xfer->len, xfer->tx_buf, xfer->rx_buf, 705 xfer->bits_per_word); 706 707 /* 708 * Enable RX FIFO Threshold Flag interrupt to be notified about 709 * transfer completion. 710 */ 711 spi_writel(as, IER, SPI_BIT(RXFTHF) | SPI_BIT(OVRES)); 712 } 713 714 /* 715 * Next transfer using PIO. 716 */ 717 static void atmel_spi_next_xfer_pio(struct spi_master *master, 718 struct spi_transfer *xfer) 719 { 720 struct atmel_spi *as = spi_master_get_devdata(master); 721 722 if (as->fifo_size) 723 atmel_spi_next_xfer_fifo(master, xfer); 724 else 725 atmel_spi_next_xfer_single(master, xfer); 726 } 727 728 /* 729 * Submit next transfer for DMA. 730 */ 731 static int atmel_spi_next_xfer_dma_submit(struct spi_master *master, 732 struct spi_transfer *xfer, 733 u32 *plen) 734 { 735 struct atmel_spi *as = spi_master_get_devdata(master); 736 struct dma_chan *rxchan = master->dma_rx; 737 struct dma_chan *txchan = master->dma_tx; 738 struct dma_async_tx_descriptor *rxdesc; 739 struct dma_async_tx_descriptor *txdesc; 740 struct dma_slave_config slave_config; 741 dma_cookie_t cookie; 742 743 dev_vdbg(master->dev.parent, "atmel_spi_next_xfer_dma_submit\n"); 744 745 /* Check that the channels are available */ 746 if (!rxchan || !txchan) 747 return -ENODEV; 748 749 /* release lock for DMA operations */ 750 atmel_spi_unlock(as); 751 752 *plen = xfer->len; 753 754 if (atmel_spi_dma_slave_config(as, &slave_config, 755 xfer->bits_per_word)) 756 goto err_exit; 757 758 /* Send both scatterlists */ 759 if (atmel_spi_is_vmalloc_xfer(xfer) && 760 IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) { 761 rxdesc = dmaengine_prep_slave_single(rxchan, 762 as->dma_addr_rx_bbuf, 763 xfer->len, 764 DMA_DEV_TO_MEM, 765 DMA_PREP_INTERRUPT | 766 DMA_CTRL_ACK); 767 } else { 768 rxdesc = dmaengine_prep_slave_sg(rxchan, 769 xfer->rx_sg.sgl, 770 xfer->rx_sg.nents, 771 DMA_DEV_TO_MEM, 772 DMA_PREP_INTERRUPT | 773 DMA_CTRL_ACK); 774 } 775 if (!rxdesc) 776 goto err_dma; 777 778 if (atmel_spi_is_vmalloc_xfer(xfer) && 779 IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) { 780 memcpy(as->addr_tx_bbuf, xfer->tx_buf, xfer->len); 781 txdesc = dmaengine_prep_slave_single(txchan, 782 as->dma_addr_tx_bbuf, 783 xfer->len, DMA_MEM_TO_DEV, 784 DMA_PREP_INTERRUPT | 785 DMA_CTRL_ACK); 786 } else { 787 txdesc = dmaengine_prep_slave_sg(txchan, 788 xfer->tx_sg.sgl, 789 xfer->tx_sg.nents, 790 DMA_MEM_TO_DEV, 791 DMA_PREP_INTERRUPT | 792 DMA_CTRL_ACK); 793 } 794 if (!txdesc) 795 goto err_dma; 796 797 dev_dbg(master->dev.parent, 798 " start dma xfer %p: len %u tx %p/%08llx rx %p/%08llx\n", 799 xfer, xfer->len, xfer->tx_buf, (unsigned long long)xfer->tx_dma, 800 xfer->rx_buf, (unsigned long long)xfer->rx_dma); 801 802 /* Enable relevant interrupts */ 803 spi_writel(as, IER, SPI_BIT(OVRES)); 804 805 /* Put the callback on the RX transfer only, that should finish last */ 806 rxdesc->callback = dma_callback; 807 rxdesc->callback_param = master; 808 809 /* Submit and fire RX and TX with TX last so we're ready to read! */ 810 cookie = rxdesc->tx_submit(rxdesc); 811 if (dma_submit_error(cookie)) 812 goto err_dma; 813 cookie = txdesc->tx_submit(txdesc); 814 if (dma_submit_error(cookie)) 815 goto err_dma; 816 rxchan->device->device_issue_pending(rxchan); 817 txchan->device->device_issue_pending(txchan); 818 819 /* take back lock */ 820 atmel_spi_lock(as); 821 return 0; 822 823 err_dma: 824 spi_writel(as, IDR, SPI_BIT(OVRES)); 825 atmel_spi_stop_dma(master); 826 err_exit: 827 atmel_spi_lock(as); 828 return -ENOMEM; 829 } 830 831 static void atmel_spi_next_xfer_data(struct spi_master *master, 832 struct spi_transfer *xfer, 833 dma_addr_t *tx_dma, 834 dma_addr_t *rx_dma, 835 u32 *plen) 836 { 837 *rx_dma = xfer->rx_dma + xfer->len - *plen; 838 *tx_dma = xfer->tx_dma + xfer->len - *plen; 839 if (*plen > master->max_dma_len) 840 *plen = master->max_dma_len; 841 } 842 843 static int atmel_spi_set_xfer_speed(struct atmel_spi *as, 844 struct spi_device *spi, 845 struct spi_transfer *xfer) 846 { 847 u32 scbr, csr; 848 unsigned long bus_hz; 849 850 /* v1 chips start out at half the peripheral bus speed. */ 851 bus_hz = as->spi_clk; 852 if (!atmel_spi_is_v2(as)) 853 bus_hz /= 2; 854 855 /* 856 * Calculate the lowest divider that satisfies the 857 * constraint, assuming div32/fdiv/mbz == 0. 858 */ 859 scbr = DIV_ROUND_UP(bus_hz, xfer->speed_hz); 860 861 /* 862 * If the resulting divider doesn't fit into the 863 * register bitfield, we can't satisfy the constraint. 864 */ 865 if (scbr >= (1 << SPI_SCBR_SIZE)) { 866 dev_err(&spi->dev, 867 "setup: %d Hz too slow, scbr %u; min %ld Hz\n", 868 xfer->speed_hz, scbr, bus_hz/255); 869 return -EINVAL; 870 } 871 if (scbr == 0) { 872 dev_err(&spi->dev, 873 "setup: %d Hz too high, scbr %u; max %ld Hz\n", 874 xfer->speed_hz, scbr, bus_hz); 875 return -EINVAL; 876 } 877 csr = spi_readl(as, CSR0 + 4 * spi->chip_select); 878 csr = SPI_BFINS(SCBR, scbr, csr); 879 spi_writel(as, CSR0 + 4 * spi->chip_select, csr); 880 881 return 0; 882 } 883 884 /* 885 * Submit next transfer for PDC. 886 * lock is held, spi irq is blocked 887 */ 888 static void atmel_spi_pdc_next_xfer(struct spi_master *master, 889 struct spi_message *msg, 890 struct spi_transfer *xfer) 891 { 892 struct atmel_spi *as = spi_master_get_devdata(master); 893 u32 len; 894 dma_addr_t tx_dma, rx_dma; 895 896 spi_writel(as, PTCR, SPI_BIT(RXTDIS) | SPI_BIT(TXTDIS)); 897 898 len = as->current_remaining_bytes; 899 atmel_spi_next_xfer_data(master, xfer, &tx_dma, &rx_dma, &len); 900 as->current_remaining_bytes -= len; 901 902 spi_writel(as, RPR, rx_dma); 903 spi_writel(as, TPR, tx_dma); 904 905 if (msg->spi->bits_per_word > 8) 906 len >>= 1; 907 spi_writel(as, RCR, len); 908 spi_writel(as, TCR, len); 909 910 dev_dbg(&msg->spi->dev, 911 " start xfer %p: len %u tx %p/%08llx rx %p/%08llx\n", 912 xfer, xfer->len, xfer->tx_buf, 913 (unsigned long long)xfer->tx_dma, xfer->rx_buf, 914 (unsigned long long)xfer->rx_dma); 915 916 if (as->current_remaining_bytes) { 917 len = as->current_remaining_bytes; 918 atmel_spi_next_xfer_data(master, xfer, &tx_dma, &rx_dma, &len); 919 as->current_remaining_bytes -= len; 920 921 spi_writel(as, RNPR, rx_dma); 922 spi_writel(as, TNPR, tx_dma); 923 924 if (msg->spi->bits_per_word > 8) 925 len >>= 1; 926 spi_writel(as, RNCR, len); 927 spi_writel(as, TNCR, len); 928 929 dev_dbg(&msg->spi->dev, 930 " next xfer %p: len %u tx %p/%08llx rx %p/%08llx\n", 931 xfer, xfer->len, xfer->tx_buf, 932 (unsigned long long)xfer->tx_dma, xfer->rx_buf, 933 (unsigned long long)xfer->rx_dma); 934 } 935 936 /* REVISIT: We're waiting for RXBUFF before we start the next 937 * transfer because we need to handle some difficult timing 938 * issues otherwise. If we wait for TXBUFE in one transfer and 939 * then starts waiting for RXBUFF in the next, it's difficult 940 * to tell the difference between the RXBUFF interrupt we're 941 * actually waiting for and the RXBUFF interrupt of the 942 * previous transfer. 943 * 944 * It should be doable, though. Just not now... 945 */ 946 spi_writel(as, IER, SPI_BIT(RXBUFF) | SPI_BIT(OVRES)); 947 spi_writel(as, PTCR, SPI_BIT(TXTEN) | SPI_BIT(RXTEN)); 948 } 949 950 /* 951 * For DMA, tx_buf/tx_dma have the same relationship as rx_buf/rx_dma: 952 * - The buffer is either valid for CPU access, else NULL 953 * - If the buffer is valid, so is its DMA address 954 * 955 * This driver manages the dma address unless message->is_dma_mapped. 956 */ 957 static int 958 atmel_spi_dma_map_xfer(struct atmel_spi *as, struct spi_transfer *xfer) 959 { 960 struct device *dev = &as->pdev->dev; 961 962 xfer->tx_dma = xfer->rx_dma = INVALID_DMA_ADDRESS; 963 if (xfer->tx_buf) { 964 /* tx_buf is a const void* where we need a void * for the dma 965 * mapping */ 966 void *nonconst_tx = (void *)xfer->tx_buf; 967 968 xfer->tx_dma = dma_map_single(dev, 969 nonconst_tx, xfer->len, 970 DMA_TO_DEVICE); 971 if (dma_mapping_error(dev, xfer->tx_dma)) 972 return -ENOMEM; 973 } 974 if (xfer->rx_buf) { 975 xfer->rx_dma = dma_map_single(dev, 976 xfer->rx_buf, xfer->len, 977 DMA_FROM_DEVICE); 978 if (dma_mapping_error(dev, xfer->rx_dma)) { 979 if (xfer->tx_buf) 980 dma_unmap_single(dev, 981 xfer->tx_dma, xfer->len, 982 DMA_TO_DEVICE); 983 return -ENOMEM; 984 } 985 } 986 return 0; 987 } 988 989 static void atmel_spi_dma_unmap_xfer(struct spi_master *master, 990 struct spi_transfer *xfer) 991 { 992 if (xfer->tx_dma != INVALID_DMA_ADDRESS) 993 dma_unmap_single(master->dev.parent, xfer->tx_dma, 994 xfer->len, DMA_TO_DEVICE); 995 if (xfer->rx_dma != INVALID_DMA_ADDRESS) 996 dma_unmap_single(master->dev.parent, xfer->rx_dma, 997 xfer->len, DMA_FROM_DEVICE); 998 } 999 1000 static void atmel_spi_disable_pdc_transfer(struct atmel_spi *as) 1001 { 1002 spi_writel(as, PTCR, SPI_BIT(RXTDIS) | SPI_BIT(TXTDIS)); 1003 } 1004 1005 static void 1006 atmel_spi_pump_single_data(struct atmel_spi *as, struct spi_transfer *xfer) 1007 { 1008 u8 *rxp; 1009 u16 *rxp16; 1010 unsigned long xfer_pos = xfer->len - as->current_remaining_bytes; 1011 1012 if (xfer->bits_per_word > 8) { 1013 rxp16 = (u16 *)(((u8 *)xfer->rx_buf) + xfer_pos); 1014 *rxp16 = spi_readl(as, RDR); 1015 } else { 1016 rxp = ((u8 *)xfer->rx_buf) + xfer_pos; 1017 *rxp = spi_readl(as, RDR); 1018 } 1019 if (xfer->bits_per_word > 8) { 1020 if (as->current_remaining_bytes > 2) 1021 as->current_remaining_bytes -= 2; 1022 else 1023 as->current_remaining_bytes = 0; 1024 } else { 1025 as->current_remaining_bytes--; 1026 } 1027 } 1028 1029 static void 1030 atmel_spi_pump_fifo_data(struct atmel_spi *as, struct spi_transfer *xfer) 1031 { 1032 u32 fifolr = spi_readl(as, FLR); 1033 u32 num_bytes, num_data = SPI_BFEXT(RXFL, fifolr); 1034 u32 offset = xfer->len - as->current_remaining_bytes; 1035 u16 *words = (u16 *)((u8 *)xfer->rx_buf + offset); 1036 u8 *bytes = (u8 *)((u8 *)xfer->rx_buf + offset); 1037 u16 rd; /* RD field is the lowest 16 bits of RDR */ 1038 1039 /* Update the number of remaining bytes to transfer */ 1040 num_bytes = ((xfer->bits_per_word > 8) ? 1041 (num_data << 1) : 1042 num_data); 1043 1044 if (as->current_remaining_bytes > num_bytes) 1045 as->current_remaining_bytes -= num_bytes; 1046 else 1047 as->current_remaining_bytes = 0; 1048 1049 /* Handle odd number of bytes when data are more than 8bit width */ 1050 if (xfer->bits_per_word > 8) 1051 as->current_remaining_bytes &= ~0x1; 1052 1053 /* Read data */ 1054 while (num_data) { 1055 rd = spi_readl(as, RDR); 1056 if (xfer->bits_per_word > 8) 1057 *words++ = rd; 1058 else 1059 *bytes++ = rd; 1060 num_data--; 1061 } 1062 } 1063 1064 /* Called from IRQ 1065 * 1066 * Must update "current_remaining_bytes" to keep track of data 1067 * to transfer. 1068 */ 1069 static void 1070 atmel_spi_pump_pio_data(struct atmel_spi *as, struct spi_transfer *xfer) 1071 { 1072 if (as->fifo_size) 1073 atmel_spi_pump_fifo_data(as, xfer); 1074 else 1075 atmel_spi_pump_single_data(as, xfer); 1076 } 1077 1078 /* Interrupt 1079 * 1080 * No need for locking in this Interrupt handler: done_status is the 1081 * only information modified. 1082 */ 1083 static irqreturn_t 1084 atmel_spi_pio_interrupt(int irq, void *dev_id) 1085 { 1086 struct spi_master *master = dev_id; 1087 struct atmel_spi *as = spi_master_get_devdata(master); 1088 u32 status, pending, imr; 1089 struct spi_transfer *xfer; 1090 int ret = IRQ_NONE; 1091 1092 imr = spi_readl(as, IMR); 1093 status = spi_readl(as, SR); 1094 pending = status & imr; 1095 1096 if (pending & SPI_BIT(OVRES)) { 1097 ret = IRQ_HANDLED; 1098 spi_writel(as, IDR, SPI_BIT(OVRES)); 1099 dev_warn(master->dev.parent, "overrun\n"); 1100 1101 /* 1102 * When we get an overrun, we disregard the current 1103 * transfer. Data will not be copied back from any 1104 * bounce buffer and msg->actual_len will not be 1105 * updated with the last xfer. 1106 * 1107 * We will also not process any remaning transfers in 1108 * the message. 1109 */ 1110 as->done_status = -EIO; 1111 smp_wmb(); 1112 1113 /* Clear any overrun happening while cleaning up */ 1114 spi_readl(as, SR); 1115 1116 complete(&as->xfer_completion); 1117 1118 } else if (pending & (SPI_BIT(RDRF) | SPI_BIT(RXFTHF))) { 1119 atmel_spi_lock(as); 1120 1121 if (as->current_remaining_bytes) { 1122 ret = IRQ_HANDLED; 1123 xfer = as->current_transfer; 1124 atmel_spi_pump_pio_data(as, xfer); 1125 if (!as->current_remaining_bytes) 1126 spi_writel(as, IDR, pending); 1127 1128 complete(&as->xfer_completion); 1129 } 1130 1131 atmel_spi_unlock(as); 1132 } else { 1133 WARN_ONCE(pending, "IRQ not handled, pending = %x\n", pending); 1134 ret = IRQ_HANDLED; 1135 spi_writel(as, IDR, pending); 1136 } 1137 1138 return ret; 1139 } 1140 1141 static irqreturn_t 1142 atmel_spi_pdc_interrupt(int irq, void *dev_id) 1143 { 1144 struct spi_master *master = dev_id; 1145 struct atmel_spi *as = spi_master_get_devdata(master); 1146 u32 status, pending, imr; 1147 int ret = IRQ_NONE; 1148 1149 imr = spi_readl(as, IMR); 1150 status = spi_readl(as, SR); 1151 pending = status & imr; 1152 1153 if (pending & SPI_BIT(OVRES)) { 1154 1155 ret = IRQ_HANDLED; 1156 1157 spi_writel(as, IDR, (SPI_BIT(RXBUFF) | SPI_BIT(ENDRX) 1158 | SPI_BIT(OVRES))); 1159 1160 /* Clear any overrun happening while cleaning up */ 1161 spi_readl(as, SR); 1162 1163 as->done_status = -EIO; 1164 1165 complete(&as->xfer_completion); 1166 1167 } else if (pending & (SPI_BIT(RXBUFF) | SPI_BIT(ENDRX))) { 1168 ret = IRQ_HANDLED; 1169 1170 spi_writel(as, IDR, pending); 1171 1172 complete(&as->xfer_completion); 1173 } 1174 1175 return ret; 1176 } 1177 1178 static int atmel_spi_setup(struct spi_device *spi) 1179 { 1180 struct atmel_spi *as; 1181 struct atmel_spi_device *asd; 1182 u32 csr; 1183 unsigned int bits = spi->bits_per_word; 1184 1185 as = spi_master_get_devdata(spi->master); 1186 1187 /* see notes above re chipselect */ 1188 if (!atmel_spi_is_v2(as) 1189 && spi->chip_select == 0 1190 && (spi->mode & SPI_CS_HIGH)) { 1191 dev_dbg(&spi->dev, "setup: can't be active-high\n"); 1192 return -EINVAL; 1193 } 1194 1195 csr = SPI_BF(BITS, bits - 8); 1196 if (spi->mode & SPI_CPOL) 1197 csr |= SPI_BIT(CPOL); 1198 if (!(spi->mode & SPI_CPHA)) 1199 csr |= SPI_BIT(NCPHA); 1200 if (!as->use_cs_gpios) 1201 csr |= SPI_BIT(CSAAT); 1202 1203 /* DLYBS is mostly irrelevant since we manage chipselect using GPIOs. 1204 */ 1205 csr |= SPI_BF(DLYBS, 0); 1206 1207 /* DLYBCT adds delays between words. This is useful for slow devices 1208 * that need a bit of time to setup the next transfer. 1209 */ 1210 csr |= SPI_BF(DLYBCT, 1211 (as->spi_clk / 1000000 * spi->word_delay_usecs) >> 5); 1212 1213 asd = spi->controller_state; 1214 if (!asd) { 1215 asd = kzalloc(sizeof(struct atmel_spi_device), GFP_KERNEL); 1216 if (!asd) 1217 return -ENOMEM; 1218 1219 /* 1220 * If use_cs_gpios is true this means that we have "cs-gpios" 1221 * defined in the device tree node so we should have 1222 * gotten the GPIO lines from the device tree inside the 1223 * SPI core. Warn if this is not the case but continue since 1224 * CS GPIOs are after all optional. 1225 */ 1226 if (as->use_cs_gpios) { 1227 if (!spi->cs_gpiod) { 1228 dev_err(&spi->dev, 1229 "host claims to use CS GPIOs but no CS found in DT by the SPI core\n"); 1230 } 1231 asd->npcs_pin = spi->cs_gpiod; 1232 } 1233 1234 spi->controller_state = asd; 1235 } 1236 1237 asd->csr = csr; 1238 1239 dev_dbg(&spi->dev, 1240 "setup: bpw %u mode 0x%x -> csr%d %08x\n", 1241 bits, spi->mode, spi->chip_select, csr); 1242 1243 if (!atmel_spi_is_v2(as)) 1244 spi_writel(as, CSR0 + 4 * spi->chip_select, csr); 1245 1246 return 0; 1247 } 1248 1249 static int atmel_spi_one_transfer(struct spi_master *master, 1250 struct spi_message *msg, 1251 struct spi_transfer *xfer) 1252 { 1253 struct atmel_spi *as; 1254 struct spi_device *spi = msg->spi; 1255 u8 bits; 1256 u32 len; 1257 struct atmel_spi_device *asd; 1258 int timeout; 1259 int ret; 1260 unsigned long dma_timeout; 1261 1262 as = spi_master_get_devdata(master); 1263 1264 if (!(xfer->tx_buf || xfer->rx_buf) && xfer->len) { 1265 dev_dbg(&spi->dev, "missing rx or tx buf\n"); 1266 return -EINVAL; 1267 } 1268 1269 asd = spi->controller_state; 1270 bits = (asd->csr >> 4) & 0xf; 1271 if (bits != xfer->bits_per_word - 8) { 1272 dev_dbg(&spi->dev, 1273 "you can't yet change bits_per_word in transfers\n"); 1274 return -ENOPROTOOPT; 1275 } 1276 1277 /* 1278 * DMA map early, for performance (empties dcache ASAP) and 1279 * better fault reporting. 1280 */ 1281 if ((!msg->is_dma_mapped) 1282 && as->use_pdc) { 1283 if (atmel_spi_dma_map_xfer(as, xfer) < 0) 1284 return -ENOMEM; 1285 } 1286 1287 atmel_spi_set_xfer_speed(as, msg->spi, xfer); 1288 1289 as->done_status = 0; 1290 as->current_transfer = xfer; 1291 as->current_remaining_bytes = xfer->len; 1292 while (as->current_remaining_bytes) { 1293 reinit_completion(&as->xfer_completion); 1294 1295 if (as->use_pdc) { 1296 atmel_spi_pdc_next_xfer(master, msg, xfer); 1297 } else if (atmel_spi_use_dma(as, xfer)) { 1298 len = as->current_remaining_bytes; 1299 ret = atmel_spi_next_xfer_dma_submit(master, 1300 xfer, &len); 1301 if (ret) { 1302 dev_err(&spi->dev, 1303 "unable to use DMA, fallback to PIO\n"); 1304 atmel_spi_next_xfer_pio(master, xfer); 1305 } else { 1306 as->current_remaining_bytes -= len; 1307 if (as->current_remaining_bytes < 0) 1308 as->current_remaining_bytes = 0; 1309 } 1310 } else { 1311 atmel_spi_next_xfer_pio(master, xfer); 1312 } 1313 1314 /* interrupts are disabled, so free the lock for schedule */ 1315 atmel_spi_unlock(as); 1316 dma_timeout = wait_for_completion_timeout(&as->xfer_completion, 1317 SPI_DMA_TIMEOUT); 1318 atmel_spi_lock(as); 1319 if (WARN_ON(dma_timeout == 0)) { 1320 dev_err(&spi->dev, "spi transfer timeout\n"); 1321 as->done_status = -EIO; 1322 } 1323 1324 if (as->done_status) 1325 break; 1326 } 1327 1328 if (as->done_status) { 1329 if (as->use_pdc) { 1330 dev_warn(master->dev.parent, 1331 "overrun (%u/%u remaining)\n", 1332 spi_readl(as, TCR), spi_readl(as, RCR)); 1333 1334 /* 1335 * Clean up DMA registers and make sure the data 1336 * registers are empty. 1337 */ 1338 spi_writel(as, RNCR, 0); 1339 spi_writel(as, TNCR, 0); 1340 spi_writel(as, RCR, 0); 1341 spi_writel(as, TCR, 0); 1342 for (timeout = 1000; timeout; timeout--) 1343 if (spi_readl(as, SR) & SPI_BIT(TXEMPTY)) 1344 break; 1345 if (!timeout) 1346 dev_warn(master->dev.parent, 1347 "timeout waiting for TXEMPTY"); 1348 while (spi_readl(as, SR) & SPI_BIT(RDRF)) 1349 spi_readl(as, RDR); 1350 1351 /* Clear any overrun happening while cleaning up */ 1352 spi_readl(as, SR); 1353 1354 } else if (atmel_spi_use_dma(as, xfer)) { 1355 atmel_spi_stop_dma(master); 1356 } 1357 1358 if (!msg->is_dma_mapped 1359 && as->use_pdc) 1360 atmel_spi_dma_unmap_xfer(master, xfer); 1361 1362 return 0; 1363 1364 } else { 1365 /* only update length if no error */ 1366 msg->actual_length += xfer->len; 1367 } 1368 1369 if (!msg->is_dma_mapped 1370 && as->use_pdc) 1371 atmel_spi_dma_unmap_xfer(master, xfer); 1372 1373 if (xfer->delay_usecs) 1374 udelay(xfer->delay_usecs); 1375 1376 if (xfer->cs_change) { 1377 if (list_is_last(&xfer->transfer_list, 1378 &msg->transfers)) { 1379 as->keep_cs = true; 1380 } else { 1381 as->cs_active = !as->cs_active; 1382 if (as->cs_active) 1383 cs_activate(as, msg->spi); 1384 else 1385 cs_deactivate(as, msg->spi); 1386 } 1387 } 1388 1389 return 0; 1390 } 1391 1392 static int atmel_spi_transfer_one_message(struct spi_master *master, 1393 struct spi_message *msg) 1394 { 1395 struct atmel_spi *as; 1396 struct spi_transfer *xfer; 1397 struct spi_device *spi = msg->spi; 1398 int ret = 0; 1399 1400 as = spi_master_get_devdata(master); 1401 1402 dev_dbg(&spi->dev, "new message %p submitted for %s\n", 1403 msg, dev_name(&spi->dev)); 1404 1405 atmel_spi_lock(as); 1406 cs_activate(as, spi); 1407 1408 as->cs_active = true; 1409 as->keep_cs = false; 1410 1411 msg->status = 0; 1412 msg->actual_length = 0; 1413 1414 list_for_each_entry(xfer, &msg->transfers, transfer_list) { 1415 ret = atmel_spi_one_transfer(master, msg, xfer); 1416 if (ret) 1417 goto msg_done; 1418 } 1419 1420 if (as->use_pdc) 1421 atmel_spi_disable_pdc_transfer(as); 1422 1423 list_for_each_entry(xfer, &msg->transfers, transfer_list) { 1424 dev_dbg(&spi->dev, 1425 " xfer %p: len %u tx %p/%pad rx %p/%pad\n", 1426 xfer, xfer->len, 1427 xfer->tx_buf, &xfer->tx_dma, 1428 xfer->rx_buf, &xfer->rx_dma); 1429 } 1430 1431 msg_done: 1432 if (!as->keep_cs) 1433 cs_deactivate(as, msg->spi); 1434 1435 atmel_spi_unlock(as); 1436 1437 msg->status = as->done_status; 1438 spi_finalize_current_message(spi->master); 1439 1440 return ret; 1441 } 1442 1443 static void atmel_spi_cleanup(struct spi_device *spi) 1444 { 1445 struct atmel_spi_device *asd = spi->controller_state; 1446 1447 if (!asd) 1448 return; 1449 1450 spi->controller_state = NULL; 1451 kfree(asd); 1452 } 1453 1454 static inline unsigned int atmel_get_version(struct atmel_spi *as) 1455 { 1456 return spi_readl(as, VERSION) & 0x00000fff; 1457 } 1458 1459 static void atmel_get_caps(struct atmel_spi *as) 1460 { 1461 unsigned int version; 1462 1463 version = atmel_get_version(as); 1464 1465 as->caps.is_spi2 = version > 0x121; 1466 as->caps.has_wdrbt = version >= 0x210; 1467 as->caps.has_dma_support = version >= 0x212; 1468 as->caps.has_pdc_support = version < 0x212; 1469 } 1470 1471 static void atmel_spi_init(struct atmel_spi *as) 1472 { 1473 spi_writel(as, CR, SPI_BIT(SWRST)); 1474 spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */ 1475 1476 /* It is recommended to enable FIFOs first thing after reset */ 1477 if (as->fifo_size) 1478 spi_writel(as, CR, SPI_BIT(FIFOEN)); 1479 1480 if (as->caps.has_wdrbt) { 1481 spi_writel(as, MR, SPI_BIT(WDRBT) | SPI_BIT(MODFDIS) 1482 | SPI_BIT(MSTR)); 1483 } else { 1484 spi_writel(as, MR, SPI_BIT(MSTR) | SPI_BIT(MODFDIS)); 1485 } 1486 1487 if (as->use_pdc) 1488 spi_writel(as, PTCR, SPI_BIT(RXTDIS) | SPI_BIT(TXTDIS)); 1489 spi_writel(as, CR, SPI_BIT(SPIEN)); 1490 } 1491 1492 static int atmel_spi_probe(struct platform_device *pdev) 1493 { 1494 struct resource *regs; 1495 int irq; 1496 struct clk *clk; 1497 int ret; 1498 struct spi_master *master; 1499 struct atmel_spi *as; 1500 1501 /* Select default pin state */ 1502 pinctrl_pm_select_default_state(&pdev->dev); 1503 1504 regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1505 if (!regs) 1506 return -ENXIO; 1507 1508 irq = platform_get_irq(pdev, 0); 1509 if (irq < 0) 1510 return irq; 1511 1512 clk = devm_clk_get(&pdev->dev, "spi_clk"); 1513 if (IS_ERR(clk)) 1514 return PTR_ERR(clk); 1515 1516 /* setup spi core then atmel-specific driver state */ 1517 ret = -ENOMEM; 1518 master = spi_alloc_master(&pdev->dev, sizeof(*as)); 1519 if (!master) 1520 goto out_free; 1521 1522 /* the spi->mode bits understood by this driver: */ 1523 master->use_gpio_descriptors = true; 1524 master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH; 1525 master->bits_per_word_mask = SPI_BPW_RANGE_MASK(8, 16); 1526 master->dev.of_node = pdev->dev.of_node; 1527 master->bus_num = pdev->id; 1528 master->num_chipselect = master->dev.of_node ? 0 : 4; 1529 master->setup = atmel_spi_setup; 1530 master->flags = (SPI_MASTER_MUST_RX | SPI_MASTER_MUST_TX); 1531 master->transfer_one_message = atmel_spi_transfer_one_message; 1532 master->cleanup = atmel_spi_cleanup; 1533 master->auto_runtime_pm = true; 1534 master->max_dma_len = SPI_MAX_DMA_XFER; 1535 master->can_dma = atmel_spi_can_dma; 1536 platform_set_drvdata(pdev, master); 1537 1538 as = spi_master_get_devdata(master); 1539 1540 spin_lock_init(&as->lock); 1541 1542 as->pdev = pdev; 1543 as->regs = devm_ioremap_resource(&pdev->dev, regs); 1544 if (IS_ERR(as->regs)) { 1545 ret = PTR_ERR(as->regs); 1546 goto out_unmap_regs; 1547 } 1548 as->phybase = regs->start; 1549 as->irq = irq; 1550 as->clk = clk; 1551 1552 init_completion(&as->xfer_completion); 1553 1554 atmel_get_caps(as); 1555 1556 /* 1557 * If there are chip selects in the device tree, those will be 1558 * discovered by the SPI core when registering the SPI master 1559 * and assigned to each SPI device. 1560 */ 1561 as->use_cs_gpios = true; 1562 if (atmel_spi_is_v2(as) && 1563 pdev->dev.of_node && 1564 !of_get_property(pdev->dev.of_node, "cs-gpios", NULL)) { 1565 as->use_cs_gpios = false; 1566 master->num_chipselect = 4; 1567 } 1568 1569 as->use_dma = false; 1570 as->use_pdc = false; 1571 if (as->caps.has_dma_support) { 1572 ret = atmel_spi_configure_dma(master, as); 1573 if (ret == 0) { 1574 as->use_dma = true; 1575 } else if (ret == -EPROBE_DEFER) { 1576 return ret; 1577 } 1578 } else if (as->caps.has_pdc_support) { 1579 as->use_pdc = true; 1580 } 1581 1582 if (IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) { 1583 as->addr_rx_bbuf = dma_alloc_coherent(&pdev->dev, 1584 SPI_MAX_DMA_XFER, 1585 &as->dma_addr_rx_bbuf, 1586 GFP_KERNEL | GFP_DMA); 1587 if (!as->addr_rx_bbuf) { 1588 as->use_dma = false; 1589 } else { 1590 as->addr_tx_bbuf = dma_alloc_coherent(&pdev->dev, 1591 SPI_MAX_DMA_XFER, 1592 &as->dma_addr_tx_bbuf, 1593 GFP_KERNEL | GFP_DMA); 1594 if (!as->addr_tx_bbuf) { 1595 as->use_dma = false; 1596 dma_free_coherent(&pdev->dev, SPI_MAX_DMA_XFER, 1597 as->addr_rx_bbuf, 1598 as->dma_addr_rx_bbuf); 1599 } 1600 } 1601 if (!as->use_dma) 1602 dev_info(master->dev.parent, 1603 " can not allocate dma coherent memory\n"); 1604 } 1605 1606 if (as->caps.has_dma_support && !as->use_dma) 1607 dev_info(&pdev->dev, "Atmel SPI Controller using PIO only\n"); 1608 1609 if (as->use_pdc) { 1610 ret = devm_request_irq(&pdev->dev, irq, atmel_spi_pdc_interrupt, 1611 0, dev_name(&pdev->dev), master); 1612 } else { 1613 ret = devm_request_irq(&pdev->dev, irq, atmel_spi_pio_interrupt, 1614 0, dev_name(&pdev->dev), master); 1615 } 1616 if (ret) 1617 goto out_unmap_regs; 1618 1619 /* Initialize the hardware */ 1620 ret = clk_prepare_enable(clk); 1621 if (ret) 1622 goto out_free_irq; 1623 1624 as->spi_clk = clk_get_rate(clk); 1625 1626 as->fifo_size = 0; 1627 if (!of_property_read_u32(pdev->dev.of_node, "atmel,fifo-size", 1628 &as->fifo_size)) { 1629 dev_info(&pdev->dev, "Using FIFO (%u data)\n", as->fifo_size); 1630 } 1631 1632 atmel_spi_init(as); 1633 1634 pm_runtime_set_autosuspend_delay(&pdev->dev, AUTOSUSPEND_TIMEOUT); 1635 pm_runtime_use_autosuspend(&pdev->dev); 1636 pm_runtime_set_active(&pdev->dev); 1637 pm_runtime_enable(&pdev->dev); 1638 1639 ret = devm_spi_register_master(&pdev->dev, master); 1640 if (ret) 1641 goto out_free_dma; 1642 1643 /* go! */ 1644 dev_info(&pdev->dev, "Atmel SPI Controller version 0x%x at 0x%08lx (irq %d)\n", 1645 atmel_get_version(as), (unsigned long)regs->start, 1646 irq); 1647 1648 return 0; 1649 1650 out_free_dma: 1651 pm_runtime_disable(&pdev->dev); 1652 pm_runtime_set_suspended(&pdev->dev); 1653 1654 if (as->use_dma) 1655 atmel_spi_release_dma(master); 1656 1657 spi_writel(as, CR, SPI_BIT(SWRST)); 1658 spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */ 1659 clk_disable_unprepare(clk); 1660 out_free_irq: 1661 out_unmap_regs: 1662 out_free: 1663 spi_master_put(master); 1664 return ret; 1665 } 1666 1667 static int atmel_spi_remove(struct platform_device *pdev) 1668 { 1669 struct spi_master *master = platform_get_drvdata(pdev); 1670 struct atmel_spi *as = spi_master_get_devdata(master); 1671 1672 pm_runtime_get_sync(&pdev->dev); 1673 1674 /* reset the hardware and block queue progress */ 1675 if (as->use_dma) { 1676 atmel_spi_stop_dma(master); 1677 atmel_spi_release_dma(master); 1678 if (IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) { 1679 dma_free_coherent(&pdev->dev, SPI_MAX_DMA_XFER, 1680 as->addr_tx_bbuf, 1681 as->dma_addr_tx_bbuf); 1682 dma_free_coherent(&pdev->dev, SPI_MAX_DMA_XFER, 1683 as->addr_rx_bbuf, 1684 as->dma_addr_rx_bbuf); 1685 } 1686 } 1687 1688 spin_lock_irq(&as->lock); 1689 spi_writel(as, CR, SPI_BIT(SWRST)); 1690 spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */ 1691 spi_readl(as, SR); 1692 spin_unlock_irq(&as->lock); 1693 1694 clk_disable_unprepare(as->clk); 1695 1696 pm_runtime_put_noidle(&pdev->dev); 1697 pm_runtime_disable(&pdev->dev); 1698 1699 return 0; 1700 } 1701 1702 #ifdef CONFIG_PM 1703 static int atmel_spi_runtime_suspend(struct device *dev) 1704 { 1705 struct spi_master *master = dev_get_drvdata(dev); 1706 struct atmel_spi *as = spi_master_get_devdata(master); 1707 1708 clk_disable_unprepare(as->clk); 1709 pinctrl_pm_select_sleep_state(dev); 1710 1711 return 0; 1712 } 1713 1714 static int atmel_spi_runtime_resume(struct device *dev) 1715 { 1716 struct spi_master *master = dev_get_drvdata(dev); 1717 struct atmel_spi *as = spi_master_get_devdata(master); 1718 1719 pinctrl_pm_select_default_state(dev); 1720 1721 return clk_prepare_enable(as->clk); 1722 } 1723 1724 #ifdef CONFIG_PM_SLEEP 1725 static int atmel_spi_suspend(struct device *dev) 1726 { 1727 struct spi_master *master = dev_get_drvdata(dev); 1728 int ret; 1729 1730 /* Stop the queue running */ 1731 ret = spi_master_suspend(master); 1732 if (ret) 1733 return ret; 1734 1735 if (!pm_runtime_suspended(dev)) 1736 atmel_spi_runtime_suspend(dev); 1737 1738 return 0; 1739 } 1740 1741 static int atmel_spi_resume(struct device *dev) 1742 { 1743 struct spi_master *master = dev_get_drvdata(dev); 1744 struct atmel_spi *as = spi_master_get_devdata(master); 1745 int ret; 1746 1747 ret = clk_prepare_enable(as->clk); 1748 if (ret) 1749 return ret; 1750 1751 atmel_spi_init(as); 1752 1753 clk_disable_unprepare(as->clk); 1754 1755 if (!pm_runtime_suspended(dev)) { 1756 ret = atmel_spi_runtime_resume(dev); 1757 if (ret) 1758 return ret; 1759 } 1760 1761 /* Start the queue running */ 1762 return spi_master_resume(master); 1763 } 1764 #endif 1765 1766 static const struct dev_pm_ops atmel_spi_pm_ops = { 1767 SET_SYSTEM_SLEEP_PM_OPS(atmel_spi_suspend, atmel_spi_resume) 1768 SET_RUNTIME_PM_OPS(atmel_spi_runtime_suspend, 1769 atmel_spi_runtime_resume, NULL) 1770 }; 1771 #define ATMEL_SPI_PM_OPS (&atmel_spi_pm_ops) 1772 #else 1773 #define ATMEL_SPI_PM_OPS NULL 1774 #endif 1775 1776 #if defined(CONFIG_OF) 1777 static const struct of_device_id atmel_spi_dt_ids[] = { 1778 { .compatible = "atmel,at91rm9200-spi" }, 1779 { /* sentinel */ } 1780 }; 1781 1782 MODULE_DEVICE_TABLE(of, atmel_spi_dt_ids); 1783 #endif 1784 1785 static struct platform_driver atmel_spi_driver = { 1786 .driver = { 1787 .name = "atmel_spi", 1788 .pm = ATMEL_SPI_PM_OPS, 1789 .of_match_table = of_match_ptr(atmel_spi_dt_ids), 1790 }, 1791 .probe = atmel_spi_probe, 1792 .remove = atmel_spi_remove, 1793 }; 1794 module_platform_driver(atmel_spi_driver); 1795 1796 MODULE_DESCRIPTION("Atmel AT32/AT91 SPI Controller driver"); 1797 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 1798 MODULE_LICENSE("GPL"); 1799 MODULE_ALIAS("platform:atmel_spi"); 1800