1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Driver for Atmel AT32 and AT91 SPI Controllers 4 * 5 * Copyright (C) 2006 Atmel Corporation 6 */ 7 8 #include <linux/kernel.h> 9 #include <linux/clk.h> 10 #include <linux/module.h> 11 #include <linux/platform_device.h> 12 #include <linux/delay.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/dmaengine.h> 15 #include <linux/err.h> 16 #include <linux/interrupt.h> 17 #include <linux/spi/spi.h> 18 #include <linux/slab.h> 19 #include <linux/platform_data/dma-atmel.h> 20 #include <linux/of.h> 21 22 #include <linux/io.h> 23 #include <linux/gpio/consumer.h> 24 #include <linux/pinctrl/consumer.h> 25 #include <linux/pm_runtime.h> 26 #include <trace/events/spi.h> 27 28 /* SPI register offsets */ 29 #define SPI_CR 0x0000 30 #define SPI_MR 0x0004 31 #define SPI_RDR 0x0008 32 #define SPI_TDR 0x000c 33 #define SPI_SR 0x0010 34 #define SPI_IER 0x0014 35 #define SPI_IDR 0x0018 36 #define SPI_IMR 0x001c 37 #define SPI_CSR0 0x0030 38 #define SPI_CSR1 0x0034 39 #define SPI_CSR2 0x0038 40 #define SPI_CSR3 0x003c 41 #define SPI_FMR 0x0040 42 #define SPI_FLR 0x0044 43 #define SPI_VERSION 0x00fc 44 #define SPI_RPR 0x0100 45 #define SPI_RCR 0x0104 46 #define SPI_TPR 0x0108 47 #define SPI_TCR 0x010c 48 #define SPI_RNPR 0x0110 49 #define SPI_RNCR 0x0114 50 #define SPI_TNPR 0x0118 51 #define SPI_TNCR 0x011c 52 #define SPI_PTCR 0x0120 53 #define SPI_PTSR 0x0124 54 55 /* Bitfields in CR */ 56 #define SPI_SPIEN_OFFSET 0 57 #define SPI_SPIEN_SIZE 1 58 #define SPI_SPIDIS_OFFSET 1 59 #define SPI_SPIDIS_SIZE 1 60 #define SPI_SWRST_OFFSET 7 61 #define SPI_SWRST_SIZE 1 62 #define SPI_LASTXFER_OFFSET 24 63 #define SPI_LASTXFER_SIZE 1 64 #define SPI_TXFCLR_OFFSET 16 65 #define SPI_TXFCLR_SIZE 1 66 #define SPI_RXFCLR_OFFSET 17 67 #define SPI_RXFCLR_SIZE 1 68 #define SPI_FIFOEN_OFFSET 30 69 #define SPI_FIFOEN_SIZE 1 70 #define SPI_FIFODIS_OFFSET 31 71 #define SPI_FIFODIS_SIZE 1 72 73 /* Bitfields in MR */ 74 #define SPI_MSTR_OFFSET 0 75 #define SPI_MSTR_SIZE 1 76 #define SPI_PS_OFFSET 1 77 #define SPI_PS_SIZE 1 78 #define SPI_PCSDEC_OFFSET 2 79 #define SPI_PCSDEC_SIZE 1 80 #define SPI_FDIV_OFFSET 3 81 #define SPI_FDIV_SIZE 1 82 #define SPI_MODFDIS_OFFSET 4 83 #define SPI_MODFDIS_SIZE 1 84 #define SPI_WDRBT_OFFSET 5 85 #define SPI_WDRBT_SIZE 1 86 #define SPI_LLB_OFFSET 7 87 #define SPI_LLB_SIZE 1 88 #define SPI_PCS_OFFSET 16 89 #define SPI_PCS_SIZE 4 90 #define SPI_DLYBCS_OFFSET 24 91 #define SPI_DLYBCS_SIZE 8 92 93 /* Bitfields in RDR */ 94 #define SPI_RD_OFFSET 0 95 #define SPI_RD_SIZE 16 96 97 /* Bitfields in TDR */ 98 #define SPI_TD_OFFSET 0 99 #define SPI_TD_SIZE 16 100 101 /* Bitfields in SR */ 102 #define SPI_RDRF_OFFSET 0 103 #define SPI_RDRF_SIZE 1 104 #define SPI_TDRE_OFFSET 1 105 #define SPI_TDRE_SIZE 1 106 #define SPI_MODF_OFFSET 2 107 #define SPI_MODF_SIZE 1 108 #define SPI_OVRES_OFFSET 3 109 #define SPI_OVRES_SIZE 1 110 #define SPI_ENDRX_OFFSET 4 111 #define SPI_ENDRX_SIZE 1 112 #define SPI_ENDTX_OFFSET 5 113 #define SPI_ENDTX_SIZE 1 114 #define SPI_RXBUFF_OFFSET 6 115 #define SPI_RXBUFF_SIZE 1 116 #define SPI_TXBUFE_OFFSET 7 117 #define SPI_TXBUFE_SIZE 1 118 #define SPI_NSSR_OFFSET 8 119 #define SPI_NSSR_SIZE 1 120 #define SPI_TXEMPTY_OFFSET 9 121 #define SPI_TXEMPTY_SIZE 1 122 #define SPI_SPIENS_OFFSET 16 123 #define SPI_SPIENS_SIZE 1 124 #define SPI_TXFEF_OFFSET 24 125 #define SPI_TXFEF_SIZE 1 126 #define SPI_TXFFF_OFFSET 25 127 #define SPI_TXFFF_SIZE 1 128 #define SPI_TXFTHF_OFFSET 26 129 #define SPI_TXFTHF_SIZE 1 130 #define SPI_RXFEF_OFFSET 27 131 #define SPI_RXFEF_SIZE 1 132 #define SPI_RXFFF_OFFSET 28 133 #define SPI_RXFFF_SIZE 1 134 #define SPI_RXFTHF_OFFSET 29 135 #define SPI_RXFTHF_SIZE 1 136 #define SPI_TXFPTEF_OFFSET 30 137 #define SPI_TXFPTEF_SIZE 1 138 #define SPI_RXFPTEF_OFFSET 31 139 #define SPI_RXFPTEF_SIZE 1 140 141 /* Bitfields in CSR0 */ 142 #define SPI_CPOL_OFFSET 0 143 #define SPI_CPOL_SIZE 1 144 #define SPI_NCPHA_OFFSET 1 145 #define SPI_NCPHA_SIZE 1 146 #define SPI_CSAAT_OFFSET 3 147 #define SPI_CSAAT_SIZE 1 148 #define SPI_BITS_OFFSET 4 149 #define SPI_BITS_SIZE 4 150 #define SPI_SCBR_OFFSET 8 151 #define SPI_SCBR_SIZE 8 152 #define SPI_DLYBS_OFFSET 16 153 #define SPI_DLYBS_SIZE 8 154 #define SPI_DLYBCT_OFFSET 24 155 #define SPI_DLYBCT_SIZE 8 156 157 /* Bitfields in RCR */ 158 #define SPI_RXCTR_OFFSET 0 159 #define SPI_RXCTR_SIZE 16 160 161 /* Bitfields in TCR */ 162 #define SPI_TXCTR_OFFSET 0 163 #define SPI_TXCTR_SIZE 16 164 165 /* Bitfields in RNCR */ 166 #define SPI_RXNCR_OFFSET 0 167 #define SPI_RXNCR_SIZE 16 168 169 /* Bitfields in TNCR */ 170 #define SPI_TXNCR_OFFSET 0 171 #define SPI_TXNCR_SIZE 16 172 173 /* Bitfields in PTCR */ 174 #define SPI_RXTEN_OFFSET 0 175 #define SPI_RXTEN_SIZE 1 176 #define SPI_RXTDIS_OFFSET 1 177 #define SPI_RXTDIS_SIZE 1 178 #define SPI_TXTEN_OFFSET 8 179 #define SPI_TXTEN_SIZE 1 180 #define SPI_TXTDIS_OFFSET 9 181 #define SPI_TXTDIS_SIZE 1 182 183 /* Bitfields in FMR */ 184 #define SPI_TXRDYM_OFFSET 0 185 #define SPI_TXRDYM_SIZE 2 186 #define SPI_RXRDYM_OFFSET 4 187 #define SPI_RXRDYM_SIZE 2 188 #define SPI_TXFTHRES_OFFSET 16 189 #define SPI_TXFTHRES_SIZE 6 190 #define SPI_RXFTHRES_OFFSET 24 191 #define SPI_RXFTHRES_SIZE 6 192 193 /* Bitfields in FLR */ 194 #define SPI_TXFL_OFFSET 0 195 #define SPI_TXFL_SIZE 6 196 #define SPI_RXFL_OFFSET 16 197 #define SPI_RXFL_SIZE 6 198 199 /* Constants for BITS */ 200 #define SPI_BITS_8_BPT 0 201 #define SPI_BITS_9_BPT 1 202 #define SPI_BITS_10_BPT 2 203 #define SPI_BITS_11_BPT 3 204 #define SPI_BITS_12_BPT 4 205 #define SPI_BITS_13_BPT 5 206 #define SPI_BITS_14_BPT 6 207 #define SPI_BITS_15_BPT 7 208 #define SPI_BITS_16_BPT 8 209 #define SPI_ONE_DATA 0 210 #define SPI_TWO_DATA 1 211 #define SPI_FOUR_DATA 2 212 213 /* Bit manipulation macros */ 214 #define SPI_BIT(name) \ 215 (1 << SPI_##name##_OFFSET) 216 #define SPI_BF(name, value) \ 217 (((value) & ((1 << SPI_##name##_SIZE) - 1)) << SPI_##name##_OFFSET) 218 #define SPI_BFEXT(name, value) \ 219 (((value) >> SPI_##name##_OFFSET) & ((1 << SPI_##name##_SIZE) - 1)) 220 #define SPI_BFINS(name, value, old) \ 221 (((old) & ~(((1 << SPI_##name##_SIZE) - 1) << SPI_##name##_OFFSET)) \ 222 | SPI_BF(name, value)) 223 224 /* Register access macros */ 225 #ifdef CONFIG_AVR32 226 #define spi_readl(port, reg) \ 227 __raw_readl((port)->regs + SPI_##reg) 228 #define spi_writel(port, reg, value) \ 229 __raw_writel((value), (port)->regs + SPI_##reg) 230 231 #define spi_readw(port, reg) \ 232 __raw_readw((port)->regs + SPI_##reg) 233 #define spi_writew(port, reg, value) \ 234 __raw_writew((value), (port)->regs + SPI_##reg) 235 236 #define spi_readb(port, reg) \ 237 __raw_readb((port)->regs + SPI_##reg) 238 #define spi_writeb(port, reg, value) \ 239 __raw_writeb((value), (port)->regs + SPI_##reg) 240 #else 241 #define spi_readl(port, reg) \ 242 readl_relaxed((port)->regs + SPI_##reg) 243 #define spi_writel(port, reg, value) \ 244 writel_relaxed((value), (port)->regs + SPI_##reg) 245 246 #define spi_readw(port, reg) \ 247 readw_relaxed((port)->regs + SPI_##reg) 248 #define spi_writew(port, reg, value) \ 249 writew_relaxed((value), (port)->regs + SPI_##reg) 250 251 #define spi_readb(port, reg) \ 252 readb_relaxed((port)->regs + SPI_##reg) 253 #define spi_writeb(port, reg, value) \ 254 writeb_relaxed((value), (port)->regs + SPI_##reg) 255 #endif 256 /* use PIO for small transfers, avoiding DMA setup/teardown overhead and 257 * cache operations; better heuristics consider wordsize and bitrate. 258 */ 259 #define DMA_MIN_BYTES 16 260 261 #define SPI_DMA_TIMEOUT (msecs_to_jiffies(1000)) 262 263 #define AUTOSUSPEND_TIMEOUT 2000 264 265 struct atmel_spi_caps { 266 bool is_spi2; 267 bool has_wdrbt; 268 bool has_dma_support; 269 bool has_pdc_support; 270 }; 271 272 /* 273 * The core SPI transfer engine just talks to a register bank to set up 274 * DMA transfers; transfer queue progress is driven by IRQs. The clock 275 * framework provides the base clock, subdivided for each spi_device. 276 */ 277 struct atmel_spi { 278 spinlock_t lock; 279 unsigned long flags; 280 281 phys_addr_t phybase; 282 void __iomem *regs; 283 int irq; 284 struct clk *clk; 285 struct platform_device *pdev; 286 unsigned long spi_clk; 287 288 struct spi_transfer *current_transfer; 289 int current_remaining_bytes; 290 int done_status; 291 dma_addr_t dma_addr_rx_bbuf; 292 dma_addr_t dma_addr_tx_bbuf; 293 void *addr_rx_bbuf; 294 void *addr_tx_bbuf; 295 296 struct completion xfer_completion; 297 298 struct atmel_spi_caps caps; 299 300 bool use_dma; 301 bool use_pdc; 302 bool use_cs_gpios; 303 304 bool keep_cs; 305 bool cs_active; 306 307 u32 fifo_size; 308 }; 309 310 /* Controller-specific per-slave state */ 311 struct atmel_spi_device { 312 struct gpio_desc *npcs_pin; 313 u32 csr; 314 }; 315 316 #define SPI_MAX_DMA_XFER 65535 /* true for both PDC and DMA */ 317 #define INVALID_DMA_ADDRESS 0xffffffff 318 319 /* 320 * Version 2 of the SPI controller has 321 * - CR.LASTXFER 322 * - SPI_MR.DIV32 may become FDIV or must-be-zero (here: always zero) 323 * - SPI_SR.TXEMPTY, SPI_SR.NSSR (and corresponding irqs) 324 * - SPI_CSRx.CSAAT 325 * - SPI_CSRx.SBCR allows faster clocking 326 */ 327 static bool atmel_spi_is_v2(struct atmel_spi *as) 328 { 329 return as->caps.is_spi2; 330 } 331 332 /* 333 * Earlier SPI controllers (e.g. on at91rm9200) have a design bug whereby 334 * they assume that spi slave device state will not change on deselect, so 335 * that automagic deselection is OK. ("NPCSx rises if no data is to be 336 * transmitted") Not so! Workaround uses nCSx pins as GPIOs; or newer 337 * controllers have CSAAT and friends. 338 * 339 * Since the CSAAT functionality is a bit weird on newer controllers as 340 * well, we use GPIO to control nCSx pins on all controllers, updating 341 * MR.PCS to avoid confusing the controller. Using GPIOs also lets us 342 * support active-high chipselects despite the controller's belief that 343 * only active-low devices/systems exists. 344 * 345 * However, at91rm9200 has a second erratum whereby nCS0 doesn't work 346 * right when driven with GPIO. ("Mode Fault does not allow more than one 347 * Master on Chip Select 0.") No workaround exists for that ... so for 348 * nCS0 on that chip, we (a) don't use the GPIO, (b) can't support CS_HIGH, 349 * and (c) will trigger that first erratum in some cases. 350 */ 351 352 static void cs_activate(struct atmel_spi *as, struct spi_device *spi) 353 { 354 struct atmel_spi_device *asd = spi->controller_state; 355 u32 mr; 356 357 if (atmel_spi_is_v2(as)) { 358 spi_writel(as, CSR0 + 4 * spi->chip_select, asd->csr); 359 /* For the low SPI version, there is a issue that PDC transfer 360 * on CS1,2,3 needs SPI_CSR0.BITS config as SPI_CSR1,2,3.BITS 361 */ 362 spi_writel(as, CSR0, asd->csr); 363 if (as->caps.has_wdrbt) { 364 spi_writel(as, MR, 365 SPI_BF(PCS, ~(0x01 << spi->chip_select)) 366 | SPI_BIT(WDRBT) 367 | SPI_BIT(MODFDIS) 368 | SPI_BIT(MSTR)); 369 } else { 370 spi_writel(as, MR, 371 SPI_BF(PCS, ~(0x01 << spi->chip_select)) 372 | SPI_BIT(MODFDIS) 373 | SPI_BIT(MSTR)); 374 } 375 376 mr = spi_readl(as, MR); 377 if (as->use_cs_gpios) 378 gpiod_set_value(asd->npcs_pin, 1); 379 } else { 380 u32 cpol = (spi->mode & SPI_CPOL) ? SPI_BIT(CPOL) : 0; 381 int i; 382 u32 csr; 383 384 /* Make sure clock polarity is correct */ 385 for (i = 0; i < spi->master->num_chipselect; i++) { 386 csr = spi_readl(as, CSR0 + 4 * i); 387 if ((csr ^ cpol) & SPI_BIT(CPOL)) 388 spi_writel(as, CSR0 + 4 * i, 389 csr ^ SPI_BIT(CPOL)); 390 } 391 392 mr = spi_readl(as, MR); 393 mr = SPI_BFINS(PCS, ~(1 << spi->chip_select), mr); 394 if (as->use_cs_gpios && spi->chip_select != 0) 395 gpiod_set_value(asd->npcs_pin, 1); 396 spi_writel(as, MR, mr); 397 } 398 399 dev_dbg(&spi->dev, "activate NPCS, mr %08x\n", mr); 400 } 401 402 static void cs_deactivate(struct atmel_spi *as, struct spi_device *spi) 403 { 404 struct atmel_spi_device *asd = spi->controller_state; 405 u32 mr; 406 407 /* only deactivate *this* device; sometimes transfers to 408 * another device may be active when this routine is called. 409 */ 410 mr = spi_readl(as, MR); 411 if (~SPI_BFEXT(PCS, mr) & (1 << spi->chip_select)) { 412 mr = SPI_BFINS(PCS, 0xf, mr); 413 spi_writel(as, MR, mr); 414 } 415 416 dev_dbg(&spi->dev, "DEactivate NPCS, mr %08x\n", mr); 417 418 if (!as->use_cs_gpios) 419 spi_writel(as, CR, SPI_BIT(LASTXFER)); 420 else if (atmel_spi_is_v2(as) || spi->chip_select != 0) 421 gpiod_set_value(asd->npcs_pin, 0); 422 } 423 424 static void atmel_spi_lock(struct atmel_spi *as) __acquires(&as->lock) 425 { 426 spin_lock_irqsave(&as->lock, as->flags); 427 } 428 429 static void atmel_spi_unlock(struct atmel_spi *as) __releases(&as->lock) 430 { 431 spin_unlock_irqrestore(&as->lock, as->flags); 432 } 433 434 static inline bool atmel_spi_is_vmalloc_xfer(struct spi_transfer *xfer) 435 { 436 return is_vmalloc_addr(xfer->tx_buf) || is_vmalloc_addr(xfer->rx_buf); 437 } 438 439 static inline bool atmel_spi_use_dma(struct atmel_spi *as, 440 struct spi_transfer *xfer) 441 { 442 return as->use_dma && xfer->len >= DMA_MIN_BYTES; 443 } 444 445 static bool atmel_spi_can_dma(struct spi_master *master, 446 struct spi_device *spi, 447 struct spi_transfer *xfer) 448 { 449 struct atmel_spi *as = spi_master_get_devdata(master); 450 451 if (IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) 452 return atmel_spi_use_dma(as, xfer) && 453 !atmel_spi_is_vmalloc_xfer(xfer); 454 else 455 return atmel_spi_use_dma(as, xfer); 456 457 } 458 459 static int atmel_spi_dma_slave_config(struct atmel_spi *as, 460 struct dma_slave_config *slave_config, 461 u8 bits_per_word) 462 { 463 struct spi_master *master = platform_get_drvdata(as->pdev); 464 int err = 0; 465 466 if (bits_per_word > 8) { 467 slave_config->dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES; 468 slave_config->src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES; 469 } else { 470 slave_config->dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE; 471 slave_config->src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE; 472 } 473 474 slave_config->dst_addr = (dma_addr_t)as->phybase + SPI_TDR; 475 slave_config->src_addr = (dma_addr_t)as->phybase + SPI_RDR; 476 slave_config->src_maxburst = 1; 477 slave_config->dst_maxburst = 1; 478 slave_config->device_fc = false; 479 480 /* 481 * This driver uses fixed peripheral select mode (PS bit set to '0' in 482 * the Mode Register). 483 * So according to the datasheet, when FIFOs are available (and 484 * enabled), the Transmit FIFO operates in Multiple Data Mode. 485 * In this mode, up to 2 data, not 4, can be written into the Transmit 486 * Data Register in a single access. 487 * However, the first data has to be written into the lowest 16 bits and 488 * the second data into the highest 16 bits of the Transmit 489 * Data Register. For 8bit data (the most frequent case), it would 490 * require to rework tx_buf so each data would actualy fit 16 bits. 491 * So we'd rather write only one data at the time. Hence the transmit 492 * path works the same whether FIFOs are available (and enabled) or not. 493 */ 494 slave_config->direction = DMA_MEM_TO_DEV; 495 if (dmaengine_slave_config(master->dma_tx, slave_config)) { 496 dev_err(&as->pdev->dev, 497 "failed to configure tx dma channel\n"); 498 err = -EINVAL; 499 } 500 501 /* 502 * This driver configures the spi controller for master mode (MSTR bit 503 * set to '1' in the Mode Register). 504 * So according to the datasheet, when FIFOs are available (and 505 * enabled), the Receive FIFO operates in Single Data Mode. 506 * So the receive path works the same whether FIFOs are available (and 507 * enabled) or not. 508 */ 509 slave_config->direction = DMA_DEV_TO_MEM; 510 if (dmaengine_slave_config(master->dma_rx, slave_config)) { 511 dev_err(&as->pdev->dev, 512 "failed to configure rx dma channel\n"); 513 err = -EINVAL; 514 } 515 516 return err; 517 } 518 519 static int atmel_spi_configure_dma(struct spi_master *master, 520 struct atmel_spi *as) 521 { 522 struct dma_slave_config slave_config; 523 struct device *dev = &as->pdev->dev; 524 int err; 525 526 dma_cap_mask_t mask; 527 dma_cap_zero(mask); 528 dma_cap_set(DMA_SLAVE, mask); 529 530 master->dma_tx = dma_request_slave_channel_reason(dev, "tx"); 531 if (IS_ERR(master->dma_tx)) { 532 err = PTR_ERR(master->dma_tx); 533 if (err == -EPROBE_DEFER) { 534 dev_warn(dev, "no DMA channel available at the moment\n"); 535 goto error_clear; 536 } 537 dev_err(dev, 538 "DMA TX channel not available, SPI unable to use DMA\n"); 539 err = -EBUSY; 540 goto error_clear; 541 } 542 543 /* 544 * No reason to check EPROBE_DEFER here since we have already requested 545 * tx channel. If it fails here, it's for another reason. 546 */ 547 master->dma_rx = dma_request_slave_channel(dev, "rx"); 548 549 if (!master->dma_rx) { 550 dev_err(dev, 551 "DMA RX channel not available, SPI unable to use DMA\n"); 552 err = -EBUSY; 553 goto error; 554 } 555 556 err = atmel_spi_dma_slave_config(as, &slave_config, 8); 557 if (err) 558 goto error; 559 560 dev_info(&as->pdev->dev, 561 "Using %s (tx) and %s (rx) for DMA transfers\n", 562 dma_chan_name(master->dma_tx), 563 dma_chan_name(master->dma_rx)); 564 565 return 0; 566 error: 567 if (master->dma_rx) 568 dma_release_channel(master->dma_rx); 569 if (!IS_ERR(master->dma_tx)) 570 dma_release_channel(master->dma_tx); 571 error_clear: 572 master->dma_tx = master->dma_rx = NULL; 573 return err; 574 } 575 576 static void atmel_spi_stop_dma(struct spi_master *master) 577 { 578 if (master->dma_rx) 579 dmaengine_terminate_all(master->dma_rx); 580 if (master->dma_tx) 581 dmaengine_terminate_all(master->dma_tx); 582 } 583 584 static void atmel_spi_release_dma(struct spi_master *master) 585 { 586 if (master->dma_rx) { 587 dma_release_channel(master->dma_rx); 588 master->dma_rx = NULL; 589 } 590 if (master->dma_tx) { 591 dma_release_channel(master->dma_tx); 592 master->dma_tx = NULL; 593 } 594 } 595 596 /* This function is called by the DMA driver from tasklet context */ 597 static void dma_callback(void *data) 598 { 599 struct spi_master *master = data; 600 struct atmel_spi *as = spi_master_get_devdata(master); 601 602 if (is_vmalloc_addr(as->current_transfer->rx_buf) && 603 IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) { 604 memcpy(as->current_transfer->rx_buf, as->addr_rx_bbuf, 605 as->current_transfer->len); 606 } 607 complete(&as->xfer_completion); 608 } 609 610 /* 611 * Next transfer using PIO without FIFO. 612 */ 613 static void atmel_spi_next_xfer_single(struct spi_master *master, 614 struct spi_transfer *xfer) 615 { 616 struct atmel_spi *as = spi_master_get_devdata(master); 617 unsigned long xfer_pos = xfer->len - as->current_remaining_bytes; 618 619 dev_vdbg(master->dev.parent, "atmel_spi_next_xfer_pio\n"); 620 621 /* Make sure data is not remaining in RDR */ 622 spi_readl(as, RDR); 623 while (spi_readl(as, SR) & SPI_BIT(RDRF)) { 624 spi_readl(as, RDR); 625 cpu_relax(); 626 } 627 628 if (xfer->bits_per_word > 8) 629 spi_writel(as, TDR, *(u16 *)(xfer->tx_buf + xfer_pos)); 630 else 631 spi_writel(as, TDR, *(u8 *)(xfer->tx_buf + xfer_pos)); 632 633 dev_dbg(master->dev.parent, 634 " start pio xfer %p: len %u tx %p rx %p bitpw %d\n", 635 xfer, xfer->len, xfer->tx_buf, xfer->rx_buf, 636 xfer->bits_per_word); 637 638 /* Enable relevant interrupts */ 639 spi_writel(as, IER, SPI_BIT(RDRF) | SPI_BIT(OVRES)); 640 } 641 642 /* 643 * Next transfer using PIO with FIFO. 644 */ 645 static void atmel_spi_next_xfer_fifo(struct spi_master *master, 646 struct spi_transfer *xfer) 647 { 648 struct atmel_spi *as = spi_master_get_devdata(master); 649 u32 current_remaining_data, num_data; 650 u32 offset = xfer->len - as->current_remaining_bytes; 651 const u16 *words = (const u16 *)((u8 *)xfer->tx_buf + offset); 652 const u8 *bytes = (const u8 *)((u8 *)xfer->tx_buf + offset); 653 u16 td0, td1; 654 u32 fifomr; 655 656 dev_vdbg(master->dev.parent, "atmel_spi_next_xfer_fifo\n"); 657 658 /* Compute the number of data to transfer in the current iteration */ 659 current_remaining_data = ((xfer->bits_per_word > 8) ? 660 ((u32)as->current_remaining_bytes >> 1) : 661 (u32)as->current_remaining_bytes); 662 num_data = min(current_remaining_data, as->fifo_size); 663 664 /* Flush RX and TX FIFOs */ 665 spi_writel(as, CR, SPI_BIT(RXFCLR) | SPI_BIT(TXFCLR)); 666 while (spi_readl(as, FLR)) 667 cpu_relax(); 668 669 /* Set RX FIFO Threshold to the number of data to transfer */ 670 fifomr = spi_readl(as, FMR); 671 spi_writel(as, FMR, SPI_BFINS(RXFTHRES, num_data, fifomr)); 672 673 /* Clear FIFO flags in the Status Register, especially RXFTHF */ 674 (void)spi_readl(as, SR); 675 676 /* Fill TX FIFO */ 677 while (num_data >= 2) { 678 if (xfer->bits_per_word > 8) { 679 td0 = *words++; 680 td1 = *words++; 681 } else { 682 td0 = *bytes++; 683 td1 = *bytes++; 684 } 685 686 spi_writel(as, TDR, (td1 << 16) | td0); 687 num_data -= 2; 688 } 689 690 if (num_data) { 691 if (xfer->bits_per_word > 8) 692 td0 = *words++; 693 else 694 td0 = *bytes++; 695 696 spi_writew(as, TDR, td0); 697 num_data--; 698 } 699 700 dev_dbg(master->dev.parent, 701 " start fifo xfer %p: len %u tx %p rx %p bitpw %d\n", 702 xfer, xfer->len, xfer->tx_buf, xfer->rx_buf, 703 xfer->bits_per_word); 704 705 /* 706 * Enable RX FIFO Threshold Flag interrupt to be notified about 707 * transfer completion. 708 */ 709 spi_writel(as, IER, SPI_BIT(RXFTHF) | SPI_BIT(OVRES)); 710 } 711 712 /* 713 * Next transfer using PIO. 714 */ 715 static void atmel_spi_next_xfer_pio(struct spi_master *master, 716 struct spi_transfer *xfer) 717 { 718 struct atmel_spi *as = spi_master_get_devdata(master); 719 720 if (as->fifo_size) 721 atmel_spi_next_xfer_fifo(master, xfer); 722 else 723 atmel_spi_next_xfer_single(master, xfer); 724 } 725 726 /* 727 * Submit next transfer for DMA. 728 */ 729 static int atmel_spi_next_xfer_dma_submit(struct spi_master *master, 730 struct spi_transfer *xfer, 731 u32 *plen) 732 { 733 struct atmel_spi *as = spi_master_get_devdata(master); 734 struct dma_chan *rxchan = master->dma_rx; 735 struct dma_chan *txchan = master->dma_tx; 736 struct dma_async_tx_descriptor *rxdesc; 737 struct dma_async_tx_descriptor *txdesc; 738 struct dma_slave_config slave_config; 739 dma_cookie_t cookie; 740 741 dev_vdbg(master->dev.parent, "atmel_spi_next_xfer_dma_submit\n"); 742 743 /* Check that the channels are available */ 744 if (!rxchan || !txchan) 745 return -ENODEV; 746 747 /* release lock for DMA operations */ 748 atmel_spi_unlock(as); 749 750 *plen = xfer->len; 751 752 if (atmel_spi_dma_slave_config(as, &slave_config, 753 xfer->bits_per_word)) 754 goto err_exit; 755 756 /* Send both scatterlists */ 757 if (atmel_spi_is_vmalloc_xfer(xfer) && 758 IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) { 759 rxdesc = dmaengine_prep_slave_single(rxchan, 760 as->dma_addr_rx_bbuf, 761 xfer->len, 762 DMA_DEV_TO_MEM, 763 DMA_PREP_INTERRUPT | 764 DMA_CTRL_ACK); 765 } else { 766 rxdesc = dmaengine_prep_slave_sg(rxchan, 767 xfer->rx_sg.sgl, 768 xfer->rx_sg.nents, 769 DMA_DEV_TO_MEM, 770 DMA_PREP_INTERRUPT | 771 DMA_CTRL_ACK); 772 } 773 if (!rxdesc) 774 goto err_dma; 775 776 if (atmel_spi_is_vmalloc_xfer(xfer) && 777 IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) { 778 memcpy(as->addr_tx_bbuf, xfer->tx_buf, xfer->len); 779 txdesc = dmaengine_prep_slave_single(txchan, 780 as->dma_addr_tx_bbuf, 781 xfer->len, DMA_MEM_TO_DEV, 782 DMA_PREP_INTERRUPT | 783 DMA_CTRL_ACK); 784 } else { 785 txdesc = dmaengine_prep_slave_sg(txchan, 786 xfer->tx_sg.sgl, 787 xfer->tx_sg.nents, 788 DMA_MEM_TO_DEV, 789 DMA_PREP_INTERRUPT | 790 DMA_CTRL_ACK); 791 } 792 if (!txdesc) 793 goto err_dma; 794 795 dev_dbg(master->dev.parent, 796 " start dma xfer %p: len %u tx %p/%08llx rx %p/%08llx\n", 797 xfer, xfer->len, xfer->tx_buf, (unsigned long long)xfer->tx_dma, 798 xfer->rx_buf, (unsigned long long)xfer->rx_dma); 799 800 /* Enable relevant interrupts */ 801 spi_writel(as, IER, SPI_BIT(OVRES)); 802 803 /* Put the callback on the RX transfer only, that should finish last */ 804 rxdesc->callback = dma_callback; 805 rxdesc->callback_param = master; 806 807 /* Submit and fire RX and TX with TX last so we're ready to read! */ 808 cookie = rxdesc->tx_submit(rxdesc); 809 if (dma_submit_error(cookie)) 810 goto err_dma; 811 cookie = txdesc->tx_submit(txdesc); 812 if (dma_submit_error(cookie)) 813 goto err_dma; 814 rxchan->device->device_issue_pending(rxchan); 815 txchan->device->device_issue_pending(txchan); 816 817 /* take back lock */ 818 atmel_spi_lock(as); 819 return 0; 820 821 err_dma: 822 spi_writel(as, IDR, SPI_BIT(OVRES)); 823 atmel_spi_stop_dma(master); 824 err_exit: 825 atmel_spi_lock(as); 826 return -ENOMEM; 827 } 828 829 static void atmel_spi_next_xfer_data(struct spi_master *master, 830 struct spi_transfer *xfer, 831 dma_addr_t *tx_dma, 832 dma_addr_t *rx_dma, 833 u32 *plen) 834 { 835 *rx_dma = xfer->rx_dma + xfer->len - *plen; 836 *tx_dma = xfer->tx_dma + xfer->len - *plen; 837 if (*plen > master->max_dma_len) 838 *plen = master->max_dma_len; 839 } 840 841 static int atmel_spi_set_xfer_speed(struct atmel_spi *as, 842 struct spi_device *spi, 843 struct spi_transfer *xfer) 844 { 845 u32 scbr, csr; 846 unsigned long bus_hz; 847 848 /* v1 chips start out at half the peripheral bus speed. */ 849 bus_hz = as->spi_clk; 850 if (!atmel_spi_is_v2(as)) 851 bus_hz /= 2; 852 853 /* 854 * Calculate the lowest divider that satisfies the 855 * constraint, assuming div32/fdiv/mbz == 0. 856 */ 857 scbr = DIV_ROUND_UP(bus_hz, xfer->speed_hz); 858 859 /* 860 * If the resulting divider doesn't fit into the 861 * register bitfield, we can't satisfy the constraint. 862 */ 863 if (scbr >= (1 << SPI_SCBR_SIZE)) { 864 dev_err(&spi->dev, 865 "setup: %d Hz too slow, scbr %u; min %ld Hz\n", 866 xfer->speed_hz, scbr, bus_hz/255); 867 return -EINVAL; 868 } 869 if (scbr == 0) { 870 dev_err(&spi->dev, 871 "setup: %d Hz too high, scbr %u; max %ld Hz\n", 872 xfer->speed_hz, scbr, bus_hz); 873 return -EINVAL; 874 } 875 csr = spi_readl(as, CSR0 + 4 * spi->chip_select); 876 csr = SPI_BFINS(SCBR, scbr, csr); 877 spi_writel(as, CSR0 + 4 * spi->chip_select, csr); 878 879 return 0; 880 } 881 882 /* 883 * Submit next transfer for PDC. 884 * lock is held, spi irq is blocked 885 */ 886 static void atmel_spi_pdc_next_xfer(struct spi_master *master, 887 struct spi_message *msg, 888 struct spi_transfer *xfer) 889 { 890 struct atmel_spi *as = spi_master_get_devdata(master); 891 u32 len; 892 dma_addr_t tx_dma, rx_dma; 893 894 spi_writel(as, PTCR, SPI_BIT(RXTDIS) | SPI_BIT(TXTDIS)); 895 896 len = as->current_remaining_bytes; 897 atmel_spi_next_xfer_data(master, xfer, &tx_dma, &rx_dma, &len); 898 as->current_remaining_bytes -= len; 899 900 spi_writel(as, RPR, rx_dma); 901 spi_writel(as, TPR, tx_dma); 902 903 if (msg->spi->bits_per_word > 8) 904 len >>= 1; 905 spi_writel(as, RCR, len); 906 spi_writel(as, TCR, len); 907 908 dev_dbg(&msg->spi->dev, 909 " start xfer %p: len %u tx %p/%08llx rx %p/%08llx\n", 910 xfer, xfer->len, xfer->tx_buf, 911 (unsigned long long)xfer->tx_dma, xfer->rx_buf, 912 (unsigned long long)xfer->rx_dma); 913 914 if (as->current_remaining_bytes) { 915 len = as->current_remaining_bytes; 916 atmel_spi_next_xfer_data(master, xfer, &tx_dma, &rx_dma, &len); 917 as->current_remaining_bytes -= len; 918 919 spi_writel(as, RNPR, rx_dma); 920 spi_writel(as, TNPR, tx_dma); 921 922 if (msg->spi->bits_per_word > 8) 923 len >>= 1; 924 spi_writel(as, RNCR, len); 925 spi_writel(as, TNCR, len); 926 927 dev_dbg(&msg->spi->dev, 928 " next xfer %p: len %u tx %p/%08llx rx %p/%08llx\n", 929 xfer, xfer->len, xfer->tx_buf, 930 (unsigned long long)xfer->tx_dma, xfer->rx_buf, 931 (unsigned long long)xfer->rx_dma); 932 } 933 934 /* REVISIT: We're waiting for RXBUFF before we start the next 935 * transfer because we need to handle some difficult timing 936 * issues otherwise. If we wait for TXBUFE in one transfer and 937 * then starts waiting for RXBUFF in the next, it's difficult 938 * to tell the difference between the RXBUFF interrupt we're 939 * actually waiting for and the RXBUFF interrupt of the 940 * previous transfer. 941 * 942 * It should be doable, though. Just not now... 943 */ 944 spi_writel(as, IER, SPI_BIT(RXBUFF) | SPI_BIT(OVRES)); 945 spi_writel(as, PTCR, SPI_BIT(TXTEN) | SPI_BIT(RXTEN)); 946 } 947 948 /* 949 * For DMA, tx_buf/tx_dma have the same relationship as rx_buf/rx_dma: 950 * - The buffer is either valid for CPU access, else NULL 951 * - If the buffer is valid, so is its DMA address 952 * 953 * This driver manages the dma address unless message->is_dma_mapped. 954 */ 955 static int 956 atmel_spi_dma_map_xfer(struct atmel_spi *as, struct spi_transfer *xfer) 957 { 958 struct device *dev = &as->pdev->dev; 959 960 xfer->tx_dma = xfer->rx_dma = INVALID_DMA_ADDRESS; 961 if (xfer->tx_buf) { 962 /* tx_buf is a const void* where we need a void * for the dma 963 * mapping */ 964 void *nonconst_tx = (void *)xfer->tx_buf; 965 966 xfer->tx_dma = dma_map_single(dev, 967 nonconst_tx, xfer->len, 968 DMA_TO_DEVICE); 969 if (dma_mapping_error(dev, xfer->tx_dma)) 970 return -ENOMEM; 971 } 972 if (xfer->rx_buf) { 973 xfer->rx_dma = dma_map_single(dev, 974 xfer->rx_buf, xfer->len, 975 DMA_FROM_DEVICE); 976 if (dma_mapping_error(dev, xfer->rx_dma)) { 977 if (xfer->tx_buf) 978 dma_unmap_single(dev, 979 xfer->tx_dma, xfer->len, 980 DMA_TO_DEVICE); 981 return -ENOMEM; 982 } 983 } 984 return 0; 985 } 986 987 static void atmel_spi_dma_unmap_xfer(struct spi_master *master, 988 struct spi_transfer *xfer) 989 { 990 if (xfer->tx_dma != INVALID_DMA_ADDRESS) 991 dma_unmap_single(master->dev.parent, xfer->tx_dma, 992 xfer->len, DMA_TO_DEVICE); 993 if (xfer->rx_dma != INVALID_DMA_ADDRESS) 994 dma_unmap_single(master->dev.parent, xfer->rx_dma, 995 xfer->len, DMA_FROM_DEVICE); 996 } 997 998 static void atmel_spi_disable_pdc_transfer(struct atmel_spi *as) 999 { 1000 spi_writel(as, PTCR, SPI_BIT(RXTDIS) | SPI_BIT(TXTDIS)); 1001 } 1002 1003 static void 1004 atmel_spi_pump_single_data(struct atmel_spi *as, struct spi_transfer *xfer) 1005 { 1006 u8 *rxp; 1007 u16 *rxp16; 1008 unsigned long xfer_pos = xfer->len - as->current_remaining_bytes; 1009 1010 if (xfer->bits_per_word > 8) { 1011 rxp16 = (u16 *)(((u8 *)xfer->rx_buf) + xfer_pos); 1012 *rxp16 = spi_readl(as, RDR); 1013 } else { 1014 rxp = ((u8 *)xfer->rx_buf) + xfer_pos; 1015 *rxp = spi_readl(as, RDR); 1016 } 1017 if (xfer->bits_per_word > 8) { 1018 if (as->current_remaining_bytes > 2) 1019 as->current_remaining_bytes -= 2; 1020 else 1021 as->current_remaining_bytes = 0; 1022 } else { 1023 as->current_remaining_bytes--; 1024 } 1025 } 1026 1027 static void 1028 atmel_spi_pump_fifo_data(struct atmel_spi *as, struct spi_transfer *xfer) 1029 { 1030 u32 fifolr = spi_readl(as, FLR); 1031 u32 num_bytes, num_data = SPI_BFEXT(RXFL, fifolr); 1032 u32 offset = xfer->len - as->current_remaining_bytes; 1033 u16 *words = (u16 *)((u8 *)xfer->rx_buf + offset); 1034 u8 *bytes = (u8 *)((u8 *)xfer->rx_buf + offset); 1035 u16 rd; /* RD field is the lowest 16 bits of RDR */ 1036 1037 /* Update the number of remaining bytes to transfer */ 1038 num_bytes = ((xfer->bits_per_word > 8) ? 1039 (num_data << 1) : 1040 num_data); 1041 1042 if (as->current_remaining_bytes > num_bytes) 1043 as->current_remaining_bytes -= num_bytes; 1044 else 1045 as->current_remaining_bytes = 0; 1046 1047 /* Handle odd number of bytes when data are more than 8bit width */ 1048 if (xfer->bits_per_word > 8) 1049 as->current_remaining_bytes &= ~0x1; 1050 1051 /* Read data */ 1052 while (num_data) { 1053 rd = spi_readl(as, RDR); 1054 if (xfer->bits_per_word > 8) 1055 *words++ = rd; 1056 else 1057 *bytes++ = rd; 1058 num_data--; 1059 } 1060 } 1061 1062 /* Called from IRQ 1063 * 1064 * Must update "current_remaining_bytes" to keep track of data 1065 * to transfer. 1066 */ 1067 static void 1068 atmel_spi_pump_pio_data(struct atmel_spi *as, struct spi_transfer *xfer) 1069 { 1070 if (as->fifo_size) 1071 atmel_spi_pump_fifo_data(as, xfer); 1072 else 1073 atmel_spi_pump_single_data(as, xfer); 1074 } 1075 1076 /* Interrupt 1077 * 1078 * No need for locking in this Interrupt handler: done_status is the 1079 * only information modified. 1080 */ 1081 static irqreturn_t 1082 atmel_spi_pio_interrupt(int irq, void *dev_id) 1083 { 1084 struct spi_master *master = dev_id; 1085 struct atmel_spi *as = spi_master_get_devdata(master); 1086 u32 status, pending, imr; 1087 struct spi_transfer *xfer; 1088 int ret = IRQ_NONE; 1089 1090 imr = spi_readl(as, IMR); 1091 status = spi_readl(as, SR); 1092 pending = status & imr; 1093 1094 if (pending & SPI_BIT(OVRES)) { 1095 ret = IRQ_HANDLED; 1096 spi_writel(as, IDR, SPI_BIT(OVRES)); 1097 dev_warn(master->dev.parent, "overrun\n"); 1098 1099 /* 1100 * When we get an overrun, we disregard the current 1101 * transfer. Data will not be copied back from any 1102 * bounce buffer and msg->actual_len will not be 1103 * updated with the last xfer. 1104 * 1105 * We will also not process any remaning transfers in 1106 * the message. 1107 */ 1108 as->done_status = -EIO; 1109 smp_wmb(); 1110 1111 /* Clear any overrun happening while cleaning up */ 1112 spi_readl(as, SR); 1113 1114 complete(&as->xfer_completion); 1115 1116 } else if (pending & (SPI_BIT(RDRF) | SPI_BIT(RXFTHF))) { 1117 atmel_spi_lock(as); 1118 1119 if (as->current_remaining_bytes) { 1120 ret = IRQ_HANDLED; 1121 xfer = as->current_transfer; 1122 atmel_spi_pump_pio_data(as, xfer); 1123 if (!as->current_remaining_bytes) 1124 spi_writel(as, IDR, pending); 1125 1126 complete(&as->xfer_completion); 1127 } 1128 1129 atmel_spi_unlock(as); 1130 } else { 1131 WARN_ONCE(pending, "IRQ not handled, pending = %x\n", pending); 1132 ret = IRQ_HANDLED; 1133 spi_writel(as, IDR, pending); 1134 } 1135 1136 return ret; 1137 } 1138 1139 static irqreturn_t 1140 atmel_spi_pdc_interrupt(int irq, void *dev_id) 1141 { 1142 struct spi_master *master = dev_id; 1143 struct atmel_spi *as = spi_master_get_devdata(master); 1144 u32 status, pending, imr; 1145 int ret = IRQ_NONE; 1146 1147 imr = spi_readl(as, IMR); 1148 status = spi_readl(as, SR); 1149 pending = status & imr; 1150 1151 if (pending & SPI_BIT(OVRES)) { 1152 1153 ret = IRQ_HANDLED; 1154 1155 spi_writel(as, IDR, (SPI_BIT(RXBUFF) | SPI_BIT(ENDRX) 1156 | SPI_BIT(OVRES))); 1157 1158 /* Clear any overrun happening while cleaning up */ 1159 spi_readl(as, SR); 1160 1161 as->done_status = -EIO; 1162 1163 complete(&as->xfer_completion); 1164 1165 } else if (pending & (SPI_BIT(RXBUFF) | SPI_BIT(ENDRX))) { 1166 ret = IRQ_HANDLED; 1167 1168 spi_writel(as, IDR, pending); 1169 1170 complete(&as->xfer_completion); 1171 } 1172 1173 return ret; 1174 } 1175 1176 static int atmel_spi_setup(struct spi_device *spi) 1177 { 1178 struct atmel_spi *as; 1179 struct atmel_spi_device *asd; 1180 u32 csr; 1181 unsigned int bits = spi->bits_per_word; 1182 1183 as = spi_master_get_devdata(spi->master); 1184 1185 /* see notes above re chipselect */ 1186 if (!atmel_spi_is_v2(as) 1187 && spi->chip_select == 0 1188 && (spi->mode & SPI_CS_HIGH)) { 1189 dev_dbg(&spi->dev, "setup: can't be active-high\n"); 1190 return -EINVAL; 1191 } 1192 1193 csr = SPI_BF(BITS, bits - 8); 1194 if (spi->mode & SPI_CPOL) 1195 csr |= SPI_BIT(CPOL); 1196 if (!(spi->mode & SPI_CPHA)) 1197 csr |= SPI_BIT(NCPHA); 1198 if (!as->use_cs_gpios) 1199 csr |= SPI_BIT(CSAAT); 1200 1201 /* DLYBS is mostly irrelevant since we manage chipselect using GPIOs. 1202 */ 1203 csr |= SPI_BF(DLYBS, 0); 1204 1205 /* DLYBCT adds delays between words. This is useful for slow devices 1206 * that need a bit of time to setup the next transfer. 1207 */ 1208 csr |= SPI_BF(DLYBCT, 1209 (as->spi_clk / 1000000 * spi->word_delay_usecs) >> 5); 1210 1211 asd = spi->controller_state; 1212 if (!asd) { 1213 asd = kzalloc(sizeof(struct atmel_spi_device), GFP_KERNEL); 1214 if (!asd) 1215 return -ENOMEM; 1216 1217 /* 1218 * If use_cs_gpios is true this means that we have "cs-gpios" 1219 * defined in the device tree node so we should have 1220 * gotten the GPIO lines from the device tree inside the 1221 * SPI core. Warn if this is not the case but continue since 1222 * CS GPIOs are after all optional. 1223 */ 1224 if (as->use_cs_gpios) { 1225 if (!spi->cs_gpiod) { 1226 dev_err(&spi->dev, 1227 "host claims to use CS GPIOs but no CS found in DT by the SPI core\n"); 1228 } 1229 asd->npcs_pin = spi->cs_gpiod; 1230 } 1231 1232 spi->controller_state = asd; 1233 } 1234 1235 asd->csr = csr; 1236 1237 dev_dbg(&spi->dev, 1238 "setup: bpw %u mode 0x%x -> csr%d %08x\n", 1239 bits, spi->mode, spi->chip_select, csr); 1240 1241 if (!atmel_spi_is_v2(as)) 1242 spi_writel(as, CSR0 + 4 * spi->chip_select, csr); 1243 1244 return 0; 1245 } 1246 1247 static int atmel_spi_one_transfer(struct spi_master *master, 1248 struct spi_message *msg, 1249 struct spi_transfer *xfer) 1250 { 1251 struct atmel_spi *as; 1252 struct spi_device *spi = msg->spi; 1253 u8 bits; 1254 u32 len; 1255 struct atmel_spi_device *asd; 1256 int timeout; 1257 int ret; 1258 unsigned long dma_timeout; 1259 1260 as = spi_master_get_devdata(master); 1261 1262 if (!(xfer->tx_buf || xfer->rx_buf) && xfer->len) { 1263 dev_dbg(&spi->dev, "missing rx or tx buf\n"); 1264 return -EINVAL; 1265 } 1266 1267 asd = spi->controller_state; 1268 bits = (asd->csr >> 4) & 0xf; 1269 if (bits != xfer->bits_per_word - 8) { 1270 dev_dbg(&spi->dev, 1271 "you can't yet change bits_per_word in transfers\n"); 1272 return -ENOPROTOOPT; 1273 } 1274 1275 /* 1276 * DMA map early, for performance (empties dcache ASAP) and 1277 * better fault reporting. 1278 */ 1279 if ((!msg->is_dma_mapped) 1280 && as->use_pdc) { 1281 if (atmel_spi_dma_map_xfer(as, xfer) < 0) 1282 return -ENOMEM; 1283 } 1284 1285 atmel_spi_set_xfer_speed(as, msg->spi, xfer); 1286 1287 as->done_status = 0; 1288 as->current_transfer = xfer; 1289 as->current_remaining_bytes = xfer->len; 1290 while (as->current_remaining_bytes) { 1291 reinit_completion(&as->xfer_completion); 1292 1293 if (as->use_pdc) { 1294 atmel_spi_pdc_next_xfer(master, msg, xfer); 1295 } else if (atmel_spi_use_dma(as, xfer)) { 1296 len = as->current_remaining_bytes; 1297 ret = atmel_spi_next_xfer_dma_submit(master, 1298 xfer, &len); 1299 if (ret) { 1300 dev_err(&spi->dev, 1301 "unable to use DMA, fallback to PIO\n"); 1302 atmel_spi_next_xfer_pio(master, xfer); 1303 } else { 1304 as->current_remaining_bytes -= len; 1305 if (as->current_remaining_bytes < 0) 1306 as->current_remaining_bytes = 0; 1307 } 1308 } else { 1309 atmel_spi_next_xfer_pio(master, xfer); 1310 } 1311 1312 /* interrupts are disabled, so free the lock for schedule */ 1313 atmel_spi_unlock(as); 1314 dma_timeout = wait_for_completion_timeout(&as->xfer_completion, 1315 SPI_DMA_TIMEOUT); 1316 atmel_spi_lock(as); 1317 if (WARN_ON(dma_timeout == 0)) { 1318 dev_err(&spi->dev, "spi transfer timeout\n"); 1319 as->done_status = -EIO; 1320 } 1321 1322 if (as->done_status) 1323 break; 1324 } 1325 1326 if (as->done_status) { 1327 if (as->use_pdc) { 1328 dev_warn(master->dev.parent, 1329 "overrun (%u/%u remaining)\n", 1330 spi_readl(as, TCR), spi_readl(as, RCR)); 1331 1332 /* 1333 * Clean up DMA registers and make sure the data 1334 * registers are empty. 1335 */ 1336 spi_writel(as, RNCR, 0); 1337 spi_writel(as, TNCR, 0); 1338 spi_writel(as, RCR, 0); 1339 spi_writel(as, TCR, 0); 1340 for (timeout = 1000; timeout; timeout--) 1341 if (spi_readl(as, SR) & SPI_BIT(TXEMPTY)) 1342 break; 1343 if (!timeout) 1344 dev_warn(master->dev.parent, 1345 "timeout waiting for TXEMPTY"); 1346 while (spi_readl(as, SR) & SPI_BIT(RDRF)) 1347 spi_readl(as, RDR); 1348 1349 /* Clear any overrun happening while cleaning up */ 1350 spi_readl(as, SR); 1351 1352 } else if (atmel_spi_use_dma(as, xfer)) { 1353 atmel_spi_stop_dma(master); 1354 } 1355 1356 if (!msg->is_dma_mapped 1357 && as->use_pdc) 1358 atmel_spi_dma_unmap_xfer(master, xfer); 1359 1360 return 0; 1361 1362 } else { 1363 /* only update length if no error */ 1364 msg->actual_length += xfer->len; 1365 } 1366 1367 if (!msg->is_dma_mapped 1368 && as->use_pdc) 1369 atmel_spi_dma_unmap_xfer(master, xfer); 1370 1371 if (xfer->delay_usecs) 1372 udelay(xfer->delay_usecs); 1373 1374 if (xfer->cs_change) { 1375 if (list_is_last(&xfer->transfer_list, 1376 &msg->transfers)) { 1377 as->keep_cs = true; 1378 } else { 1379 as->cs_active = !as->cs_active; 1380 if (as->cs_active) 1381 cs_activate(as, msg->spi); 1382 else 1383 cs_deactivate(as, msg->spi); 1384 } 1385 } 1386 1387 return 0; 1388 } 1389 1390 static int atmel_spi_transfer_one_message(struct spi_master *master, 1391 struct spi_message *msg) 1392 { 1393 struct atmel_spi *as; 1394 struct spi_transfer *xfer; 1395 struct spi_device *spi = msg->spi; 1396 int ret = 0; 1397 1398 as = spi_master_get_devdata(master); 1399 1400 dev_dbg(&spi->dev, "new message %p submitted for %s\n", 1401 msg, dev_name(&spi->dev)); 1402 1403 atmel_spi_lock(as); 1404 cs_activate(as, spi); 1405 1406 as->cs_active = true; 1407 as->keep_cs = false; 1408 1409 msg->status = 0; 1410 msg->actual_length = 0; 1411 1412 list_for_each_entry(xfer, &msg->transfers, transfer_list) { 1413 trace_spi_transfer_start(msg, xfer); 1414 1415 ret = atmel_spi_one_transfer(master, msg, xfer); 1416 if (ret) 1417 goto msg_done; 1418 1419 trace_spi_transfer_stop(msg, xfer); 1420 } 1421 1422 if (as->use_pdc) 1423 atmel_spi_disable_pdc_transfer(as); 1424 1425 list_for_each_entry(xfer, &msg->transfers, transfer_list) { 1426 dev_dbg(&spi->dev, 1427 " xfer %p: len %u tx %p/%pad rx %p/%pad\n", 1428 xfer, xfer->len, 1429 xfer->tx_buf, &xfer->tx_dma, 1430 xfer->rx_buf, &xfer->rx_dma); 1431 } 1432 1433 msg_done: 1434 if (!as->keep_cs) 1435 cs_deactivate(as, msg->spi); 1436 1437 atmel_spi_unlock(as); 1438 1439 msg->status = as->done_status; 1440 spi_finalize_current_message(spi->master); 1441 1442 return ret; 1443 } 1444 1445 static void atmel_spi_cleanup(struct spi_device *spi) 1446 { 1447 struct atmel_spi_device *asd = spi->controller_state; 1448 1449 if (!asd) 1450 return; 1451 1452 spi->controller_state = NULL; 1453 kfree(asd); 1454 } 1455 1456 static inline unsigned int atmel_get_version(struct atmel_spi *as) 1457 { 1458 return spi_readl(as, VERSION) & 0x00000fff; 1459 } 1460 1461 static void atmel_get_caps(struct atmel_spi *as) 1462 { 1463 unsigned int version; 1464 1465 version = atmel_get_version(as); 1466 1467 as->caps.is_spi2 = version > 0x121; 1468 as->caps.has_wdrbt = version >= 0x210; 1469 as->caps.has_dma_support = version >= 0x212; 1470 as->caps.has_pdc_support = version < 0x212; 1471 } 1472 1473 static void atmel_spi_init(struct atmel_spi *as) 1474 { 1475 spi_writel(as, CR, SPI_BIT(SWRST)); 1476 spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */ 1477 1478 /* It is recommended to enable FIFOs first thing after reset */ 1479 if (as->fifo_size) 1480 spi_writel(as, CR, SPI_BIT(FIFOEN)); 1481 1482 if (as->caps.has_wdrbt) { 1483 spi_writel(as, MR, SPI_BIT(WDRBT) | SPI_BIT(MODFDIS) 1484 | SPI_BIT(MSTR)); 1485 } else { 1486 spi_writel(as, MR, SPI_BIT(MSTR) | SPI_BIT(MODFDIS)); 1487 } 1488 1489 if (as->use_pdc) 1490 spi_writel(as, PTCR, SPI_BIT(RXTDIS) | SPI_BIT(TXTDIS)); 1491 spi_writel(as, CR, SPI_BIT(SPIEN)); 1492 } 1493 1494 static int atmel_spi_probe(struct platform_device *pdev) 1495 { 1496 struct resource *regs; 1497 int irq; 1498 struct clk *clk; 1499 int ret; 1500 struct spi_master *master; 1501 struct atmel_spi *as; 1502 1503 /* Select default pin state */ 1504 pinctrl_pm_select_default_state(&pdev->dev); 1505 1506 regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1507 if (!regs) 1508 return -ENXIO; 1509 1510 irq = platform_get_irq(pdev, 0); 1511 if (irq < 0) 1512 return irq; 1513 1514 clk = devm_clk_get(&pdev->dev, "spi_clk"); 1515 if (IS_ERR(clk)) 1516 return PTR_ERR(clk); 1517 1518 /* setup spi core then atmel-specific driver state */ 1519 ret = -ENOMEM; 1520 master = spi_alloc_master(&pdev->dev, sizeof(*as)); 1521 if (!master) 1522 goto out_free; 1523 1524 /* the spi->mode bits understood by this driver: */ 1525 master->use_gpio_descriptors = true; 1526 master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH; 1527 master->bits_per_word_mask = SPI_BPW_RANGE_MASK(8, 16); 1528 master->dev.of_node = pdev->dev.of_node; 1529 master->bus_num = pdev->id; 1530 master->num_chipselect = master->dev.of_node ? 0 : 4; 1531 master->setup = atmel_spi_setup; 1532 master->flags = (SPI_MASTER_MUST_RX | SPI_MASTER_MUST_TX); 1533 master->transfer_one_message = atmel_spi_transfer_one_message; 1534 master->cleanup = atmel_spi_cleanup; 1535 master->auto_runtime_pm = true; 1536 master->max_dma_len = SPI_MAX_DMA_XFER; 1537 master->can_dma = atmel_spi_can_dma; 1538 platform_set_drvdata(pdev, master); 1539 1540 as = spi_master_get_devdata(master); 1541 1542 spin_lock_init(&as->lock); 1543 1544 as->pdev = pdev; 1545 as->regs = devm_ioremap_resource(&pdev->dev, regs); 1546 if (IS_ERR(as->regs)) { 1547 ret = PTR_ERR(as->regs); 1548 goto out_unmap_regs; 1549 } 1550 as->phybase = regs->start; 1551 as->irq = irq; 1552 as->clk = clk; 1553 1554 init_completion(&as->xfer_completion); 1555 1556 atmel_get_caps(as); 1557 1558 /* 1559 * If there are chip selects in the device tree, those will be 1560 * discovered by the SPI core when registering the SPI master 1561 * and assigned to each SPI device. 1562 */ 1563 as->use_cs_gpios = true; 1564 if (atmel_spi_is_v2(as) && 1565 pdev->dev.of_node && 1566 !of_get_property(pdev->dev.of_node, "cs-gpios", NULL)) { 1567 as->use_cs_gpios = false; 1568 master->num_chipselect = 4; 1569 } 1570 1571 as->use_dma = false; 1572 as->use_pdc = false; 1573 if (as->caps.has_dma_support) { 1574 ret = atmel_spi_configure_dma(master, as); 1575 if (ret == 0) { 1576 as->use_dma = true; 1577 } else if (ret == -EPROBE_DEFER) { 1578 return ret; 1579 } 1580 } else if (as->caps.has_pdc_support) { 1581 as->use_pdc = true; 1582 } 1583 1584 if (IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) { 1585 as->addr_rx_bbuf = dma_alloc_coherent(&pdev->dev, 1586 SPI_MAX_DMA_XFER, 1587 &as->dma_addr_rx_bbuf, 1588 GFP_KERNEL | GFP_DMA); 1589 if (!as->addr_rx_bbuf) { 1590 as->use_dma = false; 1591 } else { 1592 as->addr_tx_bbuf = dma_alloc_coherent(&pdev->dev, 1593 SPI_MAX_DMA_XFER, 1594 &as->dma_addr_tx_bbuf, 1595 GFP_KERNEL | GFP_DMA); 1596 if (!as->addr_tx_bbuf) { 1597 as->use_dma = false; 1598 dma_free_coherent(&pdev->dev, SPI_MAX_DMA_XFER, 1599 as->addr_rx_bbuf, 1600 as->dma_addr_rx_bbuf); 1601 } 1602 } 1603 if (!as->use_dma) 1604 dev_info(master->dev.parent, 1605 " can not allocate dma coherent memory\n"); 1606 } 1607 1608 if (as->caps.has_dma_support && !as->use_dma) 1609 dev_info(&pdev->dev, "Atmel SPI Controller using PIO only\n"); 1610 1611 if (as->use_pdc) { 1612 ret = devm_request_irq(&pdev->dev, irq, atmel_spi_pdc_interrupt, 1613 0, dev_name(&pdev->dev), master); 1614 } else { 1615 ret = devm_request_irq(&pdev->dev, irq, atmel_spi_pio_interrupt, 1616 0, dev_name(&pdev->dev), master); 1617 } 1618 if (ret) 1619 goto out_unmap_regs; 1620 1621 /* Initialize the hardware */ 1622 ret = clk_prepare_enable(clk); 1623 if (ret) 1624 goto out_free_irq; 1625 1626 as->spi_clk = clk_get_rate(clk); 1627 1628 as->fifo_size = 0; 1629 if (!of_property_read_u32(pdev->dev.of_node, "atmel,fifo-size", 1630 &as->fifo_size)) { 1631 dev_info(&pdev->dev, "Using FIFO (%u data)\n", as->fifo_size); 1632 } 1633 1634 atmel_spi_init(as); 1635 1636 pm_runtime_set_autosuspend_delay(&pdev->dev, AUTOSUSPEND_TIMEOUT); 1637 pm_runtime_use_autosuspend(&pdev->dev); 1638 pm_runtime_set_active(&pdev->dev); 1639 pm_runtime_enable(&pdev->dev); 1640 1641 ret = devm_spi_register_master(&pdev->dev, master); 1642 if (ret) 1643 goto out_free_dma; 1644 1645 /* go! */ 1646 dev_info(&pdev->dev, "Atmel SPI Controller version 0x%x at 0x%08lx (irq %d)\n", 1647 atmel_get_version(as), (unsigned long)regs->start, 1648 irq); 1649 1650 return 0; 1651 1652 out_free_dma: 1653 pm_runtime_disable(&pdev->dev); 1654 pm_runtime_set_suspended(&pdev->dev); 1655 1656 if (as->use_dma) 1657 atmel_spi_release_dma(master); 1658 1659 spi_writel(as, CR, SPI_BIT(SWRST)); 1660 spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */ 1661 clk_disable_unprepare(clk); 1662 out_free_irq: 1663 out_unmap_regs: 1664 out_free: 1665 spi_master_put(master); 1666 return ret; 1667 } 1668 1669 static int atmel_spi_remove(struct platform_device *pdev) 1670 { 1671 struct spi_master *master = platform_get_drvdata(pdev); 1672 struct atmel_spi *as = spi_master_get_devdata(master); 1673 1674 pm_runtime_get_sync(&pdev->dev); 1675 1676 /* reset the hardware and block queue progress */ 1677 if (as->use_dma) { 1678 atmel_spi_stop_dma(master); 1679 atmel_spi_release_dma(master); 1680 if (IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) { 1681 dma_free_coherent(&pdev->dev, SPI_MAX_DMA_XFER, 1682 as->addr_tx_bbuf, 1683 as->dma_addr_tx_bbuf); 1684 dma_free_coherent(&pdev->dev, SPI_MAX_DMA_XFER, 1685 as->addr_rx_bbuf, 1686 as->dma_addr_rx_bbuf); 1687 } 1688 } 1689 1690 spin_lock_irq(&as->lock); 1691 spi_writel(as, CR, SPI_BIT(SWRST)); 1692 spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */ 1693 spi_readl(as, SR); 1694 spin_unlock_irq(&as->lock); 1695 1696 clk_disable_unprepare(as->clk); 1697 1698 pm_runtime_put_noidle(&pdev->dev); 1699 pm_runtime_disable(&pdev->dev); 1700 1701 return 0; 1702 } 1703 1704 #ifdef CONFIG_PM 1705 static int atmel_spi_runtime_suspend(struct device *dev) 1706 { 1707 struct spi_master *master = dev_get_drvdata(dev); 1708 struct atmel_spi *as = spi_master_get_devdata(master); 1709 1710 clk_disable_unprepare(as->clk); 1711 pinctrl_pm_select_sleep_state(dev); 1712 1713 return 0; 1714 } 1715 1716 static int atmel_spi_runtime_resume(struct device *dev) 1717 { 1718 struct spi_master *master = dev_get_drvdata(dev); 1719 struct atmel_spi *as = spi_master_get_devdata(master); 1720 1721 pinctrl_pm_select_default_state(dev); 1722 1723 return clk_prepare_enable(as->clk); 1724 } 1725 1726 #ifdef CONFIG_PM_SLEEP 1727 static int atmel_spi_suspend(struct device *dev) 1728 { 1729 struct spi_master *master = dev_get_drvdata(dev); 1730 int ret; 1731 1732 /* Stop the queue running */ 1733 ret = spi_master_suspend(master); 1734 if (ret) 1735 return ret; 1736 1737 if (!pm_runtime_suspended(dev)) 1738 atmel_spi_runtime_suspend(dev); 1739 1740 return 0; 1741 } 1742 1743 static int atmel_spi_resume(struct device *dev) 1744 { 1745 struct spi_master *master = dev_get_drvdata(dev); 1746 struct atmel_spi *as = spi_master_get_devdata(master); 1747 int ret; 1748 1749 ret = clk_prepare_enable(as->clk); 1750 if (ret) 1751 return ret; 1752 1753 atmel_spi_init(as); 1754 1755 clk_disable_unprepare(as->clk); 1756 1757 if (!pm_runtime_suspended(dev)) { 1758 ret = atmel_spi_runtime_resume(dev); 1759 if (ret) 1760 return ret; 1761 } 1762 1763 /* Start the queue running */ 1764 return spi_master_resume(master); 1765 } 1766 #endif 1767 1768 static const struct dev_pm_ops atmel_spi_pm_ops = { 1769 SET_SYSTEM_SLEEP_PM_OPS(atmel_spi_suspend, atmel_spi_resume) 1770 SET_RUNTIME_PM_OPS(atmel_spi_runtime_suspend, 1771 atmel_spi_runtime_resume, NULL) 1772 }; 1773 #define ATMEL_SPI_PM_OPS (&atmel_spi_pm_ops) 1774 #else 1775 #define ATMEL_SPI_PM_OPS NULL 1776 #endif 1777 1778 #if defined(CONFIG_OF) 1779 static const struct of_device_id atmel_spi_dt_ids[] = { 1780 { .compatible = "atmel,at91rm9200-spi" }, 1781 { /* sentinel */ } 1782 }; 1783 1784 MODULE_DEVICE_TABLE(of, atmel_spi_dt_ids); 1785 #endif 1786 1787 static struct platform_driver atmel_spi_driver = { 1788 .driver = { 1789 .name = "atmel_spi", 1790 .pm = ATMEL_SPI_PM_OPS, 1791 .of_match_table = of_match_ptr(atmel_spi_dt_ids), 1792 }, 1793 .probe = atmel_spi_probe, 1794 .remove = atmel_spi_remove, 1795 }; 1796 module_platform_driver(atmel_spi_driver); 1797 1798 MODULE_DESCRIPTION("Atmel AT32/AT91 SPI Controller driver"); 1799 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 1800 MODULE_LICENSE("GPL"); 1801 MODULE_ALIAS("platform:atmel_spi"); 1802