1 /* 2 * DMA support use of SYS DMAC with SDHI SD/SDIO controller 3 * 4 * Copyright (C) 2016-17 Renesas Electronics Corporation 5 * Copyright (C) 2016-17 Sang Engineering, Wolfram Sang 6 * Copyright (C) 2017 Horms Solutions, Simon Horman 7 * Copyright (C) 2010-2011 Guennadi Liakhovetski 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 as 11 * published by the Free Software Foundation. 12 */ 13 14 #include <linux/device.h> 15 #include <linux/dma-mapping.h> 16 #include <linux/dmaengine.h> 17 #include <linux/mfd/tmio.h> 18 #include <linux/mmc/host.h> 19 #include <linux/mod_devicetable.h> 20 #include <linux/module.h> 21 #include <linux/of_device.h> 22 #include <linux/pagemap.h> 23 #include <linux/scatterlist.h> 24 #include <linux/sys_soc.h> 25 26 #include "renesas_sdhi.h" 27 #include "tmio_mmc.h" 28 29 #define TMIO_MMC_MIN_DMA_LEN 8 30 31 static const struct renesas_sdhi_of_data of_default_cfg = { 32 .tmio_flags = TMIO_MMC_HAS_IDLE_WAIT, 33 }; 34 35 static const struct renesas_sdhi_of_data of_rz_compatible = { 36 .tmio_flags = TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_32BIT_DATA_PORT | 37 TMIO_MMC_HAVE_CBSY, 38 .tmio_ocr_mask = MMC_VDD_32_33, 39 .capabilities = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ, 40 }; 41 42 static const struct renesas_sdhi_of_data of_rcar_gen1_compatible = { 43 .tmio_flags = TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_CLK_ACTUAL, 44 .capabilities = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ, 45 .capabilities2 = MMC_CAP2_NO_WRITE_PROTECT, 46 }; 47 48 /* Definitions for sampling clocks */ 49 static struct renesas_sdhi_scc rcar_gen2_scc_taps[] = { 50 { 51 .clk_rate = 156000000, 52 .tap = 0x00000703, 53 }, 54 { 55 .clk_rate = 0, 56 .tap = 0x00000300, 57 }, 58 }; 59 60 static const struct renesas_sdhi_of_data of_rcar_gen2_compatible = { 61 .tmio_flags = TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_CLK_ACTUAL | 62 TMIO_MMC_HAVE_CBSY | TMIO_MMC_MIN_RCAR2, 63 .capabilities = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ | 64 MMC_CAP_CMD23, 65 .capabilities2 = MMC_CAP2_NO_WRITE_PROTECT, 66 .dma_buswidth = DMA_SLAVE_BUSWIDTH_4_BYTES, 67 .dma_rx_offset = 0x2000, 68 .scc_offset = 0x0300, 69 .taps = rcar_gen2_scc_taps, 70 .taps_num = ARRAY_SIZE(rcar_gen2_scc_taps), 71 }; 72 73 /* Definitions for sampling clocks */ 74 static struct renesas_sdhi_scc rcar_gen3_scc_taps[] = { 75 { 76 .clk_rate = 0, 77 .tap = 0x00000300, 78 }, 79 }; 80 81 static const struct renesas_sdhi_of_data of_rcar_r8a7795_compatible = { 82 .tmio_flags = TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_CLK_ACTUAL | 83 TMIO_MMC_HAVE_CBSY | TMIO_MMC_MIN_RCAR2 | 84 TMIO_MMC_HAVE_4TAP_HS400, 85 .capabilities = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ | 86 MMC_CAP_CMD23, 87 .capabilities2 = MMC_CAP2_NO_WRITE_PROTECT, 88 .bus_shift = 2, 89 .scc_offset = 0x1000, 90 .taps = rcar_gen3_scc_taps, 91 .taps_num = ARRAY_SIZE(rcar_gen3_scc_taps), 92 }; 93 94 static const struct renesas_sdhi_of_data of_rcar_gen3_compatible = { 95 .tmio_flags = TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_CLK_ACTUAL | 96 TMIO_MMC_HAVE_CBSY | TMIO_MMC_MIN_RCAR2, 97 .capabilities = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ | 98 MMC_CAP_CMD23, 99 .capabilities2 = MMC_CAP2_NO_WRITE_PROTECT, 100 .bus_shift = 2, 101 .scc_offset = 0x1000, 102 .taps = rcar_gen3_scc_taps, 103 .taps_num = ARRAY_SIZE(rcar_gen3_scc_taps), 104 }; 105 106 static const struct of_device_id renesas_sdhi_sys_dmac_of_match[] = { 107 { .compatible = "renesas,sdhi-sh73a0", .data = &of_default_cfg, }, 108 { .compatible = "renesas,sdhi-r8a73a4", .data = &of_default_cfg, }, 109 { .compatible = "renesas,sdhi-r8a7740", .data = &of_default_cfg, }, 110 { .compatible = "renesas,sdhi-r7s72100", .data = &of_rz_compatible, }, 111 { .compatible = "renesas,sdhi-r8a7778", .data = &of_rcar_gen1_compatible, }, 112 { .compatible = "renesas,sdhi-r8a7779", .data = &of_rcar_gen1_compatible, }, 113 { .compatible = "renesas,sdhi-r8a7743", .data = &of_rcar_gen2_compatible, }, 114 { .compatible = "renesas,sdhi-r8a7745", .data = &of_rcar_gen2_compatible, }, 115 { .compatible = "renesas,sdhi-r8a7790", .data = &of_rcar_gen2_compatible, }, 116 { .compatible = "renesas,sdhi-r8a7791", .data = &of_rcar_gen2_compatible, }, 117 { .compatible = "renesas,sdhi-r8a7792", .data = &of_rcar_gen2_compatible, }, 118 { .compatible = "renesas,sdhi-r8a7793", .data = &of_rcar_gen2_compatible, }, 119 { .compatible = "renesas,sdhi-r8a7794", .data = &of_rcar_gen2_compatible, }, 120 { .compatible = "renesas,sdhi-r8a7795", .data = &of_rcar_r8a7795_compatible, }, 121 { .compatible = "renesas,sdhi-r8a7796", .data = &of_rcar_r8a7795_compatible, }, 122 { .compatible = "renesas,rcar-gen1-sdhi", .data = &of_rcar_gen1_compatible, }, 123 { .compatible = "renesas,rcar-gen2-sdhi", .data = &of_rcar_gen2_compatible, }, 124 { .compatible = "renesas,rcar-gen3-sdhi", .data = &of_rcar_gen3_compatible, }, 125 { .compatible = "renesas,sdhi-shmobile" }, 126 {}, 127 }; 128 MODULE_DEVICE_TABLE(of, renesas_sdhi_sys_dmac_of_match); 129 130 static void renesas_sdhi_sys_dmac_enable_dma(struct tmio_mmc_host *host, 131 bool enable) 132 { 133 struct renesas_sdhi *priv = host_to_priv(host); 134 135 if (!host->chan_tx || !host->chan_rx) 136 return; 137 138 if (priv->dma_priv.enable) 139 priv->dma_priv.enable(host, enable); 140 } 141 142 static void renesas_sdhi_sys_dmac_abort_dma(struct tmio_mmc_host *host) 143 { 144 renesas_sdhi_sys_dmac_enable_dma(host, false); 145 146 if (host->chan_rx) 147 dmaengine_terminate_all(host->chan_rx); 148 if (host->chan_tx) 149 dmaengine_terminate_all(host->chan_tx); 150 151 renesas_sdhi_sys_dmac_enable_dma(host, true); 152 } 153 154 static void renesas_sdhi_sys_dmac_dataend_dma(struct tmio_mmc_host *host) 155 { 156 struct renesas_sdhi *priv = host_to_priv(host); 157 158 complete(&priv->dma_priv.dma_dataend); 159 } 160 161 static void renesas_sdhi_sys_dmac_dma_callback(void *arg) 162 { 163 struct tmio_mmc_host *host = arg; 164 struct renesas_sdhi *priv = host_to_priv(host); 165 166 spin_lock_irq(&host->lock); 167 168 if (!host->data) 169 goto out; 170 171 if (host->data->flags & MMC_DATA_READ) 172 dma_unmap_sg(host->chan_rx->device->dev, 173 host->sg_ptr, host->sg_len, 174 DMA_FROM_DEVICE); 175 else 176 dma_unmap_sg(host->chan_tx->device->dev, 177 host->sg_ptr, host->sg_len, 178 DMA_TO_DEVICE); 179 180 spin_unlock_irq(&host->lock); 181 182 wait_for_completion(&priv->dma_priv.dma_dataend); 183 184 spin_lock_irq(&host->lock); 185 tmio_mmc_do_data_irq(host); 186 out: 187 spin_unlock_irq(&host->lock); 188 } 189 190 static void renesas_sdhi_sys_dmac_start_dma_rx(struct tmio_mmc_host *host) 191 { 192 struct renesas_sdhi *priv = host_to_priv(host); 193 struct scatterlist *sg = host->sg_ptr, *sg_tmp; 194 struct dma_async_tx_descriptor *desc = NULL; 195 struct dma_chan *chan = host->chan_rx; 196 dma_cookie_t cookie; 197 int ret, i; 198 bool aligned = true, multiple = true; 199 unsigned int align = (1 << host->pdata->alignment_shift) - 1; 200 201 for_each_sg(sg, sg_tmp, host->sg_len, i) { 202 if (sg_tmp->offset & align) 203 aligned = false; 204 if (sg_tmp->length & align) { 205 multiple = false; 206 break; 207 } 208 } 209 210 if ((!aligned && (host->sg_len > 1 || sg->length > PAGE_SIZE || 211 (align & PAGE_MASK))) || !multiple) { 212 ret = -EINVAL; 213 goto pio; 214 } 215 216 if (sg->length < TMIO_MMC_MIN_DMA_LEN) { 217 host->force_pio = true; 218 return; 219 } 220 221 /* The only sg element can be unaligned, use our bounce buffer then */ 222 if (!aligned) { 223 sg_init_one(&host->bounce_sg, host->bounce_buf, sg->length); 224 host->sg_ptr = &host->bounce_sg; 225 sg = host->sg_ptr; 226 } 227 228 ret = dma_map_sg(chan->device->dev, sg, host->sg_len, DMA_FROM_DEVICE); 229 if (ret > 0) 230 desc = dmaengine_prep_slave_sg(chan, sg, ret, DMA_DEV_TO_MEM, 231 DMA_CTRL_ACK); 232 233 if (desc) { 234 reinit_completion(&priv->dma_priv.dma_dataend); 235 desc->callback = renesas_sdhi_sys_dmac_dma_callback; 236 desc->callback_param = host; 237 238 cookie = dmaengine_submit(desc); 239 if (cookie < 0) { 240 desc = NULL; 241 ret = cookie; 242 } 243 } 244 pio: 245 if (!desc) { 246 /* DMA failed, fall back to PIO */ 247 renesas_sdhi_sys_dmac_enable_dma(host, false); 248 if (ret >= 0) 249 ret = -EIO; 250 host->chan_rx = NULL; 251 dma_release_channel(chan); 252 /* Free the Tx channel too */ 253 chan = host->chan_tx; 254 if (chan) { 255 host->chan_tx = NULL; 256 dma_release_channel(chan); 257 } 258 dev_warn(&host->pdev->dev, 259 "DMA failed: %d, falling back to PIO\n", ret); 260 } 261 } 262 263 static void renesas_sdhi_sys_dmac_start_dma_tx(struct tmio_mmc_host *host) 264 { 265 struct renesas_sdhi *priv = host_to_priv(host); 266 struct scatterlist *sg = host->sg_ptr, *sg_tmp; 267 struct dma_async_tx_descriptor *desc = NULL; 268 struct dma_chan *chan = host->chan_tx; 269 dma_cookie_t cookie; 270 int ret, i; 271 bool aligned = true, multiple = true; 272 unsigned int align = (1 << host->pdata->alignment_shift) - 1; 273 274 for_each_sg(sg, sg_tmp, host->sg_len, i) { 275 if (sg_tmp->offset & align) 276 aligned = false; 277 if (sg_tmp->length & align) { 278 multiple = false; 279 break; 280 } 281 } 282 283 if ((!aligned && (host->sg_len > 1 || sg->length > PAGE_SIZE || 284 (align & PAGE_MASK))) || !multiple) { 285 ret = -EINVAL; 286 goto pio; 287 } 288 289 if (sg->length < TMIO_MMC_MIN_DMA_LEN) { 290 host->force_pio = true; 291 return; 292 } 293 294 /* The only sg element can be unaligned, use our bounce buffer then */ 295 if (!aligned) { 296 unsigned long flags; 297 void *sg_vaddr = tmio_mmc_kmap_atomic(sg, &flags); 298 299 sg_init_one(&host->bounce_sg, host->bounce_buf, sg->length); 300 memcpy(host->bounce_buf, sg_vaddr, host->bounce_sg.length); 301 tmio_mmc_kunmap_atomic(sg, &flags, sg_vaddr); 302 host->sg_ptr = &host->bounce_sg; 303 sg = host->sg_ptr; 304 } 305 306 ret = dma_map_sg(chan->device->dev, sg, host->sg_len, DMA_TO_DEVICE); 307 if (ret > 0) 308 desc = dmaengine_prep_slave_sg(chan, sg, ret, DMA_MEM_TO_DEV, 309 DMA_CTRL_ACK); 310 311 if (desc) { 312 reinit_completion(&priv->dma_priv.dma_dataend); 313 desc->callback = renesas_sdhi_sys_dmac_dma_callback; 314 desc->callback_param = host; 315 316 cookie = dmaengine_submit(desc); 317 if (cookie < 0) { 318 desc = NULL; 319 ret = cookie; 320 } 321 } 322 pio: 323 if (!desc) { 324 /* DMA failed, fall back to PIO */ 325 renesas_sdhi_sys_dmac_enable_dma(host, false); 326 if (ret >= 0) 327 ret = -EIO; 328 host->chan_tx = NULL; 329 dma_release_channel(chan); 330 /* Free the Rx channel too */ 331 chan = host->chan_rx; 332 if (chan) { 333 host->chan_rx = NULL; 334 dma_release_channel(chan); 335 } 336 dev_warn(&host->pdev->dev, 337 "DMA failed: %d, falling back to PIO\n", ret); 338 } 339 } 340 341 static void renesas_sdhi_sys_dmac_start_dma(struct tmio_mmc_host *host, 342 struct mmc_data *data) 343 { 344 if (data->flags & MMC_DATA_READ) { 345 if (host->chan_rx) 346 renesas_sdhi_sys_dmac_start_dma_rx(host); 347 } else { 348 if (host->chan_tx) 349 renesas_sdhi_sys_dmac_start_dma_tx(host); 350 } 351 } 352 353 static void renesas_sdhi_sys_dmac_issue_tasklet_fn(unsigned long priv) 354 { 355 struct tmio_mmc_host *host = (struct tmio_mmc_host *)priv; 356 struct dma_chan *chan = NULL; 357 358 spin_lock_irq(&host->lock); 359 360 if (host->data) { 361 if (host->data->flags & MMC_DATA_READ) 362 chan = host->chan_rx; 363 else 364 chan = host->chan_tx; 365 } 366 367 spin_unlock_irq(&host->lock); 368 369 tmio_mmc_enable_mmc_irqs(host, TMIO_STAT_DATAEND); 370 371 if (chan) 372 dma_async_issue_pending(chan); 373 } 374 375 static void renesas_sdhi_sys_dmac_request_dma(struct tmio_mmc_host *host, 376 struct tmio_mmc_data *pdata) 377 { 378 struct renesas_sdhi *priv = host_to_priv(host); 379 380 /* We can only either use DMA for both Tx and Rx or not use it at all */ 381 if (!host->pdev->dev.of_node && 382 (!pdata->chan_priv_tx || !pdata->chan_priv_rx)) 383 return; 384 385 if (!host->chan_tx && !host->chan_rx) { 386 struct resource *res = platform_get_resource(host->pdev, 387 IORESOURCE_MEM, 0); 388 struct dma_slave_config cfg = {}; 389 dma_cap_mask_t mask; 390 int ret; 391 392 if (!res) 393 return; 394 395 dma_cap_zero(mask); 396 dma_cap_set(DMA_SLAVE, mask); 397 398 host->chan_tx = dma_request_slave_channel_compat(mask, 399 priv->dma_priv.filter, pdata->chan_priv_tx, 400 &host->pdev->dev, "tx"); 401 dev_dbg(&host->pdev->dev, "%s: TX: got channel %p\n", __func__, 402 host->chan_tx); 403 404 if (!host->chan_tx) 405 return; 406 407 cfg.direction = DMA_MEM_TO_DEV; 408 cfg.dst_addr = res->start + 409 (CTL_SD_DATA_PORT << host->bus_shift); 410 cfg.dst_addr_width = priv->dma_priv.dma_buswidth; 411 if (!cfg.dst_addr_width) 412 cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES; 413 cfg.src_addr = 0; 414 ret = dmaengine_slave_config(host->chan_tx, &cfg); 415 if (ret < 0) 416 goto ecfgtx; 417 418 host->chan_rx = dma_request_slave_channel_compat(mask, 419 priv->dma_priv.filter, pdata->chan_priv_rx, 420 &host->pdev->dev, "rx"); 421 dev_dbg(&host->pdev->dev, "%s: RX: got channel %p\n", __func__, 422 host->chan_rx); 423 424 if (!host->chan_rx) 425 goto ereqrx; 426 427 cfg.direction = DMA_DEV_TO_MEM; 428 cfg.src_addr = cfg.dst_addr + host->pdata->dma_rx_offset; 429 cfg.src_addr_width = priv->dma_priv.dma_buswidth; 430 if (!cfg.src_addr_width) 431 cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES; 432 cfg.dst_addr = 0; 433 ret = dmaengine_slave_config(host->chan_rx, &cfg); 434 if (ret < 0) 435 goto ecfgrx; 436 437 host->bounce_buf = (u8 *)__get_free_page(GFP_KERNEL | GFP_DMA); 438 if (!host->bounce_buf) 439 goto ebouncebuf; 440 441 init_completion(&priv->dma_priv.dma_dataend); 442 tasklet_init(&host->dma_issue, 443 renesas_sdhi_sys_dmac_issue_tasklet_fn, 444 (unsigned long)host); 445 } 446 447 renesas_sdhi_sys_dmac_enable_dma(host, true); 448 449 return; 450 451 ebouncebuf: 452 ecfgrx: 453 dma_release_channel(host->chan_rx); 454 host->chan_rx = NULL; 455 ereqrx: 456 ecfgtx: 457 dma_release_channel(host->chan_tx); 458 host->chan_tx = NULL; 459 } 460 461 static void renesas_sdhi_sys_dmac_release_dma(struct tmio_mmc_host *host) 462 { 463 if (host->chan_tx) { 464 struct dma_chan *chan = host->chan_tx; 465 466 host->chan_tx = NULL; 467 dma_release_channel(chan); 468 } 469 if (host->chan_rx) { 470 struct dma_chan *chan = host->chan_rx; 471 472 host->chan_rx = NULL; 473 dma_release_channel(chan); 474 } 475 if (host->bounce_buf) { 476 free_pages((unsigned long)host->bounce_buf, 0); 477 host->bounce_buf = NULL; 478 } 479 } 480 481 static const struct tmio_mmc_dma_ops renesas_sdhi_sys_dmac_dma_ops = { 482 .start = renesas_sdhi_sys_dmac_start_dma, 483 .enable = renesas_sdhi_sys_dmac_enable_dma, 484 .request = renesas_sdhi_sys_dmac_request_dma, 485 .release = renesas_sdhi_sys_dmac_release_dma, 486 .abort = renesas_sdhi_sys_dmac_abort_dma, 487 .dataend = renesas_sdhi_sys_dmac_dataend_dma, 488 }; 489 490 /* 491 * Whitelist of specific R-Car Gen3 SoC ES versions to use this DMAC 492 * implementation. Currently empty as all supported ES versions use 493 * the internal DMAC. 494 */ 495 static const struct soc_device_attribute gen3_soc_whitelist[] = { 496 { /* sentinel */ } 497 }; 498 499 static int renesas_sdhi_sys_dmac_probe(struct platform_device *pdev) 500 { 501 if ((of_device_get_match_data(&pdev->dev) == &of_rcar_gen3_compatible || 502 of_device_get_match_data(&pdev->dev) == &of_rcar_r8a7795_compatible) && 503 !soc_device_match(gen3_soc_whitelist)) 504 return -ENODEV; 505 506 return renesas_sdhi_probe(pdev, &renesas_sdhi_sys_dmac_dma_ops); 507 } 508 509 static const struct dev_pm_ops renesas_sdhi_sys_dmac_dev_pm_ops = { 510 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, 511 pm_runtime_force_resume) 512 SET_RUNTIME_PM_OPS(tmio_mmc_host_runtime_suspend, 513 tmio_mmc_host_runtime_resume, 514 NULL) 515 }; 516 517 static struct platform_driver renesas_sys_dmac_sdhi_driver = { 518 .driver = { 519 .name = "sh_mobile_sdhi", 520 .pm = &renesas_sdhi_sys_dmac_dev_pm_ops, 521 .of_match_table = renesas_sdhi_sys_dmac_of_match, 522 }, 523 .probe = renesas_sdhi_sys_dmac_probe, 524 .remove = renesas_sdhi_remove, 525 }; 526 527 module_platform_driver(renesas_sys_dmac_sdhi_driver); 528 529 MODULE_DESCRIPTION("Renesas SDHI driver"); 530 MODULE_AUTHOR("Magnus Damm"); 531 MODULE_LICENSE("GPL v2"); 532 MODULE_ALIAS("platform:sh_mobile_sdhi"); 533