1 #include <linux/device.h> 2 #include <linux/dma-mapping.h> 3 #include <linux/dmaengine.h> 4 #include <linux/sizes.h> 5 #include <linux/platform_device.h> 6 #include <linux/of.h> 7 8 #include "musb_core.h" 9 10 #define RNDIS_REG(x) (0x80 + ((x - 1) * 4)) 11 12 #define EP_MODE_AUTOREG_NONE 0 13 #define EP_MODE_AUTOREG_ALL_NEOP 1 14 #define EP_MODE_AUTOREG_ALWAYS 3 15 16 #define EP_MODE_DMA_TRANSPARENT 0 17 #define EP_MODE_DMA_RNDIS 1 18 #define EP_MODE_DMA_GEN_RNDIS 3 19 20 #define USB_CTRL_TX_MODE 0x70 21 #define USB_CTRL_RX_MODE 0x74 22 #define USB_CTRL_AUTOREQ 0xd0 23 #define USB_TDOWN 0xd8 24 25 struct cppi41_dma_channel { 26 struct dma_channel channel; 27 struct cppi41_dma_controller *controller; 28 struct musb_hw_ep *hw_ep; 29 struct dma_chan *dc; 30 dma_cookie_t cookie; 31 u8 port_num; 32 u8 is_tx; 33 u8 is_allocated; 34 u8 usb_toggle; 35 36 dma_addr_t buf_addr; 37 u32 total_len; 38 u32 prog_len; 39 u32 transferred; 40 u32 packet_sz; 41 }; 42 43 #define MUSB_DMA_NUM_CHANNELS 15 44 45 struct cppi41_dma_controller { 46 struct dma_controller controller; 47 struct cppi41_dma_channel rx_channel[MUSB_DMA_NUM_CHANNELS]; 48 struct cppi41_dma_channel tx_channel[MUSB_DMA_NUM_CHANNELS]; 49 struct musb *musb; 50 u32 rx_mode; 51 u32 tx_mode; 52 u32 auto_req; 53 }; 54 55 static void save_rx_toggle(struct cppi41_dma_channel *cppi41_channel) 56 { 57 u16 csr; 58 u8 toggle; 59 60 if (cppi41_channel->is_tx) 61 return; 62 if (!is_host_active(cppi41_channel->controller->musb)) 63 return; 64 65 csr = musb_readw(cppi41_channel->hw_ep->regs, MUSB_RXCSR); 66 toggle = csr & MUSB_RXCSR_H_DATATOGGLE ? 1 : 0; 67 68 cppi41_channel->usb_toggle = toggle; 69 } 70 71 static void update_rx_toggle(struct cppi41_dma_channel *cppi41_channel) 72 { 73 u16 csr; 74 u8 toggle; 75 76 if (cppi41_channel->is_tx) 77 return; 78 if (!is_host_active(cppi41_channel->controller->musb)) 79 return; 80 81 csr = musb_readw(cppi41_channel->hw_ep->regs, MUSB_RXCSR); 82 toggle = csr & MUSB_RXCSR_H_DATATOGGLE ? 1 : 0; 83 84 /* 85 * AM335x Advisory 1.0.13: Due to internal synchronisation error the 86 * data toggle may reset from DATA1 to DATA0 during receiving data from 87 * more than one endpoint. 88 */ 89 if (!toggle && toggle == cppi41_channel->usb_toggle) { 90 csr |= MUSB_RXCSR_H_DATATOGGLE | MUSB_RXCSR_H_WR_DATATOGGLE; 91 musb_writew(cppi41_channel->hw_ep->regs, MUSB_RXCSR, csr); 92 dev_dbg(cppi41_channel->controller->musb->controller, 93 "Restoring DATA1 toggle.\n"); 94 } 95 96 cppi41_channel->usb_toggle = toggle; 97 } 98 99 static void cppi41_dma_callback(void *private_data) 100 { 101 struct dma_channel *channel = private_data; 102 struct cppi41_dma_channel *cppi41_channel = channel->private_data; 103 struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep; 104 struct musb *musb = hw_ep->musb; 105 unsigned long flags; 106 struct dma_tx_state txstate; 107 u32 transferred; 108 109 spin_lock_irqsave(&musb->lock, flags); 110 111 dmaengine_tx_status(cppi41_channel->dc, cppi41_channel->cookie, 112 &txstate); 113 transferred = cppi41_channel->prog_len - txstate.residue; 114 cppi41_channel->transferred += transferred; 115 116 dev_dbg(musb->controller, "DMA transfer done on hw_ep=%d bytes=%d/%d\n", 117 hw_ep->epnum, cppi41_channel->transferred, 118 cppi41_channel->total_len); 119 120 update_rx_toggle(cppi41_channel); 121 122 if (cppi41_channel->transferred == cppi41_channel->total_len || 123 transferred < cppi41_channel->packet_sz) { 124 125 /* done, complete */ 126 cppi41_channel->channel.actual_len = 127 cppi41_channel->transferred; 128 cppi41_channel->channel.status = MUSB_DMA_STATUS_FREE; 129 musb_dma_completion(musb, hw_ep->epnum, cppi41_channel->is_tx); 130 } else { 131 /* next iteration, reload */ 132 struct dma_chan *dc = cppi41_channel->dc; 133 struct dma_async_tx_descriptor *dma_desc; 134 enum dma_transfer_direction direction; 135 u16 csr; 136 u32 remain_bytes; 137 void __iomem *epio = cppi41_channel->hw_ep->regs; 138 139 cppi41_channel->buf_addr += cppi41_channel->packet_sz; 140 141 remain_bytes = cppi41_channel->total_len; 142 remain_bytes -= cppi41_channel->transferred; 143 remain_bytes = min(remain_bytes, cppi41_channel->packet_sz); 144 cppi41_channel->prog_len = remain_bytes; 145 146 direction = cppi41_channel->is_tx ? DMA_MEM_TO_DEV 147 : DMA_DEV_TO_MEM; 148 dma_desc = dmaengine_prep_slave_single(dc, 149 cppi41_channel->buf_addr, 150 remain_bytes, 151 direction, 152 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 153 if (WARN_ON(!dma_desc)) { 154 spin_unlock_irqrestore(&musb->lock, flags); 155 return; 156 } 157 158 dma_desc->callback = cppi41_dma_callback; 159 dma_desc->callback_param = channel; 160 cppi41_channel->cookie = dma_desc->tx_submit(dma_desc); 161 dma_async_issue_pending(dc); 162 163 if (!cppi41_channel->is_tx) { 164 csr = musb_readw(epio, MUSB_RXCSR); 165 csr |= MUSB_RXCSR_H_REQPKT; 166 musb_writew(epio, MUSB_RXCSR, csr); 167 } 168 } 169 spin_unlock_irqrestore(&musb->lock, flags); 170 } 171 172 static u32 update_ep_mode(unsigned ep, unsigned mode, u32 old) 173 { 174 unsigned shift; 175 176 shift = (ep - 1) * 2; 177 old &= ~(3 << shift); 178 old |= mode << shift; 179 return old; 180 } 181 182 static void cppi41_set_dma_mode(struct cppi41_dma_channel *cppi41_channel, 183 unsigned mode) 184 { 185 struct cppi41_dma_controller *controller = cppi41_channel->controller; 186 u32 port; 187 u32 new_mode; 188 u32 old_mode; 189 190 if (cppi41_channel->is_tx) 191 old_mode = controller->tx_mode; 192 else 193 old_mode = controller->rx_mode; 194 port = cppi41_channel->port_num; 195 new_mode = update_ep_mode(port, mode, old_mode); 196 197 if (new_mode == old_mode) 198 return; 199 if (cppi41_channel->is_tx) { 200 controller->tx_mode = new_mode; 201 musb_writel(controller->musb->ctrl_base, USB_CTRL_TX_MODE, 202 new_mode); 203 } else { 204 controller->rx_mode = new_mode; 205 musb_writel(controller->musb->ctrl_base, USB_CTRL_RX_MODE, 206 new_mode); 207 } 208 } 209 210 static void cppi41_set_autoreq_mode(struct cppi41_dma_channel *cppi41_channel, 211 unsigned mode) 212 { 213 struct cppi41_dma_controller *controller = cppi41_channel->controller; 214 u32 port; 215 u32 new_mode; 216 u32 old_mode; 217 218 old_mode = controller->auto_req; 219 port = cppi41_channel->port_num; 220 new_mode = update_ep_mode(port, mode, old_mode); 221 222 if (new_mode == old_mode) 223 return; 224 controller->auto_req = new_mode; 225 musb_writel(controller->musb->ctrl_base, USB_CTRL_AUTOREQ, new_mode); 226 } 227 228 static bool cppi41_configure_channel(struct dma_channel *channel, 229 u16 packet_sz, u8 mode, 230 dma_addr_t dma_addr, u32 len) 231 { 232 struct cppi41_dma_channel *cppi41_channel = channel->private_data; 233 struct dma_chan *dc = cppi41_channel->dc; 234 struct dma_async_tx_descriptor *dma_desc; 235 enum dma_transfer_direction direction; 236 struct musb *musb = cppi41_channel->controller->musb; 237 unsigned use_gen_rndis = 0; 238 239 dev_dbg(musb->controller, 240 "configure ep%d/%x packet_sz=%d, mode=%d, dma_addr=0x%llx, len=%d is_tx=%d\n", 241 cppi41_channel->port_num, RNDIS_REG(cppi41_channel->port_num), 242 packet_sz, mode, (unsigned long long) dma_addr, 243 len, cppi41_channel->is_tx); 244 245 cppi41_channel->buf_addr = dma_addr; 246 cppi41_channel->total_len = len; 247 cppi41_channel->transferred = 0; 248 cppi41_channel->packet_sz = packet_sz; 249 250 /* 251 * Due to AM335x' Advisory 1.0.13 we are not allowed to transfer more 252 * than max packet size at a time. 253 */ 254 if (cppi41_channel->is_tx) 255 use_gen_rndis = 1; 256 257 if (use_gen_rndis) { 258 /* RNDIS mode */ 259 if (len > packet_sz) { 260 musb_writel(musb->ctrl_base, 261 RNDIS_REG(cppi41_channel->port_num), len); 262 /* gen rndis */ 263 cppi41_set_dma_mode(cppi41_channel, 264 EP_MODE_DMA_GEN_RNDIS); 265 266 /* auto req */ 267 cppi41_set_autoreq_mode(cppi41_channel, 268 EP_MODE_AUTOREG_ALL_NEOP); 269 } else { 270 musb_writel(musb->ctrl_base, 271 RNDIS_REG(cppi41_channel->port_num), 0); 272 cppi41_set_dma_mode(cppi41_channel, 273 EP_MODE_DMA_TRANSPARENT); 274 cppi41_set_autoreq_mode(cppi41_channel, 275 EP_MODE_AUTOREG_NONE); 276 } 277 } else { 278 /* fallback mode */ 279 cppi41_set_dma_mode(cppi41_channel, EP_MODE_DMA_TRANSPARENT); 280 cppi41_set_autoreq_mode(cppi41_channel, EP_MODE_AUTOREG_NONE); 281 len = min_t(u32, packet_sz, len); 282 } 283 cppi41_channel->prog_len = len; 284 direction = cppi41_channel->is_tx ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM; 285 dma_desc = dmaengine_prep_slave_single(dc, dma_addr, len, direction, 286 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 287 if (!dma_desc) 288 return false; 289 290 dma_desc->callback = cppi41_dma_callback; 291 dma_desc->callback_param = channel; 292 cppi41_channel->cookie = dma_desc->tx_submit(dma_desc); 293 294 save_rx_toggle(cppi41_channel); 295 dma_async_issue_pending(dc); 296 return true; 297 } 298 299 static struct dma_channel *cppi41_dma_channel_allocate(struct dma_controller *c, 300 struct musb_hw_ep *hw_ep, u8 is_tx) 301 { 302 struct cppi41_dma_controller *controller = container_of(c, 303 struct cppi41_dma_controller, controller); 304 struct cppi41_dma_channel *cppi41_channel = NULL; 305 u8 ch_num = hw_ep->epnum - 1; 306 307 if (ch_num >= MUSB_DMA_NUM_CHANNELS) 308 return NULL; 309 310 if (is_tx) 311 cppi41_channel = &controller->tx_channel[ch_num]; 312 else 313 cppi41_channel = &controller->rx_channel[ch_num]; 314 315 if (!cppi41_channel->dc) 316 return NULL; 317 318 if (cppi41_channel->is_allocated) 319 return NULL; 320 321 cppi41_channel->hw_ep = hw_ep; 322 cppi41_channel->is_allocated = 1; 323 324 return &cppi41_channel->channel; 325 } 326 327 static void cppi41_dma_channel_release(struct dma_channel *channel) 328 { 329 struct cppi41_dma_channel *cppi41_channel = channel->private_data; 330 331 if (cppi41_channel->is_allocated) { 332 cppi41_channel->is_allocated = 0; 333 channel->status = MUSB_DMA_STATUS_FREE; 334 channel->actual_len = 0; 335 } 336 } 337 338 static int cppi41_dma_channel_program(struct dma_channel *channel, 339 u16 packet_sz, u8 mode, 340 dma_addr_t dma_addr, u32 len) 341 { 342 int ret; 343 344 BUG_ON(channel->status == MUSB_DMA_STATUS_UNKNOWN || 345 channel->status == MUSB_DMA_STATUS_BUSY); 346 347 channel->status = MUSB_DMA_STATUS_BUSY; 348 channel->actual_len = 0; 349 ret = cppi41_configure_channel(channel, packet_sz, mode, dma_addr, len); 350 if (!ret) 351 channel->status = MUSB_DMA_STATUS_FREE; 352 353 return ret; 354 } 355 356 static int cppi41_is_compatible(struct dma_channel *channel, u16 maxpacket, 357 void *buf, u32 length) 358 { 359 struct cppi41_dma_channel *cppi41_channel = channel->private_data; 360 struct cppi41_dma_controller *controller = cppi41_channel->controller; 361 struct musb *musb = controller->musb; 362 363 if (is_host_active(musb)) { 364 WARN_ON(1); 365 return 1; 366 } 367 if (cppi41_channel->is_tx) 368 return 1; 369 /* AM335x Advisory 1.0.13. No workaround for device RX mode */ 370 return 0; 371 } 372 373 static int cppi41_dma_channel_abort(struct dma_channel *channel) 374 { 375 struct cppi41_dma_channel *cppi41_channel = channel->private_data; 376 struct cppi41_dma_controller *controller = cppi41_channel->controller; 377 struct musb *musb = controller->musb; 378 void __iomem *epio = cppi41_channel->hw_ep->regs; 379 int tdbit; 380 int ret; 381 unsigned is_tx; 382 u16 csr; 383 384 is_tx = cppi41_channel->is_tx; 385 dev_dbg(musb->controller, "abort channel=%d, is_tx=%d\n", 386 cppi41_channel->port_num, is_tx); 387 388 if (cppi41_channel->channel.status == MUSB_DMA_STATUS_FREE) 389 return 0; 390 391 if (is_tx) { 392 csr = musb_readw(epio, MUSB_TXCSR); 393 csr &= ~MUSB_TXCSR_DMAENAB; 394 musb_writew(epio, MUSB_TXCSR, csr); 395 } else { 396 csr = musb_readw(epio, MUSB_RXCSR); 397 csr &= ~(MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_DMAENAB); 398 musb_writew(epio, MUSB_RXCSR, csr); 399 400 csr = musb_readw(epio, MUSB_RXCSR); 401 if (csr & MUSB_RXCSR_RXPKTRDY) { 402 csr |= MUSB_RXCSR_FLUSHFIFO; 403 musb_writew(epio, MUSB_RXCSR, csr); 404 musb_writew(epio, MUSB_RXCSR, csr); 405 } 406 } 407 408 tdbit = 1 << cppi41_channel->port_num; 409 if (is_tx) 410 tdbit <<= 16; 411 412 do { 413 musb_writel(musb->ctrl_base, USB_TDOWN, tdbit); 414 ret = dmaengine_terminate_all(cppi41_channel->dc); 415 } while (ret == -EAGAIN); 416 417 musb_writel(musb->ctrl_base, USB_TDOWN, tdbit); 418 419 if (is_tx) { 420 csr = musb_readw(epio, MUSB_TXCSR); 421 if (csr & MUSB_TXCSR_TXPKTRDY) { 422 csr |= MUSB_TXCSR_FLUSHFIFO; 423 musb_writew(epio, MUSB_TXCSR, csr); 424 } 425 } 426 427 cppi41_channel->channel.status = MUSB_DMA_STATUS_FREE; 428 return 0; 429 } 430 431 static void cppi41_release_all_dma_chans(struct cppi41_dma_controller *ctrl) 432 { 433 struct dma_chan *dc; 434 int i; 435 436 for (i = 0; i < MUSB_DMA_NUM_CHANNELS; i++) { 437 dc = ctrl->tx_channel[i].dc; 438 if (dc) 439 dma_release_channel(dc); 440 dc = ctrl->rx_channel[i].dc; 441 if (dc) 442 dma_release_channel(dc); 443 } 444 } 445 446 static void cppi41_dma_controller_stop(struct cppi41_dma_controller *controller) 447 { 448 cppi41_release_all_dma_chans(controller); 449 } 450 451 static int cppi41_dma_controller_start(struct cppi41_dma_controller *controller) 452 { 453 struct musb *musb = controller->musb; 454 struct device *dev = musb->controller; 455 struct device_node *np = dev->of_node; 456 struct cppi41_dma_channel *cppi41_channel; 457 int count; 458 int i; 459 int ret; 460 461 count = of_property_count_strings(np, "dma-names"); 462 if (count < 0) 463 return count; 464 465 for (i = 0; i < count; i++) { 466 struct dma_chan *dc; 467 struct dma_channel *musb_dma; 468 const char *str; 469 unsigned is_tx; 470 unsigned int port; 471 472 ret = of_property_read_string_index(np, "dma-names", i, &str); 473 if (ret) 474 goto err; 475 if (!strncmp(str, "tx", 2)) 476 is_tx = 1; 477 else if (!strncmp(str, "rx", 2)) 478 is_tx = 0; 479 else { 480 dev_err(dev, "Wrong dmatype %s\n", str); 481 goto err; 482 } 483 ret = kstrtouint(str + 2, 0, &port); 484 if (ret) 485 goto err; 486 487 if (port > MUSB_DMA_NUM_CHANNELS || !port) 488 goto err; 489 if (is_tx) 490 cppi41_channel = &controller->tx_channel[port - 1]; 491 else 492 cppi41_channel = &controller->rx_channel[port - 1]; 493 494 cppi41_channel->controller = controller; 495 cppi41_channel->port_num = port; 496 cppi41_channel->is_tx = is_tx; 497 498 musb_dma = &cppi41_channel->channel; 499 musb_dma->private_data = cppi41_channel; 500 musb_dma->status = MUSB_DMA_STATUS_FREE; 501 musb_dma->max_len = SZ_4M; 502 503 dc = dma_request_slave_channel(dev, str); 504 if (!dc) { 505 dev_err(dev, "Falied to request %s.\n", str); 506 goto err; 507 } 508 cppi41_channel->dc = dc; 509 } 510 return 0; 511 err: 512 cppi41_release_all_dma_chans(controller); 513 return -EINVAL; 514 } 515 516 void dma_controller_destroy(struct dma_controller *c) 517 { 518 struct cppi41_dma_controller *controller = container_of(c, 519 struct cppi41_dma_controller, controller); 520 521 cppi41_dma_controller_stop(controller); 522 kfree(controller); 523 } 524 525 struct dma_controller *dma_controller_create(struct musb *musb, 526 void __iomem *base) 527 { 528 struct cppi41_dma_controller *controller; 529 int ret; 530 531 if (!musb->controller->of_node) { 532 dev_err(musb->controller, "Need DT for the DMA engine.\n"); 533 return NULL; 534 } 535 536 controller = kzalloc(sizeof(*controller), GFP_KERNEL); 537 if (!controller) 538 goto kzalloc_fail; 539 540 controller->musb = musb; 541 542 controller->controller.channel_alloc = cppi41_dma_channel_allocate; 543 controller->controller.channel_release = cppi41_dma_channel_release; 544 controller->controller.channel_program = cppi41_dma_channel_program; 545 controller->controller.channel_abort = cppi41_dma_channel_abort; 546 controller->controller.is_compatible = cppi41_is_compatible; 547 548 ret = cppi41_dma_controller_start(controller); 549 if (ret) 550 goto plat_get_fail; 551 return &controller->controller; 552 553 plat_get_fail: 554 kfree(controller); 555 kzalloc_fail: 556 return NULL; 557 } 558