1 /* 2 * Renesas USB driver 3 * 4 * Copyright (C) 2011 Renesas Solutions Corp. 5 * Kuninori Morimoto <kuninori.morimoto.gx@renesas.com> 6 * 7 * This program is distributed in the hope that it will be useful, 8 * but WITHOUT ANY WARRANTY; without even the implied warranty of 9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 * GNU General Public License for more details. 11 * 12 * You should have received a copy of the GNU General Public License 13 * along with this program; if not, write to the Free Software 14 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 15 * 16 */ 17 #include <linux/delay.h> 18 #include <linux/io.h> 19 #include <linux/scatterlist.h> 20 #include "common.h" 21 #include "pipe.h" 22 23 #define usbhsf_get_cfifo(p) (&((p)->fifo_info.cfifo)) 24 #define usbhsf_is_cfifo(p, f) (usbhsf_get_cfifo(p) == f) 25 26 #define usbhsf_fifo_is_busy(f) ((f)->pipe) /* see usbhs_pipe_select_fifo */ 27 28 /* 29 * packet initialize 30 */ 31 void usbhs_pkt_init(struct usbhs_pkt *pkt) 32 { 33 INIT_LIST_HEAD(&pkt->node); 34 } 35 36 /* 37 * packet control function 38 */ 39 static int usbhsf_null_handle(struct usbhs_pkt *pkt, int *is_done) 40 { 41 struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe); 42 struct device *dev = usbhs_priv_to_dev(priv); 43 44 dev_err(dev, "null handler\n"); 45 46 return -EINVAL; 47 } 48 49 static struct usbhs_pkt_handle usbhsf_null_handler = { 50 .prepare = usbhsf_null_handle, 51 .try_run = usbhsf_null_handle, 52 }; 53 54 void usbhs_pkt_push(struct usbhs_pipe *pipe, struct usbhs_pkt *pkt, 55 void (*done)(struct usbhs_priv *priv, 56 struct usbhs_pkt *pkt), 57 void *buf, int len, int zero, int sequence) 58 { 59 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe); 60 struct device *dev = usbhs_priv_to_dev(priv); 61 unsigned long flags; 62 63 if (!done) { 64 dev_err(dev, "no done function\n"); 65 return; 66 } 67 68 /******************** spin lock ********************/ 69 usbhs_lock(priv, flags); 70 71 if (!pipe->handler) { 72 dev_err(dev, "no handler function\n"); 73 pipe->handler = &usbhsf_null_handler; 74 } 75 76 list_move_tail(&pkt->node, &pipe->list); 77 78 /* 79 * each pkt must hold own handler. 80 * because handler might be changed by its situation. 81 * dma handler -> pio handler. 82 */ 83 pkt->pipe = pipe; 84 pkt->buf = buf; 85 pkt->handler = pipe->handler; 86 pkt->length = len; 87 pkt->zero = zero; 88 pkt->actual = 0; 89 pkt->done = done; 90 pkt->sequence = sequence; 91 92 usbhs_unlock(priv, flags); 93 /******************** spin unlock ******************/ 94 } 95 96 static void __usbhsf_pkt_del(struct usbhs_pkt *pkt) 97 { 98 list_del_init(&pkt->node); 99 } 100 101 static struct usbhs_pkt *__usbhsf_pkt_get(struct usbhs_pipe *pipe) 102 { 103 if (list_empty(&pipe->list)) 104 return NULL; 105 106 return list_first_entry(&pipe->list, struct usbhs_pkt, node); 107 } 108 109 static void usbhsf_fifo_clear(struct usbhs_pipe *pipe, 110 struct usbhs_fifo *fifo); 111 static void usbhsf_fifo_unselect(struct usbhs_pipe *pipe, 112 struct usbhs_fifo *fifo); 113 static struct dma_chan *usbhsf_dma_chan_get(struct usbhs_fifo *fifo, 114 struct usbhs_pkt *pkt); 115 #define usbhsf_dma_map(p) __usbhsf_dma_map_ctrl(p, 1) 116 #define usbhsf_dma_unmap(p) __usbhsf_dma_map_ctrl(p, 0) 117 static int __usbhsf_dma_map_ctrl(struct usbhs_pkt *pkt, int map); 118 struct usbhs_pkt *usbhs_pkt_pop(struct usbhs_pipe *pipe, struct usbhs_pkt *pkt) 119 { 120 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe); 121 struct usbhs_fifo *fifo = usbhs_pipe_to_fifo(pipe); 122 unsigned long flags; 123 124 /******************** spin lock ********************/ 125 usbhs_lock(priv, flags); 126 127 usbhs_pipe_disable(pipe); 128 129 if (!pkt) 130 pkt = __usbhsf_pkt_get(pipe); 131 132 if (pkt) { 133 struct dma_chan *chan = NULL; 134 135 if (fifo) 136 chan = usbhsf_dma_chan_get(fifo, pkt); 137 if (chan) { 138 dmaengine_terminate_all(chan); 139 usbhsf_fifo_clear(pipe, fifo); 140 usbhsf_dma_unmap(pkt); 141 } 142 143 __usbhsf_pkt_del(pkt); 144 } 145 146 if (fifo) 147 usbhsf_fifo_unselect(pipe, fifo); 148 149 usbhs_unlock(priv, flags); 150 /******************** spin unlock ******************/ 151 152 return pkt; 153 } 154 155 enum { 156 USBHSF_PKT_PREPARE, 157 USBHSF_PKT_TRY_RUN, 158 USBHSF_PKT_DMA_DONE, 159 }; 160 161 static int usbhsf_pkt_handler(struct usbhs_pipe *pipe, int type) 162 { 163 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe); 164 struct usbhs_pkt *pkt; 165 struct device *dev = usbhs_priv_to_dev(priv); 166 int (*func)(struct usbhs_pkt *pkt, int *is_done); 167 unsigned long flags; 168 int ret = 0; 169 int is_done = 0; 170 171 /******************** spin lock ********************/ 172 usbhs_lock(priv, flags); 173 174 pkt = __usbhsf_pkt_get(pipe); 175 if (!pkt) 176 goto __usbhs_pkt_handler_end; 177 178 switch (type) { 179 case USBHSF_PKT_PREPARE: 180 func = pkt->handler->prepare; 181 break; 182 case USBHSF_PKT_TRY_RUN: 183 func = pkt->handler->try_run; 184 break; 185 case USBHSF_PKT_DMA_DONE: 186 func = pkt->handler->dma_done; 187 break; 188 default: 189 dev_err(dev, "unknown pkt handler\n"); 190 goto __usbhs_pkt_handler_end; 191 } 192 193 ret = func(pkt, &is_done); 194 195 if (is_done) 196 __usbhsf_pkt_del(pkt); 197 198 __usbhs_pkt_handler_end: 199 usbhs_unlock(priv, flags); 200 /******************** spin unlock ******************/ 201 202 if (is_done) { 203 pkt->done(priv, pkt); 204 usbhs_pkt_start(pipe); 205 } 206 207 return ret; 208 } 209 210 void usbhs_pkt_start(struct usbhs_pipe *pipe) 211 { 212 usbhsf_pkt_handler(pipe, USBHSF_PKT_PREPARE); 213 } 214 215 /* 216 * irq enable/disable function 217 */ 218 #define usbhsf_irq_empty_ctrl(p, e) usbhsf_irq_callback_ctrl(p, irq_bempsts, e) 219 #define usbhsf_irq_ready_ctrl(p, e) usbhsf_irq_callback_ctrl(p, irq_brdysts, e) 220 #define usbhsf_irq_callback_ctrl(pipe, status, enable) \ 221 ({ \ 222 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe); \ 223 struct usbhs_mod *mod = usbhs_mod_get_current(priv); \ 224 u16 status = (1 << usbhs_pipe_number(pipe)); \ 225 if (!mod) \ 226 return; \ 227 if (enable) \ 228 mod->status |= status; \ 229 else \ 230 mod->status &= ~status; \ 231 usbhs_irq_callback_update(priv, mod); \ 232 }) 233 234 static void usbhsf_tx_irq_ctrl(struct usbhs_pipe *pipe, int enable) 235 { 236 /* 237 * And DCP pipe can NOT use "ready interrupt" for "send" 238 * it should use "empty" interrupt. 239 * see 240 * "Operation" - "Interrupt Function" - "BRDY Interrupt" 241 * 242 * on the other hand, normal pipe can use "ready interrupt" for "send" 243 * even though it is single/double buffer 244 */ 245 if (usbhs_pipe_is_dcp(pipe)) 246 usbhsf_irq_empty_ctrl(pipe, enable); 247 else 248 usbhsf_irq_ready_ctrl(pipe, enable); 249 } 250 251 static void usbhsf_rx_irq_ctrl(struct usbhs_pipe *pipe, int enable) 252 { 253 usbhsf_irq_ready_ctrl(pipe, enable); 254 } 255 256 /* 257 * FIFO ctrl 258 */ 259 static void usbhsf_send_terminator(struct usbhs_pipe *pipe, 260 struct usbhs_fifo *fifo) 261 { 262 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe); 263 264 usbhs_bset(priv, fifo->ctr, BVAL, BVAL); 265 } 266 267 static int usbhsf_fifo_barrier(struct usbhs_priv *priv, 268 struct usbhs_fifo *fifo) 269 { 270 int timeout = 1024; 271 272 do { 273 /* The FIFO port is accessible */ 274 if (usbhs_read(priv, fifo->ctr) & FRDY) 275 return 0; 276 277 udelay(10); 278 } while (timeout--); 279 280 return -EBUSY; 281 } 282 283 static void usbhsf_fifo_clear(struct usbhs_pipe *pipe, 284 struct usbhs_fifo *fifo) 285 { 286 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe); 287 288 if (!usbhs_pipe_is_dcp(pipe)) 289 usbhsf_fifo_barrier(priv, fifo); 290 291 usbhs_write(priv, fifo->ctr, BCLR); 292 } 293 294 static int usbhsf_fifo_rcv_len(struct usbhs_priv *priv, 295 struct usbhs_fifo *fifo) 296 { 297 return usbhs_read(priv, fifo->ctr) & DTLN_MASK; 298 } 299 300 static void usbhsf_fifo_unselect(struct usbhs_pipe *pipe, 301 struct usbhs_fifo *fifo) 302 { 303 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe); 304 305 usbhs_pipe_select_fifo(pipe, NULL); 306 usbhs_write(priv, fifo->sel, 0); 307 } 308 309 static int usbhsf_fifo_select(struct usbhs_pipe *pipe, 310 struct usbhs_fifo *fifo, 311 int write) 312 { 313 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe); 314 struct device *dev = usbhs_priv_to_dev(priv); 315 int timeout = 1024; 316 u16 mask = ((1 << 5) | 0xF); /* mask of ISEL | CURPIPE */ 317 u16 base = usbhs_pipe_number(pipe); /* CURPIPE */ 318 319 if (usbhs_pipe_is_busy(pipe) || 320 usbhsf_fifo_is_busy(fifo)) 321 return -EBUSY; 322 323 if (usbhs_pipe_is_dcp(pipe)) { 324 base |= (1 == write) << 5; /* ISEL */ 325 326 if (usbhs_mod_is_host(priv)) 327 usbhs_dcp_dir_for_host(pipe, write); 328 } 329 330 /* "base" will be used below */ 331 if (usbhs_get_dparam(priv, has_sudmac) && !usbhsf_is_cfifo(priv, fifo)) 332 usbhs_write(priv, fifo->sel, base); 333 else 334 usbhs_write(priv, fifo->sel, base | MBW_32); 335 336 /* check ISEL and CURPIPE value */ 337 while (timeout--) { 338 if (base == (mask & usbhs_read(priv, fifo->sel))) { 339 usbhs_pipe_select_fifo(pipe, fifo); 340 return 0; 341 } 342 udelay(10); 343 } 344 345 dev_err(dev, "fifo select error\n"); 346 347 return -EIO; 348 } 349 350 /* 351 * DCP status stage 352 */ 353 static int usbhs_dcp_dir_switch_to_write(struct usbhs_pkt *pkt, int *is_done) 354 { 355 struct usbhs_pipe *pipe = pkt->pipe; 356 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe); 357 struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); /* CFIFO */ 358 struct device *dev = usbhs_priv_to_dev(priv); 359 int ret; 360 361 usbhs_pipe_disable(pipe); 362 363 ret = usbhsf_fifo_select(pipe, fifo, 1); 364 if (ret < 0) { 365 dev_err(dev, "%s() faile\n", __func__); 366 return ret; 367 } 368 369 usbhs_pipe_sequence_data1(pipe); /* DATA1 */ 370 371 usbhsf_fifo_clear(pipe, fifo); 372 usbhsf_send_terminator(pipe, fifo); 373 374 usbhsf_fifo_unselect(pipe, fifo); 375 376 usbhsf_tx_irq_ctrl(pipe, 1); 377 usbhs_pipe_enable(pipe); 378 379 return ret; 380 } 381 382 static int usbhs_dcp_dir_switch_to_read(struct usbhs_pkt *pkt, int *is_done) 383 { 384 struct usbhs_pipe *pipe = pkt->pipe; 385 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe); 386 struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); /* CFIFO */ 387 struct device *dev = usbhs_priv_to_dev(priv); 388 int ret; 389 390 usbhs_pipe_disable(pipe); 391 392 ret = usbhsf_fifo_select(pipe, fifo, 0); 393 if (ret < 0) { 394 dev_err(dev, "%s() fail\n", __func__); 395 return ret; 396 } 397 398 usbhs_pipe_sequence_data1(pipe); /* DATA1 */ 399 usbhsf_fifo_clear(pipe, fifo); 400 401 usbhsf_fifo_unselect(pipe, fifo); 402 403 usbhsf_rx_irq_ctrl(pipe, 1); 404 usbhs_pipe_enable(pipe); 405 406 return ret; 407 408 } 409 410 static int usbhs_dcp_dir_switch_done(struct usbhs_pkt *pkt, int *is_done) 411 { 412 struct usbhs_pipe *pipe = pkt->pipe; 413 414 if (pkt->handler == &usbhs_dcp_status_stage_in_handler) 415 usbhsf_tx_irq_ctrl(pipe, 0); 416 else 417 usbhsf_rx_irq_ctrl(pipe, 0); 418 419 pkt->actual = pkt->length; 420 *is_done = 1; 421 422 return 0; 423 } 424 425 struct usbhs_pkt_handle usbhs_dcp_status_stage_in_handler = { 426 .prepare = usbhs_dcp_dir_switch_to_write, 427 .try_run = usbhs_dcp_dir_switch_done, 428 }; 429 430 struct usbhs_pkt_handle usbhs_dcp_status_stage_out_handler = { 431 .prepare = usbhs_dcp_dir_switch_to_read, 432 .try_run = usbhs_dcp_dir_switch_done, 433 }; 434 435 /* 436 * DCP data stage (push) 437 */ 438 static int usbhsf_dcp_data_stage_try_push(struct usbhs_pkt *pkt, int *is_done) 439 { 440 struct usbhs_pipe *pipe = pkt->pipe; 441 442 usbhs_pipe_sequence_data1(pipe); /* DATA1 */ 443 444 /* 445 * change handler to PIO push 446 */ 447 pkt->handler = &usbhs_fifo_pio_push_handler; 448 449 return pkt->handler->prepare(pkt, is_done); 450 } 451 452 struct usbhs_pkt_handle usbhs_dcp_data_stage_out_handler = { 453 .prepare = usbhsf_dcp_data_stage_try_push, 454 }; 455 456 /* 457 * DCP data stage (pop) 458 */ 459 static int usbhsf_dcp_data_stage_prepare_pop(struct usbhs_pkt *pkt, 460 int *is_done) 461 { 462 struct usbhs_pipe *pipe = pkt->pipe; 463 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe); 464 struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); 465 466 if (usbhs_pipe_is_busy(pipe)) 467 return 0; 468 469 /* 470 * prepare pop for DCP should 471 * - change DCP direction, 472 * - clear fifo 473 * - DATA1 474 */ 475 usbhs_pipe_disable(pipe); 476 477 usbhs_pipe_sequence_data1(pipe); /* DATA1 */ 478 479 usbhsf_fifo_select(pipe, fifo, 0); 480 usbhsf_fifo_clear(pipe, fifo); 481 usbhsf_fifo_unselect(pipe, fifo); 482 483 /* 484 * change handler to PIO pop 485 */ 486 pkt->handler = &usbhs_fifo_pio_pop_handler; 487 488 return pkt->handler->prepare(pkt, is_done); 489 } 490 491 struct usbhs_pkt_handle usbhs_dcp_data_stage_in_handler = { 492 .prepare = usbhsf_dcp_data_stage_prepare_pop, 493 }; 494 495 /* 496 * PIO push handler 497 */ 498 static int usbhsf_pio_try_push(struct usbhs_pkt *pkt, int *is_done) 499 { 500 struct usbhs_pipe *pipe = pkt->pipe; 501 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe); 502 struct device *dev = usbhs_priv_to_dev(priv); 503 struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); /* CFIFO */ 504 void __iomem *addr = priv->base + fifo->port; 505 u8 *buf; 506 int maxp = usbhs_pipe_get_maxpacket(pipe); 507 int total_len; 508 int i, ret, len; 509 int is_short; 510 511 usbhs_pipe_data_sequence(pipe, pkt->sequence); 512 pkt->sequence = -1; /* -1 sequence will be ignored */ 513 514 usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->length); 515 516 ret = usbhsf_fifo_select(pipe, fifo, 1); 517 if (ret < 0) 518 return 0; 519 520 ret = usbhs_pipe_is_accessible(pipe); 521 if (ret < 0) { 522 /* inaccessible pipe is not an error */ 523 ret = 0; 524 goto usbhs_fifo_write_busy; 525 } 526 527 ret = usbhsf_fifo_barrier(priv, fifo); 528 if (ret < 0) 529 goto usbhs_fifo_write_busy; 530 531 buf = pkt->buf + pkt->actual; 532 len = pkt->length - pkt->actual; 533 len = min(len, maxp); 534 total_len = len; 535 is_short = total_len < maxp; 536 537 /* 538 * FIXME 539 * 540 * 32-bit access only 541 */ 542 if (len >= 4 && !((unsigned long)buf & 0x03)) { 543 iowrite32_rep(addr, buf, len / 4); 544 len %= 4; 545 buf += total_len - len; 546 } 547 548 /* the rest operation */ 549 for (i = 0; i < len; i++) 550 iowrite8(buf[i], addr + (0x03 - (i & 0x03))); 551 552 /* 553 * variable update 554 */ 555 pkt->actual += total_len; 556 557 if (pkt->actual < pkt->length) 558 *is_done = 0; /* there are remainder data */ 559 else if (is_short) 560 *is_done = 1; /* short packet */ 561 else 562 *is_done = !pkt->zero; /* send zero packet ? */ 563 564 /* 565 * pipe/irq handling 566 */ 567 if (is_short) 568 usbhsf_send_terminator(pipe, fifo); 569 570 usbhsf_tx_irq_ctrl(pipe, !*is_done); 571 usbhs_pipe_running(pipe, !*is_done); 572 usbhs_pipe_enable(pipe); 573 574 dev_dbg(dev, " send %d (%d/ %d/ %d/ %d)\n", 575 usbhs_pipe_number(pipe), 576 pkt->length, pkt->actual, *is_done, pkt->zero); 577 578 usbhsf_fifo_unselect(pipe, fifo); 579 580 return 0; 581 582 usbhs_fifo_write_busy: 583 usbhsf_fifo_unselect(pipe, fifo); 584 585 /* 586 * pipe is busy. 587 * retry in interrupt 588 */ 589 usbhsf_tx_irq_ctrl(pipe, 1); 590 usbhs_pipe_running(pipe, 1); 591 592 return ret; 593 } 594 595 static int usbhsf_pio_prepare_push(struct usbhs_pkt *pkt, int *is_done) 596 { 597 if (usbhs_pipe_is_running(pkt->pipe)) 598 return 0; 599 600 return usbhsf_pio_try_push(pkt, is_done); 601 } 602 603 struct usbhs_pkt_handle usbhs_fifo_pio_push_handler = { 604 .prepare = usbhsf_pio_prepare_push, 605 .try_run = usbhsf_pio_try_push, 606 }; 607 608 /* 609 * PIO pop handler 610 */ 611 static int usbhsf_prepare_pop(struct usbhs_pkt *pkt, int *is_done) 612 { 613 struct usbhs_pipe *pipe = pkt->pipe; 614 615 if (usbhs_pipe_is_busy(pipe)) 616 return 0; 617 618 if (usbhs_pipe_is_running(pipe)) 619 return 0; 620 621 /* 622 * pipe enable to prepare packet receive 623 */ 624 usbhs_pipe_data_sequence(pipe, pkt->sequence); 625 pkt->sequence = -1; /* -1 sequence will be ignored */ 626 627 usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->length); 628 usbhs_pipe_enable(pipe); 629 usbhs_pipe_running(pipe, 1); 630 usbhsf_rx_irq_ctrl(pipe, 1); 631 632 return 0; 633 } 634 635 static int usbhsf_pio_try_pop(struct usbhs_pkt *pkt, int *is_done) 636 { 637 struct usbhs_pipe *pipe = pkt->pipe; 638 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe); 639 struct device *dev = usbhs_priv_to_dev(priv); 640 struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); /* CFIFO */ 641 void __iomem *addr = priv->base + fifo->port; 642 u8 *buf; 643 u32 data = 0; 644 int maxp = usbhs_pipe_get_maxpacket(pipe); 645 int rcv_len, len; 646 int i, ret; 647 int total_len = 0; 648 649 ret = usbhsf_fifo_select(pipe, fifo, 0); 650 if (ret < 0) 651 return 0; 652 653 ret = usbhsf_fifo_barrier(priv, fifo); 654 if (ret < 0) 655 goto usbhs_fifo_read_busy; 656 657 rcv_len = usbhsf_fifo_rcv_len(priv, fifo); 658 659 buf = pkt->buf + pkt->actual; 660 len = pkt->length - pkt->actual; 661 len = min(len, rcv_len); 662 total_len = len; 663 664 /* 665 * update actual length first here to decide disable pipe. 666 * if this pipe keeps BUF status and all data were popped, 667 * then, next interrupt/token will be issued again 668 */ 669 pkt->actual += total_len; 670 671 if ((pkt->actual == pkt->length) || /* receive all data */ 672 (total_len < maxp)) { /* short packet */ 673 *is_done = 1; 674 usbhsf_rx_irq_ctrl(pipe, 0); 675 usbhs_pipe_running(pipe, 0); 676 usbhs_pipe_disable(pipe); /* disable pipe first */ 677 } 678 679 /* 680 * Buffer clear if Zero-Length packet 681 * 682 * see 683 * "Operation" - "FIFO Buffer Memory" - "FIFO Port Function" 684 */ 685 if (0 == rcv_len) { 686 pkt->zero = 1; 687 usbhsf_fifo_clear(pipe, fifo); 688 goto usbhs_fifo_read_end; 689 } 690 691 /* 692 * FIXME 693 * 694 * 32-bit access only 695 */ 696 if (len >= 4 && !((unsigned long)buf & 0x03)) { 697 ioread32_rep(addr, buf, len / 4); 698 len %= 4; 699 buf += total_len - len; 700 } 701 702 /* the rest operation */ 703 for (i = 0; i < len; i++) { 704 if (!(i & 0x03)) 705 data = ioread32(addr); 706 707 buf[i] = (data >> ((i & 0x03) * 8)) & 0xff; 708 } 709 710 usbhs_fifo_read_end: 711 dev_dbg(dev, " recv %d (%d/ %d/ %d/ %d)\n", 712 usbhs_pipe_number(pipe), 713 pkt->length, pkt->actual, *is_done, pkt->zero); 714 715 usbhs_fifo_read_busy: 716 usbhsf_fifo_unselect(pipe, fifo); 717 718 return ret; 719 } 720 721 struct usbhs_pkt_handle usbhs_fifo_pio_pop_handler = { 722 .prepare = usbhsf_prepare_pop, 723 .try_run = usbhsf_pio_try_pop, 724 }; 725 726 /* 727 * DCP ctrol statge handler 728 */ 729 static int usbhsf_ctrl_stage_end(struct usbhs_pkt *pkt, int *is_done) 730 { 731 usbhs_dcp_control_transfer_done(pkt->pipe); 732 733 *is_done = 1; 734 735 return 0; 736 } 737 738 struct usbhs_pkt_handle usbhs_ctrl_stage_end_handler = { 739 .prepare = usbhsf_ctrl_stage_end, 740 .try_run = usbhsf_ctrl_stage_end, 741 }; 742 743 /* 744 * DMA fifo functions 745 */ 746 static struct dma_chan *usbhsf_dma_chan_get(struct usbhs_fifo *fifo, 747 struct usbhs_pkt *pkt) 748 { 749 if (&usbhs_fifo_dma_push_handler == pkt->handler) 750 return fifo->tx_chan; 751 752 if (&usbhs_fifo_dma_pop_handler == pkt->handler) 753 return fifo->rx_chan; 754 755 return NULL; 756 } 757 758 static struct usbhs_fifo *usbhsf_get_dma_fifo(struct usbhs_priv *priv, 759 struct usbhs_pkt *pkt) 760 { 761 struct usbhs_fifo *fifo; 762 int i; 763 764 usbhs_for_each_dfifo(priv, fifo, i) { 765 if (usbhsf_dma_chan_get(fifo, pkt) && 766 !usbhsf_fifo_is_busy(fifo)) 767 return fifo; 768 } 769 770 return NULL; 771 } 772 773 #define usbhsf_dma_start(p, f) __usbhsf_dma_ctrl(p, f, DREQE) 774 #define usbhsf_dma_stop(p, f) __usbhsf_dma_ctrl(p, f, 0) 775 static void __usbhsf_dma_ctrl(struct usbhs_pipe *pipe, 776 struct usbhs_fifo *fifo, 777 u16 dreqe) 778 { 779 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe); 780 781 usbhs_bset(priv, fifo->sel, DREQE, dreqe); 782 } 783 784 static int __usbhsf_dma_map_ctrl(struct usbhs_pkt *pkt, int map) 785 { 786 struct usbhs_pipe *pipe = pkt->pipe; 787 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe); 788 struct usbhs_pipe_info *info = usbhs_priv_to_pipeinfo(priv); 789 790 return info->dma_map_ctrl(pkt, map); 791 } 792 793 static void usbhsf_dma_complete(void *arg); 794 static void xfer_work(struct work_struct *work) 795 { 796 struct usbhs_pkt *pkt = container_of(work, struct usbhs_pkt, work); 797 struct usbhs_pipe *pipe = pkt->pipe; 798 struct usbhs_fifo *fifo = usbhs_pipe_to_fifo(pipe); 799 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe); 800 struct dma_async_tx_descriptor *desc; 801 struct dma_chan *chan = usbhsf_dma_chan_get(fifo, pkt); 802 struct device *dev = usbhs_priv_to_dev(priv); 803 enum dma_transfer_direction dir; 804 805 dir = usbhs_pipe_is_dir_in(pipe) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV; 806 807 desc = dmaengine_prep_slave_single(chan, pkt->dma + pkt->actual, 808 pkt->trans, dir, 809 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 810 if (!desc) 811 return; 812 813 desc->callback = usbhsf_dma_complete; 814 desc->callback_param = pipe; 815 816 if (dmaengine_submit(desc) < 0) { 817 dev_err(dev, "Failed to submit dma descriptor\n"); 818 return; 819 } 820 821 dev_dbg(dev, " %s %d (%d/ %d)\n", 822 fifo->name, usbhs_pipe_number(pipe), pkt->length, pkt->zero); 823 824 usbhs_pipe_running(pipe, 1); 825 usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->trans); 826 usbhs_pipe_enable(pipe); 827 usbhsf_dma_start(pipe, fifo); 828 dma_async_issue_pending(chan); 829 } 830 831 /* 832 * DMA push handler 833 */ 834 static int usbhsf_dma_prepare_push(struct usbhs_pkt *pkt, int *is_done) 835 { 836 struct usbhs_pipe *pipe = pkt->pipe; 837 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe); 838 struct usbhs_fifo *fifo; 839 int len = pkt->length - pkt->actual; 840 int ret; 841 842 if (usbhs_pipe_is_busy(pipe)) 843 return 0; 844 845 /* use PIO if packet is less than pio_dma_border or pipe is DCP */ 846 if ((len < usbhs_get_dparam(priv, pio_dma_border)) || 847 usbhs_pipe_is_dcp(pipe)) 848 goto usbhsf_pio_prepare_push; 849 850 if (len & 0x7) /* 8byte alignment */ 851 goto usbhsf_pio_prepare_push; 852 853 if ((uintptr_t)(pkt->buf + pkt->actual) & 0x7) /* 8byte alignment */ 854 goto usbhsf_pio_prepare_push; 855 856 /* return at this time if the pipe is running */ 857 if (usbhs_pipe_is_running(pipe)) 858 return 0; 859 860 /* get enable DMA fifo */ 861 fifo = usbhsf_get_dma_fifo(priv, pkt); 862 if (!fifo) 863 goto usbhsf_pio_prepare_push; 864 865 if (usbhsf_dma_map(pkt) < 0) 866 goto usbhsf_pio_prepare_push; 867 868 ret = usbhsf_fifo_select(pipe, fifo, 0); 869 if (ret < 0) 870 goto usbhsf_pio_prepare_push_unmap; 871 872 pkt->trans = len; 873 874 INIT_WORK(&pkt->work, xfer_work); 875 schedule_work(&pkt->work); 876 877 return 0; 878 879 usbhsf_pio_prepare_push_unmap: 880 usbhsf_dma_unmap(pkt); 881 usbhsf_pio_prepare_push: 882 /* 883 * change handler to PIO 884 */ 885 pkt->handler = &usbhs_fifo_pio_push_handler; 886 887 return pkt->handler->prepare(pkt, is_done); 888 } 889 890 static int usbhsf_dma_push_done(struct usbhs_pkt *pkt, int *is_done) 891 { 892 struct usbhs_pipe *pipe = pkt->pipe; 893 int is_short = pkt->trans % usbhs_pipe_get_maxpacket(pipe); 894 895 pkt->actual += pkt->trans; 896 897 if (pkt->actual < pkt->length) 898 *is_done = 0; /* there are remainder data */ 899 else if (is_short) 900 *is_done = 1; /* short packet */ 901 else 902 *is_done = !pkt->zero; /* send zero packet? */ 903 904 usbhs_pipe_running(pipe, !*is_done); 905 906 usbhsf_dma_stop(pipe, pipe->fifo); 907 usbhsf_dma_unmap(pkt); 908 usbhsf_fifo_unselect(pipe, pipe->fifo); 909 910 if (!*is_done) { 911 /* change handler to PIO */ 912 pkt->handler = &usbhs_fifo_pio_push_handler; 913 return pkt->handler->try_run(pkt, is_done); 914 } 915 916 return 0; 917 } 918 919 struct usbhs_pkt_handle usbhs_fifo_dma_push_handler = { 920 .prepare = usbhsf_dma_prepare_push, 921 .dma_done = usbhsf_dma_push_done, 922 }; 923 924 /* 925 * DMA pop handler 926 */ 927 static int usbhsf_dma_try_pop(struct usbhs_pkt *pkt, int *is_done) 928 { 929 struct usbhs_pipe *pipe = pkt->pipe; 930 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe); 931 struct usbhs_fifo *fifo; 932 int len, ret; 933 934 if (usbhs_pipe_is_busy(pipe)) 935 return 0; 936 937 if (usbhs_pipe_is_dcp(pipe)) 938 goto usbhsf_pio_prepare_pop; 939 940 /* get enable DMA fifo */ 941 fifo = usbhsf_get_dma_fifo(priv, pkt); 942 if (!fifo) 943 goto usbhsf_pio_prepare_pop; 944 945 if ((uintptr_t)(pkt->buf + pkt->actual) & 0x7) /* 8byte alignment */ 946 goto usbhsf_pio_prepare_pop; 947 948 ret = usbhsf_fifo_select(pipe, fifo, 0); 949 if (ret < 0) 950 goto usbhsf_pio_prepare_pop; 951 952 /* use PIO if packet is less than pio_dma_border */ 953 len = usbhsf_fifo_rcv_len(priv, fifo); 954 len = min(pkt->length - pkt->actual, len); 955 if (len & 0x7) /* 8byte alignment */ 956 goto usbhsf_pio_prepare_pop_unselect; 957 958 if (len < usbhs_get_dparam(priv, pio_dma_border)) 959 goto usbhsf_pio_prepare_pop_unselect; 960 961 ret = usbhsf_fifo_barrier(priv, fifo); 962 if (ret < 0) 963 goto usbhsf_pio_prepare_pop_unselect; 964 965 if (usbhsf_dma_map(pkt) < 0) 966 goto usbhsf_pio_prepare_pop_unselect; 967 968 /* DMA */ 969 970 /* 971 * usbhs_fifo_dma_pop_handler :: prepare 972 * enabled irq to come here. 973 * but it is no longer needed for DMA. disable it. 974 */ 975 usbhsf_rx_irq_ctrl(pipe, 0); 976 977 pkt->trans = len; 978 979 INIT_WORK(&pkt->work, xfer_work); 980 schedule_work(&pkt->work); 981 982 return 0; 983 984 usbhsf_pio_prepare_pop_unselect: 985 usbhsf_fifo_unselect(pipe, fifo); 986 usbhsf_pio_prepare_pop: 987 988 /* 989 * change handler to PIO 990 */ 991 pkt->handler = &usbhs_fifo_pio_pop_handler; 992 993 return pkt->handler->try_run(pkt, is_done); 994 } 995 996 static int usbhsf_dma_pop_done(struct usbhs_pkt *pkt, int *is_done) 997 { 998 struct usbhs_pipe *pipe = pkt->pipe; 999 int maxp = usbhs_pipe_get_maxpacket(pipe); 1000 1001 usbhsf_dma_stop(pipe, pipe->fifo); 1002 usbhsf_dma_unmap(pkt); 1003 usbhsf_fifo_unselect(pipe, pipe->fifo); 1004 1005 pkt->actual += pkt->trans; 1006 1007 if ((pkt->actual == pkt->length) || /* receive all data */ 1008 (pkt->trans < maxp)) { /* short packet */ 1009 *is_done = 1; 1010 usbhs_pipe_running(pipe, 0); 1011 } else { 1012 /* re-enable */ 1013 usbhs_pipe_running(pipe, 0); 1014 usbhsf_prepare_pop(pkt, is_done); 1015 } 1016 1017 return 0; 1018 } 1019 1020 struct usbhs_pkt_handle usbhs_fifo_dma_pop_handler = { 1021 .prepare = usbhsf_prepare_pop, 1022 .try_run = usbhsf_dma_try_pop, 1023 .dma_done = usbhsf_dma_pop_done 1024 }; 1025 1026 /* 1027 * DMA setting 1028 */ 1029 static bool usbhsf_dma_filter(struct dma_chan *chan, void *param) 1030 { 1031 struct sh_dmae_slave *slave = param; 1032 1033 /* 1034 * FIXME 1035 * 1036 * usbhs doesn't recognize id = 0 as valid DMA 1037 */ 1038 if (0 == slave->shdma_slave.slave_id) 1039 return false; 1040 1041 chan->private = slave; 1042 1043 return true; 1044 } 1045 1046 static void usbhsf_dma_quit(struct usbhs_priv *priv, struct usbhs_fifo *fifo) 1047 { 1048 if (fifo->tx_chan) 1049 dma_release_channel(fifo->tx_chan); 1050 if (fifo->rx_chan) 1051 dma_release_channel(fifo->rx_chan); 1052 1053 fifo->tx_chan = NULL; 1054 fifo->rx_chan = NULL; 1055 } 1056 1057 static void usbhsf_dma_init(struct usbhs_priv *priv, 1058 struct usbhs_fifo *fifo) 1059 { 1060 struct device *dev = usbhs_priv_to_dev(priv); 1061 dma_cap_mask_t mask; 1062 1063 dma_cap_zero(mask); 1064 dma_cap_set(DMA_SLAVE, mask); 1065 fifo->tx_chan = dma_request_channel(mask, usbhsf_dma_filter, 1066 &fifo->tx_slave); 1067 1068 dma_cap_zero(mask); 1069 dma_cap_set(DMA_SLAVE, mask); 1070 fifo->rx_chan = dma_request_channel(mask, usbhsf_dma_filter, 1071 &fifo->rx_slave); 1072 1073 if (fifo->tx_chan || fifo->rx_chan) 1074 dev_dbg(dev, "enable DMAEngine (%s%s%s)\n", 1075 fifo->name, 1076 fifo->tx_chan ? "[TX]" : " ", 1077 fifo->rx_chan ? "[RX]" : " "); 1078 } 1079 1080 /* 1081 * irq functions 1082 */ 1083 static int usbhsf_irq_empty(struct usbhs_priv *priv, 1084 struct usbhs_irq_state *irq_state) 1085 { 1086 struct usbhs_pipe *pipe; 1087 struct device *dev = usbhs_priv_to_dev(priv); 1088 int i, ret; 1089 1090 if (!irq_state->bempsts) { 1091 dev_err(dev, "debug %s !!\n", __func__); 1092 return -EIO; 1093 } 1094 1095 dev_dbg(dev, "irq empty [0x%04x]\n", irq_state->bempsts); 1096 1097 /* 1098 * search interrupted "pipe" 1099 * not "uep". 1100 */ 1101 usbhs_for_each_pipe_with_dcp(pipe, priv, i) { 1102 if (!(irq_state->bempsts & (1 << i))) 1103 continue; 1104 1105 ret = usbhsf_pkt_handler(pipe, USBHSF_PKT_TRY_RUN); 1106 if (ret < 0) 1107 dev_err(dev, "irq_empty run_error %d : %d\n", i, ret); 1108 } 1109 1110 return 0; 1111 } 1112 1113 static int usbhsf_irq_ready(struct usbhs_priv *priv, 1114 struct usbhs_irq_state *irq_state) 1115 { 1116 struct usbhs_pipe *pipe; 1117 struct device *dev = usbhs_priv_to_dev(priv); 1118 int i, ret; 1119 1120 if (!irq_state->brdysts) { 1121 dev_err(dev, "debug %s !!\n", __func__); 1122 return -EIO; 1123 } 1124 1125 dev_dbg(dev, "irq ready [0x%04x]\n", irq_state->brdysts); 1126 1127 /* 1128 * search interrupted "pipe" 1129 * not "uep". 1130 */ 1131 usbhs_for_each_pipe_with_dcp(pipe, priv, i) { 1132 if (!(irq_state->brdysts & (1 << i))) 1133 continue; 1134 1135 ret = usbhsf_pkt_handler(pipe, USBHSF_PKT_TRY_RUN); 1136 if (ret < 0) 1137 dev_err(dev, "irq_ready run_error %d : %d\n", i, ret); 1138 } 1139 1140 return 0; 1141 } 1142 1143 static void usbhsf_dma_complete(void *arg) 1144 { 1145 struct usbhs_pipe *pipe = arg; 1146 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe); 1147 struct device *dev = usbhs_priv_to_dev(priv); 1148 int ret; 1149 1150 ret = usbhsf_pkt_handler(pipe, USBHSF_PKT_DMA_DONE); 1151 if (ret < 0) 1152 dev_err(dev, "dma_complete run_error %d : %d\n", 1153 usbhs_pipe_number(pipe), ret); 1154 } 1155 1156 void usbhs_fifo_clear_dcp(struct usbhs_pipe *pipe) 1157 { 1158 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe); 1159 struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); /* CFIFO */ 1160 1161 /* clear DCP FIFO of transmission */ 1162 if (usbhsf_fifo_select(pipe, fifo, 1) < 0) 1163 return; 1164 usbhsf_fifo_clear(pipe, fifo); 1165 usbhsf_fifo_unselect(pipe, fifo); 1166 1167 /* clear DCP FIFO of reception */ 1168 if (usbhsf_fifo_select(pipe, fifo, 0) < 0) 1169 return; 1170 usbhsf_fifo_clear(pipe, fifo); 1171 usbhsf_fifo_unselect(pipe, fifo); 1172 } 1173 1174 /* 1175 * fifo init 1176 */ 1177 void usbhs_fifo_init(struct usbhs_priv *priv) 1178 { 1179 struct usbhs_mod *mod = usbhs_mod_get_current(priv); 1180 struct usbhs_fifo *cfifo = usbhsf_get_cfifo(priv); 1181 struct usbhs_fifo *dfifo; 1182 int i; 1183 1184 mod->irq_empty = usbhsf_irq_empty; 1185 mod->irq_ready = usbhsf_irq_ready; 1186 mod->irq_bempsts = 0; 1187 mod->irq_brdysts = 0; 1188 1189 cfifo->pipe = NULL; 1190 usbhs_for_each_dfifo(priv, dfifo, i) 1191 dfifo->pipe = NULL; 1192 } 1193 1194 void usbhs_fifo_quit(struct usbhs_priv *priv) 1195 { 1196 struct usbhs_mod *mod = usbhs_mod_get_current(priv); 1197 1198 mod->irq_empty = NULL; 1199 mod->irq_ready = NULL; 1200 mod->irq_bempsts = 0; 1201 mod->irq_brdysts = 0; 1202 } 1203 1204 #define __USBHS_DFIFO_INIT(priv, fifo, channel, fifo_port) \ 1205 do { \ 1206 fifo = usbhsf_get_dnfifo(priv, channel); \ 1207 fifo->name = "D"#channel"FIFO"; \ 1208 fifo->port = fifo_port; \ 1209 fifo->sel = D##channel##FIFOSEL; \ 1210 fifo->ctr = D##channel##FIFOCTR; \ 1211 fifo->tx_slave.shdma_slave.slave_id = \ 1212 usbhs_get_dparam(priv, d##channel##_tx_id); \ 1213 fifo->rx_slave.shdma_slave.slave_id = \ 1214 usbhs_get_dparam(priv, d##channel##_rx_id); \ 1215 usbhsf_dma_init(priv, fifo); \ 1216 } while (0) 1217 1218 #define USBHS_DFIFO_INIT(priv, fifo, channel) \ 1219 __USBHS_DFIFO_INIT(priv, fifo, channel, D##channel##FIFO) 1220 #define USBHS_DFIFO_INIT_NO_PORT(priv, fifo, channel) \ 1221 __USBHS_DFIFO_INIT(priv, fifo, channel, 0) 1222 1223 int usbhs_fifo_probe(struct usbhs_priv *priv) 1224 { 1225 struct usbhs_fifo *fifo; 1226 1227 /* CFIFO */ 1228 fifo = usbhsf_get_cfifo(priv); 1229 fifo->name = "CFIFO"; 1230 fifo->port = CFIFO; 1231 fifo->sel = CFIFOSEL; 1232 fifo->ctr = CFIFOCTR; 1233 1234 /* DFIFO */ 1235 USBHS_DFIFO_INIT(priv, fifo, 0); 1236 USBHS_DFIFO_INIT(priv, fifo, 1); 1237 USBHS_DFIFO_INIT_NO_PORT(priv, fifo, 2); 1238 USBHS_DFIFO_INIT_NO_PORT(priv, fifo, 3); 1239 1240 return 0; 1241 } 1242 1243 void usbhs_fifo_remove(struct usbhs_priv *priv) 1244 { 1245 struct usbhs_fifo *fifo; 1246 int i; 1247 1248 usbhs_for_each_dfifo(priv, fifo, i) 1249 usbhsf_dma_quit(priv, fifo); 1250 } 1251