1 /* 2 * Intel I/OAT DMA Linux driver 3 * Copyright(c) 2004 - 2015 Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 * The full GNU General Public License is included in this distribution in 15 * the file called "COPYING". 16 * 17 */ 18 19 /* 20 * This driver supports an Intel I/OAT DMA engine, which does asynchronous 21 * copy operations. 22 */ 23 24 #include <linux/init.h> 25 #include <linux/module.h> 26 #include <linux/slab.h> 27 #include <linux/pci.h> 28 #include <linux/interrupt.h> 29 #include <linux/dmaengine.h> 30 #include <linux/delay.h> 31 #include <linux/dma-mapping.h> 32 #include <linux/workqueue.h> 33 #include <linux/prefetch.h> 34 #include <linux/sizes.h> 35 #include "dma.h" 36 #include "registers.h" 37 #include "hw.h" 38 39 #include "../dmaengine.h" 40 41 static char *chanerr_str[] = { 42 "DMA Transfer Destination Address Error", 43 "Next Descriptor Address Error", 44 "Descriptor Error", 45 "Chan Address Value Error", 46 "CHANCMD Error", 47 "Chipset Uncorrectable Data Integrity Error", 48 "DMA Uncorrectable Data Integrity Error", 49 "Read Data Error", 50 "Write Data Error", 51 "Descriptor Control Error", 52 "Descriptor Transfer Size Error", 53 "Completion Address Error", 54 "Interrupt Configuration Error", 55 "Super extended descriptor Address Error", 56 "Unaffiliated Error", 57 "CRC or XOR P Error", 58 "XOR Q Error", 59 "Descriptor Count Error", 60 "DIF All F detect Error", 61 "Guard Tag verification Error", 62 "Application Tag verification Error", 63 "Reference Tag verification Error", 64 "Bundle Bit Error", 65 "Result DIF All F detect Error", 66 "Result Guard Tag verification Error", 67 "Result Application Tag verification Error", 68 "Result Reference Tag verification Error", 69 NULL 70 }; 71 72 static void ioat_eh(struct ioatdma_chan *ioat_chan); 73 74 static void ioat_print_chanerrs(struct ioatdma_chan *ioat_chan, u32 chanerr) 75 { 76 int i; 77 78 for (i = 0; i < 32; i++) { 79 if ((chanerr >> i) & 1) { 80 if (chanerr_str[i]) { 81 dev_err(to_dev(ioat_chan), "Err(%d): %s\n", 82 i, chanerr_str[i]); 83 } else 84 break; 85 } 86 } 87 } 88 89 /** 90 * ioat_dma_do_interrupt - handler used for single vector interrupt mode 91 * @irq: interrupt id 92 * @data: interrupt data 93 */ 94 irqreturn_t ioat_dma_do_interrupt(int irq, void *data) 95 { 96 struct ioatdma_device *instance = data; 97 struct ioatdma_chan *ioat_chan; 98 unsigned long attnstatus; 99 int bit; 100 u8 intrctrl; 101 102 intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET); 103 104 if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN)) 105 return IRQ_NONE; 106 107 if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) { 108 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); 109 return IRQ_NONE; 110 } 111 112 attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET); 113 for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) { 114 ioat_chan = ioat_chan_by_index(instance, bit); 115 if (test_bit(IOAT_RUN, &ioat_chan->state)) 116 tasklet_schedule(&ioat_chan->cleanup_task); 117 } 118 119 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); 120 return IRQ_HANDLED; 121 } 122 123 /** 124 * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode 125 * @irq: interrupt id 126 * @data: interrupt data 127 */ 128 irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data) 129 { 130 struct ioatdma_chan *ioat_chan = data; 131 132 if (test_bit(IOAT_RUN, &ioat_chan->state)) 133 tasklet_schedule(&ioat_chan->cleanup_task); 134 135 return IRQ_HANDLED; 136 } 137 138 void ioat_stop(struct ioatdma_chan *ioat_chan) 139 { 140 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; 141 struct pci_dev *pdev = ioat_dma->pdev; 142 int chan_id = chan_num(ioat_chan); 143 struct msix_entry *msix; 144 145 /* 1/ stop irq from firing tasklets 146 * 2/ stop the tasklet from re-arming irqs 147 */ 148 clear_bit(IOAT_RUN, &ioat_chan->state); 149 150 /* flush inflight interrupts */ 151 switch (ioat_dma->irq_mode) { 152 case IOAT_MSIX: 153 msix = &ioat_dma->msix_entries[chan_id]; 154 synchronize_irq(msix->vector); 155 break; 156 case IOAT_MSI: 157 case IOAT_INTX: 158 synchronize_irq(pdev->irq); 159 break; 160 default: 161 break; 162 } 163 164 /* flush inflight timers */ 165 del_timer_sync(&ioat_chan->timer); 166 167 /* flush inflight tasklet runs */ 168 tasklet_kill(&ioat_chan->cleanup_task); 169 170 /* final cleanup now that everything is quiesced and can't re-arm */ 171 ioat_cleanup_event((unsigned long)&ioat_chan->dma_chan); 172 } 173 174 static void __ioat_issue_pending(struct ioatdma_chan *ioat_chan) 175 { 176 ioat_chan->dmacount += ioat_ring_pending(ioat_chan); 177 ioat_chan->issued = ioat_chan->head; 178 writew(ioat_chan->dmacount, 179 ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET); 180 dev_dbg(to_dev(ioat_chan), 181 "%s: head: %#x tail: %#x issued: %#x count: %#x\n", 182 __func__, ioat_chan->head, ioat_chan->tail, 183 ioat_chan->issued, ioat_chan->dmacount); 184 } 185 186 void ioat_issue_pending(struct dma_chan *c) 187 { 188 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); 189 190 if (ioat_ring_pending(ioat_chan)) { 191 spin_lock_bh(&ioat_chan->prep_lock); 192 __ioat_issue_pending(ioat_chan); 193 spin_unlock_bh(&ioat_chan->prep_lock); 194 } 195 } 196 197 /** 198 * ioat_update_pending - log pending descriptors 199 * @ioat: ioat+ channel 200 * 201 * Check if the number of unsubmitted descriptors has exceeded the 202 * watermark. Called with prep_lock held 203 */ 204 static void ioat_update_pending(struct ioatdma_chan *ioat_chan) 205 { 206 if (ioat_ring_pending(ioat_chan) > ioat_pending_level) 207 __ioat_issue_pending(ioat_chan); 208 } 209 210 static void __ioat_start_null_desc(struct ioatdma_chan *ioat_chan) 211 { 212 struct ioat_ring_ent *desc; 213 struct ioat_dma_descriptor *hw; 214 215 if (ioat_ring_space(ioat_chan) < 1) { 216 dev_err(to_dev(ioat_chan), 217 "Unable to start null desc - ring full\n"); 218 return; 219 } 220 221 dev_dbg(to_dev(ioat_chan), 222 "%s: head: %#x tail: %#x issued: %#x\n", 223 __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued); 224 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head); 225 226 hw = desc->hw; 227 hw->ctl = 0; 228 hw->ctl_f.null = 1; 229 hw->ctl_f.int_en = 1; 230 hw->ctl_f.compl_write = 1; 231 /* set size to non-zero value (channel returns error when size is 0) */ 232 hw->size = NULL_DESC_BUFFER_SIZE; 233 hw->src_addr = 0; 234 hw->dst_addr = 0; 235 async_tx_ack(&desc->txd); 236 ioat_set_chainaddr(ioat_chan, desc->txd.phys); 237 dump_desc_dbg(ioat_chan, desc); 238 /* make sure descriptors are written before we submit */ 239 wmb(); 240 ioat_chan->head += 1; 241 __ioat_issue_pending(ioat_chan); 242 } 243 244 void ioat_start_null_desc(struct ioatdma_chan *ioat_chan) 245 { 246 spin_lock_bh(&ioat_chan->prep_lock); 247 if (!test_bit(IOAT_CHAN_DOWN, &ioat_chan->state)) 248 __ioat_start_null_desc(ioat_chan); 249 spin_unlock_bh(&ioat_chan->prep_lock); 250 } 251 252 static void __ioat_restart_chan(struct ioatdma_chan *ioat_chan) 253 { 254 /* set the tail to be re-issued */ 255 ioat_chan->issued = ioat_chan->tail; 256 ioat_chan->dmacount = 0; 257 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); 258 259 dev_dbg(to_dev(ioat_chan), 260 "%s: head: %#x tail: %#x issued: %#x count: %#x\n", 261 __func__, ioat_chan->head, ioat_chan->tail, 262 ioat_chan->issued, ioat_chan->dmacount); 263 264 if (ioat_ring_pending(ioat_chan)) { 265 struct ioat_ring_ent *desc; 266 267 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail); 268 ioat_set_chainaddr(ioat_chan, desc->txd.phys); 269 __ioat_issue_pending(ioat_chan); 270 } else 271 __ioat_start_null_desc(ioat_chan); 272 } 273 274 static int ioat_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo) 275 { 276 unsigned long end = jiffies + tmo; 277 int err = 0; 278 u32 status; 279 280 status = ioat_chansts(ioat_chan); 281 if (is_ioat_active(status) || is_ioat_idle(status)) 282 ioat_suspend(ioat_chan); 283 while (is_ioat_active(status) || is_ioat_idle(status)) { 284 if (tmo && time_after(jiffies, end)) { 285 err = -ETIMEDOUT; 286 break; 287 } 288 status = ioat_chansts(ioat_chan); 289 cpu_relax(); 290 } 291 292 return err; 293 } 294 295 static int ioat_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo) 296 { 297 unsigned long end = jiffies + tmo; 298 int err = 0; 299 300 ioat_reset(ioat_chan); 301 while (ioat_reset_pending(ioat_chan)) { 302 if (end && time_after(jiffies, end)) { 303 err = -ETIMEDOUT; 304 break; 305 } 306 cpu_relax(); 307 } 308 309 return err; 310 } 311 312 static dma_cookie_t ioat_tx_submit_unlock(struct dma_async_tx_descriptor *tx) 313 __releases(&ioat_chan->prep_lock) 314 { 315 struct dma_chan *c = tx->chan; 316 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); 317 dma_cookie_t cookie; 318 319 cookie = dma_cookie_assign(tx); 320 dev_dbg(to_dev(ioat_chan), "%s: cookie: %d\n", __func__, cookie); 321 322 if (!test_and_set_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state)) 323 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); 324 325 /* make descriptor updates visible before advancing ioat->head, 326 * this is purposefully not smp_wmb() since we are also 327 * publishing the descriptor updates to a dma device 328 */ 329 wmb(); 330 331 ioat_chan->head += ioat_chan->produce; 332 333 ioat_update_pending(ioat_chan); 334 spin_unlock_bh(&ioat_chan->prep_lock); 335 336 return cookie; 337 } 338 339 static struct ioat_ring_ent * 340 ioat_alloc_ring_ent(struct dma_chan *chan, int idx, gfp_t flags) 341 { 342 struct ioat_dma_descriptor *hw; 343 struct ioat_ring_ent *desc; 344 struct ioatdma_device *ioat_dma; 345 struct ioatdma_chan *ioat_chan = to_ioat_chan(chan); 346 int chunk; 347 dma_addr_t phys; 348 u8 *pos; 349 off_t offs; 350 351 ioat_dma = to_ioatdma_device(chan->device); 352 353 chunk = idx / IOAT_DESCS_PER_2M; 354 idx &= (IOAT_DESCS_PER_2M - 1); 355 offs = idx * IOAT_DESC_SZ; 356 pos = (u8 *)ioat_chan->descs[chunk].virt + offs; 357 phys = ioat_chan->descs[chunk].hw + offs; 358 hw = (struct ioat_dma_descriptor *)pos; 359 memset(hw, 0, sizeof(*hw)); 360 361 desc = kmem_cache_zalloc(ioat_cache, flags); 362 if (!desc) 363 return NULL; 364 365 dma_async_tx_descriptor_init(&desc->txd, chan); 366 desc->txd.tx_submit = ioat_tx_submit_unlock; 367 desc->hw = hw; 368 desc->txd.phys = phys; 369 return desc; 370 } 371 372 void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan) 373 { 374 kmem_cache_free(ioat_cache, desc); 375 } 376 377 struct ioat_ring_ent ** 378 ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags) 379 { 380 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); 381 struct ioat_ring_ent **ring; 382 int total_descs = 1 << order; 383 int i, chunks; 384 385 /* allocate the array to hold the software ring */ 386 ring = kcalloc(total_descs, sizeof(*ring), flags); 387 if (!ring) 388 return NULL; 389 390 ioat_chan->desc_chunks = chunks = (total_descs * IOAT_DESC_SZ) / SZ_2M; 391 392 for (i = 0; i < chunks; i++) { 393 struct ioat_descs *descs = &ioat_chan->descs[i]; 394 395 descs->virt = dma_alloc_coherent(to_dev(ioat_chan), 396 SZ_2M, &descs->hw, flags); 397 if (!descs->virt && (i > 0)) { 398 int idx; 399 400 for (idx = 0; idx < i; idx++) { 401 dma_free_coherent(to_dev(ioat_chan), SZ_2M, 402 descs->virt, descs->hw); 403 descs->virt = NULL; 404 descs->hw = 0; 405 } 406 407 ioat_chan->desc_chunks = 0; 408 kfree(ring); 409 return NULL; 410 } 411 } 412 413 for (i = 0; i < total_descs; i++) { 414 ring[i] = ioat_alloc_ring_ent(c, i, flags); 415 if (!ring[i]) { 416 int idx; 417 418 while (i--) 419 ioat_free_ring_ent(ring[i], c); 420 421 for (idx = 0; idx < ioat_chan->desc_chunks; idx++) { 422 dma_free_coherent(to_dev(ioat_chan), 423 SZ_2M, 424 ioat_chan->descs[idx].virt, 425 ioat_chan->descs[idx].hw); 426 ioat_chan->descs[idx].virt = NULL; 427 ioat_chan->descs[idx].hw = 0; 428 } 429 430 ioat_chan->desc_chunks = 0; 431 kfree(ring); 432 return NULL; 433 } 434 set_desc_id(ring[i], i); 435 } 436 437 /* link descs */ 438 for (i = 0; i < total_descs-1; i++) { 439 struct ioat_ring_ent *next = ring[i+1]; 440 struct ioat_dma_descriptor *hw = ring[i]->hw; 441 442 hw->next = next->txd.phys; 443 } 444 ring[i]->hw->next = ring[0]->txd.phys; 445 446 return ring; 447 } 448 449 /** 450 * ioat_check_space_lock - verify space and grab ring producer lock 451 * @ioat: ioat,3 channel (ring) to operate on 452 * @num_descs: allocation length 453 */ 454 int ioat_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs) 455 __acquires(&ioat_chan->prep_lock) 456 { 457 spin_lock_bh(&ioat_chan->prep_lock); 458 /* never allow the last descriptor to be consumed, we need at 459 * least one free at all times to allow for on-the-fly ring 460 * resizing. 461 */ 462 if (likely(ioat_ring_space(ioat_chan) > num_descs)) { 463 dev_dbg(to_dev(ioat_chan), "%s: num_descs: %d (%x:%x:%x)\n", 464 __func__, num_descs, ioat_chan->head, 465 ioat_chan->tail, ioat_chan->issued); 466 ioat_chan->produce = num_descs; 467 return 0; /* with ioat->prep_lock held */ 468 } 469 spin_unlock_bh(&ioat_chan->prep_lock); 470 471 dev_dbg_ratelimited(to_dev(ioat_chan), 472 "%s: ring full! num_descs: %d (%x:%x:%x)\n", 473 __func__, num_descs, ioat_chan->head, 474 ioat_chan->tail, ioat_chan->issued); 475 476 /* progress reclaim in the allocation failure case we may be 477 * called under bh_disabled so we need to trigger the timer 478 * event directly 479 */ 480 if (time_is_before_jiffies(ioat_chan->timer.expires) 481 && timer_pending(&ioat_chan->timer)) { 482 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); 483 ioat_timer_event((unsigned long)ioat_chan); 484 } 485 486 return -ENOMEM; 487 } 488 489 static bool desc_has_ext(struct ioat_ring_ent *desc) 490 { 491 struct ioat_dma_descriptor *hw = desc->hw; 492 493 if (hw->ctl_f.op == IOAT_OP_XOR || 494 hw->ctl_f.op == IOAT_OP_XOR_VAL) { 495 struct ioat_xor_descriptor *xor = desc->xor; 496 497 if (src_cnt_to_sw(xor->ctl_f.src_cnt) > 5) 498 return true; 499 } else if (hw->ctl_f.op == IOAT_OP_PQ || 500 hw->ctl_f.op == IOAT_OP_PQ_VAL) { 501 struct ioat_pq_descriptor *pq = desc->pq; 502 503 if (src_cnt_to_sw(pq->ctl_f.src_cnt) > 3) 504 return true; 505 } 506 507 return false; 508 } 509 510 static void 511 ioat_free_sed(struct ioatdma_device *ioat_dma, struct ioat_sed_ent *sed) 512 { 513 if (!sed) 514 return; 515 516 dma_pool_free(ioat_dma->sed_hw_pool[sed->hw_pool], sed->hw, sed->dma); 517 kmem_cache_free(ioat_sed_cache, sed); 518 } 519 520 static u64 ioat_get_current_completion(struct ioatdma_chan *ioat_chan) 521 { 522 u64 phys_complete; 523 u64 completion; 524 525 completion = *ioat_chan->completion; 526 phys_complete = ioat_chansts_to_addr(completion); 527 528 dev_dbg(to_dev(ioat_chan), "%s: phys_complete: %#llx\n", __func__, 529 (unsigned long long) phys_complete); 530 531 return phys_complete; 532 } 533 534 static bool ioat_cleanup_preamble(struct ioatdma_chan *ioat_chan, 535 u64 *phys_complete) 536 { 537 *phys_complete = ioat_get_current_completion(ioat_chan); 538 if (*phys_complete == ioat_chan->last_completion) 539 return false; 540 541 clear_bit(IOAT_COMPLETION_ACK, &ioat_chan->state); 542 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); 543 544 return true; 545 } 546 547 static void 548 desc_get_errstat(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc) 549 { 550 struct ioat_dma_descriptor *hw = desc->hw; 551 552 switch (hw->ctl_f.op) { 553 case IOAT_OP_PQ_VAL: 554 case IOAT_OP_PQ_VAL_16S: 555 { 556 struct ioat_pq_descriptor *pq = desc->pq; 557 558 /* check if there's error written */ 559 if (!pq->dwbes_f.wbes) 560 return; 561 562 /* need to set a chanerr var for checking to clear later */ 563 564 if (pq->dwbes_f.p_val_err) 565 *desc->result |= SUM_CHECK_P_RESULT; 566 567 if (pq->dwbes_f.q_val_err) 568 *desc->result |= SUM_CHECK_Q_RESULT; 569 570 return; 571 } 572 default: 573 return; 574 } 575 } 576 577 /** 578 * __cleanup - reclaim used descriptors 579 * @ioat: channel (ring) to clean 580 */ 581 static void __cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete) 582 { 583 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; 584 struct ioat_ring_ent *desc; 585 bool seen_current = false; 586 int idx = ioat_chan->tail, i; 587 u16 active; 588 589 dev_dbg(to_dev(ioat_chan), "%s: head: %#x tail: %#x issued: %#x\n", 590 __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued); 591 592 /* 593 * At restart of the channel, the completion address and the 594 * channel status will be 0 due to starting a new chain. Since 595 * it's new chain and the first descriptor "fails", there is 596 * nothing to clean up. We do not want to reap the entire submitted 597 * chain due to this 0 address value and then BUG. 598 */ 599 if (!phys_complete) 600 return; 601 602 active = ioat_ring_active(ioat_chan); 603 for (i = 0; i < active && !seen_current; i++) { 604 struct dma_async_tx_descriptor *tx; 605 606 smp_read_barrier_depends(); 607 prefetch(ioat_get_ring_ent(ioat_chan, idx + i + 1)); 608 desc = ioat_get_ring_ent(ioat_chan, idx + i); 609 dump_desc_dbg(ioat_chan, desc); 610 611 /* set err stat if we are using dwbes */ 612 if (ioat_dma->cap & IOAT_CAP_DWBES) 613 desc_get_errstat(ioat_chan, desc); 614 615 tx = &desc->txd; 616 if (tx->cookie) { 617 struct dmaengine_result res; 618 619 dma_cookie_complete(tx); 620 dma_descriptor_unmap(tx); 621 res.result = DMA_TRANS_NOERROR; 622 dmaengine_desc_get_callback_invoke(tx, NULL); 623 tx->callback = NULL; 624 tx->callback_result = NULL; 625 } 626 627 if (tx->phys == phys_complete) 628 seen_current = true; 629 630 /* skip extended descriptors */ 631 if (desc_has_ext(desc)) { 632 BUG_ON(i + 1 >= active); 633 i++; 634 } 635 636 /* cleanup super extended descriptors */ 637 if (desc->sed) { 638 ioat_free_sed(ioat_dma, desc->sed); 639 desc->sed = NULL; 640 } 641 } 642 643 /* finish all descriptor reads before incrementing tail */ 644 smp_mb(); 645 ioat_chan->tail = idx + i; 646 /* no active descs have written a completion? */ 647 BUG_ON(active && !seen_current); 648 ioat_chan->last_completion = phys_complete; 649 650 if (active - i == 0) { 651 dev_dbg(to_dev(ioat_chan), "%s: cancel completion timeout\n", 652 __func__); 653 mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); 654 } 655 656 /* 5 microsecond delay per pending descriptor */ 657 writew(min((5 * (active - i)), IOAT_INTRDELAY_MASK), 658 ioat_chan->ioat_dma->reg_base + IOAT_INTRDELAY_OFFSET); 659 } 660 661 static void ioat_cleanup(struct ioatdma_chan *ioat_chan) 662 { 663 u64 phys_complete; 664 665 spin_lock_bh(&ioat_chan->cleanup_lock); 666 667 if (ioat_cleanup_preamble(ioat_chan, &phys_complete)) 668 __cleanup(ioat_chan, phys_complete); 669 670 if (is_ioat_halted(*ioat_chan->completion)) { 671 u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); 672 673 if (chanerr & 674 (IOAT_CHANERR_HANDLE_MASK | IOAT_CHANERR_RECOVER_MASK)) { 675 mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); 676 ioat_eh(ioat_chan); 677 } 678 } 679 680 spin_unlock_bh(&ioat_chan->cleanup_lock); 681 } 682 683 void ioat_cleanup_event(unsigned long data) 684 { 685 struct ioatdma_chan *ioat_chan = to_ioat_chan((void *)data); 686 687 ioat_cleanup(ioat_chan); 688 if (!test_bit(IOAT_RUN, &ioat_chan->state)) 689 return; 690 writew(IOAT_CHANCTRL_RUN, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); 691 } 692 693 static void ioat_restart_channel(struct ioatdma_chan *ioat_chan) 694 { 695 u64 phys_complete; 696 697 ioat_quiesce(ioat_chan, 0); 698 if (ioat_cleanup_preamble(ioat_chan, &phys_complete)) 699 __cleanup(ioat_chan, phys_complete); 700 701 __ioat_restart_chan(ioat_chan); 702 } 703 704 705 static void ioat_abort_descs(struct ioatdma_chan *ioat_chan) 706 { 707 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; 708 struct ioat_ring_ent *desc; 709 u16 active; 710 int idx = ioat_chan->tail, i; 711 712 /* 713 * We assume that the failed descriptor has been processed. 714 * Now we are just returning all the remaining submitted 715 * descriptors to abort. 716 */ 717 active = ioat_ring_active(ioat_chan); 718 719 /* we skip the failed descriptor that tail points to */ 720 for (i = 1; i < active; i++) { 721 struct dma_async_tx_descriptor *tx; 722 723 smp_read_barrier_depends(); 724 prefetch(ioat_get_ring_ent(ioat_chan, idx + i + 1)); 725 desc = ioat_get_ring_ent(ioat_chan, idx + i); 726 727 tx = &desc->txd; 728 if (tx->cookie) { 729 struct dmaengine_result res; 730 731 dma_cookie_complete(tx); 732 dma_descriptor_unmap(tx); 733 res.result = DMA_TRANS_ABORTED; 734 dmaengine_desc_get_callback_invoke(tx, &res); 735 tx->callback = NULL; 736 tx->callback_result = NULL; 737 } 738 739 /* skip extended descriptors */ 740 if (desc_has_ext(desc)) { 741 WARN_ON(i + 1 >= active); 742 i++; 743 } 744 745 /* cleanup super extended descriptors */ 746 if (desc->sed) { 747 ioat_free_sed(ioat_dma, desc->sed); 748 desc->sed = NULL; 749 } 750 } 751 752 smp_mb(); /* finish all descriptor reads before incrementing tail */ 753 ioat_chan->tail = idx + active; 754 755 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail); 756 ioat_chan->last_completion = *ioat_chan->completion = desc->txd.phys; 757 } 758 759 static void ioat_eh(struct ioatdma_chan *ioat_chan) 760 { 761 struct pci_dev *pdev = to_pdev(ioat_chan); 762 struct ioat_dma_descriptor *hw; 763 struct dma_async_tx_descriptor *tx; 764 u64 phys_complete; 765 struct ioat_ring_ent *desc; 766 u32 err_handled = 0; 767 u32 chanerr_int; 768 u32 chanerr; 769 bool abort = false; 770 struct dmaengine_result res; 771 772 /* cleanup so tail points to descriptor that caused the error */ 773 if (ioat_cleanup_preamble(ioat_chan, &phys_complete)) 774 __cleanup(ioat_chan, phys_complete); 775 776 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); 777 pci_read_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, &chanerr_int); 778 779 dev_dbg(to_dev(ioat_chan), "%s: error = %x:%x\n", 780 __func__, chanerr, chanerr_int); 781 782 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail); 783 hw = desc->hw; 784 dump_desc_dbg(ioat_chan, desc); 785 786 switch (hw->ctl_f.op) { 787 case IOAT_OP_XOR_VAL: 788 if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) { 789 *desc->result |= SUM_CHECK_P_RESULT; 790 err_handled |= IOAT_CHANERR_XOR_P_OR_CRC_ERR; 791 } 792 break; 793 case IOAT_OP_PQ_VAL: 794 case IOAT_OP_PQ_VAL_16S: 795 if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) { 796 *desc->result |= SUM_CHECK_P_RESULT; 797 err_handled |= IOAT_CHANERR_XOR_P_OR_CRC_ERR; 798 } 799 if (chanerr & IOAT_CHANERR_XOR_Q_ERR) { 800 *desc->result |= SUM_CHECK_Q_RESULT; 801 err_handled |= IOAT_CHANERR_XOR_Q_ERR; 802 } 803 break; 804 } 805 806 if (chanerr & IOAT_CHANERR_RECOVER_MASK) { 807 if (chanerr & IOAT_CHANERR_READ_DATA_ERR) { 808 res.result = DMA_TRANS_READ_FAILED; 809 err_handled |= IOAT_CHANERR_READ_DATA_ERR; 810 } else if (chanerr & IOAT_CHANERR_WRITE_DATA_ERR) { 811 res.result = DMA_TRANS_WRITE_FAILED; 812 err_handled |= IOAT_CHANERR_WRITE_DATA_ERR; 813 } 814 815 abort = true; 816 } else 817 res.result = DMA_TRANS_NOERROR; 818 819 /* fault on unhandled error or spurious halt */ 820 if (chanerr ^ err_handled || chanerr == 0) { 821 dev_err(to_dev(ioat_chan), "%s: fatal error (%x:%x)\n", 822 __func__, chanerr, err_handled); 823 dev_err(to_dev(ioat_chan), "Errors handled:\n"); 824 ioat_print_chanerrs(ioat_chan, err_handled); 825 dev_err(to_dev(ioat_chan), "Errors not handled:\n"); 826 ioat_print_chanerrs(ioat_chan, (chanerr & ~err_handled)); 827 828 BUG(); 829 } 830 831 /* cleanup the faulty descriptor since we are continuing */ 832 tx = &desc->txd; 833 if (tx->cookie) { 834 dma_cookie_complete(tx); 835 dma_descriptor_unmap(tx); 836 dmaengine_desc_get_callback_invoke(tx, &res); 837 tx->callback = NULL; 838 tx->callback_result = NULL; 839 } 840 841 /* mark faulting descriptor as complete */ 842 *ioat_chan->completion = desc->txd.phys; 843 844 spin_lock_bh(&ioat_chan->prep_lock); 845 /* we need abort all descriptors */ 846 if (abort) { 847 ioat_abort_descs(ioat_chan); 848 /* clean up the channel, we could be in weird state */ 849 ioat_reset_hw(ioat_chan); 850 } 851 852 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET); 853 pci_write_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, chanerr_int); 854 855 ioat_restart_channel(ioat_chan); 856 spin_unlock_bh(&ioat_chan->prep_lock); 857 } 858 859 static void check_active(struct ioatdma_chan *ioat_chan) 860 { 861 if (ioat_ring_active(ioat_chan)) { 862 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); 863 return; 864 } 865 866 if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state)) 867 mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); 868 } 869 870 void ioat_timer_event(unsigned long data) 871 { 872 struct ioatdma_chan *ioat_chan = to_ioat_chan((void *)data); 873 dma_addr_t phys_complete; 874 u64 status; 875 876 status = ioat_chansts(ioat_chan); 877 878 /* when halted due to errors check for channel 879 * programming errors before advancing the completion state 880 */ 881 if (is_ioat_halted(status)) { 882 u32 chanerr; 883 884 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); 885 dev_err(to_dev(ioat_chan), "%s: Channel halted (%x)\n", 886 __func__, chanerr); 887 dev_err(to_dev(ioat_chan), "Errors:\n"); 888 ioat_print_chanerrs(ioat_chan, chanerr); 889 890 if (test_bit(IOAT_RUN, &ioat_chan->state)) { 891 spin_lock_bh(&ioat_chan->cleanup_lock); 892 spin_lock_bh(&ioat_chan->prep_lock); 893 set_bit(IOAT_CHAN_DOWN, &ioat_chan->state); 894 spin_unlock_bh(&ioat_chan->prep_lock); 895 896 ioat_abort_descs(ioat_chan); 897 dev_warn(to_dev(ioat_chan), "Reset channel...\n"); 898 ioat_reset_hw(ioat_chan); 899 dev_warn(to_dev(ioat_chan), "Restart channel...\n"); 900 ioat_restart_channel(ioat_chan); 901 902 spin_lock_bh(&ioat_chan->prep_lock); 903 clear_bit(IOAT_CHAN_DOWN, &ioat_chan->state); 904 spin_unlock_bh(&ioat_chan->prep_lock); 905 spin_unlock_bh(&ioat_chan->cleanup_lock); 906 } 907 908 return; 909 } 910 911 spin_lock_bh(&ioat_chan->cleanup_lock); 912 913 /* handle the no-actives case */ 914 if (!ioat_ring_active(ioat_chan)) { 915 spin_lock_bh(&ioat_chan->prep_lock); 916 check_active(ioat_chan); 917 spin_unlock_bh(&ioat_chan->prep_lock); 918 spin_unlock_bh(&ioat_chan->cleanup_lock); 919 return; 920 } 921 922 /* if we haven't made progress and we have already 923 * acknowledged a pending completion once, then be more 924 * forceful with a restart 925 */ 926 if (ioat_cleanup_preamble(ioat_chan, &phys_complete)) 927 __cleanup(ioat_chan, phys_complete); 928 else if (test_bit(IOAT_COMPLETION_ACK, &ioat_chan->state)) { 929 u32 chanerr; 930 931 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); 932 dev_err(to_dev(ioat_chan), "CHANSTS: %#Lx CHANERR: %#x\n", 933 status, chanerr); 934 dev_err(to_dev(ioat_chan), "Errors:\n"); 935 ioat_print_chanerrs(ioat_chan, chanerr); 936 937 dev_dbg(to_dev(ioat_chan), "Active descriptors: %d\n", 938 ioat_ring_active(ioat_chan)); 939 940 spin_lock_bh(&ioat_chan->prep_lock); 941 set_bit(IOAT_CHAN_DOWN, &ioat_chan->state); 942 spin_unlock_bh(&ioat_chan->prep_lock); 943 944 ioat_abort_descs(ioat_chan); 945 dev_warn(to_dev(ioat_chan), "Resetting channel...\n"); 946 ioat_reset_hw(ioat_chan); 947 dev_warn(to_dev(ioat_chan), "Restarting channel...\n"); 948 ioat_restart_channel(ioat_chan); 949 950 spin_lock_bh(&ioat_chan->prep_lock); 951 clear_bit(IOAT_CHAN_DOWN, &ioat_chan->state); 952 spin_unlock_bh(&ioat_chan->prep_lock); 953 spin_unlock_bh(&ioat_chan->cleanup_lock); 954 return; 955 } else 956 set_bit(IOAT_COMPLETION_ACK, &ioat_chan->state); 957 958 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); 959 spin_unlock_bh(&ioat_chan->cleanup_lock); 960 } 961 962 enum dma_status 963 ioat_tx_status(struct dma_chan *c, dma_cookie_t cookie, 964 struct dma_tx_state *txstate) 965 { 966 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); 967 enum dma_status ret; 968 969 ret = dma_cookie_status(c, cookie, txstate); 970 if (ret == DMA_COMPLETE) 971 return ret; 972 973 ioat_cleanup(ioat_chan); 974 975 return dma_cookie_status(c, cookie, txstate); 976 } 977 978 int ioat_reset_hw(struct ioatdma_chan *ioat_chan) 979 { 980 /* throw away whatever the channel was doing and get it 981 * initialized, with ioat3 specific workarounds 982 */ 983 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; 984 struct pci_dev *pdev = ioat_dma->pdev; 985 u32 chanerr; 986 u16 dev_id; 987 int err; 988 989 ioat_quiesce(ioat_chan, msecs_to_jiffies(100)); 990 991 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); 992 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET); 993 994 if (ioat_dma->version < IOAT_VER_3_3) { 995 /* clear any pending errors */ 996 err = pci_read_config_dword(pdev, 997 IOAT_PCI_CHANERR_INT_OFFSET, &chanerr); 998 if (err) { 999 dev_err(&pdev->dev, 1000 "channel error register unreachable\n"); 1001 return err; 1002 } 1003 pci_write_config_dword(pdev, 1004 IOAT_PCI_CHANERR_INT_OFFSET, chanerr); 1005 1006 /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit 1007 * (workaround for spurious config parity error after restart) 1008 */ 1009 pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id); 1010 if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0) { 1011 pci_write_config_dword(pdev, 1012 IOAT_PCI_DMAUNCERRSTS_OFFSET, 1013 0x10); 1014 } 1015 } 1016 1017 if (is_bwd_ioat(pdev) && (ioat_dma->irq_mode == IOAT_MSIX)) { 1018 ioat_dma->msixtba0 = readq(ioat_dma->reg_base + 0x1000); 1019 ioat_dma->msixdata0 = readq(ioat_dma->reg_base + 0x1008); 1020 ioat_dma->msixpba = readq(ioat_dma->reg_base + 0x1800); 1021 } 1022 1023 1024 err = ioat_reset_sync(ioat_chan, msecs_to_jiffies(200)); 1025 if (!err) { 1026 if (is_bwd_ioat(pdev) && (ioat_dma->irq_mode == IOAT_MSIX)) { 1027 writeq(ioat_dma->msixtba0, ioat_dma->reg_base + 0x1000); 1028 writeq(ioat_dma->msixdata0, ioat_dma->reg_base + 0x1008); 1029 writeq(ioat_dma->msixpba, ioat_dma->reg_base + 0x1800); 1030 } 1031 } 1032 1033 if (err) 1034 dev_err(&pdev->dev, "Failed to reset: %d\n", err); 1035 1036 return err; 1037 } 1038