1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause 2 /* 3 * Copyright(c) 2015 - 2018 Intel Corporation. 4 */ 5 6 #include <linux/spinlock.h> 7 #include <linux/seqlock.h> 8 #include <linux/netdevice.h> 9 #include <linux/moduleparam.h> 10 #include <linux/bitops.h> 11 #include <linux/timer.h> 12 #include <linux/vmalloc.h> 13 #include <linux/highmem.h> 14 15 #include "hfi.h" 16 #include "common.h" 17 #include "qp.h" 18 #include "sdma.h" 19 #include "iowait.h" 20 #include "trace.h" 21 22 /* must be a power of 2 >= 64 <= 32768 */ 23 #define SDMA_DESCQ_CNT 2048 24 #define SDMA_DESC_INTR 64 25 #define INVALID_TAIL 0xffff 26 #define SDMA_PAD max_t(size_t, MAX_16B_PADDING, sizeof(u32)) 27 28 static uint sdma_descq_cnt = SDMA_DESCQ_CNT; 29 module_param(sdma_descq_cnt, uint, S_IRUGO); 30 MODULE_PARM_DESC(sdma_descq_cnt, "Number of SDMA descq entries"); 31 32 static uint sdma_idle_cnt = 250; 33 module_param(sdma_idle_cnt, uint, S_IRUGO); 34 MODULE_PARM_DESC(sdma_idle_cnt, "sdma interrupt idle delay (ns,default 250)"); 35 36 uint mod_num_sdma; 37 module_param_named(num_sdma, mod_num_sdma, uint, S_IRUGO); 38 MODULE_PARM_DESC(num_sdma, "Set max number SDMA engines to use"); 39 40 static uint sdma_desct_intr = SDMA_DESC_INTR; 41 module_param_named(desct_intr, sdma_desct_intr, uint, S_IRUGO | S_IWUSR); 42 MODULE_PARM_DESC(desct_intr, "Number of SDMA descriptor before interrupt"); 43 44 #define SDMA_WAIT_BATCH_SIZE 20 45 /* max wait time for a SDMA engine to indicate it has halted */ 46 #define SDMA_ERR_HALT_TIMEOUT 10 /* ms */ 47 /* all SDMA engine errors that cause a halt */ 48 49 #define SD(name) SEND_DMA_##name 50 #define ALL_SDMA_ENG_HALT_ERRS \ 51 (SD(ENG_ERR_STATUS_SDMA_WRONG_DW_ERR_SMASK) \ 52 | SD(ENG_ERR_STATUS_SDMA_GEN_MISMATCH_ERR_SMASK) \ 53 | SD(ENG_ERR_STATUS_SDMA_TOO_LONG_ERR_SMASK) \ 54 | SD(ENG_ERR_STATUS_SDMA_TAIL_OUT_OF_BOUNDS_ERR_SMASK) \ 55 | SD(ENG_ERR_STATUS_SDMA_FIRST_DESC_ERR_SMASK) \ 56 | SD(ENG_ERR_STATUS_SDMA_MEM_READ_ERR_SMASK) \ 57 | SD(ENG_ERR_STATUS_SDMA_HALT_ERR_SMASK) \ 58 | SD(ENG_ERR_STATUS_SDMA_LENGTH_MISMATCH_ERR_SMASK) \ 59 | SD(ENG_ERR_STATUS_SDMA_PACKET_DESC_OVERFLOW_ERR_SMASK) \ 60 | SD(ENG_ERR_STATUS_SDMA_HEADER_SELECT_ERR_SMASK) \ 61 | SD(ENG_ERR_STATUS_SDMA_HEADER_ADDRESS_ERR_SMASK) \ 62 | SD(ENG_ERR_STATUS_SDMA_HEADER_LENGTH_ERR_SMASK) \ 63 | SD(ENG_ERR_STATUS_SDMA_TIMEOUT_ERR_SMASK) \ 64 | SD(ENG_ERR_STATUS_SDMA_DESC_TABLE_UNC_ERR_SMASK) \ 65 | SD(ENG_ERR_STATUS_SDMA_ASSEMBLY_UNC_ERR_SMASK) \ 66 | SD(ENG_ERR_STATUS_SDMA_PACKET_TRACKING_UNC_ERR_SMASK) \ 67 | SD(ENG_ERR_STATUS_SDMA_HEADER_STORAGE_UNC_ERR_SMASK) \ 68 | SD(ENG_ERR_STATUS_SDMA_HEADER_REQUEST_FIFO_UNC_ERR_SMASK)) 69 70 /* sdma_sendctrl operations */ 71 #define SDMA_SENDCTRL_OP_ENABLE BIT(0) 72 #define SDMA_SENDCTRL_OP_INTENABLE BIT(1) 73 #define SDMA_SENDCTRL_OP_HALT BIT(2) 74 #define SDMA_SENDCTRL_OP_CLEANUP BIT(3) 75 76 /* handle long defines */ 77 #define SDMA_EGRESS_PACKET_OCCUPANCY_SMASK \ 78 SEND_EGRESS_SEND_DMA_STATUS_SDMA_EGRESS_PACKET_OCCUPANCY_SMASK 79 #define SDMA_EGRESS_PACKET_OCCUPANCY_SHIFT \ 80 SEND_EGRESS_SEND_DMA_STATUS_SDMA_EGRESS_PACKET_OCCUPANCY_SHIFT 81 82 static const char * const sdma_state_names[] = { 83 [sdma_state_s00_hw_down] = "s00_HwDown", 84 [sdma_state_s10_hw_start_up_halt_wait] = "s10_HwStartUpHaltWait", 85 [sdma_state_s15_hw_start_up_clean_wait] = "s15_HwStartUpCleanWait", 86 [sdma_state_s20_idle] = "s20_Idle", 87 [sdma_state_s30_sw_clean_up_wait] = "s30_SwCleanUpWait", 88 [sdma_state_s40_hw_clean_up_wait] = "s40_HwCleanUpWait", 89 [sdma_state_s50_hw_halt_wait] = "s50_HwHaltWait", 90 [sdma_state_s60_idle_halt_wait] = "s60_IdleHaltWait", 91 [sdma_state_s80_hw_freeze] = "s80_HwFreeze", 92 [sdma_state_s82_freeze_sw_clean] = "s82_FreezeSwClean", 93 [sdma_state_s99_running] = "s99_Running", 94 }; 95 96 #ifdef CONFIG_SDMA_VERBOSITY 97 static const char * const sdma_event_names[] = { 98 [sdma_event_e00_go_hw_down] = "e00_GoHwDown", 99 [sdma_event_e10_go_hw_start] = "e10_GoHwStart", 100 [sdma_event_e15_hw_halt_done] = "e15_HwHaltDone", 101 [sdma_event_e25_hw_clean_up_done] = "e25_HwCleanUpDone", 102 [sdma_event_e30_go_running] = "e30_GoRunning", 103 [sdma_event_e40_sw_cleaned] = "e40_SwCleaned", 104 [sdma_event_e50_hw_cleaned] = "e50_HwCleaned", 105 [sdma_event_e60_hw_halted] = "e60_HwHalted", 106 [sdma_event_e70_go_idle] = "e70_GoIdle", 107 [sdma_event_e80_hw_freeze] = "e80_HwFreeze", 108 [sdma_event_e81_hw_frozen] = "e81_HwFrozen", 109 [sdma_event_e82_hw_unfreeze] = "e82_HwUnfreeze", 110 [sdma_event_e85_link_down] = "e85_LinkDown", 111 [sdma_event_e90_sw_halted] = "e90_SwHalted", 112 }; 113 #endif 114 115 static const struct sdma_set_state_action sdma_action_table[] = { 116 [sdma_state_s00_hw_down] = { 117 .go_s99_running_tofalse = 1, 118 .op_enable = 0, 119 .op_intenable = 0, 120 .op_halt = 0, 121 .op_cleanup = 0, 122 }, 123 [sdma_state_s10_hw_start_up_halt_wait] = { 124 .op_enable = 0, 125 .op_intenable = 0, 126 .op_halt = 1, 127 .op_cleanup = 0, 128 }, 129 [sdma_state_s15_hw_start_up_clean_wait] = { 130 .op_enable = 0, 131 .op_intenable = 1, 132 .op_halt = 0, 133 .op_cleanup = 1, 134 }, 135 [sdma_state_s20_idle] = { 136 .op_enable = 0, 137 .op_intenable = 1, 138 .op_halt = 0, 139 .op_cleanup = 0, 140 }, 141 [sdma_state_s30_sw_clean_up_wait] = { 142 .op_enable = 0, 143 .op_intenable = 0, 144 .op_halt = 0, 145 .op_cleanup = 0, 146 }, 147 [sdma_state_s40_hw_clean_up_wait] = { 148 .op_enable = 0, 149 .op_intenable = 0, 150 .op_halt = 0, 151 .op_cleanup = 1, 152 }, 153 [sdma_state_s50_hw_halt_wait] = { 154 .op_enable = 0, 155 .op_intenable = 0, 156 .op_halt = 0, 157 .op_cleanup = 0, 158 }, 159 [sdma_state_s60_idle_halt_wait] = { 160 .go_s99_running_tofalse = 1, 161 .op_enable = 0, 162 .op_intenable = 0, 163 .op_halt = 1, 164 .op_cleanup = 0, 165 }, 166 [sdma_state_s80_hw_freeze] = { 167 .op_enable = 0, 168 .op_intenable = 0, 169 .op_halt = 0, 170 .op_cleanup = 0, 171 }, 172 [sdma_state_s82_freeze_sw_clean] = { 173 .op_enable = 0, 174 .op_intenable = 0, 175 .op_halt = 0, 176 .op_cleanup = 0, 177 }, 178 [sdma_state_s99_running] = { 179 .op_enable = 1, 180 .op_intenable = 1, 181 .op_halt = 0, 182 .op_cleanup = 0, 183 .go_s99_running_totrue = 1, 184 }, 185 }; 186 187 #define SDMA_TAIL_UPDATE_THRESH 0x1F 188 189 /* declare all statics here rather than keep sorting */ 190 static void sdma_complete(struct kref *); 191 static void sdma_finalput(struct sdma_state *); 192 static void sdma_get(struct sdma_state *); 193 static void sdma_hw_clean_up_task(struct tasklet_struct *); 194 static void sdma_put(struct sdma_state *); 195 static void sdma_set_state(struct sdma_engine *, enum sdma_states); 196 static void sdma_start_hw_clean_up(struct sdma_engine *); 197 static void sdma_sw_clean_up_task(struct tasklet_struct *); 198 static void sdma_sendctrl(struct sdma_engine *, unsigned); 199 static void init_sdma_regs(struct sdma_engine *, u32, uint); 200 static void sdma_process_event( 201 struct sdma_engine *sde, 202 enum sdma_events event); 203 static void __sdma_process_event( 204 struct sdma_engine *sde, 205 enum sdma_events event); 206 static void dump_sdma_state(struct sdma_engine *sde); 207 static void sdma_make_progress(struct sdma_engine *sde, u64 status); 208 static void sdma_desc_avail(struct sdma_engine *sde, uint avail); 209 static void sdma_flush_descq(struct sdma_engine *sde); 210 211 /** 212 * sdma_state_name() - return state string from enum 213 * @state: state 214 */ 215 static const char *sdma_state_name(enum sdma_states state) 216 { 217 return sdma_state_names[state]; 218 } 219 220 static void sdma_get(struct sdma_state *ss) 221 { 222 kref_get(&ss->kref); 223 } 224 225 static void sdma_complete(struct kref *kref) 226 { 227 struct sdma_state *ss = 228 container_of(kref, struct sdma_state, kref); 229 230 complete(&ss->comp); 231 } 232 233 static void sdma_put(struct sdma_state *ss) 234 { 235 kref_put(&ss->kref, sdma_complete); 236 } 237 238 static void sdma_finalput(struct sdma_state *ss) 239 { 240 sdma_put(ss); 241 wait_for_completion(&ss->comp); 242 } 243 244 static inline void write_sde_csr( 245 struct sdma_engine *sde, 246 u32 offset0, 247 u64 value) 248 { 249 write_kctxt_csr(sde->dd, sde->this_idx, offset0, value); 250 } 251 252 static inline u64 read_sde_csr( 253 struct sdma_engine *sde, 254 u32 offset0) 255 { 256 return read_kctxt_csr(sde->dd, sde->this_idx, offset0); 257 } 258 259 /* 260 * sdma_wait_for_packet_egress() - wait for the VL FIFO occupancy for 261 * sdma engine 'sde' to drop to 0. 262 */ 263 static void sdma_wait_for_packet_egress(struct sdma_engine *sde, 264 int pause) 265 { 266 u64 off = 8 * sde->this_idx; 267 struct hfi1_devdata *dd = sde->dd; 268 int lcnt = 0; 269 u64 reg_prev; 270 u64 reg = 0; 271 272 while (1) { 273 reg_prev = reg; 274 reg = read_csr(dd, off + SEND_EGRESS_SEND_DMA_STATUS); 275 276 reg &= SDMA_EGRESS_PACKET_OCCUPANCY_SMASK; 277 reg >>= SDMA_EGRESS_PACKET_OCCUPANCY_SHIFT; 278 if (reg == 0) 279 break; 280 /* counter is reest if accupancy count changes */ 281 if (reg != reg_prev) 282 lcnt = 0; 283 if (lcnt++ > 500) { 284 /* timed out - bounce the link */ 285 dd_dev_err(dd, "%s: engine %u timeout waiting for packets to egress, remaining count %u, bouncing link\n", 286 __func__, sde->this_idx, (u32)reg); 287 queue_work(dd->pport->link_wq, 288 &dd->pport->link_bounce_work); 289 break; 290 } 291 udelay(1); 292 } 293 } 294 295 /* 296 * sdma_wait() - wait for packet egress to complete for all SDMA engines, 297 * and pause for credit return. 298 */ 299 void sdma_wait(struct hfi1_devdata *dd) 300 { 301 int i; 302 303 for (i = 0; i < dd->num_sdma; i++) { 304 struct sdma_engine *sde = &dd->per_sdma[i]; 305 306 sdma_wait_for_packet_egress(sde, 0); 307 } 308 } 309 310 static inline void sdma_set_desc_cnt(struct sdma_engine *sde, unsigned cnt) 311 { 312 u64 reg; 313 314 if (!(sde->dd->flags & HFI1_HAS_SDMA_TIMEOUT)) 315 return; 316 reg = cnt; 317 reg &= SD(DESC_CNT_CNT_MASK); 318 reg <<= SD(DESC_CNT_CNT_SHIFT); 319 write_sde_csr(sde, SD(DESC_CNT), reg); 320 } 321 322 static inline void complete_tx(struct sdma_engine *sde, 323 struct sdma_txreq *tx, 324 int res) 325 { 326 /* protect against complete modifying */ 327 struct iowait *wait = tx->wait; 328 callback_t complete = tx->complete; 329 330 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER 331 trace_hfi1_sdma_out_sn(sde, tx->sn); 332 if (WARN_ON_ONCE(sde->head_sn != tx->sn)) 333 dd_dev_err(sde->dd, "expected %llu got %llu\n", 334 sde->head_sn, tx->sn); 335 sde->head_sn++; 336 #endif 337 __sdma_txclean(sde->dd, tx); 338 if (complete) 339 (*complete)(tx, res); 340 if (iowait_sdma_dec(wait)) 341 iowait_drain_wakeup(wait); 342 } 343 344 /* 345 * Complete all the sdma requests with a SDMA_TXREQ_S_ABORTED status 346 * 347 * Depending on timing there can be txreqs in two places: 348 * - in the descq ring 349 * - in the flush list 350 * 351 * To avoid ordering issues the descq ring needs to be flushed 352 * first followed by the flush list. 353 * 354 * This routine is called from two places 355 * - From a work queue item 356 * - Directly from the state machine just before setting the 357 * state to running 358 * 359 * Must be called with head_lock held 360 * 361 */ 362 static void sdma_flush(struct sdma_engine *sde) 363 { 364 struct sdma_txreq *txp, *txp_next; 365 LIST_HEAD(flushlist); 366 unsigned long flags; 367 uint seq; 368 369 /* flush from head to tail */ 370 sdma_flush_descq(sde); 371 spin_lock_irqsave(&sde->flushlist_lock, flags); 372 /* copy flush list */ 373 list_splice_init(&sde->flushlist, &flushlist); 374 spin_unlock_irqrestore(&sde->flushlist_lock, flags); 375 /* flush from flush list */ 376 list_for_each_entry_safe(txp, txp_next, &flushlist, list) 377 complete_tx(sde, txp, SDMA_TXREQ_S_ABORTED); 378 /* wakeup QPs orphaned on the dmawait list */ 379 do { 380 struct iowait *w, *nw; 381 382 seq = read_seqbegin(&sde->waitlock); 383 if (!list_empty(&sde->dmawait)) { 384 write_seqlock(&sde->waitlock); 385 list_for_each_entry_safe(w, nw, &sde->dmawait, list) { 386 if (w->wakeup) { 387 w->wakeup(w, SDMA_AVAIL_REASON); 388 list_del_init(&w->list); 389 } 390 } 391 write_sequnlock(&sde->waitlock); 392 } 393 } while (read_seqretry(&sde->waitlock, seq)); 394 } 395 396 /* 397 * Fields a work request for flushing the descq ring 398 * and the flush list 399 * 400 * If the engine has been brought to running during 401 * the scheduling delay, the flush is ignored, assuming 402 * that the process of bringing the engine to running 403 * would have done this flush prior to going to running. 404 * 405 */ 406 static void sdma_field_flush(struct work_struct *work) 407 { 408 unsigned long flags; 409 struct sdma_engine *sde = 410 container_of(work, struct sdma_engine, flush_worker); 411 412 write_seqlock_irqsave(&sde->head_lock, flags); 413 if (!__sdma_running(sde)) 414 sdma_flush(sde); 415 write_sequnlock_irqrestore(&sde->head_lock, flags); 416 } 417 418 static void sdma_err_halt_wait(struct work_struct *work) 419 { 420 struct sdma_engine *sde = container_of(work, struct sdma_engine, 421 err_halt_worker); 422 u64 statuscsr; 423 unsigned long timeout; 424 425 timeout = jiffies + msecs_to_jiffies(SDMA_ERR_HALT_TIMEOUT); 426 while (1) { 427 statuscsr = read_sde_csr(sde, SD(STATUS)); 428 statuscsr &= SD(STATUS_ENG_HALTED_SMASK); 429 if (statuscsr) 430 break; 431 if (time_after(jiffies, timeout)) { 432 dd_dev_err(sde->dd, 433 "SDMA engine %d - timeout waiting for engine to halt\n", 434 sde->this_idx); 435 /* 436 * Continue anyway. This could happen if there was 437 * an uncorrectable error in the wrong spot. 438 */ 439 break; 440 } 441 usleep_range(80, 120); 442 } 443 444 sdma_process_event(sde, sdma_event_e15_hw_halt_done); 445 } 446 447 static void sdma_err_progress_check_schedule(struct sdma_engine *sde) 448 { 449 if (!is_bx(sde->dd) && HFI1_CAP_IS_KSET(SDMA_AHG)) { 450 unsigned index; 451 struct hfi1_devdata *dd = sde->dd; 452 453 for (index = 0; index < dd->num_sdma; index++) { 454 struct sdma_engine *curr_sdma = &dd->per_sdma[index]; 455 456 if (curr_sdma != sde) 457 curr_sdma->progress_check_head = 458 curr_sdma->descq_head; 459 } 460 dd_dev_err(sde->dd, 461 "SDMA engine %d - check scheduled\n", 462 sde->this_idx); 463 mod_timer(&sde->err_progress_check_timer, jiffies + 10); 464 } 465 } 466 467 static void sdma_err_progress_check(struct timer_list *t) 468 { 469 unsigned index; 470 struct sdma_engine *sde = from_timer(sde, t, err_progress_check_timer); 471 472 dd_dev_err(sde->dd, "SDE progress check event\n"); 473 for (index = 0; index < sde->dd->num_sdma; index++) { 474 struct sdma_engine *curr_sde = &sde->dd->per_sdma[index]; 475 unsigned long flags; 476 477 /* check progress on each engine except the current one */ 478 if (curr_sde == sde) 479 continue; 480 /* 481 * We must lock interrupts when acquiring sde->lock, 482 * to avoid a deadlock if interrupt triggers and spins on 483 * the same lock on same CPU 484 */ 485 spin_lock_irqsave(&curr_sde->tail_lock, flags); 486 write_seqlock(&curr_sde->head_lock); 487 488 /* skip non-running queues */ 489 if (curr_sde->state.current_state != sdma_state_s99_running) { 490 write_sequnlock(&curr_sde->head_lock); 491 spin_unlock_irqrestore(&curr_sde->tail_lock, flags); 492 continue; 493 } 494 495 if ((curr_sde->descq_head != curr_sde->descq_tail) && 496 (curr_sde->descq_head == 497 curr_sde->progress_check_head)) 498 __sdma_process_event(curr_sde, 499 sdma_event_e90_sw_halted); 500 write_sequnlock(&curr_sde->head_lock); 501 spin_unlock_irqrestore(&curr_sde->tail_lock, flags); 502 } 503 schedule_work(&sde->err_halt_worker); 504 } 505 506 static void sdma_hw_clean_up_task(struct tasklet_struct *t) 507 { 508 struct sdma_engine *sde = from_tasklet(sde, t, 509 sdma_hw_clean_up_task); 510 u64 statuscsr; 511 512 while (1) { 513 #ifdef CONFIG_SDMA_VERBOSITY 514 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", 515 sde->this_idx, slashstrip(__FILE__), __LINE__, 516 __func__); 517 #endif 518 statuscsr = read_sde_csr(sde, SD(STATUS)); 519 statuscsr &= SD(STATUS_ENG_CLEANED_UP_SMASK); 520 if (statuscsr) 521 break; 522 udelay(10); 523 } 524 525 sdma_process_event(sde, sdma_event_e25_hw_clean_up_done); 526 } 527 528 static inline struct sdma_txreq *get_txhead(struct sdma_engine *sde) 529 { 530 return sde->tx_ring[sde->tx_head & sde->sdma_mask]; 531 } 532 533 /* 534 * flush ring for recovery 535 */ 536 static void sdma_flush_descq(struct sdma_engine *sde) 537 { 538 u16 head, tail; 539 int progress = 0; 540 struct sdma_txreq *txp = get_txhead(sde); 541 542 /* The reason for some of the complexity of this code is that 543 * not all descriptors have corresponding txps. So, we have to 544 * be able to skip over descs until we wander into the range of 545 * the next txp on the list. 546 */ 547 head = sde->descq_head & sde->sdma_mask; 548 tail = sde->descq_tail & sde->sdma_mask; 549 while (head != tail) { 550 /* advance head, wrap if needed */ 551 head = ++sde->descq_head & sde->sdma_mask; 552 /* if now past this txp's descs, do the callback */ 553 if (txp && txp->next_descq_idx == head) { 554 /* remove from list */ 555 sde->tx_ring[sde->tx_head++ & sde->sdma_mask] = NULL; 556 complete_tx(sde, txp, SDMA_TXREQ_S_ABORTED); 557 trace_hfi1_sdma_progress(sde, head, tail, txp); 558 txp = get_txhead(sde); 559 } 560 progress++; 561 } 562 if (progress) 563 sdma_desc_avail(sde, sdma_descq_freecnt(sde)); 564 } 565 566 static void sdma_sw_clean_up_task(struct tasklet_struct *t) 567 { 568 struct sdma_engine *sde = from_tasklet(sde, t, sdma_sw_clean_up_task); 569 unsigned long flags; 570 571 spin_lock_irqsave(&sde->tail_lock, flags); 572 write_seqlock(&sde->head_lock); 573 574 /* 575 * At this point, the following should always be true: 576 * - We are halted, so no more descriptors are getting retired. 577 * - We are not running, so no one is submitting new work. 578 * - Only we can send the e40_sw_cleaned, so we can't start 579 * running again until we say so. So, the active list and 580 * descq are ours to play with. 581 */ 582 583 /* 584 * In the error clean up sequence, software clean must be called 585 * before the hardware clean so we can use the hardware head in 586 * the progress routine. A hardware clean or SPC unfreeze will 587 * reset the hardware head. 588 * 589 * Process all retired requests. The progress routine will use the 590 * latest physical hardware head - we are not running so speed does 591 * not matter. 592 */ 593 sdma_make_progress(sde, 0); 594 595 sdma_flush(sde); 596 597 /* 598 * Reset our notion of head and tail. 599 * Note that the HW registers have been reset via an earlier 600 * clean up. 601 */ 602 sde->descq_tail = 0; 603 sde->descq_head = 0; 604 sde->desc_avail = sdma_descq_freecnt(sde); 605 *sde->head_dma = 0; 606 607 __sdma_process_event(sde, sdma_event_e40_sw_cleaned); 608 609 write_sequnlock(&sde->head_lock); 610 spin_unlock_irqrestore(&sde->tail_lock, flags); 611 } 612 613 static void sdma_sw_tear_down(struct sdma_engine *sde) 614 { 615 struct sdma_state *ss = &sde->state; 616 617 /* Releasing this reference means the state machine has stopped. */ 618 sdma_put(ss); 619 620 /* stop waiting for all unfreeze events to complete */ 621 atomic_set(&sde->dd->sdma_unfreeze_count, -1); 622 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); 623 } 624 625 static void sdma_start_hw_clean_up(struct sdma_engine *sde) 626 { 627 tasklet_hi_schedule(&sde->sdma_hw_clean_up_task); 628 } 629 630 static void sdma_set_state(struct sdma_engine *sde, 631 enum sdma_states next_state) 632 { 633 struct sdma_state *ss = &sde->state; 634 const struct sdma_set_state_action *action = sdma_action_table; 635 unsigned op = 0; 636 637 trace_hfi1_sdma_state( 638 sde, 639 sdma_state_names[ss->current_state], 640 sdma_state_names[next_state]); 641 642 /* debugging bookkeeping */ 643 ss->previous_state = ss->current_state; 644 ss->previous_op = ss->current_op; 645 ss->current_state = next_state; 646 647 if (ss->previous_state != sdma_state_s99_running && 648 next_state == sdma_state_s99_running) 649 sdma_flush(sde); 650 651 if (action[next_state].op_enable) 652 op |= SDMA_SENDCTRL_OP_ENABLE; 653 654 if (action[next_state].op_intenable) 655 op |= SDMA_SENDCTRL_OP_INTENABLE; 656 657 if (action[next_state].op_halt) 658 op |= SDMA_SENDCTRL_OP_HALT; 659 660 if (action[next_state].op_cleanup) 661 op |= SDMA_SENDCTRL_OP_CLEANUP; 662 663 if (action[next_state].go_s99_running_tofalse) 664 ss->go_s99_running = 0; 665 666 if (action[next_state].go_s99_running_totrue) 667 ss->go_s99_running = 1; 668 669 ss->current_op = op; 670 sdma_sendctrl(sde, ss->current_op); 671 } 672 673 /** 674 * sdma_get_descq_cnt() - called when device probed 675 * 676 * Return a validated descq count. 677 * 678 * This is currently only used in the verbs initialization to build the tx 679 * list. 680 * 681 * This will probably be deleted in favor of a more scalable approach to 682 * alloc tx's. 683 * 684 */ 685 u16 sdma_get_descq_cnt(void) 686 { 687 u16 count = sdma_descq_cnt; 688 689 if (!count) 690 return SDMA_DESCQ_CNT; 691 /* count must be a power of 2 greater than 64 and less than 692 * 32768. Otherwise return default. 693 */ 694 if (!is_power_of_2(count)) 695 return SDMA_DESCQ_CNT; 696 if (count < 64 || count > 32768) 697 return SDMA_DESCQ_CNT; 698 return count; 699 } 700 701 /** 702 * sdma_engine_get_vl() - return vl for a given sdma engine 703 * @sde: sdma engine 704 * 705 * This function returns the vl mapped to a given engine, or an error if 706 * the mapping can't be found. The mapping fields are protected by RCU. 707 */ 708 int sdma_engine_get_vl(struct sdma_engine *sde) 709 { 710 struct hfi1_devdata *dd = sde->dd; 711 struct sdma_vl_map *m; 712 u8 vl; 713 714 if (sde->this_idx >= TXE_NUM_SDMA_ENGINES) 715 return -EINVAL; 716 717 rcu_read_lock(); 718 m = rcu_dereference(dd->sdma_map); 719 if (unlikely(!m)) { 720 rcu_read_unlock(); 721 return -EINVAL; 722 } 723 vl = m->engine_to_vl[sde->this_idx]; 724 rcu_read_unlock(); 725 726 return vl; 727 } 728 729 /** 730 * sdma_select_engine_vl() - select sdma engine 731 * @dd: devdata 732 * @selector: a spreading factor 733 * @vl: this vl 734 * 735 * 736 * This function returns an engine based on the selector and a vl. The 737 * mapping fields are protected by RCU. 738 */ 739 struct sdma_engine *sdma_select_engine_vl( 740 struct hfi1_devdata *dd, 741 u32 selector, 742 u8 vl) 743 { 744 struct sdma_vl_map *m; 745 struct sdma_map_elem *e; 746 struct sdma_engine *rval; 747 748 /* NOTE This should only happen if SC->VL changed after the initial 749 * checks on the QP/AH 750 * Default will return engine 0 below 751 */ 752 if (vl >= num_vls) { 753 rval = NULL; 754 goto done; 755 } 756 757 rcu_read_lock(); 758 m = rcu_dereference(dd->sdma_map); 759 if (unlikely(!m)) { 760 rcu_read_unlock(); 761 return &dd->per_sdma[0]; 762 } 763 e = m->map[vl & m->mask]; 764 rval = e->sde[selector & e->mask]; 765 rcu_read_unlock(); 766 767 done: 768 rval = !rval ? &dd->per_sdma[0] : rval; 769 trace_hfi1_sdma_engine_select(dd, selector, vl, rval->this_idx); 770 return rval; 771 } 772 773 /** 774 * sdma_select_engine_sc() - select sdma engine 775 * @dd: devdata 776 * @selector: a spreading factor 777 * @sc5: the 5 bit sc 778 * 779 * 780 * This function returns an engine based on the selector and an sc. 781 */ 782 struct sdma_engine *sdma_select_engine_sc( 783 struct hfi1_devdata *dd, 784 u32 selector, 785 u8 sc5) 786 { 787 u8 vl = sc_to_vlt(dd, sc5); 788 789 return sdma_select_engine_vl(dd, selector, vl); 790 } 791 792 struct sdma_rht_map_elem { 793 u32 mask; 794 u8 ctr; 795 struct sdma_engine *sde[]; 796 }; 797 798 struct sdma_rht_node { 799 unsigned long cpu_id; 800 struct sdma_rht_map_elem *map[HFI1_MAX_VLS_SUPPORTED]; 801 struct rhash_head node; 802 }; 803 804 #define NR_CPUS_HINT 192 805 806 static const struct rhashtable_params sdma_rht_params = { 807 .nelem_hint = NR_CPUS_HINT, 808 .head_offset = offsetof(struct sdma_rht_node, node), 809 .key_offset = offsetof(struct sdma_rht_node, cpu_id), 810 .key_len = sizeof_field(struct sdma_rht_node, cpu_id), 811 .max_size = NR_CPUS, 812 .min_size = 8, 813 .automatic_shrinking = true, 814 }; 815 816 /* 817 * sdma_select_user_engine() - select sdma engine based on user setup 818 * @dd: devdata 819 * @selector: a spreading factor 820 * @vl: this vl 821 * 822 * This function returns an sdma engine for a user sdma request. 823 * User defined sdma engine affinity setting is honored when applicable, 824 * otherwise system default sdma engine mapping is used. To ensure correct 825 * ordering, the mapping from <selector, vl> to sde must remain unchanged. 826 */ 827 struct sdma_engine *sdma_select_user_engine(struct hfi1_devdata *dd, 828 u32 selector, u8 vl) 829 { 830 struct sdma_rht_node *rht_node; 831 struct sdma_engine *sde = NULL; 832 unsigned long cpu_id; 833 834 /* 835 * To ensure that always the same sdma engine(s) will be 836 * selected make sure the process is pinned to this CPU only. 837 */ 838 if (current->nr_cpus_allowed != 1) 839 goto out; 840 841 rcu_read_lock(); 842 cpu_id = smp_processor_id(); 843 rht_node = rhashtable_lookup(dd->sdma_rht, &cpu_id, 844 sdma_rht_params); 845 846 if (rht_node && rht_node->map[vl]) { 847 struct sdma_rht_map_elem *map = rht_node->map[vl]; 848 849 sde = map->sde[selector & map->mask]; 850 } 851 rcu_read_unlock(); 852 853 if (sde) 854 return sde; 855 856 out: 857 return sdma_select_engine_vl(dd, selector, vl); 858 } 859 860 static void sdma_populate_sde_map(struct sdma_rht_map_elem *map) 861 { 862 int i; 863 864 for (i = 0; i < roundup_pow_of_two(map->ctr ? : 1) - map->ctr; i++) 865 map->sde[map->ctr + i] = map->sde[i]; 866 } 867 868 static void sdma_cleanup_sde_map(struct sdma_rht_map_elem *map, 869 struct sdma_engine *sde) 870 { 871 unsigned int i, pow; 872 873 /* only need to check the first ctr entries for a match */ 874 for (i = 0; i < map->ctr; i++) { 875 if (map->sde[i] == sde) { 876 memmove(&map->sde[i], &map->sde[i + 1], 877 (map->ctr - i - 1) * sizeof(map->sde[0])); 878 map->ctr--; 879 pow = roundup_pow_of_two(map->ctr ? : 1); 880 map->mask = pow - 1; 881 sdma_populate_sde_map(map); 882 break; 883 } 884 } 885 } 886 887 /* 888 * Prevents concurrent reads and writes of the sdma engine cpu_mask 889 */ 890 static DEFINE_MUTEX(process_to_sde_mutex); 891 892 ssize_t sdma_set_cpu_to_sde_map(struct sdma_engine *sde, const char *buf, 893 size_t count) 894 { 895 struct hfi1_devdata *dd = sde->dd; 896 cpumask_var_t mask, new_mask; 897 unsigned long cpu; 898 int ret, vl, sz; 899 struct sdma_rht_node *rht_node; 900 901 vl = sdma_engine_get_vl(sde); 902 if (unlikely(vl < 0 || vl >= ARRAY_SIZE(rht_node->map))) 903 return -EINVAL; 904 905 ret = zalloc_cpumask_var(&mask, GFP_KERNEL); 906 if (!ret) 907 return -ENOMEM; 908 909 ret = zalloc_cpumask_var(&new_mask, GFP_KERNEL); 910 if (!ret) { 911 free_cpumask_var(mask); 912 return -ENOMEM; 913 } 914 ret = cpulist_parse(buf, mask); 915 if (ret) 916 goto out_free; 917 918 if (!cpumask_subset(mask, cpu_online_mask)) { 919 dd_dev_warn(sde->dd, "Invalid CPU mask\n"); 920 ret = -EINVAL; 921 goto out_free; 922 } 923 924 sz = sizeof(struct sdma_rht_map_elem) + 925 (TXE_NUM_SDMA_ENGINES * sizeof(struct sdma_engine *)); 926 927 mutex_lock(&process_to_sde_mutex); 928 929 for_each_cpu(cpu, mask) { 930 /* Check if we have this already mapped */ 931 if (cpumask_test_cpu(cpu, &sde->cpu_mask)) { 932 cpumask_set_cpu(cpu, new_mask); 933 continue; 934 } 935 936 rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpu, 937 sdma_rht_params); 938 if (!rht_node) { 939 rht_node = kzalloc(sizeof(*rht_node), GFP_KERNEL); 940 if (!rht_node) { 941 ret = -ENOMEM; 942 goto out; 943 } 944 945 rht_node->map[vl] = kzalloc(sz, GFP_KERNEL); 946 if (!rht_node->map[vl]) { 947 kfree(rht_node); 948 ret = -ENOMEM; 949 goto out; 950 } 951 rht_node->cpu_id = cpu; 952 rht_node->map[vl]->mask = 0; 953 rht_node->map[vl]->ctr = 1; 954 rht_node->map[vl]->sde[0] = sde; 955 956 ret = rhashtable_insert_fast(dd->sdma_rht, 957 &rht_node->node, 958 sdma_rht_params); 959 if (ret) { 960 kfree(rht_node->map[vl]); 961 kfree(rht_node); 962 dd_dev_err(sde->dd, "Failed to set process to sde affinity for cpu %lu\n", 963 cpu); 964 goto out; 965 } 966 967 } else { 968 int ctr, pow; 969 970 /* Add new user mappings */ 971 if (!rht_node->map[vl]) 972 rht_node->map[vl] = kzalloc(sz, GFP_KERNEL); 973 974 if (!rht_node->map[vl]) { 975 ret = -ENOMEM; 976 goto out; 977 } 978 979 rht_node->map[vl]->ctr++; 980 ctr = rht_node->map[vl]->ctr; 981 rht_node->map[vl]->sde[ctr - 1] = sde; 982 pow = roundup_pow_of_two(ctr); 983 rht_node->map[vl]->mask = pow - 1; 984 985 /* Populate the sde map table */ 986 sdma_populate_sde_map(rht_node->map[vl]); 987 } 988 cpumask_set_cpu(cpu, new_mask); 989 } 990 991 /* Clean up old mappings */ 992 for_each_cpu(cpu, cpu_online_mask) { 993 struct sdma_rht_node *rht_node; 994 995 /* Don't cleanup sdes that are set in the new mask */ 996 if (cpumask_test_cpu(cpu, mask)) 997 continue; 998 999 rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpu, 1000 sdma_rht_params); 1001 if (rht_node) { 1002 bool empty = true; 1003 int i; 1004 1005 /* Remove mappings for old sde */ 1006 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) 1007 if (rht_node->map[i]) 1008 sdma_cleanup_sde_map(rht_node->map[i], 1009 sde); 1010 1011 /* Free empty hash table entries */ 1012 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) { 1013 if (!rht_node->map[i]) 1014 continue; 1015 1016 if (rht_node->map[i]->ctr) { 1017 empty = false; 1018 break; 1019 } 1020 } 1021 1022 if (empty) { 1023 ret = rhashtable_remove_fast(dd->sdma_rht, 1024 &rht_node->node, 1025 sdma_rht_params); 1026 WARN_ON(ret); 1027 1028 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) 1029 kfree(rht_node->map[i]); 1030 1031 kfree(rht_node); 1032 } 1033 } 1034 } 1035 1036 cpumask_copy(&sde->cpu_mask, new_mask); 1037 out: 1038 mutex_unlock(&process_to_sde_mutex); 1039 out_free: 1040 free_cpumask_var(mask); 1041 free_cpumask_var(new_mask); 1042 return ret ? : strnlen(buf, PAGE_SIZE); 1043 } 1044 1045 ssize_t sdma_get_cpu_to_sde_map(struct sdma_engine *sde, char *buf) 1046 { 1047 mutex_lock(&process_to_sde_mutex); 1048 if (cpumask_empty(&sde->cpu_mask)) 1049 snprintf(buf, PAGE_SIZE, "%s\n", "empty"); 1050 else 1051 cpumap_print_to_pagebuf(true, buf, &sde->cpu_mask); 1052 mutex_unlock(&process_to_sde_mutex); 1053 return strnlen(buf, PAGE_SIZE); 1054 } 1055 1056 static void sdma_rht_free(void *ptr, void *arg) 1057 { 1058 struct sdma_rht_node *rht_node = ptr; 1059 int i; 1060 1061 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) 1062 kfree(rht_node->map[i]); 1063 1064 kfree(rht_node); 1065 } 1066 1067 /** 1068 * sdma_seqfile_dump_cpu_list() - debugfs dump the cpu to sdma mappings 1069 * @s: seq file 1070 * @dd: hfi1_devdata 1071 * @cpuid: cpu id 1072 * 1073 * This routine dumps the process to sde mappings per cpu 1074 */ 1075 void sdma_seqfile_dump_cpu_list(struct seq_file *s, 1076 struct hfi1_devdata *dd, 1077 unsigned long cpuid) 1078 { 1079 struct sdma_rht_node *rht_node; 1080 int i, j; 1081 1082 rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpuid, 1083 sdma_rht_params); 1084 if (!rht_node) 1085 return; 1086 1087 seq_printf(s, "cpu%3lu: ", cpuid); 1088 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) { 1089 if (!rht_node->map[i] || !rht_node->map[i]->ctr) 1090 continue; 1091 1092 seq_printf(s, " vl%d: [", i); 1093 1094 for (j = 0; j < rht_node->map[i]->ctr; j++) { 1095 if (!rht_node->map[i]->sde[j]) 1096 continue; 1097 1098 if (j > 0) 1099 seq_puts(s, ","); 1100 1101 seq_printf(s, " sdma%2d", 1102 rht_node->map[i]->sde[j]->this_idx); 1103 } 1104 seq_puts(s, " ]"); 1105 } 1106 1107 seq_puts(s, "\n"); 1108 } 1109 1110 /* 1111 * Free the indicated map struct 1112 */ 1113 static void sdma_map_free(struct sdma_vl_map *m) 1114 { 1115 int i; 1116 1117 for (i = 0; m && i < m->actual_vls; i++) 1118 kfree(m->map[i]); 1119 kfree(m); 1120 } 1121 1122 /* 1123 * Handle RCU callback 1124 */ 1125 static void sdma_map_rcu_callback(struct rcu_head *list) 1126 { 1127 struct sdma_vl_map *m = container_of(list, struct sdma_vl_map, list); 1128 1129 sdma_map_free(m); 1130 } 1131 1132 /** 1133 * sdma_map_init - called when # vls change 1134 * @dd: hfi1_devdata 1135 * @port: port number 1136 * @num_vls: number of vls 1137 * @vl_engines: per vl engine mapping (optional) 1138 * 1139 * This routine changes the mapping based on the number of vls. 1140 * 1141 * vl_engines is used to specify a non-uniform vl/engine loading. NULL 1142 * implies auto computing the loading and giving each VLs a uniform 1143 * distribution of engines per VL. 1144 * 1145 * The auto algorithm computes the sde_per_vl and the number of extra 1146 * engines. Any extra engines are added from the last VL on down. 1147 * 1148 * rcu locking is used here to control access to the mapping fields. 1149 * 1150 * If either the num_vls or num_sdma are non-power of 2, the array sizes 1151 * in the struct sdma_vl_map and the struct sdma_map_elem are rounded 1152 * up to the next highest power of 2 and the first entry is reused 1153 * in a round robin fashion. 1154 * 1155 * If an error occurs the map change is not done and the mapping is 1156 * not changed. 1157 * 1158 */ 1159 int sdma_map_init(struct hfi1_devdata *dd, u8 port, u8 num_vls, u8 *vl_engines) 1160 { 1161 int i, j; 1162 int extra, sde_per_vl; 1163 int engine = 0; 1164 u8 lvl_engines[OPA_MAX_VLS]; 1165 struct sdma_vl_map *oldmap, *newmap; 1166 1167 if (!(dd->flags & HFI1_HAS_SEND_DMA)) 1168 return 0; 1169 1170 if (!vl_engines) { 1171 /* truncate divide */ 1172 sde_per_vl = dd->num_sdma / num_vls; 1173 /* extras */ 1174 extra = dd->num_sdma % num_vls; 1175 vl_engines = lvl_engines; 1176 /* add extras from last vl down */ 1177 for (i = num_vls - 1; i >= 0; i--, extra--) 1178 vl_engines[i] = sde_per_vl + (extra > 0 ? 1 : 0); 1179 } 1180 /* build new map */ 1181 newmap = kzalloc( 1182 sizeof(struct sdma_vl_map) + 1183 roundup_pow_of_two(num_vls) * 1184 sizeof(struct sdma_map_elem *), 1185 GFP_KERNEL); 1186 if (!newmap) 1187 goto bail; 1188 newmap->actual_vls = num_vls; 1189 newmap->vls = roundup_pow_of_two(num_vls); 1190 newmap->mask = (1 << ilog2(newmap->vls)) - 1; 1191 /* initialize back-map */ 1192 for (i = 0; i < TXE_NUM_SDMA_ENGINES; i++) 1193 newmap->engine_to_vl[i] = -1; 1194 for (i = 0; i < newmap->vls; i++) { 1195 /* save for wrap around */ 1196 int first_engine = engine; 1197 1198 if (i < newmap->actual_vls) { 1199 int sz = roundup_pow_of_two(vl_engines[i]); 1200 1201 /* only allocate once */ 1202 newmap->map[i] = kzalloc( 1203 sizeof(struct sdma_map_elem) + 1204 sz * sizeof(struct sdma_engine *), 1205 GFP_KERNEL); 1206 if (!newmap->map[i]) 1207 goto bail; 1208 newmap->map[i]->mask = (1 << ilog2(sz)) - 1; 1209 /* assign engines */ 1210 for (j = 0; j < sz; j++) { 1211 newmap->map[i]->sde[j] = 1212 &dd->per_sdma[engine]; 1213 if (++engine >= first_engine + vl_engines[i]) 1214 /* wrap back to first engine */ 1215 engine = first_engine; 1216 } 1217 /* assign back-map */ 1218 for (j = 0; j < vl_engines[i]; j++) 1219 newmap->engine_to_vl[first_engine + j] = i; 1220 } else { 1221 /* just re-use entry without allocating */ 1222 newmap->map[i] = newmap->map[i % num_vls]; 1223 } 1224 engine = first_engine + vl_engines[i]; 1225 } 1226 /* newmap in hand, save old map */ 1227 spin_lock_irq(&dd->sde_map_lock); 1228 oldmap = rcu_dereference_protected(dd->sdma_map, 1229 lockdep_is_held(&dd->sde_map_lock)); 1230 1231 /* publish newmap */ 1232 rcu_assign_pointer(dd->sdma_map, newmap); 1233 1234 spin_unlock_irq(&dd->sde_map_lock); 1235 /* success, free any old map after grace period */ 1236 if (oldmap) 1237 call_rcu(&oldmap->list, sdma_map_rcu_callback); 1238 return 0; 1239 bail: 1240 /* free any partial allocation */ 1241 sdma_map_free(newmap); 1242 return -ENOMEM; 1243 } 1244 1245 /** 1246 * sdma_clean - Clean up allocated memory 1247 * @dd: struct hfi1_devdata 1248 * @num_engines: num sdma engines 1249 * 1250 * This routine can be called regardless of the success of 1251 * sdma_init() 1252 */ 1253 void sdma_clean(struct hfi1_devdata *dd, size_t num_engines) 1254 { 1255 size_t i; 1256 struct sdma_engine *sde; 1257 1258 if (dd->sdma_pad_dma) { 1259 dma_free_coherent(&dd->pcidev->dev, SDMA_PAD, 1260 (void *)dd->sdma_pad_dma, 1261 dd->sdma_pad_phys); 1262 dd->sdma_pad_dma = NULL; 1263 dd->sdma_pad_phys = 0; 1264 } 1265 if (dd->sdma_heads_dma) { 1266 dma_free_coherent(&dd->pcidev->dev, dd->sdma_heads_size, 1267 (void *)dd->sdma_heads_dma, 1268 dd->sdma_heads_phys); 1269 dd->sdma_heads_dma = NULL; 1270 dd->sdma_heads_phys = 0; 1271 } 1272 for (i = 0; dd->per_sdma && i < num_engines; ++i) { 1273 sde = &dd->per_sdma[i]; 1274 1275 sde->head_dma = NULL; 1276 sde->head_phys = 0; 1277 1278 if (sde->descq) { 1279 dma_free_coherent( 1280 &dd->pcidev->dev, 1281 sde->descq_cnt * sizeof(u64[2]), 1282 sde->descq, 1283 sde->descq_phys 1284 ); 1285 sde->descq = NULL; 1286 sde->descq_phys = 0; 1287 } 1288 kvfree(sde->tx_ring); 1289 sde->tx_ring = NULL; 1290 } 1291 if (rcu_access_pointer(dd->sdma_map)) { 1292 spin_lock_irq(&dd->sde_map_lock); 1293 sdma_map_free(rcu_access_pointer(dd->sdma_map)); 1294 RCU_INIT_POINTER(dd->sdma_map, NULL); 1295 spin_unlock_irq(&dd->sde_map_lock); 1296 synchronize_rcu(); 1297 } 1298 kfree(dd->per_sdma); 1299 dd->per_sdma = NULL; 1300 1301 if (dd->sdma_rht) { 1302 rhashtable_free_and_destroy(dd->sdma_rht, sdma_rht_free, NULL); 1303 kfree(dd->sdma_rht); 1304 dd->sdma_rht = NULL; 1305 } 1306 } 1307 1308 /** 1309 * sdma_init() - called when device probed 1310 * @dd: hfi1_devdata 1311 * @port: port number (currently only zero) 1312 * 1313 * Initializes each sde and its csrs. 1314 * Interrupts are not required to be enabled. 1315 * 1316 * Returns: 1317 * 0 - success, -errno on failure 1318 */ 1319 int sdma_init(struct hfi1_devdata *dd, u8 port) 1320 { 1321 unsigned this_idx; 1322 struct sdma_engine *sde; 1323 struct rhashtable *tmp_sdma_rht; 1324 u16 descq_cnt; 1325 void *curr_head; 1326 struct hfi1_pportdata *ppd = dd->pport + port; 1327 u32 per_sdma_credits; 1328 uint idle_cnt = sdma_idle_cnt; 1329 size_t num_engines = chip_sdma_engines(dd); 1330 int ret = -ENOMEM; 1331 1332 if (!HFI1_CAP_IS_KSET(SDMA)) { 1333 HFI1_CAP_CLEAR(SDMA_AHG); 1334 return 0; 1335 } 1336 if (mod_num_sdma && 1337 /* can't exceed chip support */ 1338 mod_num_sdma <= chip_sdma_engines(dd) && 1339 /* count must be >= vls */ 1340 mod_num_sdma >= num_vls) 1341 num_engines = mod_num_sdma; 1342 1343 dd_dev_info(dd, "SDMA mod_num_sdma: %u\n", mod_num_sdma); 1344 dd_dev_info(dd, "SDMA chip_sdma_engines: %u\n", chip_sdma_engines(dd)); 1345 dd_dev_info(dd, "SDMA chip_sdma_mem_size: %u\n", 1346 chip_sdma_mem_size(dd)); 1347 1348 per_sdma_credits = 1349 chip_sdma_mem_size(dd) / (num_engines * SDMA_BLOCK_SIZE); 1350 1351 /* set up freeze waitqueue */ 1352 init_waitqueue_head(&dd->sdma_unfreeze_wq); 1353 atomic_set(&dd->sdma_unfreeze_count, 0); 1354 1355 descq_cnt = sdma_get_descq_cnt(); 1356 dd_dev_info(dd, "SDMA engines %zu descq_cnt %u\n", 1357 num_engines, descq_cnt); 1358 1359 /* alloc memory for array of send engines */ 1360 dd->per_sdma = kcalloc_node(num_engines, sizeof(*dd->per_sdma), 1361 GFP_KERNEL, dd->node); 1362 if (!dd->per_sdma) 1363 return ret; 1364 1365 idle_cnt = ns_to_cclock(dd, idle_cnt); 1366 if (idle_cnt) 1367 dd->default_desc1 = 1368 SDMA_DESC1_HEAD_TO_HOST_FLAG; 1369 else 1370 dd->default_desc1 = 1371 SDMA_DESC1_INT_REQ_FLAG; 1372 1373 if (!sdma_desct_intr) 1374 sdma_desct_intr = SDMA_DESC_INTR; 1375 1376 /* Allocate memory for SendDMA descriptor FIFOs */ 1377 for (this_idx = 0; this_idx < num_engines; ++this_idx) { 1378 sde = &dd->per_sdma[this_idx]; 1379 sde->dd = dd; 1380 sde->ppd = ppd; 1381 sde->this_idx = this_idx; 1382 sde->descq_cnt = descq_cnt; 1383 sde->desc_avail = sdma_descq_freecnt(sde); 1384 sde->sdma_shift = ilog2(descq_cnt); 1385 sde->sdma_mask = (1 << sde->sdma_shift) - 1; 1386 1387 /* Create a mask specifically for each interrupt source */ 1388 sde->int_mask = (u64)1 << (0 * TXE_NUM_SDMA_ENGINES + 1389 this_idx); 1390 sde->progress_mask = (u64)1 << (1 * TXE_NUM_SDMA_ENGINES + 1391 this_idx); 1392 sde->idle_mask = (u64)1 << (2 * TXE_NUM_SDMA_ENGINES + 1393 this_idx); 1394 /* Create a combined mask to cover all 3 interrupt sources */ 1395 sde->imask = sde->int_mask | sde->progress_mask | 1396 sde->idle_mask; 1397 1398 spin_lock_init(&sde->tail_lock); 1399 seqlock_init(&sde->head_lock); 1400 spin_lock_init(&sde->senddmactrl_lock); 1401 spin_lock_init(&sde->flushlist_lock); 1402 seqlock_init(&sde->waitlock); 1403 /* insure there is always a zero bit */ 1404 sde->ahg_bits = 0xfffffffe00000000ULL; 1405 1406 sdma_set_state(sde, sdma_state_s00_hw_down); 1407 1408 /* set up reference counting */ 1409 kref_init(&sde->state.kref); 1410 init_completion(&sde->state.comp); 1411 1412 INIT_LIST_HEAD(&sde->flushlist); 1413 INIT_LIST_HEAD(&sde->dmawait); 1414 1415 sde->tail_csr = 1416 get_kctxt_csr_addr(dd, this_idx, SD(TAIL)); 1417 1418 tasklet_setup(&sde->sdma_hw_clean_up_task, 1419 sdma_hw_clean_up_task); 1420 tasklet_setup(&sde->sdma_sw_clean_up_task, 1421 sdma_sw_clean_up_task); 1422 INIT_WORK(&sde->err_halt_worker, sdma_err_halt_wait); 1423 INIT_WORK(&sde->flush_worker, sdma_field_flush); 1424 1425 sde->progress_check_head = 0; 1426 1427 timer_setup(&sde->err_progress_check_timer, 1428 sdma_err_progress_check, 0); 1429 1430 sde->descq = dma_alloc_coherent(&dd->pcidev->dev, 1431 descq_cnt * sizeof(u64[2]), 1432 &sde->descq_phys, GFP_KERNEL); 1433 if (!sde->descq) 1434 goto bail; 1435 sde->tx_ring = 1436 kvzalloc_node(array_size(descq_cnt, 1437 sizeof(struct sdma_txreq *)), 1438 GFP_KERNEL, dd->node); 1439 if (!sde->tx_ring) 1440 goto bail; 1441 } 1442 1443 dd->sdma_heads_size = L1_CACHE_BYTES * num_engines; 1444 /* Allocate memory for DMA of head registers to memory */ 1445 dd->sdma_heads_dma = dma_alloc_coherent(&dd->pcidev->dev, 1446 dd->sdma_heads_size, 1447 &dd->sdma_heads_phys, 1448 GFP_KERNEL); 1449 if (!dd->sdma_heads_dma) { 1450 dd_dev_err(dd, "failed to allocate SendDMA head memory\n"); 1451 goto bail; 1452 } 1453 1454 /* Allocate memory for pad */ 1455 dd->sdma_pad_dma = dma_alloc_coherent(&dd->pcidev->dev, SDMA_PAD, 1456 &dd->sdma_pad_phys, GFP_KERNEL); 1457 if (!dd->sdma_pad_dma) { 1458 dd_dev_err(dd, "failed to allocate SendDMA pad memory\n"); 1459 goto bail; 1460 } 1461 1462 /* assign each engine to different cacheline and init registers */ 1463 curr_head = (void *)dd->sdma_heads_dma; 1464 for (this_idx = 0; this_idx < num_engines; ++this_idx) { 1465 unsigned long phys_offset; 1466 1467 sde = &dd->per_sdma[this_idx]; 1468 1469 sde->head_dma = curr_head; 1470 curr_head += L1_CACHE_BYTES; 1471 phys_offset = (unsigned long)sde->head_dma - 1472 (unsigned long)dd->sdma_heads_dma; 1473 sde->head_phys = dd->sdma_heads_phys + phys_offset; 1474 init_sdma_regs(sde, per_sdma_credits, idle_cnt); 1475 } 1476 dd->flags |= HFI1_HAS_SEND_DMA; 1477 dd->flags |= idle_cnt ? HFI1_HAS_SDMA_TIMEOUT : 0; 1478 dd->num_sdma = num_engines; 1479 ret = sdma_map_init(dd, port, ppd->vls_operational, NULL); 1480 if (ret < 0) 1481 goto bail; 1482 1483 tmp_sdma_rht = kzalloc(sizeof(*tmp_sdma_rht), GFP_KERNEL); 1484 if (!tmp_sdma_rht) { 1485 ret = -ENOMEM; 1486 goto bail; 1487 } 1488 1489 ret = rhashtable_init(tmp_sdma_rht, &sdma_rht_params); 1490 if (ret < 0) { 1491 kfree(tmp_sdma_rht); 1492 goto bail; 1493 } 1494 1495 dd->sdma_rht = tmp_sdma_rht; 1496 1497 dd_dev_info(dd, "SDMA num_sdma: %u\n", dd->num_sdma); 1498 return 0; 1499 1500 bail: 1501 sdma_clean(dd, num_engines); 1502 return ret; 1503 } 1504 1505 /** 1506 * sdma_all_running() - called when the link goes up 1507 * @dd: hfi1_devdata 1508 * 1509 * This routine moves all engines to the running state. 1510 */ 1511 void sdma_all_running(struct hfi1_devdata *dd) 1512 { 1513 struct sdma_engine *sde; 1514 unsigned int i; 1515 1516 /* move all engines to running */ 1517 for (i = 0; i < dd->num_sdma; ++i) { 1518 sde = &dd->per_sdma[i]; 1519 sdma_process_event(sde, sdma_event_e30_go_running); 1520 } 1521 } 1522 1523 /** 1524 * sdma_all_idle() - called when the link goes down 1525 * @dd: hfi1_devdata 1526 * 1527 * This routine moves all engines to the idle state. 1528 */ 1529 void sdma_all_idle(struct hfi1_devdata *dd) 1530 { 1531 struct sdma_engine *sde; 1532 unsigned int i; 1533 1534 /* idle all engines */ 1535 for (i = 0; i < dd->num_sdma; ++i) { 1536 sde = &dd->per_sdma[i]; 1537 sdma_process_event(sde, sdma_event_e70_go_idle); 1538 } 1539 } 1540 1541 /** 1542 * sdma_start() - called to kick off state processing for all engines 1543 * @dd: hfi1_devdata 1544 * 1545 * This routine is for kicking off the state processing for all required 1546 * sdma engines. Interrupts need to be working at this point. 1547 * 1548 */ 1549 void sdma_start(struct hfi1_devdata *dd) 1550 { 1551 unsigned i; 1552 struct sdma_engine *sde; 1553 1554 /* kick off the engines state processing */ 1555 for (i = 0; i < dd->num_sdma; ++i) { 1556 sde = &dd->per_sdma[i]; 1557 sdma_process_event(sde, sdma_event_e10_go_hw_start); 1558 } 1559 } 1560 1561 /** 1562 * sdma_exit() - used when module is removed 1563 * @dd: hfi1_devdata 1564 */ 1565 void sdma_exit(struct hfi1_devdata *dd) 1566 { 1567 unsigned this_idx; 1568 struct sdma_engine *sde; 1569 1570 for (this_idx = 0; dd->per_sdma && this_idx < dd->num_sdma; 1571 ++this_idx) { 1572 sde = &dd->per_sdma[this_idx]; 1573 if (!list_empty(&sde->dmawait)) 1574 dd_dev_err(dd, "sde %u: dmawait list not empty!\n", 1575 sde->this_idx); 1576 sdma_process_event(sde, sdma_event_e00_go_hw_down); 1577 1578 del_timer_sync(&sde->err_progress_check_timer); 1579 1580 /* 1581 * This waits for the state machine to exit so it is not 1582 * necessary to kill the sdma_sw_clean_up_task to make sure 1583 * it is not running. 1584 */ 1585 sdma_finalput(&sde->state); 1586 } 1587 } 1588 1589 /* 1590 * unmap the indicated descriptor 1591 */ 1592 static inline void sdma_unmap_desc( 1593 struct hfi1_devdata *dd, 1594 struct sdma_desc *descp) 1595 { 1596 system_descriptor_complete(dd, descp); 1597 } 1598 1599 /* 1600 * return the mode as indicated by the first 1601 * descriptor in the tx. 1602 */ 1603 static inline u8 ahg_mode(struct sdma_txreq *tx) 1604 { 1605 return (tx->descp[0].qw[1] & SDMA_DESC1_HEADER_MODE_SMASK) 1606 >> SDMA_DESC1_HEADER_MODE_SHIFT; 1607 } 1608 1609 /** 1610 * __sdma_txclean() - clean tx of mappings, descp *kmalloc's 1611 * @dd: hfi1_devdata for unmapping 1612 * @tx: tx request to clean 1613 * 1614 * This is used in the progress routine to clean the tx or 1615 * by the ULP to toss an in-process tx build. 1616 * 1617 * The code can be called multiple times without issue. 1618 * 1619 */ 1620 void __sdma_txclean( 1621 struct hfi1_devdata *dd, 1622 struct sdma_txreq *tx) 1623 { 1624 u16 i; 1625 1626 if (tx->num_desc) { 1627 u8 skip = 0, mode = ahg_mode(tx); 1628 1629 /* unmap first */ 1630 sdma_unmap_desc(dd, &tx->descp[0]); 1631 /* determine number of AHG descriptors to skip */ 1632 if (mode > SDMA_AHG_APPLY_UPDATE1) 1633 skip = mode >> 1; 1634 for (i = 1 + skip; i < tx->num_desc; i++) 1635 sdma_unmap_desc(dd, &tx->descp[i]); 1636 tx->num_desc = 0; 1637 } 1638 kfree(tx->coalesce_buf); 1639 tx->coalesce_buf = NULL; 1640 /* kmalloc'ed descp */ 1641 if (unlikely(tx->desc_limit > ARRAY_SIZE(tx->descs))) { 1642 tx->desc_limit = ARRAY_SIZE(tx->descs); 1643 kfree(tx->descp); 1644 } 1645 } 1646 1647 static inline u16 sdma_gethead(struct sdma_engine *sde) 1648 { 1649 struct hfi1_devdata *dd = sde->dd; 1650 int use_dmahead; 1651 u16 hwhead; 1652 1653 #ifdef CONFIG_SDMA_VERBOSITY 1654 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", 1655 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); 1656 #endif 1657 1658 retry: 1659 use_dmahead = HFI1_CAP_IS_KSET(USE_SDMA_HEAD) && __sdma_running(sde) && 1660 (dd->flags & HFI1_HAS_SDMA_TIMEOUT); 1661 hwhead = use_dmahead ? 1662 (u16)le64_to_cpu(*sde->head_dma) : 1663 (u16)read_sde_csr(sde, SD(HEAD)); 1664 1665 if (unlikely(HFI1_CAP_IS_KSET(SDMA_HEAD_CHECK))) { 1666 u16 cnt; 1667 u16 swtail; 1668 u16 swhead; 1669 int sane; 1670 1671 swhead = sde->descq_head & sde->sdma_mask; 1672 /* this code is really bad for cache line trading */ 1673 swtail = READ_ONCE(sde->descq_tail) & sde->sdma_mask; 1674 cnt = sde->descq_cnt; 1675 1676 if (swhead < swtail) 1677 /* not wrapped */ 1678 sane = (hwhead >= swhead) & (hwhead <= swtail); 1679 else if (swhead > swtail) 1680 /* wrapped around */ 1681 sane = ((hwhead >= swhead) && (hwhead < cnt)) || 1682 (hwhead <= swtail); 1683 else 1684 /* empty */ 1685 sane = (hwhead == swhead); 1686 1687 if (unlikely(!sane)) { 1688 dd_dev_err(dd, "SDMA(%u) bad head (%s) hwhd=%u swhd=%u swtl=%u cnt=%u\n", 1689 sde->this_idx, 1690 use_dmahead ? "dma" : "kreg", 1691 hwhead, swhead, swtail, cnt); 1692 if (use_dmahead) { 1693 /* try one more time, using csr */ 1694 use_dmahead = 0; 1695 goto retry; 1696 } 1697 /* proceed as if no progress */ 1698 hwhead = swhead; 1699 } 1700 } 1701 return hwhead; 1702 } 1703 1704 /* 1705 * This is called when there are send DMA descriptors that might be 1706 * available. 1707 * 1708 * This is called with head_lock held. 1709 */ 1710 static void sdma_desc_avail(struct sdma_engine *sde, uint avail) 1711 { 1712 struct iowait *wait, *nw, *twait; 1713 struct iowait *waits[SDMA_WAIT_BATCH_SIZE]; 1714 uint i, n = 0, seq, tidx = 0; 1715 1716 #ifdef CONFIG_SDMA_VERBOSITY 1717 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx, 1718 slashstrip(__FILE__), __LINE__, __func__); 1719 dd_dev_err(sde->dd, "avail: %u\n", avail); 1720 #endif 1721 1722 do { 1723 seq = read_seqbegin(&sde->waitlock); 1724 if (!list_empty(&sde->dmawait)) { 1725 /* at least one item */ 1726 write_seqlock(&sde->waitlock); 1727 /* Harvest waiters wanting DMA descriptors */ 1728 list_for_each_entry_safe( 1729 wait, 1730 nw, 1731 &sde->dmawait, 1732 list) { 1733 u32 num_desc; 1734 1735 if (!wait->wakeup) 1736 continue; 1737 if (n == ARRAY_SIZE(waits)) 1738 break; 1739 iowait_init_priority(wait); 1740 num_desc = iowait_get_all_desc(wait); 1741 if (num_desc > avail) 1742 break; 1743 avail -= num_desc; 1744 /* Find the top-priority wait memeber */ 1745 if (n) { 1746 twait = waits[tidx]; 1747 tidx = 1748 iowait_priority_update_top(wait, 1749 twait, 1750 n, 1751 tidx); 1752 } 1753 list_del_init(&wait->list); 1754 waits[n++] = wait; 1755 } 1756 write_sequnlock(&sde->waitlock); 1757 break; 1758 } 1759 } while (read_seqretry(&sde->waitlock, seq)); 1760 1761 /* Schedule the top-priority entry first */ 1762 if (n) 1763 waits[tidx]->wakeup(waits[tidx], SDMA_AVAIL_REASON); 1764 1765 for (i = 0; i < n; i++) 1766 if (i != tidx) 1767 waits[i]->wakeup(waits[i], SDMA_AVAIL_REASON); 1768 } 1769 1770 /* head_lock must be held */ 1771 static void sdma_make_progress(struct sdma_engine *sde, u64 status) 1772 { 1773 struct sdma_txreq *txp = NULL; 1774 int progress = 0; 1775 u16 hwhead, swhead; 1776 int idle_check_done = 0; 1777 1778 hwhead = sdma_gethead(sde); 1779 1780 /* The reason for some of the complexity of this code is that 1781 * not all descriptors have corresponding txps. So, we have to 1782 * be able to skip over descs until we wander into the range of 1783 * the next txp on the list. 1784 */ 1785 1786 retry: 1787 txp = get_txhead(sde); 1788 swhead = sde->descq_head & sde->sdma_mask; 1789 trace_hfi1_sdma_progress(sde, hwhead, swhead, txp); 1790 while (swhead != hwhead) { 1791 /* advance head, wrap if needed */ 1792 swhead = ++sde->descq_head & sde->sdma_mask; 1793 1794 /* if now past this txp's descs, do the callback */ 1795 if (txp && txp->next_descq_idx == swhead) { 1796 /* remove from list */ 1797 sde->tx_ring[sde->tx_head++ & sde->sdma_mask] = NULL; 1798 complete_tx(sde, txp, SDMA_TXREQ_S_OK); 1799 /* see if there is another txp */ 1800 txp = get_txhead(sde); 1801 } 1802 trace_hfi1_sdma_progress(sde, hwhead, swhead, txp); 1803 progress++; 1804 } 1805 1806 /* 1807 * The SDMA idle interrupt is not guaranteed to be ordered with respect 1808 * to updates to the dma_head location in host memory. The head 1809 * value read might not be fully up to date. If there are pending 1810 * descriptors and the SDMA idle interrupt fired then read from the 1811 * CSR SDMA head instead to get the latest value from the hardware. 1812 * The hardware SDMA head should be read at most once in this invocation 1813 * of sdma_make_progress(..) which is ensured by idle_check_done flag 1814 */ 1815 if ((status & sde->idle_mask) && !idle_check_done) { 1816 u16 swtail; 1817 1818 swtail = READ_ONCE(sde->descq_tail) & sde->sdma_mask; 1819 if (swtail != hwhead) { 1820 hwhead = (u16)read_sde_csr(sde, SD(HEAD)); 1821 idle_check_done = 1; 1822 goto retry; 1823 } 1824 } 1825 1826 sde->last_status = status; 1827 if (progress) 1828 sdma_desc_avail(sde, sdma_descq_freecnt(sde)); 1829 } 1830 1831 /* 1832 * sdma_engine_interrupt() - interrupt handler for engine 1833 * @sde: sdma engine 1834 * @status: sdma interrupt reason 1835 * 1836 * Status is a mask of the 3 possible interrupts for this engine. It will 1837 * contain bits _only_ for this SDMA engine. It will contain at least one 1838 * bit, it may contain more. 1839 */ 1840 void sdma_engine_interrupt(struct sdma_engine *sde, u64 status) 1841 { 1842 trace_hfi1_sdma_engine_interrupt(sde, status); 1843 write_seqlock(&sde->head_lock); 1844 sdma_set_desc_cnt(sde, sdma_desct_intr); 1845 if (status & sde->idle_mask) 1846 sde->idle_int_cnt++; 1847 else if (status & sde->progress_mask) 1848 sde->progress_int_cnt++; 1849 else if (status & sde->int_mask) 1850 sde->sdma_int_cnt++; 1851 sdma_make_progress(sde, status); 1852 write_sequnlock(&sde->head_lock); 1853 } 1854 1855 /** 1856 * sdma_engine_error() - error handler for engine 1857 * @sde: sdma engine 1858 * @status: sdma interrupt reason 1859 */ 1860 void sdma_engine_error(struct sdma_engine *sde, u64 status) 1861 { 1862 unsigned long flags; 1863 1864 #ifdef CONFIG_SDMA_VERBOSITY 1865 dd_dev_err(sde->dd, "CONFIG SDMA(%u) error status 0x%llx state %s\n", 1866 sde->this_idx, 1867 (unsigned long long)status, 1868 sdma_state_names[sde->state.current_state]); 1869 #endif 1870 spin_lock_irqsave(&sde->tail_lock, flags); 1871 write_seqlock(&sde->head_lock); 1872 if (status & ALL_SDMA_ENG_HALT_ERRS) 1873 __sdma_process_event(sde, sdma_event_e60_hw_halted); 1874 if (status & ~SD(ENG_ERR_STATUS_SDMA_HALT_ERR_SMASK)) { 1875 dd_dev_err(sde->dd, 1876 "SDMA (%u) engine error: 0x%llx state %s\n", 1877 sde->this_idx, 1878 (unsigned long long)status, 1879 sdma_state_names[sde->state.current_state]); 1880 dump_sdma_state(sde); 1881 } 1882 write_sequnlock(&sde->head_lock); 1883 spin_unlock_irqrestore(&sde->tail_lock, flags); 1884 } 1885 1886 static void sdma_sendctrl(struct sdma_engine *sde, unsigned op) 1887 { 1888 u64 set_senddmactrl = 0; 1889 u64 clr_senddmactrl = 0; 1890 unsigned long flags; 1891 1892 #ifdef CONFIG_SDMA_VERBOSITY 1893 dd_dev_err(sde->dd, "CONFIG SDMA(%u) senddmactrl E=%d I=%d H=%d C=%d\n", 1894 sde->this_idx, 1895 (op & SDMA_SENDCTRL_OP_ENABLE) ? 1 : 0, 1896 (op & SDMA_SENDCTRL_OP_INTENABLE) ? 1 : 0, 1897 (op & SDMA_SENDCTRL_OP_HALT) ? 1 : 0, 1898 (op & SDMA_SENDCTRL_OP_CLEANUP) ? 1 : 0); 1899 #endif 1900 1901 if (op & SDMA_SENDCTRL_OP_ENABLE) 1902 set_senddmactrl |= SD(CTRL_SDMA_ENABLE_SMASK); 1903 else 1904 clr_senddmactrl |= SD(CTRL_SDMA_ENABLE_SMASK); 1905 1906 if (op & SDMA_SENDCTRL_OP_INTENABLE) 1907 set_senddmactrl |= SD(CTRL_SDMA_INT_ENABLE_SMASK); 1908 else 1909 clr_senddmactrl |= SD(CTRL_SDMA_INT_ENABLE_SMASK); 1910 1911 if (op & SDMA_SENDCTRL_OP_HALT) 1912 set_senddmactrl |= SD(CTRL_SDMA_HALT_SMASK); 1913 else 1914 clr_senddmactrl |= SD(CTRL_SDMA_HALT_SMASK); 1915 1916 spin_lock_irqsave(&sde->senddmactrl_lock, flags); 1917 1918 sde->p_senddmactrl |= set_senddmactrl; 1919 sde->p_senddmactrl &= ~clr_senddmactrl; 1920 1921 if (op & SDMA_SENDCTRL_OP_CLEANUP) 1922 write_sde_csr(sde, SD(CTRL), 1923 sde->p_senddmactrl | 1924 SD(CTRL_SDMA_CLEANUP_SMASK)); 1925 else 1926 write_sde_csr(sde, SD(CTRL), sde->p_senddmactrl); 1927 1928 spin_unlock_irqrestore(&sde->senddmactrl_lock, flags); 1929 1930 #ifdef CONFIG_SDMA_VERBOSITY 1931 sdma_dumpstate(sde); 1932 #endif 1933 } 1934 1935 static void sdma_setlengen(struct sdma_engine *sde) 1936 { 1937 #ifdef CONFIG_SDMA_VERBOSITY 1938 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", 1939 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); 1940 #endif 1941 1942 /* 1943 * Set SendDmaLenGen and clear-then-set the MSB of the generation 1944 * count to enable generation checking and load the internal 1945 * generation counter. 1946 */ 1947 write_sde_csr(sde, SD(LEN_GEN), 1948 (sde->descq_cnt / 64) << SD(LEN_GEN_LENGTH_SHIFT)); 1949 write_sde_csr(sde, SD(LEN_GEN), 1950 ((sde->descq_cnt / 64) << SD(LEN_GEN_LENGTH_SHIFT)) | 1951 (4ULL << SD(LEN_GEN_GENERATION_SHIFT))); 1952 } 1953 1954 static inline void sdma_update_tail(struct sdma_engine *sde, u16 tail) 1955 { 1956 /* Commit writes to memory and advance the tail on the chip */ 1957 smp_wmb(); /* see get_txhead() */ 1958 writeq(tail, sde->tail_csr); 1959 } 1960 1961 /* 1962 * This is called when changing to state s10_hw_start_up_halt_wait as 1963 * a result of send buffer errors or send DMA descriptor errors. 1964 */ 1965 static void sdma_hw_start_up(struct sdma_engine *sde) 1966 { 1967 u64 reg; 1968 1969 #ifdef CONFIG_SDMA_VERBOSITY 1970 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", 1971 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); 1972 #endif 1973 1974 sdma_setlengen(sde); 1975 sdma_update_tail(sde, 0); /* Set SendDmaTail */ 1976 *sde->head_dma = 0; 1977 1978 reg = SD(ENG_ERR_CLEAR_SDMA_HEADER_REQUEST_FIFO_UNC_ERR_MASK) << 1979 SD(ENG_ERR_CLEAR_SDMA_HEADER_REQUEST_FIFO_UNC_ERR_SHIFT); 1980 write_sde_csr(sde, SD(ENG_ERR_CLEAR), reg); 1981 } 1982 1983 /* 1984 * set_sdma_integrity 1985 * 1986 * Set the SEND_DMA_CHECK_ENABLE register for send DMA engine 'sde'. 1987 */ 1988 static void set_sdma_integrity(struct sdma_engine *sde) 1989 { 1990 struct hfi1_devdata *dd = sde->dd; 1991 1992 write_sde_csr(sde, SD(CHECK_ENABLE), 1993 hfi1_pkt_base_sdma_integrity(dd)); 1994 } 1995 1996 static void init_sdma_regs( 1997 struct sdma_engine *sde, 1998 u32 credits, 1999 uint idle_cnt) 2000 { 2001 u8 opval, opmask; 2002 #ifdef CONFIG_SDMA_VERBOSITY 2003 struct hfi1_devdata *dd = sde->dd; 2004 2005 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", 2006 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); 2007 #endif 2008 2009 write_sde_csr(sde, SD(BASE_ADDR), sde->descq_phys); 2010 sdma_setlengen(sde); 2011 sdma_update_tail(sde, 0); /* Set SendDmaTail */ 2012 write_sde_csr(sde, SD(RELOAD_CNT), idle_cnt); 2013 write_sde_csr(sde, SD(DESC_CNT), 0); 2014 write_sde_csr(sde, SD(HEAD_ADDR), sde->head_phys); 2015 write_sde_csr(sde, SD(MEMORY), 2016 ((u64)credits << SD(MEMORY_SDMA_MEMORY_CNT_SHIFT)) | 2017 ((u64)(credits * sde->this_idx) << 2018 SD(MEMORY_SDMA_MEMORY_INDEX_SHIFT))); 2019 write_sde_csr(sde, SD(ENG_ERR_MASK), ~0ull); 2020 set_sdma_integrity(sde); 2021 opmask = OPCODE_CHECK_MASK_DISABLED; 2022 opval = OPCODE_CHECK_VAL_DISABLED; 2023 write_sde_csr(sde, SD(CHECK_OPCODE), 2024 (opmask << SEND_CTXT_CHECK_OPCODE_MASK_SHIFT) | 2025 (opval << SEND_CTXT_CHECK_OPCODE_VALUE_SHIFT)); 2026 } 2027 2028 #ifdef CONFIG_SDMA_VERBOSITY 2029 2030 #define sdma_dumpstate_helper0(reg) do { \ 2031 csr = read_csr(sde->dd, reg); \ 2032 dd_dev_err(sde->dd, "%36s 0x%016llx\n", #reg, csr); \ 2033 } while (0) 2034 2035 #define sdma_dumpstate_helper(reg) do { \ 2036 csr = read_sde_csr(sde, reg); \ 2037 dd_dev_err(sde->dd, "%36s[%02u] 0x%016llx\n", \ 2038 #reg, sde->this_idx, csr); \ 2039 } while (0) 2040 2041 #define sdma_dumpstate_helper2(reg) do { \ 2042 csr = read_csr(sde->dd, reg + (8 * i)); \ 2043 dd_dev_err(sde->dd, "%33s_%02u 0x%016llx\n", \ 2044 #reg, i, csr); \ 2045 } while (0) 2046 2047 void sdma_dumpstate(struct sdma_engine *sde) 2048 { 2049 u64 csr; 2050 unsigned i; 2051 2052 sdma_dumpstate_helper(SD(CTRL)); 2053 sdma_dumpstate_helper(SD(STATUS)); 2054 sdma_dumpstate_helper0(SD(ERR_STATUS)); 2055 sdma_dumpstate_helper0(SD(ERR_MASK)); 2056 sdma_dumpstate_helper(SD(ENG_ERR_STATUS)); 2057 sdma_dumpstate_helper(SD(ENG_ERR_MASK)); 2058 2059 for (i = 0; i < CCE_NUM_INT_CSRS; ++i) { 2060 sdma_dumpstate_helper2(CCE_INT_STATUS); 2061 sdma_dumpstate_helper2(CCE_INT_MASK); 2062 sdma_dumpstate_helper2(CCE_INT_BLOCKED); 2063 } 2064 2065 sdma_dumpstate_helper(SD(TAIL)); 2066 sdma_dumpstate_helper(SD(HEAD)); 2067 sdma_dumpstate_helper(SD(PRIORITY_THLD)); 2068 sdma_dumpstate_helper(SD(IDLE_CNT)); 2069 sdma_dumpstate_helper(SD(RELOAD_CNT)); 2070 sdma_dumpstate_helper(SD(DESC_CNT)); 2071 sdma_dumpstate_helper(SD(DESC_FETCHED_CNT)); 2072 sdma_dumpstate_helper(SD(MEMORY)); 2073 sdma_dumpstate_helper0(SD(ENGINES)); 2074 sdma_dumpstate_helper0(SD(MEM_SIZE)); 2075 /* sdma_dumpstate_helper(SEND_EGRESS_SEND_DMA_STATUS); */ 2076 sdma_dumpstate_helper(SD(BASE_ADDR)); 2077 sdma_dumpstate_helper(SD(LEN_GEN)); 2078 sdma_dumpstate_helper(SD(HEAD_ADDR)); 2079 sdma_dumpstate_helper(SD(CHECK_ENABLE)); 2080 sdma_dumpstate_helper(SD(CHECK_VL)); 2081 sdma_dumpstate_helper(SD(CHECK_JOB_KEY)); 2082 sdma_dumpstate_helper(SD(CHECK_PARTITION_KEY)); 2083 sdma_dumpstate_helper(SD(CHECK_SLID)); 2084 sdma_dumpstate_helper(SD(CHECK_OPCODE)); 2085 } 2086 #endif 2087 2088 static void dump_sdma_state(struct sdma_engine *sde) 2089 { 2090 struct hw_sdma_desc *descqp; 2091 u64 desc[2]; 2092 u64 addr; 2093 u8 gen; 2094 u16 len; 2095 u16 head, tail, cnt; 2096 2097 head = sde->descq_head & sde->sdma_mask; 2098 tail = sde->descq_tail & sde->sdma_mask; 2099 cnt = sdma_descq_freecnt(sde); 2100 2101 dd_dev_err(sde->dd, 2102 "SDMA (%u) descq_head: %u descq_tail: %u freecnt: %u FLE %d\n", 2103 sde->this_idx, head, tail, cnt, 2104 !list_empty(&sde->flushlist)); 2105 2106 /* print info for each entry in the descriptor queue */ 2107 while (head != tail) { 2108 char flags[6] = { 'x', 'x', 'x', 'x', 0 }; 2109 2110 descqp = &sde->descq[head]; 2111 desc[0] = le64_to_cpu(descqp->qw[0]); 2112 desc[1] = le64_to_cpu(descqp->qw[1]); 2113 flags[0] = (desc[1] & SDMA_DESC1_INT_REQ_FLAG) ? 'I' : '-'; 2114 flags[1] = (desc[1] & SDMA_DESC1_HEAD_TO_HOST_FLAG) ? 2115 'H' : '-'; 2116 flags[2] = (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) ? 'F' : '-'; 2117 flags[3] = (desc[0] & SDMA_DESC0_LAST_DESC_FLAG) ? 'L' : '-'; 2118 addr = (desc[0] >> SDMA_DESC0_PHY_ADDR_SHIFT) 2119 & SDMA_DESC0_PHY_ADDR_MASK; 2120 gen = (desc[1] >> SDMA_DESC1_GENERATION_SHIFT) 2121 & SDMA_DESC1_GENERATION_MASK; 2122 len = (desc[0] >> SDMA_DESC0_BYTE_COUNT_SHIFT) 2123 & SDMA_DESC0_BYTE_COUNT_MASK; 2124 dd_dev_err(sde->dd, 2125 "SDMA sdmadesc[%u]: flags:%s addr:0x%016llx gen:%u len:%u bytes\n", 2126 head, flags, addr, gen, len); 2127 dd_dev_err(sde->dd, 2128 "\tdesc0:0x%016llx desc1 0x%016llx\n", 2129 desc[0], desc[1]); 2130 if (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) 2131 dd_dev_err(sde->dd, 2132 "\taidx: %u amode: %u alen: %u\n", 2133 (u8)((desc[1] & 2134 SDMA_DESC1_HEADER_INDEX_SMASK) >> 2135 SDMA_DESC1_HEADER_INDEX_SHIFT), 2136 (u8)((desc[1] & 2137 SDMA_DESC1_HEADER_MODE_SMASK) >> 2138 SDMA_DESC1_HEADER_MODE_SHIFT), 2139 (u8)((desc[1] & 2140 SDMA_DESC1_HEADER_DWS_SMASK) >> 2141 SDMA_DESC1_HEADER_DWS_SHIFT)); 2142 head++; 2143 head &= sde->sdma_mask; 2144 } 2145 } 2146 2147 #define SDE_FMT \ 2148 "SDE %u CPU %d STE %s C 0x%llx S 0x%016llx E 0x%llx T(HW) 0x%llx T(SW) 0x%x H(HW) 0x%llx H(SW) 0x%x H(D) 0x%llx DM 0x%llx GL 0x%llx R 0x%llx LIS 0x%llx AHGI 0x%llx TXT %u TXH %u DT %u DH %u FLNE %d DQF %u SLC 0x%llx\n" 2149 /** 2150 * sdma_seqfile_dump_sde() - debugfs dump of sde 2151 * @s: seq file 2152 * @sde: send dma engine to dump 2153 * 2154 * This routine dumps the sde to the indicated seq file. 2155 */ 2156 void sdma_seqfile_dump_sde(struct seq_file *s, struct sdma_engine *sde) 2157 { 2158 u16 head, tail; 2159 struct hw_sdma_desc *descqp; 2160 u64 desc[2]; 2161 u64 addr; 2162 u8 gen; 2163 u16 len; 2164 2165 head = sde->descq_head & sde->sdma_mask; 2166 tail = READ_ONCE(sde->descq_tail) & sde->sdma_mask; 2167 seq_printf(s, SDE_FMT, sde->this_idx, 2168 sde->cpu, 2169 sdma_state_name(sde->state.current_state), 2170 (unsigned long long)read_sde_csr(sde, SD(CTRL)), 2171 (unsigned long long)read_sde_csr(sde, SD(STATUS)), 2172 (unsigned long long)read_sde_csr(sde, SD(ENG_ERR_STATUS)), 2173 (unsigned long long)read_sde_csr(sde, SD(TAIL)), tail, 2174 (unsigned long long)read_sde_csr(sde, SD(HEAD)), head, 2175 (unsigned long long)le64_to_cpu(*sde->head_dma), 2176 (unsigned long long)read_sde_csr(sde, SD(MEMORY)), 2177 (unsigned long long)read_sde_csr(sde, SD(LEN_GEN)), 2178 (unsigned long long)read_sde_csr(sde, SD(RELOAD_CNT)), 2179 (unsigned long long)sde->last_status, 2180 (unsigned long long)sde->ahg_bits, 2181 sde->tx_tail, 2182 sde->tx_head, 2183 sde->descq_tail, 2184 sde->descq_head, 2185 !list_empty(&sde->flushlist), 2186 sde->descq_full_count, 2187 (unsigned long long)read_sde_csr(sde, SEND_DMA_CHECK_SLID)); 2188 2189 /* print info for each entry in the descriptor queue */ 2190 while (head != tail) { 2191 char flags[6] = { 'x', 'x', 'x', 'x', 0 }; 2192 2193 descqp = &sde->descq[head]; 2194 desc[0] = le64_to_cpu(descqp->qw[0]); 2195 desc[1] = le64_to_cpu(descqp->qw[1]); 2196 flags[0] = (desc[1] & SDMA_DESC1_INT_REQ_FLAG) ? 'I' : '-'; 2197 flags[1] = (desc[1] & SDMA_DESC1_HEAD_TO_HOST_FLAG) ? 2198 'H' : '-'; 2199 flags[2] = (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) ? 'F' : '-'; 2200 flags[3] = (desc[0] & SDMA_DESC0_LAST_DESC_FLAG) ? 'L' : '-'; 2201 addr = (desc[0] >> SDMA_DESC0_PHY_ADDR_SHIFT) 2202 & SDMA_DESC0_PHY_ADDR_MASK; 2203 gen = (desc[1] >> SDMA_DESC1_GENERATION_SHIFT) 2204 & SDMA_DESC1_GENERATION_MASK; 2205 len = (desc[0] >> SDMA_DESC0_BYTE_COUNT_SHIFT) 2206 & SDMA_DESC0_BYTE_COUNT_MASK; 2207 seq_printf(s, 2208 "\tdesc[%u]: flags:%s addr:0x%016llx gen:%u len:%u bytes\n", 2209 head, flags, addr, gen, len); 2210 if (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) 2211 seq_printf(s, "\t\tahgidx: %u ahgmode: %u\n", 2212 (u8)((desc[1] & 2213 SDMA_DESC1_HEADER_INDEX_SMASK) >> 2214 SDMA_DESC1_HEADER_INDEX_SHIFT), 2215 (u8)((desc[1] & 2216 SDMA_DESC1_HEADER_MODE_SMASK) >> 2217 SDMA_DESC1_HEADER_MODE_SHIFT)); 2218 head = (head + 1) & sde->sdma_mask; 2219 } 2220 } 2221 2222 /* 2223 * add the generation number into 2224 * the qw1 and return 2225 */ 2226 static inline u64 add_gen(struct sdma_engine *sde, u64 qw1) 2227 { 2228 u8 generation = (sde->descq_tail >> sde->sdma_shift) & 3; 2229 2230 qw1 &= ~SDMA_DESC1_GENERATION_SMASK; 2231 qw1 |= ((u64)generation & SDMA_DESC1_GENERATION_MASK) 2232 << SDMA_DESC1_GENERATION_SHIFT; 2233 return qw1; 2234 } 2235 2236 /* 2237 * This routine submits the indicated tx 2238 * 2239 * Space has already been guaranteed and 2240 * tail side of ring is locked. 2241 * 2242 * The hardware tail update is done 2243 * in the caller and that is facilitated 2244 * by returning the new tail. 2245 * 2246 * There is special case logic for ahg 2247 * to not add the generation number for 2248 * up to 2 descriptors that follow the 2249 * first descriptor. 2250 * 2251 */ 2252 static inline u16 submit_tx(struct sdma_engine *sde, struct sdma_txreq *tx) 2253 { 2254 int i; 2255 u16 tail; 2256 struct sdma_desc *descp = tx->descp; 2257 u8 skip = 0, mode = ahg_mode(tx); 2258 2259 tail = sde->descq_tail & sde->sdma_mask; 2260 sde->descq[tail].qw[0] = cpu_to_le64(descp->qw[0]); 2261 sde->descq[tail].qw[1] = cpu_to_le64(add_gen(sde, descp->qw[1])); 2262 trace_hfi1_sdma_descriptor(sde, descp->qw[0], descp->qw[1], 2263 tail, &sde->descq[tail]); 2264 tail = ++sde->descq_tail & sde->sdma_mask; 2265 descp++; 2266 if (mode > SDMA_AHG_APPLY_UPDATE1) 2267 skip = mode >> 1; 2268 for (i = 1; i < tx->num_desc; i++, descp++) { 2269 u64 qw1; 2270 2271 sde->descq[tail].qw[0] = cpu_to_le64(descp->qw[0]); 2272 if (skip) { 2273 /* edits don't have generation */ 2274 qw1 = descp->qw[1]; 2275 skip--; 2276 } else { 2277 /* replace generation with real one for non-edits */ 2278 qw1 = add_gen(sde, descp->qw[1]); 2279 } 2280 sde->descq[tail].qw[1] = cpu_to_le64(qw1); 2281 trace_hfi1_sdma_descriptor(sde, descp->qw[0], qw1, 2282 tail, &sde->descq[tail]); 2283 tail = ++sde->descq_tail & sde->sdma_mask; 2284 } 2285 tx->next_descq_idx = tail; 2286 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER 2287 tx->sn = sde->tail_sn++; 2288 trace_hfi1_sdma_in_sn(sde, tx->sn); 2289 WARN_ON_ONCE(sde->tx_ring[sde->tx_tail & sde->sdma_mask]); 2290 #endif 2291 sde->tx_ring[sde->tx_tail++ & sde->sdma_mask] = tx; 2292 sde->desc_avail -= tx->num_desc; 2293 return tail; 2294 } 2295 2296 /* 2297 * Check for progress 2298 */ 2299 static int sdma_check_progress( 2300 struct sdma_engine *sde, 2301 struct iowait_work *wait, 2302 struct sdma_txreq *tx, 2303 bool pkts_sent) 2304 { 2305 int ret; 2306 2307 sde->desc_avail = sdma_descq_freecnt(sde); 2308 if (tx->num_desc <= sde->desc_avail) 2309 return -EAGAIN; 2310 /* pulse the head_lock */ 2311 if (wait && iowait_ioww_to_iow(wait)->sleep) { 2312 unsigned seq; 2313 2314 seq = raw_seqcount_begin( 2315 (const seqcount_t *)&sde->head_lock.seqcount); 2316 ret = wait->iow->sleep(sde, wait, tx, seq, pkts_sent); 2317 if (ret == -EAGAIN) 2318 sde->desc_avail = sdma_descq_freecnt(sde); 2319 } else { 2320 ret = -EBUSY; 2321 } 2322 return ret; 2323 } 2324 2325 /** 2326 * sdma_send_txreq() - submit a tx req to ring 2327 * @sde: sdma engine to use 2328 * @wait: SE wait structure to use when full (may be NULL) 2329 * @tx: sdma_txreq to submit 2330 * @pkts_sent: has any packet been sent yet? 2331 * 2332 * The call submits the tx into the ring. If a iowait structure is non-NULL 2333 * the packet will be queued to the list in wait. 2334 * 2335 * Return: 2336 * 0 - Success, -EINVAL - sdma_txreq incomplete, -EBUSY - no space in 2337 * ring (wait == NULL) 2338 * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state 2339 */ 2340 int sdma_send_txreq(struct sdma_engine *sde, 2341 struct iowait_work *wait, 2342 struct sdma_txreq *tx, 2343 bool pkts_sent) 2344 { 2345 int ret = 0; 2346 u16 tail; 2347 unsigned long flags; 2348 2349 /* user should have supplied entire packet */ 2350 if (unlikely(tx->tlen)) 2351 return -EINVAL; 2352 tx->wait = iowait_ioww_to_iow(wait); 2353 spin_lock_irqsave(&sde->tail_lock, flags); 2354 retry: 2355 if (unlikely(!__sdma_running(sde))) 2356 goto unlock_noconn; 2357 if (unlikely(tx->num_desc > sde->desc_avail)) 2358 goto nodesc; 2359 tail = submit_tx(sde, tx); 2360 if (wait) 2361 iowait_sdma_inc(iowait_ioww_to_iow(wait)); 2362 sdma_update_tail(sde, tail); 2363 unlock: 2364 spin_unlock_irqrestore(&sde->tail_lock, flags); 2365 return ret; 2366 unlock_noconn: 2367 if (wait) 2368 iowait_sdma_inc(iowait_ioww_to_iow(wait)); 2369 tx->next_descq_idx = 0; 2370 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER 2371 tx->sn = sde->tail_sn++; 2372 trace_hfi1_sdma_in_sn(sde, tx->sn); 2373 #endif 2374 spin_lock(&sde->flushlist_lock); 2375 list_add_tail(&tx->list, &sde->flushlist); 2376 spin_unlock(&sde->flushlist_lock); 2377 iowait_inc_wait_count(wait, tx->num_desc); 2378 queue_work_on(sde->cpu, system_highpri_wq, &sde->flush_worker); 2379 ret = -ECOMM; 2380 goto unlock; 2381 nodesc: 2382 ret = sdma_check_progress(sde, wait, tx, pkts_sent); 2383 if (ret == -EAGAIN) { 2384 ret = 0; 2385 goto retry; 2386 } 2387 sde->descq_full_count++; 2388 goto unlock; 2389 } 2390 2391 /** 2392 * sdma_send_txlist() - submit a list of tx req to ring 2393 * @sde: sdma engine to use 2394 * @wait: SE wait structure to use when full (may be NULL) 2395 * @tx_list: list of sdma_txreqs to submit 2396 * @count_out: pointer to a u16 which, after return will contain the total number of 2397 * sdma_txreqs removed from the tx_list. This will include sdma_txreqs 2398 * whose SDMA descriptors are submitted to the ring and the sdma_txreqs 2399 * which are added to SDMA engine flush list if the SDMA engine state is 2400 * not running. 2401 * 2402 * The call submits the list into the ring. 2403 * 2404 * If the iowait structure is non-NULL and not equal to the iowait list 2405 * the unprocessed part of the list will be appended to the list in wait. 2406 * 2407 * In all cases, the tx_list will be updated so the head of the tx_list is 2408 * the list of descriptors that have yet to be transmitted. 2409 * 2410 * The intent of this call is to provide a more efficient 2411 * way of submitting multiple packets to SDMA while holding the tail 2412 * side locking. 2413 * 2414 * Return: 2415 * 0 - Success, 2416 * -EINVAL - sdma_txreq incomplete, -EBUSY - no space in ring (wait == NULL) 2417 * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state 2418 */ 2419 int sdma_send_txlist(struct sdma_engine *sde, struct iowait_work *wait, 2420 struct list_head *tx_list, u16 *count_out) 2421 { 2422 struct sdma_txreq *tx, *tx_next; 2423 int ret = 0; 2424 unsigned long flags; 2425 u16 tail = INVALID_TAIL; 2426 u32 submit_count = 0, flush_count = 0, total_count; 2427 2428 spin_lock_irqsave(&sde->tail_lock, flags); 2429 retry: 2430 list_for_each_entry_safe(tx, tx_next, tx_list, list) { 2431 tx->wait = iowait_ioww_to_iow(wait); 2432 if (unlikely(!__sdma_running(sde))) 2433 goto unlock_noconn; 2434 if (unlikely(tx->num_desc > sde->desc_avail)) 2435 goto nodesc; 2436 if (unlikely(tx->tlen)) { 2437 ret = -EINVAL; 2438 goto update_tail; 2439 } 2440 list_del_init(&tx->list); 2441 tail = submit_tx(sde, tx); 2442 submit_count++; 2443 if (tail != INVALID_TAIL && 2444 (submit_count & SDMA_TAIL_UPDATE_THRESH) == 0) { 2445 sdma_update_tail(sde, tail); 2446 tail = INVALID_TAIL; 2447 } 2448 } 2449 update_tail: 2450 total_count = submit_count + flush_count; 2451 if (wait) { 2452 iowait_sdma_add(iowait_ioww_to_iow(wait), total_count); 2453 iowait_starve_clear(submit_count > 0, 2454 iowait_ioww_to_iow(wait)); 2455 } 2456 if (tail != INVALID_TAIL) 2457 sdma_update_tail(sde, tail); 2458 spin_unlock_irqrestore(&sde->tail_lock, flags); 2459 *count_out = total_count; 2460 return ret; 2461 unlock_noconn: 2462 spin_lock(&sde->flushlist_lock); 2463 list_for_each_entry_safe(tx, tx_next, tx_list, list) { 2464 tx->wait = iowait_ioww_to_iow(wait); 2465 list_del_init(&tx->list); 2466 tx->next_descq_idx = 0; 2467 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER 2468 tx->sn = sde->tail_sn++; 2469 trace_hfi1_sdma_in_sn(sde, tx->sn); 2470 #endif 2471 list_add_tail(&tx->list, &sde->flushlist); 2472 flush_count++; 2473 iowait_inc_wait_count(wait, tx->num_desc); 2474 } 2475 spin_unlock(&sde->flushlist_lock); 2476 queue_work_on(sde->cpu, system_highpri_wq, &sde->flush_worker); 2477 ret = -ECOMM; 2478 goto update_tail; 2479 nodesc: 2480 ret = sdma_check_progress(sde, wait, tx, submit_count > 0); 2481 if (ret == -EAGAIN) { 2482 ret = 0; 2483 goto retry; 2484 } 2485 sde->descq_full_count++; 2486 goto update_tail; 2487 } 2488 2489 static void sdma_process_event(struct sdma_engine *sde, enum sdma_events event) 2490 { 2491 unsigned long flags; 2492 2493 spin_lock_irqsave(&sde->tail_lock, flags); 2494 write_seqlock(&sde->head_lock); 2495 2496 __sdma_process_event(sde, event); 2497 2498 if (sde->state.current_state == sdma_state_s99_running) 2499 sdma_desc_avail(sde, sdma_descq_freecnt(sde)); 2500 2501 write_sequnlock(&sde->head_lock); 2502 spin_unlock_irqrestore(&sde->tail_lock, flags); 2503 } 2504 2505 static void __sdma_process_event(struct sdma_engine *sde, 2506 enum sdma_events event) 2507 { 2508 struct sdma_state *ss = &sde->state; 2509 int need_progress = 0; 2510 2511 /* CONFIG SDMA temporary */ 2512 #ifdef CONFIG_SDMA_VERBOSITY 2513 dd_dev_err(sde->dd, "CONFIG SDMA(%u) [%s] %s\n", sde->this_idx, 2514 sdma_state_names[ss->current_state], 2515 sdma_event_names[event]); 2516 #endif 2517 2518 switch (ss->current_state) { 2519 case sdma_state_s00_hw_down: 2520 switch (event) { 2521 case sdma_event_e00_go_hw_down: 2522 break; 2523 case sdma_event_e30_go_running: 2524 /* 2525 * If down, but running requested (usually result 2526 * of link up, then we need to start up. 2527 * This can happen when hw down is requested while 2528 * bringing the link up with traffic active on 2529 * 7220, e.g. 2530 */ 2531 ss->go_s99_running = 1; 2532 fallthrough; /* and start dma engine */ 2533 case sdma_event_e10_go_hw_start: 2534 /* This reference means the state machine is started */ 2535 sdma_get(&sde->state); 2536 sdma_set_state(sde, 2537 sdma_state_s10_hw_start_up_halt_wait); 2538 break; 2539 case sdma_event_e15_hw_halt_done: 2540 break; 2541 case sdma_event_e25_hw_clean_up_done: 2542 break; 2543 case sdma_event_e40_sw_cleaned: 2544 sdma_sw_tear_down(sde); 2545 break; 2546 case sdma_event_e50_hw_cleaned: 2547 break; 2548 case sdma_event_e60_hw_halted: 2549 break; 2550 case sdma_event_e70_go_idle: 2551 break; 2552 case sdma_event_e80_hw_freeze: 2553 break; 2554 case sdma_event_e81_hw_frozen: 2555 break; 2556 case sdma_event_e82_hw_unfreeze: 2557 break; 2558 case sdma_event_e85_link_down: 2559 break; 2560 case sdma_event_e90_sw_halted: 2561 break; 2562 } 2563 break; 2564 2565 case sdma_state_s10_hw_start_up_halt_wait: 2566 switch (event) { 2567 case sdma_event_e00_go_hw_down: 2568 sdma_set_state(sde, sdma_state_s00_hw_down); 2569 sdma_sw_tear_down(sde); 2570 break; 2571 case sdma_event_e10_go_hw_start: 2572 break; 2573 case sdma_event_e15_hw_halt_done: 2574 sdma_set_state(sde, 2575 sdma_state_s15_hw_start_up_clean_wait); 2576 sdma_start_hw_clean_up(sde); 2577 break; 2578 case sdma_event_e25_hw_clean_up_done: 2579 break; 2580 case sdma_event_e30_go_running: 2581 ss->go_s99_running = 1; 2582 break; 2583 case sdma_event_e40_sw_cleaned: 2584 break; 2585 case sdma_event_e50_hw_cleaned: 2586 break; 2587 case sdma_event_e60_hw_halted: 2588 schedule_work(&sde->err_halt_worker); 2589 break; 2590 case sdma_event_e70_go_idle: 2591 ss->go_s99_running = 0; 2592 break; 2593 case sdma_event_e80_hw_freeze: 2594 break; 2595 case sdma_event_e81_hw_frozen: 2596 break; 2597 case sdma_event_e82_hw_unfreeze: 2598 break; 2599 case sdma_event_e85_link_down: 2600 break; 2601 case sdma_event_e90_sw_halted: 2602 break; 2603 } 2604 break; 2605 2606 case sdma_state_s15_hw_start_up_clean_wait: 2607 switch (event) { 2608 case sdma_event_e00_go_hw_down: 2609 sdma_set_state(sde, sdma_state_s00_hw_down); 2610 sdma_sw_tear_down(sde); 2611 break; 2612 case sdma_event_e10_go_hw_start: 2613 break; 2614 case sdma_event_e15_hw_halt_done: 2615 break; 2616 case sdma_event_e25_hw_clean_up_done: 2617 sdma_hw_start_up(sde); 2618 sdma_set_state(sde, ss->go_s99_running ? 2619 sdma_state_s99_running : 2620 sdma_state_s20_idle); 2621 break; 2622 case sdma_event_e30_go_running: 2623 ss->go_s99_running = 1; 2624 break; 2625 case sdma_event_e40_sw_cleaned: 2626 break; 2627 case sdma_event_e50_hw_cleaned: 2628 break; 2629 case sdma_event_e60_hw_halted: 2630 break; 2631 case sdma_event_e70_go_idle: 2632 ss->go_s99_running = 0; 2633 break; 2634 case sdma_event_e80_hw_freeze: 2635 break; 2636 case sdma_event_e81_hw_frozen: 2637 break; 2638 case sdma_event_e82_hw_unfreeze: 2639 break; 2640 case sdma_event_e85_link_down: 2641 break; 2642 case sdma_event_e90_sw_halted: 2643 break; 2644 } 2645 break; 2646 2647 case sdma_state_s20_idle: 2648 switch (event) { 2649 case sdma_event_e00_go_hw_down: 2650 sdma_set_state(sde, sdma_state_s00_hw_down); 2651 sdma_sw_tear_down(sde); 2652 break; 2653 case sdma_event_e10_go_hw_start: 2654 break; 2655 case sdma_event_e15_hw_halt_done: 2656 break; 2657 case sdma_event_e25_hw_clean_up_done: 2658 break; 2659 case sdma_event_e30_go_running: 2660 sdma_set_state(sde, sdma_state_s99_running); 2661 ss->go_s99_running = 1; 2662 break; 2663 case sdma_event_e40_sw_cleaned: 2664 break; 2665 case sdma_event_e50_hw_cleaned: 2666 break; 2667 case sdma_event_e60_hw_halted: 2668 sdma_set_state(sde, sdma_state_s50_hw_halt_wait); 2669 schedule_work(&sde->err_halt_worker); 2670 break; 2671 case sdma_event_e70_go_idle: 2672 break; 2673 case sdma_event_e85_link_down: 2674 case sdma_event_e80_hw_freeze: 2675 sdma_set_state(sde, sdma_state_s80_hw_freeze); 2676 atomic_dec(&sde->dd->sdma_unfreeze_count); 2677 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); 2678 break; 2679 case sdma_event_e81_hw_frozen: 2680 break; 2681 case sdma_event_e82_hw_unfreeze: 2682 break; 2683 case sdma_event_e90_sw_halted: 2684 break; 2685 } 2686 break; 2687 2688 case sdma_state_s30_sw_clean_up_wait: 2689 switch (event) { 2690 case sdma_event_e00_go_hw_down: 2691 sdma_set_state(sde, sdma_state_s00_hw_down); 2692 break; 2693 case sdma_event_e10_go_hw_start: 2694 break; 2695 case sdma_event_e15_hw_halt_done: 2696 break; 2697 case sdma_event_e25_hw_clean_up_done: 2698 break; 2699 case sdma_event_e30_go_running: 2700 ss->go_s99_running = 1; 2701 break; 2702 case sdma_event_e40_sw_cleaned: 2703 sdma_set_state(sde, sdma_state_s40_hw_clean_up_wait); 2704 sdma_start_hw_clean_up(sde); 2705 break; 2706 case sdma_event_e50_hw_cleaned: 2707 break; 2708 case sdma_event_e60_hw_halted: 2709 break; 2710 case sdma_event_e70_go_idle: 2711 ss->go_s99_running = 0; 2712 break; 2713 case sdma_event_e80_hw_freeze: 2714 break; 2715 case sdma_event_e81_hw_frozen: 2716 break; 2717 case sdma_event_e82_hw_unfreeze: 2718 break; 2719 case sdma_event_e85_link_down: 2720 ss->go_s99_running = 0; 2721 break; 2722 case sdma_event_e90_sw_halted: 2723 break; 2724 } 2725 break; 2726 2727 case sdma_state_s40_hw_clean_up_wait: 2728 switch (event) { 2729 case sdma_event_e00_go_hw_down: 2730 sdma_set_state(sde, sdma_state_s00_hw_down); 2731 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2732 break; 2733 case sdma_event_e10_go_hw_start: 2734 break; 2735 case sdma_event_e15_hw_halt_done: 2736 break; 2737 case sdma_event_e25_hw_clean_up_done: 2738 sdma_hw_start_up(sde); 2739 sdma_set_state(sde, ss->go_s99_running ? 2740 sdma_state_s99_running : 2741 sdma_state_s20_idle); 2742 break; 2743 case sdma_event_e30_go_running: 2744 ss->go_s99_running = 1; 2745 break; 2746 case sdma_event_e40_sw_cleaned: 2747 break; 2748 case sdma_event_e50_hw_cleaned: 2749 break; 2750 case sdma_event_e60_hw_halted: 2751 break; 2752 case sdma_event_e70_go_idle: 2753 ss->go_s99_running = 0; 2754 break; 2755 case sdma_event_e80_hw_freeze: 2756 break; 2757 case sdma_event_e81_hw_frozen: 2758 break; 2759 case sdma_event_e82_hw_unfreeze: 2760 break; 2761 case sdma_event_e85_link_down: 2762 ss->go_s99_running = 0; 2763 break; 2764 case sdma_event_e90_sw_halted: 2765 break; 2766 } 2767 break; 2768 2769 case sdma_state_s50_hw_halt_wait: 2770 switch (event) { 2771 case sdma_event_e00_go_hw_down: 2772 sdma_set_state(sde, sdma_state_s00_hw_down); 2773 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2774 break; 2775 case sdma_event_e10_go_hw_start: 2776 break; 2777 case sdma_event_e15_hw_halt_done: 2778 sdma_set_state(sde, sdma_state_s30_sw_clean_up_wait); 2779 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2780 break; 2781 case sdma_event_e25_hw_clean_up_done: 2782 break; 2783 case sdma_event_e30_go_running: 2784 ss->go_s99_running = 1; 2785 break; 2786 case sdma_event_e40_sw_cleaned: 2787 break; 2788 case sdma_event_e50_hw_cleaned: 2789 break; 2790 case sdma_event_e60_hw_halted: 2791 schedule_work(&sde->err_halt_worker); 2792 break; 2793 case sdma_event_e70_go_idle: 2794 ss->go_s99_running = 0; 2795 break; 2796 case sdma_event_e80_hw_freeze: 2797 break; 2798 case sdma_event_e81_hw_frozen: 2799 break; 2800 case sdma_event_e82_hw_unfreeze: 2801 break; 2802 case sdma_event_e85_link_down: 2803 ss->go_s99_running = 0; 2804 break; 2805 case sdma_event_e90_sw_halted: 2806 break; 2807 } 2808 break; 2809 2810 case sdma_state_s60_idle_halt_wait: 2811 switch (event) { 2812 case sdma_event_e00_go_hw_down: 2813 sdma_set_state(sde, sdma_state_s00_hw_down); 2814 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2815 break; 2816 case sdma_event_e10_go_hw_start: 2817 break; 2818 case sdma_event_e15_hw_halt_done: 2819 sdma_set_state(sde, sdma_state_s30_sw_clean_up_wait); 2820 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2821 break; 2822 case sdma_event_e25_hw_clean_up_done: 2823 break; 2824 case sdma_event_e30_go_running: 2825 ss->go_s99_running = 1; 2826 break; 2827 case sdma_event_e40_sw_cleaned: 2828 break; 2829 case sdma_event_e50_hw_cleaned: 2830 break; 2831 case sdma_event_e60_hw_halted: 2832 schedule_work(&sde->err_halt_worker); 2833 break; 2834 case sdma_event_e70_go_idle: 2835 ss->go_s99_running = 0; 2836 break; 2837 case sdma_event_e80_hw_freeze: 2838 break; 2839 case sdma_event_e81_hw_frozen: 2840 break; 2841 case sdma_event_e82_hw_unfreeze: 2842 break; 2843 case sdma_event_e85_link_down: 2844 break; 2845 case sdma_event_e90_sw_halted: 2846 break; 2847 } 2848 break; 2849 2850 case sdma_state_s80_hw_freeze: 2851 switch (event) { 2852 case sdma_event_e00_go_hw_down: 2853 sdma_set_state(sde, sdma_state_s00_hw_down); 2854 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2855 break; 2856 case sdma_event_e10_go_hw_start: 2857 break; 2858 case sdma_event_e15_hw_halt_done: 2859 break; 2860 case sdma_event_e25_hw_clean_up_done: 2861 break; 2862 case sdma_event_e30_go_running: 2863 ss->go_s99_running = 1; 2864 break; 2865 case sdma_event_e40_sw_cleaned: 2866 break; 2867 case sdma_event_e50_hw_cleaned: 2868 break; 2869 case sdma_event_e60_hw_halted: 2870 break; 2871 case sdma_event_e70_go_idle: 2872 ss->go_s99_running = 0; 2873 break; 2874 case sdma_event_e80_hw_freeze: 2875 break; 2876 case sdma_event_e81_hw_frozen: 2877 sdma_set_state(sde, sdma_state_s82_freeze_sw_clean); 2878 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2879 break; 2880 case sdma_event_e82_hw_unfreeze: 2881 break; 2882 case sdma_event_e85_link_down: 2883 break; 2884 case sdma_event_e90_sw_halted: 2885 break; 2886 } 2887 break; 2888 2889 case sdma_state_s82_freeze_sw_clean: 2890 switch (event) { 2891 case sdma_event_e00_go_hw_down: 2892 sdma_set_state(sde, sdma_state_s00_hw_down); 2893 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2894 break; 2895 case sdma_event_e10_go_hw_start: 2896 break; 2897 case sdma_event_e15_hw_halt_done: 2898 break; 2899 case sdma_event_e25_hw_clean_up_done: 2900 break; 2901 case sdma_event_e30_go_running: 2902 ss->go_s99_running = 1; 2903 break; 2904 case sdma_event_e40_sw_cleaned: 2905 /* notify caller this engine is done cleaning */ 2906 atomic_dec(&sde->dd->sdma_unfreeze_count); 2907 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); 2908 break; 2909 case sdma_event_e50_hw_cleaned: 2910 break; 2911 case sdma_event_e60_hw_halted: 2912 break; 2913 case sdma_event_e70_go_idle: 2914 ss->go_s99_running = 0; 2915 break; 2916 case sdma_event_e80_hw_freeze: 2917 break; 2918 case sdma_event_e81_hw_frozen: 2919 break; 2920 case sdma_event_e82_hw_unfreeze: 2921 sdma_hw_start_up(sde); 2922 sdma_set_state(sde, ss->go_s99_running ? 2923 sdma_state_s99_running : 2924 sdma_state_s20_idle); 2925 break; 2926 case sdma_event_e85_link_down: 2927 break; 2928 case sdma_event_e90_sw_halted: 2929 break; 2930 } 2931 break; 2932 2933 case sdma_state_s99_running: 2934 switch (event) { 2935 case sdma_event_e00_go_hw_down: 2936 sdma_set_state(sde, sdma_state_s00_hw_down); 2937 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2938 break; 2939 case sdma_event_e10_go_hw_start: 2940 break; 2941 case sdma_event_e15_hw_halt_done: 2942 break; 2943 case sdma_event_e25_hw_clean_up_done: 2944 break; 2945 case sdma_event_e30_go_running: 2946 break; 2947 case sdma_event_e40_sw_cleaned: 2948 break; 2949 case sdma_event_e50_hw_cleaned: 2950 break; 2951 case sdma_event_e60_hw_halted: 2952 need_progress = 1; 2953 sdma_err_progress_check_schedule(sde); 2954 fallthrough; 2955 case sdma_event_e90_sw_halted: 2956 /* 2957 * SW initiated halt does not perform engines 2958 * progress check 2959 */ 2960 sdma_set_state(sde, sdma_state_s50_hw_halt_wait); 2961 schedule_work(&sde->err_halt_worker); 2962 break; 2963 case sdma_event_e70_go_idle: 2964 sdma_set_state(sde, sdma_state_s60_idle_halt_wait); 2965 break; 2966 case sdma_event_e85_link_down: 2967 ss->go_s99_running = 0; 2968 fallthrough; 2969 case sdma_event_e80_hw_freeze: 2970 sdma_set_state(sde, sdma_state_s80_hw_freeze); 2971 atomic_dec(&sde->dd->sdma_unfreeze_count); 2972 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); 2973 break; 2974 case sdma_event_e81_hw_frozen: 2975 break; 2976 case sdma_event_e82_hw_unfreeze: 2977 break; 2978 } 2979 break; 2980 } 2981 2982 ss->last_event = event; 2983 if (need_progress) 2984 sdma_make_progress(sde, 0); 2985 } 2986 2987 /* 2988 * _extend_sdma_tx_descs() - helper to extend txreq 2989 * 2990 * This is called once the initial nominal allocation 2991 * of descriptors in the sdma_txreq is exhausted. 2992 * 2993 * The code will bump the allocation up to the max 2994 * of MAX_DESC (64) descriptors. There doesn't seem 2995 * much point in an interim step. The last descriptor 2996 * is reserved for coalesce buffer in order to support 2997 * cases where input packet has >MAX_DESC iovecs. 2998 * 2999 */ 3000 static int _extend_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx) 3001 { 3002 int i; 3003 struct sdma_desc *descp; 3004 3005 /* Handle last descriptor */ 3006 if (unlikely((tx->num_desc == (MAX_DESC - 1)))) { 3007 /* if tlen is 0, it is for padding, release last descriptor */ 3008 if (!tx->tlen) { 3009 tx->desc_limit = MAX_DESC; 3010 } else if (!tx->coalesce_buf) { 3011 /* allocate coalesce buffer with space for padding */ 3012 tx->coalesce_buf = kmalloc(tx->tlen + sizeof(u32), 3013 GFP_ATOMIC); 3014 if (!tx->coalesce_buf) 3015 goto enomem; 3016 tx->coalesce_idx = 0; 3017 } 3018 return 0; 3019 } 3020 3021 if (unlikely(tx->num_desc == MAX_DESC)) 3022 goto enomem; 3023 3024 descp = kmalloc_array(MAX_DESC, sizeof(struct sdma_desc), GFP_ATOMIC); 3025 if (!descp) 3026 goto enomem; 3027 tx->descp = descp; 3028 3029 /* reserve last descriptor for coalescing */ 3030 tx->desc_limit = MAX_DESC - 1; 3031 /* copy ones already built */ 3032 for (i = 0; i < tx->num_desc; i++) 3033 tx->descp[i] = tx->descs[i]; 3034 return 0; 3035 enomem: 3036 __sdma_txclean(dd, tx); 3037 return -ENOMEM; 3038 } 3039 3040 /* 3041 * ext_coal_sdma_tx_descs() - extend or coalesce sdma tx descriptors 3042 * 3043 * This is called once the initial nominal allocation of descriptors 3044 * in the sdma_txreq is exhausted. 3045 * 3046 * This function calls _extend_sdma_tx_descs to extend or allocate 3047 * coalesce buffer. If there is a allocated coalesce buffer, it will 3048 * copy the input packet data into the coalesce buffer. It also adds 3049 * coalesce buffer descriptor once when whole packet is received. 3050 * 3051 * Return: 3052 * <0 - error 3053 * 0 - coalescing, don't populate descriptor 3054 * 1 - continue with populating descriptor 3055 */ 3056 int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx, 3057 int type, void *kvaddr, struct page *page, 3058 unsigned long offset, u16 len) 3059 { 3060 int pad_len, rval; 3061 dma_addr_t addr; 3062 3063 rval = _extend_sdma_tx_descs(dd, tx); 3064 if (rval) { 3065 __sdma_txclean(dd, tx); 3066 return rval; 3067 } 3068 3069 /* If coalesce buffer is allocated, copy data into it */ 3070 if (tx->coalesce_buf) { 3071 if (type == SDMA_MAP_NONE) { 3072 __sdma_txclean(dd, tx); 3073 return -EINVAL; 3074 } 3075 3076 if (type == SDMA_MAP_PAGE) { 3077 kvaddr = kmap_local_page(page); 3078 kvaddr += offset; 3079 } else if (WARN_ON(!kvaddr)) { 3080 __sdma_txclean(dd, tx); 3081 return -EINVAL; 3082 } 3083 3084 memcpy(tx->coalesce_buf + tx->coalesce_idx, kvaddr, len); 3085 tx->coalesce_idx += len; 3086 if (type == SDMA_MAP_PAGE) 3087 kunmap_local(kvaddr); 3088 3089 /* If there is more data, return */ 3090 if (tx->tlen - tx->coalesce_idx) 3091 return 0; 3092 3093 /* Whole packet is received; add any padding */ 3094 pad_len = tx->packet_len & (sizeof(u32) - 1); 3095 if (pad_len) { 3096 pad_len = sizeof(u32) - pad_len; 3097 memset(tx->coalesce_buf + tx->coalesce_idx, 0, pad_len); 3098 /* padding is taken care of for coalescing case */ 3099 tx->packet_len += pad_len; 3100 tx->tlen += pad_len; 3101 } 3102 3103 /* dma map the coalesce buffer */ 3104 addr = dma_map_single(&dd->pcidev->dev, 3105 tx->coalesce_buf, 3106 tx->tlen, 3107 DMA_TO_DEVICE); 3108 3109 if (unlikely(dma_mapping_error(&dd->pcidev->dev, addr))) { 3110 __sdma_txclean(dd, tx); 3111 return -ENOSPC; 3112 } 3113 3114 /* Add descriptor for coalesce buffer */ 3115 tx->desc_limit = MAX_DESC; 3116 return _sdma_txadd_daddr(dd, SDMA_MAP_SINGLE, NULL, tx, 3117 addr, tx->tlen); 3118 } 3119 3120 return 1; 3121 } 3122 3123 /* Update sdes when the lmc changes */ 3124 void sdma_update_lmc(struct hfi1_devdata *dd, u64 mask, u32 lid) 3125 { 3126 struct sdma_engine *sde; 3127 int i; 3128 u64 sreg; 3129 3130 sreg = ((mask & SD(CHECK_SLID_MASK_MASK)) << 3131 SD(CHECK_SLID_MASK_SHIFT)) | 3132 (((lid & mask) & SD(CHECK_SLID_VALUE_MASK)) << 3133 SD(CHECK_SLID_VALUE_SHIFT)); 3134 3135 for (i = 0; i < dd->num_sdma; i++) { 3136 hfi1_cdbg(LINKVERB, "SendDmaEngine[%d].SLID_CHECK = 0x%x", 3137 i, (u32)sreg); 3138 sde = &dd->per_sdma[i]; 3139 write_sde_csr(sde, SD(CHECK_SLID), sreg); 3140 } 3141 } 3142 3143 /* tx not dword sized - pad */ 3144 int _pad_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx) 3145 { 3146 int rval = 0; 3147 3148 if ((unlikely(tx->num_desc + 1 == tx->desc_limit))) { 3149 rval = _extend_sdma_tx_descs(dd, tx); 3150 if (rval) { 3151 __sdma_txclean(dd, tx); 3152 return rval; 3153 } 3154 } 3155 3156 /* finish the one just added */ 3157 make_tx_sdma_desc( 3158 tx, 3159 SDMA_MAP_NONE, 3160 NULL, 3161 dd->sdma_pad_phys, 3162 sizeof(u32) - (tx->packet_len & (sizeof(u32) - 1))); 3163 tx->num_desc++; 3164 _sdma_close_tx(dd, tx); 3165 return rval; 3166 } 3167 3168 /* 3169 * Add ahg to the sdma_txreq 3170 * 3171 * The logic will consume up to 3 3172 * descriptors at the beginning of 3173 * sdma_txreq. 3174 */ 3175 void _sdma_txreq_ahgadd( 3176 struct sdma_txreq *tx, 3177 u8 num_ahg, 3178 u8 ahg_entry, 3179 u32 *ahg, 3180 u8 ahg_hlen) 3181 { 3182 u32 i, shift = 0, desc = 0; 3183 u8 mode; 3184 3185 WARN_ON_ONCE(num_ahg > 9 || (ahg_hlen & 3) || ahg_hlen == 4); 3186 /* compute mode */ 3187 if (num_ahg == 1) 3188 mode = SDMA_AHG_APPLY_UPDATE1; 3189 else if (num_ahg <= 5) 3190 mode = SDMA_AHG_APPLY_UPDATE2; 3191 else 3192 mode = SDMA_AHG_APPLY_UPDATE3; 3193 tx->num_desc++; 3194 /* initialize to consumed descriptors to zero */ 3195 switch (mode) { 3196 case SDMA_AHG_APPLY_UPDATE3: 3197 tx->num_desc++; 3198 tx->descs[2].qw[0] = 0; 3199 tx->descs[2].qw[1] = 0; 3200 fallthrough; 3201 case SDMA_AHG_APPLY_UPDATE2: 3202 tx->num_desc++; 3203 tx->descs[1].qw[0] = 0; 3204 tx->descs[1].qw[1] = 0; 3205 break; 3206 } 3207 ahg_hlen >>= 2; 3208 tx->descs[0].qw[1] |= 3209 (((u64)ahg_entry & SDMA_DESC1_HEADER_INDEX_MASK) 3210 << SDMA_DESC1_HEADER_INDEX_SHIFT) | 3211 (((u64)ahg_hlen & SDMA_DESC1_HEADER_DWS_MASK) 3212 << SDMA_DESC1_HEADER_DWS_SHIFT) | 3213 (((u64)mode & SDMA_DESC1_HEADER_MODE_MASK) 3214 << SDMA_DESC1_HEADER_MODE_SHIFT) | 3215 (((u64)ahg[0] & SDMA_DESC1_HEADER_UPDATE1_MASK) 3216 << SDMA_DESC1_HEADER_UPDATE1_SHIFT); 3217 for (i = 0; i < (num_ahg - 1); i++) { 3218 if (!shift && !(i & 2)) 3219 desc++; 3220 tx->descs[desc].qw[!!(i & 2)] |= 3221 (((u64)ahg[i + 1]) 3222 << shift); 3223 shift = (shift + 32) & 63; 3224 } 3225 } 3226 3227 /** 3228 * sdma_ahg_alloc - allocate an AHG entry 3229 * @sde: engine to allocate from 3230 * 3231 * Return: 3232 * 0-31 when successful, -EOPNOTSUPP if AHG is not enabled, 3233 * -ENOSPC if an entry is not available 3234 */ 3235 int sdma_ahg_alloc(struct sdma_engine *sde) 3236 { 3237 int nr; 3238 int oldbit; 3239 3240 if (!sde) { 3241 trace_hfi1_ahg_allocate(sde, -EINVAL); 3242 return -EINVAL; 3243 } 3244 while (1) { 3245 nr = ffz(READ_ONCE(sde->ahg_bits)); 3246 if (nr > 31) { 3247 trace_hfi1_ahg_allocate(sde, -ENOSPC); 3248 return -ENOSPC; 3249 } 3250 oldbit = test_and_set_bit(nr, &sde->ahg_bits); 3251 if (!oldbit) 3252 break; 3253 cpu_relax(); 3254 } 3255 trace_hfi1_ahg_allocate(sde, nr); 3256 return nr; 3257 } 3258 3259 /** 3260 * sdma_ahg_free - free an AHG entry 3261 * @sde: engine to return AHG entry 3262 * @ahg_index: index to free 3263 * 3264 * This routine frees the indicate AHG entry. 3265 */ 3266 void sdma_ahg_free(struct sdma_engine *sde, int ahg_index) 3267 { 3268 if (!sde) 3269 return; 3270 trace_hfi1_ahg_deallocate(sde, ahg_index); 3271 if (ahg_index < 0 || ahg_index > 31) 3272 return; 3273 clear_bit(ahg_index, &sde->ahg_bits); 3274 } 3275 3276 /* 3277 * SPC freeze handling for SDMA engines. Called when the driver knows 3278 * the SPC is going into a freeze but before the freeze is fully 3279 * settled. Generally an error interrupt. 3280 * 3281 * This event will pull the engine out of running so no more entries can be 3282 * added to the engine's queue. 3283 */ 3284 void sdma_freeze_notify(struct hfi1_devdata *dd, int link_down) 3285 { 3286 int i; 3287 enum sdma_events event = link_down ? sdma_event_e85_link_down : 3288 sdma_event_e80_hw_freeze; 3289 3290 /* set up the wait but do not wait here */ 3291 atomic_set(&dd->sdma_unfreeze_count, dd->num_sdma); 3292 3293 /* tell all engines to stop running and wait */ 3294 for (i = 0; i < dd->num_sdma; i++) 3295 sdma_process_event(&dd->per_sdma[i], event); 3296 3297 /* sdma_freeze() will wait for all engines to have stopped */ 3298 } 3299 3300 /* 3301 * SPC freeze handling for SDMA engines. Called when the driver knows 3302 * the SPC is fully frozen. 3303 */ 3304 void sdma_freeze(struct hfi1_devdata *dd) 3305 { 3306 int i; 3307 int ret; 3308 3309 /* 3310 * Make sure all engines have moved out of the running state before 3311 * continuing. 3312 */ 3313 ret = wait_event_interruptible(dd->sdma_unfreeze_wq, 3314 atomic_read(&dd->sdma_unfreeze_count) <= 3315 0); 3316 /* interrupted or count is negative, then unloading - just exit */ 3317 if (ret || atomic_read(&dd->sdma_unfreeze_count) < 0) 3318 return; 3319 3320 /* set up the count for the next wait */ 3321 atomic_set(&dd->sdma_unfreeze_count, dd->num_sdma); 3322 3323 /* tell all engines that the SPC is frozen, they can start cleaning */ 3324 for (i = 0; i < dd->num_sdma; i++) 3325 sdma_process_event(&dd->per_sdma[i], sdma_event_e81_hw_frozen); 3326 3327 /* 3328 * Wait for everyone to finish software clean before exiting. The 3329 * software clean will read engine CSRs, so must be completed before 3330 * the next step, which will clear the engine CSRs. 3331 */ 3332 (void)wait_event_interruptible(dd->sdma_unfreeze_wq, 3333 atomic_read(&dd->sdma_unfreeze_count) <= 0); 3334 /* no need to check results - done no matter what */ 3335 } 3336 3337 /* 3338 * SPC freeze handling for the SDMA engines. Called after the SPC is unfrozen. 3339 * 3340 * The SPC freeze acts like a SDMA halt and a hardware clean combined. All 3341 * that is left is a software clean. We could do it after the SPC is fully 3342 * frozen, but then we'd have to add another state to wait for the unfreeze. 3343 * Instead, just defer the software clean until the unfreeze step. 3344 */ 3345 void sdma_unfreeze(struct hfi1_devdata *dd) 3346 { 3347 int i; 3348 3349 /* tell all engines start freeze clean up */ 3350 for (i = 0; i < dd->num_sdma; i++) 3351 sdma_process_event(&dd->per_sdma[i], 3352 sdma_event_e82_hw_unfreeze); 3353 } 3354 3355 /** 3356 * _sdma_engine_progress_schedule() - schedule progress on engine 3357 * @sde: sdma_engine to schedule progress 3358 * 3359 */ 3360 void _sdma_engine_progress_schedule( 3361 struct sdma_engine *sde) 3362 { 3363 trace_hfi1_sdma_engine_progress(sde, sde->progress_mask); 3364 /* assume we have selected a good cpu */ 3365 write_csr(sde->dd, 3366 CCE_INT_FORCE + (8 * (IS_SDMA_START / 64)), 3367 sde->progress_mask); 3368 } 3369