1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2021 Broadcom. All Rights Reserved. The term
4 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
5 */
6
7 #include "efct_driver.h"
8 #include "efct_hw.h"
9
10 #define enable_tsend_auto_resp(efct) 1
11 #define enable_treceive_auto_resp(efct) 0
12
13 #define SCSI_IOFMT "[%04x][i:%04x t:%04x h:%04x]"
14
15 #define scsi_io_printf(io, fmt, ...) \
16 efc_log_debug(io->efct, "[%s]" SCSI_IOFMT fmt, \
17 io->node->display_name, io->instance_index,\
18 io->init_task_tag, io->tgt_task_tag, io->hw_tag, ##__VA_ARGS__)
19
20 #define EFCT_LOG_ENABLE_SCSI_TRACE(efct) \
21 (((efct) != NULL) ? (((efct)->logmask & (1U << 2)) != 0) : 0)
22
23 #define scsi_io_trace(io, fmt, ...) \
24 do { \
25 if (EFCT_LOG_ENABLE_SCSI_TRACE(io->efct)) \
26 scsi_io_printf(io, fmt, ##__VA_ARGS__); \
27 } while (0)
28
29 struct efct_io *
efct_scsi_io_alloc(struct efct_node * node)30 efct_scsi_io_alloc(struct efct_node *node)
31 {
32 struct efct *efct;
33 struct efct_xport *xport;
34 struct efct_io *io;
35 unsigned long flags;
36
37 efct = node->efct;
38
39 xport = efct->xport;
40
41 io = efct_io_pool_io_alloc(efct->xport->io_pool);
42 if (!io) {
43 efc_log_err(efct, "IO alloc Failed\n");
44 atomic_add_return(1, &xport->io_alloc_failed_count);
45 return NULL;
46 }
47
48 /* initialize refcount */
49 kref_init(&io->ref);
50 io->release = _efct_scsi_io_free;
51
52 /* set generic fields */
53 io->efct = efct;
54 io->node = node;
55 kref_get(&node->ref);
56
57 /* set type and name */
58 io->io_type = EFCT_IO_TYPE_IO;
59 io->display_name = "scsi_io";
60
61 io->cmd_ini = false;
62 io->cmd_tgt = true;
63
64 /* Add to node's active_ios list */
65 INIT_LIST_HEAD(&io->list_entry);
66 spin_lock_irqsave(&node->active_ios_lock, flags);
67 list_add(&io->list_entry, &node->active_ios);
68
69 spin_unlock_irqrestore(&node->active_ios_lock, flags);
70
71 return io;
72 }
73
74 void
_efct_scsi_io_free(struct kref * arg)75 _efct_scsi_io_free(struct kref *arg)
76 {
77 struct efct_io *io = container_of(arg, struct efct_io, ref);
78 struct efct *efct = io->efct;
79 struct efct_node *node = io->node;
80 unsigned long flags = 0;
81
82 scsi_io_trace(io, "freeing io 0x%p %s\n", io, io->display_name);
83
84 if (io->io_free) {
85 efc_log_err(efct, "IO already freed.\n");
86 return;
87 }
88
89 spin_lock_irqsave(&node->active_ios_lock, flags);
90 list_del_init(&io->list_entry);
91 spin_unlock_irqrestore(&node->active_ios_lock, flags);
92
93 kref_put(&node->ref, node->release);
94 io->node = NULL;
95 efct_io_pool_io_free(efct->xport->io_pool, io);
96 }
97
98 void
efct_scsi_io_free(struct efct_io * io)99 efct_scsi_io_free(struct efct_io *io)
100 {
101 scsi_io_trace(io, "freeing io 0x%p %s\n", io, io->display_name);
102 WARN_ON(!refcount_read(&io->ref.refcount));
103 kref_put(&io->ref, io->release);
104 }
105
106 static void
efct_target_io_cb(struct efct_hw_io * hio,u32 length,int status,u32 ext_status,void * app)107 efct_target_io_cb(struct efct_hw_io *hio, u32 length, int status,
108 u32 ext_status, void *app)
109 {
110 u32 flags = 0;
111 struct efct_io *io = app;
112 struct efct *efct;
113 enum efct_scsi_io_status scsi_stat = EFCT_SCSI_STATUS_GOOD;
114 efct_scsi_io_cb_t cb;
115
116 if (!io || !io->efct) {
117 pr_err("%s: IO can not be NULL\n", __func__);
118 return;
119 }
120
121 scsi_io_trace(io, "status x%x ext_status x%x\n", status, ext_status);
122
123 efct = io->efct;
124
125 io->transferred += length;
126
127 if (!io->scsi_tgt_cb) {
128 efct_scsi_check_pending(efct);
129 return;
130 }
131
132 /* Call target server completion */
133 cb = io->scsi_tgt_cb;
134
135 /* Clear the callback before invoking the callback */
136 io->scsi_tgt_cb = NULL;
137
138 /* if status was good, and auto-good-response was set,
139 * then callback target-server with IO_CMPL_RSP_SENT,
140 * otherwise send IO_CMPL
141 */
142 if (status == 0 && io->auto_resp)
143 flags |= EFCT_SCSI_IO_CMPL_RSP_SENT;
144 else
145 flags |= EFCT_SCSI_IO_CMPL;
146
147 switch (status) {
148 case SLI4_FC_WCQE_STATUS_SUCCESS:
149 scsi_stat = EFCT_SCSI_STATUS_GOOD;
150 break;
151 case SLI4_FC_WCQE_STATUS_DI_ERROR:
152 if (ext_status & SLI4_FC_DI_ERROR_GE)
153 scsi_stat = EFCT_SCSI_STATUS_DIF_GUARD_ERR;
154 else if (ext_status & SLI4_FC_DI_ERROR_AE)
155 scsi_stat = EFCT_SCSI_STATUS_DIF_APP_TAG_ERROR;
156 else if (ext_status & SLI4_FC_DI_ERROR_RE)
157 scsi_stat = EFCT_SCSI_STATUS_DIF_REF_TAG_ERROR;
158 else
159 scsi_stat = EFCT_SCSI_STATUS_DIF_UNKNOWN_ERROR;
160 break;
161 case SLI4_FC_WCQE_STATUS_LOCAL_REJECT:
162 switch (ext_status) {
163 case SLI4_FC_LOCAL_REJECT_INVALID_RELOFFSET:
164 case SLI4_FC_LOCAL_REJECT_ABORT_REQUESTED:
165 scsi_stat = EFCT_SCSI_STATUS_ABORTED;
166 break;
167 case SLI4_FC_LOCAL_REJECT_INVALID_RPI:
168 scsi_stat = EFCT_SCSI_STATUS_NEXUS_LOST;
169 break;
170 case SLI4_FC_LOCAL_REJECT_NO_XRI:
171 scsi_stat = EFCT_SCSI_STATUS_NO_IO;
172 break;
173 default:
174 /*we have seen 0x0d(TX_DMA_FAILED err)*/
175 scsi_stat = EFCT_SCSI_STATUS_ERROR;
176 break;
177 }
178 break;
179
180 case SLI4_FC_WCQE_STATUS_TARGET_WQE_TIMEOUT:
181 /* target IO timed out */
182 scsi_stat = EFCT_SCSI_STATUS_TIMEDOUT_AND_ABORTED;
183 break;
184
185 case SLI4_FC_WCQE_STATUS_SHUTDOWN:
186 /* Target IO cancelled by HW */
187 scsi_stat = EFCT_SCSI_STATUS_SHUTDOWN;
188 break;
189
190 default:
191 scsi_stat = EFCT_SCSI_STATUS_ERROR;
192 break;
193 }
194
195 cb(io, scsi_stat, flags, io->scsi_tgt_cb_arg);
196
197 efct_scsi_check_pending(efct);
198 }
199
200 static int
efct_scsi_build_sgls(struct efct_hw * hw,struct efct_hw_io * hio,struct efct_scsi_sgl * sgl,u32 sgl_count,enum efct_hw_io_type type)201 efct_scsi_build_sgls(struct efct_hw *hw, struct efct_hw_io *hio,
202 struct efct_scsi_sgl *sgl, u32 sgl_count,
203 enum efct_hw_io_type type)
204 {
205 int rc;
206 u32 i;
207 struct efct *efct = hw->os;
208
209 /* Initialize HW SGL */
210 rc = efct_hw_io_init_sges(hw, hio, type);
211 if (rc) {
212 efc_log_err(efct, "efct_hw_io_init_sges failed: %d\n", rc);
213 return -EIO;
214 }
215
216 for (i = 0; i < sgl_count; i++) {
217 /* Add data SGE */
218 rc = efct_hw_io_add_sge(hw, hio, sgl[i].addr, sgl[i].len);
219 if (rc) {
220 efc_log_err(efct, "add sge failed cnt=%d rc=%d\n",
221 sgl_count, rc);
222 return rc;
223 }
224 }
225
226 return 0;
227 }
228
efc_log_sgl(struct efct_io * io)229 static void efc_log_sgl(struct efct_io *io)
230 {
231 struct efct_hw_io *hio = io->hio;
232 struct sli4_sge *data = NULL;
233 u32 *dword = NULL;
234 u32 i;
235 u32 n_sge;
236
237 scsi_io_trace(io, "def_sgl at 0x%x 0x%08x\n",
238 upper_32_bits(hio->def_sgl.phys),
239 lower_32_bits(hio->def_sgl.phys));
240 n_sge = (hio->sgl == &hio->def_sgl) ? hio->n_sge : hio->def_sgl_count;
241 for (i = 0, data = hio->def_sgl.virt; i < n_sge; i++, data++) {
242 dword = (u32 *)data;
243
244 scsi_io_trace(io, "SGL %2d 0x%08x 0x%08x 0x%08x 0x%08x\n",
245 i, dword[0], dword[1], dword[2], dword[3]);
246
247 if (dword[2] & (1U << 31))
248 break;
249 }
250 }
251
252 static void
efct_scsi_check_pending_async_cb(struct efct_hw * hw,int status,u8 * mqe,void * arg)253 efct_scsi_check_pending_async_cb(struct efct_hw *hw, int status,
254 u8 *mqe, void *arg)
255 {
256 struct efct_io *io = arg;
257
258 if (io) {
259 efct_hw_done_t cb = io->hw_cb;
260
261 if (!io->hw_cb)
262 return;
263
264 io->hw_cb = NULL;
265 (cb)(io->hio, 0, SLI4_FC_WCQE_STATUS_DISPATCH_ERROR, 0, io);
266 }
267 }
268
269 static int
efct_scsi_io_dispatch_hw_io(struct efct_io * io,struct efct_hw_io * hio)270 efct_scsi_io_dispatch_hw_io(struct efct_io *io, struct efct_hw_io *hio)
271 {
272 int rc = 0;
273 struct efct *efct = io->efct;
274
275 /* Got a HW IO;
276 * update ini/tgt_task_tag with HW IO info and dispatch
277 */
278 io->hio = hio;
279 if (io->cmd_tgt)
280 io->tgt_task_tag = hio->indicator;
281 else if (io->cmd_ini)
282 io->init_task_tag = hio->indicator;
283 io->hw_tag = hio->reqtag;
284
285 hio->eq = io->hw_priv;
286
287 /* Copy WQ steering */
288 switch (io->wq_steering) {
289 case EFCT_SCSI_WQ_STEERING_CLASS >> EFCT_SCSI_WQ_STEERING_SHIFT:
290 hio->wq_steering = EFCT_HW_WQ_STEERING_CLASS;
291 break;
292 case EFCT_SCSI_WQ_STEERING_REQUEST >> EFCT_SCSI_WQ_STEERING_SHIFT:
293 hio->wq_steering = EFCT_HW_WQ_STEERING_REQUEST;
294 break;
295 case EFCT_SCSI_WQ_STEERING_CPU >> EFCT_SCSI_WQ_STEERING_SHIFT:
296 hio->wq_steering = EFCT_HW_WQ_STEERING_CPU;
297 break;
298 }
299
300 switch (io->io_type) {
301 case EFCT_IO_TYPE_IO:
302 rc = efct_scsi_build_sgls(&efct->hw, io->hio,
303 io->sgl, io->sgl_count, io->hio_type);
304 if (rc)
305 break;
306
307 if (EFCT_LOG_ENABLE_SCSI_TRACE(efct))
308 efc_log_sgl(io);
309
310 if (io->app_id)
311 io->iparam.fcp_tgt.app_id = io->app_id;
312
313 io->iparam.fcp_tgt.vpi = io->node->vpi;
314 io->iparam.fcp_tgt.rpi = io->node->rpi;
315 io->iparam.fcp_tgt.s_id = io->node->port_fc_id;
316 io->iparam.fcp_tgt.d_id = io->node->node_fc_id;
317 io->iparam.fcp_tgt.xmit_len = io->wire_len;
318
319 rc = efct_hw_io_send(&io->efct->hw, io->hio_type, io->hio,
320 &io->iparam, io->hw_cb, io);
321 break;
322 default:
323 scsi_io_printf(io, "Unknown IO type=%d\n", io->io_type);
324 rc = -EIO;
325 break;
326 }
327 return rc;
328 }
329
330 static int
efct_scsi_io_dispatch_no_hw_io(struct efct_io * io)331 efct_scsi_io_dispatch_no_hw_io(struct efct_io *io)
332 {
333 int rc;
334
335 switch (io->io_type) {
336 case EFCT_IO_TYPE_ABORT: {
337 struct efct_hw_io *hio_to_abort = NULL;
338
339 hio_to_abort = io->io_to_abort->hio;
340
341 if (!hio_to_abort) {
342 /*
343 * If "IO to abort" does not have an
344 * associated HW IO, immediately make callback with
345 * success. The command must have been sent to
346 * the backend, but the data phase has not yet
347 * started, so we don't have a HW IO.
348 *
349 * Note: since the backend shims should be
350 * taking a reference on io_to_abort, it should not
351 * be possible to have been completed and freed by
352 * the backend before the abort got here.
353 */
354 scsi_io_printf(io, "IO: not active\n");
355 ((efct_hw_done_t)io->hw_cb)(io->hio, 0,
356 SLI4_FC_WCQE_STATUS_SUCCESS, 0, io);
357 rc = 0;
358 break;
359 }
360
361 /* HW IO is valid, abort it */
362 scsi_io_printf(io, "aborting\n");
363 rc = efct_hw_io_abort(&io->efct->hw, hio_to_abort,
364 io->send_abts, io->hw_cb, io);
365 if (rc) {
366 int status = SLI4_FC_WCQE_STATUS_SUCCESS;
367 efct_hw_done_t cb = io->hw_cb;
368
369 if (rc != -ENOENT && rc != -EINPROGRESS) {
370 status = -1;
371 scsi_io_printf(io, "Failed to abort IO rc=%d\n",
372 rc);
373 }
374 cb(io->hio, 0, status, 0, io);
375 rc = 0;
376 }
377
378 break;
379 }
380 default:
381 scsi_io_printf(io, "Unknown IO type=%d\n", io->io_type);
382 rc = -EIO;
383 break;
384 }
385 return rc;
386 }
387
388 static struct efct_io *
efct_scsi_dispatch_pending(struct efct * efct)389 efct_scsi_dispatch_pending(struct efct *efct)
390 {
391 struct efct_xport *xport = efct->xport;
392 struct efct_io *io = NULL;
393 struct efct_hw_io *hio;
394 unsigned long flags = 0;
395 int status;
396
397 spin_lock_irqsave(&xport->io_pending_lock, flags);
398
399 if (!list_empty(&xport->io_pending_list)) {
400 io = list_first_entry(&xport->io_pending_list, struct efct_io,
401 io_pending_link);
402 list_del_init(&io->io_pending_link);
403 }
404
405 if (!io) {
406 spin_unlock_irqrestore(&xport->io_pending_lock, flags);
407 return NULL;
408 }
409
410 if (io->io_type == EFCT_IO_TYPE_ABORT) {
411 hio = NULL;
412 } else {
413 hio = efct_hw_io_alloc(&efct->hw);
414 if (!hio) {
415 /*
416 * No HW IO available.Put IO back on
417 * the front of pending list
418 */
419 list_add(&xport->io_pending_list, &io->io_pending_link);
420 io = NULL;
421 } else {
422 hio->eq = io->hw_priv;
423 }
424 }
425
426 /* Must drop the lock before dispatching the IO */
427 spin_unlock_irqrestore(&xport->io_pending_lock, flags);
428
429 if (!io)
430 return NULL;
431
432 /*
433 * We pulled an IO off the pending list,
434 * and either got an HW IO or don't need one
435 */
436 atomic_sub_return(1, &xport->io_pending_count);
437 if (!hio)
438 status = efct_scsi_io_dispatch_no_hw_io(io);
439 else
440 status = efct_scsi_io_dispatch_hw_io(io, hio);
441 if (status) {
442 /*
443 * Invoke the HW callback, but do so in the
444 * separate execution context,provided by the
445 * NOP mailbox completion processing context
446 * by using efct_hw_async_call()
447 */
448 if (efct_hw_async_call(&efct->hw,
449 efct_scsi_check_pending_async_cb, io)) {
450 efc_log_debug(efct, "call hw async failed\n");
451 }
452 }
453
454 return io;
455 }
456
457 void
efct_scsi_check_pending(struct efct * efct)458 efct_scsi_check_pending(struct efct *efct)
459 {
460 struct efct_xport *xport = efct->xport;
461 struct efct_io *io = NULL;
462 int count = 0;
463 unsigned long flags = 0;
464 int dispatch = 0;
465
466 /* Guard against recursion */
467 if (atomic_add_return(1, &xport->io_pending_recursing)) {
468 /* This function is already running. Decrement and return. */
469 atomic_sub_return(1, &xport->io_pending_recursing);
470 return;
471 }
472
473 while (efct_scsi_dispatch_pending(efct))
474 count++;
475
476 if (count) {
477 atomic_sub_return(1, &xport->io_pending_recursing);
478 return;
479 }
480
481 /*
482 * If nothing was removed from the list,
483 * we might be in a case where we need to abort an
484 * active IO and the abort is on the pending list.
485 * Look for an abort we can dispatch.
486 */
487
488 spin_lock_irqsave(&xport->io_pending_lock, flags);
489
490 list_for_each_entry(io, &xport->io_pending_list, io_pending_link) {
491 if (io->io_type == EFCT_IO_TYPE_ABORT && io->io_to_abort->hio) {
492 /* This IO has a HW IO, so it is
493 * active. Dispatch the abort.
494 */
495 dispatch = 1;
496 list_del_init(&io->io_pending_link);
497 atomic_sub_return(1, &xport->io_pending_count);
498 break;
499 }
500 }
501
502 spin_unlock_irqrestore(&xport->io_pending_lock, flags);
503
504 if (dispatch) {
505 if (efct_scsi_io_dispatch_no_hw_io(io)) {
506 if (efct_hw_async_call(&efct->hw,
507 efct_scsi_check_pending_async_cb, io)) {
508 efc_log_debug(efct, "hw async failed\n");
509 }
510 }
511 }
512
513 atomic_sub_return(1, &xport->io_pending_recursing);
514 }
515
516 int
efct_scsi_io_dispatch(struct efct_io * io,void * cb)517 efct_scsi_io_dispatch(struct efct_io *io, void *cb)
518 {
519 struct efct_hw_io *hio;
520 struct efct *efct = io->efct;
521 struct efct_xport *xport = efct->xport;
522 unsigned long flags = 0;
523
524 io->hw_cb = cb;
525
526 /*
527 * if this IO already has a HW IO, then this is either
528 * not the first phase of the IO. Send it to the HW.
529 */
530 if (io->hio)
531 return efct_scsi_io_dispatch_hw_io(io, io->hio);
532
533 /*
534 * We don't already have a HW IO associated with the IO. First check
535 * the pending list. If not empty, add IO to the tail and process the
536 * pending list.
537 */
538 spin_lock_irqsave(&xport->io_pending_lock, flags);
539 if (!list_empty(&xport->io_pending_list)) {
540 /*
541 * If this is a low latency request,
542 * the put at the front of the IO pending
543 * queue, otherwise put it at the end of the queue.
544 */
545 if (io->low_latency) {
546 INIT_LIST_HEAD(&io->io_pending_link);
547 list_add(&xport->io_pending_list, &io->io_pending_link);
548 } else {
549 INIT_LIST_HEAD(&io->io_pending_link);
550 list_add_tail(&io->io_pending_link,
551 &xport->io_pending_list);
552 }
553 spin_unlock_irqrestore(&xport->io_pending_lock, flags);
554 atomic_add_return(1, &xport->io_pending_count);
555 atomic_add_return(1, &xport->io_total_pending);
556
557 /* process pending list */
558 efct_scsi_check_pending(efct);
559 return 0;
560 }
561 spin_unlock_irqrestore(&xport->io_pending_lock, flags);
562
563 /*
564 * We don't have a HW IO associated with the IO and there's nothing
565 * on the pending list. Attempt to allocate a HW IO and dispatch it.
566 */
567 hio = efct_hw_io_alloc(&io->efct->hw);
568 if (!hio) {
569 /* Couldn't get a HW IO. Save this IO on the pending list */
570 spin_lock_irqsave(&xport->io_pending_lock, flags);
571 INIT_LIST_HEAD(&io->io_pending_link);
572 list_add_tail(&io->io_pending_link, &xport->io_pending_list);
573 spin_unlock_irqrestore(&xport->io_pending_lock, flags);
574
575 atomic_add_return(1, &xport->io_total_pending);
576 atomic_add_return(1, &xport->io_pending_count);
577 return 0;
578 }
579
580 /* We successfully allocated a HW IO; dispatch to HW */
581 return efct_scsi_io_dispatch_hw_io(io, hio);
582 }
583
584 int
efct_scsi_io_dispatch_abort(struct efct_io * io,void * cb)585 efct_scsi_io_dispatch_abort(struct efct_io *io, void *cb)
586 {
587 struct efct *efct = io->efct;
588 struct efct_xport *xport = efct->xport;
589 unsigned long flags = 0;
590
591 io->hw_cb = cb;
592
593 /*
594 * For aborts, we don't need a HW IO, but we still want
595 * to pass through the pending list to preserve ordering.
596 * Thus, if the pending list is not empty, add this abort
597 * to the pending list and process the pending list.
598 */
599 spin_lock_irqsave(&xport->io_pending_lock, flags);
600 if (!list_empty(&xport->io_pending_list)) {
601 INIT_LIST_HEAD(&io->io_pending_link);
602 list_add_tail(&io->io_pending_link, &xport->io_pending_list);
603 spin_unlock_irqrestore(&xport->io_pending_lock, flags);
604 atomic_add_return(1, &xport->io_pending_count);
605 atomic_add_return(1, &xport->io_total_pending);
606
607 /* process pending list */
608 efct_scsi_check_pending(efct);
609 return 0;
610 }
611 spin_unlock_irqrestore(&xport->io_pending_lock, flags);
612
613 /* nothing on pending list, dispatch abort */
614 return efct_scsi_io_dispatch_no_hw_io(io);
615 }
616
617 static inline int
efct_scsi_xfer_data(struct efct_io * io,u32 flags,struct efct_scsi_sgl * sgl,u32 sgl_count,u64 xwire_len,enum efct_hw_io_type type,int enable_ar,efct_scsi_io_cb_t cb,void * arg)618 efct_scsi_xfer_data(struct efct_io *io, u32 flags,
619 struct efct_scsi_sgl *sgl, u32 sgl_count, u64 xwire_len,
620 enum efct_hw_io_type type, int enable_ar,
621 efct_scsi_io_cb_t cb, void *arg)
622 {
623 struct efct *efct;
624 size_t residual = 0;
625
626 io->sgl_count = sgl_count;
627
628 efct = io->efct;
629
630 scsi_io_trace(io, "%s wire_len %llu\n",
631 (type == EFCT_HW_IO_TARGET_READ) ? "send" : "recv",
632 xwire_len);
633
634 io->hio_type = type;
635
636 io->scsi_tgt_cb = cb;
637 io->scsi_tgt_cb_arg = arg;
638
639 residual = io->exp_xfer_len - io->transferred;
640 io->wire_len = (xwire_len < residual) ? xwire_len : residual;
641 residual = (xwire_len - io->wire_len);
642
643 memset(&io->iparam, 0, sizeof(io->iparam));
644 io->iparam.fcp_tgt.ox_id = io->init_task_tag;
645 io->iparam.fcp_tgt.offset = io->transferred;
646 io->iparam.fcp_tgt.cs_ctl = io->cs_ctl;
647 io->iparam.fcp_tgt.timeout = io->timeout;
648
649 /* if this is the last data phase and there is no residual, enable
650 * auto-good-response
651 */
652 if (enable_ar && (flags & EFCT_SCSI_LAST_DATAPHASE) && residual == 0 &&
653 ((io->transferred + io->wire_len) == io->exp_xfer_len) &&
654 (!(flags & EFCT_SCSI_NO_AUTO_RESPONSE))) {
655 io->iparam.fcp_tgt.flags |= SLI4_IO_AUTO_GOOD_RESPONSE;
656 io->auto_resp = true;
657 } else {
658 io->auto_resp = false;
659 }
660
661 /* save this transfer length */
662 io->xfer_req = io->wire_len;
663
664 /* Adjust the transferred count to account for overrun
665 * when the residual is calculated in efct_scsi_send_resp
666 */
667 io->transferred += residual;
668
669 /* Adjust the SGL size if there is overrun */
670
671 if (residual) {
672 struct efct_scsi_sgl *sgl_ptr = &io->sgl[sgl_count - 1];
673
674 while (residual) {
675 size_t len = sgl_ptr->len;
676
677 if (len > residual) {
678 sgl_ptr->len = len - residual;
679 residual = 0;
680 } else {
681 sgl_ptr->len = 0;
682 residual -= len;
683 io->sgl_count--;
684 }
685 sgl_ptr--;
686 }
687 }
688
689 /* Set latency and WQ steering */
690 io->low_latency = (flags & EFCT_SCSI_LOW_LATENCY) != 0;
691 io->wq_steering = (flags & EFCT_SCSI_WQ_STEERING_MASK) >>
692 EFCT_SCSI_WQ_STEERING_SHIFT;
693 io->wq_class = (flags & EFCT_SCSI_WQ_CLASS_MASK) >>
694 EFCT_SCSI_WQ_CLASS_SHIFT;
695
696 if (efct->xport) {
697 struct efct_xport *xport = efct->xport;
698
699 if (type == EFCT_HW_IO_TARGET_READ) {
700 xport->fcp_stats.input_requests++;
701 xport->fcp_stats.input_bytes += xwire_len;
702 } else if (type == EFCT_HW_IO_TARGET_WRITE) {
703 xport->fcp_stats.output_requests++;
704 xport->fcp_stats.output_bytes += xwire_len;
705 }
706 }
707 return efct_scsi_io_dispatch(io, efct_target_io_cb);
708 }
709
710 int
efct_scsi_send_rd_data(struct efct_io * io,u32 flags,struct efct_scsi_sgl * sgl,u32 sgl_count,u64 len,efct_scsi_io_cb_t cb,void * arg)711 efct_scsi_send_rd_data(struct efct_io *io, u32 flags,
712 struct efct_scsi_sgl *sgl, u32 sgl_count, u64 len,
713 efct_scsi_io_cb_t cb, void *arg)
714 {
715 return efct_scsi_xfer_data(io, flags, sgl, sgl_count,
716 len, EFCT_HW_IO_TARGET_READ,
717 enable_tsend_auto_resp(io->efct), cb, arg);
718 }
719
720 int
efct_scsi_recv_wr_data(struct efct_io * io,u32 flags,struct efct_scsi_sgl * sgl,u32 sgl_count,u64 len,efct_scsi_io_cb_t cb,void * arg)721 efct_scsi_recv_wr_data(struct efct_io *io, u32 flags,
722 struct efct_scsi_sgl *sgl, u32 sgl_count, u64 len,
723 efct_scsi_io_cb_t cb, void *arg)
724 {
725 return efct_scsi_xfer_data(io, flags, sgl, sgl_count, len,
726 EFCT_HW_IO_TARGET_WRITE,
727 enable_treceive_auto_resp(io->efct), cb, arg);
728 }
729
730 int
efct_scsi_send_resp(struct efct_io * io,u32 flags,struct efct_scsi_cmd_resp * rsp,efct_scsi_io_cb_t cb,void * arg)731 efct_scsi_send_resp(struct efct_io *io, u32 flags,
732 struct efct_scsi_cmd_resp *rsp,
733 efct_scsi_io_cb_t cb, void *arg)
734 {
735 struct efct *efct;
736 int residual;
737 /* Always try auto resp */
738 bool auto_resp = true;
739 u8 scsi_status = 0;
740 u16 scsi_status_qualifier = 0;
741 u8 *sense_data = NULL;
742 u32 sense_data_length = 0;
743
744 efct = io->efct;
745
746 if (rsp) {
747 scsi_status = rsp->scsi_status;
748 scsi_status_qualifier = rsp->scsi_status_qualifier;
749 sense_data = rsp->sense_data;
750 sense_data_length = rsp->sense_data_length;
751 residual = rsp->residual;
752 } else {
753 residual = io->exp_xfer_len - io->transferred;
754 }
755
756 io->wire_len = 0;
757 io->hio_type = EFCT_HW_IO_TARGET_RSP;
758
759 io->scsi_tgt_cb = cb;
760 io->scsi_tgt_cb_arg = arg;
761
762 memset(&io->iparam, 0, sizeof(io->iparam));
763 io->iparam.fcp_tgt.ox_id = io->init_task_tag;
764 io->iparam.fcp_tgt.offset = 0;
765 io->iparam.fcp_tgt.cs_ctl = io->cs_ctl;
766 io->iparam.fcp_tgt.timeout = io->timeout;
767
768 /* Set low latency queueing request */
769 io->low_latency = (flags & EFCT_SCSI_LOW_LATENCY) != 0;
770 io->wq_steering = (flags & EFCT_SCSI_WQ_STEERING_MASK) >>
771 EFCT_SCSI_WQ_STEERING_SHIFT;
772 io->wq_class = (flags & EFCT_SCSI_WQ_CLASS_MASK) >>
773 EFCT_SCSI_WQ_CLASS_SHIFT;
774
775 if (scsi_status != 0 || residual || sense_data_length) {
776 struct fcp_resp_with_ext *fcprsp = io->rspbuf.virt;
777 u8 *sns_data;
778
779 if (!fcprsp) {
780 efc_log_err(efct, "NULL response buffer\n");
781 return -EIO;
782 }
783
784 sns_data = (u8 *)io->rspbuf.virt + sizeof(*fcprsp);
785
786 auto_resp = false;
787
788 memset(fcprsp, 0, sizeof(*fcprsp));
789
790 io->wire_len += sizeof(*fcprsp);
791
792 fcprsp->resp.fr_status = scsi_status;
793 fcprsp->resp.fr_retry_delay =
794 cpu_to_be16(scsi_status_qualifier);
795
796 /* set residual status if necessary */
797 if (residual != 0) {
798 /* FCP: if data transferred is less than the
799 * amount expected, then this is an underflow.
800 * If data transferred would have been greater
801 * than the amount expected this is an overflow
802 */
803 if (residual > 0) {
804 fcprsp->resp.fr_flags |= FCP_RESID_UNDER;
805 fcprsp->ext.fr_resid = cpu_to_be32(residual);
806 } else {
807 fcprsp->resp.fr_flags |= FCP_RESID_OVER;
808 fcprsp->ext.fr_resid = cpu_to_be32(-residual);
809 }
810 }
811
812 if (EFCT_SCSI_SNS_BUF_VALID(sense_data) && sense_data_length) {
813 if (sense_data_length > SCSI_SENSE_BUFFERSIZE) {
814 efc_log_err(efct, "Sense exceeds max size.\n");
815 return -EIO;
816 }
817
818 fcprsp->resp.fr_flags |= FCP_SNS_LEN_VAL;
819 memcpy(sns_data, sense_data, sense_data_length);
820 fcprsp->ext.fr_sns_len = cpu_to_be32(sense_data_length);
821 io->wire_len += sense_data_length;
822 }
823
824 io->sgl[0].addr = io->rspbuf.phys;
825 io->sgl[0].dif_addr = 0;
826 io->sgl[0].len = io->wire_len;
827 io->sgl_count = 1;
828 }
829
830 if (auto_resp)
831 io->iparam.fcp_tgt.flags |= SLI4_IO_AUTO_GOOD_RESPONSE;
832
833 return efct_scsi_io_dispatch(io, efct_target_io_cb);
834 }
835
836 static int
efct_target_bls_resp_cb(struct efct_hw_io * hio,u32 length,int status,u32 ext_status,void * app)837 efct_target_bls_resp_cb(struct efct_hw_io *hio, u32 length, int status,
838 u32 ext_status, void *app)
839 {
840 struct efct_io *io = app;
841 struct efct *efct;
842 enum efct_scsi_io_status bls_status;
843
844 efct = io->efct;
845
846 /* BLS isn't really a "SCSI" concept, but use SCSI status */
847 if (status) {
848 io_error_log(io, "s=%#x x=%#x\n", status, ext_status);
849 bls_status = EFCT_SCSI_STATUS_ERROR;
850 } else {
851 bls_status = EFCT_SCSI_STATUS_GOOD;
852 }
853
854 if (io->bls_cb) {
855 efct_scsi_io_cb_t bls_cb = io->bls_cb;
856 void *bls_cb_arg = io->bls_cb_arg;
857
858 io->bls_cb = NULL;
859 io->bls_cb_arg = NULL;
860
861 /* invoke callback */
862 bls_cb(io, bls_status, 0, bls_cb_arg);
863 }
864
865 efct_scsi_check_pending(efct);
866 return 0;
867 }
868
869 static int
efct_target_send_bls_resp(struct efct_io * io,efct_scsi_io_cb_t cb,void * arg)870 efct_target_send_bls_resp(struct efct_io *io,
871 efct_scsi_io_cb_t cb, void *arg)
872 {
873 struct efct_node *node = io->node;
874 struct sli_bls_params *bls = &io->iparam.bls;
875 struct efct *efct = node->efct;
876 struct fc_ba_acc *acc;
877 int rc;
878
879 /* fill out IO structure with everything needed to send BA_ACC */
880 memset(&io->iparam, 0, sizeof(io->iparam));
881 bls->ox_id = io->init_task_tag;
882 bls->rx_id = io->abort_rx_id;
883 bls->vpi = io->node->vpi;
884 bls->rpi = io->node->rpi;
885 bls->s_id = U32_MAX;
886 bls->d_id = io->node->node_fc_id;
887 bls->rpi_registered = true;
888
889 acc = (void *)bls->payload;
890 acc->ba_ox_id = cpu_to_be16(bls->ox_id);
891 acc->ba_rx_id = cpu_to_be16(bls->rx_id);
892 acc->ba_high_seq_cnt = cpu_to_be16(U16_MAX);
893
894 /* generic io fields have already been populated */
895
896 /* set type and BLS-specific fields */
897 io->io_type = EFCT_IO_TYPE_BLS_RESP;
898 io->display_name = "bls_rsp";
899 io->hio_type = EFCT_HW_BLS_ACC;
900 io->bls_cb = cb;
901 io->bls_cb_arg = arg;
902
903 /* dispatch IO */
904 rc = efct_hw_bls_send(efct, FC_RCTL_BA_ACC, bls,
905 efct_target_bls_resp_cb, io);
906 return rc;
907 }
908
efct_bls_send_rjt_cb(struct efct_hw_io * hio,u32 length,int status,u32 ext_status,void * app)909 static int efct_bls_send_rjt_cb(struct efct_hw_io *hio, u32 length, int status,
910 u32 ext_status, void *app)
911 {
912 struct efct_io *io = app;
913
914 efct_scsi_io_free(io);
915 return 0;
916 }
917
918 struct efct_io *
efct_bls_send_rjt(struct efct_io * io,struct fc_frame_header * hdr)919 efct_bls_send_rjt(struct efct_io *io, struct fc_frame_header *hdr)
920 {
921 struct efct_node *node = io->node;
922 struct sli_bls_params *bls = &io->iparam.bls;
923 struct efct *efct = node->efct;
924 struct fc_ba_rjt *acc;
925 int rc;
926
927 /* fill out BLS Response-specific fields */
928 io->io_type = EFCT_IO_TYPE_BLS_RESP;
929 io->display_name = "ba_rjt";
930 io->hio_type = EFCT_HW_BLS_RJT;
931 io->init_task_tag = be16_to_cpu(hdr->fh_ox_id);
932
933 /* fill out iparam fields */
934 memset(&io->iparam, 0, sizeof(io->iparam));
935 bls->ox_id = be16_to_cpu(hdr->fh_ox_id);
936 bls->rx_id = be16_to_cpu(hdr->fh_rx_id);
937 bls->vpi = io->node->vpi;
938 bls->rpi = io->node->rpi;
939 bls->s_id = U32_MAX;
940 bls->d_id = io->node->node_fc_id;
941 bls->rpi_registered = true;
942
943 acc = (void *)bls->payload;
944 acc->br_reason = ELS_RJT_UNAB;
945 acc->br_explan = ELS_EXPL_NONE;
946
947 rc = efct_hw_bls_send(efct, FC_RCTL_BA_RJT, bls, efct_bls_send_rjt_cb,
948 io);
949 if (rc) {
950 efc_log_err(efct, "efct_scsi_io_dispatch() failed: %d\n", rc);
951 efct_scsi_io_free(io);
952 io = NULL;
953 }
954 return io;
955 }
956
957 int
efct_scsi_send_tmf_resp(struct efct_io * io,enum efct_scsi_tmf_resp rspcode,u8 addl_rsp_info[3],efct_scsi_io_cb_t cb,void * arg)958 efct_scsi_send_tmf_resp(struct efct_io *io,
959 enum efct_scsi_tmf_resp rspcode,
960 u8 addl_rsp_info[3],
961 efct_scsi_io_cb_t cb, void *arg)
962 {
963 int rc;
964 struct {
965 struct fcp_resp_with_ext rsp_ext;
966 struct fcp_resp_rsp_info info;
967 } *fcprsp;
968 u8 fcp_rspcode;
969
970 io->wire_len = 0;
971
972 switch (rspcode) {
973 case EFCT_SCSI_TMF_FUNCTION_COMPLETE:
974 fcp_rspcode = FCP_TMF_CMPL;
975 break;
976 case EFCT_SCSI_TMF_FUNCTION_SUCCEEDED:
977 case EFCT_SCSI_TMF_FUNCTION_IO_NOT_FOUND:
978 fcp_rspcode = FCP_TMF_CMPL;
979 break;
980 case EFCT_SCSI_TMF_FUNCTION_REJECTED:
981 fcp_rspcode = FCP_TMF_REJECTED;
982 break;
983 case EFCT_SCSI_TMF_INCORRECT_LOGICAL_UNIT_NUMBER:
984 fcp_rspcode = FCP_TMF_INVALID_LUN;
985 break;
986 case EFCT_SCSI_TMF_SERVICE_DELIVERY:
987 fcp_rspcode = FCP_TMF_FAILED;
988 break;
989 default:
990 fcp_rspcode = FCP_TMF_REJECTED;
991 break;
992 }
993
994 io->hio_type = EFCT_HW_IO_TARGET_RSP;
995
996 io->scsi_tgt_cb = cb;
997 io->scsi_tgt_cb_arg = arg;
998
999 if (io->tmf_cmd == EFCT_SCSI_TMF_ABORT_TASK) {
1000 rc = efct_target_send_bls_resp(io, cb, arg);
1001 return rc;
1002 }
1003
1004 /* populate the FCP TMF response */
1005 fcprsp = io->rspbuf.virt;
1006 memset(fcprsp, 0, sizeof(*fcprsp));
1007
1008 fcprsp->rsp_ext.resp.fr_flags |= FCP_SNS_LEN_VAL;
1009
1010 if (addl_rsp_info) {
1011 memcpy(fcprsp->info._fr_resvd, addl_rsp_info,
1012 sizeof(fcprsp->info._fr_resvd));
1013 }
1014 fcprsp->info.rsp_code = fcp_rspcode;
1015
1016 io->wire_len = sizeof(*fcprsp);
1017
1018 fcprsp->rsp_ext.ext.fr_rsp_len =
1019 cpu_to_be32(sizeof(struct fcp_resp_rsp_info));
1020
1021 io->sgl[0].addr = io->rspbuf.phys;
1022 io->sgl[0].dif_addr = 0;
1023 io->sgl[0].len = io->wire_len;
1024 io->sgl_count = 1;
1025
1026 memset(&io->iparam, 0, sizeof(io->iparam));
1027 io->iparam.fcp_tgt.ox_id = io->init_task_tag;
1028 io->iparam.fcp_tgt.offset = 0;
1029 io->iparam.fcp_tgt.cs_ctl = io->cs_ctl;
1030 io->iparam.fcp_tgt.timeout = io->timeout;
1031
1032 rc = efct_scsi_io_dispatch(io, efct_target_io_cb);
1033
1034 return rc;
1035 }
1036
1037 static int
efct_target_abort_cb(struct efct_hw_io * hio,u32 length,int status,u32 ext_status,void * app)1038 efct_target_abort_cb(struct efct_hw_io *hio, u32 length, int status,
1039 u32 ext_status, void *app)
1040 {
1041 struct efct_io *io = app;
1042 struct efct *efct;
1043 enum efct_scsi_io_status scsi_status;
1044 efct_scsi_io_cb_t abort_cb;
1045 void *abort_cb_arg;
1046
1047 efct = io->efct;
1048
1049 if (!io->abort_cb)
1050 goto done;
1051
1052 abort_cb = io->abort_cb;
1053 abort_cb_arg = io->abort_cb_arg;
1054
1055 io->abort_cb = NULL;
1056 io->abort_cb_arg = NULL;
1057
1058 switch (status) {
1059 case SLI4_FC_WCQE_STATUS_SUCCESS:
1060 scsi_status = EFCT_SCSI_STATUS_GOOD;
1061 break;
1062 case SLI4_FC_WCQE_STATUS_LOCAL_REJECT:
1063 switch (ext_status) {
1064 case SLI4_FC_LOCAL_REJECT_NO_XRI:
1065 scsi_status = EFCT_SCSI_STATUS_NO_IO;
1066 break;
1067 case SLI4_FC_LOCAL_REJECT_ABORT_IN_PROGRESS:
1068 scsi_status = EFCT_SCSI_STATUS_ABORT_IN_PROGRESS;
1069 break;
1070 default:
1071 /*we have seen 0x15 (abort in progress)*/
1072 scsi_status = EFCT_SCSI_STATUS_ERROR;
1073 break;
1074 }
1075 break;
1076 case SLI4_FC_WCQE_STATUS_FCP_RSP_FAILURE:
1077 scsi_status = EFCT_SCSI_STATUS_CHECK_RESPONSE;
1078 break;
1079 default:
1080 scsi_status = EFCT_SCSI_STATUS_ERROR;
1081 break;
1082 }
1083 /* invoke callback */
1084 abort_cb(io->io_to_abort, scsi_status, 0, abort_cb_arg);
1085
1086 done:
1087 /* done with IO to abort,efct_ref_get(): efct_scsi_tgt_abort_io() */
1088 kref_put(&io->io_to_abort->ref, io->io_to_abort->release);
1089
1090 efct_io_pool_io_free(efct->xport->io_pool, io);
1091
1092 efct_scsi_check_pending(efct);
1093 return 0;
1094 }
1095
1096 int
efct_scsi_tgt_abort_io(struct efct_io * io,efct_scsi_io_cb_t cb,void * arg)1097 efct_scsi_tgt_abort_io(struct efct_io *io, efct_scsi_io_cb_t cb, void *arg)
1098 {
1099 struct efct *efct;
1100 struct efct_xport *xport;
1101 int rc;
1102 struct efct_io *abort_io = NULL;
1103
1104 efct = io->efct;
1105 xport = efct->xport;
1106
1107 /* take a reference on IO being aborted */
1108 if (kref_get_unless_zero(&io->ref) == 0) {
1109 /* command no longer active */
1110 scsi_io_printf(io, "command no longer active\n");
1111 return -EIO;
1112 }
1113
1114 /*
1115 * allocate a new IO to send the abort request. Use efct_io_alloc()
1116 * directly, as we need an IO object that will not fail allocation
1117 * due to allocations being disabled (in efct_scsi_io_alloc())
1118 */
1119 abort_io = efct_io_pool_io_alloc(efct->xport->io_pool);
1120 if (!abort_io) {
1121 atomic_add_return(1, &xport->io_alloc_failed_count);
1122 kref_put(&io->ref, io->release);
1123 return -EIO;
1124 }
1125
1126 /* Save the target server callback and argument */
1127 /* set generic fields */
1128 abort_io->cmd_tgt = true;
1129 abort_io->node = io->node;
1130
1131 /* set type and abort-specific fields */
1132 abort_io->io_type = EFCT_IO_TYPE_ABORT;
1133 abort_io->display_name = "tgt_abort";
1134 abort_io->io_to_abort = io;
1135 abort_io->send_abts = false;
1136 abort_io->abort_cb = cb;
1137 abort_io->abort_cb_arg = arg;
1138
1139 /* now dispatch IO */
1140 rc = efct_scsi_io_dispatch_abort(abort_io, efct_target_abort_cb);
1141 if (rc)
1142 kref_put(&io->ref, io->release);
1143 return rc;
1144 }
1145
1146 void
efct_scsi_io_complete(struct efct_io * io)1147 efct_scsi_io_complete(struct efct_io *io)
1148 {
1149 if (io->io_free) {
1150 efc_log_debug(io->efct, "completion for non-busy io tag 0x%x\n",
1151 io->tag);
1152 return;
1153 }
1154
1155 scsi_io_trace(io, "freeing io 0x%p %s\n", io, io->display_name);
1156 kref_put(&io->ref, io->release);
1157 }
1158