xref: /openbmc/linux/drivers/scsi/libfc/fc_exch.c (revision 9efac679)
1 /*
2  * Copyright(c) 2007 Intel Corporation. All rights reserved.
3  * Copyright(c) 2008 Red Hat, Inc.  All rights reserved.
4  * Copyright(c) 2008 Mike Christie
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms and conditions of the GNU General Public License,
8  * version 2, as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope it will be useful, but WITHOUT
11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
13  * more details.
14  *
15  * You should have received a copy of the GNU General Public License along with
16  * this program; if not, write to the Free Software Foundation, Inc.,
17  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18  *
19  * Maintained at www.Open-FCoE.org
20  */
21 
22 /*
23  * Fibre Channel exchange and sequence handling.
24  */
25 
26 #include <linux/timer.h>
27 #include <linux/slab.h>
28 #include <linux/err.h>
29 #include <linux/export.h>
30 #include <linux/log2.h>
31 
32 #include <scsi/fc/fc_fc2.h>
33 
34 #include <scsi/libfc.h>
35 #include <scsi/fc_encode.h>
36 
37 #include "fc_libfc.h"
38 
39 u16	fc_cpu_mask;		/* cpu mask for possible cpus */
40 EXPORT_SYMBOL(fc_cpu_mask);
41 static u16	fc_cpu_order;	/* 2's power to represent total possible cpus */
42 static struct kmem_cache *fc_em_cachep;	       /* cache for exchanges */
43 static struct workqueue_struct *fc_exch_workqueue;
44 
45 /*
46  * Structure and function definitions for managing Fibre Channel Exchanges
47  * and Sequences.
48  *
49  * The three primary structures used here are fc_exch_mgr, fc_exch, and fc_seq.
50  *
51  * fc_exch_mgr holds the exchange state for an N port
52  *
53  * fc_exch holds state for one exchange and links to its active sequence.
54  *
55  * fc_seq holds the state for an individual sequence.
56  */
57 
58 /**
59  * struct fc_exch_pool - Per cpu exchange pool
60  * @next_index:	  Next possible free exchange index
61  * @total_exches: Total allocated exchanges
62  * @lock:	  Exch pool lock
63  * @ex_list:	  List of exchanges
64  *
65  * This structure manages per cpu exchanges in array of exchange pointers.
66  * This array is allocated followed by struct fc_exch_pool memory for
67  * assigned range of exchanges to per cpu pool.
68  */
69 struct fc_exch_pool {
70 	spinlock_t	 lock;
71 	struct list_head ex_list;
72 	u16		 next_index;
73 	u16		 total_exches;
74 
75 	/* two cache of free slot in exch array */
76 	u16		 left;
77 	u16		 right;
78 } ____cacheline_aligned_in_smp;
79 
80 /**
81  * struct fc_exch_mgr - The Exchange Manager (EM).
82  * @class:	    Default class for new sequences
83  * @kref:	    Reference counter
84  * @min_xid:	    Minimum exchange ID
85  * @max_xid:	    Maximum exchange ID
86  * @ep_pool:	    Reserved exchange pointers
87  * @pool_max_index: Max exch array index in exch pool
88  * @pool:	    Per cpu exch pool
89  * @stats:	    Statistics structure
90  *
91  * This structure is the center for creating exchanges and sequences.
92  * It manages the allocation of exchange IDs.
93  */
94 struct fc_exch_mgr {
95 	struct fc_exch_pool __percpu *pool;
96 	mempool_t	*ep_pool;
97 	enum fc_class	class;
98 	struct kref	kref;
99 	u16		min_xid;
100 	u16		max_xid;
101 	u16		pool_max_index;
102 
103 	struct {
104 		atomic_t no_free_exch;
105 		atomic_t no_free_exch_xid;
106 		atomic_t xid_not_found;
107 		atomic_t xid_busy;
108 		atomic_t seq_not_found;
109 		atomic_t non_bls_resp;
110 	} stats;
111 };
112 
113 /**
114  * struct fc_exch_mgr_anchor - primary structure for list of EMs
115  * @ema_list: Exchange Manager Anchor list
116  * @mp:	      Exchange Manager associated with this anchor
117  * @match:    Routine to determine if this anchor's EM should be used
118  *
119  * When walking the list of anchors the match routine will be called
120  * for each anchor to determine if that EM should be used. The last
121  * anchor in the list will always match to handle any exchanges not
122  * handled by other EMs. The non-default EMs would be added to the
123  * anchor list by HW that provides offloads.
124  */
125 struct fc_exch_mgr_anchor {
126 	struct list_head ema_list;
127 	struct fc_exch_mgr *mp;
128 	bool (*match)(struct fc_frame *);
129 };
130 
131 static void fc_exch_rrq(struct fc_exch *);
132 static void fc_seq_ls_acc(struct fc_frame *);
133 static void fc_seq_ls_rjt(struct fc_frame *, enum fc_els_rjt_reason,
134 			  enum fc_els_rjt_explan);
135 static void fc_exch_els_rec(struct fc_frame *);
136 static void fc_exch_els_rrq(struct fc_frame *);
137 
138 /*
139  * Internal implementation notes.
140  *
141  * The exchange manager is one by default in libfc but LLD may choose
142  * to have one per CPU. The sequence manager is one per exchange manager
143  * and currently never separated.
144  *
145  * Section 9.8 in FC-FS-2 specifies:  "The SEQ_ID is a one-byte field
146  * assigned by the Sequence Initiator that shall be unique for a specific
147  * D_ID and S_ID pair while the Sequence is open."   Note that it isn't
148  * qualified by exchange ID, which one might think it would be.
149  * In practice this limits the number of open sequences and exchanges to 256
150  * per session.	 For most targets we could treat this limit as per exchange.
151  *
152  * The exchange and its sequence are freed when the last sequence is received.
153  * It's possible for the remote port to leave an exchange open without
154  * sending any sequences.
155  *
156  * Notes on reference counts:
157  *
158  * Exchanges are reference counted and exchange gets freed when the reference
159  * count becomes zero.
160  *
161  * Timeouts:
162  * Sequences are timed out for E_D_TOV and R_A_TOV.
163  *
164  * Sequence event handling:
165  *
166  * The following events may occur on initiator sequences:
167  *
168  *	Send.
169  *	    For now, the whole thing is sent.
170  *	Receive ACK
171  *	    This applies only to class F.
172  *	    The sequence is marked complete.
173  *	ULP completion.
174  *	    The upper layer calls fc_exch_done() when done
175  *	    with exchange and sequence tuple.
176  *	RX-inferred completion.
177  *	    When we receive the next sequence on the same exchange, we can
178  *	    retire the previous sequence ID.  (XXX not implemented).
179  *	Timeout.
180  *	    R_A_TOV frees the sequence ID.  If we're waiting for ACK,
181  *	    E_D_TOV causes abort and calls upper layer response handler
182  *	    with FC_EX_TIMEOUT error.
183  *	Receive RJT
184  *	    XXX defer.
185  *	Send ABTS
186  *	    On timeout.
187  *
188  * The following events may occur on recipient sequences:
189  *
190  *	Receive
191  *	    Allocate sequence for first frame received.
192  *	    Hold during receive handler.
193  *	    Release when final frame received.
194  *	    Keep status of last N of these for the ELS RES command.  XXX TBD.
195  *	Receive ABTS
196  *	    Deallocate sequence
197  *	Send RJT
198  *	    Deallocate
199  *
200  * For now, we neglect conditions where only part of a sequence was
201  * received or transmitted, or where out-of-order receipt is detected.
202  */
203 
204 /*
205  * Locking notes:
206  *
207  * The EM code run in a per-CPU worker thread.
208  *
209  * To protect against concurrency between a worker thread code and timers,
210  * sequence allocation and deallocation must be locked.
211  *  - exchange refcnt can be done atomicly without locks.
212  *  - sequence allocation must be locked by exch lock.
213  *  - If the EM pool lock and ex_lock must be taken at the same time, then the
214  *    EM pool lock must be taken before the ex_lock.
215  */
216 
217 /*
218  * opcode names for debugging.
219  */
220 static char *fc_exch_rctl_names[] = FC_RCTL_NAMES_INIT;
221 
222 /**
223  * fc_exch_name_lookup() - Lookup name by opcode
224  * @op:	       Opcode to be looked up
225  * @table:     Opcode/name table
226  * @max_index: Index not to be exceeded
227  *
228  * This routine is used to determine a human-readable string identifying
229  * a R_CTL opcode.
230  */
231 static inline const char *fc_exch_name_lookup(unsigned int op, char **table,
232 					      unsigned int max_index)
233 {
234 	const char *name = NULL;
235 
236 	if (op < max_index)
237 		name = table[op];
238 	if (!name)
239 		name = "unknown";
240 	return name;
241 }
242 
243 /**
244  * fc_exch_rctl_name() - Wrapper routine for fc_exch_name_lookup()
245  * @op: The opcode to be looked up
246  */
247 static const char *fc_exch_rctl_name(unsigned int op)
248 {
249 	return fc_exch_name_lookup(op, fc_exch_rctl_names,
250 				   ARRAY_SIZE(fc_exch_rctl_names));
251 }
252 
253 /**
254  * fc_exch_hold() - Increment an exchange's reference count
255  * @ep: Echange to be held
256  */
257 static inline void fc_exch_hold(struct fc_exch *ep)
258 {
259 	atomic_inc(&ep->ex_refcnt);
260 }
261 
262 /**
263  * fc_exch_setup_hdr() - Initialize a FC header by initializing some fields
264  *			 and determine SOF and EOF.
265  * @ep:	   The exchange to that will use the header
266  * @fp:	   The frame whose header is to be modified
267  * @f_ctl: F_CTL bits that will be used for the frame header
268  *
269  * The fields initialized by this routine are: fh_ox_id, fh_rx_id,
270  * fh_seq_id, fh_seq_cnt and the SOF and EOF.
271  */
272 static void fc_exch_setup_hdr(struct fc_exch *ep, struct fc_frame *fp,
273 			      u32 f_ctl)
274 {
275 	struct fc_frame_header *fh = fc_frame_header_get(fp);
276 	u16 fill;
277 
278 	fr_sof(fp) = ep->class;
279 	if (ep->seq.cnt)
280 		fr_sof(fp) = fc_sof_normal(ep->class);
281 
282 	if (f_ctl & FC_FC_END_SEQ) {
283 		fr_eof(fp) = FC_EOF_T;
284 		if (fc_sof_needs_ack(ep->class))
285 			fr_eof(fp) = FC_EOF_N;
286 		/*
287 		 * From F_CTL.
288 		 * The number of fill bytes to make the length a 4-byte
289 		 * multiple is the low order 2-bits of the f_ctl.
290 		 * The fill itself will have been cleared by the frame
291 		 * allocation.
292 		 * After this, the length will be even, as expected by
293 		 * the transport.
294 		 */
295 		fill = fr_len(fp) & 3;
296 		if (fill) {
297 			fill = 4 - fill;
298 			/* TODO, this may be a problem with fragmented skb */
299 			skb_put(fp_skb(fp), fill);
300 			hton24(fh->fh_f_ctl, f_ctl | fill);
301 		}
302 	} else {
303 		WARN_ON(fr_len(fp) % 4 != 0);	/* no pad to non last frame */
304 		fr_eof(fp) = FC_EOF_N;
305 	}
306 
307 	/* Initialize remaining fh fields from fc_fill_fc_hdr */
308 	fh->fh_ox_id = htons(ep->oxid);
309 	fh->fh_rx_id = htons(ep->rxid);
310 	fh->fh_seq_id = ep->seq.id;
311 	fh->fh_seq_cnt = htons(ep->seq.cnt);
312 }
313 
314 /**
315  * fc_exch_release() - Decrement an exchange's reference count
316  * @ep: Exchange to be released
317  *
318  * If the reference count reaches zero and the exchange is complete,
319  * it is freed.
320  */
321 static void fc_exch_release(struct fc_exch *ep)
322 {
323 	struct fc_exch_mgr *mp;
324 
325 	if (atomic_dec_and_test(&ep->ex_refcnt)) {
326 		mp = ep->em;
327 		if (ep->destructor)
328 			ep->destructor(&ep->seq, ep->arg);
329 		WARN_ON(!(ep->esb_stat & ESB_ST_COMPLETE));
330 		mempool_free(ep, mp->ep_pool);
331 	}
332 }
333 
334 /**
335  * fc_exch_timer_cancel() - cancel exch timer
336  * @ep:		The exchange whose timer to be canceled
337  */
338 static inline void fc_exch_timer_cancel(struct fc_exch *ep)
339 {
340 	if (cancel_delayed_work(&ep->timeout_work)) {
341 		FC_EXCH_DBG(ep, "Exchange timer canceled\n");
342 		atomic_dec(&ep->ex_refcnt); /* drop hold for timer */
343 	}
344 }
345 
346 /**
347  * fc_exch_timer_set_locked() - Start a timer for an exchange w/ the
348  *				the exchange lock held
349  * @ep:		The exchange whose timer will start
350  * @timer_msec: The timeout period
351  *
352  * Used for upper level protocols to time out the exchange.
353  * The timer is cancelled when it fires or when the exchange completes.
354  */
355 static inline void fc_exch_timer_set_locked(struct fc_exch *ep,
356 					    unsigned int timer_msec)
357 {
358 	if (ep->state & (FC_EX_RST_CLEANUP | FC_EX_DONE))
359 		return;
360 
361 	FC_EXCH_DBG(ep, "Exchange timer armed : %d msecs\n", timer_msec);
362 
363 	fc_exch_hold(ep);		/* hold for timer */
364 	if (!queue_delayed_work(fc_exch_workqueue, &ep->timeout_work,
365 				msecs_to_jiffies(timer_msec)))
366 		fc_exch_release(ep);
367 }
368 
369 /**
370  * fc_exch_timer_set() - Lock the exchange and set the timer
371  * @ep:		The exchange whose timer will start
372  * @timer_msec: The timeout period
373  */
374 static void fc_exch_timer_set(struct fc_exch *ep, unsigned int timer_msec)
375 {
376 	spin_lock_bh(&ep->ex_lock);
377 	fc_exch_timer_set_locked(ep, timer_msec);
378 	spin_unlock_bh(&ep->ex_lock);
379 }
380 
381 /**
382  * fc_exch_done_locked() - Complete an exchange with the exchange lock held
383  * @ep: The exchange that is complete
384  *
385  * Note: May sleep if invoked from outside a response handler.
386  */
387 static int fc_exch_done_locked(struct fc_exch *ep)
388 {
389 	int rc = 1;
390 
391 	/*
392 	 * We must check for completion in case there are two threads
393 	 * tyring to complete this. But the rrq code will reuse the
394 	 * ep, and in that case we only clear the resp and set it as
395 	 * complete, so it can be reused by the timer to send the rrq.
396 	 */
397 	if (ep->state & FC_EX_DONE)
398 		return rc;
399 	ep->esb_stat |= ESB_ST_COMPLETE;
400 
401 	if (!(ep->esb_stat & ESB_ST_REC_QUAL)) {
402 		ep->state |= FC_EX_DONE;
403 		fc_exch_timer_cancel(ep);
404 		rc = 0;
405 	}
406 	return rc;
407 }
408 
409 /**
410  * fc_exch_ptr_get() - Return an exchange from an exchange pool
411  * @pool:  Exchange Pool to get an exchange from
412  * @index: Index of the exchange within the pool
413  *
414  * Use the index to get an exchange from within an exchange pool. exches
415  * will point to an array of exchange pointers. The index will select
416  * the exchange within the array.
417  */
418 static inline struct fc_exch *fc_exch_ptr_get(struct fc_exch_pool *pool,
419 					      u16 index)
420 {
421 	struct fc_exch **exches = (struct fc_exch **)(pool + 1);
422 	return exches[index];
423 }
424 
425 /**
426  * fc_exch_ptr_set() - Assign an exchange to a slot in an exchange pool
427  * @pool:  The pool to assign the exchange to
428  * @index: The index in the pool where the exchange will be assigned
429  * @ep:	   The exchange to assign to the pool
430  */
431 static inline void fc_exch_ptr_set(struct fc_exch_pool *pool, u16 index,
432 				   struct fc_exch *ep)
433 {
434 	((struct fc_exch **)(pool + 1))[index] = ep;
435 }
436 
437 /**
438  * fc_exch_delete() - Delete an exchange
439  * @ep: The exchange to be deleted
440  */
441 static void fc_exch_delete(struct fc_exch *ep)
442 {
443 	struct fc_exch_pool *pool;
444 	u16 index;
445 
446 	pool = ep->pool;
447 	spin_lock_bh(&pool->lock);
448 	WARN_ON(pool->total_exches <= 0);
449 	pool->total_exches--;
450 
451 	/* update cache of free slot */
452 	index = (ep->xid - ep->em->min_xid) >> fc_cpu_order;
453 	if (pool->left == FC_XID_UNKNOWN)
454 		pool->left = index;
455 	else if (pool->right == FC_XID_UNKNOWN)
456 		pool->right = index;
457 	else
458 		pool->next_index = index;
459 
460 	fc_exch_ptr_set(pool, index, NULL);
461 	list_del(&ep->ex_list);
462 	spin_unlock_bh(&pool->lock);
463 	fc_exch_release(ep);	/* drop hold for exch in mp */
464 }
465 
466 static int fc_seq_send_locked(struct fc_lport *lport, struct fc_seq *sp,
467 			      struct fc_frame *fp)
468 {
469 	struct fc_exch *ep;
470 	struct fc_frame_header *fh = fc_frame_header_get(fp);
471 	int error = -ENXIO;
472 	u32 f_ctl;
473 	u8 fh_type = fh->fh_type;
474 
475 	ep = fc_seq_exch(sp);
476 
477 	if (ep->esb_stat & (ESB_ST_COMPLETE | ESB_ST_ABNORMAL)) {
478 		fc_frame_free(fp);
479 		goto out;
480 	}
481 
482 	WARN_ON(!(ep->esb_stat & ESB_ST_SEQ_INIT));
483 
484 	f_ctl = ntoh24(fh->fh_f_ctl);
485 	fc_exch_setup_hdr(ep, fp, f_ctl);
486 	fr_encaps(fp) = ep->encaps;
487 
488 	/*
489 	 * update sequence count if this frame is carrying
490 	 * multiple FC frames when sequence offload is enabled
491 	 * by LLD.
492 	 */
493 	if (fr_max_payload(fp))
494 		sp->cnt += DIV_ROUND_UP((fr_len(fp) - sizeof(*fh)),
495 					fr_max_payload(fp));
496 	else
497 		sp->cnt++;
498 
499 	/*
500 	 * Send the frame.
501 	 */
502 	error = lport->tt.frame_send(lport, fp);
503 
504 	if (fh_type == FC_TYPE_BLS)
505 		goto out;
506 
507 	/*
508 	 * Update the exchange and sequence flags,
509 	 * assuming all frames for the sequence have been sent.
510 	 * We can only be called to send once for each sequence.
511 	 */
512 	ep->f_ctl = f_ctl & ~FC_FC_FIRST_SEQ;	/* not first seq */
513 	if (f_ctl & FC_FC_SEQ_INIT)
514 		ep->esb_stat &= ~ESB_ST_SEQ_INIT;
515 out:
516 	return error;
517 }
518 
519 /**
520  * fc_seq_send() - Send a frame using existing sequence/exchange pair
521  * @lport: The local port that the exchange will be sent on
522  * @sp:	   The sequence to be sent
523  * @fp:	   The frame to be sent on the exchange
524  *
525  * Note: The frame will be freed either by a direct call to fc_frame_free(fp)
526  * or indirectly by calling libfc_function_template.frame_send().
527  */
528 static int fc_seq_send(struct fc_lport *lport, struct fc_seq *sp,
529 		       struct fc_frame *fp)
530 {
531 	struct fc_exch *ep;
532 	int error;
533 	ep = fc_seq_exch(sp);
534 	spin_lock_bh(&ep->ex_lock);
535 	error = fc_seq_send_locked(lport, sp, fp);
536 	spin_unlock_bh(&ep->ex_lock);
537 	return error;
538 }
539 
540 /**
541  * fc_seq_alloc() - Allocate a sequence for a given exchange
542  * @ep:	    The exchange to allocate a new sequence for
543  * @seq_id: The sequence ID to be used
544  *
545  * We don't support multiple originated sequences on the same exchange.
546  * By implication, any previously originated sequence on this exchange
547  * is complete, and we reallocate the same sequence.
548  */
549 static struct fc_seq *fc_seq_alloc(struct fc_exch *ep, u8 seq_id)
550 {
551 	struct fc_seq *sp;
552 
553 	sp = &ep->seq;
554 	sp->ssb_stat = 0;
555 	sp->cnt = 0;
556 	sp->id = seq_id;
557 	return sp;
558 }
559 
560 /**
561  * fc_seq_start_next_locked() - Allocate a new sequence on the same
562  *				exchange as the supplied sequence
563  * @sp: The sequence/exchange to get a new sequence for
564  */
565 static struct fc_seq *fc_seq_start_next_locked(struct fc_seq *sp)
566 {
567 	struct fc_exch *ep = fc_seq_exch(sp);
568 
569 	sp = fc_seq_alloc(ep, ep->seq_id++);
570 	FC_EXCH_DBG(ep, "f_ctl %6x seq %2x\n",
571 		    ep->f_ctl, sp->id);
572 	return sp;
573 }
574 
575 /**
576  * fc_seq_start_next() - Lock the exchange and get a new sequence
577  *			 for a given sequence/exchange pair
578  * @sp: The sequence/exchange to get a new exchange for
579  */
580 static struct fc_seq *fc_seq_start_next(struct fc_seq *sp)
581 {
582 	struct fc_exch *ep = fc_seq_exch(sp);
583 
584 	spin_lock_bh(&ep->ex_lock);
585 	sp = fc_seq_start_next_locked(sp);
586 	spin_unlock_bh(&ep->ex_lock);
587 
588 	return sp;
589 }
590 
591 /*
592  * Set the response handler for the exchange associated with a sequence.
593  *
594  * Note: May sleep if invoked from outside a response handler.
595  */
596 static void fc_seq_set_resp(struct fc_seq *sp,
597 			    void (*resp)(struct fc_seq *, struct fc_frame *,
598 					 void *),
599 			    void *arg)
600 {
601 	struct fc_exch *ep = fc_seq_exch(sp);
602 	DEFINE_WAIT(wait);
603 
604 	spin_lock_bh(&ep->ex_lock);
605 	while (ep->resp_active && ep->resp_task != current) {
606 		prepare_to_wait(&ep->resp_wq, &wait, TASK_UNINTERRUPTIBLE);
607 		spin_unlock_bh(&ep->ex_lock);
608 
609 		schedule();
610 
611 		spin_lock_bh(&ep->ex_lock);
612 	}
613 	finish_wait(&ep->resp_wq, &wait);
614 	ep->resp = resp;
615 	ep->arg = arg;
616 	spin_unlock_bh(&ep->ex_lock);
617 }
618 
619 /**
620  * fc_exch_abort_locked() - Abort an exchange
621  * @ep:	The exchange to be aborted
622  * @timer_msec: The period of time to wait before aborting
623  *
624  * Locking notes:  Called with exch lock held
625  *
626  * Return value: 0 on success else error code
627  */
628 static int fc_exch_abort_locked(struct fc_exch *ep,
629 				unsigned int timer_msec)
630 {
631 	struct fc_seq *sp;
632 	struct fc_frame *fp;
633 	int error;
634 
635 	if (ep->esb_stat & (ESB_ST_COMPLETE | ESB_ST_ABNORMAL) ||
636 	    ep->state & (FC_EX_DONE | FC_EX_RST_CLEANUP))
637 		return -ENXIO;
638 
639 	/*
640 	 * Send the abort on a new sequence if possible.
641 	 */
642 	sp = fc_seq_start_next_locked(&ep->seq);
643 	if (!sp)
644 		return -ENOMEM;
645 
646 	if (timer_msec)
647 		fc_exch_timer_set_locked(ep, timer_msec);
648 
649 	if (ep->sid) {
650 		/*
651 		 * Send an abort for the sequence that timed out.
652 		 */
653 		fp = fc_frame_alloc(ep->lp, 0);
654 		if (fp) {
655 			ep->esb_stat |= ESB_ST_SEQ_INIT;
656 			fc_fill_fc_hdr(fp, FC_RCTL_BA_ABTS, ep->did, ep->sid,
657 				       FC_TYPE_BLS, FC_FC_END_SEQ |
658 				       FC_FC_SEQ_INIT, 0);
659 			error = fc_seq_send_locked(ep->lp, sp, fp);
660 		} else {
661 			error = -ENOBUFS;
662 		}
663 	} else {
664 		/*
665 		 * If not logged into the fabric, don't send ABTS but leave
666 		 * sequence active until next timeout.
667 		 */
668 		error = 0;
669 	}
670 	ep->esb_stat |= ESB_ST_ABNORMAL;
671 	return error;
672 }
673 
674 /**
675  * fc_seq_exch_abort() - Abort an exchange and sequence
676  * @req_sp:	The sequence to be aborted
677  * @timer_msec: The period of time to wait before aborting
678  *
679  * Generally called because of a timeout or an abort from the upper layer.
680  *
681  * Return value: 0 on success else error code
682  */
683 static int fc_seq_exch_abort(const struct fc_seq *req_sp,
684 			     unsigned int timer_msec)
685 {
686 	struct fc_exch *ep;
687 	int error;
688 
689 	ep = fc_seq_exch(req_sp);
690 	spin_lock_bh(&ep->ex_lock);
691 	error = fc_exch_abort_locked(ep, timer_msec);
692 	spin_unlock_bh(&ep->ex_lock);
693 	return error;
694 }
695 
696 /**
697  * fc_invoke_resp() - invoke ep->resp()
698  *
699  * Notes:
700  * It is assumed that after initialization finished (this means the
701  * first unlock of ex_lock after fc_exch_alloc()) ep->resp and ep->arg are
702  * modified only via fc_seq_set_resp(). This guarantees that none of these
703  * two variables changes if ep->resp_active > 0.
704  *
705  * If an fc_seq_set_resp() call is busy modifying ep->resp and ep->arg when
706  * this function is invoked, the first spin_lock_bh() call in this function
707  * will wait until fc_seq_set_resp() has finished modifying these variables.
708  *
709  * Since fc_exch_done() invokes fc_seq_set_resp() it is guaranteed that that
710  * ep->resp() won't be invoked after fc_exch_done() has returned.
711  *
712  * The response handler itself may invoke fc_exch_done(), which will clear the
713  * ep->resp pointer.
714  *
715  * Return value:
716  * Returns true if and only if ep->resp has been invoked.
717  */
718 static bool fc_invoke_resp(struct fc_exch *ep, struct fc_seq *sp,
719 			   struct fc_frame *fp)
720 {
721 	void (*resp)(struct fc_seq *, struct fc_frame *fp, void *arg);
722 	void *arg;
723 	bool res = false;
724 
725 	spin_lock_bh(&ep->ex_lock);
726 	ep->resp_active++;
727 	if (ep->resp_task != current)
728 		ep->resp_task = !ep->resp_task ? current : NULL;
729 	resp = ep->resp;
730 	arg = ep->arg;
731 	spin_unlock_bh(&ep->ex_lock);
732 
733 	if (resp) {
734 		resp(sp, fp, arg);
735 		res = true;
736 	}
737 
738 	spin_lock_bh(&ep->ex_lock);
739 	if (--ep->resp_active == 0)
740 		ep->resp_task = NULL;
741 	spin_unlock_bh(&ep->ex_lock);
742 
743 	if (ep->resp_active == 0)
744 		wake_up(&ep->resp_wq);
745 
746 	return res;
747 }
748 
749 /**
750  * fc_exch_timeout() - Handle exchange timer expiration
751  * @work: The work_struct identifying the exchange that timed out
752  */
753 static void fc_exch_timeout(struct work_struct *work)
754 {
755 	struct fc_exch *ep = container_of(work, struct fc_exch,
756 					  timeout_work.work);
757 	struct fc_seq *sp = &ep->seq;
758 	u32 e_stat;
759 	int rc = 1;
760 
761 	FC_EXCH_DBG(ep, "Exchange timed out\n");
762 
763 	spin_lock_bh(&ep->ex_lock);
764 	if (ep->state & (FC_EX_RST_CLEANUP | FC_EX_DONE))
765 		goto unlock;
766 
767 	e_stat = ep->esb_stat;
768 	if (e_stat & ESB_ST_COMPLETE) {
769 		ep->esb_stat = e_stat & ~ESB_ST_REC_QUAL;
770 		spin_unlock_bh(&ep->ex_lock);
771 		if (e_stat & ESB_ST_REC_QUAL)
772 			fc_exch_rrq(ep);
773 		goto done;
774 	} else {
775 		if (e_stat & ESB_ST_ABNORMAL)
776 			rc = fc_exch_done_locked(ep);
777 		spin_unlock_bh(&ep->ex_lock);
778 		if (!rc)
779 			fc_exch_delete(ep);
780 		fc_invoke_resp(ep, sp, ERR_PTR(-FC_EX_TIMEOUT));
781 		fc_seq_set_resp(sp, NULL, ep->arg);
782 		fc_seq_exch_abort(sp, 2 * ep->r_a_tov);
783 		goto done;
784 	}
785 unlock:
786 	spin_unlock_bh(&ep->ex_lock);
787 done:
788 	/*
789 	 * This release matches the hold taken when the timer was set.
790 	 */
791 	fc_exch_release(ep);
792 }
793 
794 /**
795  * fc_exch_em_alloc() - Allocate an exchange from a specified EM.
796  * @lport: The local port that the exchange is for
797  * @mp:	   The exchange manager that will allocate the exchange
798  *
799  * Returns pointer to allocated fc_exch with exch lock held.
800  */
801 static struct fc_exch *fc_exch_em_alloc(struct fc_lport *lport,
802 					struct fc_exch_mgr *mp)
803 {
804 	struct fc_exch *ep;
805 	unsigned int cpu;
806 	u16 index;
807 	struct fc_exch_pool *pool;
808 
809 	/* allocate memory for exchange */
810 	ep = mempool_alloc(mp->ep_pool, GFP_ATOMIC);
811 	if (!ep) {
812 		atomic_inc(&mp->stats.no_free_exch);
813 		goto out;
814 	}
815 	memset(ep, 0, sizeof(*ep));
816 
817 	cpu = get_cpu();
818 	pool = per_cpu_ptr(mp->pool, cpu);
819 	spin_lock_bh(&pool->lock);
820 	put_cpu();
821 
822 	/* peek cache of free slot */
823 	if (pool->left != FC_XID_UNKNOWN) {
824 		index = pool->left;
825 		pool->left = FC_XID_UNKNOWN;
826 		goto hit;
827 	}
828 	if (pool->right != FC_XID_UNKNOWN) {
829 		index = pool->right;
830 		pool->right = FC_XID_UNKNOWN;
831 		goto hit;
832 	}
833 
834 	index = pool->next_index;
835 	/* allocate new exch from pool */
836 	while (fc_exch_ptr_get(pool, index)) {
837 		index = index == mp->pool_max_index ? 0 : index + 1;
838 		if (index == pool->next_index)
839 			goto err;
840 	}
841 	pool->next_index = index == mp->pool_max_index ? 0 : index + 1;
842 hit:
843 	fc_exch_hold(ep);	/* hold for exch in mp */
844 	spin_lock_init(&ep->ex_lock);
845 	/*
846 	 * Hold exch lock for caller to prevent fc_exch_reset()
847 	 * from releasing exch	while fc_exch_alloc() caller is
848 	 * still working on exch.
849 	 */
850 	spin_lock_bh(&ep->ex_lock);
851 
852 	fc_exch_ptr_set(pool, index, ep);
853 	list_add_tail(&ep->ex_list, &pool->ex_list);
854 	fc_seq_alloc(ep, ep->seq_id++);
855 	pool->total_exches++;
856 	spin_unlock_bh(&pool->lock);
857 
858 	/*
859 	 *  update exchange
860 	 */
861 	ep->oxid = ep->xid = (index << fc_cpu_order | cpu) + mp->min_xid;
862 	ep->em = mp;
863 	ep->pool = pool;
864 	ep->lp = lport;
865 	ep->f_ctl = FC_FC_FIRST_SEQ;	/* next seq is first seq */
866 	ep->rxid = FC_XID_UNKNOWN;
867 	ep->class = mp->class;
868 	ep->resp_active = 0;
869 	init_waitqueue_head(&ep->resp_wq);
870 	INIT_DELAYED_WORK(&ep->timeout_work, fc_exch_timeout);
871 out:
872 	return ep;
873 err:
874 	spin_unlock_bh(&pool->lock);
875 	atomic_inc(&mp->stats.no_free_exch_xid);
876 	mempool_free(ep, mp->ep_pool);
877 	return NULL;
878 }
879 
880 /**
881  * fc_exch_alloc() - Allocate an exchange from an EM on a
882  *		     local port's list of EMs.
883  * @lport: The local port that will own the exchange
884  * @fp:	   The FC frame that the exchange will be for
885  *
886  * This function walks the list of exchange manager(EM)
887  * anchors to select an EM for a new exchange allocation. The
888  * EM is selected when a NULL match function pointer is encountered
889  * or when a call to a match function returns true.
890  */
891 static inline struct fc_exch *fc_exch_alloc(struct fc_lport *lport,
892 					    struct fc_frame *fp)
893 {
894 	struct fc_exch_mgr_anchor *ema;
895 
896 	list_for_each_entry(ema, &lport->ema_list, ema_list)
897 		if (!ema->match || ema->match(fp))
898 			return fc_exch_em_alloc(lport, ema->mp);
899 	return NULL;
900 }
901 
902 /**
903  * fc_exch_find() - Lookup and hold an exchange
904  * @mp:	 The exchange manager to lookup the exchange from
905  * @xid: The XID of the exchange to look up
906  */
907 static struct fc_exch *fc_exch_find(struct fc_exch_mgr *mp, u16 xid)
908 {
909 	struct fc_exch_pool *pool;
910 	struct fc_exch *ep = NULL;
911 	u16 cpu = xid & fc_cpu_mask;
912 
913 	if (cpu >= nr_cpu_ids || !cpu_possible(cpu)) {
914 		printk_ratelimited(KERN_ERR
915 			"libfc: lookup request for XID = %d, "
916 			"indicates invalid CPU %d\n", xid, cpu);
917 		return NULL;
918 	}
919 
920 	if ((xid >= mp->min_xid) && (xid <= mp->max_xid)) {
921 		pool = per_cpu_ptr(mp->pool, cpu);
922 		spin_lock_bh(&pool->lock);
923 		ep = fc_exch_ptr_get(pool, (xid - mp->min_xid) >> fc_cpu_order);
924 		if (ep) {
925 			WARN_ON(ep->xid != xid);
926 			fc_exch_hold(ep);
927 		}
928 		spin_unlock_bh(&pool->lock);
929 	}
930 	return ep;
931 }
932 
933 
934 /**
935  * fc_exch_done() - Indicate that an exchange/sequence tuple is complete and
936  *		    the memory allocated for the related objects may be freed.
937  * @sp: The sequence that has completed
938  *
939  * Note: May sleep if invoked from outside a response handler.
940  */
941 static void fc_exch_done(struct fc_seq *sp)
942 {
943 	struct fc_exch *ep = fc_seq_exch(sp);
944 	int rc;
945 
946 	spin_lock_bh(&ep->ex_lock);
947 	rc = fc_exch_done_locked(ep);
948 	spin_unlock_bh(&ep->ex_lock);
949 
950 	fc_seq_set_resp(sp, NULL, ep->arg);
951 	if (!rc)
952 		fc_exch_delete(ep);
953 }
954 
955 /**
956  * fc_exch_resp() - Allocate a new exchange for a response frame
957  * @lport: The local port that the exchange was for
958  * @mp:	   The exchange manager to allocate the exchange from
959  * @fp:	   The response frame
960  *
961  * Sets the responder ID in the frame header.
962  */
963 static struct fc_exch *fc_exch_resp(struct fc_lport *lport,
964 				    struct fc_exch_mgr *mp,
965 				    struct fc_frame *fp)
966 {
967 	struct fc_exch *ep;
968 	struct fc_frame_header *fh;
969 
970 	ep = fc_exch_alloc(lport, fp);
971 	if (ep) {
972 		ep->class = fc_frame_class(fp);
973 
974 		/*
975 		 * Set EX_CTX indicating we're responding on this exchange.
976 		 */
977 		ep->f_ctl |= FC_FC_EX_CTX;	/* we're responding */
978 		ep->f_ctl &= ~FC_FC_FIRST_SEQ;	/* not new */
979 		fh = fc_frame_header_get(fp);
980 		ep->sid = ntoh24(fh->fh_d_id);
981 		ep->did = ntoh24(fh->fh_s_id);
982 		ep->oid = ep->did;
983 
984 		/*
985 		 * Allocated exchange has placed the XID in the
986 		 * originator field. Move it to the responder field,
987 		 * and set the originator XID from the frame.
988 		 */
989 		ep->rxid = ep->xid;
990 		ep->oxid = ntohs(fh->fh_ox_id);
991 		ep->esb_stat |= ESB_ST_RESP | ESB_ST_SEQ_INIT;
992 		if ((ntoh24(fh->fh_f_ctl) & FC_FC_SEQ_INIT) == 0)
993 			ep->esb_stat &= ~ESB_ST_SEQ_INIT;
994 
995 		fc_exch_hold(ep);	/* hold for caller */
996 		spin_unlock_bh(&ep->ex_lock);	/* lock from fc_exch_alloc */
997 	}
998 	return ep;
999 }
1000 
1001 /**
1002  * fc_seq_lookup_recip() - Find a sequence where the other end
1003  *			   originated the sequence
1004  * @lport: The local port that the frame was sent to
1005  * @mp:	   The Exchange Manager to lookup the exchange from
1006  * @fp:	   The frame associated with the sequence we're looking for
1007  *
1008  * If fc_pf_rjt_reason is FC_RJT_NONE then this function will have a hold
1009  * on the ep that should be released by the caller.
1010  */
1011 static enum fc_pf_rjt_reason fc_seq_lookup_recip(struct fc_lport *lport,
1012 						 struct fc_exch_mgr *mp,
1013 						 struct fc_frame *fp)
1014 {
1015 	struct fc_frame_header *fh = fc_frame_header_get(fp);
1016 	struct fc_exch *ep = NULL;
1017 	struct fc_seq *sp = NULL;
1018 	enum fc_pf_rjt_reason reject = FC_RJT_NONE;
1019 	u32 f_ctl;
1020 	u16 xid;
1021 
1022 	f_ctl = ntoh24(fh->fh_f_ctl);
1023 	WARN_ON((f_ctl & FC_FC_SEQ_CTX) != 0);
1024 
1025 	/*
1026 	 * Lookup or create the exchange if we will be creating the sequence.
1027 	 */
1028 	if (f_ctl & FC_FC_EX_CTX) {
1029 		xid = ntohs(fh->fh_ox_id);	/* we originated exch */
1030 		ep = fc_exch_find(mp, xid);
1031 		if (!ep) {
1032 			atomic_inc(&mp->stats.xid_not_found);
1033 			reject = FC_RJT_OX_ID;
1034 			goto out;
1035 		}
1036 		if (ep->rxid == FC_XID_UNKNOWN)
1037 			ep->rxid = ntohs(fh->fh_rx_id);
1038 		else if (ep->rxid != ntohs(fh->fh_rx_id)) {
1039 			reject = FC_RJT_OX_ID;
1040 			goto rel;
1041 		}
1042 	} else {
1043 		xid = ntohs(fh->fh_rx_id);	/* we are the responder */
1044 
1045 		/*
1046 		 * Special case for MDS issuing an ELS TEST with a
1047 		 * bad rxid of 0.
1048 		 * XXX take this out once we do the proper reject.
1049 		 */
1050 		if (xid == 0 && fh->fh_r_ctl == FC_RCTL_ELS_REQ &&
1051 		    fc_frame_payload_op(fp) == ELS_TEST) {
1052 			fh->fh_rx_id = htons(FC_XID_UNKNOWN);
1053 			xid = FC_XID_UNKNOWN;
1054 		}
1055 
1056 		/*
1057 		 * new sequence - find the exchange
1058 		 */
1059 		ep = fc_exch_find(mp, xid);
1060 		if ((f_ctl & FC_FC_FIRST_SEQ) && fc_sof_is_init(fr_sof(fp))) {
1061 			if (ep) {
1062 				atomic_inc(&mp->stats.xid_busy);
1063 				reject = FC_RJT_RX_ID;
1064 				goto rel;
1065 			}
1066 			ep = fc_exch_resp(lport, mp, fp);
1067 			if (!ep) {
1068 				reject = FC_RJT_EXCH_EST;	/* XXX */
1069 				goto out;
1070 			}
1071 			xid = ep->xid;	/* get our XID */
1072 		} else if (!ep) {
1073 			atomic_inc(&mp->stats.xid_not_found);
1074 			reject = FC_RJT_RX_ID;	/* XID not found */
1075 			goto out;
1076 		}
1077 	}
1078 
1079 	spin_lock_bh(&ep->ex_lock);
1080 	/*
1081 	 * At this point, we have the exchange held.
1082 	 * Find or create the sequence.
1083 	 */
1084 	if (fc_sof_is_init(fr_sof(fp))) {
1085 		sp = &ep->seq;
1086 		sp->ssb_stat |= SSB_ST_RESP;
1087 		sp->id = fh->fh_seq_id;
1088 	} else {
1089 		sp = &ep->seq;
1090 		if (sp->id != fh->fh_seq_id) {
1091 			atomic_inc(&mp->stats.seq_not_found);
1092 			if (f_ctl & FC_FC_END_SEQ) {
1093 				/*
1094 				 * Update sequence_id based on incoming last
1095 				 * frame of sequence exchange. This is needed
1096 				 * for FC target where DDP has been used
1097 				 * on target where, stack is indicated only
1098 				 * about last frame's (payload _header) header.
1099 				 * Whereas "seq_id" which is part of
1100 				 * frame_header is allocated by initiator
1101 				 * which is totally different from "seq_id"
1102 				 * allocated when XFER_RDY was sent by target.
1103 				 * To avoid false -ve which results into not
1104 				 * sending RSP, hence write request on other
1105 				 * end never finishes.
1106 				 */
1107 				sp->ssb_stat |= SSB_ST_RESP;
1108 				sp->id = fh->fh_seq_id;
1109 			} else {
1110 				spin_unlock_bh(&ep->ex_lock);
1111 
1112 				/* sequence/exch should exist */
1113 				reject = FC_RJT_SEQ_ID;
1114 				goto rel;
1115 			}
1116 		}
1117 	}
1118 	WARN_ON(ep != fc_seq_exch(sp));
1119 
1120 	if (f_ctl & FC_FC_SEQ_INIT)
1121 		ep->esb_stat |= ESB_ST_SEQ_INIT;
1122 	spin_unlock_bh(&ep->ex_lock);
1123 
1124 	fr_seq(fp) = sp;
1125 out:
1126 	return reject;
1127 rel:
1128 	fc_exch_done(&ep->seq);
1129 	fc_exch_release(ep);	/* hold from fc_exch_find/fc_exch_resp */
1130 	return reject;
1131 }
1132 
1133 /**
1134  * fc_seq_lookup_orig() - Find a sequence where this end
1135  *			  originated the sequence
1136  * @mp:	   The Exchange Manager to lookup the exchange from
1137  * @fp:	   The frame associated with the sequence we're looking for
1138  *
1139  * Does not hold the sequence for the caller.
1140  */
1141 static struct fc_seq *fc_seq_lookup_orig(struct fc_exch_mgr *mp,
1142 					 struct fc_frame *fp)
1143 {
1144 	struct fc_frame_header *fh = fc_frame_header_get(fp);
1145 	struct fc_exch *ep;
1146 	struct fc_seq *sp = NULL;
1147 	u32 f_ctl;
1148 	u16 xid;
1149 
1150 	f_ctl = ntoh24(fh->fh_f_ctl);
1151 	WARN_ON((f_ctl & FC_FC_SEQ_CTX) != FC_FC_SEQ_CTX);
1152 	xid = ntohs((f_ctl & FC_FC_EX_CTX) ? fh->fh_ox_id : fh->fh_rx_id);
1153 	ep = fc_exch_find(mp, xid);
1154 	if (!ep)
1155 		return NULL;
1156 	if (ep->seq.id == fh->fh_seq_id) {
1157 		/*
1158 		 * Save the RX_ID if we didn't previously know it.
1159 		 */
1160 		sp = &ep->seq;
1161 		if ((f_ctl & FC_FC_EX_CTX) != 0 &&
1162 		    ep->rxid == FC_XID_UNKNOWN) {
1163 			ep->rxid = ntohs(fh->fh_rx_id);
1164 		}
1165 	}
1166 	fc_exch_release(ep);
1167 	return sp;
1168 }
1169 
1170 /**
1171  * fc_exch_set_addr() - Set the source and destination IDs for an exchange
1172  * @ep:	     The exchange to set the addresses for
1173  * @orig_id: The originator's ID
1174  * @resp_id: The responder's ID
1175  *
1176  * Note this must be done before the first sequence of the exchange is sent.
1177  */
1178 static void fc_exch_set_addr(struct fc_exch *ep,
1179 			     u32 orig_id, u32 resp_id)
1180 {
1181 	ep->oid = orig_id;
1182 	if (ep->esb_stat & ESB_ST_RESP) {
1183 		ep->sid = resp_id;
1184 		ep->did = orig_id;
1185 	} else {
1186 		ep->sid = orig_id;
1187 		ep->did = resp_id;
1188 	}
1189 }
1190 
1191 /**
1192  * fc_seq_els_rsp_send() - Send an ELS response using information from
1193  *			   the existing sequence/exchange.
1194  * @fp:	      The received frame
1195  * @els_cmd:  The ELS command to be sent
1196  * @els_data: The ELS data to be sent
1197  *
1198  * The received frame is not freed.
1199  */
1200 static void fc_seq_els_rsp_send(struct fc_frame *fp, enum fc_els_cmd els_cmd,
1201 				struct fc_seq_els_data *els_data)
1202 {
1203 	switch (els_cmd) {
1204 	case ELS_LS_RJT:
1205 		fc_seq_ls_rjt(fp, els_data->reason, els_data->explan);
1206 		break;
1207 	case ELS_LS_ACC:
1208 		fc_seq_ls_acc(fp);
1209 		break;
1210 	case ELS_RRQ:
1211 		fc_exch_els_rrq(fp);
1212 		break;
1213 	case ELS_REC:
1214 		fc_exch_els_rec(fp);
1215 		break;
1216 	default:
1217 		FC_LPORT_DBG(fr_dev(fp), "Invalid ELS CMD:%x\n", els_cmd);
1218 	}
1219 }
1220 
1221 /**
1222  * fc_seq_send_last() - Send a sequence that is the last in the exchange
1223  * @sp:	     The sequence that is to be sent
1224  * @fp:	     The frame that will be sent on the sequence
1225  * @rctl:    The R_CTL information to be sent
1226  * @fh_type: The frame header type
1227  */
1228 static void fc_seq_send_last(struct fc_seq *sp, struct fc_frame *fp,
1229 			     enum fc_rctl rctl, enum fc_fh_type fh_type)
1230 {
1231 	u32 f_ctl;
1232 	struct fc_exch *ep = fc_seq_exch(sp);
1233 
1234 	f_ctl = FC_FC_LAST_SEQ | FC_FC_END_SEQ | FC_FC_SEQ_INIT;
1235 	f_ctl |= ep->f_ctl;
1236 	fc_fill_fc_hdr(fp, rctl, ep->did, ep->sid, fh_type, f_ctl, 0);
1237 	fc_seq_send_locked(ep->lp, sp, fp);
1238 }
1239 
1240 /**
1241  * fc_seq_send_ack() - Send an acknowledgement that we've received a frame
1242  * @sp:	   The sequence to send the ACK on
1243  * @rx_fp: The received frame that is being acknoledged
1244  *
1245  * Send ACK_1 (or equiv.) indicating we received something.
1246  */
1247 static void fc_seq_send_ack(struct fc_seq *sp, const struct fc_frame *rx_fp)
1248 {
1249 	struct fc_frame *fp;
1250 	struct fc_frame_header *rx_fh;
1251 	struct fc_frame_header *fh;
1252 	struct fc_exch *ep = fc_seq_exch(sp);
1253 	struct fc_lport *lport = ep->lp;
1254 	unsigned int f_ctl;
1255 
1256 	/*
1257 	 * Don't send ACKs for class 3.
1258 	 */
1259 	if (fc_sof_needs_ack(fr_sof(rx_fp))) {
1260 		fp = fc_frame_alloc(lport, 0);
1261 		if (!fp)
1262 			return;
1263 
1264 		fh = fc_frame_header_get(fp);
1265 		fh->fh_r_ctl = FC_RCTL_ACK_1;
1266 		fh->fh_type = FC_TYPE_BLS;
1267 
1268 		/*
1269 		 * Form f_ctl by inverting EX_CTX and SEQ_CTX (bits 23, 22).
1270 		 * Echo FIRST_SEQ, LAST_SEQ, END_SEQ, END_CONN, SEQ_INIT.
1271 		 * Bits 9-8 are meaningful (retransmitted or unidirectional).
1272 		 * Last ACK uses bits 7-6 (continue sequence),
1273 		 * bits 5-4 are meaningful (what kind of ACK to use).
1274 		 */
1275 		rx_fh = fc_frame_header_get(rx_fp);
1276 		f_ctl = ntoh24(rx_fh->fh_f_ctl);
1277 		f_ctl &= FC_FC_EX_CTX | FC_FC_SEQ_CTX |
1278 			FC_FC_FIRST_SEQ | FC_FC_LAST_SEQ |
1279 			FC_FC_END_SEQ | FC_FC_END_CONN | FC_FC_SEQ_INIT |
1280 			FC_FC_RETX_SEQ | FC_FC_UNI_TX;
1281 		f_ctl ^= FC_FC_EX_CTX | FC_FC_SEQ_CTX;
1282 		hton24(fh->fh_f_ctl, f_ctl);
1283 
1284 		fc_exch_setup_hdr(ep, fp, f_ctl);
1285 		fh->fh_seq_id = rx_fh->fh_seq_id;
1286 		fh->fh_seq_cnt = rx_fh->fh_seq_cnt;
1287 		fh->fh_parm_offset = htonl(1);	/* ack single frame */
1288 
1289 		fr_sof(fp) = fr_sof(rx_fp);
1290 		if (f_ctl & FC_FC_END_SEQ)
1291 			fr_eof(fp) = FC_EOF_T;
1292 		else
1293 			fr_eof(fp) = FC_EOF_N;
1294 
1295 		lport->tt.frame_send(lport, fp);
1296 	}
1297 }
1298 
1299 /**
1300  * fc_exch_send_ba_rjt() - Send BLS Reject
1301  * @rx_fp:  The frame being rejected
1302  * @reason: The reason the frame is being rejected
1303  * @explan: The explanation for the rejection
1304  *
1305  * This is for rejecting BA_ABTS only.
1306  */
1307 static void fc_exch_send_ba_rjt(struct fc_frame *rx_fp,
1308 				enum fc_ba_rjt_reason reason,
1309 				enum fc_ba_rjt_explan explan)
1310 {
1311 	struct fc_frame *fp;
1312 	struct fc_frame_header *rx_fh;
1313 	struct fc_frame_header *fh;
1314 	struct fc_ba_rjt *rp;
1315 	struct fc_lport *lport;
1316 	unsigned int f_ctl;
1317 
1318 	lport = fr_dev(rx_fp);
1319 	fp = fc_frame_alloc(lport, sizeof(*rp));
1320 	if (!fp)
1321 		return;
1322 	fh = fc_frame_header_get(fp);
1323 	rx_fh = fc_frame_header_get(rx_fp);
1324 
1325 	memset(fh, 0, sizeof(*fh) + sizeof(*rp));
1326 
1327 	rp = fc_frame_payload_get(fp, sizeof(*rp));
1328 	rp->br_reason = reason;
1329 	rp->br_explan = explan;
1330 
1331 	/*
1332 	 * seq_id, cs_ctl, df_ctl and param/offset are zero.
1333 	 */
1334 	memcpy(fh->fh_s_id, rx_fh->fh_d_id, 3);
1335 	memcpy(fh->fh_d_id, rx_fh->fh_s_id, 3);
1336 	fh->fh_ox_id = rx_fh->fh_ox_id;
1337 	fh->fh_rx_id = rx_fh->fh_rx_id;
1338 	fh->fh_seq_cnt = rx_fh->fh_seq_cnt;
1339 	fh->fh_r_ctl = FC_RCTL_BA_RJT;
1340 	fh->fh_type = FC_TYPE_BLS;
1341 
1342 	/*
1343 	 * Form f_ctl by inverting EX_CTX and SEQ_CTX (bits 23, 22).
1344 	 * Echo FIRST_SEQ, LAST_SEQ, END_SEQ, END_CONN, SEQ_INIT.
1345 	 * Bits 9-8 are meaningful (retransmitted or unidirectional).
1346 	 * Last ACK uses bits 7-6 (continue sequence),
1347 	 * bits 5-4 are meaningful (what kind of ACK to use).
1348 	 * Always set LAST_SEQ, END_SEQ.
1349 	 */
1350 	f_ctl = ntoh24(rx_fh->fh_f_ctl);
1351 	f_ctl &= FC_FC_EX_CTX | FC_FC_SEQ_CTX |
1352 		FC_FC_END_CONN | FC_FC_SEQ_INIT |
1353 		FC_FC_RETX_SEQ | FC_FC_UNI_TX;
1354 	f_ctl ^= FC_FC_EX_CTX | FC_FC_SEQ_CTX;
1355 	f_ctl |= FC_FC_LAST_SEQ | FC_FC_END_SEQ;
1356 	f_ctl &= ~FC_FC_FIRST_SEQ;
1357 	hton24(fh->fh_f_ctl, f_ctl);
1358 
1359 	fr_sof(fp) = fc_sof_class(fr_sof(rx_fp));
1360 	fr_eof(fp) = FC_EOF_T;
1361 	if (fc_sof_needs_ack(fr_sof(fp)))
1362 		fr_eof(fp) = FC_EOF_N;
1363 
1364 	lport->tt.frame_send(lport, fp);
1365 }
1366 
1367 /**
1368  * fc_exch_recv_abts() - Handle an incoming ABTS
1369  * @ep:	   The exchange the abort was on
1370  * @rx_fp: The ABTS frame
1371  *
1372  * This would be for target mode usually, but could be due to lost
1373  * FCP transfer ready, confirm or RRQ. We always handle this as an
1374  * exchange abort, ignoring the parameter.
1375  */
1376 static void fc_exch_recv_abts(struct fc_exch *ep, struct fc_frame *rx_fp)
1377 {
1378 	struct fc_frame *fp;
1379 	struct fc_ba_acc *ap;
1380 	struct fc_frame_header *fh;
1381 	struct fc_seq *sp;
1382 
1383 	if (!ep)
1384 		goto reject;
1385 
1386 	fp = fc_frame_alloc(ep->lp, sizeof(*ap));
1387 	if (!fp)
1388 		goto free;
1389 
1390 	spin_lock_bh(&ep->ex_lock);
1391 	if (ep->esb_stat & ESB_ST_COMPLETE) {
1392 		spin_unlock_bh(&ep->ex_lock);
1393 
1394 		fc_frame_free(fp);
1395 		goto reject;
1396 	}
1397 	if (!(ep->esb_stat & ESB_ST_REC_QUAL)) {
1398 		ep->esb_stat |= ESB_ST_REC_QUAL;
1399 		fc_exch_hold(ep);		/* hold for REC_QUAL */
1400 	}
1401 	fc_exch_timer_set_locked(ep, ep->r_a_tov);
1402 	fh = fc_frame_header_get(fp);
1403 	ap = fc_frame_payload_get(fp, sizeof(*ap));
1404 	memset(ap, 0, sizeof(*ap));
1405 	sp = &ep->seq;
1406 	ap->ba_high_seq_cnt = htons(0xffff);
1407 	if (sp->ssb_stat & SSB_ST_RESP) {
1408 		ap->ba_seq_id = sp->id;
1409 		ap->ba_seq_id_val = FC_BA_SEQ_ID_VAL;
1410 		ap->ba_high_seq_cnt = fh->fh_seq_cnt;
1411 		ap->ba_low_seq_cnt = htons(sp->cnt);
1412 	}
1413 	sp = fc_seq_start_next_locked(sp);
1414 	fc_seq_send_last(sp, fp, FC_RCTL_BA_ACC, FC_TYPE_BLS);
1415 	ep->esb_stat |= ESB_ST_ABNORMAL;
1416 	spin_unlock_bh(&ep->ex_lock);
1417 
1418 free:
1419 	fc_frame_free(rx_fp);
1420 	return;
1421 
1422 reject:
1423 	fc_exch_send_ba_rjt(rx_fp, FC_BA_RJT_UNABLE, FC_BA_RJT_INV_XID);
1424 	goto free;
1425 }
1426 
1427 /**
1428  * fc_seq_assign() - Assign exchange and sequence for incoming request
1429  * @lport: The local port that received the request
1430  * @fp:    The request frame
1431  *
1432  * On success, the sequence pointer will be returned and also in fr_seq(@fp).
1433  * A reference will be held on the exchange/sequence for the caller, which
1434  * must call fc_seq_release().
1435  */
1436 static struct fc_seq *fc_seq_assign(struct fc_lport *lport, struct fc_frame *fp)
1437 {
1438 	struct fc_exch_mgr_anchor *ema;
1439 
1440 	WARN_ON(lport != fr_dev(fp));
1441 	WARN_ON(fr_seq(fp));
1442 	fr_seq(fp) = NULL;
1443 
1444 	list_for_each_entry(ema, &lport->ema_list, ema_list)
1445 		if ((!ema->match || ema->match(fp)) &&
1446 		    fc_seq_lookup_recip(lport, ema->mp, fp) == FC_RJT_NONE)
1447 			break;
1448 	return fr_seq(fp);
1449 }
1450 
1451 /**
1452  * fc_seq_release() - Release the hold
1453  * @sp:    The sequence.
1454  */
1455 static void fc_seq_release(struct fc_seq *sp)
1456 {
1457 	fc_exch_release(fc_seq_exch(sp));
1458 }
1459 
1460 /**
1461  * fc_exch_recv_req() - Handler for an incoming request
1462  * @lport: The local port that received the request
1463  * @mp:	   The EM that the exchange is on
1464  * @fp:	   The request frame
1465  *
1466  * This is used when the other end is originating the exchange
1467  * and the sequence.
1468  */
1469 static void fc_exch_recv_req(struct fc_lport *lport, struct fc_exch_mgr *mp,
1470 			     struct fc_frame *fp)
1471 {
1472 	struct fc_frame_header *fh = fc_frame_header_get(fp);
1473 	struct fc_seq *sp = NULL;
1474 	struct fc_exch *ep = NULL;
1475 	enum fc_pf_rjt_reason reject;
1476 
1477 	/* We can have the wrong fc_lport at this point with NPIV, which is a
1478 	 * problem now that we know a new exchange needs to be allocated
1479 	 */
1480 	lport = fc_vport_id_lookup(lport, ntoh24(fh->fh_d_id));
1481 	if (!lport) {
1482 		fc_frame_free(fp);
1483 		return;
1484 	}
1485 	fr_dev(fp) = lport;
1486 
1487 	BUG_ON(fr_seq(fp));		/* XXX remove later */
1488 
1489 	/*
1490 	 * If the RX_ID is 0xffff, don't allocate an exchange.
1491 	 * The upper-level protocol may request one later, if needed.
1492 	 */
1493 	if (fh->fh_rx_id == htons(FC_XID_UNKNOWN))
1494 		return lport->tt.lport_recv(lport, fp);
1495 
1496 	reject = fc_seq_lookup_recip(lport, mp, fp);
1497 	if (reject == FC_RJT_NONE) {
1498 		sp = fr_seq(fp);	/* sequence will be held */
1499 		ep = fc_seq_exch(sp);
1500 		fc_seq_send_ack(sp, fp);
1501 		ep->encaps = fr_encaps(fp);
1502 
1503 		/*
1504 		 * Call the receive function.
1505 		 *
1506 		 * The receive function may allocate a new sequence
1507 		 * over the old one, so we shouldn't change the
1508 		 * sequence after this.
1509 		 *
1510 		 * The frame will be freed by the receive function.
1511 		 * If new exch resp handler is valid then call that
1512 		 * first.
1513 		 */
1514 		if (!fc_invoke_resp(ep, sp, fp))
1515 			lport->tt.lport_recv(lport, fp);
1516 		fc_exch_release(ep);	/* release from lookup */
1517 	} else {
1518 		FC_LPORT_DBG(lport, "exch/seq lookup failed: reject %x\n",
1519 			     reject);
1520 		fc_frame_free(fp);
1521 	}
1522 }
1523 
1524 /**
1525  * fc_exch_recv_seq_resp() - Handler for an incoming response where the other
1526  *			     end is the originator of the sequence that is a
1527  *			     response to our initial exchange
1528  * @mp: The EM that the exchange is on
1529  * @fp: The response frame
1530  */
1531 static void fc_exch_recv_seq_resp(struct fc_exch_mgr *mp, struct fc_frame *fp)
1532 {
1533 	struct fc_frame_header *fh = fc_frame_header_get(fp);
1534 	struct fc_seq *sp;
1535 	struct fc_exch *ep;
1536 	enum fc_sof sof;
1537 	u32 f_ctl;
1538 	int rc;
1539 
1540 	ep = fc_exch_find(mp, ntohs(fh->fh_ox_id));
1541 	if (!ep) {
1542 		atomic_inc(&mp->stats.xid_not_found);
1543 		goto out;
1544 	}
1545 	if (ep->esb_stat & ESB_ST_COMPLETE) {
1546 		atomic_inc(&mp->stats.xid_not_found);
1547 		goto rel;
1548 	}
1549 	if (ep->rxid == FC_XID_UNKNOWN)
1550 		ep->rxid = ntohs(fh->fh_rx_id);
1551 	if (ep->sid != 0 && ep->sid != ntoh24(fh->fh_d_id)) {
1552 		atomic_inc(&mp->stats.xid_not_found);
1553 		goto rel;
1554 	}
1555 	if (ep->did != ntoh24(fh->fh_s_id) &&
1556 	    ep->did != FC_FID_FLOGI) {
1557 		atomic_inc(&mp->stats.xid_not_found);
1558 		goto rel;
1559 	}
1560 	sof = fr_sof(fp);
1561 	sp = &ep->seq;
1562 	if (fc_sof_is_init(sof)) {
1563 		sp->ssb_stat |= SSB_ST_RESP;
1564 		sp->id = fh->fh_seq_id;
1565 	} else if (sp->id != fh->fh_seq_id) {
1566 		atomic_inc(&mp->stats.seq_not_found);
1567 		goto rel;
1568 	}
1569 
1570 	f_ctl = ntoh24(fh->fh_f_ctl);
1571 	fr_seq(fp) = sp;
1572 
1573 	spin_lock_bh(&ep->ex_lock);
1574 	if (f_ctl & FC_FC_SEQ_INIT)
1575 		ep->esb_stat |= ESB_ST_SEQ_INIT;
1576 	spin_unlock_bh(&ep->ex_lock);
1577 
1578 	if (fc_sof_needs_ack(sof))
1579 		fc_seq_send_ack(sp, fp);
1580 
1581 	if (fh->fh_type != FC_TYPE_FCP && fr_eof(fp) == FC_EOF_T &&
1582 	    (f_ctl & (FC_FC_LAST_SEQ | FC_FC_END_SEQ)) ==
1583 	    (FC_FC_LAST_SEQ | FC_FC_END_SEQ)) {
1584 		spin_lock_bh(&ep->ex_lock);
1585 		rc = fc_exch_done_locked(ep);
1586 		WARN_ON(fc_seq_exch(sp) != ep);
1587 		spin_unlock_bh(&ep->ex_lock);
1588 		if (!rc)
1589 			fc_exch_delete(ep);
1590 	}
1591 
1592 	/*
1593 	 * Call the receive function.
1594 	 * The sequence is held (has a refcnt) for us,
1595 	 * but not for the receive function.
1596 	 *
1597 	 * The receive function may allocate a new sequence
1598 	 * over the old one, so we shouldn't change the
1599 	 * sequence after this.
1600 	 *
1601 	 * The frame will be freed by the receive function.
1602 	 * If new exch resp handler is valid then call that
1603 	 * first.
1604 	 */
1605 	if (!fc_invoke_resp(ep, sp, fp))
1606 		fc_frame_free(fp);
1607 
1608 	fc_exch_release(ep);
1609 	return;
1610 rel:
1611 	fc_exch_release(ep);
1612 out:
1613 	fc_frame_free(fp);
1614 }
1615 
1616 /**
1617  * fc_exch_recv_resp() - Handler for a sequence where other end is
1618  *			 responding to our sequence
1619  * @mp: The EM that the exchange is on
1620  * @fp: The response frame
1621  */
1622 static void fc_exch_recv_resp(struct fc_exch_mgr *mp, struct fc_frame *fp)
1623 {
1624 	struct fc_seq *sp;
1625 
1626 	sp = fc_seq_lookup_orig(mp, fp);	/* doesn't hold sequence */
1627 
1628 	if (!sp)
1629 		atomic_inc(&mp->stats.xid_not_found);
1630 	else
1631 		atomic_inc(&mp->stats.non_bls_resp);
1632 
1633 	fc_frame_free(fp);
1634 }
1635 
1636 /**
1637  * fc_exch_abts_resp() - Handler for a response to an ABT
1638  * @ep: The exchange that the frame is on
1639  * @fp: The response frame
1640  *
1641  * This response would be to an ABTS cancelling an exchange or sequence.
1642  * The response can be either BA_ACC or BA_RJT
1643  */
1644 static void fc_exch_abts_resp(struct fc_exch *ep, struct fc_frame *fp)
1645 {
1646 	struct fc_frame_header *fh;
1647 	struct fc_ba_acc *ap;
1648 	struct fc_seq *sp;
1649 	u16 low;
1650 	u16 high;
1651 	int rc = 1, has_rec = 0;
1652 
1653 	fh = fc_frame_header_get(fp);
1654 	FC_EXCH_DBG(ep, "exch: BLS rctl %x - %s\n", fh->fh_r_ctl,
1655 		    fc_exch_rctl_name(fh->fh_r_ctl));
1656 
1657 	if (cancel_delayed_work_sync(&ep->timeout_work)) {
1658 		FC_EXCH_DBG(ep, "Exchange timer canceled due to ABTS response\n");
1659 		fc_exch_release(ep);	/* release from pending timer hold */
1660 	}
1661 
1662 	spin_lock_bh(&ep->ex_lock);
1663 	switch (fh->fh_r_ctl) {
1664 	case FC_RCTL_BA_ACC:
1665 		ap = fc_frame_payload_get(fp, sizeof(*ap));
1666 		if (!ap)
1667 			break;
1668 
1669 		/*
1670 		 * Decide whether to establish a Recovery Qualifier.
1671 		 * We do this if there is a non-empty SEQ_CNT range and
1672 		 * SEQ_ID is the same as the one we aborted.
1673 		 */
1674 		low = ntohs(ap->ba_low_seq_cnt);
1675 		high = ntohs(ap->ba_high_seq_cnt);
1676 		if ((ep->esb_stat & ESB_ST_REC_QUAL) == 0 &&
1677 		    (ap->ba_seq_id_val != FC_BA_SEQ_ID_VAL ||
1678 		     ap->ba_seq_id == ep->seq_id) && low != high) {
1679 			ep->esb_stat |= ESB_ST_REC_QUAL;
1680 			fc_exch_hold(ep);  /* hold for recovery qualifier */
1681 			has_rec = 1;
1682 		}
1683 		break;
1684 	case FC_RCTL_BA_RJT:
1685 		break;
1686 	default:
1687 		break;
1688 	}
1689 
1690 	/* do we need to do some other checks here. Can we reuse more of
1691 	 * fc_exch_recv_seq_resp
1692 	 */
1693 	sp = &ep->seq;
1694 	/*
1695 	 * do we want to check END_SEQ as well as LAST_SEQ here?
1696 	 */
1697 	if (ep->fh_type != FC_TYPE_FCP &&
1698 	    ntoh24(fh->fh_f_ctl) & FC_FC_LAST_SEQ)
1699 		rc = fc_exch_done_locked(ep);
1700 	spin_unlock_bh(&ep->ex_lock);
1701 
1702 	fc_exch_hold(ep);
1703 	if (!rc)
1704 		fc_exch_delete(ep);
1705 	if (!fc_invoke_resp(ep, sp, fp))
1706 		fc_frame_free(fp);
1707 	if (has_rec)
1708 		fc_exch_timer_set(ep, ep->r_a_tov);
1709 	fc_exch_release(ep);
1710 }
1711 
1712 /**
1713  * fc_exch_recv_bls() - Handler for a BLS sequence
1714  * @mp: The EM that the exchange is on
1715  * @fp: The request frame
1716  *
1717  * The BLS frame is always a sequence initiated by the remote side.
1718  * We may be either the originator or recipient of the exchange.
1719  */
1720 static void fc_exch_recv_bls(struct fc_exch_mgr *mp, struct fc_frame *fp)
1721 {
1722 	struct fc_frame_header *fh;
1723 	struct fc_exch *ep;
1724 	u32 f_ctl;
1725 
1726 	fh = fc_frame_header_get(fp);
1727 	f_ctl = ntoh24(fh->fh_f_ctl);
1728 	fr_seq(fp) = NULL;
1729 
1730 	ep = fc_exch_find(mp, (f_ctl & FC_FC_EX_CTX) ?
1731 			  ntohs(fh->fh_ox_id) : ntohs(fh->fh_rx_id));
1732 	if (ep && (f_ctl & FC_FC_SEQ_INIT)) {
1733 		spin_lock_bh(&ep->ex_lock);
1734 		ep->esb_stat |= ESB_ST_SEQ_INIT;
1735 		spin_unlock_bh(&ep->ex_lock);
1736 	}
1737 	if (f_ctl & FC_FC_SEQ_CTX) {
1738 		/*
1739 		 * A response to a sequence we initiated.
1740 		 * This should only be ACKs for class 2 or F.
1741 		 */
1742 		switch (fh->fh_r_ctl) {
1743 		case FC_RCTL_ACK_1:
1744 		case FC_RCTL_ACK_0:
1745 			break;
1746 		default:
1747 			if (ep)
1748 				FC_EXCH_DBG(ep, "BLS rctl %x - %s received\n",
1749 					    fh->fh_r_ctl,
1750 					    fc_exch_rctl_name(fh->fh_r_ctl));
1751 			break;
1752 		}
1753 		fc_frame_free(fp);
1754 	} else {
1755 		switch (fh->fh_r_ctl) {
1756 		case FC_RCTL_BA_RJT:
1757 		case FC_RCTL_BA_ACC:
1758 			if (ep)
1759 				fc_exch_abts_resp(ep, fp);
1760 			else
1761 				fc_frame_free(fp);
1762 			break;
1763 		case FC_RCTL_BA_ABTS:
1764 			fc_exch_recv_abts(ep, fp);
1765 			break;
1766 		default:			/* ignore junk */
1767 			fc_frame_free(fp);
1768 			break;
1769 		}
1770 	}
1771 	if (ep)
1772 		fc_exch_release(ep);	/* release hold taken by fc_exch_find */
1773 }
1774 
1775 /**
1776  * fc_seq_ls_acc() - Accept sequence with LS_ACC
1777  * @rx_fp: The received frame, not freed here.
1778  *
1779  * If this fails due to allocation or transmit congestion, assume the
1780  * originator will repeat the sequence.
1781  */
1782 static void fc_seq_ls_acc(struct fc_frame *rx_fp)
1783 {
1784 	struct fc_lport *lport;
1785 	struct fc_els_ls_acc *acc;
1786 	struct fc_frame *fp;
1787 
1788 	lport = fr_dev(rx_fp);
1789 	fp = fc_frame_alloc(lport, sizeof(*acc));
1790 	if (!fp)
1791 		return;
1792 	acc = fc_frame_payload_get(fp, sizeof(*acc));
1793 	memset(acc, 0, sizeof(*acc));
1794 	acc->la_cmd = ELS_LS_ACC;
1795 	fc_fill_reply_hdr(fp, rx_fp, FC_RCTL_ELS_REP, 0);
1796 	lport->tt.frame_send(lport, fp);
1797 }
1798 
1799 /**
1800  * fc_seq_ls_rjt() - Reject a sequence with ELS LS_RJT
1801  * @rx_fp: The received frame, not freed here.
1802  * @reason: The reason the sequence is being rejected
1803  * @explan: The explanation for the rejection
1804  *
1805  * If this fails due to allocation or transmit congestion, assume the
1806  * originator will repeat the sequence.
1807  */
1808 static void fc_seq_ls_rjt(struct fc_frame *rx_fp, enum fc_els_rjt_reason reason,
1809 			  enum fc_els_rjt_explan explan)
1810 {
1811 	struct fc_lport *lport;
1812 	struct fc_els_ls_rjt *rjt;
1813 	struct fc_frame *fp;
1814 
1815 	lport = fr_dev(rx_fp);
1816 	fp = fc_frame_alloc(lport, sizeof(*rjt));
1817 	if (!fp)
1818 		return;
1819 	rjt = fc_frame_payload_get(fp, sizeof(*rjt));
1820 	memset(rjt, 0, sizeof(*rjt));
1821 	rjt->er_cmd = ELS_LS_RJT;
1822 	rjt->er_reason = reason;
1823 	rjt->er_explan = explan;
1824 	fc_fill_reply_hdr(fp, rx_fp, FC_RCTL_ELS_REP, 0);
1825 	lport->tt.frame_send(lport, fp);
1826 }
1827 
1828 /**
1829  * fc_exch_reset() - Reset an exchange
1830  * @ep: The exchange to be reset
1831  *
1832  * Note: May sleep if invoked from outside a response handler.
1833  */
1834 static void fc_exch_reset(struct fc_exch *ep)
1835 {
1836 	struct fc_seq *sp;
1837 	int rc = 1;
1838 
1839 	spin_lock_bh(&ep->ex_lock);
1840 	fc_exch_abort_locked(ep, 0);
1841 	ep->state |= FC_EX_RST_CLEANUP;
1842 	fc_exch_timer_cancel(ep);
1843 	if (ep->esb_stat & ESB_ST_REC_QUAL)
1844 		atomic_dec(&ep->ex_refcnt);	/* drop hold for rec_qual */
1845 	ep->esb_stat &= ~ESB_ST_REC_QUAL;
1846 	sp = &ep->seq;
1847 	rc = fc_exch_done_locked(ep);
1848 	spin_unlock_bh(&ep->ex_lock);
1849 
1850 	fc_exch_hold(ep);
1851 
1852 	if (!rc)
1853 		fc_exch_delete(ep);
1854 
1855 	fc_invoke_resp(ep, sp, ERR_PTR(-FC_EX_CLOSED));
1856 	fc_seq_set_resp(sp, NULL, ep->arg);
1857 	fc_exch_release(ep);
1858 }
1859 
1860 /**
1861  * fc_exch_pool_reset() - Reset a per cpu exchange pool
1862  * @lport: The local port that the exchange pool is on
1863  * @pool:  The exchange pool to be reset
1864  * @sid:   The source ID
1865  * @did:   The destination ID
1866  *
1867  * Resets a per cpu exches pool, releasing all of its sequences
1868  * and exchanges. If sid is non-zero then reset only exchanges
1869  * we sourced from the local port's FID. If did is non-zero then
1870  * only reset exchanges destined for the local port's FID.
1871  */
1872 static void fc_exch_pool_reset(struct fc_lport *lport,
1873 			       struct fc_exch_pool *pool,
1874 			       u32 sid, u32 did)
1875 {
1876 	struct fc_exch *ep;
1877 	struct fc_exch *next;
1878 
1879 	spin_lock_bh(&pool->lock);
1880 restart:
1881 	list_for_each_entry_safe(ep, next, &pool->ex_list, ex_list) {
1882 		if ((lport == ep->lp) &&
1883 		    (sid == 0 || sid == ep->sid) &&
1884 		    (did == 0 || did == ep->did)) {
1885 			fc_exch_hold(ep);
1886 			spin_unlock_bh(&pool->lock);
1887 
1888 			fc_exch_reset(ep);
1889 
1890 			fc_exch_release(ep);
1891 			spin_lock_bh(&pool->lock);
1892 
1893 			/*
1894 			 * must restart loop incase while lock
1895 			 * was down multiple eps were released.
1896 			 */
1897 			goto restart;
1898 		}
1899 	}
1900 	pool->next_index = 0;
1901 	pool->left = FC_XID_UNKNOWN;
1902 	pool->right = FC_XID_UNKNOWN;
1903 	spin_unlock_bh(&pool->lock);
1904 }
1905 
1906 /**
1907  * fc_exch_mgr_reset() - Reset all EMs of a local port
1908  * @lport: The local port whose EMs are to be reset
1909  * @sid:   The source ID
1910  * @did:   The destination ID
1911  *
1912  * Reset all EMs associated with a given local port. Release all
1913  * sequences and exchanges. If sid is non-zero then reset only the
1914  * exchanges sent from the local port's FID. If did is non-zero then
1915  * reset only exchanges destined for the local port's FID.
1916  */
1917 void fc_exch_mgr_reset(struct fc_lport *lport, u32 sid, u32 did)
1918 {
1919 	struct fc_exch_mgr_anchor *ema;
1920 	unsigned int cpu;
1921 
1922 	list_for_each_entry(ema, &lport->ema_list, ema_list) {
1923 		for_each_possible_cpu(cpu)
1924 			fc_exch_pool_reset(lport,
1925 					   per_cpu_ptr(ema->mp->pool, cpu),
1926 					   sid, did);
1927 	}
1928 }
1929 EXPORT_SYMBOL(fc_exch_mgr_reset);
1930 
1931 /**
1932  * fc_exch_lookup() - find an exchange
1933  * @lport: The local port
1934  * @xid: The exchange ID
1935  *
1936  * Returns exchange pointer with hold for caller, or NULL if not found.
1937  */
1938 static struct fc_exch *fc_exch_lookup(struct fc_lport *lport, u32 xid)
1939 {
1940 	struct fc_exch_mgr_anchor *ema;
1941 
1942 	list_for_each_entry(ema, &lport->ema_list, ema_list)
1943 		if (ema->mp->min_xid <= xid && xid <= ema->mp->max_xid)
1944 			return fc_exch_find(ema->mp, xid);
1945 	return NULL;
1946 }
1947 
1948 /**
1949  * fc_exch_els_rec() - Handler for ELS REC (Read Exchange Concise) requests
1950  * @rfp: The REC frame, not freed here.
1951  *
1952  * Note that the requesting port may be different than the S_ID in the request.
1953  */
1954 static void fc_exch_els_rec(struct fc_frame *rfp)
1955 {
1956 	struct fc_lport *lport;
1957 	struct fc_frame *fp;
1958 	struct fc_exch *ep;
1959 	struct fc_els_rec *rp;
1960 	struct fc_els_rec_acc *acc;
1961 	enum fc_els_rjt_reason reason = ELS_RJT_LOGIC;
1962 	enum fc_els_rjt_explan explan;
1963 	u32 sid;
1964 	u16 rxid;
1965 	u16 oxid;
1966 
1967 	lport = fr_dev(rfp);
1968 	rp = fc_frame_payload_get(rfp, sizeof(*rp));
1969 	explan = ELS_EXPL_INV_LEN;
1970 	if (!rp)
1971 		goto reject;
1972 	sid = ntoh24(rp->rec_s_id);
1973 	rxid = ntohs(rp->rec_rx_id);
1974 	oxid = ntohs(rp->rec_ox_id);
1975 
1976 	ep = fc_exch_lookup(lport,
1977 			    sid == fc_host_port_id(lport->host) ? oxid : rxid);
1978 	explan = ELS_EXPL_OXID_RXID;
1979 	if (!ep)
1980 		goto reject;
1981 	if (ep->oid != sid || oxid != ep->oxid)
1982 		goto rel;
1983 	if (rxid != FC_XID_UNKNOWN && rxid != ep->rxid)
1984 		goto rel;
1985 	fp = fc_frame_alloc(lport, sizeof(*acc));
1986 	if (!fp)
1987 		goto out;
1988 
1989 	acc = fc_frame_payload_get(fp, sizeof(*acc));
1990 	memset(acc, 0, sizeof(*acc));
1991 	acc->reca_cmd = ELS_LS_ACC;
1992 	acc->reca_ox_id = rp->rec_ox_id;
1993 	memcpy(acc->reca_ofid, rp->rec_s_id, 3);
1994 	acc->reca_rx_id = htons(ep->rxid);
1995 	if (ep->sid == ep->oid)
1996 		hton24(acc->reca_rfid, ep->did);
1997 	else
1998 		hton24(acc->reca_rfid, ep->sid);
1999 	acc->reca_fc4value = htonl(ep->seq.rec_data);
2000 	acc->reca_e_stat = htonl(ep->esb_stat & (ESB_ST_RESP |
2001 						 ESB_ST_SEQ_INIT |
2002 						 ESB_ST_COMPLETE));
2003 	fc_fill_reply_hdr(fp, rfp, FC_RCTL_ELS_REP, 0);
2004 	lport->tt.frame_send(lport, fp);
2005 out:
2006 	fc_exch_release(ep);
2007 	return;
2008 
2009 rel:
2010 	fc_exch_release(ep);
2011 reject:
2012 	fc_seq_ls_rjt(rfp, reason, explan);
2013 }
2014 
2015 /**
2016  * fc_exch_rrq_resp() - Handler for RRQ responses
2017  * @sp:	 The sequence that the RRQ is on
2018  * @fp:	 The RRQ frame
2019  * @arg: The exchange that the RRQ is on
2020  *
2021  * TODO: fix error handler.
2022  */
2023 static void fc_exch_rrq_resp(struct fc_seq *sp, struct fc_frame *fp, void *arg)
2024 {
2025 	struct fc_exch *aborted_ep = arg;
2026 	unsigned int op;
2027 
2028 	if (IS_ERR(fp)) {
2029 		int err = PTR_ERR(fp);
2030 
2031 		if (err == -FC_EX_CLOSED || err == -FC_EX_TIMEOUT)
2032 			goto cleanup;
2033 		FC_EXCH_DBG(aborted_ep, "Cannot process RRQ, "
2034 			    "frame error %d\n", err);
2035 		return;
2036 	}
2037 
2038 	op = fc_frame_payload_op(fp);
2039 	fc_frame_free(fp);
2040 
2041 	switch (op) {
2042 	case ELS_LS_RJT:
2043 		FC_EXCH_DBG(aborted_ep, "LS_RJT for RRQ\n");
2044 		/* fall through */
2045 	case ELS_LS_ACC:
2046 		goto cleanup;
2047 	default:
2048 		FC_EXCH_DBG(aborted_ep, "unexpected response op %x for RRQ\n",
2049 			    op);
2050 		return;
2051 	}
2052 
2053 cleanup:
2054 	fc_exch_done(&aborted_ep->seq);
2055 	/* drop hold for rec qual */
2056 	fc_exch_release(aborted_ep);
2057 }
2058 
2059 
2060 /**
2061  * fc_exch_seq_send() - Send a frame using a new exchange and sequence
2062  * @lport:	The local port to send the frame on
2063  * @fp:		The frame to be sent
2064  * @resp:	The response handler for this request
2065  * @destructor: The destructor for the exchange
2066  * @arg:	The argument to be passed to the response handler
2067  * @timer_msec: The timeout period for the exchange
2068  *
2069  * The frame pointer with some of the header's fields must be
2070  * filled before calling this routine, those fields are:
2071  *
2072  * - routing control
2073  * - FC port did
2074  * - FC port sid
2075  * - FC header type
2076  * - frame control
2077  * - parameter or relative offset
2078  */
2079 static struct fc_seq *fc_exch_seq_send(struct fc_lport *lport,
2080 				       struct fc_frame *fp,
2081 				       void (*resp)(struct fc_seq *,
2082 						    struct fc_frame *fp,
2083 						    void *arg),
2084 				       void (*destructor)(struct fc_seq *,
2085 							  void *),
2086 				       void *arg, u32 timer_msec)
2087 {
2088 	struct fc_exch *ep;
2089 	struct fc_seq *sp = NULL;
2090 	struct fc_frame_header *fh;
2091 	struct fc_fcp_pkt *fsp = NULL;
2092 	int rc = 1;
2093 
2094 	ep = fc_exch_alloc(lport, fp);
2095 	if (!ep) {
2096 		fc_frame_free(fp);
2097 		return NULL;
2098 	}
2099 	ep->esb_stat |= ESB_ST_SEQ_INIT;
2100 	fh = fc_frame_header_get(fp);
2101 	fc_exch_set_addr(ep, ntoh24(fh->fh_s_id), ntoh24(fh->fh_d_id));
2102 	ep->resp = resp;
2103 	ep->destructor = destructor;
2104 	ep->arg = arg;
2105 	ep->r_a_tov = FC_DEF_R_A_TOV;
2106 	ep->lp = lport;
2107 	sp = &ep->seq;
2108 
2109 	ep->fh_type = fh->fh_type; /* save for possbile timeout handling */
2110 	ep->f_ctl = ntoh24(fh->fh_f_ctl);
2111 	fc_exch_setup_hdr(ep, fp, ep->f_ctl);
2112 	sp->cnt++;
2113 
2114 	if (ep->xid <= lport->lro_xid && fh->fh_r_ctl == FC_RCTL_DD_UNSOL_CMD) {
2115 		fsp = fr_fsp(fp);
2116 		fc_fcp_ddp_setup(fr_fsp(fp), ep->xid);
2117 	}
2118 
2119 	if (unlikely(lport->tt.frame_send(lport, fp)))
2120 		goto err;
2121 
2122 	if (timer_msec)
2123 		fc_exch_timer_set_locked(ep, timer_msec);
2124 	ep->f_ctl &= ~FC_FC_FIRST_SEQ;	/* not first seq */
2125 
2126 	if (ep->f_ctl & FC_FC_SEQ_INIT)
2127 		ep->esb_stat &= ~ESB_ST_SEQ_INIT;
2128 	spin_unlock_bh(&ep->ex_lock);
2129 	return sp;
2130 err:
2131 	if (fsp)
2132 		fc_fcp_ddp_done(fsp);
2133 	rc = fc_exch_done_locked(ep);
2134 	spin_unlock_bh(&ep->ex_lock);
2135 	if (!rc)
2136 		fc_exch_delete(ep);
2137 	return NULL;
2138 }
2139 
2140 /**
2141  * fc_exch_rrq() - Send an ELS RRQ (Reinstate Recovery Qualifier) command
2142  * @ep: The exchange to send the RRQ on
2143  *
2144  * This tells the remote port to stop blocking the use of
2145  * the exchange and the seq_cnt range.
2146  */
2147 static void fc_exch_rrq(struct fc_exch *ep)
2148 {
2149 	struct fc_lport *lport;
2150 	struct fc_els_rrq *rrq;
2151 	struct fc_frame *fp;
2152 	u32 did;
2153 
2154 	lport = ep->lp;
2155 
2156 	fp = fc_frame_alloc(lport, sizeof(*rrq));
2157 	if (!fp)
2158 		goto retry;
2159 
2160 	rrq = fc_frame_payload_get(fp, sizeof(*rrq));
2161 	memset(rrq, 0, sizeof(*rrq));
2162 	rrq->rrq_cmd = ELS_RRQ;
2163 	hton24(rrq->rrq_s_id, ep->sid);
2164 	rrq->rrq_ox_id = htons(ep->oxid);
2165 	rrq->rrq_rx_id = htons(ep->rxid);
2166 
2167 	did = ep->did;
2168 	if (ep->esb_stat & ESB_ST_RESP)
2169 		did = ep->sid;
2170 
2171 	fc_fill_fc_hdr(fp, FC_RCTL_ELS_REQ, did,
2172 		       lport->port_id, FC_TYPE_ELS,
2173 		       FC_FC_FIRST_SEQ | FC_FC_END_SEQ | FC_FC_SEQ_INIT, 0);
2174 
2175 	if (fc_exch_seq_send(lport, fp, fc_exch_rrq_resp, NULL, ep,
2176 			     lport->e_d_tov))
2177 		return;
2178 
2179 retry:
2180 	spin_lock_bh(&ep->ex_lock);
2181 	if (ep->state & (FC_EX_RST_CLEANUP | FC_EX_DONE)) {
2182 		spin_unlock_bh(&ep->ex_lock);
2183 		/* drop hold for rec qual */
2184 		fc_exch_release(ep);
2185 		return;
2186 	}
2187 	ep->esb_stat |= ESB_ST_REC_QUAL;
2188 	fc_exch_timer_set_locked(ep, ep->r_a_tov);
2189 	spin_unlock_bh(&ep->ex_lock);
2190 }
2191 
2192 /**
2193  * fc_exch_els_rrq() - Handler for ELS RRQ (Reset Recovery Qualifier) requests
2194  * @fp: The RRQ frame, not freed here.
2195  */
2196 static void fc_exch_els_rrq(struct fc_frame *fp)
2197 {
2198 	struct fc_lport *lport;
2199 	struct fc_exch *ep = NULL;	/* request or subject exchange */
2200 	struct fc_els_rrq *rp;
2201 	u32 sid;
2202 	u16 xid;
2203 	enum fc_els_rjt_explan explan;
2204 
2205 	lport = fr_dev(fp);
2206 	rp = fc_frame_payload_get(fp, sizeof(*rp));
2207 	explan = ELS_EXPL_INV_LEN;
2208 	if (!rp)
2209 		goto reject;
2210 
2211 	/*
2212 	 * lookup subject exchange.
2213 	 */
2214 	sid = ntoh24(rp->rrq_s_id);		/* subject source */
2215 	xid = fc_host_port_id(lport->host) == sid ?
2216 			ntohs(rp->rrq_ox_id) : ntohs(rp->rrq_rx_id);
2217 	ep = fc_exch_lookup(lport, xid);
2218 	explan = ELS_EXPL_OXID_RXID;
2219 	if (!ep)
2220 		goto reject;
2221 	spin_lock_bh(&ep->ex_lock);
2222 	if (ep->oxid != ntohs(rp->rrq_ox_id))
2223 		goto unlock_reject;
2224 	if (ep->rxid != ntohs(rp->rrq_rx_id) &&
2225 	    ep->rxid != FC_XID_UNKNOWN)
2226 		goto unlock_reject;
2227 	explan = ELS_EXPL_SID;
2228 	if (ep->sid != sid)
2229 		goto unlock_reject;
2230 
2231 	/*
2232 	 * Clear Recovery Qualifier state, and cancel timer if complete.
2233 	 */
2234 	if (ep->esb_stat & ESB_ST_REC_QUAL) {
2235 		ep->esb_stat &= ~ESB_ST_REC_QUAL;
2236 		atomic_dec(&ep->ex_refcnt);	/* drop hold for rec qual */
2237 	}
2238 	if (ep->esb_stat & ESB_ST_COMPLETE)
2239 		fc_exch_timer_cancel(ep);
2240 
2241 	spin_unlock_bh(&ep->ex_lock);
2242 
2243 	/*
2244 	 * Send LS_ACC.
2245 	 */
2246 	fc_seq_ls_acc(fp);
2247 	goto out;
2248 
2249 unlock_reject:
2250 	spin_unlock_bh(&ep->ex_lock);
2251 reject:
2252 	fc_seq_ls_rjt(fp, ELS_RJT_LOGIC, explan);
2253 out:
2254 	if (ep)
2255 		fc_exch_release(ep);	/* drop hold from fc_exch_find */
2256 }
2257 
2258 /**
2259  * fc_exch_update_stats() - update exches stats to lport
2260  * @lport: The local port to update exchange manager stats
2261  */
2262 void fc_exch_update_stats(struct fc_lport *lport)
2263 {
2264 	struct fc_host_statistics *st;
2265 	struct fc_exch_mgr_anchor *ema;
2266 	struct fc_exch_mgr *mp;
2267 
2268 	st = &lport->host_stats;
2269 
2270 	list_for_each_entry(ema, &lport->ema_list, ema_list) {
2271 		mp = ema->mp;
2272 		st->fc_no_free_exch += atomic_read(&mp->stats.no_free_exch);
2273 		st->fc_no_free_exch_xid +=
2274 				atomic_read(&mp->stats.no_free_exch_xid);
2275 		st->fc_xid_not_found += atomic_read(&mp->stats.xid_not_found);
2276 		st->fc_xid_busy += atomic_read(&mp->stats.xid_busy);
2277 		st->fc_seq_not_found += atomic_read(&mp->stats.seq_not_found);
2278 		st->fc_non_bls_resp += atomic_read(&mp->stats.non_bls_resp);
2279 	}
2280 }
2281 EXPORT_SYMBOL(fc_exch_update_stats);
2282 
2283 /**
2284  * fc_exch_mgr_add() - Add an exchange manager to a local port's list of EMs
2285  * @lport: The local port to add the exchange manager to
2286  * @mp:	   The exchange manager to be added to the local port
2287  * @match: The match routine that indicates when this EM should be used
2288  */
2289 struct fc_exch_mgr_anchor *fc_exch_mgr_add(struct fc_lport *lport,
2290 					   struct fc_exch_mgr *mp,
2291 					   bool (*match)(struct fc_frame *))
2292 {
2293 	struct fc_exch_mgr_anchor *ema;
2294 
2295 	ema = kmalloc(sizeof(*ema), GFP_ATOMIC);
2296 	if (!ema)
2297 		return ema;
2298 
2299 	ema->mp = mp;
2300 	ema->match = match;
2301 	/* add EM anchor to EM anchors list */
2302 	list_add_tail(&ema->ema_list, &lport->ema_list);
2303 	kref_get(&mp->kref);
2304 	return ema;
2305 }
2306 EXPORT_SYMBOL(fc_exch_mgr_add);
2307 
2308 /**
2309  * fc_exch_mgr_destroy() - Destroy an exchange manager
2310  * @kref: The reference to the EM to be destroyed
2311  */
2312 static void fc_exch_mgr_destroy(struct kref *kref)
2313 {
2314 	struct fc_exch_mgr *mp = container_of(kref, struct fc_exch_mgr, kref);
2315 
2316 	mempool_destroy(mp->ep_pool);
2317 	free_percpu(mp->pool);
2318 	kfree(mp);
2319 }
2320 
2321 /**
2322  * fc_exch_mgr_del() - Delete an EM from a local port's list
2323  * @ema: The exchange manager anchor identifying the EM to be deleted
2324  */
2325 void fc_exch_mgr_del(struct fc_exch_mgr_anchor *ema)
2326 {
2327 	/* remove EM anchor from EM anchors list */
2328 	list_del(&ema->ema_list);
2329 	kref_put(&ema->mp->kref, fc_exch_mgr_destroy);
2330 	kfree(ema);
2331 }
2332 EXPORT_SYMBOL(fc_exch_mgr_del);
2333 
2334 /**
2335  * fc_exch_mgr_list_clone() - Share all exchange manager objects
2336  * @src: Source lport to clone exchange managers from
2337  * @dst: New lport that takes references to all the exchange managers
2338  */
2339 int fc_exch_mgr_list_clone(struct fc_lport *src, struct fc_lport *dst)
2340 {
2341 	struct fc_exch_mgr_anchor *ema, *tmp;
2342 
2343 	list_for_each_entry(ema, &src->ema_list, ema_list) {
2344 		if (!fc_exch_mgr_add(dst, ema->mp, ema->match))
2345 			goto err;
2346 	}
2347 	return 0;
2348 err:
2349 	list_for_each_entry_safe(ema, tmp, &dst->ema_list, ema_list)
2350 		fc_exch_mgr_del(ema);
2351 	return -ENOMEM;
2352 }
2353 EXPORT_SYMBOL(fc_exch_mgr_list_clone);
2354 
2355 /**
2356  * fc_exch_mgr_alloc() - Allocate an exchange manager
2357  * @lport:   The local port that the new EM will be associated with
2358  * @class:   The default FC class for new exchanges
2359  * @min_xid: The minimum XID for exchanges from the new EM
2360  * @max_xid: The maximum XID for exchanges from the new EM
2361  * @match:   The match routine for the new EM
2362  */
2363 struct fc_exch_mgr *fc_exch_mgr_alloc(struct fc_lport *lport,
2364 				      enum fc_class class,
2365 				      u16 min_xid, u16 max_xid,
2366 				      bool (*match)(struct fc_frame *))
2367 {
2368 	struct fc_exch_mgr *mp;
2369 	u16 pool_exch_range;
2370 	size_t pool_size;
2371 	unsigned int cpu;
2372 	struct fc_exch_pool *pool;
2373 
2374 	if (max_xid <= min_xid || max_xid == FC_XID_UNKNOWN ||
2375 	    (min_xid & fc_cpu_mask) != 0) {
2376 		FC_LPORT_DBG(lport, "Invalid min_xid 0x:%x and max_xid 0x:%x\n",
2377 			     min_xid, max_xid);
2378 		return NULL;
2379 	}
2380 
2381 	/*
2382 	 * allocate memory for EM
2383 	 */
2384 	mp = kzalloc(sizeof(struct fc_exch_mgr), GFP_ATOMIC);
2385 	if (!mp)
2386 		return NULL;
2387 
2388 	mp->class = class;
2389 	/* adjust em exch xid range for offload */
2390 	mp->min_xid = min_xid;
2391 
2392        /* reduce range so per cpu pool fits into PCPU_MIN_UNIT_SIZE pool */
2393 	pool_exch_range = (PCPU_MIN_UNIT_SIZE - sizeof(*pool)) /
2394 		sizeof(struct fc_exch *);
2395 	if ((max_xid - min_xid + 1) / (fc_cpu_mask + 1) > pool_exch_range) {
2396 		mp->max_xid = pool_exch_range * (fc_cpu_mask + 1) +
2397 			min_xid - 1;
2398 	} else {
2399 		mp->max_xid = max_xid;
2400 		pool_exch_range = (mp->max_xid - mp->min_xid + 1) /
2401 			(fc_cpu_mask + 1);
2402 	}
2403 
2404 	mp->ep_pool = mempool_create_slab_pool(2, fc_em_cachep);
2405 	if (!mp->ep_pool)
2406 		goto free_mp;
2407 
2408 	/*
2409 	 * Setup per cpu exch pool with entire exchange id range equally
2410 	 * divided across all cpus. The exch pointers array memory is
2411 	 * allocated for exch range per pool.
2412 	 */
2413 	mp->pool_max_index = pool_exch_range - 1;
2414 
2415 	/*
2416 	 * Allocate and initialize per cpu exch pool
2417 	 */
2418 	pool_size = sizeof(*pool) + pool_exch_range * sizeof(struct fc_exch *);
2419 	mp->pool = __alloc_percpu(pool_size, __alignof__(struct fc_exch_pool));
2420 	if (!mp->pool)
2421 		goto free_mempool;
2422 	for_each_possible_cpu(cpu) {
2423 		pool = per_cpu_ptr(mp->pool, cpu);
2424 		pool->next_index = 0;
2425 		pool->left = FC_XID_UNKNOWN;
2426 		pool->right = FC_XID_UNKNOWN;
2427 		spin_lock_init(&pool->lock);
2428 		INIT_LIST_HEAD(&pool->ex_list);
2429 	}
2430 
2431 	kref_init(&mp->kref);
2432 	if (!fc_exch_mgr_add(lport, mp, match)) {
2433 		free_percpu(mp->pool);
2434 		goto free_mempool;
2435 	}
2436 
2437 	/*
2438 	 * Above kref_init() sets mp->kref to 1 and then
2439 	 * call to fc_exch_mgr_add incremented mp->kref again,
2440 	 * so adjust that extra increment.
2441 	 */
2442 	kref_put(&mp->kref, fc_exch_mgr_destroy);
2443 	return mp;
2444 
2445 free_mempool:
2446 	mempool_destroy(mp->ep_pool);
2447 free_mp:
2448 	kfree(mp);
2449 	return NULL;
2450 }
2451 EXPORT_SYMBOL(fc_exch_mgr_alloc);
2452 
2453 /**
2454  * fc_exch_mgr_free() - Free all exchange managers on a local port
2455  * @lport: The local port whose EMs are to be freed
2456  */
2457 void fc_exch_mgr_free(struct fc_lport *lport)
2458 {
2459 	struct fc_exch_mgr_anchor *ema, *next;
2460 
2461 	flush_workqueue(fc_exch_workqueue);
2462 	list_for_each_entry_safe(ema, next, &lport->ema_list, ema_list)
2463 		fc_exch_mgr_del(ema);
2464 }
2465 EXPORT_SYMBOL(fc_exch_mgr_free);
2466 
2467 /**
2468  * fc_find_ema() - Lookup and return appropriate Exchange Manager Anchor depending
2469  * upon 'xid'.
2470  * @f_ctl: f_ctl
2471  * @lport: The local port the frame was received on
2472  * @fh: The received frame header
2473  */
2474 static struct fc_exch_mgr_anchor *fc_find_ema(u32 f_ctl,
2475 					      struct fc_lport *lport,
2476 					      struct fc_frame_header *fh)
2477 {
2478 	struct fc_exch_mgr_anchor *ema;
2479 	u16 xid;
2480 
2481 	if (f_ctl & FC_FC_EX_CTX)
2482 		xid = ntohs(fh->fh_ox_id);
2483 	else {
2484 		xid = ntohs(fh->fh_rx_id);
2485 		if (xid == FC_XID_UNKNOWN)
2486 			return list_entry(lport->ema_list.prev,
2487 					  typeof(*ema), ema_list);
2488 	}
2489 
2490 	list_for_each_entry(ema, &lport->ema_list, ema_list) {
2491 		if ((xid >= ema->mp->min_xid) &&
2492 		    (xid <= ema->mp->max_xid))
2493 			return ema;
2494 	}
2495 	return NULL;
2496 }
2497 /**
2498  * fc_exch_recv() - Handler for received frames
2499  * @lport: The local port the frame was received on
2500  * @fp:	The received frame
2501  */
2502 void fc_exch_recv(struct fc_lport *lport, struct fc_frame *fp)
2503 {
2504 	struct fc_frame_header *fh = fc_frame_header_get(fp);
2505 	struct fc_exch_mgr_anchor *ema;
2506 	u32 f_ctl;
2507 
2508 	/* lport lock ? */
2509 	if (!lport || lport->state == LPORT_ST_DISABLED) {
2510 		FC_LPORT_DBG(lport, "Receiving frames for an lport that "
2511 			     "has not been initialized correctly\n");
2512 		fc_frame_free(fp);
2513 		return;
2514 	}
2515 
2516 	f_ctl = ntoh24(fh->fh_f_ctl);
2517 	ema = fc_find_ema(f_ctl, lport, fh);
2518 	if (!ema) {
2519 		FC_LPORT_DBG(lport, "Unable to find Exchange Manager Anchor,"
2520 				    "fc_ctl <0x%x>, xid <0x%x>\n",
2521 				     f_ctl,
2522 				     (f_ctl & FC_FC_EX_CTX) ?
2523 				     ntohs(fh->fh_ox_id) :
2524 				     ntohs(fh->fh_rx_id));
2525 		fc_frame_free(fp);
2526 		return;
2527 	}
2528 
2529 	/*
2530 	 * If frame is marked invalid, just drop it.
2531 	 */
2532 	switch (fr_eof(fp)) {
2533 	case FC_EOF_T:
2534 		if (f_ctl & FC_FC_END_SEQ)
2535 			skb_trim(fp_skb(fp), fr_len(fp) - FC_FC_FILL(f_ctl));
2536 		/* fall through */
2537 	case FC_EOF_N:
2538 		if (fh->fh_type == FC_TYPE_BLS)
2539 			fc_exch_recv_bls(ema->mp, fp);
2540 		else if ((f_ctl & (FC_FC_EX_CTX | FC_FC_SEQ_CTX)) ==
2541 			 FC_FC_EX_CTX)
2542 			fc_exch_recv_seq_resp(ema->mp, fp);
2543 		else if (f_ctl & FC_FC_SEQ_CTX)
2544 			fc_exch_recv_resp(ema->mp, fp);
2545 		else	/* no EX_CTX and no SEQ_CTX */
2546 			fc_exch_recv_req(lport, ema->mp, fp);
2547 		break;
2548 	default:
2549 		FC_LPORT_DBG(lport, "dropping invalid frame (eof %x)",
2550 			     fr_eof(fp));
2551 		fc_frame_free(fp);
2552 	}
2553 }
2554 EXPORT_SYMBOL(fc_exch_recv);
2555 
2556 /**
2557  * fc_exch_init() - Initialize the exchange layer for a local port
2558  * @lport: The local port to initialize the exchange layer for
2559  */
2560 int fc_exch_init(struct fc_lport *lport)
2561 {
2562 	if (!lport->tt.seq_start_next)
2563 		lport->tt.seq_start_next = fc_seq_start_next;
2564 
2565 	if (!lport->tt.seq_set_resp)
2566 		lport->tt.seq_set_resp = fc_seq_set_resp;
2567 
2568 	if (!lport->tt.exch_seq_send)
2569 		lport->tt.exch_seq_send = fc_exch_seq_send;
2570 
2571 	if (!lport->tt.seq_send)
2572 		lport->tt.seq_send = fc_seq_send;
2573 
2574 	if (!lport->tt.seq_els_rsp_send)
2575 		lport->tt.seq_els_rsp_send = fc_seq_els_rsp_send;
2576 
2577 	if (!lport->tt.exch_done)
2578 		lport->tt.exch_done = fc_exch_done;
2579 
2580 	if (!lport->tt.exch_mgr_reset)
2581 		lport->tt.exch_mgr_reset = fc_exch_mgr_reset;
2582 
2583 	if (!lport->tt.seq_exch_abort)
2584 		lport->tt.seq_exch_abort = fc_seq_exch_abort;
2585 
2586 	if (!lport->tt.seq_assign)
2587 		lport->tt.seq_assign = fc_seq_assign;
2588 
2589 	if (!lport->tt.seq_release)
2590 		lport->tt.seq_release = fc_seq_release;
2591 
2592 	return 0;
2593 }
2594 EXPORT_SYMBOL(fc_exch_init);
2595 
2596 /**
2597  * fc_setup_exch_mgr() - Setup an exchange manager
2598  */
2599 int fc_setup_exch_mgr(void)
2600 {
2601 	fc_em_cachep = kmem_cache_create("libfc_em", sizeof(struct fc_exch),
2602 					 0, SLAB_HWCACHE_ALIGN, NULL);
2603 	if (!fc_em_cachep)
2604 		return -ENOMEM;
2605 
2606 	/*
2607 	 * Initialize fc_cpu_mask and fc_cpu_order. The
2608 	 * fc_cpu_mask is set for nr_cpu_ids rounded up
2609 	 * to order of 2's * power and order is stored
2610 	 * in fc_cpu_order as this is later required in
2611 	 * mapping between an exch id and exch array index
2612 	 * in per cpu exch pool.
2613 	 *
2614 	 * This round up is required to align fc_cpu_mask
2615 	 * to exchange id's lower bits such that all incoming
2616 	 * frames of an exchange gets delivered to the same
2617 	 * cpu on which exchange originated by simple bitwise
2618 	 * AND operation between fc_cpu_mask and exchange id.
2619 	 */
2620 	fc_cpu_order = ilog2(roundup_pow_of_two(nr_cpu_ids));
2621 	fc_cpu_mask = (1 << fc_cpu_order) - 1;
2622 
2623 	fc_exch_workqueue = create_singlethread_workqueue("fc_exch_workqueue");
2624 	if (!fc_exch_workqueue)
2625 		goto err;
2626 	return 0;
2627 err:
2628 	kmem_cache_destroy(fc_em_cachep);
2629 	return -ENOMEM;
2630 }
2631 
2632 /**
2633  * fc_destroy_exch_mgr() - Destroy an exchange manager
2634  */
2635 void fc_destroy_exch_mgr(void)
2636 {
2637 	destroy_workqueue(fc_exch_workqueue);
2638 	kmem_cache_destroy(fc_em_cachep);
2639 }
2640