xref: /openbmc/linux/drivers/net/ethernet/ibm/ibmvnic.c (revision f8a11425075ff11b4b5784f077cb84f3d2dfb3f0)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /**************************************************************************/
3 /*                                                                        */
4 /*  IBM System i and System p Virtual NIC Device Driver                   */
5 /*  Copyright (C) 2014 IBM Corp.                                          */
6 /*  Santiago Leon (santi_leon@yahoo.com)                                  */
7 /*  Thomas Falcon (tlfalcon@linux.vnet.ibm.com)                           */
8 /*  John Allen (jallen@linux.vnet.ibm.com)                                */
9 /*                                                                        */
10 /*                                                                        */
11 /* This module contains the implementation of a virtual ethernet device   */
12 /* for use with IBM i/p Series LPAR Linux. It utilizes the logical LAN    */
13 /* option of the RS/6000 Platform Architecture to interface with virtual  */
14 /* ethernet NICs that are presented to the partition by the hypervisor.   */
15 /*									   */
16 /* Messages are passed between the VNIC driver and the VNIC server using  */
17 /* Command/Response Queues (CRQs) and sub CRQs (sCRQs). CRQs are used to  */
18 /* issue and receive commands that initiate communication with the server */
19 /* on driver initialization. Sub CRQs (sCRQs) are similar to CRQs, but    */
20 /* are used by the driver to notify the server that a packet is           */
21 /* ready for transmission or that a buffer has been added to receive a    */
22 /* packet. Subsequently, sCRQs are used by the server to notify the       */
23 /* driver that a packet transmission has been completed or that a packet  */
24 /* has been received and placed in a waiting buffer.                      */
25 /*                                                                        */
26 /* In lieu of a more conventional "on-the-fly" DMA mapping strategy in    */
27 /* which skbs are DMA mapped and immediately unmapped when the transmit   */
28 /* or receive has been completed, the VNIC driver is required to use      */
29 /* "long term mapping". This entails that large, continuous DMA mapped    */
30 /* buffers are allocated on driver initialization and these buffers are   */
31 /* then continuously reused to pass skbs to and from the VNIC server.     */
32 /*                                                                        */
33 /**************************************************************************/
34 
35 #include <linux/module.h>
36 #include <linux/moduleparam.h>
37 #include <linux/types.h>
38 #include <linux/errno.h>
39 #include <linux/completion.h>
40 #include <linux/ioport.h>
41 #include <linux/dma-mapping.h>
42 #include <linux/kernel.h>
43 #include <linux/netdevice.h>
44 #include <linux/etherdevice.h>
45 #include <linux/skbuff.h>
46 #include <linux/init.h>
47 #include <linux/delay.h>
48 #include <linux/mm.h>
49 #include <linux/ethtool.h>
50 #include <linux/proc_fs.h>
51 #include <linux/if_arp.h>
52 #include <linux/in.h>
53 #include <linux/ip.h>
54 #include <linux/ipv6.h>
55 #include <linux/irq.h>
56 #include <linux/kthread.h>
57 #include <linux/seq_file.h>
58 #include <linux/interrupt.h>
59 #include <net/net_namespace.h>
60 #include <asm/hvcall.h>
61 #include <linux/atomic.h>
62 #include <asm/vio.h>
63 #include <asm/iommu.h>
64 #include <linux/uaccess.h>
65 #include <asm/firmware.h>
66 #include <linux/workqueue.h>
67 #include <linux/if_vlan.h>
68 #include <linux/utsname.h>
69 
70 #include "ibmvnic.h"
71 
72 static const char ibmvnic_driver_name[] = "ibmvnic";
73 static const char ibmvnic_driver_string[] = "IBM System i/p Virtual NIC Driver";
74 
75 MODULE_AUTHOR("Santiago Leon");
76 MODULE_DESCRIPTION("IBM System i/p Virtual NIC Driver");
77 MODULE_LICENSE("GPL");
78 MODULE_VERSION(IBMVNIC_DRIVER_VERSION);
79 
80 static int ibmvnic_version = IBMVNIC_INITIAL_VERSION;
81 static void release_sub_crqs(struct ibmvnic_adapter *, bool);
82 static int ibmvnic_reset_crq(struct ibmvnic_adapter *);
83 static int ibmvnic_send_crq_init(struct ibmvnic_adapter *);
84 static int ibmvnic_reenable_crq_queue(struct ibmvnic_adapter *);
85 static int ibmvnic_send_crq(struct ibmvnic_adapter *, union ibmvnic_crq *);
86 static int send_subcrq_indirect(struct ibmvnic_adapter *, u64, u64, u64);
87 static irqreturn_t ibmvnic_interrupt_rx(int irq, void *instance);
88 static int enable_scrq_irq(struct ibmvnic_adapter *,
89 			   struct ibmvnic_sub_crq_queue *);
90 static int disable_scrq_irq(struct ibmvnic_adapter *,
91 			    struct ibmvnic_sub_crq_queue *);
92 static int pending_scrq(struct ibmvnic_adapter *,
93 			struct ibmvnic_sub_crq_queue *);
94 static union sub_crq *ibmvnic_next_scrq(struct ibmvnic_adapter *,
95 					struct ibmvnic_sub_crq_queue *);
96 static int ibmvnic_poll(struct napi_struct *napi, int data);
97 static void send_query_map(struct ibmvnic_adapter *adapter);
98 static int send_request_map(struct ibmvnic_adapter *, dma_addr_t, u32, u8);
99 static int send_request_unmap(struct ibmvnic_adapter *, u8);
100 static int send_login(struct ibmvnic_adapter *adapter);
101 static void send_query_cap(struct ibmvnic_adapter *adapter);
102 static int init_sub_crqs(struct ibmvnic_adapter *);
103 static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter);
104 static int ibmvnic_reset_init(struct ibmvnic_adapter *, bool reset);
105 static void release_crq_queue(struct ibmvnic_adapter *);
106 static int __ibmvnic_set_mac(struct net_device *, u8 *);
107 static int init_crq_queue(struct ibmvnic_adapter *adapter);
108 static int send_query_phys_parms(struct ibmvnic_adapter *adapter);
109 
110 struct ibmvnic_stat {
111 	char name[ETH_GSTRING_LEN];
112 	int offset;
113 };
114 
115 #define IBMVNIC_STAT_OFF(stat) (offsetof(struct ibmvnic_adapter, stats) + \
116 			     offsetof(struct ibmvnic_statistics, stat))
117 #define IBMVNIC_GET_STAT(a, off) (*((u64 *)(((unsigned long)(a)) + (off))))
118 
119 static const struct ibmvnic_stat ibmvnic_stats[] = {
120 	{"rx_packets", IBMVNIC_STAT_OFF(rx_packets)},
121 	{"rx_bytes", IBMVNIC_STAT_OFF(rx_bytes)},
122 	{"tx_packets", IBMVNIC_STAT_OFF(tx_packets)},
123 	{"tx_bytes", IBMVNIC_STAT_OFF(tx_bytes)},
124 	{"ucast_tx_packets", IBMVNIC_STAT_OFF(ucast_tx_packets)},
125 	{"ucast_rx_packets", IBMVNIC_STAT_OFF(ucast_rx_packets)},
126 	{"mcast_tx_packets", IBMVNIC_STAT_OFF(mcast_tx_packets)},
127 	{"mcast_rx_packets", IBMVNIC_STAT_OFF(mcast_rx_packets)},
128 	{"bcast_tx_packets", IBMVNIC_STAT_OFF(bcast_tx_packets)},
129 	{"bcast_rx_packets", IBMVNIC_STAT_OFF(bcast_rx_packets)},
130 	{"align_errors", IBMVNIC_STAT_OFF(align_errors)},
131 	{"fcs_errors", IBMVNIC_STAT_OFF(fcs_errors)},
132 	{"single_collision_frames", IBMVNIC_STAT_OFF(single_collision_frames)},
133 	{"multi_collision_frames", IBMVNIC_STAT_OFF(multi_collision_frames)},
134 	{"sqe_test_errors", IBMVNIC_STAT_OFF(sqe_test_errors)},
135 	{"deferred_tx", IBMVNIC_STAT_OFF(deferred_tx)},
136 	{"late_collisions", IBMVNIC_STAT_OFF(late_collisions)},
137 	{"excess_collisions", IBMVNIC_STAT_OFF(excess_collisions)},
138 	{"internal_mac_tx_errors", IBMVNIC_STAT_OFF(internal_mac_tx_errors)},
139 	{"carrier_sense", IBMVNIC_STAT_OFF(carrier_sense)},
140 	{"too_long_frames", IBMVNIC_STAT_OFF(too_long_frames)},
141 	{"internal_mac_rx_errors", IBMVNIC_STAT_OFF(internal_mac_rx_errors)},
142 };
143 
144 static int send_crq_init_complete(struct ibmvnic_adapter *adapter)
145 {
146 	union ibmvnic_crq crq;
147 
148 	memset(&crq, 0, sizeof(crq));
149 	crq.generic.first = IBMVNIC_CRQ_INIT_CMD;
150 	crq.generic.cmd = IBMVNIC_CRQ_INIT_COMPLETE;
151 
152 	return ibmvnic_send_crq(adapter, &crq);
153 }
154 
155 static int send_version_xchg(struct ibmvnic_adapter *adapter)
156 {
157 	union ibmvnic_crq crq;
158 
159 	memset(&crq, 0, sizeof(crq));
160 	crq.version_exchange.first = IBMVNIC_CRQ_CMD;
161 	crq.version_exchange.cmd = VERSION_EXCHANGE;
162 	crq.version_exchange.version = cpu_to_be16(ibmvnic_version);
163 
164 	return ibmvnic_send_crq(adapter, &crq);
165 }
166 
167 static long h_reg_sub_crq(unsigned long unit_address, unsigned long token,
168 			  unsigned long length, unsigned long *number,
169 			  unsigned long *irq)
170 {
171 	unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
172 	long rc;
173 
174 	rc = plpar_hcall(H_REG_SUB_CRQ, retbuf, unit_address, token, length);
175 	*number = retbuf[0];
176 	*irq = retbuf[1];
177 
178 	return rc;
179 }
180 
181 /**
182  * ibmvnic_wait_for_completion - Check device state and wait for completion
183  * @adapter: private device data
184  * @comp_done: completion structure to wait for
185  * @timeout: time to wait in milliseconds
186  *
187  * Wait for a completion signal or until the timeout limit is reached
188  * while checking that the device is still active.
189  */
190 static int ibmvnic_wait_for_completion(struct ibmvnic_adapter *adapter,
191 				       struct completion *comp_done,
192 				       unsigned long timeout)
193 {
194 	struct net_device *netdev;
195 	unsigned long div_timeout;
196 	u8 retry;
197 
198 	netdev = adapter->netdev;
199 	retry = 5;
200 	div_timeout = msecs_to_jiffies(timeout / retry);
201 	while (true) {
202 		if (!adapter->crq.active) {
203 			netdev_err(netdev, "Device down!\n");
204 			return -ENODEV;
205 		}
206 		if (!retry--)
207 			break;
208 		if (wait_for_completion_timeout(comp_done, div_timeout))
209 			return 0;
210 	}
211 	netdev_err(netdev, "Operation timed out.\n");
212 	return -ETIMEDOUT;
213 }
214 
215 static int alloc_long_term_buff(struct ibmvnic_adapter *adapter,
216 				struct ibmvnic_long_term_buff *ltb, int size)
217 {
218 	struct device *dev = &adapter->vdev->dev;
219 	int rc;
220 
221 	ltb->size = size;
222 	ltb->buff = dma_alloc_coherent(dev, ltb->size, &ltb->addr,
223 				       GFP_KERNEL);
224 
225 	if (!ltb->buff) {
226 		dev_err(dev, "Couldn't alloc long term buffer\n");
227 		return -ENOMEM;
228 	}
229 	ltb->map_id = adapter->map_id;
230 	adapter->map_id++;
231 
232 	mutex_lock(&adapter->fw_lock);
233 	adapter->fw_done_rc = 0;
234 	reinit_completion(&adapter->fw_done);
235 	rc = send_request_map(adapter, ltb->addr,
236 			      ltb->size, ltb->map_id);
237 	if (rc) {
238 		dma_free_coherent(dev, ltb->size, ltb->buff, ltb->addr);
239 		mutex_unlock(&adapter->fw_lock);
240 		return rc;
241 	}
242 
243 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);
244 	if (rc) {
245 		dev_err(dev,
246 			"Long term map request aborted or timed out,rc = %d\n",
247 			rc);
248 		dma_free_coherent(dev, ltb->size, ltb->buff, ltb->addr);
249 		mutex_unlock(&adapter->fw_lock);
250 		return rc;
251 	}
252 
253 	if (adapter->fw_done_rc) {
254 		dev_err(dev, "Couldn't map long term buffer,rc = %d\n",
255 			adapter->fw_done_rc);
256 		dma_free_coherent(dev, ltb->size, ltb->buff, ltb->addr);
257 		mutex_unlock(&adapter->fw_lock);
258 		return -1;
259 	}
260 	mutex_unlock(&adapter->fw_lock);
261 	return 0;
262 }
263 
264 static void free_long_term_buff(struct ibmvnic_adapter *adapter,
265 				struct ibmvnic_long_term_buff *ltb)
266 {
267 	struct device *dev = &adapter->vdev->dev;
268 
269 	if (!ltb->buff)
270 		return;
271 
272 	/* VIOS automatically unmaps the long term buffer at remote
273 	 * end for the following resets:
274 	 * FAILOVER, MOBILITY, TIMEOUT.
275 	 */
276 	if (adapter->reset_reason != VNIC_RESET_FAILOVER &&
277 	    adapter->reset_reason != VNIC_RESET_MOBILITY &&
278 	    adapter->reset_reason != VNIC_RESET_TIMEOUT)
279 		send_request_unmap(adapter, ltb->map_id);
280 	dma_free_coherent(dev, ltb->size, ltb->buff, ltb->addr);
281 }
282 
283 static int reset_long_term_buff(struct ibmvnic_long_term_buff *ltb)
284 {
285 	if (!ltb->buff)
286 		return -EINVAL;
287 
288 	memset(ltb->buff, 0, ltb->size);
289 	return 0;
290 }
291 
292 static void deactivate_rx_pools(struct ibmvnic_adapter *adapter)
293 {
294 	int i;
295 
296 	for (i = 0; i < adapter->num_active_rx_pools; i++)
297 		adapter->rx_pool[i].active = 0;
298 }
299 
300 static void replenish_rx_pool(struct ibmvnic_adapter *adapter,
301 			      struct ibmvnic_rx_pool *pool)
302 {
303 	int count = pool->size - atomic_read(&pool->available);
304 	u64 handle = adapter->rx_scrq[pool->index]->handle;
305 	struct device *dev = &adapter->vdev->dev;
306 	struct ibmvnic_ind_xmit_queue *ind_bufp;
307 	struct ibmvnic_sub_crq_queue *rx_scrq;
308 	union sub_crq *sub_crq;
309 	int buffers_added = 0;
310 	unsigned long lpar_rc;
311 	struct sk_buff *skb;
312 	unsigned int offset;
313 	dma_addr_t dma_addr;
314 	unsigned char *dst;
315 	int shift = 0;
316 	int index;
317 	int i;
318 
319 	if (!pool->active)
320 		return;
321 
322 	rx_scrq = adapter->rx_scrq[pool->index];
323 	ind_bufp = &rx_scrq->ind_buf;
324 	for (i = 0; i < count; ++i) {
325 		skb = netdev_alloc_skb(adapter->netdev, pool->buff_size);
326 		if (!skb) {
327 			dev_err(dev, "Couldn't replenish rx buff\n");
328 			adapter->replenish_no_mem++;
329 			break;
330 		}
331 
332 		index = pool->free_map[pool->next_free];
333 
334 		if (pool->rx_buff[index].skb)
335 			dev_err(dev, "Inconsistent free_map!\n");
336 
337 		/* Copy the skb to the long term mapped DMA buffer */
338 		offset = index * pool->buff_size;
339 		dst = pool->long_term_buff.buff + offset;
340 		memset(dst, 0, pool->buff_size);
341 		dma_addr = pool->long_term_buff.addr + offset;
342 		pool->rx_buff[index].data = dst;
343 
344 		pool->free_map[pool->next_free] = IBMVNIC_INVALID_MAP;
345 		pool->rx_buff[index].dma = dma_addr;
346 		pool->rx_buff[index].skb = skb;
347 		pool->rx_buff[index].pool_index = pool->index;
348 		pool->rx_buff[index].size = pool->buff_size;
349 
350 		sub_crq = &ind_bufp->indir_arr[ind_bufp->index++];
351 		memset(sub_crq, 0, sizeof(*sub_crq));
352 		sub_crq->rx_add.first = IBMVNIC_CRQ_CMD;
353 		sub_crq->rx_add.correlator =
354 		    cpu_to_be64((u64)&pool->rx_buff[index]);
355 		sub_crq->rx_add.ioba = cpu_to_be32(dma_addr);
356 		sub_crq->rx_add.map_id = pool->long_term_buff.map_id;
357 
358 		/* The length field of the sCRQ is defined to be 24 bits so the
359 		 * buffer size needs to be left shifted by a byte before it is
360 		 * converted to big endian to prevent the last byte from being
361 		 * truncated.
362 		 */
363 #ifdef __LITTLE_ENDIAN__
364 		shift = 8;
365 #endif
366 		sub_crq->rx_add.len = cpu_to_be32(pool->buff_size << shift);
367 		pool->next_free = (pool->next_free + 1) % pool->size;
368 		if (ind_bufp->index == IBMVNIC_MAX_IND_DESCS ||
369 		    i == count - 1) {
370 			lpar_rc =
371 				send_subcrq_indirect(adapter, handle,
372 						     (u64)ind_bufp->indir_dma,
373 						     (u64)ind_bufp->index);
374 			if (lpar_rc != H_SUCCESS)
375 				goto failure;
376 			buffers_added += ind_bufp->index;
377 			adapter->replenish_add_buff_success += ind_bufp->index;
378 			ind_bufp->index = 0;
379 		}
380 	}
381 	atomic_add(buffers_added, &pool->available);
382 	return;
383 
384 failure:
385 	if (lpar_rc != H_PARAMETER && lpar_rc != H_CLOSED)
386 		dev_err_ratelimited(dev, "rx: replenish packet buffer failed\n");
387 	for (i = ind_bufp->index - 1; i >= 0; --i) {
388 		struct ibmvnic_rx_buff *rx_buff;
389 
390 		pool->next_free = pool->next_free == 0 ?
391 				  pool->size - 1 : pool->next_free - 1;
392 		sub_crq = &ind_bufp->indir_arr[i];
393 		rx_buff = (struct ibmvnic_rx_buff *)
394 				be64_to_cpu(sub_crq->rx_add.correlator);
395 		index = (int)(rx_buff - pool->rx_buff);
396 		pool->free_map[pool->next_free] = index;
397 		dev_kfree_skb_any(pool->rx_buff[index].skb);
398 		pool->rx_buff[index].skb = NULL;
399 	}
400 	adapter->replenish_add_buff_failure += ind_bufp->index;
401 	atomic_add(buffers_added, &pool->available);
402 	ind_bufp->index = 0;
403 	if (lpar_rc == H_CLOSED || adapter->failover_pending) {
404 		/* Disable buffer pool replenishment and report carrier off if
405 		 * queue is closed or pending failover.
406 		 * Firmware guarantees that a signal will be sent to the
407 		 * driver, triggering a reset.
408 		 */
409 		deactivate_rx_pools(adapter);
410 		netif_carrier_off(adapter->netdev);
411 	}
412 }
413 
414 static void replenish_pools(struct ibmvnic_adapter *adapter)
415 {
416 	int i;
417 
418 	adapter->replenish_task_cycles++;
419 	for (i = 0; i < adapter->num_active_rx_pools; i++) {
420 		if (adapter->rx_pool[i].active)
421 			replenish_rx_pool(adapter, &adapter->rx_pool[i]);
422 	}
423 
424 	netdev_dbg(adapter->netdev, "Replenished %d pools\n", i);
425 }
426 
427 static void release_stats_buffers(struct ibmvnic_adapter *adapter)
428 {
429 	kfree(adapter->tx_stats_buffers);
430 	kfree(adapter->rx_stats_buffers);
431 	adapter->tx_stats_buffers = NULL;
432 	adapter->rx_stats_buffers = NULL;
433 }
434 
435 static int init_stats_buffers(struct ibmvnic_adapter *adapter)
436 {
437 	adapter->tx_stats_buffers =
438 				kcalloc(IBMVNIC_MAX_QUEUES,
439 					sizeof(struct ibmvnic_tx_queue_stats),
440 					GFP_KERNEL);
441 	if (!adapter->tx_stats_buffers)
442 		return -ENOMEM;
443 
444 	adapter->rx_stats_buffers =
445 				kcalloc(IBMVNIC_MAX_QUEUES,
446 					sizeof(struct ibmvnic_rx_queue_stats),
447 					GFP_KERNEL);
448 	if (!adapter->rx_stats_buffers)
449 		return -ENOMEM;
450 
451 	return 0;
452 }
453 
454 static void release_stats_token(struct ibmvnic_adapter *adapter)
455 {
456 	struct device *dev = &adapter->vdev->dev;
457 
458 	if (!adapter->stats_token)
459 		return;
460 
461 	dma_unmap_single(dev, adapter->stats_token,
462 			 sizeof(struct ibmvnic_statistics),
463 			 DMA_FROM_DEVICE);
464 	adapter->stats_token = 0;
465 }
466 
467 static int init_stats_token(struct ibmvnic_adapter *adapter)
468 {
469 	struct device *dev = &adapter->vdev->dev;
470 	dma_addr_t stok;
471 
472 	stok = dma_map_single(dev, &adapter->stats,
473 			      sizeof(struct ibmvnic_statistics),
474 			      DMA_FROM_DEVICE);
475 	if (dma_mapping_error(dev, stok)) {
476 		dev_err(dev, "Couldn't map stats buffer\n");
477 		return -1;
478 	}
479 
480 	adapter->stats_token = stok;
481 	netdev_dbg(adapter->netdev, "Stats token initialized (%llx)\n", stok);
482 	return 0;
483 }
484 
485 static int reset_rx_pools(struct ibmvnic_adapter *adapter)
486 {
487 	struct ibmvnic_rx_pool *rx_pool;
488 	u64 buff_size;
489 	int rx_scrqs;
490 	int i, j, rc;
491 
492 	if (!adapter->rx_pool)
493 		return -1;
494 
495 	buff_size = adapter->cur_rx_buf_sz;
496 	rx_scrqs = adapter->num_active_rx_pools;
497 	for (i = 0; i < rx_scrqs; i++) {
498 		rx_pool = &adapter->rx_pool[i];
499 
500 		netdev_dbg(adapter->netdev, "Re-setting rx_pool[%d]\n", i);
501 
502 		if (rx_pool->buff_size != buff_size) {
503 			free_long_term_buff(adapter, &rx_pool->long_term_buff);
504 			rx_pool->buff_size = ALIGN(buff_size, L1_CACHE_BYTES);
505 			rc = alloc_long_term_buff(adapter,
506 						  &rx_pool->long_term_buff,
507 						  rx_pool->size *
508 						  rx_pool->buff_size);
509 		} else {
510 			rc = reset_long_term_buff(&rx_pool->long_term_buff);
511 		}
512 
513 		if (rc)
514 			return rc;
515 
516 		for (j = 0; j < rx_pool->size; j++)
517 			rx_pool->free_map[j] = j;
518 
519 		memset(rx_pool->rx_buff, 0,
520 		       rx_pool->size * sizeof(struct ibmvnic_rx_buff));
521 
522 		atomic_set(&rx_pool->available, 0);
523 		rx_pool->next_alloc = 0;
524 		rx_pool->next_free = 0;
525 		rx_pool->active = 1;
526 	}
527 
528 	return 0;
529 }
530 
531 static void release_rx_pools(struct ibmvnic_adapter *adapter)
532 {
533 	struct ibmvnic_rx_pool *rx_pool;
534 	int i, j;
535 
536 	if (!adapter->rx_pool)
537 		return;
538 
539 	for (i = 0; i < adapter->num_active_rx_pools; i++) {
540 		rx_pool = &adapter->rx_pool[i];
541 
542 		netdev_dbg(adapter->netdev, "Releasing rx_pool[%d]\n", i);
543 
544 		kfree(rx_pool->free_map);
545 		free_long_term_buff(adapter, &rx_pool->long_term_buff);
546 
547 		if (!rx_pool->rx_buff)
548 			continue;
549 
550 		for (j = 0; j < rx_pool->size; j++) {
551 			if (rx_pool->rx_buff[j].skb) {
552 				dev_kfree_skb_any(rx_pool->rx_buff[j].skb);
553 				rx_pool->rx_buff[j].skb = NULL;
554 			}
555 		}
556 
557 		kfree(rx_pool->rx_buff);
558 	}
559 
560 	kfree(adapter->rx_pool);
561 	adapter->rx_pool = NULL;
562 	adapter->num_active_rx_pools = 0;
563 }
564 
565 static int init_rx_pools(struct net_device *netdev)
566 {
567 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
568 	struct device *dev = &adapter->vdev->dev;
569 	struct ibmvnic_rx_pool *rx_pool;
570 	int rxadd_subcrqs;
571 	u64 buff_size;
572 	int i, j;
573 
574 	rxadd_subcrqs = adapter->num_active_rx_scrqs;
575 	buff_size = adapter->cur_rx_buf_sz;
576 
577 	adapter->rx_pool = kcalloc(rxadd_subcrqs,
578 				   sizeof(struct ibmvnic_rx_pool),
579 				   GFP_KERNEL);
580 	if (!adapter->rx_pool) {
581 		dev_err(dev, "Failed to allocate rx pools\n");
582 		return -1;
583 	}
584 
585 	adapter->num_active_rx_pools = rxadd_subcrqs;
586 
587 	for (i = 0; i < rxadd_subcrqs; i++) {
588 		rx_pool = &adapter->rx_pool[i];
589 
590 		netdev_dbg(adapter->netdev,
591 			   "Initializing rx_pool[%d], %lld buffs, %lld bytes each\n",
592 			   i, adapter->req_rx_add_entries_per_subcrq,
593 			   buff_size);
594 
595 		rx_pool->size = adapter->req_rx_add_entries_per_subcrq;
596 		rx_pool->index = i;
597 		rx_pool->buff_size = ALIGN(buff_size, L1_CACHE_BYTES);
598 		rx_pool->active = 1;
599 
600 		rx_pool->free_map = kcalloc(rx_pool->size, sizeof(int),
601 					    GFP_KERNEL);
602 		if (!rx_pool->free_map) {
603 			release_rx_pools(adapter);
604 			return -1;
605 		}
606 
607 		rx_pool->rx_buff = kcalloc(rx_pool->size,
608 					   sizeof(struct ibmvnic_rx_buff),
609 					   GFP_KERNEL);
610 		if (!rx_pool->rx_buff) {
611 			dev_err(dev, "Couldn't alloc rx buffers\n");
612 			release_rx_pools(adapter);
613 			return -1;
614 		}
615 
616 		if (alloc_long_term_buff(adapter, &rx_pool->long_term_buff,
617 					 rx_pool->size * rx_pool->buff_size)) {
618 			release_rx_pools(adapter);
619 			return -1;
620 		}
621 
622 		for (j = 0; j < rx_pool->size; ++j)
623 			rx_pool->free_map[j] = j;
624 
625 		atomic_set(&rx_pool->available, 0);
626 		rx_pool->next_alloc = 0;
627 		rx_pool->next_free = 0;
628 	}
629 
630 	return 0;
631 }
632 
633 static int reset_one_tx_pool(struct ibmvnic_tx_pool *tx_pool)
634 {
635 	int rc, i;
636 
637 	rc = reset_long_term_buff(&tx_pool->long_term_buff);
638 	if (rc)
639 		return rc;
640 
641 	memset(tx_pool->tx_buff, 0,
642 	       tx_pool->num_buffers *
643 	       sizeof(struct ibmvnic_tx_buff));
644 
645 	for (i = 0; i < tx_pool->num_buffers; i++)
646 		tx_pool->free_map[i] = i;
647 
648 	tx_pool->consumer_index = 0;
649 	tx_pool->producer_index = 0;
650 
651 	return 0;
652 }
653 
654 static int reset_tx_pools(struct ibmvnic_adapter *adapter)
655 {
656 	int tx_scrqs;
657 	int i, rc;
658 
659 	if (!adapter->tx_pool)
660 		return -1;
661 
662 	tx_scrqs = adapter->num_active_tx_pools;
663 	for (i = 0; i < tx_scrqs; i++) {
664 		rc = reset_one_tx_pool(&adapter->tso_pool[i]);
665 		if (rc)
666 			return rc;
667 		rc = reset_one_tx_pool(&adapter->tx_pool[i]);
668 		if (rc)
669 			return rc;
670 	}
671 
672 	return 0;
673 }
674 
675 static void release_vpd_data(struct ibmvnic_adapter *adapter)
676 {
677 	if (!adapter->vpd)
678 		return;
679 
680 	kfree(adapter->vpd->buff);
681 	kfree(adapter->vpd);
682 
683 	adapter->vpd = NULL;
684 }
685 
686 static void release_one_tx_pool(struct ibmvnic_adapter *adapter,
687 				struct ibmvnic_tx_pool *tx_pool)
688 {
689 	kfree(tx_pool->tx_buff);
690 	kfree(tx_pool->free_map);
691 	free_long_term_buff(adapter, &tx_pool->long_term_buff);
692 }
693 
694 static void release_tx_pools(struct ibmvnic_adapter *adapter)
695 {
696 	int i;
697 
698 	if (!adapter->tx_pool)
699 		return;
700 
701 	for (i = 0; i < adapter->num_active_tx_pools; i++) {
702 		release_one_tx_pool(adapter, &adapter->tx_pool[i]);
703 		release_one_tx_pool(adapter, &adapter->tso_pool[i]);
704 	}
705 
706 	kfree(adapter->tx_pool);
707 	adapter->tx_pool = NULL;
708 	kfree(adapter->tso_pool);
709 	adapter->tso_pool = NULL;
710 	adapter->num_active_tx_pools = 0;
711 }
712 
713 static int init_one_tx_pool(struct net_device *netdev,
714 			    struct ibmvnic_tx_pool *tx_pool,
715 			    int num_entries, int buf_size)
716 {
717 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
718 	int i;
719 
720 	tx_pool->tx_buff = kcalloc(num_entries,
721 				   sizeof(struct ibmvnic_tx_buff),
722 				   GFP_KERNEL);
723 	if (!tx_pool->tx_buff)
724 		return -1;
725 
726 	if (alloc_long_term_buff(adapter, &tx_pool->long_term_buff,
727 				 num_entries * buf_size))
728 		return -1;
729 
730 	tx_pool->free_map = kcalloc(num_entries, sizeof(int), GFP_KERNEL);
731 	if (!tx_pool->free_map)
732 		return -1;
733 
734 	for (i = 0; i < num_entries; i++)
735 		tx_pool->free_map[i] = i;
736 
737 	tx_pool->consumer_index = 0;
738 	tx_pool->producer_index = 0;
739 	tx_pool->num_buffers = num_entries;
740 	tx_pool->buf_size = buf_size;
741 
742 	return 0;
743 }
744 
745 static int init_tx_pools(struct net_device *netdev)
746 {
747 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
748 	int tx_subcrqs;
749 	u64 buff_size;
750 	int i, rc;
751 
752 	tx_subcrqs = adapter->num_active_tx_scrqs;
753 	adapter->tx_pool = kcalloc(tx_subcrqs,
754 				   sizeof(struct ibmvnic_tx_pool), GFP_KERNEL);
755 	if (!adapter->tx_pool)
756 		return -1;
757 
758 	adapter->tso_pool = kcalloc(tx_subcrqs,
759 				    sizeof(struct ibmvnic_tx_pool), GFP_KERNEL);
760 	if (!adapter->tso_pool)
761 		return -1;
762 
763 	adapter->num_active_tx_pools = tx_subcrqs;
764 
765 	for (i = 0; i < tx_subcrqs; i++) {
766 		buff_size = adapter->req_mtu + VLAN_HLEN;
767 		buff_size = ALIGN(buff_size, L1_CACHE_BYTES);
768 		rc = init_one_tx_pool(netdev, &adapter->tx_pool[i],
769 				      adapter->req_tx_entries_per_subcrq,
770 				      buff_size);
771 		if (rc) {
772 			release_tx_pools(adapter);
773 			return rc;
774 		}
775 
776 		rc = init_one_tx_pool(netdev, &adapter->tso_pool[i],
777 				      IBMVNIC_TSO_BUFS,
778 				      IBMVNIC_TSO_BUF_SZ);
779 		if (rc) {
780 			release_tx_pools(adapter);
781 			return rc;
782 		}
783 	}
784 
785 	return 0;
786 }
787 
788 static void ibmvnic_napi_enable(struct ibmvnic_adapter *adapter)
789 {
790 	int i;
791 
792 	if (adapter->napi_enabled)
793 		return;
794 
795 	for (i = 0; i < adapter->req_rx_queues; i++)
796 		napi_enable(&adapter->napi[i]);
797 
798 	adapter->napi_enabled = true;
799 }
800 
801 static void ibmvnic_napi_disable(struct ibmvnic_adapter *adapter)
802 {
803 	int i;
804 
805 	if (!adapter->napi_enabled)
806 		return;
807 
808 	for (i = 0; i < adapter->req_rx_queues; i++) {
809 		netdev_dbg(adapter->netdev, "Disabling napi[%d]\n", i);
810 		napi_disable(&adapter->napi[i]);
811 	}
812 
813 	adapter->napi_enabled = false;
814 }
815 
816 static int init_napi(struct ibmvnic_adapter *adapter)
817 {
818 	int i;
819 
820 	adapter->napi = kcalloc(adapter->req_rx_queues,
821 				sizeof(struct napi_struct), GFP_KERNEL);
822 	if (!adapter->napi)
823 		return -ENOMEM;
824 
825 	for (i = 0; i < adapter->req_rx_queues; i++) {
826 		netdev_dbg(adapter->netdev, "Adding napi[%d]\n", i);
827 		netif_napi_add(adapter->netdev, &adapter->napi[i],
828 			       ibmvnic_poll, NAPI_POLL_WEIGHT);
829 	}
830 
831 	adapter->num_active_rx_napi = adapter->req_rx_queues;
832 	return 0;
833 }
834 
835 static void release_napi(struct ibmvnic_adapter *adapter)
836 {
837 	int i;
838 
839 	if (!adapter->napi)
840 		return;
841 
842 	for (i = 0; i < adapter->num_active_rx_napi; i++) {
843 		netdev_dbg(adapter->netdev, "Releasing napi[%d]\n", i);
844 		netif_napi_del(&adapter->napi[i]);
845 	}
846 
847 	kfree(adapter->napi);
848 	adapter->napi = NULL;
849 	adapter->num_active_rx_napi = 0;
850 	adapter->napi_enabled = false;
851 }
852 
853 static const char *adapter_state_to_string(enum vnic_state state)
854 {
855 	switch (state) {
856 	case VNIC_PROBING:
857 		return "PROBING";
858 	case VNIC_PROBED:
859 		return "PROBED";
860 	case VNIC_OPENING:
861 		return "OPENING";
862 	case VNIC_OPEN:
863 		return "OPEN";
864 	case VNIC_CLOSING:
865 		return "CLOSING";
866 	case VNIC_CLOSED:
867 		return "CLOSED";
868 	case VNIC_REMOVING:
869 		return "REMOVING";
870 	case VNIC_REMOVED:
871 		return "REMOVED";
872 	case VNIC_DOWN:
873 		return "DOWN";
874 	}
875 	return "UNKNOWN";
876 }
877 
878 static int ibmvnic_login(struct net_device *netdev)
879 {
880 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
881 	unsigned long timeout = msecs_to_jiffies(20000);
882 	int retry_count = 0;
883 	int retries = 10;
884 	bool retry;
885 	int rc;
886 
887 	do {
888 		retry = false;
889 		if (retry_count > retries) {
890 			netdev_warn(netdev, "Login attempts exceeded\n");
891 			return -1;
892 		}
893 
894 		adapter->init_done_rc = 0;
895 		reinit_completion(&adapter->init_done);
896 		rc = send_login(adapter);
897 		if (rc)
898 			return rc;
899 
900 		if (!wait_for_completion_timeout(&adapter->init_done,
901 						 timeout)) {
902 			netdev_warn(netdev, "Login timed out, retrying...\n");
903 			retry = true;
904 			adapter->init_done_rc = 0;
905 			retry_count++;
906 			continue;
907 		}
908 
909 		if (adapter->init_done_rc == ABORTED) {
910 			netdev_warn(netdev, "Login aborted, retrying...\n");
911 			retry = true;
912 			adapter->init_done_rc = 0;
913 			retry_count++;
914 			/* FW or device may be busy, so
915 			 * wait a bit before retrying login
916 			 */
917 			msleep(500);
918 		} else if (adapter->init_done_rc == PARTIALSUCCESS) {
919 			retry_count++;
920 			release_sub_crqs(adapter, 1);
921 
922 			retry = true;
923 			netdev_dbg(netdev,
924 				   "Received partial success, retrying...\n");
925 			adapter->init_done_rc = 0;
926 			reinit_completion(&adapter->init_done);
927 			send_query_cap(adapter);
928 			if (!wait_for_completion_timeout(&adapter->init_done,
929 							 timeout)) {
930 				netdev_warn(netdev,
931 					    "Capabilities query timed out\n");
932 				return -1;
933 			}
934 
935 			rc = init_sub_crqs(adapter);
936 			if (rc) {
937 				netdev_warn(netdev,
938 					    "SCRQ initialization failed\n");
939 				return -1;
940 			}
941 
942 			rc = init_sub_crq_irqs(adapter);
943 			if (rc) {
944 				netdev_warn(netdev,
945 					    "SCRQ irq initialization failed\n");
946 				return -1;
947 			}
948 		} else if (adapter->init_done_rc) {
949 			netdev_warn(netdev, "Adapter login failed\n");
950 			return -1;
951 		}
952 	} while (retry);
953 
954 	__ibmvnic_set_mac(netdev, adapter->mac_addr);
955 
956 	netdev_dbg(netdev, "[S:%s] Login succeeded\n", adapter_state_to_string(adapter->state));
957 	return 0;
958 }
959 
960 static void release_login_buffer(struct ibmvnic_adapter *adapter)
961 {
962 	kfree(adapter->login_buf);
963 	adapter->login_buf = NULL;
964 }
965 
966 static void release_login_rsp_buffer(struct ibmvnic_adapter *adapter)
967 {
968 	kfree(adapter->login_rsp_buf);
969 	adapter->login_rsp_buf = NULL;
970 }
971 
972 static void release_resources(struct ibmvnic_adapter *adapter)
973 {
974 	release_vpd_data(adapter);
975 
976 	release_tx_pools(adapter);
977 	release_rx_pools(adapter);
978 
979 	release_napi(adapter);
980 	release_login_buffer(adapter);
981 	release_login_rsp_buffer(adapter);
982 }
983 
984 static int set_link_state(struct ibmvnic_adapter *adapter, u8 link_state)
985 {
986 	struct net_device *netdev = adapter->netdev;
987 	unsigned long timeout = msecs_to_jiffies(20000);
988 	union ibmvnic_crq crq;
989 	bool resend;
990 	int rc;
991 
992 	netdev_dbg(netdev, "setting link state %d\n", link_state);
993 
994 	memset(&crq, 0, sizeof(crq));
995 	crq.logical_link_state.first = IBMVNIC_CRQ_CMD;
996 	crq.logical_link_state.cmd = LOGICAL_LINK_STATE;
997 	crq.logical_link_state.link_state = link_state;
998 
999 	do {
1000 		resend = false;
1001 
1002 		reinit_completion(&adapter->init_done);
1003 		rc = ibmvnic_send_crq(adapter, &crq);
1004 		if (rc) {
1005 			netdev_err(netdev, "Failed to set link state\n");
1006 			return rc;
1007 		}
1008 
1009 		if (!wait_for_completion_timeout(&adapter->init_done,
1010 						 timeout)) {
1011 			netdev_err(netdev, "timeout setting link state\n");
1012 			return -1;
1013 		}
1014 
1015 		if (adapter->init_done_rc == PARTIALSUCCESS) {
1016 			/* Partuial success, delay and re-send */
1017 			mdelay(1000);
1018 			resend = true;
1019 		} else if (adapter->init_done_rc) {
1020 			netdev_warn(netdev, "Unable to set link state, rc=%d\n",
1021 				    adapter->init_done_rc);
1022 			return adapter->init_done_rc;
1023 		}
1024 	} while (resend);
1025 
1026 	return 0;
1027 }
1028 
1029 static int set_real_num_queues(struct net_device *netdev)
1030 {
1031 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1032 	int rc;
1033 
1034 	netdev_dbg(netdev, "Setting real tx/rx queues (%llx/%llx)\n",
1035 		   adapter->req_tx_queues, adapter->req_rx_queues);
1036 
1037 	rc = netif_set_real_num_tx_queues(netdev, adapter->req_tx_queues);
1038 	if (rc) {
1039 		netdev_err(netdev, "failed to set the number of tx queues\n");
1040 		return rc;
1041 	}
1042 
1043 	rc = netif_set_real_num_rx_queues(netdev, adapter->req_rx_queues);
1044 	if (rc)
1045 		netdev_err(netdev, "failed to set the number of rx queues\n");
1046 
1047 	return rc;
1048 }
1049 
1050 static int ibmvnic_get_vpd(struct ibmvnic_adapter *adapter)
1051 {
1052 	struct device *dev = &adapter->vdev->dev;
1053 	union ibmvnic_crq crq;
1054 	int len = 0;
1055 	int rc;
1056 
1057 	if (adapter->vpd->buff)
1058 		len = adapter->vpd->len;
1059 
1060 	mutex_lock(&adapter->fw_lock);
1061 	adapter->fw_done_rc = 0;
1062 	reinit_completion(&adapter->fw_done);
1063 
1064 	crq.get_vpd_size.first = IBMVNIC_CRQ_CMD;
1065 	crq.get_vpd_size.cmd = GET_VPD_SIZE;
1066 	rc = ibmvnic_send_crq(adapter, &crq);
1067 	if (rc) {
1068 		mutex_unlock(&adapter->fw_lock);
1069 		return rc;
1070 	}
1071 
1072 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);
1073 	if (rc) {
1074 		dev_err(dev, "Could not retrieve VPD size, rc = %d\n", rc);
1075 		mutex_unlock(&adapter->fw_lock);
1076 		return rc;
1077 	}
1078 	mutex_unlock(&adapter->fw_lock);
1079 
1080 	if (!adapter->vpd->len)
1081 		return -ENODATA;
1082 
1083 	if (!adapter->vpd->buff)
1084 		adapter->vpd->buff = kzalloc(adapter->vpd->len, GFP_KERNEL);
1085 	else if (adapter->vpd->len != len)
1086 		adapter->vpd->buff =
1087 			krealloc(adapter->vpd->buff,
1088 				 adapter->vpd->len, GFP_KERNEL);
1089 
1090 	if (!adapter->vpd->buff) {
1091 		dev_err(dev, "Could allocate VPD buffer\n");
1092 		return -ENOMEM;
1093 	}
1094 
1095 	adapter->vpd->dma_addr =
1096 		dma_map_single(dev, adapter->vpd->buff, adapter->vpd->len,
1097 			       DMA_FROM_DEVICE);
1098 	if (dma_mapping_error(dev, adapter->vpd->dma_addr)) {
1099 		dev_err(dev, "Could not map VPD buffer\n");
1100 		kfree(adapter->vpd->buff);
1101 		adapter->vpd->buff = NULL;
1102 		return -ENOMEM;
1103 	}
1104 
1105 	mutex_lock(&adapter->fw_lock);
1106 	adapter->fw_done_rc = 0;
1107 	reinit_completion(&adapter->fw_done);
1108 
1109 	crq.get_vpd.first = IBMVNIC_CRQ_CMD;
1110 	crq.get_vpd.cmd = GET_VPD;
1111 	crq.get_vpd.ioba = cpu_to_be32(adapter->vpd->dma_addr);
1112 	crq.get_vpd.len = cpu_to_be32((u32)adapter->vpd->len);
1113 	rc = ibmvnic_send_crq(adapter, &crq);
1114 	if (rc) {
1115 		kfree(adapter->vpd->buff);
1116 		adapter->vpd->buff = NULL;
1117 		mutex_unlock(&adapter->fw_lock);
1118 		return rc;
1119 	}
1120 
1121 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);
1122 	if (rc) {
1123 		dev_err(dev, "Unable to retrieve VPD, rc = %d\n", rc);
1124 		kfree(adapter->vpd->buff);
1125 		adapter->vpd->buff = NULL;
1126 		mutex_unlock(&adapter->fw_lock);
1127 		return rc;
1128 	}
1129 
1130 	mutex_unlock(&adapter->fw_lock);
1131 	return 0;
1132 }
1133 
1134 static int init_resources(struct ibmvnic_adapter *adapter)
1135 {
1136 	struct net_device *netdev = adapter->netdev;
1137 	int rc;
1138 
1139 	rc = set_real_num_queues(netdev);
1140 	if (rc)
1141 		return rc;
1142 
1143 	adapter->vpd = kzalloc(sizeof(*adapter->vpd), GFP_KERNEL);
1144 	if (!adapter->vpd)
1145 		return -ENOMEM;
1146 
1147 	/* Vital Product Data (VPD) */
1148 	rc = ibmvnic_get_vpd(adapter);
1149 	if (rc) {
1150 		netdev_err(netdev, "failed to initialize Vital Product Data (VPD)\n");
1151 		return rc;
1152 	}
1153 
1154 	adapter->map_id = 1;
1155 
1156 	rc = init_napi(adapter);
1157 	if (rc)
1158 		return rc;
1159 
1160 	send_query_map(adapter);
1161 
1162 	rc = init_rx_pools(netdev);
1163 	if (rc)
1164 		return rc;
1165 
1166 	rc = init_tx_pools(netdev);
1167 	return rc;
1168 }
1169 
1170 static int __ibmvnic_open(struct net_device *netdev)
1171 {
1172 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1173 	enum vnic_state prev_state = adapter->state;
1174 	int i, rc;
1175 
1176 	adapter->state = VNIC_OPENING;
1177 	replenish_pools(adapter);
1178 	ibmvnic_napi_enable(adapter);
1179 
1180 	/* We're ready to receive frames, enable the sub-crq interrupts and
1181 	 * set the logical link state to up
1182 	 */
1183 	for (i = 0; i < adapter->req_rx_queues; i++) {
1184 		netdev_dbg(netdev, "Enabling rx_scrq[%d] irq\n", i);
1185 		if (prev_state == VNIC_CLOSED)
1186 			enable_irq(adapter->rx_scrq[i]->irq);
1187 		enable_scrq_irq(adapter, adapter->rx_scrq[i]);
1188 	}
1189 
1190 	for (i = 0; i < adapter->req_tx_queues; i++) {
1191 		netdev_dbg(netdev, "Enabling tx_scrq[%d] irq\n", i);
1192 		if (prev_state == VNIC_CLOSED)
1193 			enable_irq(adapter->tx_scrq[i]->irq);
1194 		enable_scrq_irq(adapter, adapter->tx_scrq[i]);
1195 		netdev_tx_reset_queue(netdev_get_tx_queue(netdev, i));
1196 	}
1197 
1198 	rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_UP);
1199 	if (rc) {
1200 		ibmvnic_napi_disable(adapter);
1201 		release_resources(adapter);
1202 		return rc;
1203 	}
1204 
1205 	netif_tx_start_all_queues(netdev);
1206 
1207 	adapter->state = VNIC_OPEN;
1208 	return rc;
1209 }
1210 
1211 static int ibmvnic_open(struct net_device *netdev)
1212 {
1213 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1214 	int rc;
1215 
1216 	ASSERT_RTNL();
1217 
1218 	/* If device failover is pending or we are about to reset, just set
1219 	 * device state and return. Device operation will be handled by reset
1220 	 * routine.
1221 	 *
1222 	 * It should be safe to overwrite the adapter->state here. Since
1223 	 * we hold the rtnl, either the reset has not actually started or
1224 	 * the rtnl got dropped during the set_link_state() in do_reset().
1225 	 * In the former case, no one else is changing the state (again we
1226 	 * have the rtnl) and in the latter case, do_reset() will detect and
1227 	 * honor our setting below.
1228 	 */
1229 	if (adapter->failover_pending || (test_bit(0, &adapter->resetting))) {
1230 		netdev_dbg(netdev, "[S:%s FOP:%d] Resetting, deferring open\n",
1231 			   adapter_state_to_string(adapter->state),
1232 			   adapter->failover_pending);
1233 		adapter->state = VNIC_OPEN;
1234 		rc = 0;
1235 		goto out;
1236 	}
1237 
1238 	if (adapter->state != VNIC_CLOSED) {
1239 		rc = ibmvnic_login(netdev);
1240 		if (rc)
1241 			goto out;
1242 
1243 		rc = init_resources(adapter);
1244 		if (rc) {
1245 			netdev_err(netdev, "failed to initialize resources\n");
1246 			release_resources(adapter);
1247 			goto out;
1248 		}
1249 	}
1250 
1251 	rc = __ibmvnic_open(netdev);
1252 
1253 out:
1254 	/* If open failed and there is a pending failover or in-progress reset,
1255 	 * set device state and return. Device operation will be handled by
1256 	 * reset routine. See also comments above regarding rtnl.
1257 	 */
1258 	if (rc &&
1259 	    (adapter->failover_pending || (test_bit(0, &adapter->resetting)))) {
1260 		adapter->state = VNIC_OPEN;
1261 		rc = 0;
1262 	}
1263 	return rc;
1264 }
1265 
1266 static void clean_rx_pools(struct ibmvnic_adapter *adapter)
1267 {
1268 	struct ibmvnic_rx_pool *rx_pool;
1269 	struct ibmvnic_rx_buff *rx_buff;
1270 	u64 rx_entries;
1271 	int rx_scrqs;
1272 	int i, j;
1273 
1274 	if (!adapter->rx_pool)
1275 		return;
1276 
1277 	rx_scrqs = adapter->num_active_rx_pools;
1278 	rx_entries = adapter->req_rx_add_entries_per_subcrq;
1279 
1280 	/* Free any remaining skbs in the rx buffer pools */
1281 	for (i = 0; i < rx_scrqs; i++) {
1282 		rx_pool = &adapter->rx_pool[i];
1283 		if (!rx_pool || !rx_pool->rx_buff)
1284 			continue;
1285 
1286 		netdev_dbg(adapter->netdev, "Cleaning rx_pool[%d]\n", i);
1287 		for (j = 0; j < rx_entries; j++) {
1288 			rx_buff = &rx_pool->rx_buff[j];
1289 			if (rx_buff && rx_buff->skb) {
1290 				dev_kfree_skb_any(rx_buff->skb);
1291 				rx_buff->skb = NULL;
1292 			}
1293 		}
1294 	}
1295 }
1296 
1297 static void clean_one_tx_pool(struct ibmvnic_adapter *adapter,
1298 			      struct ibmvnic_tx_pool *tx_pool)
1299 {
1300 	struct ibmvnic_tx_buff *tx_buff;
1301 	u64 tx_entries;
1302 	int i;
1303 
1304 	if (!tx_pool || !tx_pool->tx_buff)
1305 		return;
1306 
1307 	tx_entries = tx_pool->num_buffers;
1308 
1309 	for (i = 0; i < tx_entries; i++) {
1310 		tx_buff = &tx_pool->tx_buff[i];
1311 		if (tx_buff && tx_buff->skb) {
1312 			dev_kfree_skb_any(tx_buff->skb);
1313 			tx_buff->skb = NULL;
1314 		}
1315 	}
1316 }
1317 
1318 static void clean_tx_pools(struct ibmvnic_adapter *adapter)
1319 {
1320 	int tx_scrqs;
1321 	int i;
1322 
1323 	if (!adapter->tx_pool || !adapter->tso_pool)
1324 		return;
1325 
1326 	tx_scrqs = adapter->num_active_tx_pools;
1327 
1328 	/* Free any remaining skbs in the tx buffer pools */
1329 	for (i = 0; i < tx_scrqs; i++) {
1330 		netdev_dbg(adapter->netdev, "Cleaning tx_pool[%d]\n", i);
1331 		clean_one_tx_pool(adapter, &adapter->tx_pool[i]);
1332 		clean_one_tx_pool(adapter, &adapter->tso_pool[i]);
1333 	}
1334 }
1335 
1336 static void ibmvnic_disable_irqs(struct ibmvnic_adapter *adapter)
1337 {
1338 	struct net_device *netdev = adapter->netdev;
1339 	int i;
1340 
1341 	if (adapter->tx_scrq) {
1342 		for (i = 0; i < adapter->req_tx_queues; i++)
1343 			if (adapter->tx_scrq[i]->irq) {
1344 				netdev_dbg(netdev,
1345 					   "Disabling tx_scrq[%d] irq\n", i);
1346 				disable_scrq_irq(adapter, adapter->tx_scrq[i]);
1347 				disable_irq(adapter->tx_scrq[i]->irq);
1348 			}
1349 	}
1350 
1351 	if (adapter->rx_scrq) {
1352 		for (i = 0; i < adapter->req_rx_queues; i++) {
1353 			if (adapter->rx_scrq[i]->irq) {
1354 				netdev_dbg(netdev,
1355 					   "Disabling rx_scrq[%d] irq\n", i);
1356 				disable_scrq_irq(adapter, adapter->rx_scrq[i]);
1357 				disable_irq(adapter->rx_scrq[i]->irq);
1358 			}
1359 		}
1360 	}
1361 }
1362 
1363 static void ibmvnic_cleanup(struct net_device *netdev)
1364 {
1365 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1366 
1367 	/* ensure that transmissions are stopped if called by do_reset */
1368 	if (test_bit(0, &adapter->resetting))
1369 		netif_tx_disable(netdev);
1370 	else
1371 		netif_tx_stop_all_queues(netdev);
1372 
1373 	ibmvnic_napi_disable(adapter);
1374 	ibmvnic_disable_irqs(adapter);
1375 
1376 	clean_rx_pools(adapter);
1377 	clean_tx_pools(adapter);
1378 }
1379 
1380 static int __ibmvnic_close(struct net_device *netdev)
1381 {
1382 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1383 	int rc = 0;
1384 
1385 	adapter->state = VNIC_CLOSING;
1386 	rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_DN);
1387 	adapter->state = VNIC_CLOSED;
1388 	return rc;
1389 }
1390 
1391 static int ibmvnic_close(struct net_device *netdev)
1392 {
1393 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1394 	int rc;
1395 
1396 	netdev_dbg(netdev, "[S:%s FOP:%d FRR:%d] Closing\n",
1397 		   adapter_state_to_string(adapter->state),
1398 		   adapter->failover_pending,
1399 		   adapter->force_reset_recovery);
1400 
1401 	/* If device failover is pending, just set device state and return.
1402 	 * Device operation will be handled by reset routine.
1403 	 */
1404 	if (adapter->failover_pending) {
1405 		adapter->state = VNIC_CLOSED;
1406 		return 0;
1407 	}
1408 
1409 	rc = __ibmvnic_close(netdev);
1410 	ibmvnic_cleanup(netdev);
1411 
1412 	return rc;
1413 }
1414 
1415 /**
1416  * build_hdr_data - creates L2/L3/L4 header data buffer
1417  * @hdr_field: bitfield determining needed headers
1418  * @skb: socket buffer
1419  * @hdr_len: array of header lengths
1420  * @hdr_data: buffer to write the header to
1421  *
1422  * Reads hdr_field to determine which headers are needed by firmware.
1423  * Builds a buffer containing these headers.  Saves individual header
1424  * lengths and total buffer length to be used to build descriptors.
1425  */
1426 static int build_hdr_data(u8 hdr_field, struct sk_buff *skb,
1427 			  int *hdr_len, u8 *hdr_data)
1428 {
1429 	int len = 0;
1430 	u8 *hdr;
1431 
1432 	if (skb_vlan_tagged(skb) && !skb_vlan_tag_present(skb))
1433 		hdr_len[0] = sizeof(struct vlan_ethhdr);
1434 	else
1435 		hdr_len[0] = sizeof(struct ethhdr);
1436 
1437 	if (skb->protocol == htons(ETH_P_IP)) {
1438 		hdr_len[1] = ip_hdr(skb)->ihl * 4;
1439 		if (ip_hdr(skb)->protocol == IPPROTO_TCP)
1440 			hdr_len[2] = tcp_hdrlen(skb);
1441 		else if (ip_hdr(skb)->protocol == IPPROTO_UDP)
1442 			hdr_len[2] = sizeof(struct udphdr);
1443 	} else if (skb->protocol == htons(ETH_P_IPV6)) {
1444 		hdr_len[1] = sizeof(struct ipv6hdr);
1445 		if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
1446 			hdr_len[2] = tcp_hdrlen(skb);
1447 		else if (ipv6_hdr(skb)->nexthdr == IPPROTO_UDP)
1448 			hdr_len[2] = sizeof(struct udphdr);
1449 	} else if (skb->protocol == htons(ETH_P_ARP)) {
1450 		hdr_len[1] = arp_hdr_len(skb->dev);
1451 		hdr_len[2] = 0;
1452 	}
1453 
1454 	memset(hdr_data, 0, 120);
1455 	if ((hdr_field >> 6) & 1) {
1456 		hdr = skb_mac_header(skb);
1457 		memcpy(hdr_data, hdr, hdr_len[0]);
1458 		len += hdr_len[0];
1459 	}
1460 
1461 	if ((hdr_field >> 5) & 1) {
1462 		hdr = skb_network_header(skb);
1463 		memcpy(hdr_data + len, hdr, hdr_len[1]);
1464 		len += hdr_len[1];
1465 	}
1466 
1467 	if ((hdr_field >> 4) & 1) {
1468 		hdr = skb_transport_header(skb);
1469 		memcpy(hdr_data + len, hdr, hdr_len[2]);
1470 		len += hdr_len[2];
1471 	}
1472 	return len;
1473 }
1474 
1475 /**
1476  * create_hdr_descs - create header and header extension descriptors
1477  * @hdr_field: bitfield determining needed headers
1478  * @hdr_data: buffer containing header data
1479  * @len: length of data buffer
1480  * @hdr_len: array of individual header lengths
1481  * @scrq_arr: descriptor array
1482  *
1483  * Creates header and, if needed, header extension descriptors and
1484  * places them in a descriptor array, scrq_arr
1485  */
1486 
1487 static int create_hdr_descs(u8 hdr_field, u8 *hdr_data, int len, int *hdr_len,
1488 			    union sub_crq *scrq_arr)
1489 {
1490 	union sub_crq hdr_desc;
1491 	int tmp_len = len;
1492 	int num_descs = 0;
1493 	u8 *data, *cur;
1494 	int tmp;
1495 
1496 	while (tmp_len > 0) {
1497 		cur = hdr_data + len - tmp_len;
1498 
1499 		memset(&hdr_desc, 0, sizeof(hdr_desc));
1500 		if (cur != hdr_data) {
1501 			data = hdr_desc.hdr_ext.data;
1502 			tmp = tmp_len > 29 ? 29 : tmp_len;
1503 			hdr_desc.hdr_ext.first = IBMVNIC_CRQ_CMD;
1504 			hdr_desc.hdr_ext.type = IBMVNIC_HDR_EXT_DESC;
1505 			hdr_desc.hdr_ext.len = tmp;
1506 		} else {
1507 			data = hdr_desc.hdr.data;
1508 			tmp = tmp_len > 24 ? 24 : tmp_len;
1509 			hdr_desc.hdr.first = IBMVNIC_CRQ_CMD;
1510 			hdr_desc.hdr.type = IBMVNIC_HDR_DESC;
1511 			hdr_desc.hdr.len = tmp;
1512 			hdr_desc.hdr.l2_len = (u8)hdr_len[0];
1513 			hdr_desc.hdr.l3_len = cpu_to_be16((u16)hdr_len[1]);
1514 			hdr_desc.hdr.l4_len = (u8)hdr_len[2];
1515 			hdr_desc.hdr.flag = hdr_field << 1;
1516 		}
1517 		memcpy(data, cur, tmp);
1518 		tmp_len -= tmp;
1519 		*scrq_arr = hdr_desc;
1520 		scrq_arr++;
1521 		num_descs++;
1522 	}
1523 
1524 	return num_descs;
1525 }
1526 
1527 /**
1528  * build_hdr_descs_arr - build a header descriptor array
1529  * @skb: tx socket buffer
1530  * @indir_arr: indirect array
1531  * @num_entries: number of descriptors to be sent
1532  * @hdr_field: bit field determining which headers will be sent
1533  *
1534  * This function will build a TX descriptor array with applicable
1535  * L2/L3/L4 packet header descriptors to be sent by send_subcrq_indirect.
1536  */
1537 
1538 static void build_hdr_descs_arr(struct sk_buff *skb,
1539 				union sub_crq *indir_arr,
1540 				int *num_entries, u8 hdr_field)
1541 {
1542 	int hdr_len[3] = {0, 0, 0};
1543 	u8 hdr_data[140] = {0};
1544 	int tot_len;
1545 
1546 	tot_len = build_hdr_data(hdr_field, skb, hdr_len,
1547 				 hdr_data);
1548 	*num_entries += create_hdr_descs(hdr_field, hdr_data, tot_len, hdr_len,
1549 					 indir_arr + 1);
1550 }
1551 
1552 static int ibmvnic_xmit_workarounds(struct sk_buff *skb,
1553 				    struct net_device *netdev)
1554 {
1555 	/* For some backing devices, mishandling of small packets
1556 	 * can result in a loss of connection or TX stall. Device
1557 	 * architects recommend that no packet should be smaller
1558 	 * than the minimum MTU value provided to the driver, so
1559 	 * pad any packets to that length
1560 	 */
1561 	if (skb->len < netdev->min_mtu)
1562 		return skb_put_padto(skb, netdev->min_mtu);
1563 
1564 	return 0;
1565 }
1566 
1567 static void ibmvnic_tx_scrq_clean_buffer(struct ibmvnic_adapter *adapter,
1568 					 struct ibmvnic_sub_crq_queue *tx_scrq)
1569 {
1570 	struct ibmvnic_ind_xmit_queue *ind_bufp;
1571 	struct ibmvnic_tx_buff *tx_buff;
1572 	struct ibmvnic_tx_pool *tx_pool;
1573 	union sub_crq tx_scrq_entry;
1574 	int queue_num;
1575 	int entries;
1576 	int index;
1577 	int i;
1578 
1579 	ind_bufp = &tx_scrq->ind_buf;
1580 	entries = (u64)ind_bufp->index;
1581 	queue_num = tx_scrq->pool_index;
1582 
1583 	for (i = entries - 1; i >= 0; --i) {
1584 		tx_scrq_entry = ind_bufp->indir_arr[i];
1585 		if (tx_scrq_entry.v1.type != IBMVNIC_TX_DESC)
1586 			continue;
1587 		index = be32_to_cpu(tx_scrq_entry.v1.correlator);
1588 		if (index & IBMVNIC_TSO_POOL_MASK) {
1589 			tx_pool = &adapter->tso_pool[queue_num];
1590 			index &= ~IBMVNIC_TSO_POOL_MASK;
1591 		} else {
1592 			tx_pool = &adapter->tx_pool[queue_num];
1593 		}
1594 		tx_pool->free_map[tx_pool->consumer_index] = index;
1595 		tx_pool->consumer_index = tx_pool->consumer_index == 0 ?
1596 					  tx_pool->num_buffers - 1 :
1597 					  tx_pool->consumer_index - 1;
1598 		tx_buff = &tx_pool->tx_buff[index];
1599 		adapter->netdev->stats.tx_packets--;
1600 		adapter->netdev->stats.tx_bytes -= tx_buff->skb->len;
1601 		adapter->tx_stats_buffers[queue_num].packets--;
1602 		adapter->tx_stats_buffers[queue_num].bytes -=
1603 						tx_buff->skb->len;
1604 		dev_kfree_skb_any(tx_buff->skb);
1605 		tx_buff->skb = NULL;
1606 		adapter->netdev->stats.tx_dropped++;
1607 	}
1608 	ind_bufp->index = 0;
1609 	if (atomic_sub_return(entries, &tx_scrq->used) <=
1610 	    (adapter->req_tx_entries_per_subcrq / 2) &&
1611 	    __netif_subqueue_stopped(adapter->netdev, queue_num)) {
1612 		netif_wake_subqueue(adapter->netdev, queue_num);
1613 		netdev_dbg(adapter->netdev, "Started queue %d\n",
1614 			   queue_num);
1615 	}
1616 }
1617 
1618 static int ibmvnic_tx_scrq_flush(struct ibmvnic_adapter *adapter,
1619 				 struct ibmvnic_sub_crq_queue *tx_scrq)
1620 {
1621 	struct ibmvnic_ind_xmit_queue *ind_bufp;
1622 	u64 dma_addr;
1623 	u64 entries;
1624 	u64 handle;
1625 	int rc;
1626 
1627 	ind_bufp = &tx_scrq->ind_buf;
1628 	dma_addr = (u64)ind_bufp->indir_dma;
1629 	entries = (u64)ind_bufp->index;
1630 	handle = tx_scrq->handle;
1631 
1632 	if (!entries)
1633 		return 0;
1634 	rc = send_subcrq_indirect(adapter, handle, dma_addr, entries);
1635 	if (rc)
1636 		ibmvnic_tx_scrq_clean_buffer(adapter, tx_scrq);
1637 	else
1638 		ind_bufp->index = 0;
1639 	return 0;
1640 }
1641 
1642 static netdev_tx_t ibmvnic_xmit(struct sk_buff *skb, struct net_device *netdev)
1643 {
1644 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1645 	int queue_num = skb_get_queue_mapping(skb);
1646 	u8 *hdrs = (u8 *)&adapter->tx_rx_desc_req;
1647 	struct device *dev = &adapter->vdev->dev;
1648 	struct ibmvnic_ind_xmit_queue *ind_bufp;
1649 	struct ibmvnic_tx_buff *tx_buff = NULL;
1650 	struct ibmvnic_sub_crq_queue *tx_scrq;
1651 	struct ibmvnic_tx_pool *tx_pool;
1652 	unsigned int tx_send_failed = 0;
1653 	netdev_tx_t ret = NETDEV_TX_OK;
1654 	unsigned int tx_map_failed = 0;
1655 	union sub_crq indir_arr[16];
1656 	unsigned int tx_dropped = 0;
1657 	unsigned int tx_packets = 0;
1658 	unsigned int tx_bytes = 0;
1659 	dma_addr_t data_dma_addr;
1660 	struct netdev_queue *txq;
1661 	unsigned long lpar_rc;
1662 	union sub_crq tx_crq;
1663 	unsigned int offset;
1664 	int num_entries = 1;
1665 	unsigned char *dst;
1666 	int index = 0;
1667 	u8 proto = 0;
1668 
1669 	tx_scrq = adapter->tx_scrq[queue_num];
1670 	txq = netdev_get_tx_queue(netdev, queue_num);
1671 	ind_bufp = &tx_scrq->ind_buf;
1672 
1673 	if (test_bit(0, &adapter->resetting)) {
1674 		if (!netif_subqueue_stopped(netdev, skb))
1675 			netif_stop_subqueue(netdev, queue_num);
1676 		dev_kfree_skb_any(skb);
1677 
1678 		tx_send_failed++;
1679 		tx_dropped++;
1680 		ret = NETDEV_TX_OK;
1681 		ibmvnic_tx_scrq_flush(adapter, tx_scrq);
1682 		goto out;
1683 	}
1684 
1685 	if (ibmvnic_xmit_workarounds(skb, netdev)) {
1686 		tx_dropped++;
1687 		tx_send_failed++;
1688 		ret = NETDEV_TX_OK;
1689 		ibmvnic_tx_scrq_flush(adapter, tx_scrq);
1690 		goto out;
1691 	}
1692 	if (skb_is_gso(skb))
1693 		tx_pool = &adapter->tso_pool[queue_num];
1694 	else
1695 		tx_pool = &adapter->tx_pool[queue_num];
1696 
1697 	index = tx_pool->free_map[tx_pool->consumer_index];
1698 
1699 	if (index == IBMVNIC_INVALID_MAP) {
1700 		dev_kfree_skb_any(skb);
1701 		tx_send_failed++;
1702 		tx_dropped++;
1703 		ret = NETDEV_TX_OK;
1704 		ibmvnic_tx_scrq_flush(adapter, tx_scrq);
1705 		goto out;
1706 	}
1707 
1708 	tx_pool->free_map[tx_pool->consumer_index] = IBMVNIC_INVALID_MAP;
1709 
1710 	offset = index * tx_pool->buf_size;
1711 	dst = tx_pool->long_term_buff.buff + offset;
1712 	memset(dst, 0, tx_pool->buf_size);
1713 	data_dma_addr = tx_pool->long_term_buff.addr + offset;
1714 
1715 	if (skb_shinfo(skb)->nr_frags) {
1716 		int cur, i;
1717 
1718 		/* Copy the head */
1719 		skb_copy_from_linear_data(skb, dst, skb_headlen(skb));
1720 		cur = skb_headlen(skb);
1721 
1722 		/* Copy the frags */
1723 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1724 			const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1725 
1726 			memcpy(dst + cur, skb_frag_address(frag),
1727 			       skb_frag_size(frag));
1728 			cur += skb_frag_size(frag);
1729 		}
1730 	} else {
1731 		skb_copy_from_linear_data(skb, dst, skb->len);
1732 	}
1733 
1734 	/* post changes to long_term_buff *dst before VIOS accessing it */
1735 	dma_wmb();
1736 
1737 	tx_pool->consumer_index =
1738 	    (tx_pool->consumer_index + 1) % tx_pool->num_buffers;
1739 
1740 	tx_buff = &tx_pool->tx_buff[index];
1741 	tx_buff->skb = skb;
1742 	tx_buff->index = index;
1743 	tx_buff->pool_index = queue_num;
1744 
1745 	memset(&tx_crq, 0, sizeof(tx_crq));
1746 	tx_crq.v1.first = IBMVNIC_CRQ_CMD;
1747 	tx_crq.v1.type = IBMVNIC_TX_DESC;
1748 	tx_crq.v1.n_crq_elem = 1;
1749 	tx_crq.v1.n_sge = 1;
1750 	tx_crq.v1.flags1 = IBMVNIC_TX_COMP_NEEDED;
1751 
1752 	if (skb_is_gso(skb))
1753 		tx_crq.v1.correlator =
1754 			cpu_to_be32(index | IBMVNIC_TSO_POOL_MASK);
1755 	else
1756 		tx_crq.v1.correlator = cpu_to_be32(index);
1757 	tx_crq.v1.dma_reg = cpu_to_be16(tx_pool->long_term_buff.map_id);
1758 	tx_crq.v1.sge_len = cpu_to_be32(skb->len);
1759 	tx_crq.v1.ioba = cpu_to_be64(data_dma_addr);
1760 
1761 	if (adapter->vlan_header_insertion && skb_vlan_tag_present(skb)) {
1762 		tx_crq.v1.flags2 |= IBMVNIC_TX_VLAN_INSERT;
1763 		tx_crq.v1.vlan_id = cpu_to_be16(skb->vlan_tci);
1764 	}
1765 
1766 	if (skb->protocol == htons(ETH_P_IP)) {
1767 		tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_IPV4;
1768 		proto = ip_hdr(skb)->protocol;
1769 	} else if (skb->protocol == htons(ETH_P_IPV6)) {
1770 		tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_IPV6;
1771 		proto = ipv6_hdr(skb)->nexthdr;
1772 	}
1773 
1774 	if (proto == IPPROTO_TCP)
1775 		tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_TCP;
1776 	else if (proto == IPPROTO_UDP)
1777 		tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_UDP;
1778 
1779 	if (skb->ip_summed == CHECKSUM_PARTIAL) {
1780 		tx_crq.v1.flags1 |= IBMVNIC_TX_CHKSUM_OFFLOAD;
1781 		hdrs += 2;
1782 	}
1783 	if (skb_is_gso(skb)) {
1784 		tx_crq.v1.flags1 |= IBMVNIC_TX_LSO;
1785 		tx_crq.v1.mss = cpu_to_be16(skb_shinfo(skb)->gso_size);
1786 		hdrs += 2;
1787 	}
1788 
1789 	if ((*hdrs >> 7) & 1)
1790 		build_hdr_descs_arr(skb, indir_arr, &num_entries, *hdrs);
1791 
1792 	tx_crq.v1.n_crq_elem = num_entries;
1793 	tx_buff->num_entries = num_entries;
1794 	/* flush buffer if current entry can not fit */
1795 	if (num_entries + ind_bufp->index > IBMVNIC_MAX_IND_DESCS) {
1796 		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq);
1797 		if (lpar_rc != H_SUCCESS)
1798 			goto tx_flush_err;
1799 	}
1800 
1801 	indir_arr[0] = tx_crq;
1802 	memcpy(&ind_bufp->indir_arr[ind_bufp->index], &indir_arr[0],
1803 	       num_entries * sizeof(struct ibmvnic_generic_scrq));
1804 	ind_bufp->index += num_entries;
1805 	if (__netdev_tx_sent_queue(txq, skb->len,
1806 				   netdev_xmit_more() &&
1807 				   ind_bufp->index < IBMVNIC_MAX_IND_DESCS)) {
1808 		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq);
1809 		if (lpar_rc != H_SUCCESS)
1810 			goto tx_err;
1811 	}
1812 
1813 	if (atomic_add_return(num_entries, &tx_scrq->used)
1814 					>= adapter->req_tx_entries_per_subcrq) {
1815 		netdev_dbg(netdev, "Stopping queue %d\n", queue_num);
1816 		netif_stop_subqueue(netdev, queue_num);
1817 	}
1818 
1819 	tx_packets++;
1820 	tx_bytes += skb->len;
1821 	txq->trans_start = jiffies;
1822 	ret = NETDEV_TX_OK;
1823 	goto out;
1824 
1825 tx_flush_err:
1826 	dev_kfree_skb_any(skb);
1827 	tx_buff->skb = NULL;
1828 	tx_pool->consumer_index = tx_pool->consumer_index == 0 ?
1829 				  tx_pool->num_buffers - 1 :
1830 				  tx_pool->consumer_index - 1;
1831 	tx_dropped++;
1832 tx_err:
1833 	if (lpar_rc != H_CLOSED && lpar_rc != H_PARAMETER)
1834 		dev_err_ratelimited(dev, "tx: send failed\n");
1835 
1836 	if (lpar_rc == H_CLOSED || adapter->failover_pending) {
1837 		/* Disable TX and report carrier off if queue is closed
1838 		 * or pending failover.
1839 		 * Firmware guarantees that a signal will be sent to the
1840 		 * driver, triggering a reset or some other action.
1841 		 */
1842 		netif_tx_stop_all_queues(netdev);
1843 		netif_carrier_off(netdev);
1844 	}
1845 out:
1846 	netdev->stats.tx_dropped += tx_dropped;
1847 	netdev->stats.tx_bytes += tx_bytes;
1848 	netdev->stats.tx_packets += tx_packets;
1849 	adapter->tx_send_failed += tx_send_failed;
1850 	adapter->tx_map_failed += tx_map_failed;
1851 	adapter->tx_stats_buffers[queue_num].packets += tx_packets;
1852 	adapter->tx_stats_buffers[queue_num].bytes += tx_bytes;
1853 	adapter->tx_stats_buffers[queue_num].dropped_packets += tx_dropped;
1854 
1855 	return ret;
1856 }
1857 
1858 static void ibmvnic_set_multi(struct net_device *netdev)
1859 {
1860 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1861 	struct netdev_hw_addr *ha;
1862 	union ibmvnic_crq crq;
1863 
1864 	memset(&crq, 0, sizeof(crq));
1865 	crq.request_capability.first = IBMVNIC_CRQ_CMD;
1866 	crq.request_capability.cmd = REQUEST_CAPABILITY;
1867 
1868 	if (netdev->flags & IFF_PROMISC) {
1869 		if (!adapter->promisc_supported)
1870 			return;
1871 	} else {
1872 		if (netdev->flags & IFF_ALLMULTI) {
1873 			/* Accept all multicast */
1874 			memset(&crq, 0, sizeof(crq));
1875 			crq.multicast_ctrl.first = IBMVNIC_CRQ_CMD;
1876 			crq.multicast_ctrl.cmd = MULTICAST_CTRL;
1877 			crq.multicast_ctrl.flags = IBMVNIC_ENABLE_ALL;
1878 			ibmvnic_send_crq(adapter, &crq);
1879 		} else if (netdev_mc_empty(netdev)) {
1880 			/* Reject all multicast */
1881 			memset(&crq, 0, sizeof(crq));
1882 			crq.multicast_ctrl.first = IBMVNIC_CRQ_CMD;
1883 			crq.multicast_ctrl.cmd = MULTICAST_CTRL;
1884 			crq.multicast_ctrl.flags = IBMVNIC_DISABLE_ALL;
1885 			ibmvnic_send_crq(adapter, &crq);
1886 		} else {
1887 			/* Accept one or more multicast(s) */
1888 			netdev_for_each_mc_addr(ha, netdev) {
1889 				memset(&crq, 0, sizeof(crq));
1890 				crq.multicast_ctrl.first = IBMVNIC_CRQ_CMD;
1891 				crq.multicast_ctrl.cmd = MULTICAST_CTRL;
1892 				crq.multicast_ctrl.flags = IBMVNIC_ENABLE_MC;
1893 				ether_addr_copy(&crq.multicast_ctrl.mac_addr[0],
1894 						ha->addr);
1895 				ibmvnic_send_crq(adapter, &crq);
1896 			}
1897 		}
1898 	}
1899 }
1900 
1901 static int __ibmvnic_set_mac(struct net_device *netdev, u8 *dev_addr)
1902 {
1903 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1904 	union ibmvnic_crq crq;
1905 	int rc;
1906 
1907 	if (!is_valid_ether_addr(dev_addr)) {
1908 		rc = -EADDRNOTAVAIL;
1909 		goto err;
1910 	}
1911 
1912 	memset(&crq, 0, sizeof(crq));
1913 	crq.change_mac_addr.first = IBMVNIC_CRQ_CMD;
1914 	crq.change_mac_addr.cmd = CHANGE_MAC_ADDR;
1915 	ether_addr_copy(&crq.change_mac_addr.mac_addr[0], dev_addr);
1916 
1917 	mutex_lock(&adapter->fw_lock);
1918 	adapter->fw_done_rc = 0;
1919 	reinit_completion(&adapter->fw_done);
1920 
1921 	rc = ibmvnic_send_crq(adapter, &crq);
1922 	if (rc) {
1923 		rc = -EIO;
1924 		mutex_unlock(&adapter->fw_lock);
1925 		goto err;
1926 	}
1927 
1928 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);
1929 	/* netdev->dev_addr is changed in handle_change_mac_rsp function */
1930 	if (rc || adapter->fw_done_rc) {
1931 		rc = -EIO;
1932 		mutex_unlock(&adapter->fw_lock);
1933 		goto err;
1934 	}
1935 	mutex_unlock(&adapter->fw_lock);
1936 	return 0;
1937 err:
1938 	ether_addr_copy(adapter->mac_addr, netdev->dev_addr);
1939 	return rc;
1940 }
1941 
1942 static int ibmvnic_set_mac(struct net_device *netdev, void *p)
1943 {
1944 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1945 	struct sockaddr *addr = p;
1946 	int rc;
1947 
1948 	rc = 0;
1949 	if (!is_valid_ether_addr(addr->sa_data))
1950 		return -EADDRNOTAVAIL;
1951 
1952 	ether_addr_copy(adapter->mac_addr, addr->sa_data);
1953 	if (adapter->state != VNIC_PROBED)
1954 		rc = __ibmvnic_set_mac(netdev, addr->sa_data);
1955 
1956 	return rc;
1957 }
1958 
1959 static const char *reset_reason_to_string(enum ibmvnic_reset_reason reason)
1960 {
1961 	switch (reason) {
1962 	case VNIC_RESET_FAILOVER:
1963 		return "FAILOVER";
1964 	case VNIC_RESET_MOBILITY:
1965 		return "MOBILITY";
1966 	case VNIC_RESET_FATAL:
1967 		return "FATAL";
1968 	case VNIC_RESET_NON_FATAL:
1969 		return "NON_FATAL";
1970 	case VNIC_RESET_TIMEOUT:
1971 		return "TIMEOUT";
1972 	case VNIC_RESET_CHANGE_PARAM:
1973 		return "CHANGE_PARAM";
1974 	case VNIC_RESET_PASSIVE_INIT:
1975 		return "PASSIVE_INIT";
1976 	}
1977 	return "UNKNOWN";
1978 }
1979 
1980 /*
1981  * do_reset returns zero if we are able to keep processing reset events, or
1982  * non-zero if we hit a fatal error and must halt.
1983  */
1984 static int do_reset(struct ibmvnic_adapter *adapter,
1985 		    struct ibmvnic_rwi *rwi, u32 reset_state)
1986 {
1987 	u64 old_num_rx_queues, old_num_tx_queues;
1988 	u64 old_num_rx_slots, old_num_tx_slots;
1989 	struct net_device *netdev = adapter->netdev;
1990 	int rc;
1991 
1992 	netdev_dbg(adapter->netdev,
1993 		   "[S:%s FOP:%d] Reset reason: %s, reset_state: %s\n",
1994 		   adapter_state_to_string(adapter->state),
1995 		   adapter->failover_pending,
1996 		   reset_reason_to_string(rwi->reset_reason),
1997 		   adapter_state_to_string(reset_state));
1998 
1999 	adapter->reset_reason = rwi->reset_reason;
2000 	/* requestor of VNIC_RESET_CHANGE_PARAM already has the rtnl lock */
2001 	if (!(adapter->reset_reason == VNIC_RESET_CHANGE_PARAM))
2002 		rtnl_lock();
2003 
2004 	/* Now that we have the rtnl lock, clear any pending failover.
2005 	 * This will ensure ibmvnic_open() has either completed or will
2006 	 * block until failover is complete.
2007 	 */
2008 	if (rwi->reset_reason == VNIC_RESET_FAILOVER)
2009 		adapter->failover_pending = false;
2010 
2011 	/* read the state and check (again) after getting rtnl */
2012 	reset_state = adapter->state;
2013 
2014 	if (reset_state == VNIC_REMOVING || reset_state == VNIC_REMOVED) {
2015 		rc = -EBUSY;
2016 		goto out;
2017 	}
2018 
2019 	netif_carrier_off(netdev);
2020 
2021 	old_num_rx_queues = adapter->req_rx_queues;
2022 	old_num_tx_queues = adapter->req_tx_queues;
2023 	old_num_rx_slots = adapter->req_rx_add_entries_per_subcrq;
2024 	old_num_tx_slots = adapter->req_tx_entries_per_subcrq;
2025 
2026 	ibmvnic_cleanup(netdev);
2027 
2028 	if (reset_state == VNIC_OPEN &&
2029 	    adapter->reset_reason != VNIC_RESET_MOBILITY &&
2030 	    adapter->reset_reason != VNIC_RESET_FAILOVER) {
2031 		if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {
2032 			rc = __ibmvnic_close(netdev);
2033 			if (rc)
2034 				goto out;
2035 		} else {
2036 			adapter->state = VNIC_CLOSING;
2037 
2038 			/* Release the RTNL lock before link state change and
2039 			 * re-acquire after the link state change to allow
2040 			 * linkwatch_event to grab the RTNL lock and run during
2041 			 * a reset.
2042 			 */
2043 			rtnl_unlock();
2044 			rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_DN);
2045 			rtnl_lock();
2046 			if (rc)
2047 				goto out;
2048 
2049 			if (adapter->state == VNIC_OPEN) {
2050 				/* When we dropped rtnl, ibmvnic_open() got
2051 				 * it and noticed that we are resetting and
2052 				 * set the adapter state to OPEN. Update our
2053 				 * new "target" state, and resume the reset
2054 				 * from VNIC_CLOSING state.
2055 				 */
2056 				netdev_dbg(netdev,
2057 					   "Open changed state from %s, updating.\n",
2058 					   adapter_state_to_string(reset_state));
2059 				reset_state = VNIC_OPEN;
2060 				adapter->state = VNIC_CLOSING;
2061 			}
2062 
2063 			if (adapter->state != VNIC_CLOSING) {
2064 				/* If someone else changed the adapter state
2065 				 * when we dropped the rtnl, fail the reset
2066 				 */
2067 				rc = -1;
2068 				goto out;
2069 			}
2070 			adapter->state = VNIC_CLOSED;
2071 		}
2072 	}
2073 
2074 	if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {
2075 		release_resources(adapter);
2076 		release_sub_crqs(adapter, 1);
2077 		release_crq_queue(adapter);
2078 	}
2079 
2080 	if (adapter->reset_reason != VNIC_RESET_NON_FATAL) {
2081 		/* remove the closed state so when we call open it appears
2082 		 * we are coming from the probed state.
2083 		 */
2084 		adapter->state = VNIC_PROBED;
2085 
2086 		if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {
2087 			rc = init_crq_queue(adapter);
2088 		} else if (adapter->reset_reason == VNIC_RESET_MOBILITY) {
2089 			rc = ibmvnic_reenable_crq_queue(adapter);
2090 			release_sub_crqs(adapter, 1);
2091 		} else {
2092 			rc = ibmvnic_reset_crq(adapter);
2093 			if (rc == H_CLOSED || rc == H_SUCCESS) {
2094 				rc = vio_enable_interrupts(adapter->vdev);
2095 				if (rc)
2096 					netdev_err(adapter->netdev,
2097 						   "Reset failed to enable interrupts. rc=%d\n",
2098 						   rc);
2099 			}
2100 		}
2101 
2102 		if (rc) {
2103 			netdev_err(adapter->netdev,
2104 				   "Reset couldn't initialize crq. rc=%d\n", rc);
2105 			goto out;
2106 		}
2107 
2108 		rc = ibmvnic_reset_init(adapter, true);
2109 		if (rc) {
2110 			rc = IBMVNIC_INIT_FAILED;
2111 			goto out;
2112 		}
2113 
2114 		/* If the adapter was in PROBE or DOWN state prior to the reset,
2115 		 * exit here.
2116 		 */
2117 		if (reset_state == VNIC_PROBED || reset_state == VNIC_DOWN) {
2118 			rc = 0;
2119 			goto out;
2120 		}
2121 
2122 		rc = ibmvnic_login(netdev);
2123 		if (rc)
2124 			goto out;
2125 
2126 		if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {
2127 			rc = init_resources(adapter);
2128 			if (rc)
2129 				goto out;
2130 		} else if (adapter->req_rx_queues != old_num_rx_queues ||
2131 		    adapter->req_tx_queues != old_num_tx_queues ||
2132 		    adapter->req_rx_add_entries_per_subcrq !=
2133 		    old_num_rx_slots ||
2134 		    adapter->req_tx_entries_per_subcrq !=
2135 		    old_num_tx_slots ||
2136 		    !adapter->rx_pool ||
2137 		    !adapter->tso_pool ||
2138 		    !adapter->tx_pool) {
2139 			release_rx_pools(adapter);
2140 			release_tx_pools(adapter);
2141 			release_napi(adapter);
2142 			release_vpd_data(adapter);
2143 
2144 			rc = init_resources(adapter);
2145 			if (rc)
2146 				goto out;
2147 
2148 		} else {
2149 			rc = reset_tx_pools(adapter);
2150 			if (rc) {
2151 				netdev_dbg(adapter->netdev, "reset tx pools failed (%d)\n",
2152 					   rc);
2153 				goto out;
2154 			}
2155 
2156 			rc = reset_rx_pools(adapter);
2157 			if (rc) {
2158 				netdev_dbg(adapter->netdev, "reset rx pools failed (%d)\n",
2159 					   rc);
2160 				goto out;
2161 			}
2162 		}
2163 		ibmvnic_disable_irqs(adapter);
2164 	}
2165 	adapter->state = VNIC_CLOSED;
2166 
2167 	if (reset_state == VNIC_CLOSED) {
2168 		rc = 0;
2169 		goto out;
2170 	}
2171 
2172 	rc = __ibmvnic_open(netdev);
2173 	if (rc) {
2174 		rc = IBMVNIC_OPEN_FAILED;
2175 		goto out;
2176 	}
2177 
2178 	/* refresh device's multicast list */
2179 	ibmvnic_set_multi(netdev);
2180 
2181 	if (adapter->reset_reason == VNIC_RESET_FAILOVER ||
2182 	    adapter->reset_reason == VNIC_RESET_MOBILITY)
2183 		__netdev_notify_peers(netdev);
2184 
2185 	rc = 0;
2186 
2187 out:
2188 	/* restore the adapter state if reset failed */
2189 	if (rc)
2190 		adapter->state = reset_state;
2191 	/* requestor of VNIC_RESET_CHANGE_PARAM should still hold the rtnl lock */
2192 	if (!(adapter->reset_reason == VNIC_RESET_CHANGE_PARAM))
2193 		rtnl_unlock();
2194 
2195 	netdev_dbg(adapter->netdev, "[S:%s FOP:%d] Reset done, rc %d\n",
2196 		   adapter_state_to_string(adapter->state),
2197 		   adapter->failover_pending, rc);
2198 	return rc;
2199 }
2200 
2201 static int do_hard_reset(struct ibmvnic_adapter *adapter,
2202 			 struct ibmvnic_rwi *rwi, u32 reset_state)
2203 {
2204 	struct net_device *netdev = adapter->netdev;
2205 	int rc;
2206 
2207 	netdev_dbg(adapter->netdev, "Hard resetting driver (%s)\n",
2208 		   reset_reason_to_string(rwi->reset_reason));
2209 
2210 	/* read the state and check (again) after getting rtnl */
2211 	reset_state = adapter->state;
2212 
2213 	if (reset_state == VNIC_REMOVING || reset_state == VNIC_REMOVED) {
2214 		rc = -EBUSY;
2215 		goto out;
2216 	}
2217 
2218 	netif_carrier_off(netdev);
2219 	adapter->reset_reason = rwi->reset_reason;
2220 
2221 	ibmvnic_cleanup(netdev);
2222 	release_resources(adapter);
2223 	release_sub_crqs(adapter, 0);
2224 	release_crq_queue(adapter);
2225 
2226 	/* remove the closed state so when we call open it appears
2227 	 * we are coming from the probed state.
2228 	 */
2229 	adapter->state = VNIC_PROBED;
2230 
2231 	reinit_completion(&adapter->init_done);
2232 	rc = init_crq_queue(adapter);
2233 	if (rc) {
2234 		netdev_err(adapter->netdev,
2235 			   "Couldn't initialize crq. rc=%d\n", rc);
2236 		goto out;
2237 	}
2238 
2239 	rc = ibmvnic_reset_init(adapter, false);
2240 	if (rc)
2241 		goto out;
2242 
2243 	/* If the adapter was in PROBE or DOWN state prior to the reset,
2244 	 * exit here.
2245 	 */
2246 	if (reset_state == VNIC_PROBED || reset_state == VNIC_DOWN)
2247 		goto out;
2248 
2249 	rc = ibmvnic_login(netdev);
2250 	if (rc)
2251 		goto out;
2252 
2253 	rc = init_resources(adapter);
2254 	if (rc)
2255 		goto out;
2256 
2257 	ibmvnic_disable_irqs(adapter);
2258 	adapter->state = VNIC_CLOSED;
2259 
2260 	if (reset_state == VNIC_CLOSED)
2261 		goto out;
2262 
2263 	rc = __ibmvnic_open(netdev);
2264 	if (rc) {
2265 		rc = IBMVNIC_OPEN_FAILED;
2266 		goto out;
2267 	}
2268 
2269 	__netdev_notify_peers(netdev);
2270 out:
2271 	/* restore adapter state if reset failed */
2272 	if (rc)
2273 		adapter->state = reset_state;
2274 	netdev_dbg(adapter->netdev, "[S:%s FOP:%d] Hard reset done, rc %d\n",
2275 		   adapter_state_to_string(adapter->state),
2276 		   adapter->failover_pending, rc);
2277 	return rc;
2278 }
2279 
2280 static struct ibmvnic_rwi *get_next_rwi(struct ibmvnic_adapter *adapter)
2281 {
2282 	struct ibmvnic_rwi *rwi;
2283 	unsigned long flags;
2284 
2285 	spin_lock_irqsave(&adapter->rwi_lock, flags);
2286 
2287 	if (!list_empty(&adapter->rwi_list)) {
2288 		rwi = list_first_entry(&adapter->rwi_list, struct ibmvnic_rwi,
2289 				       list);
2290 		list_del(&rwi->list);
2291 	} else {
2292 		rwi = NULL;
2293 	}
2294 
2295 	spin_unlock_irqrestore(&adapter->rwi_lock, flags);
2296 	return rwi;
2297 }
2298 
2299 /**
2300  * do_passive_init - complete probing when partner device is detected.
2301  * @adapter: ibmvnic_adapter struct
2302  *
2303  * If the ibmvnic device does not have a partner device to communicate with at boot
2304  * and that partner device comes online at a later time, this function is called
2305  * to complete the initialization process of ibmvnic device.
2306  * Caller is expected to hold rtnl_lock().
2307  *
2308  * Returns non-zero if sub-CRQs are not initialized properly leaving the device
2309  * in the down state.
2310  * Returns 0 upon success and the device is in PROBED state.
2311  */
2312 
2313 static int do_passive_init(struct ibmvnic_adapter *adapter)
2314 {
2315 	unsigned long timeout = msecs_to_jiffies(30000);
2316 	struct net_device *netdev = adapter->netdev;
2317 	struct device *dev = &adapter->vdev->dev;
2318 	int rc;
2319 
2320 	netdev_dbg(netdev, "Partner device found, probing.\n");
2321 
2322 	adapter->state = VNIC_PROBING;
2323 	reinit_completion(&adapter->init_done);
2324 	adapter->init_done_rc = 0;
2325 	adapter->crq.active = true;
2326 
2327 	rc = send_crq_init_complete(adapter);
2328 	if (rc)
2329 		goto out;
2330 
2331 	rc = send_version_xchg(adapter);
2332 	if (rc)
2333 		netdev_dbg(adapter->netdev, "send_version_xchg failed, rc=%d\n", rc);
2334 
2335 	if (!wait_for_completion_timeout(&adapter->init_done, timeout)) {
2336 		dev_err(dev, "Initialization sequence timed out\n");
2337 		rc = -ETIMEDOUT;
2338 		goto out;
2339 	}
2340 
2341 	rc = init_sub_crqs(adapter);
2342 	if (rc) {
2343 		dev_err(dev, "Initialization of sub crqs failed, rc=%d\n", rc);
2344 		goto out;
2345 	}
2346 
2347 	rc = init_sub_crq_irqs(adapter);
2348 	if (rc) {
2349 		dev_err(dev, "Failed to initialize sub crq irqs\n, rc=%d", rc);
2350 		goto init_failed;
2351 	}
2352 
2353 	netdev->mtu = adapter->req_mtu - ETH_HLEN;
2354 	netdev->min_mtu = adapter->min_mtu - ETH_HLEN;
2355 	netdev->max_mtu = adapter->max_mtu - ETH_HLEN;
2356 
2357 	adapter->state = VNIC_PROBED;
2358 	netdev_dbg(netdev, "Probed successfully. Waiting for signal from partner device.\n");
2359 
2360 	return 0;
2361 
2362 init_failed:
2363 	release_sub_crqs(adapter, 1);
2364 out:
2365 	adapter->state = VNIC_DOWN;
2366 	return rc;
2367 }
2368 
2369 static void __ibmvnic_reset(struct work_struct *work)
2370 {
2371 	struct ibmvnic_rwi *rwi;
2372 	struct ibmvnic_adapter *adapter;
2373 	bool saved_state = false;
2374 	unsigned long flags;
2375 	u32 reset_state;
2376 	int rc = 0;
2377 
2378 	adapter = container_of(work, struct ibmvnic_adapter, ibmvnic_reset);
2379 
2380 	if (test_and_set_bit_lock(0, &adapter->resetting)) {
2381 		queue_delayed_work(system_long_wq,
2382 				   &adapter->ibmvnic_delayed_reset,
2383 				   IBMVNIC_RESET_DELAY);
2384 		return;
2385 	}
2386 
2387 	rwi = get_next_rwi(adapter);
2388 	while (rwi) {
2389 		spin_lock_irqsave(&adapter->state_lock, flags);
2390 
2391 		if (adapter->state == VNIC_REMOVING ||
2392 		    adapter->state == VNIC_REMOVED) {
2393 			spin_unlock_irqrestore(&adapter->state_lock, flags);
2394 			kfree(rwi);
2395 			rc = EBUSY;
2396 			break;
2397 		}
2398 
2399 		if (!saved_state) {
2400 			reset_state = adapter->state;
2401 			saved_state = true;
2402 		}
2403 		spin_unlock_irqrestore(&adapter->state_lock, flags);
2404 
2405 		if (rwi->reset_reason == VNIC_RESET_PASSIVE_INIT) {
2406 			rtnl_lock();
2407 			rc = do_passive_init(adapter);
2408 			rtnl_unlock();
2409 			if (!rc)
2410 				netif_carrier_on(adapter->netdev);
2411 		} else if (adapter->force_reset_recovery) {
2412 			/* Since we are doing a hard reset now, clear the
2413 			 * failover_pending flag so we don't ignore any
2414 			 * future MOBILITY or other resets.
2415 			 */
2416 			adapter->failover_pending = false;
2417 
2418 			/* Transport event occurred during previous reset */
2419 			if (adapter->wait_for_reset) {
2420 				/* Previous was CHANGE_PARAM; caller locked */
2421 				adapter->force_reset_recovery = false;
2422 				rc = do_hard_reset(adapter, rwi, reset_state);
2423 			} else {
2424 				rtnl_lock();
2425 				adapter->force_reset_recovery = false;
2426 				rc = do_hard_reset(adapter, rwi, reset_state);
2427 				rtnl_unlock();
2428 			}
2429 			if (rc) {
2430 				/* give backing device time to settle down */
2431 				netdev_dbg(adapter->netdev,
2432 					   "[S:%s] Hard reset failed, waiting 60 secs\n",
2433 					   adapter_state_to_string(adapter->state));
2434 				set_current_state(TASK_UNINTERRUPTIBLE);
2435 				schedule_timeout(60 * HZ);
2436 			}
2437 		} else {
2438 			rc = do_reset(adapter, rwi, reset_state);
2439 		}
2440 		kfree(rwi);
2441 		adapter->last_reset_time = jiffies;
2442 
2443 		if (rc)
2444 			netdev_dbg(adapter->netdev, "Reset failed, rc=%d\n", rc);
2445 
2446 		rwi = get_next_rwi(adapter);
2447 
2448 		if (rwi && (rwi->reset_reason == VNIC_RESET_FAILOVER ||
2449 			    rwi->reset_reason == VNIC_RESET_MOBILITY))
2450 			adapter->force_reset_recovery = true;
2451 	}
2452 
2453 	if (adapter->wait_for_reset) {
2454 		adapter->reset_done_rc = rc;
2455 		complete(&adapter->reset_done);
2456 	}
2457 
2458 	clear_bit_unlock(0, &adapter->resetting);
2459 
2460 	netdev_dbg(adapter->netdev,
2461 		   "[S:%s FRR:%d WFR:%d] Done processing resets\n",
2462 		   adapter_state_to_string(adapter->state),
2463 		   adapter->force_reset_recovery,
2464 		   adapter->wait_for_reset);
2465 }
2466 
2467 static void __ibmvnic_delayed_reset(struct work_struct *work)
2468 {
2469 	struct ibmvnic_adapter *adapter;
2470 
2471 	adapter = container_of(work, struct ibmvnic_adapter,
2472 			       ibmvnic_delayed_reset.work);
2473 	__ibmvnic_reset(&adapter->ibmvnic_reset);
2474 }
2475 
2476 static int ibmvnic_reset(struct ibmvnic_adapter *adapter,
2477 			 enum ibmvnic_reset_reason reason)
2478 {
2479 	struct list_head *entry, *tmp_entry;
2480 	struct ibmvnic_rwi *rwi, *tmp;
2481 	struct net_device *netdev = adapter->netdev;
2482 	unsigned long flags;
2483 	int ret;
2484 
2485 	spin_lock_irqsave(&adapter->rwi_lock, flags);
2486 
2487 	/* If failover is pending don't schedule any other reset.
2488 	 * Instead let the failover complete. If there is already a
2489 	 * a failover reset scheduled, we will detect and drop the
2490 	 * duplicate reset when walking the ->rwi_list below.
2491 	 */
2492 	if (adapter->state == VNIC_REMOVING ||
2493 	    adapter->state == VNIC_REMOVED ||
2494 	    (adapter->failover_pending && reason != VNIC_RESET_FAILOVER)) {
2495 		ret = EBUSY;
2496 		netdev_dbg(netdev, "Adapter removing or pending failover, skipping reset\n");
2497 		goto err;
2498 	}
2499 
2500 	if (adapter->state == VNIC_PROBING) {
2501 		netdev_warn(netdev, "Adapter reset during probe\n");
2502 		adapter->init_done_rc = EAGAIN;
2503 		ret = EAGAIN;
2504 		goto err;
2505 	}
2506 
2507 	list_for_each_entry(tmp, &adapter->rwi_list, list) {
2508 		if (tmp->reset_reason == reason) {
2509 			netdev_dbg(netdev, "Skipping matching reset, reason=%s\n",
2510 				   reset_reason_to_string(reason));
2511 			ret = EBUSY;
2512 			goto err;
2513 		}
2514 	}
2515 
2516 	rwi = kzalloc(sizeof(*rwi), GFP_ATOMIC);
2517 	if (!rwi) {
2518 		ret = ENOMEM;
2519 		goto err;
2520 	}
2521 	/* if we just received a transport event,
2522 	 * flush reset queue and process this reset
2523 	 */
2524 	if (adapter->force_reset_recovery && !list_empty(&adapter->rwi_list)) {
2525 		list_for_each_safe(entry, tmp_entry, &adapter->rwi_list)
2526 			list_del(entry);
2527 	}
2528 	rwi->reset_reason = reason;
2529 	list_add_tail(&rwi->list, &adapter->rwi_list);
2530 	netdev_dbg(adapter->netdev, "Scheduling reset (reason %s)\n",
2531 		   reset_reason_to_string(reason));
2532 	queue_work(system_long_wq, &adapter->ibmvnic_reset);
2533 
2534 	ret = 0;
2535 err:
2536 	/* ibmvnic_close() below can block, so drop the lock first */
2537 	spin_unlock_irqrestore(&adapter->rwi_lock, flags);
2538 
2539 	if (ret == ENOMEM)
2540 		ibmvnic_close(netdev);
2541 
2542 	return -ret;
2543 }
2544 
2545 static void ibmvnic_tx_timeout(struct net_device *dev, unsigned int txqueue)
2546 {
2547 	struct ibmvnic_adapter *adapter = netdev_priv(dev);
2548 
2549 	if (test_bit(0, &adapter->resetting)) {
2550 		netdev_err(adapter->netdev,
2551 			   "Adapter is resetting, skip timeout reset\n");
2552 		return;
2553 	}
2554 	/* No queuing up reset until at least 5 seconds (default watchdog val)
2555 	 * after last reset
2556 	 */
2557 	if (time_before(jiffies, (adapter->last_reset_time + dev->watchdog_timeo))) {
2558 		netdev_dbg(dev, "Not yet time to tx timeout.\n");
2559 		return;
2560 	}
2561 	ibmvnic_reset(adapter, VNIC_RESET_TIMEOUT);
2562 }
2563 
2564 static void remove_buff_from_pool(struct ibmvnic_adapter *adapter,
2565 				  struct ibmvnic_rx_buff *rx_buff)
2566 {
2567 	struct ibmvnic_rx_pool *pool = &adapter->rx_pool[rx_buff->pool_index];
2568 
2569 	rx_buff->skb = NULL;
2570 
2571 	pool->free_map[pool->next_alloc] = (int)(rx_buff - pool->rx_buff);
2572 	pool->next_alloc = (pool->next_alloc + 1) % pool->size;
2573 
2574 	atomic_dec(&pool->available);
2575 }
2576 
2577 static int ibmvnic_poll(struct napi_struct *napi, int budget)
2578 {
2579 	struct ibmvnic_sub_crq_queue *rx_scrq;
2580 	struct ibmvnic_adapter *adapter;
2581 	struct net_device *netdev;
2582 	int frames_processed;
2583 	int scrq_num;
2584 
2585 	netdev = napi->dev;
2586 	adapter = netdev_priv(netdev);
2587 	scrq_num = (int)(napi - adapter->napi);
2588 	frames_processed = 0;
2589 	rx_scrq = adapter->rx_scrq[scrq_num];
2590 
2591 restart_poll:
2592 	while (frames_processed < budget) {
2593 		struct sk_buff *skb;
2594 		struct ibmvnic_rx_buff *rx_buff;
2595 		union sub_crq *next;
2596 		u32 length;
2597 		u16 offset;
2598 		u8 flags = 0;
2599 
2600 		if (unlikely(test_bit(0, &adapter->resetting) &&
2601 			     adapter->reset_reason != VNIC_RESET_NON_FATAL)) {
2602 			enable_scrq_irq(adapter, rx_scrq);
2603 			napi_complete_done(napi, frames_processed);
2604 			return frames_processed;
2605 		}
2606 
2607 		if (!pending_scrq(adapter, rx_scrq))
2608 			break;
2609 		next = ibmvnic_next_scrq(adapter, rx_scrq);
2610 		rx_buff = (struct ibmvnic_rx_buff *)
2611 			  be64_to_cpu(next->rx_comp.correlator);
2612 		/* do error checking */
2613 		if (next->rx_comp.rc) {
2614 			netdev_dbg(netdev, "rx buffer returned with rc %x\n",
2615 				   be16_to_cpu(next->rx_comp.rc));
2616 			/* free the entry */
2617 			next->rx_comp.first = 0;
2618 			dev_kfree_skb_any(rx_buff->skb);
2619 			remove_buff_from_pool(adapter, rx_buff);
2620 			continue;
2621 		} else if (!rx_buff->skb) {
2622 			/* free the entry */
2623 			next->rx_comp.first = 0;
2624 			remove_buff_from_pool(adapter, rx_buff);
2625 			continue;
2626 		}
2627 
2628 		length = be32_to_cpu(next->rx_comp.len);
2629 		offset = be16_to_cpu(next->rx_comp.off_frame_data);
2630 		flags = next->rx_comp.flags;
2631 		skb = rx_buff->skb;
2632 		/* load long_term_buff before copying to skb */
2633 		dma_rmb();
2634 		skb_copy_to_linear_data(skb, rx_buff->data + offset,
2635 					length);
2636 
2637 		/* VLAN Header has been stripped by the system firmware and
2638 		 * needs to be inserted by the driver
2639 		 */
2640 		if (adapter->rx_vlan_header_insertion &&
2641 		    (flags & IBMVNIC_VLAN_STRIPPED))
2642 			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
2643 					       ntohs(next->rx_comp.vlan_tci));
2644 
2645 		/* free the entry */
2646 		next->rx_comp.first = 0;
2647 		remove_buff_from_pool(adapter, rx_buff);
2648 
2649 		skb_put(skb, length);
2650 		skb->protocol = eth_type_trans(skb, netdev);
2651 		skb_record_rx_queue(skb, scrq_num);
2652 
2653 		if (flags & IBMVNIC_IP_CHKSUM_GOOD &&
2654 		    flags & IBMVNIC_TCP_UDP_CHKSUM_GOOD) {
2655 			skb->ip_summed = CHECKSUM_UNNECESSARY;
2656 		}
2657 
2658 		length = skb->len;
2659 		napi_gro_receive(napi, skb); /* send it up */
2660 		netdev->stats.rx_packets++;
2661 		netdev->stats.rx_bytes += length;
2662 		adapter->rx_stats_buffers[scrq_num].packets++;
2663 		adapter->rx_stats_buffers[scrq_num].bytes += length;
2664 		frames_processed++;
2665 	}
2666 
2667 	if (adapter->state != VNIC_CLOSING &&
2668 	    ((atomic_read(&adapter->rx_pool[scrq_num].available) <
2669 	      adapter->req_rx_add_entries_per_subcrq / 2) ||
2670 	      frames_processed < budget))
2671 		replenish_rx_pool(adapter, &adapter->rx_pool[scrq_num]);
2672 	if (frames_processed < budget) {
2673 		if (napi_complete_done(napi, frames_processed)) {
2674 			enable_scrq_irq(adapter, rx_scrq);
2675 			if (pending_scrq(adapter, rx_scrq)) {
2676 				if (napi_reschedule(napi)) {
2677 					disable_scrq_irq(adapter, rx_scrq);
2678 					goto restart_poll;
2679 				}
2680 			}
2681 		}
2682 	}
2683 	return frames_processed;
2684 }
2685 
2686 static int wait_for_reset(struct ibmvnic_adapter *adapter)
2687 {
2688 	int rc, ret;
2689 
2690 	adapter->fallback.mtu = adapter->req_mtu;
2691 	adapter->fallback.rx_queues = adapter->req_rx_queues;
2692 	adapter->fallback.tx_queues = adapter->req_tx_queues;
2693 	adapter->fallback.rx_entries = adapter->req_rx_add_entries_per_subcrq;
2694 	adapter->fallback.tx_entries = adapter->req_tx_entries_per_subcrq;
2695 
2696 	reinit_completion(&adapter->reset_done);
2697 	adapter->wait_for_reset = true;
2698 	rc = ibmvnic_reset(adapter, VNIC_RESET_CHANGE_PARAM);
2699 
2700 	if (rc) {
2701 		ret = rc;
2702 		goto out;
2703 	}
2704 	rc = ibmvnic_wait_for_completion(adapter, &adapter->reset_done, 60000);
2705 	if (rc) {
2706 		ret = -ENODEV;
2707 		goto out;
2708 	}
2709 
2710 	ret = 0;
2711 	if (adapter->reset_done_rc) {
2712 		ret = -EIO;
2713 		adapter->desired.mtu = adapter->fallback.mtu;
2714 		adapter->desired.rx_queues = adapter->fallback.rx_queues;
2715 		adapter->desired.tx_queues = adapter->fallback.tx_queues;
2716 		adapter->desired.rx_entries = adapter->fallback.rx_entries;
2717 		adapter->desired.tx_entries = adapter->fallback.tx_entries;
2718 
2719 		reinit_completion(&adapter->reset_done);
2720 		adapter->wait_for_reset = true;
2721 		rc = ibmvnic_reset(adapter, VNIC_RESET_CHANGE_PARAM);
2722 		if (rc) {
2723 			ret = rc;
2724 			goto out;
2725 		}
2726 		rc = ibmvnic_wait_for_completion(adapter, &adapter->reset_done,
2727 						 60000);
2728 		if (rc) {
2729 			ret = -ENODEV;
2730 			goto out;
2731 		}
2732 	}
2733 out:
2734 	adapter->wait_for_reset = false;
2735 
2736 	return ret;
2737 }
2738 
2739 static int ibmvnic_change_mtu(struct net_device *netdev, int new_mtu)
2740 {
2741 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2742 
2743 	adapter->desired.mtu = new_mtu + ETH_HLEN;
2744 
2745 	return wait_for_reset(adapter);
2746 }
2747 
2748 static netdev_features_t ibmvnic_features_check(struct sk_buff *skb,
2749 						struct net_device *dev,
2750 						netdev_features_t features)
2751 {
2752 	/* Some backing hardware adapters can not
2753 	 * handle packets with a MSS less than 224
2754 	 * or with only one segment.
2755 	 */
2756 	if (skb_is_gso(skb)) {
2757 		if (skb_shinfo(skb)->gso_size < 224 ||
2758 		    skb_shinfo(skb)->gso_segs == 1)
2759 			features &= ~NETIF_F_GSO_MASK;
2760 	}
2761 
2762 	return features;
2763 }
2764 
2765 static const struct net_device_ops ibmvnic_netdev_ops = {
2766 	.ndo_open		= ibmvnic_open,
2767 	.ndo_stop		= ibmvnic_close,
2768 	.ndo_start_xmit		= ibmvnic_xmit,
2769 	.ndo_set_rx_mode	= ibmvnic_set_multi,
2770 	.ndo_set_mac_address	= ibmvnic_set_mac,
2771 	.ndo_validate_addr	= eth_validate_addr,
2772 	.ndo_tx_timeout		= ibmvnic_tx_timeout,
2773 	.ndo_change_mtu		= ibmvnic_change_mtu,
2774 	.ndo_features_check     = ibmvnic_features_check,
2775 };
2776 
2777 /* ethtool functions */
2778 
2779 static int ibmvnic_get_link_ksettings(struct net_device *netdev,
2780 				      struct ethtool_link_ksettings *cmd)
2781 {
2782 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2783 	int rc;
2784 
2785 	rc = send_query_phys_parms(adapter);
2786 	if (rc) {
2787 		adapter->speed = SPEED_UNKNOWN;
2788 		adapter->duplex = DUPLEX_UNKNOWN;
2789 	}
2790 	cmd->base.speed = adapter->speed;
2791 	cmd->base.duplex = adapter->duplex;
2792 	cmd->base.port = PORT_FIBRE;
2793 	cmd->base.phy_address = 0;
2794 	cmd->base.autoneg = AUTONEG_ENABLE;
2795 
2796 	return 0;
2797 }
2798 
2799 static void ibmvnic_get_drvinfo(struct net_device *netdev,
2800 				struct ethtool_drvinfo *info)
2801 {
2802 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2803 
2804 	strscpy(info->driver, ibmvnic_driver_name, sizeof(info->driver));
2805 	strscpy(info->version, IBMVNIC_DRIVER_VERSION, sizeof(info->version));
2806 	strscpy(info->fw_version, adapter->fw_version,
2807 		sizeof(info->fw_version));
2808 }
2809 
2810 static u32 ibmvnic_get_msglevel(struct net_device *netdev)
2811 {
2812 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2813 
2814 	return adapter->msg_enable;
2815 }
2816 
2817 static void ibmvnic_set_msglevel(struct net_device *netdev, u32 data)
2818 {
2819 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2820 
2821 	adapter->msg_enable = data;
2822 }
2823 
2824 static u32 ibmvnic_get_link(struct net_device *netdev)
2825 {
2826 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2827 
2828 	/* Don't need to send a query because we request a logical link up at
2829 	 * init and then we wait for link state indications
2830 	 */
2831 	return adapter->logical_link_state;
2832 }
2833 
2834 static void ibmvnic_get_ringparam(struct net_device *netdev,
2835 				  struct ethtool_ringparam *ring)
2836 {
2837 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2838 
2839 	if (adapter->priv_flags & IBMVNIC_USE_SERVER_MAXES) {
2840 		ring->rx_max_pending = adapter->max_rx_add_entries_per_subcrq;
2841 		ring->tx_max_pending = adapter->max_tx_entries_per_subcrq;
2842 	} else {
2843 		ring->rx_max_pending = IBMVNIC_MAX_QUEUE_SZ;
2844 		ring->tx_max_pending = IBMVNIC_MAX_QUEUE_SZ;
2845 	}
2846 	ring->rx_mini_max_pending = 0;
2847 	ring->rx_jumbo_max_pending = 0;
2848 	ring->rx_pending = adapter->req_rx_add_entries_per_subcrq;
2849 	ring->tx_pending = adapter->req_tx_entries_per_subcrq;
2850 	ring->rx_mini_pending = 0;
2851 	ring->rx_jumbo_pending = 0;
2852 }
2853 
2854 static int ibmvnic_set_ringparam(struct net_device *netdev,
2855 				 struct ethtool_ringparam *ring)
2856 {
2857 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2858 	int ret;
2859 
2860 	ret = 0;
2861 	adapter->desired.rx_entries = ring->rx_pending;
2862 	adapter->desired.tx_entries = ring->tx_pending;
2863 
2864 	ret = wait_for_reset(adapter);
2865 
2866 	if (!ret &&
2867 	    (adapter->req_rx_add_entries_per_subcrq != ring->rx_pending ||
2868 	     adapter->req_tx_entries_per_subcrq != ring->tx_pending))
2869 		netdev_info(netdev,
2870 			    "Could not match full ringsize request. Requested: RX %d, TX %d; Allowed: RX %llu, TX %llu\n",
2871 			    ring->rx_pending, ring->tx_pending,
2872 			    adapter->req_rx_add_entries_per_subcrq,
2873 			    adapter->req_tx_entries_per_subcrq);
2874 	return ret;
2875 }
2876 
2877 static void ibmvnic_get_channels(struct net_device *netdev,
2878 				 struct ethtool_channels *channels)
2879 {
2880 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2881 
2882 	if (adapter->priv_flags & IBMVNIC_USE_SERVER_MAXES) {
2883 		channels->max_rx = adapter->max_rx_queues;
2884 		channels->max_tx = adapter->max_tx_queues;
2885 	} else {
2886 		channels->max_rx = IBMVNIC_MAX_QUEUES;
2887 		channels->max_tx = IBMVNIC_MAX_QUEUES;
2888 	}
2889 
2890 	channels->max_other = 0;
2891 	channels->max_combined = 0;
2892 	channels->rx_count = adapter->req_rx_queues;
2893 	channels->tx_count = adapter->req_tx_queues;
2894 	channels->other_count = 0;
2895 	channels->combined_count = 0;
2896 }
2897 
2898 static int ibmvnic_set_channels(struct net_device *netdev,
2899 				struct ethtool_channels *channels)
2900 {
2901 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2902 	int ret;
2903 
2904 	ret = 0;
2905 	adapter->desired.rx_queues = channels->rx_count;
2906 	adapter->desired.tx_queues = channels->tx_count;
2907 
2908 	ret = wait_for_reset(adapter);
2909 
2910 	if (!ret &&
2911 	    (adapter->req_rx_queues != channels->rx_count ||
2912 	     adapter->req_tx_queues != channels->tx_count))
2913 		netdev_info(netdev,
2914 			    "Could not match full channels request. Requested: RX %d, TX %d; Allowed: RX %llu, TX %llu\n",
2915 			    channels->rx_count, channels->tx_count,
2916 			    adapter->req_rx_queues, adapter->req_tx_queues);
2917 	return ret;
2918 }
2919 
2920 static void ibmvnic_get_strings(struct net_device *dev, u32 stringset, u8 *data)
2921 {
2922 	struct ibmvnic_adapter *adapter = netdev_priv(dev);
2923 	int i;
2924 
2925 	switch (stringset) {
2926 	case ETH_SS_STATS:
2927 		for (i = 0; i < ARRAY_SIZE(ibmvnic_stats);
2928 				i++, data += ETH_GSTRING_LEN)
2929 			memcpy(data, ibmvnic_stats[i].name, ETH_GSTRING_LEN);
2930 
2931 		for (i = 0; i < adapter->req_tx_queues; i++) {
2932 			snprintf(data, ETH_GSTRING_LEN, "tx%d_packets", i);
2933 			data += ETH_GSTRING_LEN;
2934 
2935 			snprintf(data, ETH_GSTRING_LEN, "tx%d_bytes", i);
2936 			data += ETH_GSTRING_LEN;
2937 
2938 			snprintf(data, ETH_GSTRING_LEN,
2939 				 "tx%d_dropped_packets", i);
2940 			data += ETH_GSTRING_LEN;
2941 		}
2942 
2943 		for (i = 0; i < adapter->req_rx_queues; i++) {
2944 			snprintf(data, ETH_GSTRING_LEN, "rx%d_packets", i);
2945 			data += ETH_GSTRING_LEN;
2946 
2947 			snprintf(data, ETH_GSTRING_LEN, "rx%d_bytes", i);
2948 			data += ETH_GSTRING_LEN;
2949 
2950 			snprintf(data, ETH_GSTRING_LEN, "rx%d_interrupts", i);
2951 			data += ETH_GSTRING_LEN;
2952 		}
2953 		break;
2954 
2955 	case ETH_SS_PRIV_FLAGS:
2956 		for (i = 0; i < ARRAY_SIZE(ibmvnic_priv_flags); i++)
2957 			strcpy(data + i * ETH_GSTRING_LEN,
2958 			       ibmvnic_priv_flags[i]);
2959 		break;
2960 	default:
2961 		return;
2962 	}
2963 }
2964 
2965 static int ibmvnic_get_sset_count(struct net_device *dev, int sset)
2966 {
2967 	struct ibmvnic_adapter *adapter = netdev_priv(dev);
2968 
2969 	switch (sset) {
2970 	case ETH_SS_STATS:
2971 		return ARRAY_SIZE(ibmvnic_stats) +
2972 		       adapter->req_tx_queues * NUM_TX_STATS +
2973 		       adapter->req_rx_queues * NUM_RX_STATS;
2974 	case ETH_SS_PRIV_FLAGS:
2975 		return ARRAY_SIZE(ibmvnic_priv_flags);
2976 	default:
2977 		return -EOPNOTSUPP;
2978 	}
2979 }
2980 
2981 static void ibmvnic_get_ethtool_stats(struct net_device *dev,
2982 				      struct ethtool_stats *stats, u64 *data)
2983 {
2984 	struct ibmvnic_adapter *adapter = netdev_priv(dev);
2985 	union ibmvnic_crq crq;
2986 	int i, j;
2987 	int rc;
2988 
2989 	memset(&crq, 0, sizeof(crq));
2990 	crq.request_statistics.first = IBMVNIC_CRQ_CMD;
2991 	crq.request_statistics.cmd = REQUEST_STATISTICS;
2992 	crq.request_statistics.ioba = cpu_to_be32(adapter->stats_token);
2993 	crq.request_statistics.len =
2994 	    cpu_to_be32(sizeof(struct ibmvnic_statistics));
2995 
2996 	/* Wait for data to be written */
2997 	reinit_completion(&adapter->stats_done);
2998 	rc = ibmvnic_send_crq(adapter, &crq);
2999 	if (rc)
3000 		return;
3001 	rc = ibmvnic_wait_for_completion(adapter, &adapter->stats_done, 10000);
3002 	if (rc)
3003 		return;
3004 
3005 	for (i = 0; i < ARRAY_SIZE(ibmvnic_stats); i++)
3006 		data[i] = be64_to_cpu(IBMVNIC_GET_STAT
3007 				      (adapter, ibmvnic_stats[i].offset));
3008 
3009 	for (j = 0; j < adapter->req_tx_queues; j++) {
3010 		data[i] = adapter->tx_stats_buffers[j].packets;
3011 		i++;
3012 		data[i] = adapter->tx_stats_buffers[j].bytes;
3013 		i++;
3014 		data[i] = adapter->tx_stats_buffers[j].dropped_packets;
3015 		i++;
3016 	}
3017 
3018 	for (j = 0; j < adapter->req_rx_queues; j++) {
3019 		data[i] = adapter->rx_stats_buffers[j].packets;
3020 		i++;
3021 		data[i] = adapter->rx_stats_buffers[j].bytes;
3022 		i++;
3023 		data[i] = adapter->rx_stats_buffers[j].interrupts;
3024 		i++;
3025 	}
3026 }
3027 
3028 static u32 ibmvnic_get_priv_flags(struct net_device *netdev)
3029 {
3030 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
3031 
3032 	return adapter->priv_flags;
3033 }
3034 
3035 static int ibmvnic_set_priv_flags(struct net_device *netdev, u32 flags)
3036 {
3037 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
3038 	bool which_maxes = !!(flags & IBMVNIC_USE_SERVER_MAXES);
3039 
3040 	if (which_maxes)
3041 		adapter->priv_flags |= IBMVNIC_USE_SERVER_MAXES;
3042 	else
3043 		adapter->priv_flags &= ~IBMVNIC_USE_SERVER_MAXES;
3044 
3045 	return 0;
3046 }
3047 
3048 static const struct ethtool_ops ibmvnic_ethtool_ops = {
3049 	.get_drvinfo		= ibmvnic_get_drvinfo,
3050 	.get_msglevel		= ibmvnic_get_msglevel,
3051 	.set_msglevel		= ibmvnic_set_msglevel,
3052 	.get_link		= ibmvnic_get_link,
3053 	.get_ringparam		= ibmvnic_get_ringparam,
3054 	.set_ringparam		= ibmvnic_set_ringparam,
3055 	.get_channels		= ibmvnic_get_channels,
3056 	.set_channels		= ibmvnic_set_channels,
3057 	.get_strings            = ibmvnic_get_strings,
3058 	.get_sset_count         = ibmvnic_get_sset_count,
3059 	.get_ethtool_stats	= ibmvnic_get_ethtool_stats,
3060 	.get_link_ksettings	= ibmvnic_get_link_ksettings,
3061 	.get_priv_flags		= ibmvnic_get_priv_flags,
3062 	.set_priv_flags		= ibmvnic_set_priv_flags,
3063 };
3064 
3065 /* Routines for managing CRQs/sCRQs  */
3066 
3067 static int reset_one_sub_crq_queue(struct ibmvnic_adapter *adapter,
3068 				   struct ibmvnic_sub_crq_queue *scrq)
3069 {
3070 	int rc;
3071 
3072 	if (!scrq) {
3073 		netdev_dbg(adapter->netdev, "Invalid scrq reset.\n");
3074 		return -EINVAL;
3075 	}
3076 
3077 	if (scrq->irq) {
3078 		free_irq(scrq->irq, scrq);
3079 		irq_dispose_mapping(scrq->irq);
3080 		scrq->irq = 0;
3081 	}
3082 
3083 	if (scrq->msgs) {
3084 		memset(scrq->msgs, 0, 4 * PAGE_SIZE);
3085 		atomic_set(&scrq->used, 0);
3086 		scrq->cur = 0;
3087 		scrq->ind_buf.index = 0;
3088 	} else {
3089 		netdev_dbg(adapter->netdev, "Invalid scrq reset\n");
3090 		return -EINVAL;
3091 	}
3092 
3093 	rc = h_reg_sub_crq(adapter->vdev->unit_address, scrq->msg_token,
3094 			   4 * PAGE_SIZE, &scrq->crq_num, &scrq->hw_irq);
3095 	return rc;
3096 }
3097 
3098 static int reset_sub_crq_queues(struct ibmvnic_adapter *adapter)
3099 {
3100 	int i, rc;
3101 
3102 	if (!adapter->tx_scrq || !adapter->rx_scrq)
3103 		return -EINVAL;
3104 
3105 	for (i = 0; i < adapter->req_tx_queues; i++) {
3106 		netdev_dbg(adapter->netdev, "Re-setting tx_scrq[%d]\n", i);
3107 		rc = reset_one_sub_crq_queue(adapter, adapter->tx_scrq[i]);
3108 		if (rc)
3109 			return rc;
3110 	}
3111 
3112 	for (i = 0; i < adapter->req_rx_queues; i++) {
3113 		netdev_dbg(adapter->netdev, "Re-setting rx_scrq[%d]\n", i);
3114 		rc = reset_one_sub_crq_queue(adapter, adapter->rx_scrq[i]);
3115 		if (rc)
3116 			return rc;
3117 	}
3118 
3119 	return rc;
3120 }
3121 
3122 static void release_sub_crq_queue(struct ibmvnic_adapter *adapter,
3123 				  struct ibmvnic_sub_crq_queue *scrq,
3124 				  bool do_h_free)
3125 {
3126 	struct device *dev = &adapter->vdev->dev;
3127 	long rc;
3128 
3129 	netdev_dbg(adapter->netdev, "Releasing sub-CRQ\n");
3130 
3131 	if (do_h_free) {
3132 		/* Close the sub-crqs */
3133 		do {
3134 			rc = plpar_hcall_norets(H_FREE_SUB_CRQ,
3135 						adapter->vdev->unit_address,
3136 						scrq->crq_num);
3137 		} while (rc == H_BUSY || H_IS_LONG_BUSY(rc));
3138 
3139 		if (rc) {
3140 			netdev_err(adapter->netdev,
3141 				   "Failed to release sub-CRQ %16lx, rc = %ld\n",
3142 				   scrq->crq_num, rc);
3143 		}
3144 	}
3145 
3146 	dma_free_coherent(dev,
3147 			  IBMVNIC_IND_ARR_SZ,
3148 			  scrq->ind_buf.indir_arr,
3149 			  scrq->ind_buf.indir_dma);
3150 
3151 	dma_unmap_single(dev, scrq->msg_token, 4 * PAGE_SIZE,
3152 			 DMA_BIDIRECTIONAL);
3153 	free_pages((unsigned long)scrq->msgs, 2);
3154 	kfree(scrq);
3155 }
3156 
3157 static struct ibmvnic_sub_crq_queue *init_sub_crq_queue(struct ibmvnic_adapter
3158 							*adapter)
3159 {
3160 	struct device *dev = &adapter->vdev->dev;
3161 	struct ibmvnic_sub_crq_queue *scrq;
3162 	int rc;
3163 
3164 	scrq = kzalloc(sizeof(*scrq), GFP_KERNEL);
3165 	if (!scrq)
3166 		return NULL;
3167 
3168 	scrq->msgs =
3169 		(union sub_crq *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 2);
3170 	if (!scrq->msgs) {
3171 		dev_warn(dev, "Couldn't allocate crq queue messages page\n");
3172 		goto zero_page_failed;
3173 	}
3174 
3175 	scrq->msg_token = dma_map_single(dev, scrq->msgs, 4 * PAGE_SIZE,
3176 					 DMA_BIDIRECTIONAL);
3177 	if (dma_mapping_error(dev, scrq->msg_token)) {
3178 		dev_warn(dev, "Couldn't map crq queue messages page\n");
3179 		goto map_failed;
3180 	}
3181 
3182 	rc = h_reg_sub_crq(adapter->vdev->unit_address, scrq->msg_token,
3183 			   4 * PAGE_SIZE, &scrq->crq_num, &scrq->hw_irq);
3184 
3185 	if (rc == H_RESOURCE)
3186 		rc = ibmvnic_reset_crq(adapter);
3187 
3188 	if (rc == H_CLOSED) {
3189 		dev_warn(dev, "Partner adapter not ready, waiting.\n");
3190 	} else if (rc) {
3191 		dev_warn(dev, "Error %d registering sub-crq\n", rc);
3192 		goto reg_failed;
3193 	}
3194 
3195 	scrq->adapter = adapter;
3196 	scrq->size = 4 * PAGE_SIZE / sizeof(*scrq->msgs);
3197 	scrq->ind_buf.index = 0;
3198 
3199 	scrq->ind_buf.indir_arr =
3200 		dma_alloc_coherent(dev,
3201 				   IBMVNIC_IND_ARR_SZ,
3202 				   &scrq->ind_buf.indir_dma,
3203 				   GFP_KERNEL);
3204 
3205 	if (!scrq->ind_buf.indir_arr)
3206 		goto indir_failed;
3207 
3208 	spin_lock_init(&scrq->lock);
3209 
3210 	netdev_dbg(adapter->netdev,
3211 		   "sub-crq initialized, num %lx, hw_irq=%lx, irq=%x\n",
3212 		   scrq->crq_num, scrq->hw_irq, scrq->irq);
3213 
3214 	return scrq;
3215 
3216 indir_failed:
3217 	do {
3218 		rc = plpar_hcall_norets(H_FREE_SUB_CRQ,
3219 					adapter->vdev->unit_address,
3220 					scrq->crq_num);
3221 	} while (rc == H_BUSY || rc == H_IS_LONG_BUSY(rc));
3222 reg_failed:
3223 	dma_unmap_single(dev, scrq->msg_token, 4 * PAGE_SIZE,
3224 			 DMA_BIDIRECTIONAL);
3225 map_failed:
3226 	free_pages((unsigned long)scrq->msgs, 2);
3227 zero_page_failed:
3228 	kfree(scrq);
3229 
3230 	return NULL;
3231 }
3232 
3233 static void release_sub_crqs(struct ibmvnic_adapter *adapter, bool do_h_free)
3234 {
3235 	int i;
3236 
3237 	if (adapter->tx_scrq) {
3238 		for (i = 0; i < adapter->num_active_tx_scrqs; i++) {
3239 			if (!adapter->tx_scrq[i])
3240 				continue;
3241 
3242 			netdev_dbg(adapter->netdev, "Releasing tx_scrq[%d]\n",
3243 				   i);
3244 			if (adapter->tx_scrq[i]->irq) {
3245 				free_irq(adapter->tx_scrq[i]->irq,
3246 					 adapter->tx_scrq[i]);
3247 				irq_dispose_mapping(adapter->tx_scrq[i]->irq);
3248 				adapter->tx_scrq[i]->irq = 0;
3249 			}
3250 
3251 			release_sub_crq_queue(adapter, adapter->tx_scrq[i],
3252 					      do_h_free);
3253 		}
3254 
3255 		kfree(adapter->tx_scrq);
3256 		adapter->tx_scrq = NULL;
3257 		adapter->num_active_tx_scrqs = 0;
3258 	}
3259 
3260 	if (adapter->rx_scrq) {
3261 		for (i = 0; i < adapter->num_active_rx_scrqs; i++) {
3262 			if (!adapter->rx_scrq[i])
3263 				continue;
3264 
3265 			netdev_dbg(adapter->netdev, "Releasing rx_scrq[%d]\n",
3266 				   i);
3267 			if (adapter->rx_scrq[i]->irq) {
3268 				free_irq(adapter->rx_scrq[i]->irq,
3269 					 adapter->rx_scrq[i]);
3270 				irq_dispose_mapping(adapter->rx_scrq[i]->irq);
3271 				adapter->rx_scrq[i]->irq = 0;
3272 			}
3273 
3274 			release_sub_crq_queue(adapter, adapter->rx_scrq[i],
3275 					      do_h_free);
3276 		}
3277 
3278 		kfree(adapter->rx_scrq);
3279 		adapter->rx_scrq = NULL;
3280 		adapter->num_active_rx_scrqs = 0;
3281 	}
3282 }
3283 
3284 static int disable_scrq_irq(struct ibmvnic_adapter *adapter,
3285 			    struct ibmvnic_sub_crq_queue *scrq)
3286 {
3287 	struct device *dev = &adapter->vdev->dev;
3288 	unsigned long rc;
3289 
3290 	rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,
3291 				H_DISABLE_VIO_INTERRUPT, scrq->hw_irq, 0, 0);
3292 	if (rc)
3293 		dev_err(dev, "Couldn't disable scrq irq 0x%lx. rc=%ld\n",
3294 			scrq->hw_irq, rc);
3295 	return rc;
3296 }
3297 
3298 static int enable_scrq_irq(struct ibmvnic_adapter *adapter,
3299 			   struct ibmvnic_sub_crq_queue *scrq)
3300 {
3301 	struct device *dev = &adapter->vdev->dev;
3302 	unsigned long rc;
3303 
3304 	if (scrq->hw_irq > 0x100000000ULL) {
3305 		dev_err(dev, "bad hw_irq = %lx\n", scrq->hw_irq);
3306 		return 1;
3307 	}
3308 
3309 	if (test_bit(0, &adapter->resetting) &&
3310 	    adapter->reset_reason == VNIC_RESET_MOBILITY) {
3311 		u64 val = (0xff000000) | scrq->hw_irq;
3312 
3313 		rc = plpar_hcall_norets(H_EOI, val);
3314 		/* H_EOI would fail with rc = H_FUNCTION when running
3315 		 * in XIVE mode which is expected, but not an error.
3316 		 */
3317 		if (rc && rc != H_FUNCTION)
3318 			dev_err(dev, "H_EOI FAILED irq 0x%llx. rc=%ld\n",
3319 				val, rc);
3320 	}
3321 
3322 	rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,
3323 				H_ENABLE_VIO_INTERRUPT, scrq->hw_irq, 0, 0);
3324 	if (rc)
3325 		dev_err(dev, "Couldn't enable scrq irq 0x%lx. rc=%ld\n",
3326 			scrq->hw_irq, rc);
3327 	return rc;
3328 }
3329 
3330 static int ibmvnic_complete_tx(struct ibmvnic_adapter *adapter,
3331 			       struct ibmvnic_sub_crq_queue *scrq)
3332 {
3333 	struct device *dev = &adapter->vdev->dev;
3334 	struct ibmvnic_tx_pool *tx_pool;
3335 	struct ibmvnic_tx_buff *txbuff;
3336 	struct netdev_queue *txq;
3337 	union sub_crq *next;
3338 	int index;
3339 	int i;
3340 
3341 restart_loop:
3342 	while (pending_scrq(adapter, scrq)) {
3343 		unsigned int pool = scrq->pool_index;
3344 		int num_entries = 0;
3345 		int total_bytes = 0;
3346 		int num_packets = 0;
3347 
3348 		next = ibmvnic_next_scrq(adapter, scrq);
3349 		for (i = 0; i < next->tx_comp.num_comps; i++) {
3350 			index = be32_to_cpu(next->tx_comp.correlators[i]);
3351 			if (index & IBMVNIC_TSO_POOL_MASK) {
3352 				tx_pool = &adapter->tso_pool[pool];
3353 				index &= ~IBMVNIC_TSO_POOL_MASK;
3354 			} else {
3355 				tx_pool = &adapter->tx_pool[pool];
3356 			}
3357 
3358 			txbuff = &tx_pool->tx_buff[index];
3359 			num_packets++;
3360 			num_entries += txbuff->num_entries;
3361 			if (txbuff->skb) {
3362 				total_bytes += txbuff->skb->len;
3363 				if (next->tx_comp.rcs[i]) {
3364 					dev_err(dev, "tx error %x\n",
3365 						next->tx_comp.rcs[i]);
3366 					dev_kfree_skb_irq(txbuff->skb);
3367 				} else {
3368 					dev_consume_skb_irq(txbuff->skb);
3369 				}
3370 				txbuff->skb = NULL;
3371 			} else {
3372 				netdev_warn(adapter->netdev,
3373 					    "TX completion received with NULL socket buffer\n");
3374 			}
3375 			tx_pool->free_map[tx_pool->producer_index] = index;
3376 			tx_pool->producer_index =
3377 				(tx_pool->producer_index + 1) %
3378 					tx_pool->num_buffers;
3379 		}
3380 		/* remove tx_comp scrq*/
3381 		next->tx_comp.first = 0;
3382 
3383 		txq = netdev_get_tx_queue(adapter->netdev, scrq->pool_index);
3384 		netdev_tx_completed_queue(txq, num_packets, total_bytes);
3385 
3386 		if (atomic_sub_return(num_entries, &scrq->used) <=
3387 		    (adapter->req_tx_entries_per_subcrq / 2) &&
3388 		    __netif_subqueue_stopped(adapter->netdev,
3389 					     scrq->pool_index)) {
3390 			netif_wake_subqueue(adapter->netdev, scrq->pool_index);
3391 			netdev_dbg(adapter->netdev, "Started queue %d\n",
3392 				   scrq->pool_index);
3393 		}
3394 	}
3395 
3396 	enable_scrq_irq(adapter, scrq);
3397 
3398 	if (pending_scrq(adapter, scrq)) {
3399 		disable_scrq_irq(adapter, scrq);
3400 		goto restart_loop;
3401 	}
3402 
3403 	return 0;
3404 }
3405 
3406 static irqreturn_t ibmvnic_interrupt_tx(int irq, void *instance)
3407 {
3408 	struct ibmvnic_sub_crq_queue *scrq = instance;
3409 	struct ibmvnic_adapter *adapter = scrq->adapter;
3410 
3411 	disable_scrq_irq(adapter, scrq);
3412 	ibmvnic_complete_tx(adapter, scrq);
3413 
3414 	return IRQ_HANDLED;
3415 }
3416 
3417 static irqreturn_t ibmvnic_interrupt_rx(int irq, void *instance)
3418 {
3419 	struct ibmvnic_sub_crq_queue *scrq = instance;
3420 	struct ibmvnic_adapter *adapter = scrq->adapter;
3421 
3422 	/* When booting a kdump kernel we can hit pending interrupts
3423 	 * prior to completing driver initialization.
3424 	 */
3425 	if (unlikely(adapter->state != VNIC_OPEN))
3426 		return IRQ_NONE;
3427 
3428 	adapter->rx_stats_buffers[scrq->scrq_num].interrupts++;
3429 
3430 	if (napi_schedule_prep(&adapter->napi[scrq->scrq_num])) {
3431 		disable_scrq_irq(adapter, scrq);
3432 		__napi_schedule(&adapter->napi[scrq->scrq_num]);
3433 	}
3434 
3435 	return IRQ_HANDLED;
3436 }
3437 
3438 static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter)
3439 {
3440 	struct device *dev = &adapter->vdev->dev;
3441 	struct ibmvnic_sub_crq_queue *scrq;
3442 	int i = 0, j = 0;
3443 	int rc = 0;
3444 
3445 	for (i = 0; i < adapter->req_tx_queues; i++) {
3446 		netdev_dbg(adapter->netdev, "Initializing tx_scrq[%d] irq\n",
3447 			   i);
3448 		scrq = adapter->tx_scrq[i];
3449 		scrq->irq = irq_create_mapping(NULL, scrq->hw_irq);
3450 
3451 		if (!scrq->irq) {
3452 			rc = -EINVAL;
3453 			dev_err(dev, "Error mapping irq\n");
3454 			goto req_tx_irq_failed;
3455 		}
3456 
3457 		snprintf(scrq->name, sizeof(scrq->name), "ibmvnic-%x-tx%d",
3458 			 adapter->vdev->unit_address, i);
3459 		rc = request_irq(scrq->irq, ibmvnic_interrupt_tx,
3460 				 0, scrq->name, scrq);
3461 
3462 		if (rc) {
3463 			dev_err(dev, "Couldn't register tx irq 0x%x. rc=%d\n",
3464 				scrq->irq, rc);
3465 			irq_dispose_mapping(scrq->irq);
3466 			goto req_tx_irq_failed;
3467 		}
3468 	}
3469 
3470 	for (i = 0; i < adapter->req_rx_queues; i++) {
3471 		netdev_dbg(adapter->netdev, "Initializing rx_scrq[%d] irq\n",
3472 			   i);
3473 		scrq = adapter->rx_scrq[i];
3474 		scrq->irq = irq_create_mapping(NULL, scrq->hw_irq);
3475 		if (!scrq->irq) {
3476 			rc = -EINVAL;
3477 			dev_err(dev, "Error mapping irq\n");
3478 			goto req_rx_irq_failed;
3479 		}
3480 		snprintf(scrq->name, sizeof(scrq->name), "ibmvnic-%x-rx%d",
3481 			 adapter->vdev->unit_address, i);
3482 		rc = request_irq(scrq->irq, ibmvnic_interrupt_rx,
3483 				 0, scrq->name, scrq);
3484 		if (rc) {
3485 			dev_err(dev, "Couldn't register rx irq 0x%x. rc=%d\n",
3486 				scrq->irq, rc);
3487 			irq_dispose_mapping(scrq->irq);
3488 			goto req_rx_irq_failed;
3489 		}
3490 	}
3491 	return rc;
3492 
3493 req_rx_irq_failed:
3494 	for (j = 0; j < i; j++) {
3495 		free_irq(adapter->rx_scrq[j]->irq, adapter->rx_scrq[j]);
3496 		irq_dispose_mapping(adapter->rx_scrq[j]->irq);
3497 	}
3498 	i = adapter->req_tx_queues;
3499 req_tx_irq_failed:
3500 	for (j = 0; j < i; j++) {
3501 		free_irq(adapter->tx_scrq[j]->irq, adapter->tx_scrq[j]);
3502 		irq_dispose_mapping(adapter->tx_scrq[j]->irq);
3503 	}
3504 	release_sub_crqs(adapter, 1);
3505 	return rc;
3506 }
3507 
3508 static int init_sub_crqs(struct ibmvnic_adapter *adapter)
3509 {
3510 	struct device *dev = &adapter->vdev->dev;
3511 	struct ibmvnic_sub_crq_queue **allqueues;
3512 	int registered_queues = 0;
3513 	int total_queues;
3514 	int more = 0;
3515 	int i;
3516 
3517 	total_queues = adapter->req_tx_queues + adapter->req_rx_queues;
3518 
3519 	allqueues = kcalloc(total_queues, sizeof(*allqueues), GFP_KERNEL);
3520 	if (!allqueues)
3521 		return -1;
3522 
3523 	for (i = 0; i < total_queues; i++) {
3524 		allqueues[i] = init_sub_crq_queue(adapter);
3525 		if (!allqueues[i]) {
3526 			dev_warn(dev, "Couldn't allocate all sub-crqs\n");
3527 			break;
3528 		}
3529 		registered_queues++;
3530 	}
3531 
3532 	/* Make sure we were able to register the minimum number of queues */
3533 	if (registered_queues <
3534 	    adapter->min_tx_queues + adapter->min_rx_queues) {
3535 		dev_err(dev, "Fatal: Couldn't init  min number of sub-crqs\n");
3536 		goto tx_failed;
3537 	}
3538 
3539 	/* Distribute the failed allocated queues*/
3540 	for (i = 0; i < total_queues - registered_queues + more ; i++) {
3541 		netdev_dbg(adapter->netdev, "Reducing number of queues\n");
3542 		switch (i % 3) {
3543 		case 0:
3544 			if (adapter->req_rx_queues > adapter->min_rx_queues)
3545 				adapter->req_rx_queues--;
3546 			else
3547 				more++;
3548 			break;
3549 		case 1:
3550 			if (adapter->req_tx_queues > adapter->min_tx_queues)
3551 				adapter->req_tx_queues--;
3552 			else
3553 				more++;
3554 			break;
3555 		}
3556 	}
3557 
3558 	adapter->tx_scrq = kcalloc(adapter->req_tx_queues,
3559 				   sizeof(*adapter->tx_scrq), GFP_KERNEL);
3560 	if (!adapter->tx_scrq)
3561 		goto tx_failed;
3562 
3563 	for (i = 0; i < adapter->req_tx_queues; i++) {
3564 		adapter->tx_scrq[i] = allqueues[i];
3565 		adapter->tx_scrq[i]->pool_index = i;
3566 		adapter->num_active_tx_scrqs++;
3567 	}
3568 
3569 	adapter->rx_scrq = kcalloc(adapter->req_rx_queues,
3570 				   sizeof(*adapter->rx_scrq), GFP_KERNEL);
3571 	if (!adapter->rx_scrq)
3572 		goto rx_failed;
3573 
3574 	for (i = 0; i < adapter->req_rx_queues; i++) {
3575 		adapter->rx_scrq[i] = allqueues[i + adapter->req_tx_queues];
3576 		adapter->rx_scrq[i]->scrq_num = i;
3577 		adapter->num_active_rx_scrqs++;
3578 	}
3579 
3580 	kfree(allqueues);
3581 	return 0;
3582 
3583 rx_failed:
3584 	kfree(adapter->tx_scrq);
3585 	adapter->tx_scrq = NULL;
3586 tx_failed:
3587 	for (i = 0; i < registered_queues; i++)
3588 		release_sub_crq_queue(adapter, allqueues[i], 1);
3589 	kfree(allqueues);
3590 	return -1;
3591 }
3592 
3593 static void send_request_cap(struct ibmvnic_adapter *adapter, int retry)
3594 {
3595 	struct device *dev = &adapter->vdev->dev;
3596 	union ibmvnic_crq crq;
3597 	int max_entries;
3598 
3599 	if (!retry) {
3600 		/* Sub-CRQ entries are 32 byte long */
3601 		int entries_page = 4 * PAGE_SIZE / (sizeof(u64) * 4);
3602 
3603 		if (adapter->min_tx_entries_per_subcrq > entries_page ||
3604 		    adapter->min_rx_add_entries_per_subcrq > entries_page) {
3605 			dev_err(dev, "Fatal, invalid entries per sub-crq\n");
3606 			return;
3607 		}
3608 
3609 		if (adapter->desired.mtu)
3610 			adapter->req_mtu = adapter->desired.mtu;
3611 		else
3612 			adapter->req_mtu = adapter->netdev->mtu + ETH_HLEN;
3613 
3614 		if (!adapter->desired.tx_entries)
3615 			adapter->desired.tx_entries =
3616 					adapter->max_tx_entries_per_subcrq;
3617 		if (!adapter->desired.rx_entries)
3618 			adapter->desired.rx_entries =
3619 					adapter->max_rx_add_entries_per_subcrq;
3620 
3621 		max_entries = IBMVNIC_MAX_LTB_SIZE /
3622 			      (adapter->req_mtu + IBMVNIC_BUFFER_HLEN);
3623 
3624 		if ((adapter->req_mtu + IBMVNIC_BUFFER_HLEN) *
3625 			adapter->desired.tx_entries > IBMVNIC_MAX_LTB_SIZE) {
3626 			adapter->desired.tx_entries = max_entries;
3627 		}
3628 
3629 		if ((adapter->req_mtu + IBMVNIC_BUFFER_HLEN) *
3630 			adapter->desired.rx_entries > IBMVNIC_MAX_LTB_SIZE) {
3631 			adapter->desired.rx_entries = max_entries;
3632 		}
3633 
3634 		if (adapter->desired.tx_entries)
3635 			adapter->req_tx_entries_per_subcrq =
3636 					adapter->desired.tx_entries;
3637 		else
3638 			adapter->req_tx_entries_per_subcrq =
3639 					adapter->max_tx_entries_per_subcrq;
3640 
3641 		if (adapter->desired.rx_entries)
3642 			adapter->req_rx_add_entries_per_subcrq =
3643 					adapter->desired.rx_entries;
3644 		else
3645 			adapter->req_rx_add_entries_per_subcrq =
3646 					adapter->max_rx_add_entries_per_subcrq;
3647 
3648 		if (adapter->desired.tx_queues)
3649 			adapter->req_tx_queues =
3650 					adapter->desired.tx_queues;
3651 		else
3652 			adapter->req_tx_queues =
3653 					adapter->opt_tx_comp_sub_queues;
3654 
3655 		if (adapter->desired.rx_queues)
3656 			adapter->req_rx_queues =
3657 					adapter->desired.rx_queues;
3658 		else
3659 			adapter->req_rx_queues =
3660 					adapter->opt_rx_comp_queues;
3661 
3662 		adapter->req_rx_add_queues = adapter->max_rx_add_queues;
3663 	}
3664 
3665 	memset(&crq, 0, sizeof(crq));
3666 	crq.request_capability.first = IBMVNIC_CRQ_CMD;
3667 	crq.request_capability.cmd = REQUEST_CAPABILITY;
3668 
3669 	crq.request_capability.capability = cpu_to_be16(REQ_TX_QUEUES);
3670 	crq.request_capability.number = cpu_to_be64(adapter->req_tx_queues);
3671 	atomic_inc(&adapter->running_cap_crqs);
3672 	ibmvnic_send_crq(adapter, &crq);
3673 
3674 	crq.request_capability.capability = cpu_to_be16(REQ_RX_QUEUES);
3675 	crq.request_capability.number = cpu_to_be64(adapter->req_rx_queues);
3676 	atomic_inc(&adapter->running_cap_crqs);
3677 	ibmvnic_send_crq(adapter, &crq);
3678 
3679 	crq.request_capability.capability = cpu_to_be16(REQ_RX_ADD_QUEUES);
3680 	crq.request_capability.number = cpu_to_be64(adapter->req_rx_add_queues);
3681 	atomic_inc(&adapter->running_cap_crqs);
3682 	ibmvnic_send_crq(adapter, &crq);
3683 
3684 	crq.request_capability.capability =
3685 	    cpu_to_be16(REQ_TX_ENTRIES_PER_SUBCRQ);
3686 	crq.request_capability.number =
3687 	    cpu_to_be64(adapter->req_tx_entries_per_subcrq);
3688 	atomic_inc(&adapter->running_cap_crqs);
3689 	ibmvnic_send_crq(adapter, &crq);
3690 
3691 	crq.request_capability.capability =
3692 	    cpu_to_be16(REQ_RX_ADD_ENTRIES_PER_SUBCRQ);
3693 	crq.request_capability.number =
3694 	    cpu_to_be64(adapter->req_rx_add_entries_per_subcrq);
3695 	atomic_inc(&adapter->running_cap_crqs);
3696 	ibmvnic_send_crq(adapter, &crq);
3697 
3698 	crq.request_capability.capability = cpu_to_be16(REQ_MTU);
3699 	crq.request_capability.number = cpu_to_be64(adapter->req_mtu);
3700 	atomic_inc(&adapter->running_cap_crqs);
3701 	ibmvnic_send_crq(adapter, &crq);
3702 
3703 	if (adapter->netdev->flags & IFF_PROMISC) {
3704 		if (adapter->promisc_supported) {
3705 			crq.request_capability.capability =
3706 			    cpu_to_be16(PROMISC_REQUESTED);
3707 			crq.request_capability.number = cpu_to_be64(1);
3708 			atomic_inc(&adapter->running_cap_crqs);
3709 			ibmvnic_send_crq(adapter, &crq);
3710 		}
3711 	} else {
3712 		crq.request_capability.capability =
3713 		    cpu_to_be16(PROMISC_REQUESTED);
3714 		crq.request_capability.number = cpu_to_be64(0);
3715 		atomic_inc(&adapter->running_cap_crqs);
3716 		ibmvnic_send_crq(adapter, &crq);
3717 	}
3718 }
3719 
3720 static int pending_scrq(struct ibmvnic_adapter *adapter,
3721 			struct ibmvnic_sub_crq_queue *scrq)
3722 {
3723 	union sub_crq *entry = &scrq->msgs[scrq->cur];
3724 	int rc;
3725 
3726 	rc = !!(entry->generic.first & IBMVNIC_CRQ_CMD_RSP);
3727 
3728 	/* Ensure that the SCRQ valid flag is loaded prior to loading the
3729 	 * contents of the SCRQ descriptor
3730 	 */
3731 	dma_rmb();
3732 
3733 	return rc;
3734 }
3735 
3736 static union sub_crq *ibmvnic_next_scrq(struct ibmvnic_adapter *adapter,
3737 					struct ibmvnic_sub_crq_queue *scrq)
3738 {
3739 	union sub_crq *entry;
3740 	unsigned long flags;
3741 
3742 	spin_lock_irqsave(&scrq->lock, flags);
3743 	entry = &scrq->msgs[scrq->cur];
3744 	if (entry->generic.first & IBMVNIC_CRQ_CMD_RSP) {
3745 		if (++scrq->cur == scrq->size)
3746 			scrq->cur = 0;
3747 	} else {
3748 		entry = NULL;
3749 	}
3750 	spin_unlock_irqrestore(&scrq->lock, flags);
3751 
3752 	/* Ensure that the SCRQ valid flag is loaded prior to loading the
3753 	 * contents of the SCRQ descriptor
3754 	 */
3755 	dma_rmb();
3756 
3757 	return entry;
3758 }
3759 
3760 static union ibmvnic_crq *ibmvnic_next_crq(struct ibmvnic_adapter *adapter)
3761 {
3762 	struct ibmvnic_crq_queue *queue = &adapter->crq;
3763 	union ibmvnic_crq *crq;
3764 
3765 	crq = &queue->msgs[queue->cur];
3766 	if (crq->generic.first & IBMVNIC_CRQ_CMD_RSP) {
3767 		if (++queue->cur == queue->size)
3768 			queue->cur = 0;
3769 	} else {
3770 		crq = NULL;
3771 	}
3772 
3773 	return crq;
3774 }
3775 
3776 static void print_subcrq_error(struct device *dev, int rc, const char *func)
3777 {
3778 	switch (rc) {
3779 	case H_PARAMETER:
3780 		dev_warn_ratelimited(dev,
3781 				     "%s failed: Send request is malformed or adapter failover pending. (rc=%d)\n",
3782 				     func, rc);
3783 		break;
3784 	case H_CLOSED:
3785 		dev_warn_ratelimited(dev,
3786 				     "%s failed: Backing queue closed. Adapter is down or failover pending. (rc=%d)\n",
3787 				     func, rc);
3788 		break;
3789 	default:
3790 		dev_err_ratelimited(dev, "%s failed: (rc=%d)\n", func, rc);
3791 		break;
3792 	}
3793 }
3794 
3795 static int send_subcrq_indirect(struct ibmvnic_adapter *adapter,
3796 				u64 remote_handle, u64 ioba, u64 num_entries)
3797 {
3798 	unsigned int ua = adapter->vdev->unit_address;
3799 	struct device *dev = &adapter->vdev->dev;
3800 	int rc;
3801 
3802 	/* Make sure the hypervisor sees the complete request */
3803 	dma_wmb();
3804 	rc = plpar_hcall_norets(H_SEND_SUB_CRQ_INDIRECT, ua,
3805 				cpu_to_be64(remote_handle),
3806 				ioba, num_entries);
3807 
3808 	if (rc)
3809 		print_subcrq_error(dev, rc, __func__);
3810 
3811 	return rc;
3812 }
3813 
3814 static int ibmvnic_send_crq(struct ibmvnic_adapter *adapter,
3815 			    union ibmvnic_crq *crq)
3816 {
3817 	unsigned int ua = adapter->vdev->unit_address;
3818 	struct device *dev = &adapter->vdev->dev;
3819 	u64 *u64_crq = (u64 *)crq;
3820 	int rc;
3821 
3822 	netdev_dbg(adapter->netdev, "Sending CRQ: %016lx %016lx\n",
3823 		   (unsigned long)cpu_to_be64(u64_crq[0]),
3824 		   (unsigned long)cpu_to_be64(u64_crq[1]));
3825 
3826 	if (!adapter->crq.active &&
3827 	    crq->generic.first != IBMVNIC_CRQ_INIT_CMD) {
3828 		dev_warn(dev, "Invalid request detected while CRQ is inactive, possible device state change during reset\n");
3829 		return -EINVAL;
3830 	}
3831 
3832 	/* Make sure the hypervisor sees the complete request */
3833 	dma_wmb();
3834 
3835 	rc = plpar_hcall_norets(H_SEND_CRQ, ua,
3836 				cpu_to_be64(u64_crq[0]),
3837 				cpu_to_be64(u64_crq[1]));
3838 
3839 	if (rc) {
3840 		if (rc == H_CLOSED) {
3841 			dev_warn(dev, "CRQ Queue closed\n");
3842 			/* do not reset, report the fail, wait for passive init from server */
3843 		}
3844 
3845 		dev_warn(dev, "Send error (rc=%d)\n", rc);
3846 	}
3847 
3848 	return rc;
3849 }
3850 
3851 static int ibmvnic_send_crq_init(struct ibmvnic_adapter *adapter)
3852 {
3853 	struct device *dev = &adapter->vdev->dev;
3854 	union ibmvnic_crq crq;
3855 	int retries = 100;
3856 	int rc;
3857 
3858 	memset(&crq, 0, sizeof(crq));
3859 	crq.generic.first = IBMVNIC_CRQ_INIT_CMD;
3860 	crq.generic.cmd = IBMVNIC_CRQ_INIT;
3861 	netdev_dbg(adapter->netdev, "Sending CRQ init\n");
3862 
3863 	do {
3864 		rc = ibmvnic_send_crq(adapter, &crq);
3865 		if (rc != H_CLOSED)
3866 			break;
3867 		retries--;
3868 		msleep(50);
3869 
3870 	} while (retries > 0);
3871 
3872 	if (rc) {
3873 		dev_err(dev, "Failed to send init request, rc = %d\n", rc);
3874 		return rc;
3875 	}
3876 
3877 	return 0;
3878 }
3879 
3880 struct vnic_login_client_data {
3881 	u8	type;
3882 	__be16	len;
3883 	char	name[];
3884 } __packed;
3885 
3886 static int vnic_client_data_len(struct ibmvnic_adapter *adapter)
3887 {
3888 	int len;
3889 
3890 	/* Calculate the amount of buffer space needed for the
3891 	 * vnic client data in the login buffer. There are four entries,
3892 	 * OS name, LPAR name, device name, and a null last entry.
3893 	 */
3894 	len = 4 * sizeof(struct vnic_login_client_data);
3895 	len += 6; /* "Linux" plus NULL */
3896 	len += strlen(utsname()->nodename) + 1;
3897 	len += strlen(adapter->netdev->name) + 1;
3898 
3899 	return len;
3900 }
3901 
3902 static void vnic_add_client_data(struct ibmvnic_adapter *adapter,
3903 				 struct vnic_login_client_data *vlcd)
3904 {
3905 	const char *os_name = "Linux";
3906 	int len;
3907 
3908 	/* Type 1 - LPAR OS */
3909 	vlcd->type = 1;
3910 	len = strlen(os_name) + 1;
3911 	vlcd->len = cpu_to_be16(len);
3912 	strncpy(vlcd->name, os_name, len);
3913 	vlcd = (struct vnic_login_client_data *)(vlcd->name + len);
3914 
3915 	/* Type 2 - LPAR name */
3916 	vlcd->type = 2;
3917 	len = strlen(utsname()->nodename) + 1;
3918 	vlcd->len = cpu_to_be16(len);
3919 	strncpy(vlcd->name, utsname()->nodename, len);
3920 	vlcd = (struct vnic_login_client_data *)(vlcd->name + len);
3921 
3922 	/* Type 3 - device name */
3923 	vlcd->type = 3;
3924 	len = strlen(adapter->netdev->name) + 1;
3925 	vlcd->len = cpu_to_be16(len);
3926 	strncpy(vlcd->name, adapter->netdev->name, len);
3927 }
3928 
3929 static int send_login(struct ibmvnic_adapter *adapter)
3930 {
3931 	struct ibmvnic_login_rsp_buffer *login_rsp_buffer;
3932 	struct ibmvnic_login_buffer *login_buffer;
3933 	struct device *dev = &adapter->vdev->dev;
3934 	struct vnic_login_client_data *vlcd;
3935 	dma_addr_t rsp_buffer_token;
3936 	dma_addr_t buffer_token;
3937 	size_t rsp_buffer_size;
3938 	union ibmvnic_crq crq;
3939 	int client_data_len;
3940 	size_t buffer_size;
3941 	__be64 *tx_list_p;
3942 	__be64 *rx_list_p;
3943 	int rc;
3944 	int i;
3945 
3946 	if (!adapter->tx_scrq || !adapter->rx_scrq) {
3947 		netdev_err(adapter->netdev,
3948 			   "RX or TX queues are not allocated, device login failed\n");
3949 		return -1;
3950 	}
3951 
3952 	release_login_buffer(adapter);
3953 	release_login_rsp_buffer(adapter);
3954 
3955 	client_data_len = vnic_client_data_len(adapter);
3956 
3957 	buffer_size =
3958 	    sizeof(struct ibmvnic_login_buffer) +
3959 	    sizeof(u64) * (adapter->req_tx_queues + adapter->req_rx_queues) +
3960 	    client_data_len;
3961 
3962 	login_buffer = kzalloc(buffer_size, GFP_ATOMIC);
3963 	if (!login_buffer)
3964 		goto buf_alloc_failed;
3965 
3966 	buffer_token = dma_map_single(dev, login_buffer, buffer_size,
3967 				      DMA_TO_DEVICE);
3968 	if (dma_mapping_error(dev, buffer_token)) {
3969 		dev_err(dev, "Couldn't map login buffer\n");
3970 		goto buf_map_failed;
3971 	}
3972 
3973 	rsp_buffer_size = sizeof(struct ibmvnic_login_rsp_buffer) +
3974 			  sizeof(u64) * adapter->req_tx_queues +
3975 			  sizeof(u64) * adapter->req_rx_queues +
3976 			  sizeof(u64) * adapter->req_rx_queues +
3977 			  sizeof(u8) * IBMVNIC_TX_DESC_VERSIONS;
3978 
3979 	login_rsp_buffer = kmalloc(rsp_buffer_size, GFP_ATOMIC);
3980 	if (!login_rsp_buffer)
3981 		goto buf_rsp_alloc_failed;
3982 
3983 	rsp_buffer_token = dma_map_single(dev, login_rsp_buffer,
3984 					  rsp_buffer_size, DMA_FROM_DEVICE);
3985 	if (dma_mapping_error(dev, rsp_buffer_token)) {
3986 		dev_err(dev, "Couldn't map login rsp buffer\n");
3987 		goto buf_rsp_map_failed;
3988 	}
3989 
3990 	adapter->login_buf = login_buffer;
3991 	adapter->login_buf_token = buffer_token;
3992 	adapter->login_buf_sz = buffer_size;
3993 	adapter->login_rsp_buf = login_rsp_buffer;
3994 	adapter->login_rsp_buf_token = rsp_buffer_token;
3995 	adapter->login_rsp_buf_sz = rsp_buffer_size;
3996 
3997 	login_buffer->len = cpu_to_be32(buffer_size);
3998 	login_buffer->version = cpu_to_be32(INITIAL_VERSION_LB);
3999 	login_buffer->num_txcomp_subcrqs = cpu_to_be32(adapter->req_tx_queues);
4000 	login_buffer->off_txcomp_subcrqs =
4001 	    cpu_to_be32(sizeof(struct ibmvnic_login_buffer));
4002 	login_buffer->num_rxcomp_subcrqs = cpu_to_be32(adapter->req_rx_queues);
4003 	login_buffer->off_rxcomp_subcrqs =
4004 	    cpu_to_be32(sizeof(struct ibmvnic_login_buffer) +
4005 			sizeof(u64) * adapter->req_tx_queues);
4006 	login_buffer->login_rsp_ioba = cpu_to_be32(rsp_buffer_token);
4007 	login_buffer->login_rsp_len = cpu_to_be32(rsp_buffer_size);
4008 
4009 	tx_list_p = (__be64 *)((char *)login_buffer +
4010 				      sizeof(struct ibmvnic_login_buffer));
4011 	rx_list_p = (__be64 *)((char *)login_buffer +
4012 				      sizeof(struct ibmvnic_login_buffer) +
4013 				      sizeof(u64) * adapter->req_tx_queues);
4014 
4015 	for (i = 0; i < adapter->req_tx_queues; i++) {
4016 		if (adapter->tx_scrq[i]) {
4017 			tx_list_p[i] =
4018 				cpu_to_be64(adapter->tx_scrq[i]->crq_num);
4019 		}
4020 	}
4021 
4022 	for (i = 0; i < adapter->req_rx_queues; i++) {
4023 		if (adapter->rx_scrq[i]) {
4024 			rx_list_p[i] =
4025 				cpu_to_be64(adapter->rx_scrq[i]->crq_num);
4026 		}
4027 	}
4028 
4029 	/* Insert vNIC login client data */
4030 	vlcd = (struct vnic_login_client_data *)
4031 		((char *)rx_list_p + (sizeof(u64) * adapter->req_rx_queues));
4032 	login_buffer->client_data_offset =
4033 			cpu_to_be32((char *)vlcd - (char *)login_buffer);
4034 	login_buffer->client_data_len = cpu_to_be32(client_data_len);
4035 
4036 	vnic_add_client_data(adapter, vlcd);
4037 
4038 	netdev_dbg(adapter->netdev, "Login Buffer:\n");
4039 	for (i = 0; i < (adapter->login_buf_sz - 1) / 8 + 1; i++) {
4040 		netdev_dbg(adapter->netdev, "%016lx\n",
4041 			   ((unsigned long *)(adapter->login_buf))[i]);
4042 	}
4043 
4044 	memset(&crq, 0, sizeof(crq));
4045 	crq.login.first = IBMVNIC_CRQ_CMD;
4046 	crq.login.cmd = LOGIN;
4047 	crq.login.ioba = cpu_to_be32(buffer_token);
4048 	crq.login.len = cpu_to_be32(buffer_size);
4049 
4050 	adapter->login_pending = true;
4051 	rc = ibmvnic_send_crq(adapter, &crq);
4052 	if (rc) {
4053 		adapter->login_pending = false;
4054 		netdev_err(adapter->netdev, "Failed to send login, rc=%d\n", rc);
4055 		goto buf_rsp_map_failed;
4056 	}
4057 
4058 	return 0;
4059 
4060 buf_rsp_map_failed:
4061 	kfree(login_rsp_buffer);
4062 	adapter->login_rsp_buf = NULL;
4063 buf_rsp_alloc_failed:
4064 	dma_unmap_single(dev, buffer_token, buffer_size, DMA_TO_DEVICE);
4065 buf_map_failed:
4066 	kfree(login_buffer);
4067 	adapter->login_buf = NULL;
4068 buf_alloc_failed:
4069 	return -1;
4070 }
4071 
4072 static int send_request_map(struct ibmvnic_adapter *adapter, dma_addr_t addr,
4073 			    u32 len, u8 map_id)
4074 {
4075 	union ibmvnic_crq crq;
4076 
4077 	memset(&crq, 0, sizeof(crq));
4078 	crq.request_map.first = IBMVNIC_CRQ_CMD;
4079 	crq.request_map.cmd = REQUEST_MAP;
4080 	crq.request_map.map_id = map_id;
4081 	crq.request_map.ioba = cpu_to_be32(addr);
4082 	crq.request_map.len = cpu_to_be32(len);
4083 	return ibmvnic_send_crq(adapter, &crq);
4084 }
4085 
4086 static int send_request_unmap(struct ibmvnic_adapter *adapter, u8 map_id)
4087 {
4088 	union ibmvnic_crq crq;
4089 
4090 	memset(&crq, 0, sizeof(crq));
4091 	crq.request_unmap.first = IBMVNIC_CRQ_CMD;
4092 	crq.request_unmap.cmd = REQUEST_UNMAP;
4093 	crq.request_unmap.map_id = map_id;
4094 	return ibmvnic_send_crq(adapter, &crq);
4095 }
4096 
4097 static void send_query_map(struct ibmvnic_adapter *adapter)
4098 {
4099 	union ibmvnic_crq crq;
4100 
4101 	memset(&crq, 0, sizeof(crq));
4102 	crq.query_map.first = IBMVNIC_CRQ_CMD;
4103 	crq.query_map.cmd = QUERY_MAP;
4104 	ibmvnic_send_crq(adapter, &crq);
4105 }
4106 
4107 /* Send a series of CRQs requesting various capabilities of the VNIC server */
4108 static void send_query_cap(struct ibmvnic_adapter *adapter)
4109 {
4110 	union ibmvnic_crq crq;
4111 
4112 	atomic_set(&adapter->running_cap_crqs, 0);
4113 	memset(&crq, 0, sizeof(crq));
4114 	crq.query_capability.first = IBMVNIC_CRQ_CMD;
4115 	crq.query_capability.cmd = QUERY_CAPABILITY;
4116 
4117 	crq.query_capability.capability = cpu_to_be16(MIN_TX_QUEUES);
4118 	atomic_inc(&adapter->running_cap_crqs);
4119 	ibmvnic_send_crq(adapter, &crq);
4120 
4121 	crq.query_capability.capability = cpu_to_be16(MIN_RX_QUEUES);
4122 	atomic_inc(&adapter->running_cap_crqs);
4123 	ibmvnic_send_crq(adapter, &crq);
4124 
4125 	crq.query_capability.capability = cpu_to_be16(MIN_RX_ADD_QUEUES);
4126 	atomic_inc(&adapter->running_cap_crqs);
4127 	ibmvnic_send_crq(adapter, &crq);
4128 
4129 	crq.query_capability.capability = cpu_to_be16(MAX_TX_QUEUES);
4130 	atomic_inc(&adapter->running_cap_crqs);
4131 	ibmvnic_send_crq(adapter, &crq);
4132 
4133 	crq.query_capability.capability = cpu_to_be16(MAX_RX_QUEUES);
4134 	atomic_inc(&adapter->running_cap_crqs);
4135 	ibmvnic_send_crq(adapter, &crq);
4136 
4137 	crq.query_capability.capability = cpu_to_be16(MAX_RX_ADD_QUEUES);
4138 	atomic_inc(&adapter->running_cap_crqs);
4139 	ibmvnic_send_crq(adapter, &crq);
4140 
4141 	crq.query_capability.capability =
4142 	    cpu_to_be16(MIN_TX_ENTRIES_PER_SUBCRQ);
4143 	atomic_inc(&adapter->running_cap_crqs);
4144 	ibmvnic_send_crq(adapter, &crq);
4145 
4146 	crq.query_capability.capability =
4147 	    cpu_to_be16(MIN_RX_ADD_ENTRIES_PER_SUBCRQ);
4148 	atomic_inc(&adapter->running_cap_crqs);
4149 	ibmvnic_send_crq(adapter, &crq);
4150 
4151 	crq.query_capability.capability =
4152 	    cpu_to_be16(MAX_TX_ENTRIES_PER_SUBCRQ);
4153 	atomic_inc(&adapter->running_cap_crqs);
4154 	ibmvnic_send_crq(adapter, &crq);
4155 
4156 	crq.query_capability.capability =
4157 	    cpu_to_be16(MAX_RX_ADD_ENTRIES_PER_SUBCRQ);
4158 	atomic_inc(&adapter->running_cap_crqs);
4159 	ibmvnic_send_crq(adapter, &crq);
4160 
4161 	crq.query_capability.capability = cpu_to_be16(TCP_IP_OFFLOAD);
4162 	atomic_inc(&adapter->running_cap_crqs);
4163 	ibmvnic_send_crq(adapter, &crq);
4164 
4165 	crq.query_capability.capability = cpu_to_be16(PROMISC_SUPPORTED);
4166 	atomic_inc(&adapter->running_cap_crqs);
4167 	ibmvnic_send_crq(adapter, &crq);
4168 
4169 	crq.query_capability.capability = cpu_to_be16(MIN_MTU);
4170 	atomic_inc(&adapter->running_cap_crqs);
4171 	ibmvnic_send_crq(adapter, &crq);
4172 
4173 	crq.query_capability.capability = cpu_to_be16(MAX_MTU);
4174 	atomic_inc(&adapter->running_cap_crqs);
4175 	ibmvnic_send_crq(adapter, &crq);
4176 
4177 	crq.query_capability.capability = cpu_to_be16(MAX_MULTICAST_FILTERS);
4178 	atomic_inc(&adapter->running_cap_crqs);
4179 	ibmvnic_send_crq(adapter, &crq);
4180 
4181 	crq.query_capability.capability = cpu_to_be16(VLAN_HEADER_INSERTION);
4182 	atomic_inc(&adapter->running_cap_crqs);
4183 	ibmvnic_send_crq(adapter, &crq);
4184 
4185 	crq.query_capability.capability = cpu_to_be16(RX_VLAN_HEADER_INSERTION);
4186 	atomic_inc(&adapter->running_cap_crqs);
4187 	ibmvnic_send_crq(adapter, &crq);
4188 
4189 	crq.query_capability.capability = cpu_to_be16(MAX_TX_SG_ENTRIES);
4190 	atomic_inc(&adapter->running_cap_crqs);
4191 	ibmvnic_send_crq(adapter, &crq);
4192 
4193 	crq.query_capability.capability = cpu_to_be16(RX_SG_SUPPORTED);
4194 	atomic_inc(&adapter->running_cap_crqs);
4195 	ibmvnic_send_crq(adapter, &crq);
4196 
4197 	crq.query_capability.capability = cpu_to_be16(OPT_TX_COMP_SUB_QUEUES);
4198 	atomic_inc(&adapter->running_cap_crqs);
4199 	ibmvnic_send_crq(adapter, &crq);
4200 
4201 	crq.query_capability.capability = cpu_to_be16(OPT_RX_COMP_QUEUES);
4202 	atomic_inc(&adapter->running_cap_crqs);
4203 	ibmvnic_send_crq(adapter, &crq);
4204 
4205 	crq.query_capability.capability =
4206 			cpu_to_be16(OPT_RX_BUFADD_Q_PER_RX_COMP_Q);
4207 	atomic_inc(&adapter->running_cap_crqs);
4208 	ibmvnic_send_crq(adapter, &crq);
4209 
4210 	crq.query_capability.capability =
4211 			cpu_to_be16(OPT_TX_ENTRIES_PER_SUBCRQ);
4212 	atomic_inc(&adapter->running_cap_crqs);
4213 	ibmvnic_send_crq(adapter, &crq);
4214 
4215 	crq.query_capability.capability =
4216 			cpu_to_be16(OPT_RXBA_ENTRIES_PER_SUBCRQ);
4217 	atomic_inc(&adapter->running_cap_crqs);
4218 	ibmvnic_send_crq(adapter, &crq);
4219 
4220 	crq.query_capability.capability = cpu_to_be16(TX_RX_DESC_REQ);
4221 	atomic_inc(&adapter->running_cap_crqs);
4222 	ibmvnic_send_crq(adapter, &crq);
4223 }
4224 
4225 static void send_query_ip_offload(struct ibmvnic_adapter *adapter)
4226 {
4227 	int buf_sz = sizeof(struct ibmvnic_query_ip_offload_buffer);
4228 	struct device *dev = &adapter->vdev->dev;
4229 	union ibmvnic_crq crq;
4230 
4231 	adapter->ip_offload_tok =
4232 		dma_map_single(dev,
4233 			       &adapter->ip_offload_buf,
4234 			       buf_sz,
4235 			       DMA_FROM_DEVICE);
4236 
4237 	if (dma_mapping_error(dev, adapter->ip_offload_tok)) {
4238 		if (!firmware_has_feature(FW_FEATURE_CMO))
4239 			dev_err(dev, "Couldn't map offload buffer\n");
4240 		return;
4241 	}
4242 
4243 	memset(&crq, 0, sizeof(crq));
4244 	crq.query_ip_offload.first = IBMVNIC_CRQ_CMD;
4245 	crq.query_ip_offload.cmd = QUERY_IP_OFFLOAD;
4246 	crq.query_ip_offload.len = cpu_to_be32(buf_sz);
4247 	crq.query_ip_offload.ioba =
4248 	    cpu_to_be32(adapter->ip_offload_tok);
4249 
4250 	ibmvnic_send_crq(adapter, &crq);
4251 }
4252 
4253 static void send_control_ip_offload(struct ibmvnic_adapter *adapter)
4254 {
4255 	struct ibmvnic_control_ip_offload_buffer *ctrl_buf = &adapter->ip_offload_ctrl;
4256 	struct ibmvnic_query_ip_offload_buffer *buf = &adapter->ip_offload_buf;
4257 	struct device *dev = &adapter->vdev->dev;
4258 	netdev_features_t old_hw_features = 0;
4259 	union ibmvnic_crq crq;
4260 
4261 	adapter->ip_offload_ctrl_tok =
4262 		dma_map_single(dev,
4263 			       ctrl_buf,
4264 			       sizeof(adapter->ip_offload_ctrl),
4265 			       DMA_TO_DEVICE);
4266 
4267 	if (dma_mapping_error(dev, adapter->ip_offload_ctrl_tok)) {
4268 		dev_err(dev, "Couldn't map ip offload control buffer\n");
4269 		return;
4270 	}
4271 
4272 	ctrl_buf->len = cpu_to_be32(sizeof(adapter->ip_offload_ctrl));
4273 	ctrl_buf->version = cpu_to_be32(INITIAL_VERSION_IOB);
4274 	ctrl_buf->ipv4_chksum = buf->ipv4_chksum;
4275 	ctrl_buf->ipv6_chksum = buf->ipv6_chksum;
4276 	ctrl_buf->tcp_ipv4_chksum = buf->tcp_ipv4_chksum;
4277 	ctrl_buf->udp_ipv4_chksum = buf->udp_ipv4_chksum;
4278 	ctrl_buf->tcp_ipv6_chksum = buf->tcp_ipv6_chksum;
4279 	ctrl_buf->udp_ipv6_chksum = buf->udp_ipv6_chksum;
4280 	ctrl_buf->large_tx_ipv4 = buf->large_tx_ipv4;
4281 	ctrl_buf->large_tx_ipv6 = buf->large_tx_ipv6;
4282 
4283 	/* large_rx disabled for now, additional features needed */
4284 	ctrl_buf->large_rx_ipv4 = 0;
4285 	ctrl_buf->large_rx_ipv6 = 0;
4286 
4287 	if (adapter->state != VNIC_PROBING) {
4288 		old_hw_features = adapter->netdev->hw_features;
4289 		adapter->netdev->hw_features = 0;
4290 	}
4291 
4292 	adapter->netdev->hw_features = NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO;
4293 
4294 	if (buf->tcp_ipv4_chksum || buf->udp_ipv4_chksum)
4295 		adapter->netdev->hw_features |= NETIF_F_IP_CSUM;
4296 
4297 	if (buf->tcp_ipv6_chksum || buf->udp_ipv6_chksum)
4298 		adapter->netdev->hw_features |= NETIF_F_IPV6_CSUM;
4299 
4300 	if ((adapter->netdev->features &
4301 	    (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)))
4302 		adapter->netdev->hw_features |= NETIF_F_RXCSUM;
4303 
4304 	if (buf->large_tx_ipv4)
4305 		adapter->netdev->hw_features |= NETIF_F_TSO;
4306 	if (buf->large_tx_ipv6)
4307 		adapter->netdev->hw_features |= NETIF_F_TSO6;
4308 
4309 	if (adapter->state == VNIC_PROBING) {
4310 		adapter->netdev->features |= adapter->netdev->hw_features;
4311 	} else if (old_hw_features != adapter->netdev->hw_features) {
4312 		netdev_features_t tmp = 0;
4313 
4314 		/* disable features no longer supported */
4315 		adapter->netdev->features &= adapter->netdev->hw_features;
4316 		/* turn on features now supported if previously enabled */
4317 		tmp = (old_hw_features ^ adapter->netdev->hw_features) &
4318 			adapter->netdev->hw_features;
4319 		adapter->netdev->features |=
4320 				tmp & adapter->netdev->wanted_features;
4321 	}
4322 
4323 	memset(&crq, 0, sizeof(crq));
4324 	crq.control_ip_offload.first = IBMVNIC_CRQ_CMD;
4325 	crq.control_ip_offload.cmd = CONTROL_IP_OFFLOAD;
4326 	crq.control_ip_offload.len =
4327 	    cpu_to_be32(sizeof(adapter->ip_offload_ctrl));
4328 	crq.control_ip_offload.ioba = cpu_to_be32(adapter->ip_offload_ctrl_tok);
4329 	ibmvnic_send_crq(adapter, &crq);
4330 }
4331 
4332 static void handle_vpd_size_rsp(union ibmvnic_crq *crq,
4333 				struct ibmvnic_adapter *adapter)
4334 {
4335 	struct device *dev = &adapter->vdev->dev;
4336 
4337 	if (crq->get_vpd_size_rsp.rc.code) {
4338 		dev_err(dev, "Error retrieving VPD size, rc=%x\n",
4339 			crq->get_vpd_size_rsp.rc.code);
4340 		complete(&adapter->fw_done);
4341 		return;
4342 	}
4343 
4344 	adapter->vpd->len = be64_to_cpu(crq->get_vpd_size_rsp.len);
4345 	complete(&adapter->fw_done);
4346 }
4347 
4348 static void handle_vpd_rsp(union ibmvnic_crq *crq,
4349 			   struct ibmvnic_adapter *adapter)
4350 {
4351 	struct device *dev = &adapter->vdev->dev;
4352 	unsigned char *substr = NULL;
4353 	u8 fw_level_len = 0;
4354 
4355 	memset(adapter->fw_version, 0, 32);
4356 
4357 	dma_unmap_single(dev, adapter->vpd->dma_addr, adapter->vpd->len,
4358 			 DMA_FROM_DEVICE);
4359 
4360 	if (crq->get_vpd_rsp.rc.code) {
4361 		dev_err(dev, "Error retrieving VPD from device, rc=%x\n",
4362 			crq->get_vpd_rsp.rc.code);
4363 		goto complete;
4364 	}
4365 
4366 	/* get the position of the firmware version info
4367 	 * located after the ASCII 'RM' substring in the buffer
4368 	 */
4369 	substr = strnstr(adapter->vpd->buff, "RM", adapter->vpd->len);
4370 	if (!substr) {
4371 		dev_info(dev, "Warning - No FW level has been provided in the VPD buffer by the VIOS Server\n");
4372 		goto complete;
4373 	}
4374 
4375 	/* get length of firmware level ASCII substring */
4376 	if ((substr + 2) < (adapter->vpd->buff + adapter->vpd->len)) {
4377 		fw_level_len = *(substr + 2);
4378 	} else {
4379 		dev_info(dev, "Length of FW substr extrapolated VDP buff\n");
4380 		goto complete;
4381 	}
4382 
4383 	/* copy firmware version string from vpd into adapter */
4384 	if ((substr + 3 + fw_level_len) <
4385 	    (adapter->vpd->buff + adapter->vpd->len)) {
4386 		strncpy((char *)adapter->fw_version, substr + 3, fw_level_len);
4387 	} else {
4388 		dev_info(dev, "FW substr extrapolated VPD buff\n");
4389 	}
4390 
4391 complete:
4392 	if (adapter->fw_version[0] == '\0')
4393 		strscpy((char *)adapter->fw_version, "N/A", sizeof(adapter->fw_version));
4394 	complete(&adapter->fw_done);
4395 }
4396 
4397 static void handle_query_ip_offload_rsp(struct ibmvnic_adapter *adapter)
4398 {
4399 	struct device *dev = &adapter->vdev->dev;
4400 	struct ibmvnic_query_ip_offload_buffer *buf = &adapter->ip_offload_buf;
4401 	int i;
4402 
4403 	dma_unmap_single(dev, adapter->ip_offload_tok,
4404 			 sizeof(adapter->ip_offload_buf), DMA_FROM_DEVICE);
4405 
4406 	netdev_dbg(adapter->netdev, "Query IP Offload Buffer:\n");
4407 	for (i = 0; i < (sizeof(adapter->ip_offload_buf) - 1) / 8 + 1; i++)
4408 		netdev_dbg(adapter->netdev, "%016lx\n",
4409 			   ((unsigned long *)(buf))[i]);
4410 
4411 	netdev_dbg(adapter->netdev, "ipv4_chksum = %d\n", buf->ipv4_chksum);
4412 	netdev_dbg(adapter->netdev, "ipv6_chksum = %d\n", buf->ipv6_chksum);
4413 	netdev_dbg(adapter->netdev, "tcp_ipv4_chksum = %d\n",
4414 		   buf->tcp_ipv4_chksum);
4415 	netdev_dbg(adapter->netdev, "tcp_ipv6_chksum = %d\n",
4416 		   buf->tcp_ipv6_chksum);
4417 	netdev_dbg(adapter->netdev, "udp_ipv4_chksum = %d\n",
4418 		   buf->udp_ipv4_chksum);
4419 	netdev_dbg(adapter->netdev, "udp_ipv6_chksum = %d\n",
4420 		   buf->udp_ipv6_chksum);
4421 	netdev_dbg(adapter->netdev, "large_tx_ipv4 = %d\n",
4422 		   buf->large_tx_ipv4);
4423 	netdev_dbg(adapter->netdev, "large_tx_ipv6 = %d\n",
4424 		   buf->large_tx_ipv6);
4425 	netdev_dbg(adapter->netdev, "large_rx_ipv4 = %d\n",
4426 		   buf->large_rx_ipv4);
4427 	netdev_dbg(adapter->netdev, "large_rx_ipv6 = %d\n",
4428 		   buf->large_rx_ipv6);
4429 	netdev_dbg(adapter->netdev, "max_ipv4_hdr_sz = %d\n",
4430 		   buf->max_ipv4_header_size);
4431 	netdev_dbg(adapter->netdev, "max_ipv6_hdr_sz = %d\n",
4432 		   buf->max_ipv6_header_size);
4433 	netdev_dbg(adapter->netdev, "max_tcp_hdr_size = %d\n",
4434 		   buf->max_tcp_header_size);
4435 	netdev_dbg(adapter->netdev, "max_udp_hdr_size = %d\n",
4436 		   buf->max_udp_header_size);
4437 	netdev_dbg(adapter->netdev, "max_large_tx_size = %d\n",
4438 		   buf->max_large_tx_size);
4439 	netdev_dbg(adapter->netdev, "max_large_rx_size = %d\n",
4440 		   buf->max_large_rx_size);
4441 	netdev_dbg(adapter->netdev, "ipv6_ext_hdr = %d\n",
4442 		   buf->ipv6_extension_header);
4443 	netdev_dbg(adapter->netdev, "tcp_pseudosum_req = %d\n",
4444 		   buf->tcp_pseudosum_req);
4445 	netdev_dbg(adapter->netdev, "num_ipv6_ext_hd = %d\n",
4446 		   buf->num_ipv6_ext_headers);
4447 	netdev_dbg(adapter->netdev, "off_ipv6_ext_hd = %d\n",
4448 		   buf->off_ipv6_ext_headers);
4449 
4450 	send_control_ip_offload(adapter);
4451 }
4452 
4453 static const char *ibmvnic_fw_err_cause(u16 cause)
4454 {
4455 	switch (cause) {
4456 	case ADAPTER_PROBLEM:
4457 		return "adapter problem";
4458 	case BUS_PROBLEM:
4459 		return "bus problem";
4460 	case FW_PROBLEM:
4461 		return "firmware problem";
4462 	case DD_PROBLEM:
4463 		return "device driver problem";
4464 	case EEH_RECOVERY:
4465 		return "EEH recovery";
4466 	case FW_UPDATED:
4467 		return "firmware updated";
4468 	case LOW_MEMORY:
4469 		return "low Memory";
4470 	default:
4471 		return "unknown";
4472 	}
4473 }
4474 
4475 static void handle_error_indication(union ibmvnic_crq *crq,
4476 				    struct ibmvnic_adapter *adapter)
4477 {
4478 	struct device *dev = &adapter->vdev->dev;
4479 	u16 cause;
4480 
4481 	cause = be16_to_cpu(crq->error_indication.error_cause);
4482 
4483 	dev_warn_ratelimited(dev,
4484 			     "Firmware reports %serror, cause: %s. Starting recovery...\n",
4485 			     crq->error_indication.flags
4486 				& IBMVNIC_FATAL_ERROR ? "FATAL " : "",
4487 			     ibmvnic_fw_err_cause(cause));
4488 
4489 	if (crq->error_indication.flags & IBMVNIC_FATAL_ERROR)
4490 		ibmvnic_reset(adapter, VNIC_RESET_FATAL);
4491 	else
4492 		ibmvnic_reset(adapter, VNIC_RESET_NON_FATAL);
4493 }
4494 
4495 static int handle_change_mac_rsp(union ibmvnic_crq *crq,
4496 				 struct ibmvnic_adapter *adapter)
4497 {
4498 	struct net_device *netdev = adapter->netdev;
4499 	struct device *dev = &adapter->vdev->dev;
4500 	long rc;
4501 
4502 	rc = crq->change_mac_addr_rsp.rc.code;
4503 	if (rc) {
4504 		dev_err(dev, "Error %ld in CHANGE_MAC_ADDR_RSP\n", rc);
4505 		goto out;
4506 	}
4507 	/* crq->change_mac_addr.mac_addr is the requested one
4508 	 * crq->change_mac_addr_rsp.mac_addr is the returned valid one.
4509 	 */
4510 	ether_addr_copy(netdev->dev_addr,
4511 			&crq->change_mac_addr_rsp.mac_addr[0]);
4512 	ether_addr_copy(adapter->mac_addr,
4513 			&crq->change_mac_addr_rsp.mac_addr[0]);
4514 out:
4515 	complete(&adapter->fw_done);
4516 	return rc;
4517 }
4518 
4519 static void handle_request_cap_rsp(union ibmvnic_crq *crq,
4520 				   struct ibmvnic_adapter *adapter)
4521 {
4522 	struct device *dev = &adapter->vdev->dev;
4523 	u64 *req_value;
4524 	char *name;
4525 
4526 	atomic_dec(&adapter->running_cap_crqs);
4527 	switch (be16_to_cpu(crq->request_capability_rsp.capability)) {
4528 	case REQ_TX_QUEUES:
4529 		req_value = &adapter->req_tx_queues;
4530 		name = "tx";
4531 		break;
4532 	case REQ_RX_QUEUES:
4533 		req_value = &adapter->req_rx_queues;
4534 		name = "rx";
4535 		break;
4536 	case REQ_RX_ADD_QUEUES:
4537 		req_value = &adapter->req_rx_add_queues;
4538 		name = "rx_add";
4539 		break;
4540 	case REQ_TX_ENTRIES_PER_SUBCRQ:
4541 		req_value = &adapter->req_tx_entries_per_subcrq;
4542 		name = "tx_entries_per_subcrq";
4543 		break;
4544 	case REQ_RX_ADD_ENTRIES_PER_SUBCRQ:
4545 		req_value = &adapter->req_rx_add_entries_per_subcrq;
4546 		name = "rx_add_entries_per_subcrq";
4547 		break;
4548 	case REQ_MTU:
4549 		req_value = &adapter->req_mtu;
4550 		name = "mtu";
4551 		break;
4552 	case PROMISC_REQUESTED:
4553 		req_value = &adapter->promisc;
4554 		name = "promisc";
4555 		break;
4556 	default:
4557 		dev_err(dev, "Got invalid cap request rsp %d\n",
4558 			crq->request_capability.capability);
4559 		return;
4560 	}
4561 
4562 	switch (crq->request_capability_rsp.rc.code) {
4563 	case SUCCESS:
4564 		break;
4565 	case PARTIALSUCCESS:
4566 		dev_info(dev, "req=%lld, rsp=%ld in %s queue, retrying.\n",
4567 			 *req_value,
4568 			 (long)be64_to_cpu(crq->request_capability_rsp.number),
4569 			 name);
4570 
4571 		if (be16_to_cpu(crq->request_capability_rsp.capability) ==
4572 		    REQ_MTU) {
4573 			pr_err("mtu of %llu is not supported. Reverting.\n",
4574 			       *req_value);
4575 			*req_value = adapter->fallback.mtu;
4576 		} else {
4577 			*req_value =
4578 				be64_to_cpu(crq->request_capability_rsp.number);
4579 		}
4580 
4581 		send_request_cap(adapter, 1);
4582 		return;
4583 	default:
4584 		dev_err(dev, "Error %d in request cap rsp\n",
4585 			crq->request_capability_rsp.rc.code);
4586 		return;
4587 	}
4588 
4589 	/* Done receiving requested capabilities, query IP offload support */
4590 	if (atomic_read(&adapter->running_cap_crqs) == 0) {
4591 		adapter->wait_capability = false;
4592 		send_query_ip_offload(adapter);
4593 	}
4594 }
4595 
4596 static int handle_login_rsp(union ibmvnic_crq *login_rsp_crq,
4597 			    struct ibmvnic_adapter *adapter)
4598 {
4599 	struct device *dev = &adapter->vdev->dev;
4600 	struct net_device *netdev = adapter->netdev;
4601 	struct ibmvnic_login_rsp_buffer *login_rsp = adapter->login_rsp_buf;
4602 	struct ibmvnic_login_buffer *login = adapter->login_buf;
4603 	u64 *tx_handle_array;
4604 	u64 *rx_handle_array;
4605 	int num_tx_pools;
4606 	int num_rx_pools;
4607 	u64 *size_array;
4608 	int i;
4609 
4610 	/* CHECK: Test/set of login_pending does not need to be atomic
4611 	 * because only ibmvnic_tasklet tests/clears this.
4612 	 */
4613 	if (!adapter->login_pending) {
4614 		netdev_warn(netdev, "Ignoring unexpected login response\n");
4615 		return 0;
4616 	}
4617 	adapter->login_pending = false;
4618 
4619 	dma_unmap_single(dev, adapter->login_buf_token, adapter->login_buf_sz,
4620 			 DMA_TO_DEVICE);
4621 	dma_unmap_single(dev, adapter->login_rsp_buf_token,
4622 			 adapter->login_rsp_buf_sz, DMA_FROM_DEVICE);
4623 
4624 	/* If the number of queues requested can't be allocated by the
4625 	 * server, the login response will return with code 1. We will need
4626 	 * to resend the login buffer with fewer queues requested.
4627 	 */
4628 	if (login_rsp_crq->generic.rc.code) {
4629 		adapter->init_done_rc = login_rsp_crq->generic.rc.code;
4630 		complete(&adapter->init_done);
4631 		return 0;
4632 	}
4633 
4634 	netdev->mtu = adapter->req_mtu - ETH_HLEN;
4635 
4636 	netdev_dbg(adapter->netdev, "Login Response Buffer:\n");
4637 	for (i = 0; i < (adapter->login_rsp_buf_sz - 1) / 8 + 1; i++) {
4638 		netdev_dbg(adapter->netdev, "%016lx\n",
4639 			   ((unsigned long *)(adapter->login_rsp_buf))[i]);
4640 	}
4641 
4642 	/* Sanity checks */
4643 	if (login->num_txcomp_subcrqs != login_rsp->num_txsubm_subcrqs ||
4644 	    (be32_to_cpu(login->num_rxcomp_subcrqs) *
4645 	     adapter->req_rx_add_queues !=
4646 	     be32_to_cpu(login_rsp->num_rxadd_subcrqs))) {
4647 		dev_err(dev, "FATAL: Inconsistent login and login rsp\n");
4648 		ibmvnic_reset(adapter, VNIC_RESET_FATAL);
4649 		return -EIO;
4650 	}
4651 	size_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +
4652 		be32_to_cpu(adapter->login_rsp_buf->off_rxadd_buff_size));
4653 	/* variable buffer sizes are not supported, so just read the
4654 	 * first entry.
4655 	 */
4656 	adapter->cur_rx_buf_sz = be64_to_cpu(size_array[0]);
4657 
4658 	num_tx_pools = be32_to_cpu(adapter->login_rsp_buf->num_txsubm_subcrqs);
4659 	num_rx_pools = be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);
4660 
4661 	tx_handle_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +
4662 				  be32_to_cpu(adapter->login_rsp_buf->off_txsubm_subcrqs));
4663 	rx_handle_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +
4664 				  be32_to_cpu(adapter->login_rsp_buf->off_rxadd_subcrqs));
4665 
4666 	for (i = 0; i < num_tx_pools; i++)
4667 		adapter->tx_scrq[i]->handle = tx_handle_array[i];
4668 
4669 	for (i = 0; i < num_rx_pools; i++)
4670 		adapter->rx_scrq[i]->handle = rx_handle_array[i];
4671 
4672 	adapter->num_active_tx_scrqs = num_tx_pools;
4673 	adapter->num_active_rx_scrqs = num_rx_pools;
4674 	release_login_rsp_buffer(adapter);
4675 	release_login_buffer(adapter);
4676 	complete(&adapter->init_done);
4677 
4678 	return 0;
4679 }
4680 
4681 static void handle_request_unmap_rsp(union ibmvnic_crq *crq,
4682 				     struct ibmvnic_adapter *adapter)
4683 {
4684 	struct device *dev = &adapter->vdev->dev;
4685 	long rc;
4686 
4687 	rc = crq->request_unmap_rsp.rc.code;
4688 	if (rc)
4689 		dev_err(dev, "Error %ld in REQUEST_UNMAP_RSP\n", rc);
4690 }
4691 
4692 static void handle_query_map_rsp(union ibmvnic_crq *crq,
4693 				 struct ibmvnic_adapter *adapter)
4694 {
4695 	struct net_device *netdev = adapter->netdev;
4696 	struct device *dev = &adapter->vdev->dev;
4697 	long rc;
4698 
4699 	rc = crq->query_map_rsp.rc.code;
4700 	if (rc) {
4701 		dev_err(dev, "Error %ld in QUERY_MAP_RSP\n", rc);
4702 		return;
4703 	}
4704 	netdev_dbg(netdev, "page_size = %d\ntot_pages = %d\nfree_pages = %d\n",
4705 		   crq->query_map_rsp.page_size, crq->query_map_rsp.tot_pages,
4706 		   crq->query_map_rsp.free_pages);
4707 }
4708 
4709 static void handle_query_cap_rsp(union ibmvnic_crq *crq,
4710 				 struct ibmvnic_adapter *adapter)
4711 {
4712 	struct net_device *netdev = adapter->netdev;
4713 	struct device *dev = &adapter->vdev->dev;
4714 	long rc;
4715 
4716 	atomic_dec(&adapter->running_cap_crqs);
4717 	netdev_dbg(netdev, "Outstanding queries: %d\n",
4718 		   atomic_read(&adapter->running_cap_crqs));
4719 	rc = crq->query_capability.rc.code;
4720 	if (rc) {
4721 		dev_err(dev, "Error %ld in QUERY_CAP_RSP\n", rc);
4722 		goto out;
4723 	}
4724 
4725 	switch (be16_to_cpu(crq->query_capability.capability)) {
4726 	case MIN_TX_QUEUES:
4727 		adapter->min_tx_queues =
4728 		    be64_to_cpu(crq->query_capability.number);
4729 		netdev_dbg(netdev, "min_tx_queues = %lld\n",
4730 			   adapter->min_tx_queues);
4731 		break;
4732 	case MIN_RX_QUEUES:
4733 		adapter->min_rx_queues =
4734 		    be64_to_cpu(crq->query_capability.number);
4735 		netdev_dbg(netdev, "min_rx_queues = %lld\n",
4736 			   adapter->min_rx_queues);
4737 		break;
4738 	case MIN_RX_ADD_QUEUES:
4739 		adapter->min_rx_add_queues =
4740 		    be64_to_cpu(crq->query_capability.number);
4741 		netdev_dbg(netdev, "min_rx_add_queues = %lld\n",
4742 			   adapter->min_rx_add_queues);
4743 		break;
4744 	case MAX_TX_QUEUES:
4745 		adapter->max_tx_queues =
4746 		    be64_to_cpu(crq->query_capability.number);
4747 		netdev_dbg(netdev, "max_tx_queues = %lld\n",
4748 			   adapter->max_tx_queues);
4749 		break;
4750 	case MAX_RX_QUEUES:
4751 		adapter->max_rx_queues =
4752 		    be64_to_cpu(crq->query_capability.number);
4753 		netdev_dbg(netdev, "max_rx_queues = %lld\n",
4754 			   adapter->max_rx_queues);
4755 		break;
4756 	case MAX_RX_ADD_QUEUES:
4757 		adapter->max_rx_add_queues =
4758 		    be64_to_cpu(crq->query_capability.number);
4759 		netdev_dbg(netdev, "max_rx_add_queues = %lld\n",
4760 			   adapter->max_rx_add_queues);
4761 		break;
4762 	case MIN_TX_ENTRIES_PER_SUBCRQ:
4763 		adapter->min_tx_entries_per_subcrq =
4764 		    be64_to_cpu(crq->query_capability.number);
4765 		netdev_dbg(netdev, "min_tx_entries_per_subcrq = %lld\n",
4766 			   adapter->min_tx_entries_per_subcrq);
4767 		break;
4768 	case MIN_RX_ADD_ENTRIES_PER_SUBCRQ:
4769 		adapter->min_rx_add_entries_per_subcrq =
4770 		    be64_to_cpu(crq->query_capability.number);
4771 		netdev_dbg(netdev, "min_rx_add_entrs_per_subcrq = %lld\n",
4772 			   adapter->min_rx_add_entries_per_subcrq);
4773 		break;
4774 	case MAX_TX_ENTRIES_PER_SUBCRQ:
4775 		adapter->max_tx_entries_per_subcrq =
4776 		    be64_to_cpu(crq->query_capability.number);
4777 		netdev_dbg(netdev, "max_tx_entries_per_subcrq = %lld\n",
4778 			   adapter->max_tx_entries_per_subcrq);
4779 		break;
4780 	case MAX_RX_ADD_ENTRIES_PER_SUBCRQ:
4781 		adapter->max_rx_add_entries_per_subcrq =
4782 		    be64_to_cpu(crq->query_capability.number);
4783 		netdev_dbg(netdev, "max_rx_add_entrs_per_subcrq = %lld\n",
4784 			   adapter->max_rx_add_entries_per_subcrq);
4785 		break;
4786 	case TCP_IP_OFFLOAD:
4787 		adapter->tcp_ip_offload =
4788 		    be64_to_cpu(crq->query_capability.number);
4789 		netdev_dbg(netdev, "tcp_ip_offload = %lld\n",
4790 			   adapter->tcp_ip_offload);
4791 		break;
4792 	case PROMISC_SUPPORTED:
4793 		adapter->promisc_supported =
4794 		    be64_to_cpu(crq->query_capability.number);
4795 		netdev_dbg(netdev, "promisc_supported = %lld\n",
4796 			   adapter->promisc_supported);
4797 		break;
4798 	case MIN_MTU:
4799 		adapter->min_mtu = be64_to_cpu(crq->query_capability.number);
4800 		netdev->min_mtu = adapter->min_mtu - ETH_HLEN;
4801 		netdev_dbg(netdev, "min_mtu = %lld\n", adapter->min_mtu);
4802 		break;
4803 	case MAX_MTU:
4804 		adapter->max_mtu = be64_to_cpu(crq->query_capability.number);
4805 		netdev->max_mtu = adapter->max_mtu - ETH_HLEN;
4806 		netdev_dbg(netdev, "max_mtu = %lld\n", adapter->max_mtu);
4807 		break;
4808 	case MAX_MULTICAST_FILTERS:
4809 		adapter->max_multicast_filters =
4810 		    be64_to_cpu(crq->query_capability.number);
4811 		netdev_dbg(netdev, "max_multicast_filters = %lld\n",
4812 			   adapter->max_multicast_filters);
4813 		break;
4814 	case VLAN_HEADER_INSERTION:
4815 		adapter->vlan_header_insertion =
4816 		    be64_to_cpu(crq->query_capability.number);
4817 		if (adapter->vlan_header_insertion)
4818 			netdev->features |= NETIF_F_HW_VLAN_STAG_TX;
4819 		netdev_dbg(netdev, "vlan_header_insertion = %lld\n",
4820 			   adapter->vlan_header_insertion);
4821 		break;
4822 	case RX_VLAN_HEADER_INSERTION:
4823 		adapter->rx_vlan_header_insertion =
4824 		    be64_to_cpu(crq->query_capability.number);
4825 		netdev_dbg(netdev, "rx_vlan_header_insertion = %lld\n",
4826 			   adapter->rx_vlan_header_insertion);
4827 		break;
4828 	case MAX_TX_SG_ENTRIES:
4829 		adapter->max_tx_sg_entries =
4830 		    be64_to_cpu(crq->query_capability.number);
4831 		netdev_dbg(netdev, "max_tx_sg_entries = %lld\n",
4832 			   adapter->max_tx_sg_entries);
4833 		break;
4834 	case RX_SG_SUPPORTED:
4835 		adapter->rx_sg_supported =
4836 		    be64_to_cpu(crq->query_capability.number);
4837 		netdev_dbg(netdev, "rx_sg_supported = %lld\n",
4838 			   adapter->rx_sg_supported);
4839 		break;
4840 	case OPT_TX_COMP_SUB_QUEUES:
4841 		adapter->opt_tx_comp_sub_queues =
4842 		    be64_to_cpu(crq->query_capability.number);
4843 		netdev_dbg(netdev, "opt_tx_comp_sub_queues = %lld\n",
4844 			   adapter->opt_tx_comp_sub_queues);
4845 		break;
4846 	case OPT_RX_COMP_QUEUES:
4847 		adapter->opt_rx_comp_queues =
4848 		    be64_to_cpu(crq->query_capability.number);
4849 		netdev_dbg(netdev, "opt_rx_comp_queues = %lld\n",
4850 			   adapter->opt_rx_comp_queues);
4851 		break;
4852 	case OPT_RX_BUFADD_Q_PER_RX_COMP_Q:
4853 		adapter->opt_rx_bufadd_q_per_rx_comp_q =
4854 		    be64_to_cpu(crq->query_capability.number);
4855 		netdev_dbg(netdev, "opt_rx_bufadd_q_per_rx_comp_q = %lld\n",
4856 			   adapter->opt_rx_bufadd_q_per_rx_comp_q);
4857 		break;
4858 	case OPT_TX_ENTRIES_PER_SUBCRQ:
4859 		adapter->opt_tx_entries_per_subcrq =
4860 		    be64_to_cpu(crq->query_capability.number);
4861 		netdev_dbg(netdev, "opt_tx_entries_per_subcrq = %lld\n",
4862 			   adapter->opt_tx_entries_per_subcrq);
4863 		break;
4864 	case OPT_RXBA_ENTRIES_PER_SUBCRQ:
4865 		adapter->opt_rxba_entries_per_subcrq =
4866 		    be64_to_cpu(crq->query_capability.number);
4867 		netdev_dbg(netdev, "opt_rxba_entries_per_subcrq = %lld\n",
4868 			   adapter->opt_rxba_entries_per_subcrq);
4869 		break;
4870 	case TX_RX_DESC_REQ:
4871 		adapter->tx_rx_desc_req = crq->query_capability.number;
4872 		netdev_dbg(netdev, "tx_rx_desc_req = %llx\n",
4873 			   adapter->tx_rx_desc_req);
4874 		break;
4875 
4876 	default:
4877 		netdev_err(netdev, "Got invalid cap rsp %d\n",
4878 			   crq->query_capability.capability);
4879 	}
4880 
4881 out:
4882 	if (atomic_read(&adapter->running_cap_crqs) == 0) {
4883 		adapter->wait_capability = false;
4884 		send_request_cap(adapter, 0);
4885 	}
4886 }
4887 
4888 static int send_query_phys_parms(struct ibmvnic_adapter *adapter)
4889 {
4890 	union ibmvnic_crq crq;
4891 	int rc;
4892 
4893 	memset(&crq, 0, sizeof(crq));
4894 	crq.query_phys_parms.first = IBMVNIC_CRQ_CMD;
4895 	crq.query_phys_parms.cmd = QUERY_PHYS_PARMS;
4896 
4897 	mutex_lock(&adapter->fw_lock);
4898 	adapter->fw_done_rc = 0;
4899 	reinit_completion(&adapter->fw_done);
4900 
4901 	rc = ibmvnic_send_crq(adapter, &crq);
4902 	if (rc) {
4903 		mutex_unlock(&adapter->fw_lock);
4904 		return rc;
4905 	}
4906 
4907 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);
4908 	if (rc) {
4909 		mutex_unlock(&adapter->fw_lock);
4910 		return rc;
4911 	}
4912 
4913 	mutex_unlock(&adapter->fw_lock);
4914 	return adapter->fw_done_rc ? -EIO : 0;
4915 }
4916 
4917 static int handle_query_phys_parms_rsp(union ibmvnic_crq *crq,
4918 				       struct ibmvnic_adapter *adapter)
4919 {
4920 	struct net_device *netdev = adapter->netdev;
4921 	int rc;
4922 	__be32 rspeed = cpu_to_be32(crq->query_phys_parms_rsp.speed);
4923 
4924 	rc = crq->query_phys_parms_rsp.rc.code;
4925 	if (rc) {
4926 		netdev_err(netdev, "Error %d in QUERY_PHYS_PARMS\n", rc);
4927 		return rc;
4928 	}
4929 	switch (rspeed) {
4930 	case IBMVNIC_10MBPS:
4931 		adapter->speed = SPEED_10;
4932 		break;
4933 	case IBMVNIC_100MBPS:
4934 		adapter->speed = SPEED_100;
4935 		break;
4936 	case IBMVNIC_1GBPS:
4937 		adapter->speed = SPEED_1000;
4938 		break;
4939 	case IBMVNIC_10GBPS:
4940 		adapter->speed = SPEED_10000;
4941 		break;
4942 	case IBMVNIC_25GBPS:
4943 		adapter->speed = SPEED_25000;
4944 		break;
4945 	case IBMVNIC_40GBPS:
4946 		adapter->speed = SPEED_40000;
4947 		break;
4948 	case IBMVNIC_50GBPS:
4949 		adapter->speed = SPEED_50000;
4950 		break;
4951 	case IBMVNIC_100GBPS:
4952 		adapter->speed = SPEED_100000;
4953 		break;
4954 	case IBMVNIC_200GBPS:
4955 		adapter->speed = SPEED_200000;
4956 		break;
4957 	default:
4958 		if (netif_carrier_ok(netdev))
4959 			netdev_warn(netdev, "Unknown speed 0x%08x\n", rspeed);
4960 		adapter->speed = SPEED_UNKNOWN;
4961 	}
4962 	if (crq->query_phys_parms_rsp.flags1 & IBMVNIC_FULL_DUPLEX)
4963 		adapter->duplex = DUPLEX_FULL;
4964 	else if (crq->query_phys_parms_rsp.flags1 & IBMVNIC_HALF_DUPLEX)
4965 		adapter->duplex = DUPLEX_HALF;
4966 	else
4967 		adapter->duplex = DUPLEX_UNKNOWN;
4968 
4969 	return rc;
4970 }
4971 
4972 static void ibmvnic_handle_crq(union ibmvnic_crq *crq,
4973 			       struct ibmvnic_adapter *adapter)
4974 {
4975 	struct ibmvnic_generic_crq *gen_crq = &crq->generic;
4976 	struct net_device *netdev = adapter->netdev;
4977 	struct device *dev = &adapter->vdev->dev;
4978 	u64 *u64_crq = (u64 *)crq;
4979 	long rc;
4980 
4981 	netdev_dbg(netdev, "Handling CRQ: %016lx %016lx\n",
4982 		   (unsigned long)cpu_to_be64(u64_crq[0]),
4983 		   (unsigned long)cpu_to_be64(u64_crq[1]));
4984 	switch (gen_crq->first) {
4985 	case IBMVNIC_CRQ_INIT_RSP:
4986 		switch (gen_crq->cmd) {
4987 		case IBMVNIC_CRQ_INIT:
4988 			dev_info(dev, "Partner initialized\n");
4989 			adapter->from_passive_init = true;
4990 			/* Discard any stale login responses from prev reset.
4991 			 * CHECK: should we clear even on INIT_COMPLETE?
4992 			 */
4993 			adapter->login_pending = false;
4994 
4995 			if (!completion_done(&adapter->init_done)) {
4996 				complete(&adapter->init_done);
4997 				adapter->init_done_rc = -EIO;
4998 			}
4999 
5000 			if (adapter->state == VNIC_DOWN)
5001 				rc = ibmvnic_reset(adapter, VNIC_RESET_PASSIVE_INIT);
5002 			else
5003 				rc = ibmvnic_reset(adapter, VNIC_RESET_FAILOVER);
5004 
5005 			if (rc && rc != -EBUSY) {
5006 				/* We were unable to schedule the failover
5007 				 * reset either because the adapter was still
5008 				 * probing (eg: during kexec) or we could not
5009 				 * allocate memory. Clear the failover_pending
5010 				 * flag since no one else will. We ignore
5011 				 * EBUSY because it means either FAILOVER reset
5012 				 * is already scheduled or the adapter is
5013 				 * being removed.
5014 				 */
5015 				netdev_err(netdev,
5016 					   "Error %ld scheduling failover reset\n",
5017 					   rc);
5018 				adapter->failover_pending = false;
5019 			}
5020 			break;
5021 		case IBMVNIC_CRQ_INIT_COMPLETE:
5022 			dev_info(dev, "Partner initialization complete\n");
5023 			adapter->crq.active = true;
5024 			send_version_xchg(adapter);
5025 			break;
5026 		default:
5027 			dev_err(dev, "Unknown crq cmd: %d\n", gen_crq->cmd);
5028 		}
5029 		return;
5030 	case IBMVNIC_CRQ_XPORT_EVENT:
5031 		netif_carrier_off(netdev);
5032 		adapter->crq.active = false;
5033 		/* terminate any thread waiting for a response
5034 		 * from the device
5035 		 */
5036 		if (!completion_done(&adapter->fw_done)) {
5037 			adapter->fw_done_rc = -EIO;
5038 			complete(&adapter->fw_done);
5039 		}
5040 		if (!completion_done(&adapter->stats_done))
5041 			complete(&adapter->stats_done);
5042 		if (test_bit(0, &adapter->resetting))
5043 			adapter->force_reset_recovery = true;
5044 		if (gen_crq->cmd == IBMVNIC_PARTITION_MIGRATED) {
5045 			dev_info(dev, "Migrated, re-enabling adapter\n");
5046 			ibmvnic_reset(adapter, VNIC_RESET_MOBILITY);
5047 		} else if (gen_crq->cmd == IBMVNIC_DEVICE_FAILOVER) {
5048 			dev_info(dev, "Backing device failover detected\n");
5049 			adapter->failover_pending = true;
5050 		} else {
5051 			/* The adapter lost the connection */
5052 			dev_err(dev, "Virtual Adapter failed (rc=%d)\n",
5053 				gen_crq->cmd);
5054 			ibmvnic_reset(adapter, VNIC_RESET_FATAL);
5055 		}
5056 		return;
5057 	case IBMVNIC_CRQ_CMD_RSP:
5058 		break;
5059 	default:
5060 		dev_err(dev, "Got an invalid msg type 0x%02x\n",
5061 			gen_crq->first);
5062 		return;
5063 	}
5064 
5065 	switch (gen_crq->cmd) {
5066 	case VERSION_EXCHANGE_RSP:
5067 		rc = crq->version_exchange_rsp.rc.code;
5068 		if (rc) {
5069 			dev_err(dev, "Error %ld in VERSION_EXCHG_RSP\n", rc);
5070 			break;
5071 		}
5072 		ibmvnic_version =
5073 			    be16_to_cpu(crq->version_exchange_rsp.version);
5074 		dev_info(dev, "Partner protocol version is %d\n",
5075 			 ibmvnic_version);
5076 		send_query_cap(adapter);
5077 		break;
5078 	case QUERY_CAPABILITY_RSP:
5079 		handle_query_cap_rsp(crq, adapter);
5080 		break;
5081 	case QUERY_MAP_RSP:
5082 		handle_query_map_rsp(crq, adapter);
5083 		break;
5084 	case REQUEST_MAP_RSP:
5085 		adapter->fw_done_rc = crq->request_map_rsp.rc.code;
5086 		complete(&adapter->fw_done);
5087 		break;
5088 	case REQUEST_UNMAP_RSP:
5089 		handle_request_unmap_rsp(crq, adapter);
5090 		break;
5091 	case REQUEST_CAPABILITY_RSP:
5092 		handle_request_cap_rsp(crq, adapter);
5093 		break;
5094 	case LOGIN_RSP:
5095 		netdev_dbg(netdev, "Got Login Response\n");
5096 		handle_login_rsp(crq, adapter);
5097 		break;
5098 	case LOGICAL_LINK_STATE_RSP:
5099 		netdev_dbg(netdev,
5100 			   "Got Logical Link State Response, state: %d rc: %d\n",
5101 			   crq->logical_link_state_rsp.link_state,
5102 			   crq->logical_link_state_rsp.rc.code);
5103 		adapter->logical_link_state =
5104 		    crq->logical_link_state_rsp.link_state;
5105 		adapter->init_done_rc = crq->logical_link_state_rsp.rc.code;
5106 		complete(&adapter->init_done);
5107 		break;
5108 	case LINK_STATE_INDICATION:
5109 		netdev_dbg(netdev, "Got Logical Link State Indication\n");
5110 		adapter->phys_link_state =
5111 		    crq->link_state_indication.phys_link_state;
5112 		adapter->logical_link_state =
5113 		    crq->link_state_indication.logical_link_state;
5114 		if (adapter->phys_link_state && adapter->logical_link_state)
5115 			netif_carrier_on(netdev);
5116 		else
5117 			netif_carrier_off(netdev);
5118 		break;
5119 	case CHANGE_MAC_ADDR_RSP:
5120 		netdev_dbg(netdev, "Got MAC address change Response\n");
5121 		adapter->fw_done_rc = handle_change_mac_rsp(crq, adapter);
5122 		break;
5123 	case ERROR_INDICATION:
5124 		netdev_dbg(netdev, "Got Error Indication\n");
5125 		handle_error_indication(crq, adapter);
5126 		break;
5127 	case REQUEST_STATISTICS_RSP:
5128 		netdev_dbg(netdev, "Got Statistics Response\n");
5129 		complete(&adapter->stats_done);
5130 		break;
5131 	case QUERY_IP_OFFLOAD_RSP:
5132 		netdev_dbg(netdev, "Got Query IP offload Response\n");
5133 		handle_query_ip_offload_rsp(adapter);
5134 		break;
5135 	case MULTICAST_CTRL_RSP:
5136 		netdev_dbg(netdev, "Got multicast control Response\n");
5137 		break;
5138 	case CONTROL_IP_OFFLOAD_RSP:
5139 		netdev_dbg(netdev, "Got Control IP offload Response\n");
5140 		dma_unmap_single(dev, adapter->ip_offload_ctrl_tok,
5141 				 sizeof(adapter->ip_offload_ctrl),
5142 				 DMA_TO_DEVICE);
5143 		complete(&adapter->init_done);
5144 		break;
5145 	case COLLECT_FW_TRACE_RSP:
5146 		netdev_dbg(netdev, "Got Collect firmware trace Response\n");
5147 		complete(&adapter->fw_done);
5148 		break;
5149 	case GET_VPD_SIZE_RSP:
5150 		handle_vpd_size_rsp(crq, adapter);
5151 		break;
5152 	case GET_VPD_RSP:
5153 		handle_vpd_rsp(crq, adapter);
5154 		break;
5155 	case QUERY_PHYS_PARMS_RSP:
5156 		adapter->fw_done_rc = handle_query_phys_parms_rsp(crq, adapter);
5157 		complete(&adapter->fw_done);
5158 		break;
5159 	default:
5160 		netdev_err(netdev, "Got an invalid cmd type 0x%02x\n",
5161 			   gen_crq->cmd);
5162 	}
5163 }
5164 
5165 static irqreturn_t ibmvnic_interrupt(int irq, void *instance)
5166 {
5167 	struct ibmvnic_adapter *adapter = instance;
5168 
5169 	tasklet_schedule(&adapter->tasklet);
5170 	return IRQ_HANDLED;
5171 }
5172 
5173 static void ibmvnic_tasklet(struct tasklet_struct *t)
5174 {
5175 	struct ibmvnic_adapter *adapter = from_tasklet(adapter, t, tasklet);
5176 	struct ibmvnic_crq_queue *queue = &adapter->crq;
5177 	union ibmvnic_crq *crq;
5178 	unsigned long flags;
5179 	bool done = false;
5180 
5181 	spin_lock_irqsave(&queue->lock, flags);
5182 	while (!done) {
5183 		/* Pull all the valid messages off the CRQ */
5184 		while ((crq = ibmvnic_next_crq(adapter)) != NULL) {
5185 			/* This barrier makes sure ibmvnic_next_crq()'s
5186 			 * crq->generic.first & IBMVNIC_CRQ_CMD_RSP is loaded
5187 			 * before ibmvnic_handle_crq()'s
5188 			 * switch(gen_crq->first) and switch(gen_crq->cmd).
5189 			 */
5190 			dma_rmb();
5191 			ibmvnic_handle_crq(crq, adapter);
5192 			crq->generic.first = 0;
5193 		}
5194 
5195 		/* remain in tasklet until all
5196 		 * capabilities responses are received
5197 		 */
5198 		if (!adapter->wait_capability)
5199 			done = true;
5200 	}
5201 	/* if capabilities CRQ's were sent in this tasklet, the following
5202 	 * tasklet must wait until all responses are received
5203 	 */
5204 	if (atomic_read(&adapter->running_cap_crqs) != 0)
5205 		adapter->wait_capability = true;
5206 	spin_unlock_irqrestore(&queue->lock, flags);
5207 }
5208 
5209 static int ibmvnic_reenable_crq_queue(struct ibmvnic_adapter *adapter)
5210 {
5211 	struct vio_dev *vdev = adapter->vdev;
5212 	int rc;
5213 
5214 	do {
5215 		rc = plpar_hcall_norets(H_ENABLE_CRQ, vdev->unit_address);
5216 	} while (rc == H_IN_PROGRESS || rc == H_BUSY || H_IS_LONG_BUSY(rc));
5217 
5218 	if (rc)
5219 		dev_err(&vdev->dev, "Error enabling adapter (rc=%d)\n", rc);
5220 
5221 	return rc;
5222 }
5223 
5224 static int ibmvnic_reset_crq(struct ibmvnic_adapter *adapter)
5225 {
5226 	struct ibmvnic_crq_queue *crq = &adapter->crq;
5227 	struct device *dev = &adapter->vdev->dev;
5228 	struct vio_dev *vdev = adapter->vdev;
5229 	int rc;
5230 
5231 	/* Close the CRQ */
5232 	do {
5233 		rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);
5234 	} while (rc == H_BUSY || H_IS_LONG_BUSY(rc));
5235 
5236 	/* Clean out the queue */
5237 	if (!crq->msgs)
5238 		return -EINVAL;
5239 
5240 	memset(crq->msgs, 0, PAGE_SIZE);
5241 	crq->cur = 0;
5242 	crq->active = false;
5243 
5244 	/* And re-open it again */
5245 	rc = plpar_hcall_norets(H_REG_CRQ, vdev->unit_address,
5246 				crq->msg_token, PAGE_SIZE);
5247 
5248 	if (rc == H_CLOSED)
5249 		/* Adapter is good, but other end is not ready */
5250 		dev_warn(dev, "Partner adapter not ready\n");
5251 	else if (rc != 0)
5252 		dev_warn(dev, "Couldn't register crq (rc=%d)\n", rc);
5253 
5254 	return rc;
5255 }
5256 
5257 static void release_crq_queue(struct ibmvnic_adapter *adapter)
5258 {
5259 	struct ibmvnic_crq_queue *crq = &adapter->crq;
5260 	struct vio_dev *vdev = adapter->vdev;
5261 	long rc;
5262 
5263 	if (!crq->msgs)
5264 		return;
5265 
5266 	netdev_dbg(adapter->netdev, "Releasing CRQ\n");
5267 	free_irq(vdev->irq, adapter);
5268 	tasklet_kill(&adapter->tasklet);
5269 	do {
5270 		rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);
5271 	} while (rc == H_BUSY || H_IS_LONG_BUSY(rc));
5272 
5273 	dma_unmap_single(&vdev->dev, crq->msg_token, PAGE_SIZE,
5274 			 DMA_BIDIRECTIONAL);
5275 	free_page((unsigned long)crq->msgs);
5276 	crq->msgs = NULL;
5277 	crq->active = false;
5278 }
5279 
5280 static int init_crq_queue(struct ibmvnic_adapter *adapter)
5281 {
5282 	struct ibmvnic_crq_queue *crq = &adapter->crq;
5283 	struct device *dev = &adapter->vdev->dev;
5284 	struct vio_dev *vdev = adapter->vdev;
5285 	int rc, retrc = -ENOMEM;
5286 
5287 	if (crq->msgs)
5288 		return 0;
5289 
5290 	crq->msgs = (union ibmvnic_crq *)get_zeroed_page(GFP_KERNEL);
5291 	/* Should we allocate more than one page? */
5292 
5293 	if (!crq->msgs)
5294 		return -ENOMEM;
5295 
5296 	crq->size = PAGE_SIZE / sizeof(*crq->msgs);
5297 	crq->msg_token = dma_map_single(dev, crq->msgs, PAGE_SIZE,
5298 					DMA_BIDIRECTIONAL);
5299 	if (dma_mapping_error(dev, crq->msg_token))
5300 		goto map_failed;
5301 
5302 	rc = plpar_hcall_norets(H_REG_CRQ, vdev->unit_address,
5303 				crq->msg_token, PAGE_SIZE);
5304 
5305 	if (rc == H_RESOURCE)
5306 		/* maybe kexecing and resource is busy. try a reset */
5307 		rc = ibmvnic_reset_crq(adapter);
5308 	retrc = rc;
5309 
5310 	if (rc == H_CLOSED) {
5311 		dev_warn(dev, "Partner adapter not ready\n");
5312 	} else if (rc) {
5313 		dev_warn(dev, "Error %d opening adapter\n", rc);
5314 		goto reg_crq_failed;
5315 	}
5316 
5317 	retrc = 0;
5318 
5319 	tasklet_setup(&adapter->tasklet, (void *)ibmvnic_tasklet);
5320 
5321 	netdev_dbg(adapter->netdev, "registering irq 0x%x\n", vdev->irq);
5322 	snprintf(crq->name, sizeof(crq->name), "ibmvnic-%x",
5323 		 adapter->vdev->unit_address);
5324 	rc = request_irq(vdev->irq, ibmvnic_interrupt, 0, crq->name, adapter);
5325 	if (rc) {
5326 		dev_err(dev, "Couldn't register irq 0x%x. rc=%d\n",
5327 			vdev->irq, rc);
5328 		goto req_irq_failed;
5329 	}
5330 
5331 	rc = vio_enable_interrupts(vdev);
5332 	if (rc) {
5333 		dev_err(dev, "Error %d enabling interrupts\n", rc);
5334 		goto req_irq_failed;
5335 	}
5336 
5337 	crq->cur = 0;
5338 	spin_lock_init(&crq->lock);
5339 
5340 	return retrc;
5341 
5342 req_irq_failed:
5343 	tasklet_kill(&adapter->tasklet);
5344 	do {
5345 		rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);
5346 	} while (rc == H_BUSY || H_IS_LONG_BUSY(rc));
5347 reg_crq_failed:
5348 	dma_unmap_single(dev, crq->msg_token, PAGE_SIZE, DMA_BIDIRECTIONAL);
5349 map_failed:
5350 	free_page((unsigned long)crq->msgs);
5351 	crq->msgs = NULL;
5352 	return retrc;
5353 }
5354 
5355 static int ibmvnic_reset_init(struct ibmvnic_adapter *adapter, bool reset)
5356 {
5357 	struct device *dev = &adapter->vdev->dev;
5358 	unsigned long timeout = msecs_to_jiffies(20000);
5359 	u64 old_num_rx_queues = adapter->req_rx_queues;
5360 	u64 old_num_tx_queues = adapter->req_tx_queues;
5361 	int rc;
5362 
5363 	adapter->from_passive_init = false;
5364 
5365 	if (reset)
5366 		reinit_completion(&adapter->init_done);
5367 
5368 	adapter->init_done_rc = 0;
5369 	rc = ibmvnic_send_crq_init(adapter);
5370 	if (rc) {
5371 		dev_err(dev, "Send crq init failed with error %d\n", rc);
5372 		return rc;
5373 	}
5374 
5375 	if (!wait_for_completion_timeout(&adapter->init_done, timeout)) {
5376 		dev_err(dev, "Initialization sequence timed out\n");
5377 		return -1;
5378 	}
5379 
5380 	if (adapter->init_done_rc) {
5381 		release_crq_queue(adapter);
5382 		return adapter->init_done_rc;
5383 	}
5384 
5385 	if (adapter->from_passive_init) {
5386 		adapter->state = VNIC_OPEN;
5387 		adapter->from_passive_init = false;
5388 		return -1;
5389 	}
5390 
5391 	if (reset &&
5392 	    test_bit(0, &adapter->resetting) && !adapter->wait_for_reset &&
5393 	    adapter->reset_reason != VNIC_RESET_MOBILITY) {
5394 		if (adapter->req_rx_queues != old_num_rx_queues ||
5395 		    adapter->req_tx_queues != old_num_tx_queues) {
5396 			release_sub_crqs(adapter, 0);
5397 			rc = init_sub_crqs(adapter);
5398 		} else {
5399 			rc = reset_sub_crq_queues(adapter);
5400 		}
5401 	} else {
5402 		rc = init_sub_crqs(adapter);
5403 	}
5404 
5405 	if (rc) {
5406 		dev_err(dev, "Initialization of sub crqs failed\n");
5407 		release_crq_queue(adapter);
5408 		return rc;
5409 	}
5410 
5411 	rc = init_sub_crq_irqs(adapter);
5412 	if (rc) {
5413 		dev_err(dev, "Failed to initialize sub crq irqs\n");
5414 		release_crq_queue(adapter);
5415 	}
5416 
5417 	return rc;
5418 }
5419 
5420 static struct device_attribute dev_attr_failover;
5421 
5422 static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id)
5423 {
5424 	struct ibmvnic_adapter *adapter;
5425 	struct net_device *netdev;
5426 	unsigned char *mac_addr_p;
5427 	bool init_success;
5428 	int rc;
5429 
5430 	dev_dbg(&dev->dev, "entering ibmvnic_probe for UA 0x%x\n",
5431 		dev->unit_address);
5432 
5433 	mac_addr_p = (unsigned char *)vio_get_attribute(dev,
5434 							VETH_MAC_ADDR, NULL);
5435 	if (!mac_addr_p) {
5436 		dev_err(&dev->dev,
5437 			"(%s:%3.3d) ERROR: Can't find MAC_ADDR attribute\n",
5438 			__FILE__, __LINE__);
5439 		return 0;
5440 	}
5441 
5442 	netdev = alloc_etherdev_mq(sizeof(struct ibmvnic_adapter),
5443 				   IBMVNIC_MAX_QUEUES);
5444 	if (!netdev)
5445 		return -ENOMEM;
5446 
5447 	adapter = netdev_priv(netdev);
5448 	adapter->state = VNIC_PROBING;
5449 	dev_set_drvdata(&dev->dev, netdev);
5450 	adapter->vdev = dev;
5451 	adapter->netdev = netdev;
5452 	adapter->login_pending = false;
5453 
5454 	ether_addr_copy(adapter->mac_addr, mac_addr_p);
5455 	ether_addr_copy(netdev->dev_addr, adapter->mac_addr);
5456 	netdev->irq = dev->irq;
5457 	netdev->netdev_ops = &ibmvnic_netdev_ops;
5458 	netdev->ethtool_ops = &ibmvnic_ethtool_ops;
5459 	SET_NETDEV_DEV(netdev, &dev->dev);
5460 
5461 	INIT_WORK(&adapter->ibmvnic_reset, __ibmvnic_reset);
5462 	INIT_DELAYED_WORK(&adapter->ibmvnic_delayed_reset,
5463 			  __ibmvnic_delayed_reset);
5464 	INIT_LIST_HEAD(&adapter->rwi_list);
5465 	spin_lock_init(&adapter->rwi_lock);
5466 	spin_lock_init(&adapter->state_lock);
5467 	mutex_init(&adapter->fw_lock);
5468 	init_completion(&adapter->init_done);
5469 	init_completion(&adapter->fw_done);
5470 	init_completion(&adapter->reset_done);
5471 	init_completion(&adapter->stats_done);
5472 	clear_bit(0, &adapter->resetting);
5473 
5474 	init_success = false;
5475 	do {
5476 		rc = init_crq_queue(adapter);
5477 		if (rc) {
5478 			dev_err(&dev->dev, "Couldn't initialize crq. rc=%d\n",
5479 				rc);
5480 			goto ibmvnic_init_fail;
5481 		}
5482 
5483 		rc = ibmvnic_reset_init(adapter, false);
5484 	} while (rc == EAGAIN);
5485 
5486 	/* We are ignoring the error from ibmvnic_reset_init() assuming that the
5487 	 * partner is not ready. CRQ is not active. When the partner becomes
5488 	 * ready, we will do the passive init reset.
5489 	 */
5490 
5491 	if (!rc)
5492 		init_success = true;
5493 
5494 	rc = init_stats_buffers(adapter);
5495 	if (rc)
5496 		goto ibmvnic_init_fail;
5497 
5498 	rc = init_stats_token(adapter);
5499 	if (rc)
5500 		goto ibmvnic_stats_fail;
5501 
5502 	rc = device_create_file(&dev->dev, &dev_attr_failover);
5503 	if (rc)
5504 		goto ibmvnic_dev_file_err;
5505 
5506 	netif_carrier_off(netdev);
5507 	rc = register_netdev(netdev);
5508 	if (rc) {
5509 		dev_err(&dev->dev, "failed to register netdev rc=%d\n", rc);
5510 		goto ibmvnic_register_fail;
5511 	}
5512 	dev_info(&dev->dev, "ibmvnic registered\n");
5513 
5514 	if (init_success) {
5515 		adapter->state = VNIC_PROBED;
5516 		netdev->mtu = adapter->req_mtu - ETH_HLEN;
5517 		netdev->min_mtu = adapter->min_mtu - ETH_HLEN;
5518 		netdev->max_mtu = adapter->max_mtu - ETH_HLEN;
5519 	} else {
5520 		adapter->state = VNIC_DOWN;
5521 	}
5522 
5523 	adapter->wait_for_reset = false;
5524 	adapter->last_reset_time = jiffies;
5525 	return 0;
5526 
5527 ibmvnic_register_fail:
5528 	device_remove_file(&dev->dev, &dev_attr_failover);
5529 
5530 ibmvnic_dev_file_err:
5531 	release_stats_token(adapter);
5532 
5533 ibmvnic_stats_fail:
5534 	release_stats_buffers(adapter);
5535 
5536 ibmvnic_init_fail:
5537 	release_sub_crqs(adapter, 1);
5538 	release_crq_queue(adapter);
5539 	mutex_destroy(&adapter->fw_lock);
5540 	free_netdev(netdev);
5541 
5542 	return rc;
5543 }
5544 
5545 static void ibmvnic_remove(struct vio_dev *dev)
5546 {
5547 	struct net_device *netdev = dev_get_drvdata(&dev->dev);
5548 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
5549 	unsigned long flags;
5550 
5551 	spin_lock_irqsave(&adapter->state_lock, flags);
5552 
5553 	/* If ibmvnic_reset() is scheduling a reset, wait for it to
5554 	 * finish. Then, set the state to REMOVING to prevent it from
5555 	 * scheduling any more work and to have reset functions ignore
5556 	 * any resets that have already been scheduled. Drop the lock
5557 	 * after setting state, so __ibmvnic_reset() which is called
5558 	 * from the flush_work() below, can make progress.
5559 	 */
5560 	spin_lock(&adapter->rwi_lock);
5561 	adapter->state = VNIC_REMOVING;
5562 	spin_unlock(&adapter->rwi_lock);
5563 
5564 	spin_unlock_irqrestore(&adapter->state_lock, flags);
5565 
5566 	flush_work(&adapter->ibmvnic_reset);
5567 	flush_delayed_work(&adapter->ibmvnic_delayed_reset);
5568 
5569 	rtnl_lock();
5570 	unregister_netdevice(netdev);
5571 
5572 	release_resources(adapter);
5573 	release_sub_crqs(adapter, 1);
5574 	release_crq_queue(adapter);
5575 
5576 	release_stats_token(adapter);
5577 	release_stats_buffers(adapter);
5578 
5579 	adapter->state = VNIC_REMOVED;
5580 
5581 	rtnl_unlock();
5582 	mutex_destroy(&adapter->fw_lock);
5583 	device_remove_file(&dev->dev, &dev_attr_failover);
5584 	free_netdev(netdev);
5585 	dev_set_drvdata(&dev->dev, NULL);
5586 }
5587 
5588 static ssize_t failover_store(struct device *dev, struct device_attribute *attr,
5589 			      const char *buf, size_t count)
5590 {
5591 	struct net_device *netdev = dev_get_drvdata(dev);
5592 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
5593 	unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
5594 	__be64 session_token;
5595 	long rc;
5596 
5597 	if (!sysfs_streq(buf, "1"))
5598 		return -EINVAL;
5599 
5600 	rc = plpar_hcall(H_VIOCTL, retbuf, adapter->vdev->unit_address,
5601 			 H_GET_SESSION_TOKEN, 0, 0, 0);
5602 	if (rc) {
5603 		netdev_err(netdev, "Couldn't retrieve session token, rc %ld\n",
5604 			   rc);
5605 		goto last_resort;
5606 	}
5607 
5608 	session_token = (__be64)retbuf[0];
5609 	netdev_dbg(netdev, "Initiating client failover, session id %llx\n",
5610 		   be64_to_cpu(session_token));
5611 	rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,
5612 				H_SESSION_ERR_DETECTED, session_token, 0, 0);
5613 	if (rc)
5614 		netdev_err(netdev,
5615 			   "H_VIOCTL initiated failover failed, rc %ld\n",
5616 			   rc);
5617 
5618 last_resort:
5619 	netdev_dbg(netdev, "Trying to send CRQ_CMD, the last resort\n");
5620 	ibmvnic_reset(adapter, VNIC_RESET_FAILOVER);
5621 
5622 	return count;
5623 }
5624 static DEVICE_ATTR_WO(failover);
5625 
5626 static unsigned long ibmvnic_get_desired_dma(struct vio_dev *vdev)
5627 {
5628 	struct net_device *netdev = dev_get_drvdata(&vdev->dev);
5629 	struct ibmvnic_adapter *adapter;
5630 	struct iommu_table *tbl;
5631 	unsigned long ret = 0;
5632 	int i;
5633 
5634 	tbl = get_iommu_table_base(&vdev->dev);
5635 
5636 	/* netdev inits at probe time along with the structures we need below*/
5637 	if (!netdev)
5638 		return IOMMU_PAGE_ALIGN(IBMVNIC_IO_ENTITLEMENT_DEFAULT, tbl);
5639 
5640 	adapter = netdev_priv(netdev);
5641 
5642 	ret += PAGE_SIZE; /* the crq message queue */
5643 	ret += IOMMU_PAGE_ALIGN(sizeof(struct ibmvnic_statistics), tbl);
5644 
5645 	for (i = 0; i < adapter->req_tx_queues + adapter->req_rx_queues; i++)
5646 		ret += 4 * PAGE_SIZE; /* the scrq message queue */
5647 
5648 	for (i = 0; i < adapter->num_active_rx_pools; i++)
5649 		ret += adapter->rx_pool[i].size *
5650 		    IOMMU_PAGE_ALIGN(adapter->rx_pool[i].buff_size, tbl);
5651 
5652 	return ret;
5653 }
5654 
5655 static int ibmvnic_resume(struct device *dev)
5656 {
5657 	struct net_device *netdev = dev_get_drvdata(dev);
5658 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
5659 
5660 	if (adapter->state != VNIC_OPEN)
5661 		return 0;
5662 
5663 	tasklet_schedule(&adapter->tasklet);
5664 
5665 	return 0;
5666 }
5667 
5668 static const struct vio_device_id ibmvnic_device_table[] = {
5669 	{"network", "IBM,vnic"},
5670 	{"", "" }
5671 };
5672 MODULE_DEVICE_TABLE(vio, ibmvnic_device_table);
5673 
5674 static const struct dev_pm_ops ibmvnic_pm_ops = {
5675 	.resume = ibmvnic_resume
5676 };
5677 
5678 static struct vio_driver ibmvnic_driver = {
5679 	.id_table       = ibmvnic_device_table,
5680 	.probe          = ibmvnic_probe,
5681 	.remove         = ibmvnic_remove,
5682 	.get_desired_dma = ibmvnic_get_desired_dma,
5683 	.name		= ibmvnic_driver_name,
5684 	.pm		= &ibmvnic_pm_ops,
5685 };
5686 
5687 /* module functions */
5688 static int __init ibmvnic_module_init(void)
5689 {
5690 	pr_info("%s: %s %s\n", ibmvnic_driver_name, ibmvnic_driver_string,
5691 		IBMVNIC_DRIVER_VERSION);
5692 
5693 	return vio_register_driver(&ibmvnic_driver);
5694 }
5695 
5696 static void __exit ibmvnic_module_exit(void)
5697 {
5698 	vio_unregister_driver(&ibmvnic_driver);
5699 }
5700 
5701 module_init(ibmvnic_module_init);
5702 module_exit(ibmvnic_module_exit);
5703