xref: /openbmc/linux/drivers/net/ethernet/ibm/ibmvnic.c (revision f97769fd)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /**************************************************************************/
3 /*                                                                        */
4 /*  IBM System i and System p Virtual NIC Device Driver                   */
5 /*  Copyright (C) 2014 IBM Corp.                                          */
6 /*  Santiago Leon (santi_leon@yahoo.com)                                  */
7 /*  Thomas Falcon (tlfalcon@linux.vnet.ibm.com)                           */
8 /*  John Allen (jallen@linux.vnet.ibm.com)                                */
9 /*                                                                        */
10 /*                                                                        */
11 /* This module contains the implementation of a virtual ethernet device   */
12 /* for use with IBM i/p Series LPAR Linux. It utilizes the logical LAN    */
13 /* option of the RS/6000 Platform Architecture to interface with virtual  */
14 /* ethernet NICs that are presented to the partition by the hypervisor.   */
15 /*									   */
16 /* Messages are passed between the VNIC driver and the VNIC server using  */
17 /* Command/Response Queues (CRQs) and sub CRQs (sCRQs). CRQs are used to  */
18 /* issue and receive commands that initiate communication with the server */
19 /* on driver initialization. Sub CRQs (sCRQs) are similar to CRQs, but    */
20 /* are used by the driver to notify the server that a packet is           */
21 /* ready for transmission or that a buffer has been added to receive a    */
22 /* packet. Subsequently, sCRQs are used by the server to notify the       */
23 /* driver that a packet transmission has been completed or that a packet  */
24 /* has been received and placed in a waiting buffer.                      */
25 /*                                                                        */
26 /* In lieu of a more conventional "on-the-fly" DMA mapping strategy in    */
27 /* which skbs are DMA mapped and immediately unmapped when the transmit   */
28 /* or receive has been completed, the VNIC driver is required to use      */
29 /* "long term mapping". This entails that large, continuous DMA mapped    */
30 /* buffers are allocated on driver initialization and these buffers are   */
31 /* then continuously reused to pass skbs to and from the VNIC server.     */
32 /*                                                                        */
33 /**************************************************************************/
34 
35 #include <linux/module.h>
36 #include <linux/moduleparam.h>
37 #include <linux/types.h>
38 #include <linux/errno.h>
39 #include <linux/completion.h>
40 #include <linux/ioport.h>
41 #include <linux/dma-mapping.h>
42 #include <linux/kernel.h>
43 #include <linux/netdevice.h>
44 #include <linux/etherdevice.h>
45 #include <linux/skbuff.h>
46 #include <linux/init.h>
47 #include <linux/delay.h>
48 #include <linux/mm.h>
49 #include <linux/ethtool.h>
50 #include <linux/proc_fs.h>
51 #include <linux/if_arp.h>
52 #include <linux/in.h>
53 #include <linux/ip.h>
54 #include <linux/ipv6.h>
55 #include <linux/irq.h>
56 #include <linux/kthread.h>
57 #include <linux/seq_file.h>
58 #include <linux/interrupt.h>
59 #include <net/net_namespace.h>
60 #include <asm/hvcall.h>
61 #include <linux/atomic.h>
62 #include <asm/vio.h>
63 #include <asm/iommu.h>
64 #include <linux/uaccess.h>
65 #include <asm/firmware.h>
66 #include <linux/workqueue.h>
67 #include <linux/if_vlan.h>
68 #include <linux/utsname.h>
69 
70 #include "ibmvnic.h"
71 
72 static const char ibmvnic_driver_name[] = "ibmvnic";
73 static const char ibmvnic_driver_string[] = "IBM System i/p Virtual NIC Driver";
74 
75 MODULE_AUTHOR("Santiago Leon");
76 MODULE_DESCRIPTION("IBM System i/p Virtual NIC Driver");
77 MODULE_LICENSE("GPL");
78 MODULE_VERSION(IBMVNIC_DRIVER_VERSION);
79 
80 static int ibmvnic_version = IBMVNIC_INITIAL_VERSION;
81 static int ibmvnic_remove(struct vio_dev *);
82 static void release_sub_crqs(struct ibmvnic_adapter *, bool);
83 static int ibmvnic_reset_crq(struct ibmvnic_adapter *);
84 static int ibmvnic_send_crq_init(struct ibmvnic_adapter *);
85 static int ibmvnic_reenable_crq_queue(struct ibmvnic_adapter *);
86 static int ibmvnic_send_crq(struct ibmvnic_adapter *, union ibmvnic_crq *);
87 static int send_subcrq(struct ibmvnic_adapter *adapter, u64 remote_handle,
88 		       union sub_crq *sub_crq);
89 static int send_subcrq_indirect(struct ibmvnic_adapter *, u64, u64, u64);
90 static irqreturn_t ibmvnic_interrupt_rx(int irq, void *instance);
91 static int enable_scrq_irq(struct ibmvnic_adapter *,
92 			   struct ibmvnic_sub_crq_queue *);
93 static int disable_scrq_irq(struct ibmvnic_adapter *,
94 			    struct ibmvnic_sub_crq_queue *);
95 static int pending_scrq(struct ibmvnic_adapter *,
96 			struct ibmvnic_sub_crq_queue *);
97 static union sub_crq *ibmvnic_next_scrq(struct ibmvnic_adapter *,
98 					struct ibmvnic_sub_crq_queue *);
99 static int ibmvnic_poll(struct napi_struct *napi, int data);
100 static void send_map_query(struct ibmvnic_adapter *adapter);
101 static int send_request_map(struct ibmvnic_adapter *, dma_addr_t, __be32, u8);
102 static int send_request_unmap(struct ibmvnic_adapter *, u8);
103 static int send_login(struct ibmvnic_adapter *adapter);
104 static void send_cap_queries(struct ibmvnic_adapter *adapter);
105 static int init_sub_crqs(struct ibmvnic_adapter *);
106 static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter);
107 static int ibmvnic_init(struct ibmvnic_adapter *);
108 static int ibmvnic_reset_init(struct ibmvnic_adapter *);
109 static void release_crq_queue(struct ibmvnic_adapter *);
110 static int __ibmvnic_set_mac(struct net_device *, u8 *);
111 static int init_crq_queue(struct ibmvnic_adapter *adapter);
112 static int send_query_phys_parms(struct ibmvnic_adapter *adapter);
113 
114 struct ibmvnic_stat {
115 	char name[ETH_GSTRING_LEN];
116 	int offset;
117 };
118 
119 #define IBMVNIC_STAT_OFF(stat) (offsetof(struct ibmvnic_adapter, stats) + \
120 			     offsetof(struct ibmvnic_statistics, stat))
121 #define IBMVNIC_GET_STAT(a, off) (*((u64 *)(((unsigned long)(a)) + off)))
122 
123 static const struct ibmvnic_stat ibmvnic_stats[] = {
124 	{"rx_packets", IBMVNIC_STAT_OFF(rx_packets)},
125 	{"rx_bytes", IBMVNIC_STAT_OFF(rx_bytes)},
126 	{"tx_packets", IBMVNIC_STAT_OFF(tx_packets)},
127 	{"tx_bytes", IBMVNIC_STAT_OFF(tx_bytes)},
128 	{"ucast_tx_packets", IBMVNIC_STAT_OFF(ucast_tx_packets)},
129 	{"ucast_rx_packets", IBMVNIC_STAT_OFF(ucast_rx_packets)},
130 	{"mcast_tx_packets", IBMVNIC_STAT_OFF(mcast_tx_packets)},
131 	{"mcast_rx_packets", IBMVNIC_STAT_OFF(mcast_rx_packets)},
132 	{"bcast_tx_packets", IBMVNIC_STAT_OFF(bcast_tx_packets)},
133 	{"bcast_rx_packets", IBMVNIC_STAT_OFF(bcast_rx_packets)},
134 	{"align_errors", IBMVNIC_STAT_OFF(align_errors)},
135 	{"fcs_errors", IBMVNIC_STAT_OFF(fcs_errors)},
136 	{"single_collision_frames", IBMVNIC_STAT_OFF(single_collision_frames)},
137 	{"multi_collision_frames", IBMVNIC_STAT_OFF(multi_collision_frames)},
138 	{"sqe_test_errors", IBMVNIC_STAT_OFF(sqe_test_errors)},
139 	{"deferred_tx", IBMVNIC_STAT_OFF(deferred_tx)},
140 	{"late_collisions", IBMVNIC_STAT_OFF(late_collisions)},
141 	{"excess_collisions", IBMVNIC_STAT_OFF(excess_collisions)},
142 	{"internal_mac_tx_errors", IBMVNIC_STAT_OFF(internal_mac_tx_errors)},
143 	{"carrier_sense", IBMVNIC_STAT_OFF(carrier_sense)},
144 	{"too_long_frames", IBMVNIC_STAT_OFF(too_long_frames)},
145 	{"internal_mac_rx_errors", IBMVNIC_STAT_OFF(internal_mac_rx_errors)},
146 };
147 
148 static long h_reg_sub_crq(unsigned long unit_address, unsigned long token,
149 			  unsigned long length, unsigned long *number,
150 			  unsigned long *irq)
151 {
152 	unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
153 	long rc;
154 
155 	rc = plpar_hcall(H_REG_SUB_CRQ, retbuf, unit_address, token, length);
156 	*number = retbuf[0];
157 	*irq = retbuf[1];
158 
159 	return rc;
160 }
161 
162 /**
163  * ibmvnic_wait_for_completion - Check device state and wait for completion
164  * @adapter: private device data
165  * @comp_done: completion structure to wait for
166  * @timeout: time to wait in milliseconds
167  *
168  * Wait for a completion signal or until the timeout limit is reached
169  * while checking that the device is still active.
170  */
171 static int ibmvnic_wait_for_completion(struct ibmvnic_adapter *adapter,
172 				       struct completion *comp_done,
173 				       unsigned long timeout)
174 {
175 	struct net_device *netdev;
176 	unsigned long div_timeout;
177 	u8 retry;
178 
179 	netdev = adapter->netdev;
180 	retry = 5;
181 	div_timeout = msecs_to_jiffies(timeout / retry);
182 	while (true) {
183 		if (!adapter->crq.active) {
184 			netdev_err(netdev, "Device down!\n");
185 			return -ENODEV;
186 		}
187 		if (!retry--)
188 			break;
189 		if (wait_for_completion_timeout(comp_done, div_timeout))
190 			return 0;
191 	}
192 	netdev_err(netdev, "Operation timed out.\n");
193 	return -ETIMEDOUT;
194 }
195 
196 static int alloc_long_term_buff(struct ibmvnic_adapter *adapter,
197 				struct ibmvnic_long_term_buff *ltb, int size)
198 {
199 	struct device *dev = &adapter->vdev->dev;
200 	int rc;
201 
202 	ltb->size = size;
203 	ltb->buff = dma_alloc_coherent(dev, ltb->size, &ltb->addr,
204 				       GFP_KERNEL);
205 
206 	if (!ltb->buff) {
207 		dev_err(dev, "Couldn't alloc long term buffer\n");
208 		return -ENOMEM;
209 	}
210 	ltb->map_id = adapter->map_id;
211 	adapter->map_id++;
212 
213 	mutex_lock(&adapter->fw_lock);
214 	adapter->fw_done_rc = 0;
215 	reinit_completion(&adapter->fw_done);
216 	rc = send_request_map(adapter, ltb->addr,
217 			      ltb->size, ltb->map_id);
218 	if (rc) {
219 		dma_free_coherent(dev, ltb->size, ltb->buff, ltb->addr);
220 		mutex_unlock(&adapter->fw_lock);
221 		return rc;
222 	}
223 
224 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);
225 	if (rc) {
226 		dev_err(dev,
227 			"Long term map request aborted or timed out,rc = %d\n",
228 			rc);
229 		dma_free_coherent(dev, ltb->size, ltb->buff, ltb->addr);
230 		mutex_unlock(&adapter->fw_lock);
231 		return rc;
232 	}
233 
234 	if (adapter->fw_done_rc) {
235 		dev_err(dev, "Couldn't map long term buffer,rc = %d\n",
236 			adapter->fw_done_rc);
237 		dma_free_coherent(dev, ltb->size, ltb->buff, ltb->addr);
238 		mutex_unlock(&adapter->fw_lock);
239 		return -1;
240 	}
241 	mutex_unlock(&adapter->fw_lock);
242 	return 0;
243 }
244 
245 static void free_long_term_buff(struct ibmvnic_adapter *adapter,
246 				struct ibmvnic_long_term_buff *ltb)
247 {
248 	struct device *dev = &adapter->vdev->dev;
249 
250 	if (!ltb->buff)
251 		return;
252 
253 	if (adapter->reset_reason != VNIC_RESET_FAILOVER &&
254 	    adapter->reset_reason != VNIC_RESET_MOBILITY)
255 		send_request_unmap(adapter, ltb->map_id);
256 	dma_free_coherent(dev, ltb->size, ltb->buff, ltb->addr);
257 }
258 
259 static int reset_long_term_buff(struct ibmvnic_adapter *adapter,
260 				struct ibmvnic_long_term_buff *ltb)
261 {
262 	struct device *dev = &adapter->vdev->dev;
263 	int rc;
264 
265 	memset(ltb->buff, 0, ltb->size);
266 
267 	mutex_lock(&adapter->fw_lock);
268 	adapter->fw_done_rc = 0;
269 
270 	reinit_completion(&adapter->fw_done);
271 	rc = send_request_map(adapter, ltb->addr, ltb->size, ltb->map_id);
272 	if (rc) {
273 		mutex_unlock(&adapter->fw_lock);
274 		return rc;
275 	}
276 
277 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);
278 	if (rc) {
279 		dev_info(dev,
280 			 "Reset failed, long term map request timed out or aborted\n");
281 		mutex_unlock(&adapter->fw_lock);
282 		return rc;
283 	}
284 
285 	if (adapter->fw_done_rc) {
286 		dev_info(dev,
287 			 "Reset failed, attempting to free and reallocate buffer\n");
288 		free_long_term_buff(adapter, ltb);
289 		mutex_unlock(&adapter->fw_lock);
290 		return alloc_long_term_buff(adapter, ltb, ltb->size);
291 	}
292 	mutex_unlock(&adapter->fw_lock);
293 	return 0;
294 }
295 
296 static void deactivate_rx_pools(struct ibmvnic_adapter *adapter)
297 {
298 	int i;
299 
300 	for (i = 0; i < be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);
301 	     i++)
302 		adapter->rx_pool[i].active = 0;
303 }
304 
305 static void replenish_rx_pool(struct ibmvnic_adapter *adapter,
306 			      struct ibmvnic_rx_pool *pool)
307 {
308 	int count = pool->size - atomic_read(&pool->available);
309 	struct device *dev = &adapter->vdev->dev;
310 	int buffers_added = 0;
311 	unsigned long lpar_rc;
312 	union sub_crq sub_crq;
313 	struct sk_buff *skb;
314 	unsigned int offset;
315 	dma_addr_t dma_addr;
316 	unsigned char *dst;
317 	u64 *handle_array;
318 	int shift = 0;
319 	int index;
320 	int i;
321 
322 	if (!pool->active)
323 		return;
324 
325 	handle_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +
326 				      be32_to_cpu(adapter->login_rsp_buf->
327 				      off_rxadd_subcrqs));
328 
329 	for (i = 0; i < count; ++i) {
330 		skb = alloc_skb(pool->buff_size, GFP_ATOMIC);
331 		if (!skb) {
332 			dev_err(dev, "Couldn't replenish rx buff\n");
333 			adapter->replenish_no_mem++;
334 			break;
335 		}
336 
337 		index = pool->free_map[pool->next_free];
338 
339 		if (pool->rx_buff[index].skb)
340 			dev_err(dev, "Inconsistent free_map!\n");
341 
342 		/* Copy the skb to the long term mapped DMA buffer */
343 		offset = index * pool->buff_size;
344 		dst = pool->long_term_buff.buff + offset;
345 		memset(dst, 0, pool->buff_size);
346 		dma_addr = pool->long_term_buff.addr + offset;
347 		pool->rx_buff[index].data = dst;
348 
349 		pool->free_map[pool->next_free] = IBMVNIC_INVALID_MAP;
350 		pool->rx_buff[index].dma = dma_addr;
351 		pool->rx_buff[index].skb = skb;
352 		pool->rx_buff[index].pool_index = pool->index;
353 		pool->rx_buff[index].size = pool->buff_size;
354 
355 		memset(&sub_crq, 0, sizeof(sub_crq));
356 		sub_crq.rx_add.first = IBMVNIC_CRQ_CMD;
357 		sub_crq.rx_add.correlator =
358 		    cpu_to_be64((u64)&pool->rx_buff[index]);
359 		sub_crq.rx_add.ioba = cpu_to_be32(dma_addr);
360 		sub_crq.rx_add.map_id = pool->long_term_buff.map_id;
361 
362 		/* The length field of the sCRQ is defined to be 24 bits so the
363 		 * buffer size needs to be left shifted by a byte before it is
364 		 * converted to big endian to prevent the last byte from being
365 		 * truncated.
366 		 */
367 #ifdef __LITTLE_ENDIAN__
368 		shift = 8;
369 #endif
370 		sub_crq.rx_add.len = cpu_to_be32(pool->buff_size << shift);
371 
372 		lpar_rc = send_subcrq(adapter, handle_array[pool->index],
373 				      &sub_crq);
374 		if (lpar_rc != H_SUCCESS)
375 			goto failure;
376 
377 		buffers_added++;
378 		adapter->replenish_add_buff_success++;
379 		pool->next_free = (pool->next_free + 1) % pool->size;
380 	}
381 	atomic_add(buffers_added, &pool->available);
382 	return;
383 
384 failure:
385 	if (lpar_rc != H_PARAMETER && lpar_rc != H_CLOSED)
386 		dev_err_ratelimited(dev, "rx: replenish packet buffer failed\n");
387 	pool->free_map[pool->next_free] = index;
388 	pool->rx_buff[index].skb = NULL;
389 
390 	dev_kfree_skb_any(skb);
391 	adapter->replenish_add_buff_failure++;
392 	atomic_add(buffers_added, &pool->available);
393 
394 	if (lpar_rc == H_CLOSED || adapter->failover_pending) {
395 		/* Disable buffer pool replenishment and report carrier off if
396 		 * queue is closed or pending failover.
397 		 * Firmware guarantees that a signal will be sent to the
398 		 * driver, triggering a reset.
399 		 */
400 		deactivate_rx_pools(adapter);
401 		netif_carrier_off(adapter->netdev);
402 	}
403 }
404 
405 static void replenish_pools(struct ibmvnic_adapter *adapter)
406 {
407 	int i;
408 
409 	adapter->replenish_task_cycles++;
410 	for (i = 0; i < be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);
411 	     i++) {
412 		if (adapter->rx_pool[i].active)
413 			replenish_rx_pool(adapter, &adapter->rx_pool[i]);
414 	}
415 }
416 
417 static void release_stats_buffers(struct ibmvnic_adapter *adapter)
418 {
419 	kfree(adapter->tx_stats_buffers);
420 	kfree(adapter->rx_stats_buffers);
421 	adapter->tx_stats_buffers = NULL;
422 	adapter->rx_stats_buffers = NULL;
423 }
424 
425 static int init_stats_buffers(struct ibmvnic_adapter *adapter)
426 {
427 	adapter->tx_stats_buffers =
428 				kcalloc(IBMVNIC_MAX_QUEUES,
429 					sizeof(struct ibmvnic_tx_queue_stats),
430 					GFP_KERNEL);
431 	if (!adapter->tx_stats_buffers)
432 		return -ENOMEM;
433 
434 	adapter->rx_stats_buffers =
435 				kcalloc(IBMVNIC_MAX_QUEUES,
436 					sizeof(struct ibmvnic_rx_queue_stats),
437 					GFP_KERNEL);
438 	if (!adapter->rx_stats_buffers)
439 		return -ENOMEM;
440 
441 	return 0;
442 }
443 
444 static void release_stats_token(struct ibmvnic_adapter *adapter)
445 {
446 	struct device *dev = &adapter->vdev->dev;
447 
448 	if (!adapter->stats_token)
449 		return;
450 
451 	dma_unmap_single(dev, adapter->stats_token,
452 			 sizeof(struct ibmvnic_statistics),
453 			 DMA_FROM_DEVICE);
454 	adapter->stats_token = 0;
455 }
456 
457 static int init_stats_token(struct ibmvnic_adapter *adapter)
458 {
459 	struct device *dev = &adapter->vdev->dev;
460 	dma_addr_t stok;
461 
462 	stok = dma_map_single(dev, &adapter->stats,
463 			      sizeof(struct ibmvnic_statistics),
464 			      DMA_FROM_DEVICE);
465 	if (dma_mapping_error(dev, stok)) {
466 		dev_err(dev, "Couldn't map stats buffer\n");
467 		return -1;
468 	}
469 
470 	adapter->stats_token = stok;
471 	netdev_dbg(adapter->netdev, "Stats token initialized (%llx)\n", stok);
472 	return 0;
473 }
474 
475 static int reset_rx_pools(struct ibmvnic_adapter *adapter)
476 {
477 	struct ibmvnic_rx_pool *rx_pool;
478 	int rx_scrqs;
479 	int i, j, rc;
480 	u64 *size_array;
481 
482 	if (!adapter->rx_pool)
483 		return -1;
484 
485 	size_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +
486 		be32_to_cpu(adapter->login_rsp_buf->off_rxadd_buff_size));
487 
488 	rx_scrqs = be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);
489 	for (i = 0; i < rx_scrqs; i++) {
490 		rx_pool = &adapter->rx_pool[i];
491 
492 		netdev_dbg(adapter->netdev, "Re-setting rx_pool[%d]\n", i);
493 
494 		if (rx_pool->buff_size != be64_to_cpu(size_array[i])) {
495 			free_long_term_buff(adapter, &rx_pool->long_term_buff);
496 			rx_pool->buff_size = be64_to_cpu(size_array[i]);
497 			rc = alloc_long_term_buff(adapter,
498 						  &rx_pool->long_term_buff,
499 						  rx_pool->size *
500 						  rx_pool->buff_size);
501 		} else {
502 			rc = reset_long_term_buff(adapter,
503 						  &rx_pool->long_term_buff);
504 		}
505 
506 		if (rc)
507 			return rc;
508 
509 		for (j = 0; j < rx_pool->size; j++)
510 			rx_pool->free_map[j] = j;
511 
512 		memset(rx_pool->rx_buff, 0,
513 		       rx_pool->size * sizeof(struct ibmvnic_rx_buff));
514 
515 		atomic_set(&rx_pool->available, 0);
516 		rx_pool->next_alloc = 0;
517 		rx_pool->next_free = 0;
518 		rx_pool->active = 1;
519 	}
520 
521 	return 0;
522 }
523 
524 static void release_rx_pools(struct ibmvnic_adapter *adapter)
525 {
526 	struct ibmvnic_rx_pool *rx_pool;
527 	int i, j;
528 
529 	if (!adapter->rx_pool)
530 		return;
531 
532 	for (i = 0; i < adapter->num_active_rx_pools; i++) {
533 		rx_pool = &adapter->rx_pool[i];
534 
535 		netdev_dbg(adapter->netdev, "Releasing rx_pool[%d]\n", i);
536 
537 		kfree(rx_pool->free_map);
538 		free_long_term_buff(adapter, &rx_pool->long_term_buff);
539 
540 		if (!rx_pool->rx_buff)
541 			continue;
542 
543 		for (j = 0; j < rx_pool->size; j++) {
544 			if (rx_pool->rx_buff[j].skb) {
545 				dev_kfree_skb_any(rx_pool->rx_buff[j].skb);
546 				rx_pool->rx_buff[j].skb = NULL;
547 			}
548 		}
549 
550 		kfree(rx_pool->rx_buff);
551 	}
552 
553 	kfree(adapter->rx_pool);
554 	adapter->rx_pool = NULL;
555 	adapter->num_active_rx_pools = 0;
556 }
557 
558 static int init_rx_pools(struct net_device *netdev)
559 {
560 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
561 	struct device *dev = &adapter->vdev->dev;
562 	struct ibmvnic_rx_pool *rx_pool;
563 	int rxadd_subcrqs;
564 	u64 *size_array;
565 	int i, j;
566 
567 	rxadd_subcrqs =
568 		be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);
569 	size_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +
570 		be32_to_cpu(adapter->login_rsp_buf->off_rxadd_buff_size));
571 
572 	adapter->rx_pool = kcalloc(rxadd_subcrqs,
573 				   sizeof(struct ibmvnic_rx_pool),
574 				   GFP_KERNEL);
575 	if (!adapter->rx_pool) {
576 		dev_err(dev, "Failed to allocate rx pools\n");
577 		return -1;
578 	}
579 
580 	adapter->num_active_rx_pools = rxadd_subcrqs;
581 
582 	for (i = 0; i < rxadd_subcrqs; i++) {
583 		rx_pool = &adapter->rx_pool[i];
584 
585 		netdev_dbg(adapter->netdev,
586 			   "Initializing rx_pool[%d], %lld buffs, %lld bytes each\n",
587 			   i, adapter->req_rx_add_entries_per_subcrq,
588 			   be64_to_cpu(size_array[i]));
589 
590 		rx_pool->size = adapter->req_rx_add_entries_per_subcrq;
591 		rx_pool->index = i;
592 		rx_pool->buff_size = be64_to_cpu(size_array[i]);
593 		rx_pool->active = 1;
594 
595 		rx_pool->free_map = kcalloc(rx_pool->size, sizeof(int),
596 					    GFP_KERNEL);
597 		if (!rx_pool->free_map) {
598 			release_rx_pools(adapter);
599 			return -1;
600 		}
601 
602 		rx_pool->rx_buff = kcalloc(rx_pool->size,
603 					   sizeof(struct ibmvnic_rx_buff),
604 					   GFP_KERNEL);
605 		if (!rx_pool->rx_buff) {
606 			dev_err(dev, "Couldn't alloc rx buffers\n");
607 			release_rx_pools(adapter);
608 			return -1;
609 		}
610 
611 		if (alloc_long_term_buff(adapter, &rx_pool->long_term_buff,
612 					 rx_pool->size * rx_pool->buff_size)) {
613 			release_rx_pools(adapter);
614 			return -1;
615 		}
616 
617 		for (j = 0; j < rx_pool->size; ++j)
618 			rx_pool->free_map[j] = j;
619 
620 		atomic_set(&rx_pool->available, 0);
621 		rx_pool->next_alloc = 0;
622 		rx_pool->next_free = 0;
623 	}
624 
625 	return 0;
626 }
627 
628 static int reset_one_tx_pool(struct ibmvnic_adapter *adapter,
629 			     struct ibmvnic_tx_pool *tx_pool)
630 {
631 	int rc, i;
632 
633 	rc = reset_long_term_buff(adapter, &tx_pool->long_term_buff);
634 	if (rc)
635 		return rc;
636 
637 	memset(tx_pool->tx_buff, 0,
638 	       tx_pool->num_buffers *
639 	       sizeof(struct ibmvnic_tx_buff));
640 
641 	for (i = 0; i < tx_pool->num_buffers; i++)
642 		tx_pool->free_map[i] = i;
643 
644 	tx_pool->consumer_index = 0;
645 	tx_pool->producer_index = 0;
646 
647 	return 0;
648 }
649 
650 static int reset_tx_pools(struct ibmvnic_adapter *adapter)
651 {
652 	int tx_scrqs;
653 	int i, rc;
654 
655 	if (!adapter->tx_pool)
656 		return -1;
657 
658 	tx_scrqs = be32_to_cpu(adapter->login_rsp_buf->num_txsubm_subcrqs);
659 	for (i = 0; i < tx_scrqs; i++) {
660 		rc = reset_one_tx_pool(adapter, &adapter->tso_pool[i]);
661 		if (rc)
662 			return rc;
663 		rc = reset_one_tx_pool(adapter, &adapter->tx_pool[i]);
664 		if (rc)
665 			return rc;
666 	}
667 
668 	return 0;
669 }
670 
671 static void release_vpd_data(struct ibmvnic_adapter *adapter)
672 {
673 	if (!adapter->vpd)
674 		return;
675 
676 	kfree(adapter->vpd->buff);
677 	kfree(adapter->vpd);
678 
679 	adapter->vpd = NULL;
680 }
681 
682 static void release_one_tx_pool(struct ibmvnic_adapter *adapter,
683 				struct ibmvnic_tx_pool *tx_pool)
684 {
685 	kfree(tx_pool->tx_buff);
686 	kfree(tx_pool->free_map);
687 	free_long_term_buff(adapter, &tx_pool->long_term_buff);
688 }
689 
690 static void release_tx_pools(struct ibmvnic_adapter *adapter)
691 {
692 	int i;
693 
694 	if (!adapter->tx_pool)
695 		return;
696 
697 	for (i = 0; i < adapter->num_active_tx_pools; i++) {
698 		release_one_tx_pool(adapter, &adapter->tx_pool[i]);
699 		release_one_tx_pool(adapter, &adapter->tso_pool[i]);
700 	}
701 
702 	kfree(adapter->tx_pool);
703 	adapter->tx_pool = NULL;
704 	kfree(adapter->tso_pool);
705 	adapter->tso_pool = NULL;
706 	adapter->num_active_tx_pools = 0;
707 }
708 
709 static int init_one_tx_pool(struct net_device *netdev,
710 			    struct ibmvnic_tx_pool *tx_pool,
711 			    int num_entries, int buf_size)
712 {
713 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
714 	int i;
715 
716 	tx_pool->tx_buff = kcalloc(num_entries,
717 				   sizeof(struct ibmvnic_tx_buff),
718 				   GFP_KERNEL);
719 	if (!tx_pool->tx_buff)
720 		return -1;
721 
722 	if (alloc_long_term_buff(adapter, &tx_pool->long_term_buff,
723 				 num_entries * buf_size))
724 		return -1;
725 
726 	tx_pool->free_map = kcalloc(num_entries, sizeof(int), GFP_KERNEL);
727 	if (!tx_pool->free_map)
728 		return -1;
729 
730 	for (i = 0; i < num_entries; i++)
731 		tx_pool->free_map[i] = i;
732 
733 	tx_pool->consumer_index = 0;
734 	tx_pool->producer_index = 0;
735 	tx_pool->num_buffers = num_entries;
736 	tx_pool->buf_size = buf_size;
737 
738 	return 0;
739 }
740 
741 static int init_tx_pools(struct net_device *netdev)
742 {
743 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
744 	int tx_subcrqs;
745 	int i, rc;
746 
747 	tx_subcrqs = be32_to_cpu(adapter->login_rsp_buf->num_txsubm_subcrqs);
748 	adapter->tx_pool = kcalloc(tx_subcrqs,
749 				   sizeof(struct ibmvnic_tx_pool), GFP_KERNEL);
750 	if (!adapter->tx_pool)
751 		return -1;
752 
753 	adapter->tso_pool = kcalloc(tx_subcrqs,
754 				    sizeof(struct ibmvnic_tx_pool), GFP_KERNEL);
755 	if (!adapter->tso_pool)
756 		return -1;
757 
758 	adapter->num_active_tx_pools = tx_subcrqs;
759 
760 	for (i = 0; i < tx_subcrqs; i++) {
761 		rc = init_one_tx_pool(netdev, &adapter->tx_pool[i],
762 				      adapter->req_tx_entries_per_subcrq,
763 				      adapter->req_mtu + VLAN_HLEN);
764 		if (rc) {
765 			release_tx_pools(adapter);
766 			return rc;
767 		}
768 
769 		rc = init_one_tx_pool(netdev, &adapter->tso_pool[i],
770 				      IBMVNIC_TSO_BUFS,
771 				      IBMVNIC_TSO_BUF_SZ);
772 		if (rc) {
773 			release_tx_pools(adapter);
774 			return rc;
775 		}
776 	}
777 
778 	return 0;
779 }
780 
781 static void ibmvnic_napi_enable(struct ibmvnic_adapter *adapter)
782 {
783 	int i;
784 
785 	if (adapter->napi_enabled)
786 		return;
787 
788 	for (i = 0; i < adapter->req_rx_queues; i++)
789 		napi_enable(&adapter->napi[i]);
790 
791 	adapter->napi_enabled = true;
792 }
793 
794 static void ibmvnic_napi_disable(struct ibmvnic_adapter *adapter)
795 {
796 	int i;
797 
798 	if (!adapter->napi_enabled)
799 		return;
800 
801 	for (i = 0; i < adapter->req_rx_queues; i++) {
802 		netdev_dbg(adapter->netdev, "Disabling napi[%d]\n", i);
803 		napi_disable(&adapter->napi[i]);
804 	}
805 
806 	adapter->napi_enabled = false;
807 }
808 
809 static int init_napi(struct ibmvnic_adapter *adapter)
810 {
811 	int i;
812 
813 	adapter->napi = kcalloc(adapter->req_rx_queues,
814 				sizeof(struct napi_struct), GFP_KERNEL);
815 	if (!adapter->napi)
816 		return -ENOMEM;
817 
818 	for (i = 0; i < adapter->req_rx_queues; i++) {
819 		netdev_dbg(adapter->netdev, "Adding napi[%d]\n", i);
820 		netif_napi_add(adapter->netdev, &adapter->napi[i],
821 			       ibmvnic_poll, NAPI_POLL_WEIGHT);
822 	}
823 
824 	adapter->num_active_rx_napi = adapter->req_rx_queues;
825 	return 0;
826 }
827 
828 static void release_napi(struct ibmvnic_adapter *adapter)
829 {
830 	int i;
831 
832 	if (!adapter->napi)
833 		return;
834 
835 	for (i = 0; i < adapter->num_active_rx_napi; i++) {
836 		netdev_dbg(adapter->netdev, "Releasing napi[%d]\n", i);
837 		netif_napi_del(&adapter->napi[i]);
838 	}
839 
840 	kfree(adapter->napi);
841 	adapter->napi = NULL;
842 	adapter->num_active_rx_napi = 0;
843 	adapter->napi_enabled = false;
844 }
845 
846 static int ibmvnic_login(struct net_device *netdev)
847 {
848 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
849 	unsigned long timeout = msecs_to_jiffies(30000);
850 	int retry_count = 0;
851 	int retries = 10;
852 	bool retry;
853 	int rc;
854 
855 	do {
856 		retry = false;
857 		if (retry_count > retries) {
858 			netdev_warn(netdev, "Login attempts exceeded\n");
859 			return -1;
860 		}
861 
862 		adapter->init_done_rc = 0;
863 		reinit_completion(&adapter->init_done);
864 		rc = send_login(adapter);
865 		if (rc) {
866 			netdev_warn(netdev, "Unable to login\n");
867 			return rc;
868 		}
869 
870 		if (!wait_for_completion_timeout(&adapter->init_done,
871 						 timeout)) {
872 			netdev_warn(netdev, "Login timed out, retrying...\n");
873 			retry = true;
874 			adapter->init_done_rc = 0;
875 			retry_count++;
876 			continue;
877 		}
878 
879 		if (adapter->init_done_rc == ABORTED) {
880 			netdev_warn(netdev, "Login aborted, retrying...\n");
881 			retry = true;
882 			adapter->init_done_rc = 0;
883 			retry_count++;
884 			/* FW or device may be busy, so
885 			 * wait a bit before retrying login
886 			 */
887 			msleep(500);
888 		} else if (adapter->init_done_rc == PARTIALSUCCESS) {
889 			retry_count++;
890 			release_sub_crqs(adapter, 1);
891 
892 			retry = true;
893 			netdev_dbg(netdev,
894 				   "Received partial success, retrying...\n");
895 			adapter->init_done_rc = 0;
896 			reinit_completion(&adapter->init_done);
897 			send_cap_queries(adapter);
898 			if (!wait_for_completion_timeout(&adapter->init_done,
899 							 timeout)) {
900 				netdev_warn(netdev,
901 					    "Capabilities query timed out\n");
902 				return -1;
903 			}
904 
905 			rc = init_sub_crqs(adapter);
906 			if (rc) {
907 				netdev_warn(netdev,
908 					    "SCRQ initialization failed\n");
909 				return -1;
910 			}
911 
912 			rc = init_sub_crq_irqs(adapter);
913 			if (rc) {
914 				netdev_warn(netdev,
915 					    "SCRQ irq initialization failed\n");
916 				return -1;
917 			}
918 		} else if (adapter->init_done_rc) {
919 			netdev_warn(netdev, "Adapter login failed\n");
920 			return -1;
921 		}
922 	} while (retry);
923 
924 	__ibmvnic_set_mac(netdev, adapter->mac_addr);
925 
926 	return 0;
927 }
928 
929 static void release_login_buffer(struct ibmvnic_adapter *adapter)
930 {
931 	kfree(adapter->login_buf);
932 	adapter->login_buf = NULL;
933 }
934 
935 static void release_login_rsp_buffer(struct ibmvnic_adapter *adapter)
936 {
937 	kfree(adapter->login_rsp_buf);
938 	adapter->login_rsp_buf = NULL;
939 }
940 
941 static void release_resources(struct ibmvnic_adapter *adapter)
942 {
943 	release_vpd_data(adapter);
944 
945 	release_tx_pools(adapter);
946 	release_rx_pools(adapter);
947 
948 	release_napi(adapter);
949 	release_login_rsp_buffer(adapter);
950 }
951 
952 static int set_link_state(struct ibmvnic_adapter *adapter, u8 link_state)
953 {
954 	struct net_device *netdev = adapter->netdev;
955 	unsigned long timeout = msecs_to_jiffies(30000);
956 	union ibmvnic_crq crq;
957 	bool resend;
958 	int rc;
959 
960 	netdev_dbg(netdev, "setting link state %d\n", link_state);
961 
962 	memset(&crq, 0, sizeof(crq));
963 	crq.logical_link_state.first = IBMVNIC_CRQ_CMD;
964 	crq.logical_link_state.cmd = LOGICAL_LINK_STATE;
965 	crq.logical_link_state.link_state = link_state;
966 
967 	do {
968 		resend = false;
969 
970 		reinit_completion(&adapter->init_done);
971 		rc = ibmvnic_send_crq(adapter, &crq);
972 		if (rc) {
973 			netdev_err(netdev, "Failed to set link state\n");
974 			return rc;
975 		}
976 
977 		if (!wait_for_completion_timeout(&adapter->init_done,
978 						 timeout)) {
979 			netdev_err(netdev, "timeout setting link state\n");
980 			return -1;
981 		}
982 
983 		if (adapter->init_done_rc == 1) {
984 			/* Partuial success, delay and re-send */
985 			mdelay(1000);
986 			resend = true;
987 		} else if (adapter->init_done_rc) {
988 			netdev_warn(netdev, "Unable to set link state, rc=%d\n",
989 				    adapter->init_done_rc);
990 			return adapter->init_done_rc;
991 		}
992 	} while (resend);
993 
994 	return 0;
995 }
996 
997 static int set_real_num_queues(struct net_device *netdev)
998 {
999 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1000 	int rc;
1001 
1002 	netdev_dbg(netdev, "Setting real tx/rx queues (%llx/%llx)\n",
1003 		   adapter->req_tx_queues, adapter->req_rx_queues);
1004 
1005 	rc = netif_set_real_num_tx_queues(netdev, adapter->req_tx_queues);
1006 	if (rc) {
1007 		netdev_err(netdev, "failed to set the number of tx queues\n");
1008 		return rc;
1009 	}
1010 
1011 	rc = netif_set_real_num_rx_queues(netdev, adapter->req_rx_queues);
1012 	if (rc)
1013 		netdev_err(netdev, "failed to set the number of rx queues\n");
1014 
1015 	return rc;
1016 }
1017 
1018 static int ibmvnic_get_vpd(struct ibmvnic_adapter *adapter)
1019 {
1020 	struct device *dev = &adapter->vdev->dev;
1021 	union ibmvnic_crq crq;
1022 	int len = 0;
1023 	int rc;
1024 
1025 	if (adapter->vpd->buff)
1026 		len = adapter->vpd->len;
1027 
1028 	mutex_lock(&adapter->fw_lock);
1029 	adapter->fw_done_rc = 0;
1030 	reinit_completion(&adapter->fw_done);
1031 
1032 	crq.get_vpd_size.first = IBMVNIC_CRQ_CMD;
1033 	crq.get_vpd_size.cmd = GET_VPD_SIZE;
1034 	rc = ibmvnic_send_crq(adapter, &crq);
1035 	if (rc) {
1036 		mutex_unlock(&adapter->fw_lock);
1037 		return rc;
1038 	}
1039 
1040 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);
1041 	if (rc) {
1042 		dev_err(dev, "Could not retrieve VPD size, rc = %d\n", rc);
1043 		mutex_unlock(&adapter->fw_lock);
1044 		return rc;
1045 	}
1046 	mutex_unlock(&adapter->fw_lock);
1047 
1048 	if (!adapter->vpd->len)
1049 		return -ENODATA;
1050 
1051 	if (!adapter->vpd->buff)
1052 		adapter->vpd->buff = kzalloc(adapter->vpd->len, GFP_KERNEL);
1053 	else if (adapter->vpd->len != len)
1054 		adapter->vpd->buff =
1055 			krealloc(adapter->vpd->buff,
1056 				 adapter->vpd->len, GFP_KERNEL);
1057 
1058 	if (!adapter->vpd->buff) {
1059 		dev_err(dev, "Could allocate VPD buffer\n");
1060 		return -ENOMEM;
1061 	}
1062 
1063 	adapter->vpd->dma_addr =
1064 		dma_map_single(dev, adapter->vpd->buff, adapter->vpd->len,
1065 			       DMA_FROM_DEVICE);
1066 	if (dma_mapping_error(dev, adapter->vpd->dma_addr)) {
1067 		dev_err(dev, "Could not map VPD buffer\n");
1068 		kfree(adapter->vpd->buff);
1069 		adapter->vpd->buff = NULL;
1070 		return -ENOMEM;
1071 	}
1072 
1073 	mutex_lock(&adapter->fw_lock);
1074 	adapter->fw_done_rc = 0;
1075 	reinit_completion(&adapter->fw_done);
1076 
1077 	crq.get_vpd.first = IBMVNIC_CRQ_CMD;
1078 	crq.get_vpd.cmd = GET_VPD;
1079 	crq.get_vpd.ioba = cpu_to_be32(adapter->vpd->dma_addr);
1080 	crq.get_vpd.len = cpu_to_be32((u32)adapter->vpd->len);
1081 	rc = ibmvnic_send_crq(adapter, &crq);
1082 	if (rc) {
1083 		kfree(adapter->vpd->buff);
1084 		adapter->vpd->buff = NULL;
1085 		mutex_unlock(&adapter->fw_lock);
1086 		return rc;
1087 	}
1088 
1089 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);
1090 	if (rc) {
1091 		dev_err(dev, "Unable to retrieve VPD, rc = %d\n", rc);
1092 		kfree(adapter->vpd->buff);
1093 		adapter->vpd->buff = NULL;
1094 		mutex_unlock(&adapter->fw_lock);
1095 		return rc;
1096 	}
1097 
1098 	mutex_unlock(&adapter->fw_lock);
1099 	return 0;
1100 }
1101 
1102 static int init_resources(struct ibmvnic_adapter *adapter)
1103 {
1104 	struct net_device *netdev = adapter->netdev;
1105 	int rc;
1106 
1107 	rc = set_real_num_queues(netdev);
1108 	if (rc)
1109 		return rc;
1110 
1111 	adapter->vpd = kzalloc(sizeof(*adapter->vpd), GFP_KERNEL);
1112 	if (!adapter->vpd)
1113 		return -ENOMEM;
1114 
1115 	/* Vital Product Data (VPD) */
1116 	rc = ibmvnic_get_vpd(adapter);
1117 	if (rc) {
1118 		netdev_err(netdev, "failed to initialize Vital Product Data (VPD)\n");
1119 		return rc;
1120 	}
1121 
1122 	adapter->map_id = 1;
1123 
1124 	rc = init_napi(adapter);
1125 	if (rc)
1126 		return rc;
1127 
1128 	send_map_query(adapter);
1129 
1130 	rc = init_rx_pools(netdev);
1131 	if (rc)
1132 		return rc;
1133 
1134 	rc = init_tx_pools(netdev);
1135 	return rc;
1136 }
1137 
1138 static int __ibmvnic_open(struct net_device *netdev)
1139 {
1140 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1141 	enum vnic_state prev_state = adapter->state;
1142 	int i, rc;
1143 
1144 	adapter->state = VNIC_OPENING;
1145 	replenish_pools(adapter);
1146 	ibmvnic_napi_enable(adapter);
1147 
1148 	/* We're ready to receive frames, enable the sub-crq interrupts and
1149 	 * set the logical link state to up
1150 	 */
1151 	for (i = 0; i < adapter->req_rx_queues; i++) {
1152 		netdev_dbg(netdev, "Enabling rx_scrq[%d] irq\n", i);
1153 		if (prev_state == VNIC_CLOSED)
1154 			enable_irq(adapter->rx_scrq[i]->irq);
1155 		enable_scrq_irq(adapter, adapter->rx_scrq[i]);
1156 	}
1157 
1158 	for (i = 0; i < adapter->req_tx_queues; i++) {
1159 		netdev_dbg(netdev, "Enabling tx_scrq[%d] irq\n", i);
1160 		if (prev_state == VNIC_CLOSED)
1161 			enable_irq(adapter->tx_scrq[i]->irq);
1162 		enable_scrq_irq(adapter, adapter->tx_scrq[i]);
1163 	}
1164 
1165 	rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_UP);
1166 	if (rc) {
1167 		for (i = 0; i < adapter->req_rx_queues; i++)
1168 			napi_disable(&adapter->napi[i]);
1169 		release_resources(adapter);
1170 		return rc;
1171 	}
1172 
1173 	netif_tx_start_all_queues(netdev);
1174 
1175 	if (prev_state == VNIC_CLOSED) {
1176 		for (i = 0; i < adapter->req_rx_queues; i++)
1177 			napi_schedule(&adapter->napi[i]);
1178 	}
1179 
1180 	adapter->state = VNIC_OPEN;
1181 	return rc;
1182 }
1183 
1184 static int ibmvnic_open(struct net_device *netdev)
1185 {
1186 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1187 	int rc;
1188 
1189 	/* If device failover is pending, just set device state and return.
1190 	 * Device operation will be handled by reset routine.
1191 	 */
1192 	if (adapter->failover_pending) {
1193 		adapter->state = VNIC_OPEN;
1194 		return 0;
1195 	}
1196 
1197 	if (adapter->state != VNIC_CLOSED) {
1198 		rc = ibmvnic_login(netdev);
1199 		if (rc)
1200 			return rc;
1201 
1202 		rc = init_resources(adapter);
1203 		if (rc) {
1204 			netdev_err(netdev, "failed to initialize resources\n");
1205 			release_resources(adapter);
1206 			return rc;
1207 		}
1208 	}
1209 
1210 	rc = __ibmvnic_open(netdev);
1211 
1212 	return rc;
1213 }
1214 
1215 static void clean_rx_pools(struct ibmvnic_adapter *adapter)
1216 {
1217 	struct ibmvnic_rx_pool *rx_pool;
1218 	struct ibmvnic_rx_buff *rx_buff;
1219 	u64 rx_entries;
1220 	int rx_scrqs;
1221 	int i, j;
1222 
1223 	if (!adapter->rx_pool)
1224 		return;
1225 
1226 	rx_scrqs = adapter->num_active_rx_pools;
1227 	rx_entries = adapter->req_rx_add_entries_per_subcrq;
1228 
1229 	/* Free any remaining skbs in the rx buffer pools */
1230 	for (i = 0; i < rx_scrqs; i++) {
1231 		rx_pool = &adapter->rx_pool[i];
1232 		if (!rx_pool || !rx_pool->rx_buff)
1233 			continue;
1234 
1235 		netdev_dbg(adapter->netdev, "Cleaning rx_pool[%d]\n", i);
1236 		for (j = 0; j < rx_entries; j++) {
1237 			rx_buff = &rx_pool->rx_buff[j];
1238 			if (rx_buff && rx_buff->skb) {
1239 				dev_kfree_skb_any(rx_buff->skb);
1240 				rx_buff->skb = NULL;
1241 			}
1242 		}
1243 	}
1244 }
1245 
1246 static void clean_one_tx_pool(struct ibmvnic_adapter *adapter,
1247 			      struct ibmvnic_tx_pool *tx_pool)
1248 {
1249 	struct ibmvnic_tx_buff *tx_buff;
1250 	u64 tx_entries;
1251 	int i;
1252 
1253 	if (!tx_pool || !tx_pool->tx_buff)
1254 		return;
1255 
1256 	tx_entries = tx_pool->num_buffers;
1257 
1258 	for (i = 0; i < tx_entries; i++) {
1259 		tx_buff = &tx_pool->tx_buff[i];
1260 		if (tx_buff && tx_buff->skb) {
1261 			dev_kfree_skb_any(tx_buff->skb);
1262 			tx_buff->skb = NULL;
1263 		}
1264 	}
1265 }
1266 
1267 static void clean_tx_pools(struct ibmvnic_adapter *adapter)
1268 {
1269 	int tx_scrqs;
1270 	int i;
1271 
1272 	if (!adapter->tx_pool || !adapter->tso_pool)
1273 		return;
1274 
1275 	tx_scrqs = adapter->num_active_tx_pools;
1276 
1277 	/* Free any remaining skbs in the tx buffer pools */
1278 	for (i = 0; i < tx_scrqs; i++) {
1279 		netdev_dbg(adapter->netdev, "Cleaning tx_pool[%d]\n", i);
1280 		clean_one_tx_pool(adapter, &adapter->tx_pool[i]);
1281 		clean_one_tx_pool(adapter, &adapter->tso_pool[i]);
1282 	}
1283 }
1284 
1285 static void ibmvnic_disable_irqs(struct ibmvnic_adapter *adapter)
1286 {
1287 	struct net_device *netdev = adapter->netdev;
1288 	int i;
1289 
1290 	if (adapter->tx_scrq) {
1291 		for (i = 0; i < adapter->req_tx_queues; i++)
1292 			if (adapter->tx_scrq[i]->irq) {
1293 				netdev_dbg(netdev,
1294 					   "Disabling tx_scrq[%d] irq\n", i);
1295 				disable_scrq_irq(adapter, adapter->tx_scrq[i]);
1296 				disable_irq(adapter->tx_scrq[i]->irq);
1297 			}
1298 	}
1299 
1300 	if (adapter->rx_scrq) {
1301 		for (i = 0; i < adapter->req_rx_queues; i++) {
1302 			if (adapter->rx_scrq[i]->irq) {
1303 				netdev_dbg(netdev,
1304 					   "Disabling rx_scrq[%d] irq\n", i);
1305 				disable_scrq_irq(adapter, adapter->rx_scrq[i]);
1306 				disable_irq(adapter->rx_scrq[i]->irq);
1307 			}
1308 		}
1309 	}
1310 }
1311 
1312 static void ibmvnic_cleanup(struct net_device *netdev)
1313 {
1314 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1315 
1316 	/* ensure that transmissions are stopped if called by do_reset */
1317 	if (test_bit(0, &adapter->resetting))
1318 		netif_tx_disable(netdev);
1319 	else
1320 		netif_tx_stop_all_queues(netdev);
1321 
1322 	ibmvnic_napi_disable(adapter);
1323 	ibmvnic_disable_irqs(adapter);
1324 
1325 	clean_rx_pools(adapter);
1326 	clean_tx_pools(adapter);
1327 }
1328 
1329 static int __ibmvnic_close(struct net_device *netdev)
1330 {
1331 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1332 	int rc = 0;
1333 
1334 	adapter->state = VNIC_CLOSING;
1335 	rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_DN);
1336 	if (rc)
1337 		return rc;
1338 	adapter->state = VNIC_CLOSED;
1339 	return 0;
1340 }
1341 
1342 static int ibmvnic_close(struct net_device *netdev)
1343 {
1344 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1345 	int rc;
1346 
1347 	/* If device failover is pending, just set device state and return.
1348 	 * Device operation will be handled by reset routine.
1349 	 */
1350 	if (adapter->failover_pending) {
1351 		adapter->state = VNIC_CLOSED;
1352 		return 0;
1353 	}
1354 
1355 	rc = __ibmvnic_close(netdev);
1356 	ibmvnic_cleanup(netdev);
1357 
1358 	return rc;
1359 }
1360 
1361 /**
1362  * build_hdr_data - creates L2/L3/L4 header data buffer
1363  * @hdr_field - bitfield determining needed headers
1364  * @skb - socket buffer
1365  * @hdr_len - array of header lengths
1366  * @tot_len - total length of data
1367  *
1368  * Reads hdr_field to determine which headers are needed by firmware.
1369  * Builds a buffer containing these headers.  Saves individual header
1370  * lengths and total buffer length to be used to build descriptors.
1371  */
1372 static int build_hdr_data(u8 hdr_field, struct sk_buff *skb,
1373 			  int *hdr_len, u8 *hdr_data)
1374 {
1375 	int len = 0;
1376 	u8 *hdr;
1377 
1378 	if (skb_vlan_tagged(skb) && !skb_vlan_tag_present(skb))
1379 		hdr_len[0] = sizeof(struct vlan_ethhdr);
1380 	else
1381 		hdr_len[0] = sizeof(struct ethhdr);
1382 
1383 	if (skb->protocol == htons(ETH_P_IP)) {
1384 		hdr_len[1] = ip_hdr(skb)->ihl * 4;
1385 		if (ip_hdr(skb)->protocol == IPPROTO_TCP)
1386 			hdr_len[2] = tcp_hdrlen(skb);
1387 		else if (ip_hdr(skb)->protocol == IPPROTO_UDP)
1388 			hdr_len[2] = sizeof(struct udphdr);
1389 	} else if (skb->protocol == htons(ETH_P_IPV6)) {
1390 		hdr_len[1] = sizeof(struct ipv6hdr);
1391 		if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
1392 			hdr_len[2] = tcp_hdrlen(skb);
1393 		else if (ipv6_hdr(skb)->nexthdr == IPPROTO_UDP)
1394 			hdr_len[2] = sizeof(struct udphdr);
1395 	} else if (skb->protocol == htons(ETH_P_ARP)) {
1396 		hdr_len[1] = arp_hdr_len(skb->dev);
1397 		hdr_len[2] = 0;
1398 	}
1399 
1400 	memset(hdr_data, 0, 120);
1401 	if ((hdr_field >> 6) & 1) {
1402 		hdr = skb_mac_header(skb);
1403 		memcpy(hdr_data, hdr, hdr_len[0]);
1404 		len += hdr_len[0];
1405 	}
1406 
1407 	if ((hdr_field >> 5) & 1) {
1408 		hdr = skb_network_header(skb);
1409 		memcpy(hdr_data + len, hdr, hdr_len[1]);
1410 		len += hdr_len[1];
1411 	}
1412 
1413 	if ((hdr_field >> 4) & 1) {
1414 		hdr = skb_transport_header(skb);
1415 		memcpy(hdr_data + len, hdr, hdr_len[2]);
1416 		len += hdr_len[2];
1417 	}
1418 	return len;
1419 }
1420 
1421 /**
1422  * create_hdr_descs - create header and header extension descriptors
1423  * @hdr_field - bitfield determining needed headers
1424  * @data - buffer containing header data
1425  * @len - length of data buffer
1426  * @hdr_len - array of individual header lengths
1427  * @scrq_arr - descriptor array
1428  *
1429  * Creates header and, if needed, header extension descriptors and
1430  * places them in a descriptor array, scrq_arr
1431  */
1432 
1433 static int create_hdr_descs(u8 hdr_field, u8 *hdr_data, int len, int *hdr_len,
1434 			    union sub_crq *scrq_arr)
1435 {
1436 	union sub_crq hdr_desc;
1437 	int tmp_len = len;
1438 	int num_descs = 0;
1439 	u8 *data, *cur;
1440 	int tmp;
1441 
1442 	while (tmp_len > 0) {
1443 		cur = hdr_data + len - tmp_len;
1444 
1445 		memset(&hdr_desc, 0, sizeof(hdr_desc));
1446 		if (cur != hdr_data) {
1447 			data = hdr_desc.hdr_ext.data;
1448 			tmp = tmp_len > 29 ? 29 : tmp_len;
1449 			hdr_desc.hdr_ext.first = IBMVNIC_CRQ_CMD;
1450 			hdr_desc.hdr_ext.type = IBMVNIC_HDR_EXT_DESC;
1451 			hdr_desc.hdr_ext.len = tmp;
1452 		} else {
1453 			data = hdr_desc.hdr.data;
1454 			tmp = tmp_len > 24 ? 24 : tmp_len;
1455 			hdr_desc.hdr.first = IBMVNIC_CRQ_CMD;
1456 			hdr_desc.hdr.type = IBMVNIC_HDR_DESC;
1457 			hdr_desc.hdr.len = tmp;
1458 			hdr_desc.hdr.l2_len = (u8)hdr_len[0];
1459 			hdr_desc.hdr.l3_len = cpu_to_be16((u16)hdr_len[1]);
1460 			hdr_desc.hdr.l4_len = (u8)hdr_len[2];
1461 			hdr_desc.hdr.flag = hdr_field << 1;
1462 		}
1463 		memcpy(data, cur, tmp);
1464 		tmp_len -= tmp;
1465 		*scrq_arr = hdr_desc;
1466 		scrq_arr++;
1467 		num_descs++;
1468 	}
1469 
1470 	return num_descs;
1471 }
1472 
1473 /**
1474  * build_hdr_descs_arr - build a header descriptor array
1475  * @skb - socket buffer
1476  * @num_entries - number of descriptors to be sent
1477  * @subcrq - first TX descriptor
1478  * @hdr_field - bit field determining which headers will be sent
1479  *
1480  * This function will build a TX descriptor array with applicable
1481  * L2/L3/L4 packet header descriptors to be sent by send_subcrq_indirect.
1482  */
1483 
1484 static void build_hdr_descs_arr(struct ibmvnic_tx_buff *txbuff,
1485 				int *num_entries, u8 hdr_field)
1486 {
1487 	int hdr_len[3] = {0, 0, 0};
1488 	int tot_len;
1489 	u8 *hdr_data = txbuff->hdr_data;
1490 
1491 	tot_len = build_hdr_data(hdr_field, txbuff->skb, hdr_len,
1492 				 txbuff->hdr_data);
1493 	*num_entries += create_hdr_descs(hdr_field, hdr_data, tot_len, hdr_len,
1494 			 txbuff->indir_arr + 1);
1495 }
1496 
1497 static int ibmvnic_xmit_workarounds(struct sk_buff *skb,
1498 				    struct net_device *netdev)
1499 {
1500 	/* For some backing devices, mishandling of small packets
1501 	 * can result in a loss of connection or TX stall. Device
1502 	 * architects recommend that no packet should be smaller
1503 	 * than the minimum MTU value provided to the driver, so
1504 	 * pad any packets to that length
1505 	 */
1506 	if (skb->len < netdev->min_mtu)
1507 		return skb_put_padto(skb, netdev->min_mtu);
1508 
1509 	return 0;
1510 }
1511 
1512 static netdev_tx_t ibmvnic_xmit(struct sk_buff *skb, struct net_device *netdev)
1513 {
1514 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1515 	int queue_num = skb_get_queue_mapping(skb);
1516 	u8 *hdrs = (u8 *)&adapter->tx_rx_desc_req;
1517 	struct device *dev = &adapter->vdev->dev;
1518 	struct ibmvnic_tx_buff *tx_buff = NULL;
1519 	struct ibmvnic_sub_crq_queue *tx_scrq;
1520 	struct ibmvnic_tx_pool *tx_pool;
1521 	unsigned int tx_send_failed = 0;
1522 	unsigned int tx_map_failed = 0;
1523 	unsigned int tx_dropped = 0;
1524 	unsigned int tx_packets = 0;
1525 	unsigned int tx_bytes = 0;
1526 	dma_addr_t data_dma_addr;
1527 	struct netdev_queue *txq;
1528 	unsigned long lpar_rc;
1529 	union sub_crq tx_crq;
1530 	unsigned int offset;
1531 	int num_entries = 1;
1532 	unsigned char *dst;
1533 	u64 *handle_array;
1534 	int index = 0;
1535 	u8 proto = 0;
1536 	netdev_tx_t ret = NETDEV_TX_OK;
1537 
1538 	if (test_bit(0, &adapter->resetting)) {
1539 		if (!netif_subqueue_stopped(netdev, skb))
1540 			netif_stop_subqueue(netdev, queue_num);
1541 		dev_kfree_skb_any(skb);
1542 
1543 		tx_send_failed++;
1544 		tx_dropped++;
1545 		ret = NETDEV_TX_OK;
1546 		goto out;
1547 	}
1548 
1549 	if (ibmvnic_xmit_workarounds(skb, netdev)) {
1550 		tx_dropped++;
1551 		tx_send_failed++;
1552 		ret = NETDEV_TX_OK;
1553 		goto out;
1554 	}
1555 	if (skb_is_gso(skb))
1556 		tx_pool = &adapter->tso_pool[queue_num];
1557 	else
1558 		tx_pool = &adapter->tx_pool[queue_num];
1559 
1560 	tx_scrq = adapter->tx_scrq[queue_num];
1561 	txq = netdev_get_tx_queue(netdev, skb_get_queue_mapping(skb));
1562 	handle_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +
1563 		be32_to_cpu(adapter->login_rsp_buf->off_txsubm_subcrqs));
1564 
1565 	index = tx_pool->free_map[tx_pool->consumer_index];
1566 
1567 	if (index == IBMVNIC_INVALID_MAP) {
1568 		dev_kfree_skb_any(skb);
1569 		tx_send_failed++;
1570 		tx_dropped++;
1571 		ret = NETDEV_TX_OK;
1572 		goto out;
1573 	}
1574 
1575 	tx_pool->free_map[tx_pool->consumer_index] = IBMVNIC_INVALID_MAP;
1576 
1577 	offset = index * tx_pool->buf_size;
1578 	dst = tx_pool->long_term_buff.buff + offset;
1579 	memset(dst, 0, tx_pool->buf_size);
1580 	data_dma_addr = tx_pool->long_term_buff.addr + offset;
1581 
1582 	if (skb_shinfo(skb)->nr_frags) {
1583 		int cur, i;
1584 
1585 		/* Copy the head */
1586 		skb_copy_from_linear_data(skb, dst, skb_headlen(skb));
1587 		cur = skb_headlen(skb);
1588 
1589 		/* Copy the frags */
1590 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1591 			const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1592 
1593 			memcpy(dst + cur,
1594 			       page_address(skb_frag_page(frag)) +
1595 			       skb_frag_off(frag), skb_frag_size(frag));
1596 			cur += skb_frag_size(frag);
1597 		}
1598 	} else {
1599 		skb_copy_from_linear_data(skb, dst, skb->len);
1600 	}
1601 
1602 	tx_pool->consumer_index =
1603 	    (tx_pool->consumer_index + 1) % tx_pool->num_buffers;
1604 
1605 	tx_buff = &tx_pool->tx_buff[index];
1606 	tx_buff->skb = skb;
1607 	tx_buff->data_dma[0] = data_dma_addr;
1608 	tx_buff->data_len[0] = skb->len;
1609 	tx_buff->index = index;
1610 	tx_buff->pool_index = queue_num;
1611 	tx_buff->last_frag = true;
1612 
1613 	memset(&tx_crq, 0, sizeof(tx_crq));
1614 	tx_crq.v1.first = IBMVNIC_CRQ_CMD;
1615 	tx_crq.v1.type = IBMVNIC_TX_DESC;
1616 	tx_crq.v1.n_crq_elem = 1;
1617 	tx_crq.v1.n_sge = 1;
1618 	tx_crq.v1.flags1 = IBMVNIC_TX_COMP_NEEDED;
1619 
1620 	if (skb_is_gso(skb))
1621 		tx_crq.v1.correlator =
1622 			cpu_to_be32(index | IBMVNIC_TSO_POOL_MASK);
1623 	else
1624 		tx_crq.v1.correlator = cpu_to_be32(index);
1625 	tx_crq.v1.dma_reg = cpu_to_be16(tx_pool->long_term_buff.map_id);
1626 	tx_crq.v1.sge_len = cpu_to_be32(skb->len);
1627 	tx_crq.v1.ioba = cpu_to_be64(data_dma_addr);
1628 
1629 	if (adapter->vlan_header_insertion && skb_vlan_tag_present(skb)) {
1630 		tx_crq.v1.flags2 |= IBMVNIC_TX_VLAN_INSERT;
1631 		tx_crq.v1.vlan_id = cpu_to_be16(skb->vlan_tci);
1632 	}
1633 
1634 	if (skb->protocol == htons(ETH_P_IP)) {
1635 		tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_IPV4;
1636 		proto = ip_hdr(skb)->protocol;
1637 	} else if (skb->protocol == htons(ETH_P_IPV6)) {
1638 		tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_IPV6;
1639 		proto = ipv6_hdr(skb)->nexthdr;
1640 	}
1641 
1642 	if (proto == IPPROTO_TCP)
1643 		tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_TCP;
1644 	else if (proto == IPPROTO_UDP)
1645 		tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_UDP;
1646 
1647 	if (skb->ip_summed == CHECKSUM_PARTIAL) {
1648 		tx_crq.v1.flags1 |= IBMVNIC_TX_CHKSUM_OFFLOAD;
1649 		hdrs += 2;
1650 	}
1651 	if (skb_is_gso(skb)) {
1652 		tx_crq.v1.flags1 |= IBMVNIC_TX_LSO;
1653 		tx_crq.v1.mss = cpu_to_be16(skb_shinfo(skb)->gso_size);
1654 		hdrs += 2;
1655 	}
1656 	/* determine if l2/3/4 headers are sent to firmware */
1657 	if ((*hdrs >> 7) & 1) {
1658 		build_hdr_descs_arr(tx_buff, &num_entries, *hdrs);
1659 		tx_crq.v1.n_crq_elem = num_entries;
1660 		tx_buff->num_entries = num_entries;
1661 		tx_buff->indir_arr[0] = tx_crq;
1662 		tx_buff->indir_dma = dma_map_single(dev, tx_buff->indir_arr,
1663 						    sizeof(tx_buff->indir_arr),
1664 						    DMA_TO_DEVICE);
1665 		if (dma_mapping_error(dev, tx_buff->indir_dma)) {
1666 			dev_kfree_skb_any(skb);
1667 			tx_buff->skb = NULL;
1668 			if (!firmware_has_feature(FW_FEATURE_CMO))
1669 				dev_err(dev, "tx: unable to map descriptor array\n");
1670 			tx_map_failed++;
1671 			tx_dropped++;
1672 			ret = NETDEV_TX_OK;
1673 			goto tx_err_out;
1674 		}
1675 		lpar_rc = send_subcrq_indirect(adapter, handle_array[queue_num],
1676 					       (u64)tx_buff->indir_dma,
1677 					       (u64)num_entries);
1678 		dma_unmap_single(dev, tx_buff->indir_dma,
1679 				 sizeof(tx_buff->indir_arr), DMA_TO_DEVICE);
1680 	} else {
1681 		tx_buff->num_entries = num_entries;
1682 		lpar_rc = send_subcrq(adapter, handle_array[queue_num],
1683 				      &tx_crq);
1684 	}
1685 	if (lpar_rc != H_SUCCESS) {
1686 		if (lpar_rc != H_CLOSED && lpar_rc != H_PARAMETER)
1687 			dev_err_ratelimited(dev, "tx: send failed\n");
1688 		dev_kfree_skb_any(skb);
1689 		tx_buff->skb = NULL;
1690 
1691 		if (lpar_rc == H_CLOSED || adapter->failover_pending) {
1692 			/* Disable TX and report carrier off if queue is closed
1693 			 * or pending failover.
1694 			 * Firmware guarantees that a signal will be sent to the
1695 			 * driver, triggering a reset or some other action.
1696 			 */
1697 			netif_tx_stop_all_queues(netdev);
1698 			netif_carrier_off(netdev);
1699 		}
1700 
1701 		tx_send_failed++;
1702 		tx_dropped++;
1703 		ret = NETDEV_TX_OK;
1704 		goto tx_err_out;
1705 	}
1706 
1707 	if (atomic_add_return(num_entries, &tx_scrq->used)
1708 					>= adapter->req_tx_entries_per_subcrq) {
1709 		netdev_dbg(netdev, "Stopping queue %d\n", queue_num);
1710 		netif_stop_subqueue(netdev, queue_num);
1711 	}
1712 
1713 	tx_packets++;
1714 	tx_bytes += skb->len;
1715 	txq->trans_start = jiffies;
1716 	ret = NETDEV_TX_OK;
1717 	goto out;
1718 
1719 tx_err_out:
1720 	/* roll back consumer index and map array*/
1721 	if (tx_pool->consumer_index == 0)
1722 		tx_pool->consumer_index =
1723 			tx_pool->num_buffers - 1;
1724 	else
1725 		tx_pool->consumer_index--;
1726 	tx_pool->free_map[tx_pool->consumer_index] = index;
1727 out:
1728 	netdev->stats.tx_dropped += tx_dropped;
1729 	netdev->stats.tx_bytes += tx_bytes;
1730 	netdev->stats.tx_packets += tx_packets;
1731 	adapter->tx_send_failed += tx_send_failed;
1732 	adapter->tx_map_failed += tx_map_failed;
1733 	adapter->tx_stats_buffers[queue_num].packets += tx_packets;
1734 	adapter->tx_stats_buffers[queue_num].bytes += tx_bytes;
1735 	adapter->tx_stats_buffers[queue_num].dropped_packets += tx_dropped;
1736 
1737 	return ret;
1738 }
1739 
1740 static void ibmvnic_set_multi(struct net_device *netdev)
1741 {
1742 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1743 	struct netdev_hw_addr *ha;
1744 	union ibmvnic_crq crq;
1745 
1746 	memset(&crq, 0, sizeof(crq));
1747 	crq.request_capability.first = IBMVNIC_CRQ_CMD;
1748 	crq.request_capability.cmd = REQUEST_CAPABILITY;
1749 
1750 	if (netdev->flags & IFF_PROMISC) {
1751 		if (!adapter->promisc_supported)
1752 			return;
1753 	} else {
1754 		if (netdev->flags & IFF_ALLMULTI) {
1755 			/* Accept all multicast */
1756 			memset(&crq, 0, sizeof(crq));
1757 			crq.multicast_ctrl.first = IBMVNIC_CRQ_CMD;
1758 			crq.multicast_ctrl.cmd = MULTICAST_CTRL;
1759 			crq.multicast_ctrl.flags = IBMVNIC_ENABLE_ALL;
1760 			ibmvnic_send_crq(adapter, &crq);
1761 		} else if (netdev_mc_empty(netdev)) {
1762 			/* Reject all multicast */
1763 			memset(&crq, 0, sizeof(crq));
1764 			crq.multicast_ctrl.first = IBMVNIC_CRQ_CMD;
1765 			crq.multicast_ctrl.cmd = MULTICAST_CTRL;
1766 			crq.multicast_ctrl.flags = IBMVNIC_DISABLE_ALL;
1767 			ibmvnic_send_crq(adapter, &crq);
1768 		} else {
1769 			/* Accept one or more multicast(s) */
1770 			netdev_for_each_mc_addr(ha, netdev) {
1771 				memset(&crq, 0, sizeof(crq));
1772 				crq.multicast_ctrl.first = IBMVNIC_CRQ_CMD;
1773 				crq.multicast_ctrl.cmd = MULTICAST_CTRL;
1774 				crq.multicast_ctrl.flags = IBMVNIC_ENABLE_MC;
1775 				ether_addr_copy(&crq.multicast_ctrl.mac_addr[0],
1776 						ha->addr);
1777 				ibmvnic_send_crq(adapter, &crq);
1778 			}
1779 		}
1780 	}
1781 }
1782 
1783 static int __ibmvnic_set_mac(struct net_device *netdev, u8 *dev_addr)
1784 {
1785 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1786 	union ibmvnic_crq crq;
1787 	int rc;
1788 
1789 	if (!is_valid_ether_addr(dev_addr)) {
1790 		rc = -EADDRNOTAVAIL;
1791 		goto err;
1792 	}
1793 
1794 	memset(&crq, 0, sizeof(crq));
1795 	crq.change_mac_addr.first = IBMVNIC_CRQ_CMD;
1796 	crq.change_mac_addr.cmd = CHANGE_MAC_ADDR;
1797 	ether_addr_copy(&crq.change_mac_addr.mac_addr[0], dev_addr);
1798 
1799 	mutex_lock(&adapter->fw_lock);
1800 	adapter->fw_done_rc = 0;
1801 	reinit_completion(&adapter->fw_done);
1802 
1803 	rc = ibmvnic_send_crq(adapter, &crq);
1804 	if (rc) {
1805 		rc = -EIO;
1806 		mutex_unlock(&adapter->fw_lock);
1807 		goto err;
1808 	}
1809 
1810 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);
1811 	/* netdev->dev_addr is changed in handle_change_mac_rsp function */
1812 	if (rc || adapter->fw_done_rc) {
1813 		rc = -EIO;
1814 		mutex_unlock(&adapter->fw_lock);
1815 		goto err;
1816 	}
1817 	mutex_unlock(&adapter->fw_lock);
1818 	return 0;
1819 err:
1820 	ether_addr_copy(adapter->mac_addr, netdev->dev_addr);
1821 	return rc;
1822 }
1823 
1824 static int ibmvnic_set_mac(struct net_device *netdev, void *p)
1825 {
1826 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1827 	struct sockaddr *addr = p;
1828 	int rc;
1829 
1830 	rc = 0;
1831 	ether_addr_copy(adapter->mac_addr, addr->sa_data);
1832 	if (adapter->state != VNIC_PROBED)
1833 		rc = __ibmvnic_set_mac(netdev, addr->sa_data);
1834 
1835 	return rc;
1836 }
1837 
1838 /**
1839  * do_change_param_reset returns zero if we are able to keep processing reset
1840  * events, or non-zero if we hit a fatal error and must halt.
1841  */
1842 static int do_change_param_reset(struct ibmvnic_adapter *adapter,
1843 				 struct ibmvnic_rwi *rwi,
1844 				 u32 reset_state)
1845 {
1846 	struct net_device *netdev = adapter->netdev;
1847 	int i, rc;
1848 
1849 	netdev_dbg(adapter->netdev, "Change param resetting driver (%d)\n",
1850 		   rwi->reset_reason);
1851 
1852 	netif_carrier_off(netdev);
1853 	adapter->reset_reason = rwi->reset_reason;
1854 
1855 	ibmvnic_cleanup(netdev);
1856 
1857 	if (reset_state == VNIC_OPEN) {
1858 		rc = __ibmvnic_close(netdev);
1859 		if (rc)
1860 			return rc;
1861 	}
1862 
1863 	release_resources(adapter);
1864 	release_sub_crqs(adapter, 1);
1865 	release_crq_queue(adapter);
1866 
1867 	adapter->state = VNIC_PROBED;
1868 
1869 	rc = init_crq_queue(adapter);
1870 
1871 	if (rc) {
1872 		netdev_err(adapter->netdev,
1873 			   "Couldn't initialize crq. rc=%d\n", rc);
1874 		return rc;
1875 	}
1876 
1877 	rc = ibmvnic_reset_init(adapter);
1878 	if (rc)
1879 		return IBMVNIC_INIT_FAILED;
1880 
1881 	/* If the adapter was in PROBE state prior to the reset,
1882 	 * exit here.
1883 	 */
1884 	if (reset_state == VNIC_PROBED)
1885 		return 0;
1886 
1887 	rc = ibmvnic_login(netdev);
1888 	if (rc) {
1889 		adapter->state = reset_state;
1890 		return rc;
1891 	}
1892 
1893 	rc = init_resources(adapter);
1894 	if (rc)
1895 		return rc;
1896 
1897 	ibmvnic_disable_irqs(adapter);
1898 
1899 	adapter->state = VNIC_CLOSED;
1900 
1901 	if (reset_state == VNIC_CLOSED)
1902 		return 0;
1903 
1904 	rc = __ibmvnic_open(netdev);
1905 	if (rc)
1906 		return IBMVNIC_OPEN_FAILED;
1907 
1908 	/* refresh device's multicast list */
1909 	ibmvnic_set_multi(netdev);
1910 
1911 	/* kick napi */
1912 	for (i = 0; i < adapter->req_rx_queues; i++)
1913 		napi_schedule(&adapter->napi[i]);
1914 
1915 	return 0;
1916 }
1917 
1918 /**
1919  * do_reset returns zero if we are able to keep processing reset events, or
1920  * non-zero if we hit a fatal error and must halt.
1921  */
1922 static int do_reset(struct ibmvnic_adapter *adapter,
1923 		    struct ibmvnic_rwi *rwi, u32 reset_state)
1924 {
1925 	u64 old_num_rx_queues, old_num_tx_queues;
1926 	u64 old_num_rx_slots, old_num_tx_slots;
1927 	struct net_device *netdev = adapter->netdev;
1928 	int i, rc;
1929 
1930 	netdev_dbg(adapter->netdev, "Re-setting driver (%d)\n",
1931 		   rwi->reset_reason);
1932 
1933 	rtnl_lock();
1934 
1935 	netif_carrier_off(netdev);
1936 	adapter->reset_reason = rwi->reset_reason;
1937 
1938 	old_num_rx_queues = adapter->req_rx_queues;
1939 	old_num_tx_queues = adapter->req_tx_queues;
1940 	old_num_rx_slots = adapter->req_rx_add_entries_per_subcrq;
1941 	old_num_tx_slots = adapter->req_tx_entries_per_subcrq;
1942 
1943 	ibmvnic_cleanup(netdev);
1944 
1945 	if (reset_state == VNIC_OPEN &&
1946 	    adapter->reset_reason != VNIC_RESET_MOBILITY &&
1947 	    adapter->reset_reason != VNIC_RESET_FAILOVER) {
1948 		adapter->state = VNIC_CLOSING;
1949 
1950 		/* Release the RTNL lock before link state change and
1951 		 * re-acquire after the link state change to allow
1952 		 * linkwatch_event to grab the RTNL lock and run during
1953 		 * a reset.
1954 		 */
1955 		rtnl_unlock();
1956 		rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_DN);
1957 		rtnl_lock();
1958 		if (rc)
1959 			goto out;
1960 
1961 		if (adapter->state != VNIC_CLOSING) {
1962 			rc = -1;
1963 			goto out;
1964 		}
1965 
1966 		adapter->state = VNIC_CLOSED;
1967 	}
1968 
1969 	if (adapter->reset_reason != VNIC_RESET_NON_FATAL) {
1970 		/* remove the closed state so when we call open it appears
1971 		 * we are coming from the probed state.
1972 		 */
1973 		adapter->state = VNIC_PROBED;
1974 
1975 		if (adapter->reset_reason == VNIC_RESET_MOBILITY) {
1976 			rc = ibmvnic_reenable_crq_queue(adapter);
1977 			release_sub_crqs(adapter, 1);
1978 		} else {
1979 			rc = ibmvnic_reset_crq(adapter);
1980 			if (rc == H_CLOSED || rc == H_SUCCESS) {
1981 				rc = vio_enable_interrupts(adapter->vdev);
1982 				if (rc)
1983 					netdev_err(adapter->netdev,
1984 						   "Reset failed to enable interrupts. rc=%d\n",
1985 						   rc);
1986 			}
1987 		}
1988 
1989 		if (rc) {
1990 			netdev_err(adapter->netdev,
1991 				   "Reset couldn't initialize crq. rc=%d\n", rc);
1992 			goto out;
1993 		}
1994 
1995 		rc = ibmvnic_reset_init(adapter);
1996 		if (rc) {
1997 			rc = IBMVNIC_INIT_FAILED;
1998 			goto out;
1999 		}
2000 
2001 		/* If the adapter was in PROBE state prior to the reset,
2002 		 * exit here.
2003 		 */
2004 		if (reset_state == VNIC_PROBED) {
2005 			rc = 0;
2006 			goto out;
2007 		}
2008 
2009 		rc = ibmvnic_login(netdev);
2010 		if (rc) {
2011 			adapter->state = reset_state;
2012 			goto out;
2013 		}
2014 
2015 		if (adapter->req_rx_queues != old_num_rx_queues ||
2016 		    adapter->req_tx_queues != old_num_tx_queues ||
2017 		    adapter->req_rx_add_entries_per_subcrq !=
2018 		    old_num_rx_slots ||
2019 		    adapter->req_tx_entries_per_subcrq !=
2020 		    old_num_tx_slots ||
2021 		    !adapter->rx_pool ||
2022 		    !adapter->tso_pool ||
2023 		    !adapter->tx_pool) {
2024 			release_rx_pools(adapter);
2025 			release_tx_pools(adapter);
2026 			release_napi(adapter);
2027 			release_vpd_data(adapter);
2028 
2029 			rc = init_resources(adapter);
2030 			if (rc)
2031 				goto out;
2032 
2033 		} else {
2034 			rc = reset_tx_pools(adapter);
2035 			if (rc)
2036 				netdev_dbg(adapter->netdev, "reset tx pools failed (%d)\n",
2037 						rc);
2038 				goto out;
2039 
2040 			rc = reset_rx_pools(adapter);
2041 			if (rc)
2042 				netdev_dbg(adapter->netdev, "reset rx pools failed (%d)\n",
2043 						rc);
2044 				goto out;
2045 		}
2046 		ibmvnic_disable_irqs(adapter);
2047 	}
2048 	adapter->state = VNIC_CLOSED;
2049 
2050 	if (reset_state == VNIC_CLOSED) {
2051 		rc = 0;
2052 		goto out;
2053 	}
2054 
2055 	rc = __ibmvnic_open(netdev);
2056 	if (rc) {
2057 		rc = IBMVNIC_OPEN_FAILED;
2058 		goto out;
2059 	}
2060 
2061 	/* refresh device's multicast list */
2062 	ibmvnic_set_multi(netdev);
2063 
2064 	/* kick napi */
2065 	for (i = 0; i < adapter->req_rx_queues; i++)
2066 		napi_schedule(&adapter->napi[i]);
2067 
2068 	if (adapter->reset_reason != VNIC_RESET_FAILOVER)
2069 		call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, netdev);
2070 
2071 	rc = 0;
2072 
2073 out:
2074 	rtnl_unlock();
2075 
2076 	return rc;
2077 }
2078 
2079 static int do_hard_reset(struct ibmvnic_adapter *adapter,
2080 			 struct ibmvnic_rwi *rwi, u32 reset_state)
2081 {
2082 	struct net_device *netdev = adapter->netdev;
2083 	int rc;
2084 
2085 	netdev_dbg(adapter->netdev, "Hard resetting driver (%d)\n",
2086 		   rwi->reset_reason);
2087 
2088 	netif_carrier_off(netdev);
2089 	adapter->reset_reason = rwi->reset_reason;
2090 
2091 	ibmvnic_cleanup(netdev);
2092 	release_resources(adapter);
2093 	release_sub_crqs(adapter, 0);
2094 	release_crq_queue(adapter);
2095 
2096 	/* remove the closed state so when we call open it appears
2097 	 * we are coming from the probed state.
2098 	 */
2099 	adapter->state = VNIC_PROBED;
2100 
2101 	reinit_completion(&adapter->init_done);
2102 	rc = init_crq_queue(adapter);
2103 	if (rc) {
2104 		netdev_err(adapter->netdev,
2105 			   "Couldn't initialize crq. rc=%d\n", rc);
2106 		return rc;
2107 	}
2108 
2109 	rc = ibmvnic_init(adapter);
2110 	if (rc)
2111 		return rc;
2112 
2113 	/* If the adapter was in PROBE state prior to the reset,
2114 	 * exit here.
2115 	 */
2116 	if (reset_state == VNIC_PROBED)
2117 		return 0;
2118 
2119 	rc = ibmvnic_login(netdev);
2120 	if (rc) {
2121 		adapter->state = VNIC_PROBED;
2122 		return 0;
2123 	}
2124 
2125 	rc = init_resources(adapter);
2126 	if (rc)
2127 		return rc;
2128 
2129 	ibmvnic_disable_irqs(adapter);
2130 	adapter->state = VNIC_CLOSED;
2131 
2132 	if (reset_state == VNIC_CLOSED)
2133 		return 0;
2134 
2135 	rc = __ibmvnic_open(netdev);
2136 	if (rc)
2137 		return IBMVNIC_OPEN_FAILED;
2138 
2139 	return 0;
2140 }
2141 
2142 static struct ibmvnic_rwi *get_next_rwi(struct ibmvnic_adapter *adapter)
2143 {
2144 	struct ibmvnic_rwi *rwi;
2145 	unsigned long flags;
2146 
2147 	spin_lock_irqsave(&adapter->rwi_lock, flags);
2148 
2149 	if (!list_empty(&adapter->rwi_list)) {
2150 		rwi = list_first_entry(&adapter->rwi_list, struct ibmvnic_rwi,
2151 				       list);
2152 		list_del(&rwi->list);
2153 	} else {
2154 		rwi = NULL;
2155 	}
2156 
2157 	spin_unlock_irqrestore(&adapter->rwi_lock, flags);
2158 	return rwi;
2159 }
2160 
2161 static void free_all_rwi(struct ibmvnic_adapter *adapter)
2162 {
2163 	struct ibmvnic_rwi *rwi;
2164 
2165 	rwi = get_next_rwi(adapter);
2166 	while (rwi) {
2167 		kfree(rwi);
2168 		rwi = get_next_rwi(adapter);
2169 	}
2170 }
2171 
2172 static void __ibmvnic_reset(struct work_struct *work)
2173 {
2174 	struct ibmvnic_rwi *rwi;
2175 	struct ibmvnic_adapter *adapter;
2176 	bool saved_state = false;
2177 	unsigned long flags;
2178 	u32 reset_state;
2179 	int rc = 0;
2180 
2181 	adapter = container_of(work, struct ibmvnic_adapter, ibmvnic_reset);
2182 
2183 	if (test_and_set_bit_lock(0, &adapter->resetting)) {
2184 		schedule_delayed_work(&adapter->ibmvnic_delayed_reset,
2185 				      IBMVNIC_RESET_DELAY);
2186 		return;
2187 	}
2188 
2189 	rwi = get_next_rwi(adapter);
2190 	while (rwi) {
2191 		spin_lock_irqsave(&adapter->state_lock, flags);
2192 
2193 		if (adapter->state == VNIC_REMOVING ||
2194 		    adapter->state == VNIC_REMOVED) {
2195 			spin_unlock_irqrestore(&adapter->state_lock, flags);
2196 			kfree(rwi);
2197 			rc = EBUSY;
2198 			break;
2199 		}
2200 
2201 		if (!saved_state) {
2202 			reset_state = adapter->state;
2203 			adapter->state = VNIC_RESETTING;
2204 			saved_state = true;
2205 		}
2206 		spin_unlock_irqrestore(&adapter->state_lock, flags);
2207 
2208 		if (rwi->reset_reason == VNIC_RESET_CHANGE_PARAM) {
2209 			/* CHANGE_PARAM requestor holds rtnl_lock */
2210 			rc = do_change_param_reset(adapter, rwi, reset_state);
2211 		} else if (adapter->force_reset_recovery) {
2212 			/* Transport event occurred during previous reset */
2213 			if (adapter->wait_for_reset) {
2214 				/* Previous was CHANGE_PARAM; caller locked */
2215 				adapter->force_reset_recovery = false;
2216 				rc = do_hard_reset(adapter, rwi, reset_state);
2217 			} else {
2218 				rtnl_lock();
2219 				adapter->force_reset_recovery = false;
2220 				rc = do_hard_reset(adapter, rwi, reset_state);
2221 				rtnl_unlock();
2222 			}
2223 		} else if (!(rwi->reset_reason == VNIC_RESET_FATAL &&
2224 				adapter->from_passive_init)) {
2225 			rc = do_reset(adapter, rwi, reset_state);
2226 		}
2227 		kfree(rwi);
2228 		if (rc == IBMVNIC_OPEN_FAILED) {
2229 			if (list_empty(&adapter->rwi_list))
2230 				adapter->state = VNIC_CLOSED;
2231 			else
2232 				adapter->state = reset_state;
2233 			rc = 0;
2234 		} else if (rc && rc != IBMVNIC_INIT_FAILED &&
2235 		    !adapter->force_reset_recovery)
2236 			break;
2237 
2238 		rwi = get_next_rwi(adapter);
2239 
2240 		if (rwi && (rwi->reset_reason == VNIC_RESET_FAILOVER ||
2241 			    rwi->reset_reason == VNIC_RESET_MOBILITY))
2242 			adapter->force_reset_recovery = true;
2243 	}
2244 
2245 	if (adapter->wait_for_reset) {
2246 		adapter->reset_done_rc = rc;
2247 		complete(&adapter->reset_done);
2248 	}
2249 
2250 	if (rc) {
2251 		netdev_dbg(adapter->netdev, "Reset failed\n");
2252 		free_all_rwi(adapter);
2253 	}
2254 
2255 	clear_bit_unlock(0, &adapter->resetting);
2256 }
2257 
2258 static void __ibmvnic_delayed_reset(struct work_struct *work)
2259 {
2260 	struct ibmvnic_adapter *adapter;
2261 
2262 	adapter = container_of(work, struct ibmvnic_adapter,
2263 			       ibmvnic_delayed_reset.work);
2264 	__ibmvnic_reset(&adapter->ibmvnic_reset);
2265 }
2266 
2267 static int ibmvnic_reset(struct ibmvnic_adapter *adapter,
2268 			 enum ibmvnic_reset_reason reason)
2269 {
2270 	struct list_head *entry, *tmp_entry;
2271 	struct ibmvnic_rwi *rwi, *tmp;
2272 	struct net_device *netdev = adapter->netdev;
2273 	unsigned long flags;
2274 	int ret;
2275 
2276 	if (adapter->state == VNIC_REMOVING ||
2277 	    adapter->state == VNIC_REMOVED ||
2278 	    adapter->failover_pending) {
2279 		ret = EBUSY;
2280 		netdev_dbg(netdev, "Adapter removing or pending failover, skipping reset\n");
2281 		goto err;
2282 	}
2283 
2284 	if (adapter->state == VNIC_PROBING) {
2285 		netdev_warn(netdev, "Adapter reset during probe\n");
2286 		ret = adapter->init_done_rc = EAGAIN;
2287 		goto err;
2288 	}
2289 
2290 	spin_lock_irqsave(&adapter->rwi_lock, flags);
2291 
2292 	list_for_each(entry, &adapter->rwi_list) {
2293 		tmp = list_entry(entry, struct ibmvnic_rwi, list);
2294 		if (tmp->reset_reason == reason) {
2295 			netdev_dbg(netdev, "Skipping matching reset\n");
2296 			spin_unlock_irqrestore(&adapter->rwi_lock, flags);
2297 			ret = EBUSY;
2298 			goto err;
2299 		}
2300 	}
2301 
2302 	rwi = kzalloc(sizeof(*rwi), GFP_ATOMIC);
2303 	if (!rwi) {
2304 		spin_unlock_irqrestore(&adapter->rwi_lock, flags);
2305 		ibmvnic_close(netdev);
2306 		ret = ENOMEM;
2307 		goto err;
2308 	}
2309 	/* if we just received a transport event,
2310 	 * flush reset queue and process this reset
2311 	 */
2312 	if (adapter->force_reset_recovery && !list_empty(&adapter->rwi_list)) {
2313 		list_for_each_safe(entry, tmp_entry, &adapter->rwi_list)
2314 			list_del(entry);
2315 	}
2316 	rwi->reset_reason = reason;
2317 	list_add_tail(&rwi->list, &adapter->rwi_list);
2318 	spin_unlock_irqrestore(&adapter->rwi_lock, flags);
2319 	netdev_dbg(adapter->netdev, "Scheduling reset (reason %d)\n", reason);
2320 	schedule_work(&adapter->ibmvnic_reset);
2321 
2322 	return 0;
2323 err:
2324 	return -ret;
2325 }
2326 
2327 static void ibmvnic_tx_timeout(struct net_device *dev, unsigned int txqueue)
2328 {
2329 	struct ibmvnic_adapter *adapter = netdev_priv(dev);
2330 
2331 	ibmvnic_reset(adapter, VNIC_RESET_TIMEOUT);
2332 }
2333 
2334 static void remove_buff_from_pool(struct ibmvnic_adapter *adapter,
2335 				  struct ibmvnic_rx_buff *rx_buff)
2336 {
2337 	struct ibmvnic_rx_pool *pool = &adapter->rx_pool[rx_buff->pool_index];
2338 
2339 	rx_buff->skb = NULL;
2340 
2341 	pool->free_map[pool->next_alloc] = (int)(rx_buff - pool->rx_buff);
2342 	pool->next_alloc = (pool->next_alloc + 1) % pool->size;
2343 
2344 	atomic_dec(&pool->available);
2345 }
2346 
2347 static int ibmvnic_poll(struct napi_struct *napi, int budget)
2348 {
2349 	struct net_device *netdev = napi->dev;
2350 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2351 	int scrq_num = (int)(napi - adapter->napi);
2352 	int frames_processed = 0;
2353 
2354 restart_poll:
2355 	while (frames_processed < budget) {
2356 		struct sk_buff *skb;
2357 		struct ibmvnic_rx_buff *rx_buff;
2358 		union sub_crq *next;
2359 		u32 length;
2360 		u16 offset;
2361 		u8 flags = 0;
2362 
2363 		if (unlikely(test_bit(0, &adapter->resetting) &&
2364 			     adapter->reset_reason != VNIC_RESET_NON_FATAL)) {
2365 			enable_scrq_irq(adapter, adapter->rx_scrq[scrq_num]);
2366 			napi_complete_done(napi, frames_processed);
2367 			return frames_processed;
2368 		}
2369 
2370 		if (!pending_scrq(adapter, adapter->rx_scrq[scrq_num]))
2371 			break;
2372 		next = ibmvnic_next_scrq(adapter, adapter->rx_scrq[scrq_num]);
2373 		rx_buff =
2374 		    (struct ibmvnic_rx_buff *)be64_to_cpu(next->
2375 							  rx_comp.correlator);
2376 		/* do error checking */
2377 		if (next->rx_comp.rc) {
2378 			netdev_dbg(netdev, "rx buffer returned with rc %x\n",
2379 				   be16_to_cpu(next->rx_comp.rc));
2380 			/* free the entry */
2381 			next->rx_comp.first = 0;
2382 			dev_kfree_skb_any(rx_buff->skb);
2383 			remove_buff_from_pool(adapter, rx_buff);
2384 			continue;
2385 		} else if (!rx_buff->skb) {
2386 			/* free the entry */
2387 			next->rx_comp.first = 0;
2388 			remove_buff_from_pool(adapter, rx_buff);
2389 			continue;
2390 		}
2391 
2392 		length = be32_to_cpu(next->rx_comp.len);
2393 		offset = be16_to_cpu(next->rx_comp.off_frame_data);
2394 		flags = next->rx_comp.flags;
2395 		skb = rx_buff->skb;
2396 		skb_copy_to_linear_data(skb, rx_buff->data + offset,
2397 					length);
2398 
2399 		/* VLAN Header has been stripped by the system firmware and
2400 		 * needs to be inserted by the driver
2401 		 */
2402 		if (adapter->rx_vlan_header_insertion &&
2403 		    (flags & IBMVNIC_VLAN_STRIPPED))
2404 			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
2405 					       ntohs(next->rx_comp.vlan_tci));
2406 
2407 		/* free the entry */
2408 		next->rx_comp.first = 0;
2409 		remove_buff_from_pool(adapter, rx_buff);
2410 
2411 		skb_put(skb, length);
2412 		skb->protocol = eth_type_trans(skb, netdev);
2413 		skb_record_rx_queue(skb, scrq_num);
2414 
2415 		if (flags & IBMVNIC_IP_CHKSUM_GOOD &&
2416 		    flags & IBMVNIC_TCP_UDP_CHKSUM_GOOD) {
2417 			skb->ip_summed = CHECKSUM_UNNECESSARY;
2418 		}
2419 
2420 		length = skb->len;
2421 		napi_gro_receive(napi, skb); /* send it up */
2422 		netdev->stats.rx_packets++;
2423 		netdev->stats.rx_bytes += length;
2424 		adapter->rx_stats_buffers[scrq_num].packets++;
2425 		adapter->rx_stats_buffers[scrq_num].bytes += length;
2426 		frames_processed++;
2427 	}
2428 
2429 	if (adapter->state != VNIC_CLOSING)
2430 		replenish_rx_pool(adapter, &adapter->rx_pool[scrq_num]);
2431 
2432 	if (frames_processed < budget) {
2433 		enable_scrq_irq(adapter, adapter->rx_scrq[scrq_num]);
2434 		napi_complete_done(napi, frames_processed);
2435 		if (pending_scrq(adapter, adapter->rx_scrq[scrq_num]) &&
2436 		    napi_reschedule(napi)) {
2437 			disable_scrq_irq(adapter, adapter->rx_scrq[scrq_num]);
2438 			goto restart_poll;
2439 		}
2440 	}
2441 	return frames_processed;
2442 }
2443 
2444 static int wait_for_reset(struct ibmvnic_adapter *adapter)
2445 {
2446 	int rc, ret;
2447 
2448 	adapter->fallback.mtu = adapter->req_mtu;
2449 	adapter->fallback.rx_queues = adapter->req_rx_queues;
2450 	adapter->fallback.tx_queues = adapter->req_tx_queues;
2451 	adapter->fallback.rx_entries = adapter->req_rx_add_entries_per_subcrq;
2452 	adapter->fallback.tx_entries = adapter->req_tx_entries_per_subcrq;
2453 
2454 	reinit_completion(&adapter->reset_done);
2455 	adapter->wait_for_reset = true;
2456 	rc = ibmvnic_reset(adapter, VNIC_RESET_CHANGE_PARAM);
2457 
2458 	if (rc) {
2459 		ret = rc;
2460 		goto out;
2461 	}
2462 	rc = ibmvnic_wait_for_completion(adapter, &adapter->reset_done, 60000);
2463 	if (rc) {
2464 		ret = -ENODEV;
2465 		goto out;
2466 	}
2467 
2468 	ret = 0;
2469 	if (adapter->reset_done_rc) {
2470 		ret = -EIO;
2471 		adapter->desired.mtu = adapter->fallback.mtu;
2472 		adapter->desired.rx_queues = adapter->fallback.rx_queues;
2473 		adapter->desired.tx_queues = adapter->fallback.tx_queues;
2474 		adapter->desired.rx_entries = adapter->fallback.rx_entries;
2475 		adapter->desired.tx_entries = adapter->fallback.tx_entries;
2476 
2477 		reinit_completion(&adapter->reset_done);
2478 		adapter->wait_for_reset = true;
2479 		rc = ibmvnic_reset(adapter, VNIC_RESET_CHANGE_PARAM);
2480 		if (rc) {
2481 			ret = rc;
2482 			goto out;
2483 		}
2484 		rc = ibmvnic_wait_for_completion(adapter, &adapter->reset_done,
2485 						 60000);
2486 		if (rc) {
2487 			ret = -ENODEV;
2488 			goto out;
2489 		}
2490 	}
2491 out:
2492 	adapter->wait_for_reset = false;
2493 
2494 	return ret;
2495 }
2496 
2497 static int ibmvnic_change_mtu(struct net_device *netdev, int new_mtu)
2498 {
2499 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2500 
2501 	adapter->desired.mtu = new_mtu + ETH_HLEN;
2502 
2503 	return wait_for_reset(adapter);
2504 }
2505 
2506 static netdev_features_t ibmvnic_features_check(struct sk_buff *skb,
2507 						struct net_device *dev,
2508 						netdev_features_t features)
2509 {
2510 	/* Some backing hardware adapters can not
2511 	 * handle packets with a MSS less than 224
2512 	 * or with only one segment.
2513 	 */
2514 	if (skb_is_gso(skb)) {
2515 		if (skb_shinfo(skb)->gso_size < 224 ||
2516 		    skb_shinfo(skb)->gso_segs == 1)
2517 			features &= ~NETIF_F_GSO_MASK;
2518 	}
2519 
2520 	return features;
2521 }
2522 
2523 static const struct net_device_ops ibmvnic_netdev_ops = {
2524 	.ndo_open		= ibmvnic_open,
2525 	.ndo_stop		= ibmvnic_close,
2526 	.ndo_start_xmit		= ibmvnic_xmit,
2527 	.ndo_set_rx_mode	= ibmvnic_set_multi,
2528 	.ndo_set_mac_address	= ibmvnic_set_mac,
2529 	.ndo_validate_addr	= eth_validate_addr,
2530 	.ndo_tx_timeout		= ibmvnic_tx_timeout,
2531 	.ndo_change_mtu		= ibmvnic_change_mtu,
2532 	.ndo_features_check     = ibmvnic_features_check,
2533 };
2534 
2535 /* ethtool functions */
2536 
2537 static int ibmvnic_get_link_ksettings(struct net_device *netdev,
2538 				      struct ethtool_link_ksettings *cmd)
2539 {
2540 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2541 	int rc;
2542 
2543 	rc = send_query_phys_parms(adapter);
2544 	if (rc) {
2545 		adapter->speed = SPEED_UNKNOWN;
2546 		adapter->duplex = DUPLEX_UNKNOWN;
2547 	}
2548 	cmd->base.speed = adapter->speed;
2549 	cmd->base.duplex = adapter->duplex;
2550 	cmd->base.port = PORT_FIBRE;
2551 	cmd->base.phy_address = 0;
2552 	cmd->base.autoneg = AUTONEG_ENABLE;
2553 
2554 	return 0;
2555 }
2556 
2557 static void ibmvnic_get_drvinfo(struct net_device *netdev,
2558 				struct ethtool_drvinfo *info)
2559 {
2560 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2561 
2562 	strlcpy(info->driver, ibmvnic_driver_name, sizeof(info->driver));
2563 	strlcpy(info->version, IBMVNIC_DRIVER_VERSION, sizeof(info->version));
2564 	strlcpy(info->fw_version, adapter->fw_version,
2565 		sizeof(info->fw_version));
2566 }
2567 
2568 static u32 ibmvnic_get_msglevel(struct net_device *netdev)
2569 {
2570 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2571 
2572 	return adapter->msg_enable;
2573 }
2574 
2575 static void ibmvnic_set_msglevel(struct net_device *netdev, u32 data)
2576 {
2577 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2578 
2579 	adapter->msg_enable = data;
2580 }
2581 
2582 static u32 ibmvnic_get_link(struct net_device *netdev)
2583 {
2584 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2585 
2586 	/* Don't need to send a query because we request a logical link up at
2587 	 * init and then we wait for link state indications
2588 	 */
2589 	return adapter->logical_link_state;
2590 }
2591 
2592 static void ibmvnic_get_ringparam(struct net_device *netdev,
2593 				  struct ethtool_ringparam *ring)
2594 {
2595 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2596 
2597 	if (adapter->priv_flags & IBMVNIC_USE_SERVER_MAXES) {
2598 		ring->rx_max_pending = adapter->max_rx_add_entries_per_subcrq;
2599 		ring->tx_max_pending = adapter->max_tx_entries_per_subcrq;
2600 	} else {
2601 		ring->rx_max_pending = IBMVNIC_MAX_QUEUE_SZ;
2602 		ring->tx_max_pending = IBMVNIC_MAX_QUEUE_SZ;
2603 	}
2604 	ring->rx_mini_max_pending = 0;
2605 	ring->rx_jumbo_max_pending = 0;
2606 	ring->rx_pending = adapter->req_rx_add_entries_per_subcrq;
2607 	ring->tx_pending = adapter->req_tx_entries_per_subcrq;
2608 	ring->rx_mini_pending = 0;
2609 	ring->rx_jumbo_pending = 0;
2610 }
2611 
2612 static int ibmvnic_set_ringparam(struct net_device *netdev,
2613 				 struct ethtool_ringparam *ring)
2614 {
2615 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2616 	int ret;
2617 
2618 	ret = 0;
2619 	adapter->desired.rx_entries = ring->rx_pending;
2620 	adapter->desired.tx_entries = ring->tx_pending;
2621 
2622 	ret = wait_for_reset(adapter);
2623 
2624 	if (!ret &&
2625 	    (adapter->req_rx_add_entries_per_subcrq != ring->rx_pending ||
2626 	     adapter->req_tx_entries_per_subcrq != ring->tx_pending))
2627 		netdev_info(netdev,
2628 			    "Could not match full ringsize request. Requested: RX %d, TX %d; Allowed: RX %llu, TX %llu\n",
2629 			    ring->rx_pending, ring->tx_pending,
2630 			    adapter->req_rx_add_entries_per_subcrq,
2631 			    adapter->req_tx_entries_per_subcrq);
2632 	return ret;
2633 }
2634 
2635 static void ibmvnic_get_channels(struct net_device *netdev,
2636 				 struct ethtool_channels *channels)
2637 {
2638 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2639 
2640 	if (adapter->priv_flags & IBMVNIC_USE_SERVER_MAXES) {
2641 		channels->max_rx = adapter->max_rx_queues;
2642 		channels->max_tx = adapter->max_tx_queues;
2643 	} else {
2644 		channels->max_rx = IBMVNIC_MAX_QUEUES;
2645 		channels->max_tx = IBMVNIC_MAX_QUEUES;
2646 	}
2647 
2648 	channels->max_other = 0;
2649 	channels->max_combined = 0;
2650 	channels->rx_count = adapter->req_rx_queues;
2651 	channels->tx_count = adapter->req_tx_queues;
2652 	channels->other_count = 0;
2653 	channels->combined_count = 0;
2654 }
2655 
2656 static int ibmvnic_set_channels(struct net_device *netdev,
2657 				struct ethtool_channels *channels)
2658 {
2659 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2660 	int ret;
2661 
2662 	ret = 0;
2663 	adapter->desired.rx_queues = channels->rx_count;
2664 	adapter->desired.tx_queues = channels->tx_count;
2665 
2666 	ret = wait_for_reset(adapter);
2667 
2668 	if (!ret &&
2669 	    (adapter->req_rx_queues != channels->rx_count ||
2670 	     adapter->req_tx_queues != channels->tx_count))
2671 		netdev_info(netdev,
2672 			    "Could not match full channels request. Requested: RX %d, TX %d; Allowed: RX %llu, TX %llu\n",
2673 			    channels->rx_count, channels->tx_count,
2674 			    adapter->req_rx_queues, adapter->req_tx_queues);
2675 	return ret;
2676 
2677 }
2678 
2679 static void ibmvnic_get_strings(struct net_device *dev, u32 stringset, u8 *data)
2680 {
2681 	struct ibmvnic_adapter *adapter = netdev_priv(dev);
2682 	int i;
2683 
2684 	switch (stringset) {
2685 	case ETH_SS_STATS:
2686 		for (i = 0; i < ARRAY_SIZE(ibmvnic_stats);
2687 				i++, data += ETH_GSTRING_LEN)
2688 			memcpy(data, ibmvnic_stats[i].name, ETH_GSTRING_LEN);
2689 
2690 		for (i = 0; i < adapter->req_tx_queues; i++) {
2691 			snprintf(data, ETH_GSTRING_LEN, "tx%d_packets", i);
2692 			data += ETH_GSTRING_LEN;
2693 
2694 			snprintf(data, ETH_GSTRING_LEN, "tx%d_bytes", i);
2695 			data += ETH_GSTRING_LEN;
2696 
2697 			snprintf(data, ETH_GSTRING_LEN,
2698 				 "tx%d_dropped_packets", i);
2699 			data += ETH_GSTRING_LEN;
2700 		}
2701 
2702 		for (i = 0; i < adapter->req_rx_queues; i++) {
2703 			snprintf(data, ETH_GSTRING_LEN, "rx%d_packets", i);
2704 			data += ETH_GSTRING_LEN;
2705 
2706 			snprintf(data, ETH_GSTRING_LEN, "rx%d_bytes", i);
2707 			data += ETH_GSTRING_LEN;
2708 
2709 			snprintf(data, ETH_GSTRING_LEN, "rx%d_interrupts", i);
2710 			data += ETH_GSTRING_LEN;
2711 		}
2712 		break;
2713 
2714 	case ETH_SS_PRIV_FLAGS:
2715 		for (i = 0; i < ARRAY_SIZE(ibmvnic_priv_flags); i++)
2716 			strcpy(data + i * ETH_GSTRING_LEN,
2717 			       ibmvnic_priv_flags[i]);
2718 		break;
2719 	default:
2720 		return;
2721 	}
2722 }
2723 
2724 static int ibmvnic_get_sset_count(struct net_device *dev, int sset)
2725 {
2726 	struct ibmvnic_adapter *adapter = netdev_priv(dev);
2727 
2728 	switch (sset) {
2729 	case ETH_SS_STATS:
2730 		return ARRAY_SIZE(ibmvnic_stats) +
2731 		       adapter->req_tx_queues * NUM_TX_STATS +
2732 		       adapter->req_rx_queues * NUM_RX_STATS;
2733 	case ETH_SS_PRIV_FLAGS:
2734 		return ARRAY_SIZE(ibmvnic_priv_flags);
2735 	default:
2736 		return -EOPNOTSUPP;
2737 	}
2738 }
2739 
2740 static void ibmvnic_get_ethtool_stats(struct net_device *dev,
2741 				      struct ethtool_stats *stats, u64 *data)
2742 {
2743 	struct ibmvnic_adapter *adapter = netdev_priv(dev);
2744 	union ibmvnic_crq crq;
2745 	int i, j;
2746 	int rc;
2747 
2748 	memset(&crq, 0, sizeof(crq));
2749 	crq.request_statistics.first = IBMVNIC_CRQ_CMD;
2750 	crq.request_statistics.cmd = REQUEST_STATISTICS;
2751 	crq.request_statistics.ioba = cpu_to_be32(adapter->stats_token);
2752 	crq.request_statistics.len =
2753 	    cpu_to_be32(sizeof(struct ibmvnic_statistics));
2754 
2755 	/* Wait for data to be written */
2756 	reinit_completion(&adapter->stats_done);
2757 	rc = ibmvnic_send_crq(adapter, &crq);
2758 	if (rc)
2759 		return;
2760 	rc = ibmvnic_wait_for_completion(adapter, &adapter->stats_done, 10000);
2761 	if (rc)
2762 		return;
2763 
2764 	for (i = 0; i < ARRAY_SIZE(ibmvnic_stats); i++)
2765 		data[i] = be64_to_cpu(IBMVNIC_GET_STAT(adapter,
2766 						ibmvnic_stats[i].offset));
2767 
2768 	for (j = 0; j < adapter->req_tx_queues; j++) {
2769 		data[i] = adapter->tx_stats_buffers[j].packets;
2770 		i++;
2771 		data[i] = adapter->tx_stats_buffers[j].bytes;
2772 		i++;
2773 		data[i] = adapter->tx_stats_buffers[j].dropped_packets;
2774 		i++;
2775 	}
2776 
2777 	for (j = 0; j < adapter->req_rx_queues; j++) {
2778 		data[i] = adapter->rx_stats_buffers[j].packets;
2779 		i++;
2780 		data[i] = adapter->rx_stats_buffers[j].bytes;
2781 		i++;
2782 		data[i] = adapter->rx_stats_buffers[j].interrupts;
2783 		i++;
2784 	}
2785 }
2786 
2787 static u32 ibmvnic_get_priv_flags(struct net_device *netdev)
2788 {
2789 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2790 
2791 	return adapter->priv_flags;
2792 }
2793 
2794 static int ibmvnic_set_priv_flags(struct net_device *netdev, u32 flags)
2795 {
2796 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2797 	bool which_maxes = !!(flags & IBMVNIC_USE_SERVER_MAXES);
2798 
2799 	if (which_maxes)
2800 		adapter->priv_flags |= IBMVNIC_USE_SERVER_MAXES;
2801 	else
2802 		adapter->priv_flags &= ~IBMVNIC_USE_SERVER_MAXES;
2803 
2804 	return 0;
2805 }
2806 static const struct ethtool_ops ibmvnic_ethtool_ops = {
2807 	.get_drvinfo		= ibmvnic_get_drvinfo,
2808 	.get_msglevel		= ibmvnic_get_msglevel,
2809 	.set_msglevel		= ibmvnic_set_msglevel,
2810 	.get_link		= ibmvnic_get_link,
2811 	.get_ringparam		= ibmvnic_get_ringparam,
2812 	.set_ringparam		= ibmvnic_set_ringparam,
2813 	.get_channels		= ibmvnic_get_channels,
2814 	.set_channels		= ibmvnic_set_channels,
2815 	.get_strings            = ibmvnic_get_strings,
2816 	.get_sset_count         = ibmvnic_get_sset_count,
2817 	.get_ethtool_stats	= ibmvnic_get_ethtool_stats,
2818 	.get_link_ksettings	= ibmvnic_get_link_ksettings,
2819 	.get_priv_flags		= ibmvnic_get_priv_flags,
2820 	.set_priv_flags		= ibmvnic_set_priv_flags,
2821 };
2822 
2823 /* Routines for managing CRQs/sCRQs  */
2824 
2825 static int reset_one_sub_crq_queue(struct ibmvnic_adapter *adapter,
2826 				   struct ibmvnic_sub_crq_queue *scrq)
2827 {
2828 	int rc;
2829 
2830 	if (scrq->irq) {
2831 		free_irq(scrq->irq, scrq);
2832 		irq_dispose_mapping(scrq->irq);
2833 		scrq->irq = 0;
2834 	}
2835 
2836 	memset(scrq->msgs, 0, 4 * PAGE_SIZE);
2837 	atomic_set(&scrq->used, 0);
2838 	scrq->cur = 0;
2839 
2840 	rc = h_reg_sub_crq(adapter->vdev->unit_address, scrq->msg_token,
2841 			   4 * PAGE_SIZE, &scrq->crq_num, &scrq->hw_irq);
2842 	return rc;
2843 }
2844 
2845 static int reset_sub_crq_queues(struct ibmvnic_adapter *adapter)
2846 {
2847 	int i, rc;
2848 
2849 	for (i = 0; i < adapter->req_tx_queues; i++) {
2850 		netdev_dbg(adapter->netdev, "Re-setting tx_scrq[%d]\n", i);
2851 		rc = reset_one_sub_crq_queue(adapter, adapter->tx_scrq[i]);
2852 		if (rc)
2853 			return rc;
2854 	}
2855 
2856 	for (i = 0; i < adapter->req_rx_queues; i++) {
2857 		netdev_dbg(adapter->netdev, "Re-setting rx_scrq[%d]\n", i);
2858 		rc = reset_one_sub_crq_queue(adapter, adapter->rx_scrq[i]);
2859 		if (rc)
2860 			return rc;
2861 	}
2862 
2863 	return rc;
2864 }
2865 
2866 static void release_sub_crq_queue(struct ibmvnic_adapter *adapter,
2867 				  struct ibmvnic_sub_crq_queue *scrq,
2868 				  bool do_h_free)
2869 {
2870 	struct device *dev = &adapter->vdev->dev;
2871 	long rc;
2872 
2873 	netdev_dbg(adapter->netdev, "Releasing sub-CRQ\n");
2874 
2875 	if (do_h_free) {
2876 		/* Close the sub-crqs */
2877 		do {
2878 			rc = plpar_hcall_norets(H_FREE_SUB_CRQ,
2879 						adapter->vdev->unit_address,
2880 						scrq->crq_num);
2881 		} while (rc == H_BUSY || H_IS_LONG_BUSY(rc));
2882 
2883 		if (rc) {
2884 			netdev_err(adapter->netdev,
2885 				   "Failed to release sub-CRQ %16lx, rc = %ld\n",
2886 				   scrq->crq_num, rc);
2887 		}
2888 	}
2889 
2890 	dma_unmap_single(dev, scrq->msg_token, 4 * PAGE_SIZE,
2891 			 DMA_BIDIRECTIONAL);
2892 	free_pages((unsigned long)scrq->msgs, 2);
2893 	kfree(scrq);
2894 }
2895 
2896 static struct ibmvnic_sub_crq_queue *init_sub_crq_queue(struct ibmvnic_adapter
2897 							*adapter)
2898 {
2899 	struct device *dev = &adapter->vdev->dev;
2900 	struct ibmvnic_sub_crq_queue *scrq;
2901 	int rc;
2902 
2903 	scrq = kzalloc(sizeof(*scrq), GFP_KERNEL);
2904 	if (!scrq)
2905 		return NULL;
2906 
2907 	scrq->msgs =
2908 		(union sub_crq *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 2);
2909 	if (!scrq->msgs) {
2910 		dev_warn(dev, "Couldn't allocate crq queue messages page\n");
2911 		goto zero_page_failed;
2912 	}
2913 
2914 	scrq->msg_token = dma_map_single(dev, scrq->msgs, 4 * PAGE_SIZE,
2915 					 DMA_BIDIRECTIONAL);
2916 	if (dma_mapping_error(dev, scrq->msg_token)) {
2917 		dev_warn(dev, "Couldn't map crq queue messages page\n");
2918 		goto map_failed;
2919 	}
2920 
2921 	rc = h_reg_sub_crq(adapter->vdev->unit_address, scrq->msg_token,
2922 			   4 * PAGE_SIZE, &scrq->crq_num, &scrq->hw_irq);
2923 
2924 	if (rc == H_RESOURCE)
2925 		rc = ibmvnic_reset_crq(adapter);
2926 
2927 	if (rc == H_CLOSED) {
2928 		dev_warn(dev, "Partner adapter not ready, waiting.\n");
2929 	} else if (rc) {
2930 		dev_warn(dev, "Error %d registering sub-crq\n", rc);
2931 		goto reg_failed;
2932 	}
2933 
2934 	scrq->adapter = adapter;
2935 	scrq->size = 4 * PAGE_SIZE / sizeof(*scrq->msgs);
2936 	spin_lock_init(&scrq->lock);
2937 
2938 	netdev_dbg(adapter->netdev,
2939 		   "sub-crq initialized, num %lx, hw_irq=%lx, irq=%x\n",
2940 		   scrq->crq_num, scrq->hw_irq, scrq->irq);
2941 
2942 	return scrq;
2943 
2944 reg_failed:
2945 	dma_unmap_single(dev, scrq->msg_token, 4 * PAGE_SIZE,
2946 			 DMA_BIDIRECTIONAL);
2947 map_failed:
2948 	free_pages((unsigned long)scrq->msgs, 2);
2949 zero_page_failed:
2950 	kfree(scrq);
2951 
2952 	return NULL;
2953 }
2954 
2955 static void release_sub_crqs(struct ibmvnic_adapter *adapter, bool do_h_free)
2956 {
2957 	int i;
2958 
2959 	if (adapter->tx_scrq) {
2960 		for (i = 0; i < adapter->num_active_tx_scrqs; i++) {
2961 			if (!adapter->tx_scrq[i])
2962 				continue;
2963 
2964 			netdev_dbg(adapter->netdev, "Releasing tx_scrq[%d]\n",
2965 				   i);
2966 			if (adapter->tx_scrq[i]->irq) {
2967 				free_irq(adapter->tx_scrq[i]->irq,
2968 					 adapter->tx_scrq[i]);
2969 				irq_dispose_mapping(adapter->tx_scrq[i]->irq);
2970 				adapter->tx_scrq[i]->irq = 0;
2971 			}
2972 
2973 			release_sub_crq_queue(adapter, adapter->tx_scrq[i],
2974 					      do_h_free);
2975 		}
2976 
2977 		kfree(adapter->tx_scrq);
2978 		adapter->tx_scrq = NULL;
2979 		adapter->num_active_tx_scrqs = 0;
2980 	}
2981 
2982 	if (adapter->rx_scrq) {
2983 		for (i = 0; i < adapter->num_active_rx_scrqs; i++) {
2984 			if (!adapter->rx_scrq[i])
2985 				continue;
2986 
2987 			netdev_dbg(adapter->netdev, "Releasing rx_scrq[%d]\n",
2988 				   i);
2989 			if (adapter->rx_scrq[i]->irq) {
2990 				free_irq(adapter->rx_scrq[i]->irq,
2991 					 adapter->rx_scrq[i]);
2992 				irq_dispose_mapping(adapter->rx_scrq[i]->irq);
2993 				adapter->rx_scrq[i]->irq = 0;
2994 			}
2995 
2996 			release_sub_crq_queue(adapter, adapter->rx_scrq[i],
2997 					      do_h_free);
2998 		}
2999 
3000 		kfree(adapter->rx_scrq);
3001 		adapter->rx_scrq = NULL;
3002 		adapter->num_active_rx_scrqs = 0;
3003 	}
3004 }
3005 
3006 static int disable_scrq_irq(struct ibmvnic_adapter *adapter,
3007 			    struct ibmvnic_sub_crq_queue *scrq)
3008 {
3009 	struct device *dev = &adapter->vdev->dev;
3010 	unsigned long rc;
3011 
3012 	rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,
3013 				H_DISABLE_VIO_INTERRUPT, scrq->hw_irq, 0, 0);
3014 	if (rc)
3015 		dev_err(dev, "Couldn't disable scrq irq 0x%lx. rc=%ld\n",
3016 			scrq->hw_irq, rc);
3017 	return rc;
3018 }
3019 
3020 static int enable_scrq_irq(struct ibmvnic_adapter *adapter,
3021 			   struct ibmvnic_sub_crq_queue *scrq)
3022 {
3023 	struct device *dev = &adapter->vdev->dev;
3024 	unsigned long rc;
3025 
3026 	if (scrq->hw_irq > 0x100000000ULL) {
3027 		dev_err(dev, "bad hw_irq = %lx\n", scrq->hw_irq);
3028 		return 1;
3029 	}
3030 
3031 	if (test_bit(0, &adapter->resetting) &&
3032 	    adapter->reset_reason == VNIC_RESET_MOBILITY) {
3033 		u64 val = (0xff000000) | scrq->hw_irq;
3034 
3035 		rc = plpar_hcall_norets(H_EOI, val);
3036 		/* H_EOI would fail with rc = H_FUNCTION when running
3037 		 * in XIVE mode which is expected, but not an error.
3038 		 */
3039 		if (rc && (rc != H_FUNCTION))
3040 			dev_err(dev, "H_EOI FAILED irq 0x%llx. rc=%ld\n",
3041 				val, rc);
3042 	}
3043 
3044 	rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,
3045 				H_ENABLE_VIO_INTERRUPT, scrq->hw_irq, 0, 0);
3046 	if (rc)
3047 		dev_err(dev, "Couldn't enable scrq irq 0x%lx. rc=%ld\n",
3048 			scrq->hw_irq, rc);
3049 	return rc;
3050 }
3051 
3052 static int ibmvnic_complete_tx(struct ibmvnic_adapter *adapter,
3053 			       struct ibmvnic_sub_crq_queue *scrq)
3054 {
3055 	struct device *dev = &adapter->vdev->dev;
3056 	struct ibmvnic_tx_pool *tx_pool;
3057 	struct ibmvnic_tx_buff *txbuff;
3058 	union sub_crq *next;
3059 	int index;
3060 	int i, j;
3061 
3062 restart_loop:
3063 	while (pending_scrq(adapter, scrq)) {
3064 		unsigned int pool = scrq->pool_index;
3065 		int num_entries = 0;
3066 
3067 		next = ibmvnic_next_scrq(adapter, scrq);
3068 		for (i = 0; i < next->tx_comp.num_comps; i++) {
3069 			if (next->tx_comp.rcs[i]) {
3070 				dev_err(dev, "tx error %x\n",
3071 					next->tx_comp.rcs[i]);
3072 				continue;
3073 			}
3074 			index = be32_to_cpu(next->tx_comp.correlators[i]);
3075 			if (index & IBMVNIC_TSO_POOL_MASK) {
3076 				tx_pool = &adapter->tso_pool[pool];
3077 				index &= ~IBMVNIC_TSO_POOL_MASK;
3078 			} else {
3079 				tx_pool = &adapter->tx_pool[pool];
3080 			}
3081 
3082 			txbuff = &tx_pool->tx_buff[index];
3083 
3084 			for (j = 0; j < IBMVNIC_MAX_FRAGS_PER_CRQ; j++) {
3085 				if (!txbuff->data_dma[j])
3086 					continue;
3087 
3088 				txbuff->data_dma[j] = 0;
3089 			}
3090 
3091 			if (txbuff->last_frag) {
3092 				dev_kfree_skb_any(txbuff->skb);
3093 				txbuff->skb = NULL;
3094 			}
3095 
3096 			num_entries += txbuff->num_entries;
3097 
3098 			tx_pool->free_map[tx_pool->producer_index] = index;
3099 			tx_pool->producer_index =
3100 				(tx_pool->producer_index + 1) %
3101 					tx_pool->num_buffers;
3102 		}
3103 		/* remove tx_comp scrq*/
3104 		next->tx_comp.first = 0;
3105 
3106 		if (atomic_sub_return(num_entries, &scrq->used) <=
3107 		    (adapter->req_tx_entries_per_subcrq / 2) &&
3108 		    __netif_subqueue_stopped(adapter->netdev,
3109 					     scrq->pool_index)) {
3110 			netif_wake_subqueue(adapter->netdev, scrq->pool_index);
3111 			netdev_dbg(adapter->netdev, "Started queue %d\n",
3112 				   scrq->pool_index);
3113 		}
3114 	}
3115 
3116 	enable_scrq_irq(adapter, scrq);
3117 
3118 	if (pending_scrq(adapter, scrq)) {
3119 		disable_scrq_irq(adapter, scrq);
3120 		goto restart_loop;
3121 	}
3122 
3123 	return 0;
3124 }
3125 
3126 static irqreturn_t ibmvnic_interrupt_tx(int irq, void *instance)
3127 {
3128 	struct ibmvnic_sub_crq_queue *scrq = instance;
3129 	struct ibmvnic_adapter *adapter = scrq->adapter;
3130 
3131 	disable_scrq_irq(adapter, scrq);
3132 	ibmvnic_complete_tx(adapter, scrq);
3133 
3134 	return IRQ_HANDLED;
3135 }
3136 
3137 static irqreturn_t ibmvnic_interrupt_rx(int irq, void *instance)
3138 {
3139 	struct ibmvnic_sub_crq_queue *scrq = instance;
3140 	struct ibmvnic_adapter *adapter = scrq->adapter;
3141 
3142 	/* When booting a kdump kernel we can hit pending interrupts
3143 	 * prior to completing driver initialization.
3144 	 */
3145 	if (unlikely(adapter->state != VNIC_OPEN))
3146 		return IRQ_NONE;
3147 
3148 	adapter->rx_stats_buffers[scrq->scrq_num].interrupts++;
3149 
3150 	if (napi_schedule_prep(&adapter->napi[scrq->scrq_num])) {
3151 		disable_scrq_irq(adapter, scrq);
3152 		__napi_schedule(&adapter->napi[scrq->scrq_num]);
3153 	}
3154 
3155 	return IRQ_HANDLED;
3156 }
3157 
3158 static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter)
3159 {
3160 	struct device *dev = &adapter->vdev->dev;
3161 	struct ibmvnic_sub_crq_queue *scrq;
3162 	int i = 0, j = 0;
3163 	int rc = 0;
3164 
3165 	for (i = 0; i < adapter->req_tx_queues; i++) {
3166 		netdev_dbg(adapter->netdev, "Initializing tx_scrq[%d] irq\n",
3167 			   i);
3168 		scrq = adapter->tx_scrq[i];
3169 		scrq->irq = irq_create_mapping(NULL, scrq->hw_irq);
3170 
3171 		if (!scrq->irq) {
3172 			rc = -EINVAL;
3173 			dev_err(dev, "Error mapping irq\n");
3174 			goto req_tx_irq_failed;
3175 		}
3176 
3177 		snprintf(scrq->name, sizeof(scrq->name), "ibmvnic-%x-tx%d",
3178 			 adapter->vdev->unit_address, i);
3179 		rc = request_irq(scrq->irq, ibmvnic_interrupt_tx,
3180 				 0, scrq->name, scrq);
3181 
3182 		if (rc) {
3183 			dev_err(dev, "Couldn't register tx irq 0x%x. rc=%d\n",
3184 				scrq->irq, rc);
3185 			irq_dispose_mapping(scrq->irq);
3186 			goto req_tx_irq_failed;
3187 		}
3188 	}
3189 
3190 	for (i = 0; i < adapter->req_rx_queues; i++) {
3191 		netdev_dbg(adapter->netdev, "Initializing rx_scrq[%d] irq\n",
3192 			   i);
3193 		scrq = adapter->rx_scrq[i];
3194 		scrq->irq = irq_create_mapping(NULL, scrq->hw_irq);
3195 		if (!scrq->irq) {
3196 			rc = -EINVAL;
3197 			dev_err(dev, "Error mapping irq\n");
3198 			goto req_rx_irq_failed;
3199 		}
3200 		snprintf(scrq->name, sizeof(scrq->name), "ibmvnic-%x-rx%d",
3201 			 adapter->vdev->unit_address, i);
3202 		rc = request_irq(scrq->irq, ibmvnic_interrupt_rx,
3203 				 0, scrq->name, scrq);
3204 		if (rc) {
3205 			dev_err(dev, "Couldn't register rx irq 0x%x. rc=%d\n",
3206 				scrq->irq, rc);
3207 			irq_dispose_mapping(scrq->irq);
3208 			goto req_rx_irq_failed;
3209 		}
3210 	}
3211 	return rc;
3212 
3213 req_rx_irq_failed:
3214 	for (j = 0; j < i; j++) {
3215 		free_irq(adapter->rx_scrq[j]->irq, adapter->rx_scrq[j]);
3216 		irq_dispose_mapping(adapter->rx_scrq[j]->irq);
3217 	}
3218 	i = adapter->req_tx_queues;
3219 req_tx_irq_failed:
3220 	for (j = 0; j < i; j++) {
3221 		free_irq(adapter->tx_scrq[j]->irq, adapter->tx_scrq[j]);
3222 		irq_dispose_mapping(adapter->tx_scrq[j]->irq);
3223 	}
3224 	release_sub_crqs(adapter, 1);
3225 	return rc;
3226 }
3227 
3228 static int init_sub_crqs(struct ibmvnic_adapter *adapter)
3229 {
3230 	struct device *dev = &adapter->vdev->dev;
3231 	struct ibmvnic_sub_crq_queue **allqueues;
3232 	int registered_queues = 0;
3233 	int total_queues;
3234 	int more = 0;
3235 	int i;
3236 
3237 	total_queues = adapter->req_tx_queues + adapter->req_rx_queues;
3238 
3239 	allqueues = kcalloc(total_queues, sizeof(*allqueues), GFP_KERNEL);
3240 	if (!allqueues)
3241 		return -1;
3242 
3243 	for (i = 0; i < total_queues; i++) {
3244 		allqueues[i] = init_sub_crq_queue(adapter);
3245 		if (!allqueues[i]) {
3246 			dev_warn(dev, "Couldn't allocate all sub-crqs\n");
3247 			break;
3248 		}
3249 		registered_queues++;
3250 	}
3251 
3252 	/* Make sure we were able to register the minimum number of queues */
3253 	if (registered_queues <
3254 	    adapter->min_tx_queues + adapter->min_rx_queues) {
3255 		dev_err(dev, "Fatal: Couldn't init  min number of sub-crqs\n");
3256 		goto tx_failed;
3257 	}
3258 
3259 	/* Distribute the failed allocated queues*/
3260 	for (i = 0; i < total_queues - registered_queues + more ; i++) {
3261 		netdev_dbg(adapter->netdev, "Reducing number of queues\n");
3262 		switch (i % 3) {
3263 		case 0:
3264 			if (adapter->req_rx_queues > adapter->min_rx_queues)
3265 				adapter->req_rx_queues--;
3266 			else
3267 				more++;
3268 			break;
3269 		case 1:
3270 			if (adapter->req_tx_queues > adapter->min_tx_queues)
3271 				adapter->req_tx_queues--;
3272 			else
3273 				more++;
3274 			break;
3275 		}
3276 	}
3277 
3278 	adapter->tx_scrq = kcalloc(adapter->req_tx_queues,
3279 				   sizeof(*adapter->tx_scrq), GFP_KERNEL);
3280 	if (!adapter->tx_scrq)
3281 		goto tx_failed;
3282 
3283 	for (i = 0; i < adapter->req_tx_queues; i++) {
3284 		adapter->tx_scrq[i] = allqueues[i];
3285 		adapter->tx_scrq[i]->pool_index = i;
3286 		adapter->num_active_tx_scrqs++;
3287 	}
3288 
3289 	adapter->rx_scrq = kcalloc(adapter->req_rx_queues,
3290 				   sizeof(*adapter->rx_scrq), GFP_KERNEL);
3291 	if (!adapter->rx_scrq)
3292 		goto rx_failed;
3293 
3294 	for (i = 0; i < adapter->req_rx_queues; i++) {
3295 		adapter->rx_scrq[i] = allqueues[i + adapter->req_tx_queues];
3296 		adapter->rx_scrq[i]->scrq_num = i;
3297 		adapter->num_active_rx_scrqs++;
3298 	}
3299 
3300 	kfree(allqueues);
3301 	return 0;
3302 
3303 rx_failed:
3304 	kfree(adapter->tx_scrq);
3305 	adapter->tx_scrq = NULL;
3306 tx_failed:
3307 	for (i = 0; i < registered_queues; i++)
3308 		release_sub_crq_queue(adapter, allqueues[i], 1);
3309 	kfree(allqueues);
3310 	return -1;
3311 }
3312 
3313 static void ibmvnic_send_req_caps(struct ibmvnic_adapter *adapter, int retry)
3314 {
3315 	struct device *dev = &adapter->vdev->dev;
3316 	union ibmvnic_crq crq;
3317 	int max_entries;
3318 
3319 	if (!retry) {
3320 		/* Sub-CRQ entries are 32 byte long */
3321 		int entries_page = 4 * PAGE_SIZE / (sizeof(u64) * 4);
3322 
3323 		if (adapter->min_tx_entries_per_subcrq > entries_page ||
3324 		    adapter->min_rx_add_entries_per_subcrq > entries_page) {
3325 			dev_err(dev, "Fatal, invalid entries per sub-crq\n");
3326 			return;
3327 		}
3328 
3329 		if (adapter->desired.mtu)
3330 			adapter->req_mtu = adapter->desired.mtu;
3331 		else
3332 			adapter->req_mtu = adapter->netdev->mtu + ETH_HLEN;
3333 
3334 		if (!adapter->desired.tx_entries)
3335 			adapter->desired.tx_entries =
3336 					adapter->max_tx_entries_per_subcrq;
3337 		if (!adapter->desired.rx_entries)
3338 			adapter->desired.rx_entries =
3339 					adapter->max_rx_add_entries_per_subcrq;
3340 
3341 		max_entries = IBMVNIC_MAX_LTB_SIZE /
3342 			      (adapter->req_mtu + IBMVNIC_BUFFER_HLEN);
3343 
3344 		if ((adapter->req_mtu + IBMVNIC_BUFFER_HLEN) *
3345 			adapter->desired.tx_entries > IBMVNIC_MAX_LTB_SIZE) {
3346 			adapter->desired.tx_entries = max_entries;
3347 		}
3348 
3349 		if ((adapter->req_mtu + IBMVNIC_BUFFER_HLEN) *
3350 			adapter->desired.rx_entries > IBMVNIC_MAX_LTB_SIZE) {
3351 			adapter->desired.rx_entries = max_entries;
3352 		}
3353 
3354 		if (adapter->desired.tx_entries)
3355 			adapter->req_tx_entries_per_subcrq =
3356 					adapter->desired.tx_entries;
3357 		else
3358 			adapter->req_tx_entries_per_subcrq =
3359 					adapter->max_tx_entries_per_subcrq;
3360 
3361 		if (adapter->desired.rx_entries)
3362 			adapter->req_rx_add_entries_per_subcrq =
3363 					adapter->desired.rx_entries;
3364 		else
3365 			adapter->req_rx_add_entries_per_subcrq =
3366 					adapter->max_rx_add_entries_per_subcrq;
3367 
3368 		if (adapter->desired.tx_queues)
3369 			adapter->req_tx_queues =
3370 					adapter->desired.tx_queues;
3371 		else
3372 			adapter->req_tx_queues =
3373 					adapter->opt_tx_comp_sub_queues;
3374 
3375 		if (adapter->desired.rx_queues)
3376 			adapter->req_rx_queues =
3377 					adapter->desired.rx_queues;
3378 		else
3379 			adapter->req_rx_queues =
3380 					adapter->opt_rx_comp_queues;
3381 
3382 		adapter->req_rx_add_queues = adapter->max_rx_add_queues;
3383 	}
3384 
3385 	memset(&crq, 0, sizeof(crq));
3386 	crq.request_capability.first = IBMVNIC_CRQ_CMD;
3387 	crq.request_capability.cmd = REQUEST_CAPABILITY;
3388 
3389 	crq.request_capability.capability = cpu_to_be16(REQ_TX_QUEUES);
3390 	crq.request_capability.number = cpu_to_be64(adapter->req_tx_queues);
3391 	atomic_inc(&adapter->running_cap_crqs);
3392 	ibmvnic_send_crq(adapter, &crq);
3393 
3394 	crq.request_capability.capability = cpu_to_be16(REQ_RX_QUEUES);
3395 	crq.request_capability.number = cpu_to_be64(adapter->req_rx_queues);
3396 	atomic_inc(&adapter->running_cap_crqs);
3397 	ibmvnic_send_crq(adapter, &crq);
3398 
3399 	crq.request_capability.capability = cpu_to_be16(REQ_RX_ADD_QUEUES);
3400 	crq.request_capability.number = cpu_to_be64(adapter->req_rx_add_queues);
3401 	atomic_inc(&adapter->running_cap_crqs);
3402 	ibmvnic_send_crq(adapter, &crq);
3403 
3404 	crq.request_capability.capability =
3405 	    cpu_to_be16(REQ_TX_ENTRIES_PER_SUBCRQ);
3406 	crq.request_capability.number =
3407 	    cpu_to_be64(adapter->req_tx_entries_per_subcrq);
3408 	atomic_inc(&adapter->running_cap_crqs);
3409 	ibmvnic_send_crq(adapter, &crq);
3410 
3411 	crq.request_capability.capability =
3412 	    cpu_to_be16(REQ_RX_ADD_ENTRIES_PER_SUBCRQ);
3413 	crq.request_capability.number =
3414 	    cpu_to_be64(adapter->req_rx_add_entries_per_subcrq);
3415 	atomic_inc(&adapter->running_cap_crqs);
3416 	ibmvnic_send_crq(adapter, &crq);
3417 
3418 	crq.request_capability.capability = cpu_to_be16(REQ_MTU);
3419 	crq.request_capability.number = cpu_to_be64(adapter->req_mtu);
3420 	atomic_inc(&adapter->running_cap_crqs);
3421 	ibmvnic_send_crq(adapter, &crq);
3422 
3423 	if (adapter->netdev->flags & IFF_PROMISC) {
3424 		if (adapter->promisc_supported) {
3425 			crq.request_capability.capability =
3426 			    cpu_to_be16(PROMISC_REQUESTED);
3427 			crq.request_capability.number = cpu_to_be64(1);
3428 			atomic_inc(&adapter->running_cap_crqs);
3429 			ibmvnic_send_crq(adapter, &crq);
3430 		}
3431 	} else {
3432 		crq.request_capability.capability =
3433 		    cpu_to_be16(PROMISC_REQUESTED);
3434 		crq.request_capability.number = cpu_to_be64(0);
3435 		atomic_inc(&adapter->running_cap_crqs);
3436 		ibmvnic_send_crq(adapter, &crq);
3437 	}
3438 }
3439 
3440 static int pending_scrq(struct ibmvnic_adapter *adapter,
3441 			struct ibmvnic_sub_crq_queue *scrq)
3442 {
3443 	union sub_crq *entry = &scrq->msgs[scrq->cur];
3444 
3445 	if (entry->generic.first & IBMVNIC_CRQ_CMD_RSP)
3446 		return 1;
3447 	else
3448 		return 0;
3449 }
3450 
3451 static union sub_crq *ibmvnic_next_scrq(struct ibmvnic_adapter *adapter,
3452 					struct ibmvnic_sub_crq_queue *scrq)
3453 {
3454 	union sub_crq *entry;
3455 	unsigned long flags;
3456 
3457 	spin_lock_irqsave(&scrq->lock, flags);
3458 	entry = &scrq->msgs[scrq->cur];
3459 	if (entry->generic.first & IBMVNIC_CRQ_CMD_RSP) {
3460 		if (++scrq->cur == scrq->size)
3461 			scrq->cur = 0;
3462 	} else {
3463 		entry = NULL;
3464 	}
3465 	spin_unlock_irqrestore(&scrq->lock, flags);
3466 
3467 	return entry;
3468 }
3469 
3470 static union ibmvnic_crq *ibmvnic_next_crq(struct ibmvnic_adapter *adapter)
3471 {
3472 	struct ibmvnic_crq_queue *queue = &adapter->crq;
3473 	union ibmvnic_crq *crq;
3474 
3475 	crq = &queue->msgs[queue->cur];
3476 	if (crq->generic.first & IBMVNIC_CRQ_CMD_RSP) {
3477 		if (++queue->cur == queue->size)
3478 			queue->cur = 0;
3479 	} else {
3480 		crq = NULL;
3481 	}
3482 
3483 	return crq;
3484 }
3485 
3486 static void print_subcrq_error(struct device *dev, int rc, const char *func)
3487 {
3488 	switch (rc) {
3489 	case H_PARAMETER:
3490 		dev_warn_ratelimited(dev,
3491 				     "%s failed: Send request is malformed or adapter failover pending. (rc=%d)\n",
3492 				     func, rc);
3493 		break;
3494 	case H_CLOSED:
3495 		dev_warn_ratelimited(dev,
3496 				     "%s failed: Backing queue closed. Adapter is down or failover pending. (rc=%d)\n",
3497 				     func, rc);
3498 		break;
3499 	default:
3500 		dev_err_ratelimited(dev, "%s failed: (rc=%d)\n", func, rc);
3501 		break;
3502 	}
3503 }
3504 
3505 static int send_subcrq(struct ibmvnic_adapter *adapter, u64 remote_handle,
3506 		       union sub_crq *sub_crq)
3507 {
3508 	unsigned int ua = adapter->vdev->unit_address;
3509 	struct device *dev = &adapter->vdev->dev;
3510 	u64 *u64_crq = (u64 *)sub_crq;
3511 	int rc;
3512 
3513 	netdev_dbg(adapter->netdev,
3514 		   "Sending sCRQ %016lx: %016lx %016lx %016lx %016lx\n",
3515 		   (unsigned long int)cpu_to_be64(remote_handle),
3516 		   (unsigned long int)cpu_to_be64(u64_crq[0]),
3517 		   (unsigned long int)cpu_to_be64(u64_crq[1]),
3518 		   (unsigned long int)cpu_to_be64(u64_crq[2]),
3519 		   (unsigned long int)cpu_to_be64(u64_crq[3]));
3520 
3521 	/* Make sure the hypervisor sees the complete request */
3522 	mb();
3523 
3524 	rc = plpar_hcall_norets(H_SEND_SUB_CRQ, ua,
3525 				cpu_to_be64(remote_handle),
3526 				cpu_to_be64(u64_crq[0]),
3527 				cpu_to_be64(u64_crq[1]),
3528 				cpu_to_be64(u64_crq[2]),
3529 				cpu_to_be64(u64_crq[3]));
3530 
3531 	if (rc)
3532 		print_subcrq_error(dev, rc, __func__);
3533 
3534 	return rc;
3535 }
3536 
3537 static int send_subcrq_indirect(struct ibmvnic_adapter *adapter,
3538 				u64 remote_handle, u64 ioba, u64 num_entries)
3539 {
3540 	unsigned int ua = adapter->vdev->unit_address;
3541 	struct device *dev = &adapter->vdev->dev;
3542 	int rc;
3543 
3544 	/* Make sure the hypervisor sees the complete request */
3545 	mb();
3546 	rc = plpar_hcall_norets(H_SEND_SUB_CRQ_INDIRECT, ua,
3547 				cpu_to_be64(remote_handle),
3548 				ioba, num_entries);
3549 
3550 	if (rc)
3551 		print_subcrq_error(dev, rc, __func__);
3552 
3553 	return rc;
3554 }
3555 
3556 static int ibmvnic_send_crq(struct ibmvnic_adapter *adapter,
3557 			    union ibmvnic_crq *crq)
3558 {
3559 	unsigned int ua = adapter->vdev->unit_address;
3560 	struct device *dev = &adapter->vdev->dev;
3561 	u64 *u64_crq = (u64 *)crq;
3562 	int rc;
3563 
3564 	netdev_dbg(adapter->netdev, "Sending CRQ: %016lx %016lx\n",
3565 		   (unsigned long int)cpu_to_be64(u64_crq[0]),
3566 		   (unsigned long int)cpu_to_be64(u64_crq[1]));
3567 
3568 	if (!adapter->crq.active &&
3569 	    crq->generic.first != IBMVNIC_CRQ_INIT_CMD) {
3570 		dev_warn(dev, "Invalid request detected while CRQ is inactive, possible device state change during reset\n");
3571 		return -EINVAL;
3572 	}
3573 
3574 	/* Make sure the hypervisor sees the complete request */
3575 	mb();
3576 
3577 	rc = plpar_hcall_norets(H_SEND_CRQ, ua,
3578 				cpu_to_be64(u64_crq[0]),
3579 				cpu_to_be64(u64_crq[1]));
3580 
3581 	if (rc) {
3582 		if (rc == H_CLOSED) {
3583 			dev_warn(dev, "CRQ Queue closed\n");
3584 			if (test_bit(0, &adapter->resetting))
3585 				ibmvnic_reset(adapter, VNIC_RESET_FATAL);
3586 		}
3587 
3588 		dev_warn(dev, "Send error (rc=%d)\n", rc);
3589 	}
3590 
3591 	return rc;
3592 }
3593 
3594 static int ibmvnic_send_crq_init(struct ibmvnic_adapter *adapter)
3595 {
3596 	union ibmvnic_crq crq;
3597 
3598 	memset(&crq, 0, sizeof(crq));
3599 	crq.generic.first = IBMVNIC_CRQ_INIT_CMD;
3600 	crq.generic.cmd = IBMVNIC_CRQ_INIT;
3601 	netdev_dbg(adapter->netdev, "Sending CRQ init\n");
3602 
3603 	return ibmvnic_send_crq(adapter, &crq);
3604 }
3605 
3606 static int send_version_xchg(struct ibmvnic_adapter *adapter)
3607 {
3608 	union ibmvnic_crq crq;
3609 
3610 	memset(&crq, 0, sizeof(crq));
3611 	crq.version_exchange.first = IBMVNIC_CRQ_CMD;
3612 	crq.version_exchange.cmd = VERSION_EXCHANGE;
3613 	crq.version_exchange.version = cpu_to_be16(ibmvnic_version);
3614 
3615 	return ibmvnic_send_crq(adapter, &crq);
3616 }
3617 
3618 struct vnic_login_client_data {
3619 	u8	type;
3620 	__be16	len;
3621 	char	name[];
3622 } __packed;
3623 
3624 static int vnic_client_data_len(struct ibmvnic_adapter *adapter)
3625 {
3626 	int len;
3627 
3628 	/* Calculate the amount of buffer space needed for the
3629 	 * vnic client data in the login buffer. There are four entries,
3630 	 * OS name, LPAR name, device name, and a null last entry.
3631 	 */
3632 	len = 4 * sizeof(struct vnic_login_client_data);
3633 	len += 6; /* "Linux" plus NULL */
3634 	len += strlen(utsname()->nodename) + 1;
3635 	len += strlen(adapter->netdev->name) + 1;
3636 
3637 	return len;
3638 }
3639 
3640 static void vnic_add_client_data(struct ibmvnic_adapter *adapter,
3641 				 struct vnic_login_client_data *vlcd)
3642 {
3643 	const char *os_name = "Linux";
3644 	int len;
3645 
3646 	/* Type 1 - LPAR OS */
3647 	vlcd->type = 1;
3648 	len = strlen(os_name) + 1;
3649 	vlcd->len = cpu_to_be16(len);
3650 	strncpy(vlcd->name, os_name, len);
3651 	vlcd = (struct vnic_login_client_data *)(vlcd->name + len);
3652 
3653 	/* Type 2 - LPAR name */
3654 	vlcd->type = 2;
3655 	len = strlen(utsname()->nodename) + 1;
3656 	vlcd->len = cpu_to_be16(len);
3657 	strncpy(vlcd->name, utsname()->nodename, len);
3658 	vlcd = (struct vnic_login_client_data *)(vlcd->name + len);
3659 
3660 	/* Type 3 - device name */
3661 	vlcd->type = 3;
3662 	len = strlen(adapter->netdev->name) + 1;
3663 	vlcd->len = cpu_to_be16(len);
3664 	strncpy(vlcd->name, adapter->netdev->name, len);
3665 }
3666 
3667 static int send_login(struct ibmvnic_adapter *adapter)
3668 {
3669 	struct ibmvnic_login_rsp_buffer *login_rsp_buffer;
3670 	struct ibmvnic_login_buffer *login_buffer;
3671 	struct device *dev = &adapter->vdev->dev;
3672 	dma_addr_t rsp_buffer_token;
3673 	dma_addr_t buffer_token;
3674 	size_t rsp_buffer_size;
3675 	union ibmvnic_crq crq;
3676 	size_t buffer_size;
3677 	__be64 *tx_list_p;
3678 	__be64 *rx_list_p;
3679 	int client_data_len;
3680 	struct vnic_login_client_data *vlcd;
3681 	int i;
3682 
3683 	if (!adapter->tx_scrq || !adapter->rx_scrq) {
3684 		netdev_err(adapter->netdev,
3685 			   "RX or TX queues are not allocated, device login failed\n");
3686 		return -1;
3687 	}
3688 
3689 	release_login_rsp_buffer(adapter);
3690 	client_data_len = vnic_client_data_len(adapter);
3691 
3692 	buffer_size =
3693 	    sizeof(struct ibmvnic_login_buffer) +
3694 	    sizeof(u64) * (adapter->req_tx_queues + adapter->req_rx_queues) +
3695 	    client_data_len;
3696 
3697 	login_buffer = kzalloc(buffer_size, GFP_ATOMIC);
3698 	if (!login_buffer)
3699 		goto buf_alloc_failed;
3700 
3701 	buffer_token = dma_map_single(dev, login_buffer, buffer_size,
3702 				      DMA_TO_DEVICE);
3703 	if (dma_mapping_error(dev, buffer_token)) {
3704 		dev_err(dev, "Couldn't map login buffer\n");
3705 		goto buf_map_failed;
3706 	}
3707 
3708 	rsp_buffer_size = sizeof(struct ibmvnic_login_rsp_buffer) +
3709 			  sizeof(u64) * adapter->req_tx_queues +
3710 			  sizeof(u64) * adapter->req_rx_queues +
3711 			  sizeof(u64) * adapter->req_rx_queues +
3712 			  sizeof(u8) * IBMVNIC_TX_DESC_VERSIONS;
3713 
3714 	login_rsp_buffer = kmalloc(rsp_buffer_size, GFP_ATOMIC);
3715 	if (!login_rsp_buffer)
3716 		goto buf_rsp_alloc_failed;
3717 
3718 	rsp_buffer_token = dma_map_single(dev, login_rsp_buffer,
3719 					  rsp_buffer_size, DMA_FROM_DEVICE);
3720 	if (dma_mapping_error(dev, rsp_buffer_token)) {
3721 		dev_err(dev, "Couldn't map login rsp buffer\n");
3722 		goto buf_rsp_map_failed;
3723 	}
3724 
3725 	adapter->login_buf = login_buffer;
3726 	adapter->login_buf_token = buffer_token;
3727 	adapter->login_buf_sz = buffer_size;
3728 	adapter->login_rsp_buf = login_rsp_buffer;
3729 	adapter->login_rsp_buf_token = rsp_buffer_token;
3730 	adapter->login_rsp_buf_sz = rsp_buffer_size;
3731 
3732 	login_buffer->len = cpu_to_be32(buffer_size);
3733 	login_buffer->version = cpu_to_be32(INITIAL_VERSION_LB);
3734 	login_buffer->num_txcomp_subcrqs = cpu_to_be32(adapter->req_tx_queues);
3735 	login_buffer->off_txcomp_subcrqs =
3736 	    cpu_to_be32(sizeof(struct ibmvnic_login_buffer));
3737 	login_buffer->num_rxcomp_subcrqs = cpu_to_be32(adapter->req_rx_queues);
3738 	login_buffer->off_rxcomp_subcrqs =
3739 	    cpu_to_be32(sizeof(struct ibmvnic_login_buffer) +
3740 			sizeof(u64) * adapter->req_tx_queues);
3741 	login_buffer->login_rsp_ioba = cpu_to_be32(rsp_buffer_token);
3742 	login_buffer->login_rsp_len = cpu_to_be32(rsp_buffer_size);
3743 
3744 	tx_list_p = (__be64 *)((char *)login_buffer +
3745 				      sizeof(struct ibmvnic_login_buffer));
3746 	rx_list_p = (__be64 *)((char *)login_buffer +
3747 				      sizeof(struct ibmvnic_login_buffer) +
3748 				      sizeof(u64) * adapter->req_tx_queues);
3749 
3750 	for (i = 0; i < adapter->req_tx_queues; i++) {
3751 		if (adapter->tx_scrq[i]) {
3752 			tx_list_p[i] = cpu_to_be64(adapter->tx_scrq[i]->
3753 						   crq_num);
3754 		}
3755 	}
3756 
3757 	for (i = 0; i < adapter->req_rx_queues; i++) {
3758 		if (adapter->rx_scrq[i]) {
3759 			rx_list_p[i] = cpu_to_be64(adapter->rx_scrq[i]->
3760 						   crq_num);
3761 		}
3762 	}
3763 
3764 	/* Insert vNIC login client data */
3765 	vlcd = (struct vnic_login_client_data *)
3766 		((char *)rx_list_p + (sizeof(u64) * adapter->req_rx_queues));
3767 	login_buffer->client_data_offset =
3768 			cpu_to_be32((char *)vlcd - (char *)login_buffer);
3769 	login_buffer->client_data_len = cpu_to_be32(client_data_len);
3770 
3771 	vnic_add_client_data(adapter, vlcd);
3772 
3773 	netdev_dbg(adapter->netdev, "Login Buffer:\n");
3774 	for (i = 0; i < (adapter->login_buf_sz - 1) / 8 + 1; i++) {
3775 		netdev_dbg(adapter->netdev, "%016lx\n",
3776 			   ((unsigned long int *)(adapter->login_buf))[i]);
3777 	}
3778 
3779 	memset(&crq, 0, sizeof(crq));
3780 	crq.login.first = IBMVNIC_CRQ_CMD;
3781 	crq.login.cmd = LOGIN;
3782 	crq.login.ioba = cpu_to_be32(buffer_token);
3783 	crq.login.len = cpu_to_be32(buffer_size);
3784 	ibmvnic_send_crq(adapter, &crq);
3785 
3786 	return 0;
3787 
3788 buf_rsp_map_failed:
3789 	kfree(login_rsp_buffer);
3790 buf_rsp_alloc_failed:
3791 	dma_unmap_single(dev, buffer_token, buffer_size, DMA_TO_DEVICE);
3792 buf_map_failed:
3793 	kfree(login_buffer);
3794 buf_alloc_failed:
3795 	return -1;
3796 }
3797 
3798 static int send_request_map(struct ibmvnic_adapter *adapter, dma_addr_t addr,
3799 			    u32 len, u8 map_id)
3800 {
3801 	union ibmvnic_crq crq;
3802 
3803 	memset(&crq, 0, sizeof(crq));
3804 	crq.request_map.first = IBMVNIC_CRQ_CMD;
3805 	crq.request_map.cmd = REQUEST_MAP;
3806 	crq.request_map.map_id = map_id;
3807 	crq.request_map.ioba = cpu_to_be32(addr);
3808 	crq.request_map.len = cpu_to_be32(len);
3809 	return ibmvnic_send_crq(adapter, &crq);
3810 }
3811 
3812 static int send_request_unmap(struct ibmvnic_adapter *adapter, u8 map_id)
3813 {
3814 	union ibmvnic_crq crq;
3815 
3816 	memset(&crq, 0, sizeof(crq));
3817 	crq.request_unmap.first = IBMVNIC_CRQ_CMD;
3818 	crq.request_unmap.cmd = REQUEST_UNMAP;
3819 	crq.request_unmap.map_id = map_id;
3820 	return ibmvnic_send_crq(adapter, &crq);
3821 }
3822 
3823 static void send_map_query(struct ibmvnic_adapter *adapter)
3824 {
3825 	union ibmvnic_crq crq;
3826 
3827 	memset(&crq, 0, sizeof(crq));
3828 	crq.query_map.first = IBMVNIC_CRQ_CMD;
3829 	crq.query_map.cmd = QUERY_MAP;
3830 	ibmvnic_send_crq(adapter, &crq);
3831 }
3832 
3833 /* Send a series of CRQs requesting various capabilities of the VNIC server */
3834 static void send_cap_queries(struct ibmvnic_adapter *adapter)
3835 {
3836 	union ibmvnic_crq crq;
3837 
3838 	atomic_set(&adapter->running_cap_crqs, 0);
3839 	memset(&crq, 0, sizeof(crq));
3840 	crq.query_capability.first = IBMVNIC_CRQ_CMD;
3841 	crq.query_capability.cmd = QUERY_CAPABILITY;
3842 
3843 	crq.query_capability.capability = cpu_to_be16(MIN_TX_QUEUES);
3844 	atomic_inc(&adapter->running_cap_crqs);
3845 	ibmvnic_send_crq(adapter, &crq);
3846 
3847 	crq.query_capability.capability = cpu_to_be16(MIN_RX_QUEUES);
3848 	atomic_inc(&adapter->running_cap_crqs);
3849 	ibmvnic_send_crq(adapter, &crq);
3850 
3851 	crq.query_capability.capability = cpu_to_be16(MIN_RX_ADD_QUEUES);
3852 	atomic_inc(&adapter->running_cap_crqs);
3853 	ibmvnic_send_crq(adapter, &crq);
3854 
3855 	crq.query_capability.capability = cpu_to_be16(MAX_TX_QUEUES);
3856 	atomic_inc(&adapter->running_cap_crqs);
3857 	ibmvnic_send_crq(adapter, &crq);
3858 
3859 	crq.query_capability.capability = cpu_to_be16(MAX_RX_QUEUES);
3860 	atomic_inc(&adapter->running_cap_crqs);
3861 	ibmvnic_send_crq(adapter, &crq);
3862 
3863 	crq.query_capability.capability = cpu_to_be16(MAX_RX_ADD_QUEUES);
3864 	atomic_inc(&adapter->running_cap_crqs);
3865 	ibmvnic_send_crq(adapter, &crq);
3866 
3867 	crq.query_capability.capability =
3868 	    cpu_to_be16(MIN_TX_ENTRIES_PER_SUBCRQ);
3869 	atomic_inc(&adapter->running_cap_crqs);
3870 	ibmvnic_send_crq(adapter, &crq);
3871 
3872 	crq.query_capability.capability =
3873 	    cpu_to_be16(MIN_RX_ADD_ENTRIES_PER_SUBCRQ);
3874 	atomic_inc(&adapter->running_cap_crqs);
3875 	ibmvnic_send_crq(adapter, &crq);
3876 
3877 	crq.query_capability.capability =
3878 	    cpu_to_be16(MAX_TX_ENTRIES_PER_SUBCRQ);
3879 	atomic_inc(&adapter->running_cap_crqs);
3880 	ibmvnic_send_crq(adapter, &crq);
3881 
3882 	crq.query_capability.capability =
3883 	    cpu_to_be16(MAX_RX_ADD_ENTRIES_PER_SUBCRQ);
3884 	atomic_inc(&adapter->running_cap_crqs);
3885 	ibmvnic_send_crq(adapter, &crq);
3886 
3887 	crq.query_capability.capability = cpu_to_be16(TCP_IP_OFFLOAD);
3888 	atomic_inc(&adapter->running_cap_crqs);
3889 	ibmvnic_send_crq(adapter, &crq);
3890 
3891 	crq.query_capability.capability = cpu_to_be16(PROMISC_SUPPORTED);
3892 	atomic_inc(&adapter->running_cap_crqs);
3893 	ibmvnic_send_crq(adapter, &crq);
3894 
3895 	crq.query_capability.capability = cpu_to_be16(MIN_MTU);
3896 	atomic_inc(&adapter->running_cap_crqs);
3897 	ibmvnic_send_crq(adapter, &crq);
3898 
3899 	crq.query_capability.capability = cpu_to_be16(MAX_MTU);
3900 	atomic_inc(&adapter->running_cap_crqs);
3901 	ibmvnic_send_crq(adapter, &crq);
3902 
3903 	crq.query_capability.capability = cpu_to_be16(MAX_MULTICAST_FILTERS);
3904 	atomic_inc(&adapter->running_cap_crqs);
3905 	ibmvnic_send_crq(adapter, &crq);
3906 
3907 	crq.query_capability.capability = cpu_to_be16(VLAN_HEADER_INSERTION);
3908 	atomic_inc(&adapter->running_cap_crqs);
3909 	ibmvnic_send_crq(adapter, &crq);
3910 
3911 	crq.query_capability.capability = cpu_to_be16(RX_VLAN_HEADER_INSERTION);
3912 	atomic_inc(&adapter->running_cap_crqs);
3913 	ibmvnic_send_crq(adapter, &crq);
3914 
3915 	crq.query_capability.capability = cpu_to_be16(MAX_TX_SG_ENTRIES);
3916 	atomic_inc(&adapter->running_cap_crqs);
3917 	ibmvnic_send_crq(adapter, &crq);
3918 
3919 	crq.query_capability.capability = cpu_to_be16(RX_SG_SUPPORTED);
3920 	atomic_inc(&adapter->running_cap_crqs);
3921 	ibmvnic_send_crq(adapter, &crq);
3922 
3923 	crq.query_capability.capability = cpu_to_be16(OPT_TX_COMP_SUB_QUEUES);
3924 	atomic_inc(&adapter->running_cap_crqs);
3925 	ibmvnic_send_crq(adapter, &crq);
3926 
3927 	crq.query_capability.capability = cpu_to_be16(OPT_RX_COMP_QUEUES);
3928 	atomic_inc(&adapter->running_cap_crqs);
3929 	ibmvnic_send_crq(adapter, &crq);
3930 
3931 	crq.query_capability.capability =
3932 			cpu_to_be16(OPT_RX_BUFADD_Q_PER_RX_COMP_Q);
3933 	atomic_inc(&adapter->running_cap_crqs);
3934 	ibmvnic_send_crq(adapter, &crq);
3935 
3936 	crq.query_capability.capability =
3937 			cpu_to_be16(OPT_TX_ENTRIES_PER_SUBCRQ);
3938 	atomic_inc(&adapter->running_cap_crqs);
3939 	ibmvnic_send_crq(adapter, &crq);
3940 
3941 	crq.query_capability.capability =
3942 			cpu_to_be16(OPT_RXBA_ENTRIES_PER_SUBCRQ);
3943 	atomic_inc(&adapter->running_cap_crqs);
3944 	ibmvnic_send_crq(adapter, &crq);
3945 
3946 	crq.query_capability.capability = cpu_to_be16(TX_RX_DESC_REQ);
3947 	atomic_inc(&adapter->running_cap_crqs);
3948 	ibmvnic_send_crq(adapter, &crq);
3949 }
3950 
3951 static void handle_vpd_size_rsp(union ibmvnic_crq *crq,
3952 				struct ibmvnic_adapter *adapter)
3953 {
3954 	struct device *dev = &adapter->vdev->dev;
3955 
3956 	if (crq->get_vpd_size_rsp.rc.code) {
3957 		dev_err(dev, "Error retrieving VPD size, rc=%x\n",
3958 			crq->get_vpd_size_rsp.rc.code);
3959 		complete(&adapter->fw_done);
3960 		return;
3961 	}
3962 
3963 	adapter->vpd->len = be64_to_cpu(crq->get_vpd_size_rsp.len);
3964 	complete(&adapter->fw_done);
3965 }
3966 
3967 static void handle_vpd_rsp(union ibmvnic_crq *crq,
3968 			   struct ibmvnic_adapter *adapter)
3969 {
3970 	struct device *dev = &adapter->vdev->dev;
3971 	unsigned char *substr = NULL;
3972 	u8 fw_level_len = 0;
3973 
3974 	memset(adapter->fw_version, 0, 32);
3975 
3976 	dma_unmap_single(dev, adapter->vpd->dma_addr, adapter->vpd->len,
3977 			 DMA_FROM_DEVICE);
3978 
3979 	if (crq->get_vpd_rsp.rc.code) {
3980 		dev_err(dev, "Error retrieving VPD from device, rc=%x\n",
3981 			crq->get_vpd_rsp.rc.code);
3982 		goto complete;
3983 	}
3984 
3985 	/* get the position of the firmware version info
3986 	 * located after the ASCII 'RM' substring in the buffer
3987 	 */
3988 	substr = strnstr(adapter->vpd->buff, "RM", adapter->vpd->len);
3989 	if (!substr) {
3990 		dev_info(dev, "Warning - No FW level has been provided in the VPD buffer by the VIOS Server\n");
3991 		goto complete;
3992 	}
3993 
3994 	/* get length of firmware level ASCII substring */
3995 	if ((substr + 2) < (adapter->vpd->buff + adapter->vpd->len)) {
3996 		fw_level_len = *(substr + 2);
3997 	} else {
3998 		dev_info(dev, "Length of FW substr extrapolated VDP buff\n");
3999 		goto complete;
4000 	}
4001 
4002 	/* copy firmware version string from vpd into adapter */
4003 	if ((substr + 3 + fw_level_len) <
4004 	    (adapter->vpd->buff + adapter->vpd->len)) {
4005 		strncpy((char *)adapter->fw_version, substr + 3, fw_level_len);
4006 	} else {
4007 		dev_info(dev, "FW substr extrapolated VPD buff\n");
4008 	}
4009 
4010 complete:
4011 	if (adapter->fw_version[0] == '\0')
4012 		strncpy((char *)adapter->fw_version, "N/A", 3 * sizeof(char));
4013 	complete(&adapter->fw_done);
4014 }
4015 
4016 static void handle_query_ip_offload_rsp(struct ibmvnic_adapter *adapter)
4017 {
4018 	struct device *dev = &adapter->vdev->dev;
4019 	struct ibmvnic_query_ip_offload_buffer *buf = &adapter->ip_offload_buf;
4020 	netdev_features_t old_hw_features = 0;
4021 	union ibmvnic_crq crq;
4022 	int i;
4023 
4024 	dma_unmap_single(dev, adapter->ip_offload_tok,
4025 			 sizeof(adapter->ip_offload_buf), DMA_FROM_DEVICE);
4026 
4027 	netdev_dbg(adapter->netdev, "Query IP Offload Buffer:\n");
4028 	for (i = 0; i < (sizeof(adapter->ip_offload_buf) - 1) / 8 + 1; i++)
4029 		netdev_dbg(adapter->netdev, "%016lx\n",
4030 			   ((unsigned long int *)(buf))[i]);
4031 
4032 	netdev_dbg(adapter->netdev, "ipv4_chksum = %d\n", buf->ipv4_chksum);
4033 	netdev_dbg(adapter->netdev, "ipv6_chksum = %d\n", buf->ipv6_chksum);
4034 	netdev_dbg(adapter->netdev, "tcp_ipv4_chksum = %d\n",
4035 		   buf->tcp_ipv4_chksum);
4036 	netdev_dbg(adapter->netdev, "tcp_ipv6_chksum = %d\n",
4037 		   buf->tcp_ipv6_chksum);
4038 	netdev_dbg(adapter->netdev, "udp_ipv4_chksum = %d\n",
4039 		   buf->udp_ipv4_chksum);
4040 	netdev_dbg(adapter->netdev, "udp_ipv6_chksum = %d\n",
4041 		   buf->udp_ipv6_chksum);
4042 	netdev_dbg(adapter->netdev, "large_tx_ipv4 = %d\n",
4043 		   buf->large_tx_ipv4);
4044 	netdev_dbg(adapter->netdev, "large_tx_ipv6 = %d\n",
4045 		   buf->large_tx_ipv6);
4046 	netdev_dbg(adapter->netdev, "large_rx_ipv4 = %d\n",
4047 		   buf->large_rx_ipv4);
4048 	netdev_dbg(adapter->netdev, "large_rx_ipv6 = %d\n",
4049 		   buf->large_rx_ipv6);
4050 	netdev_dbg(adapter->netdev, "max_ipv4_hdr_sz = %d\n",
4051 		   buf->max_ipv4_header_size);
4052 	netdev_dbg(adapter->netdev, "max_ipv6_hdr_sz = %d\n",
4053 		   buf->max_ipv6_header_size);
4054 	netdev_dbg(adapter->netdev, "max_tcp_hdr_size = %d\n",
4055 		   buf->max_tcp_header_size);
4056 	netdev_dbg(adapter->netdev, "max_udp_hdr_size = %d\n",
4057 		   buf->max_udp_header_size);
4058 	netdev_dbg(adapter->netdev, "max_large_tx_size = %d\n",
4059 		   buf->max_large_tx_size);
4060 	netdev_dbg(adapter->netdev, "max_large_rx_size = %d\n",
4061 		   buf->max_large_rx_size);
4062 	netdev_dbg(adapter->netdev, "ipv6_ext_hdr = %d\n",
4063 		   buf->ipv6_extension_header);
4064 	netdev_dbg(adapter->netdev, "tcp_pseudosum_req = %d\n",
4065 		   buf->tcp_pseudosum_req);
4066 	netdev_dbg(adapter->netdev, "num_ipv6_ext_hd = %d\n",
4067 		   buf->num_ipv6_ext_headers);
4068 	netdev_dbg(adapter->netdev, "off_ipv6_ext_hd = %d\n",
4069 		   buf->off_ipv6_ext_headers);
4070 
4071 	adapter->ip_offload_ctrl_tok =
4072 	    dma_map_single(dev, &adapter->ip_offload_ctrl,
4073 			   sizeof(adapter->ip_offload_ctrl), DMA_TO_DEVICE);
4074 
4075 	if (dma_mapping_error(dev, adapter->ip_offload_ctrl_tok)) {
4076 		dev_err(dev, "Couldn't map ip offload control buffer\n");
4077 		return;
4078 	}
4079 
4080 	adapter->ip_offload_ctrl.len =
4081 	    cpu_to_be32(sizeof(adapter->ip_offload_ctrl));
4082 	adapter->ip_offload_ctrl.version = cpu_to_be32(INITIAL_VERSION_IOB);
4083 	adapter->ip_offload_ctrl.ipv4_chksum = buf->ipv4_chksum;
4084 	adapter->ip_offload_ctrl.ipv6_chksum = buf->ipv6_chksum;
4085 	adapter->ip_offload_ctrl.tcp_ipv4_chksum = buf->tcp_ipv4_chksum;
4086 	adapter->ip_offload_ctrl.udp_ipv4_chksum = buf->udp_ipv4_chksum;
4087 	adapter->ip_offload_ctrl.tcp_ipv6_chksum = buf->tcp_ipv6_chksum;
4088 	adapter->ip_offload_ctrl.udp_ipv6_chksum = buf->udp_ipv6_chksum;
4089 	adapter->ip_offload_ctrl.large_tx_ipv4 = buf->large_tx_ipv4;
4090 	adapter->ip_offload_ctrl.large_tx_ipv6 = buf->large_tx_ipv6;
4091 
4092 	/* large_rx disabled for now, additional features needed */
4093 	adapter->ip_offload_ctrl.large_rx_ipv4 = 0;
4094 	adapter->ip_offload_ctrl.large_rx_ipv6 = 0;
4095 
4096 	if (adapter->state != VNIC_PROBING) {
4097 		old_hw_features = adapter->netdev->hw_features;
4098 		adapter->netdev->hw_features = 0;
4099 	}
4100 
4101 	adapter->netdev->hw_features = NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO;
4102 
4103 	if (buf->tcp_ipv4_chksum || buf->udp_ipv4_chksum)
4104 		adapter->netdev->hw_features |= NETIF_F_IP_CSUM;
4105 
4106 	if (buf->tcp_ipv6_chksum || buf->udp_ipv6_chksum)
4107 		adapter->netdev->hw_features |= NETIF_F_IPV6_CSUM;
4108 
4109 	if ((adapter->netdev->features &
4110 	    (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)))
4111 		adapter->netdev->hw_features |= NETIF_F_RXCSUM;
4112 
4113 	if (buf->large_tx_ipv4)
4114 		adapter->netdev->hw_features |= NETIF_F_TSO;
4115 	if (buf->large_tx_ipv6)
4116 		adapter->netdev->hw_features |= NETIF_F_TSO6;
4117 
4118 	if (adapter->state == VNIC_PROBING) {
4119 		adapter->netdev->features |= adapter->netdev->hw_features;
4120 	} else if (old_hw_features != adapter->netdev->hw_features) {
4121 		netdev_features_t tmp = 0;
4122 
4123 		/* disable features no longer supported */
4124 		adapter->netdev->features &= adapter->netdev->hw_features;
4125 		/* turn on features now supported if previously enabled */
4126 		tmp = (old_hw_features ^ adapter->netdev->hw_features) &
4127 			adapter->netdev->hw_features;
4128 		adapter->netdev->features |=
4129 				tmp & adapter->netdev->wanted_features;
4130 	}
4131 
4132 	memset(&crq, 0, sizeof(crq));
4133 	crq.control_ip_offload.first = IBMVNIC_CRQ_CMD;
4134 	crq.control_ip_offload.cmd = CONTROL_IP_OFFLOAD;
4135 	crq.control_ip_offload.len =
4136 	    cpu_to_be32(sizeof(adapter->ip_offload_ctrl));
4137 	crq.control_ip_offload.ioba = cpu_to_be32(adapter->ip_offload_ctrl_tok);
4138 	ibmvnic_send_crq(adapter, &crq);
4139 }
4140 
4141 static const char *ibmvnic_fw_err_cause(u16 cause)
4142 {
4143 	switch (cause) {
4144 	case ADAPTER_PROBLEM:
4145 		return "adapter problem";
4146 	case BUS_PROBLEM:
4147 		return "bus problem";
4148 	case FW_PROBLEM:
4149 		return "firmware problem";
4150 	case DD_PROBLEM:
4151 		return "device driver problem";
4152 	case EEH_RECOVERY:
4153 		return "EEH recovery";
4154 	case FW_UPDATED:
4155 		return "firmware updated";
4156 	case LOW_MEMORY:
4157 		return "low Memory";
4158 	default:
4159 		return "unknown";
4160 	}
4161 }
4162 
4163 static void handle_error_indication(union ibmvnic_crq *crq,
4164 				    struct ibmvnic_adapter *adapter)
4165 {
4166 	struct device *dev = &adapter->vdev->dev;
4167 	u16 cause;
4168 
4169 	cause = be16_to_cpu(crq->error_indication.error_cause);
4170 
4171 	dev_warn_ratelimited(dev,
4172 			     "Firmware reports %serror, cause: %s. Starting recovery...\n",
4173 			     crq->error_indication.flags
4174 				& IBMVNIC_FATAL_ERROR ? "FATAL " : "",
4175 			     ibmvnic_fw_err_cause(cause));
4176 
4177 	if (crq->error_indication.flags & IBMVNIC_FATAL_ERROR)
4178 		ibmvnic_reset(adapter, VNIC_RESET_FATAL);
4179 	else
4180 		ibmvnic_reset(adapter, VNIC_RESET_NON_FATAL);
4181 }
4182 
4183 static int handle_change_mac_rsp(union ibmvnic_crq *crq,
4184 				 struct ibmvnic_adapter *adapter)
4185 {
4186 	struct net_device *netdev = adapter->netdev;
4187 	struct device *dev = &adapter->vdev->dev;
4188 	long rc;
4189 
4190 	rc = crq->change_mac_addr_rsp.rc.code;
4191 	if (rc) {
4192 		dev_err(dev, "Error %ld in CHANGE_MAC_ADDR_RSP\n", rc);
4193 		goto out;
4194 	}
4195 	ether_addr_copy(netdev->dev_addr,
4196 			&crq->change_mac_addr_rsp.mac_addr[0]);
4197 out:
4198 	complete(&adapter->fw_done);
4199 	return rc;
4200 }
4201 
4202 static void handle_request_cap_rsp(union ibmvnic_crq *crq,
4203 				   struct ibmvnic_adapter *adapter)
4204 {
4205 	struct device *dev = &adapter->vdev->dev;
4206 	u64 *req_value;
4207 	char *name;
4208 
4209 	atomic_dec(&adapter->running_cap_crqs);
4210 	switch (be16_to_cpu(crq->request_capability_rsp.capability)) {
4211 	case REQ_TX_QUEUES:
4212 		req_value = &adapter->req_tx_queues;
4213 		name = "tx";
4214 		break;
4215 	case REQ_RX_QUEUES:
4216 		req_value = &adapter->req_rx_queues;
4217 		name = "rx";
4218 		break;
4219 	case REQ_RX_ADD_QUEUES:
4220 		req_value = &adapter->req_rx_add_queues;
4221 		name = "rx_add";
4222 		break;
4223 	case REQ_TX_ENTRIES_PER_SUBCRQ:
4224 		req_value = &adapter->req_tx_entries_per_subcrq;
4225 		name = "tx_entries_per_subcrq";
4226 		break;
4227 	case REQ_RX_ADD_ENTRIES_PER_SUBCRQ:
4228 		req_value = &adapter->req_rx_add_entries_per_subcrq;
4229 		name = "rx_add_entries_per_subcrq";
4230 		break;
4231 	case REQ_MTU:
4232 		req_value = &adapter->req_mtu;
4233 		name = "mtu";
4234 		break;
4235 	case PROMISC_REQUESTED:
4236 		req_value = &adapter->promisc;
4237 		name = "promisc";
4238 		break;
4239 	default:
4240 		dev_err(dev, "Got invalid cap request rsp %d\n",
4241 			crq->request_capability.capability);
4242 		return;
4243 	}
4244 
4245 	switch (crq->request_capability_rsp.rc.code) {
4246 	case SUCCESS:
4247 		break;
4248 	case PARTIALSUCCESS:
4249 		dev_info(dev, "req=%lld, rsp=%ld in %s queue, retrying.\n",
4250 			 *req_value,
4251 			 (long int)be64_to_cpu(crq->request_capability_rsp.
4252 					       number), name);
4253 
4254 		if (be16_to_cpu(crq->request_capability_rsp.capability) ==
4255 		    REQ_MTU) {
4256 			pr_err("mtu of %llu is not supported. Reverting.\n",
4257 			       *req_value);
4258 			*req_value = adapter->fallback.mtu;
4259 		} else {
4260 			*req_value =
4261 				be64_to_cpu(crq->request_capability_rsp.number);
4262 		}
4263 
4264 		ibmvnic_send_req_caps(adapter, 1);
4265 		return;
4266 	default:
4267 		dev_err(dev, "Error %d in request cap rsp\n",
4268 			crq->request_capability_rsp.rc.code);
4269 		return;
4270 	}
4271 
4272 	/* Done receiving requested capabilities, query IP offload support */
4273 	if (atomic_read(&adapter->running_cap_crqs) == 0) {
4274 		union ibmvnic_crq newcrq;
4275 		int buf_sz = sizeof(struct ibmvnic_query_ip_offload_buffer);
4276 		struct ibmvnic_query_ip_offload_buffer *ip_offload_buf =
4277 		    &adapter->ip_offload_buf;
4278 
4279 		adapter->wait_capability = false;
4280 		adapter->ip_offload_tok = dma_map_single(dev, ip_offload_buf,
4281 							 buf_sz,
4282 							 DMA_FROM_DEVICE);
4283 
4284 		if (dma_mapping_error(dev, adapter->ip_offload_tok)) {
4285 			if (!firmware_has_feature(FW_FEATURE_CMO))
4286 				dev_err(dev, "Couldn't map offload buffer\n");
4287 			return;
4288 		}
4289 
4290 		memset(&newcrq, 0, sizeof(newcrq));
4291 		newcrq.query_ip_offload.first = IBMVNIC_CRQ_CMD;
4292 		newcrq.query_ip_offload.cmd = QUERY_IP_OFFLOAD;
4293 		newcrq.query_ip_offload.len = cpu_to_be32(buf_sz);
4294 		newcrq.query_ip_offload.ioba =
4295 		    cpu_to_be32(adapter->ip_offload_tok);
4296 
4297 		ibmvnic_send_crq(adapter, &newcrq);
4298 	}
4299 }
4300 
4301 static int handle_login_rsp(union ibmvnic_crq *login_rsp_crq,
4302 			    struct ibmvnic_adapter *adapter)
4303 {
4304 	struct device *dev = &adapter->vdev->dev;
4305 	struct net_device *netdev = adapter->netdev;
4306 	struct ibmvnic_login_rsp_buffer *login_rsp = adapter->login_rsp_buf;
4307 	struct ibmvnic_login_buffer *login = adapter->login_buf;
4308 	int i;
4309 
4310 	dma_unmap_single(dev, adapter->login_buf_token, adapter->login_buf_sz,
4311 			 DMA_TO_DEVICE);
4312 	dma_unmap_single(dev, adapter->login_rsp_buf_token,
4313 			 adapter->login_rsp_buf_sz, DMA_FROM_DEVICE);
4314 
4315 	/* If the number of queues requested can't be allocated by the
4316 	 * server, the login response will return with code 1. We will need
4317 	 * to resend the login buffer with fewer queues requested.
4318 	 */
4319 	if (login_rsp_crq->generic.rc.code) {
4320 		adapter->init_done_rc = login_rsp_crq->generic.rc.code;
4321 		complete(&adapter->init_done);
4322 		return 0;
4323 	}
4324 
4325 	netdev->mtu = adapter->req_mtu - ETH_HLEN;
4326 
4327 	netdev_dbg(adapter->netdev, "Login Response Buffer:\n");
4328 	for (i = 0; i < (adapter->login_rsp_buf_sz - 1) / 8 + 1; i++) {
4329 		netdev_dbg(adapter->netdev, "%016lx\n",
4330 			   ((unsigned long int *)(adapter->login_rsp_buf))[i]);
4331 	}
4332 
4333 	/* Sanity checks */
4334 	if (login->num_txcomp_subcrqs != login_rsp->num_txsubm_subcrqs ||
4335 	    (be32_to_cpu(login->num_rxcomp_subcrqs) *
4336 	     adapter->req_rx_add_queues !=
4337 	     be32_to_cpu(login_rsp->num_rxadd_subcrqs))) {
4338 		dev_err(dev, "FATAL: Inconsistent login and login rsp\n");
4339 		ibmvnic_remove(adapter->vdev);
4340 		return -EIO;
4341 	}
4342 	release_login_buffer(adapter);
4343 	complete(&adapter->init_done);
4344 
4345 	return 0;
4346 }
4347 
4348 static void handle_request_unmap_rsp(union ibmvnic_crq *crq,
4349 				     struct ibmvnic_adapter *adapter)
4350 {
4351 	struct device *dev = &adapter->vdev->dev;
4352 	long rc;
4353 
4354 	rc = crq->request_unmap_rsp.rc.code;
4355 	if (rc)
4356 		dev_err(dev, "Error %ld in REQUEST_UNMAP_RSP\n", rc);
4357 }
4358 
4359 static void handle_query_map_rsp(union ibmvnic_crq *crq,
4360 				 struct ibmvnic_adapter *adapter)
4361 {
4362 	struct net_device *netdev = adapter->netdev;
4363 	struct device *dev = &adapter->vdev->dev;
4364 	long rc;
4365 
4366 	rc = crq->query_map_rsp.rc.code;
4367 	if (rc) {
4368 		dev_err(dev, "Error %ld in QUERY_MAP_RSP\n", rc);
4369 		return;
4370 	}
4371 	netdev_dbg(netdev, "page_size = %d\ntot_pages = %d\nfree_pages = %d\n",
4372 		   crq->query_map_rsp.page_size, crq->query_map_rsp.tot_pages,
4373 		   crq->query_map_rsp.free_pages);
4374 }
4375 
4376 static void handle_query_cap_rsp(union ibmvnic_crq *crq,
4377 				 struct ibmvnic_adapter *adapter)
4378 {
4379 	struct net_device *netdev = adapter->netdev;
4380 	struct device *dev = &adapter->vdev->dev;
4381 	long rc;
4382 
4383 	atomic_dec(&adapter->running_cap_crqs);
4384 	netdev_dbg(netdev, "Outstanding queries: %d\n",
4385 		   atomic_read(&adapter->running_cap_crqs));
4386 	rc = crq->query_capability.rc.code;
4387 	if (rc) {
4388 		dev_err(dev, "Error %ld in QUERY_CAP_RSP\n", rc);
4389 		goto out;
4390 	}
4391 
4392 	switch (be16_to_cpu(crq->query_capability.capability)) {
4393 	case MIN_TX_QUEUES:
4394 		adapter->min_tx_queues =
4395 		    be64_to_cpu(crq->query_capability.number);
4396 		netdev_dbg(netdev, "min_tx_queues = %lld\n",
4397 			   adapter->min_tx_queues);
4398 		break;
4399 	case MIN_RX_QUEUES:
4400 		adapter->min_rx_queues =
4401 		    be64_to_cpu(crq->query_capability.number);
4402 		netdev_dbg(netdev, "min_rx_queues = %lld\n",
4403 			   adapter->min_rx_queues);
4404 		break;
4405 	case MIN_RX_ADD_QUEUES:
4406 		adapter->min_rx_add_queues =
4407 		    be64_to_cpu(crq->query_capability.number);
4408 		netdev_dbg(netdev, "min_rx_add_queues = %lld\n",
4409 			   adapter->min_rx_add_queues);
4410 		break;
4411 	case MAX_TX_QUEUES:
4412 		adapter->max_tx_queues =
4413 		    be64_to_cpu(crq->query_capability.number);
4414 		netdev_dbg(netdev, "max_tx_queues = %lld\n",
4415 			   adapter->max_tx_queues);
4416 		break;
4417 	case MAX_RX_QUEUES:
4418 		adapter->max_rx_queues =
4419 		    be64_to_cpu(crq->query_capability.number);
4420 		netdev_dbg(netdev, "max_rx_queues = %lld\n",
4421 			   adapter->max_rx_queues);
4422 		break;
4423 	case MAX_RX_ADD_QUEUES:
4424 		adapter->max_rx_add_queues =
4425 		    be64_to_cpu(crq->query_capability.number);
4426 		netdev_dbg(netdev, "max_rx_add_queues = %lld\n",
4427 			   adapter->max_rx_add_queues);
4428 		break;
4429 	case MIN_TX_ENTRIES_PER_SUBCRQ:
4430 		adapter->min_tx_entries_per_subcrq =
4431 		    be64_to_cpu(crq->query_capability.number);
4432 		netdev_dbg(netdev, "min_tx_entries_per_subcrq = %lld\n",
4433 			   adapter->min_tx_entries_per_subcrq);
4434 		break;
4435 	case MIN_RX_ADD_ENTRIES_PER_SUBCRQ:
4436 		adapter->min_rx_add_entries_per_subcrq =
4437 		    be64_to_cpu(crq->query_capability.number);
4438 		netdev_dbg(netdev, "min_rx_add_entrs_per_subcrq = %lld\n",
4439 			   adapter->min_rx_add_entries_per_subcrq);
4440 		break;
4441 	case MAX_TX_ENTRIES_PER_SUBCRQ:
4442 		adapter->max_tx_entries_per_subcrq =
4443 		    be64_to_cpu(crq->query_capability.number);
4444 		netdev_dbg(netdev, "max_tx_entries_per_subcrq = %lld\n",
4445 			   adapter->max_tx_entries_per_subcrq);
4446 		break;
4447 	case MAX_RX_ADD_ENTRIES_PER_SUBCRQ:
4448 		adapter->max_rx_add_entries_per_subcrq =
4449 		    be64_to_cpu(crq->query_capability.number);
4450 		netdev_dbg(netdev, "max_rx_add_entrs_per_subcrq = %lld\n",
4451 			   adapter->max_rx_add_entries_per_subcrq);
4452 		break;
4453 	case TCP_IP_OFFLOAD:
4454 		adapter->tcp_ip_offload =
4455 		    be64_to_cpu(crq->query_capability.number);
4456 		netdev_dbg(netdev, "tcp_ip_offload = %lld\n",
4457 			   adapter->tcp_ip_offload);
4458 		break;
4459 	case PROMISC_SUPPORTED:
4460 		adapter->promisc_supported =
4461 		    be64_to_cpu(crq->query_capability.number);
4462 		netdev_dbg(netdev, "promisc_supported = %lld\n",
4463 			   adapter->promisc_supported);
4464 		break;
4465 	case MIN_MTU:
4466 		adapter->min_mtu = be64_to_cpu(crq->query_capability.number);
4467 		netdev->min_mtu = adapter->min_mtu - ETH_HLEN;
4468 		netdev_dbg(netdev, "min_mtu = %lld\n", adapter->min_mtu);
4469 		break;
4470 	case MAX_MTU:
4471 		adapter->max_mtu = be64_to_cpu(crq->query_capability.number);
4472 		netdev->max_mtu = adapter->max_mtu - ETH_HLEN;
4473 		netdev_dbg(netdev, "max_mtu = %lld\n", adapter->max_mtu);
4474 		break;
4475 	case MAX_MULTICAST_FILTERS:
4476 		adapter->max_multicast_filters =
4477 		    be64_to_cpu(crq->query_capability.number);
4478 		netdev_dbg(netdev, "max_multicast_filters = %lld\n",
4479 			   adapter->max_multicast_filters);
4480 		break;
4481 	case VLAN_HEADER_INSERTION:
4482 		adapter->vlan_header_insertion =
4483 		    be64_to_cpu(crq->query_capability.number);
4484 		if (adapter->vlan_header_insertion)
4485 			netdev->features |= NETIF_F_HW_VLAN_STAG_TX;
4486 		netdev_dbg(netdev, "vlan_header_insertion = %lld\n",
4487 			   adapter->vlan_header_insertion);
4488 		break;
4489 	case RX_VLAN_HEADER_INSERTION:
4490 		adapter->rx_vlan_header_insertion =
4491 		    be64_to_cpu(crq->query_capability.number);
4492 		netdev_dbg(netdev, "rx_vlan_header_insertion = %lld\n",
4493 			   adapter->rx_vlan_header_insertion);
4494 		break;
4495 	case MAX_TX_SG_ENTRIES:
4496 		adapter->max_tx_sg_entries =
4497 		    be64_to_cpu(crq->query_capability.number);
4498 		netdev_dbg(netdev, "max_tx_sg_entries = %lld\n",
4499 			   adapter->max_tx_sg_entries);
4500 		break;
4501 	case RX_SG_SUPPORTED:
4502 		adapter->rx_sg_supported =
4503 		    be64_to_cpu(crq->query_capability.number);
4504 		netdev_dbg(netdev, "rx_sg_supported = %lld\n",
4505 			   adapter->rx_sg_supported);
4506 		break;
4507 	case OPT_TX_COMP_SUB_QUEUES:
4508 		adapter->opt_tx_comp_sub_queues =
4509 		    be64_to_cpu(crq->query_capability.number);
4510 		netdev_dbg(netdev, "opt_tx_comp_sub_queues = %lld\n",
4511 			   adapter->opt_tx_comp_sub_queues);
4512 		break;
4513 	case OPT_RX_COMP_QUEUES:
4514 		adapter->opt_rx_comp_queues =
4515 		    be64_to_cpu(crq->query_capability.number);
4516 		netdev_dbg(netdev, "opt_rx_comp_queues = %lld\n",
4517 			   adapter->opt_rx_comp_queues);
4518 		break;
4519 	case OPT_RX_BUFADD_Q_PER_RX_COMP_Q:
4520 		adapter->opt_rx_bufadd_q_per_rx_comp_q =
4521 		    be64_to_cpu(crq->query_capability.number);
4522 		netdev_dbg(netdev, "opt_rx_bufadd_q_per_rx_comp_q = %lld\n",
4523 			   adapter->opt_rx_bufadd_q_per_rx_comp_q);
4524 		break;
4525 	case OPT_TX_ENTRIES_PER_SUBCRQ:
4526 		adapter->opt_tx_entries_per_subcrq =
4527 		    be64_to_cpu(crq->query_capability.number);
4528 		netdev_dbg(netdev, "opt_tx_entries_per_subcrq = %lld\n",
4529 			   adapter->opt_tx_entries_per_subcrq);
4530 		break;
4531 	case OPT_RXBA_ENTRIES_PER_SUBCRQ:
4532 		adapter->opt_rxba_entries_per_subcrq =
4533 		    be64_to_cpu(crq->query_capability.number);
4534 		netdev_dbg(netdev, "opt_rxba_entries_per_subcrq = %lld\n",
4535 			   adapter->opt_rxba_entries_per_subcrq);
4536 		break;
4537 	case TX_RX_DESC_REQ:
4538 		adapter->tx_rx_desc_req = crq->query_capability.number;
4539 		netdev_dbg(netdev, "tx_rx_desc_req = %llx\n",
4540 			   adapter->tx_rx_desc_req);
4541 		break;
4542 
4543 	default:
4544 		netdev_err(netdev, "Got invalid cap rsp %d\n",
4545 			   crq->query_capability.capability);
4546 	}
4547 
4548 out:
4549 	if (atomic_read(&adapter->running_cap_crqs) == 0) {
4550 		adapter->wait_capability = false;
4551 		ibmvnic_send_req_caps(adapter, 0);
4552 	}
4553 }
4554 
4555 static int send_query_phys_parms(struct ibmvnic_adapter *adapter)
4556 {
4557 	union ibmvnic_crq crq;
4558 	int rc;
4559 
4560 	memset(&crq, 0, sizeof(crq));
4561 	crq.query_phys_parms.first = IBMVNIC_CRQ_CMD;
4562 	crq.query_phys_parms.cmd = QUERY_PHYS_PARMS;
4563 
4564 	mutex_lock(&adapter->fw_lock);
4565 	adapter->fw_done_rc = 0;
4566 	reinit_completion(&adapter->fw_done);
4567 
4568 	rc = ibmvnic_send_crq(adapter, &crq);
4569 	if (rc) {
4570 		mutex_unlock(&adapter->fw_lock);
4571 		return rc;
4572 	}
4573 
4574 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);
4575 	if (rc) {
4576 		mutex_unlock(&adapter->fw_lock);
4577 		return rc;
4578 	}
4579 
4580 	mutex_unlock(&adapter->fw_lock);
4581 	return adapter->fw_done_rc ? -EIO : 0;
4582 }
4583 
4584 static int handle_query_phys_parms_rsp(union ibmvnic_crq *crq,
4585 				       struct ibmvnic_adapter *adapter)
4586 {
4587 	struct net_device *netdev = adapter->netdev;
4588 	int rc;
4589 	__be32 rspeed = cpu_to_be32(crq->query_phys_parms_rsp.speed);
4590 
4591 	rc = crq->query_phys_parms_rsp.rc.code;
4592 	if (rc) {
4593 		netdev_err(netdev, "Error %d in QUERY_PHYS_PARMS\n", rc);
4594 		return rc;
4595 	}
4596 	switch (rspeed) {
4597 	case IBMVNIC_10MBPS:
4598 		adapter->speed = SPEED_10;
4599 		break;
4600 	case IBMVNIC_100MBPS:
4601 		adapter->speed = SPEED_100;
4602 		break;
4603 	case IBMVNIC_1GBPS:
4604 		adapter->speed = SPEED_1000;
4605 		break;
4606 	case IBMVNIC_10GBP:
4607 		adapter->speed = SPEED_10000;
4608 		break;
4609 	case IBMVNIC_25GBPS:
4610 		adapter->speed = SPEED_25000;
4611 		break;
4612 	case IBMVNIC_40GBPS:
4613 		adapter->speed = SPEED_40000;
4614 		break;
4615 	case IBMVNIC_50GBPS:
4616 		adapter->speed = SPEED_50000;
4617 		break;
4618 	case IBMVNIC_100GBPS:
4619 		adapter->speed = SPEED_100000;
4620 		break;
4621 	default:
4622 		if (netif_carrier_ok(netdev))
4623 			netdev_warn(netdev, "Unknown speed 0x%08x\n", rspeed);
4624 		adapter->speed = SPEED_UNKNOWN;
4625 	}
4626 	if (crq->query_phys_parms_rsp.flags1 & IBMVNIC_FULL_DUPLEX)
4627 		adapter->duplex = DUPLEX_FULL;
4628 	else if (crq->query_phys_parms_rsp.flags1 & IBMVNIC_HALF_DUPLEX)
4629 		adapter->duplex = DUPLEX_HALF;
4630 	else
4631 		adapter->duplex = DUPLEX_UNKNOWN;
4632 
4633 	return rc;
4634 }
4635 
4636 static void ibmvnic_handle_crq(union ibmvnic_crq *crq,
4637 			       struct ibmvnic_adapter *adapter)
4638 {
4639 	struct ibmvnic_generic_crq *gen_crq = &crq->generic;
4640 	struct net_device *netdev = adapter->netdev;
4641 	struct device *dev = &adapter->vdev->dev;
4642 	u64 *u64_crq = (u64 *)crq;
4643 	long rc;
4644 
4645 	netdev_dbg(netdev, "Handling CRQ: %016lx %016lx\n",
4646 		   (unsigned long int)cpu_to_be64(u64_crq[0]),
4647 		   (unsigned long int)cpu_to_be64(u64_crq[1]));
4648 	switch (gen_crq->first) {
4649 	case IBMVNIC_CRQ_INIT_RSP:
4650 		switch (gen_crq->cmd) {
4651 		case IBMVNIC_CRQ_INIT:
4652 			dev_info(dev, "Partner initialized\n");
4653 			adapter->from_passive_init = true;
4654 			adapter->failover_pending = false;
4655 			if (!completion_done(&adapter->init_done)) {
4656 				complete(&adapter->init_done);
4657 				adapter->init_done_rc = -EIO;
4658 			}
4659 			ibmvnic_reset(adapter, VNIC_RESET_FAILOVER);
4660 			break;
4661 		case IBMVNIC_CRQ_INIT_COMPLETE:
4662 			dev_info(dev, "Partner initialization complete\n");
4663 			adapter->crq.active = true;
4664 			send_version_xchg(adapter);
4665 			break;
4666 		default:
4667 			dev_err(dev, "Unknown crq cmd: %d\n", gen_crq->cmd);
4668 		}
4669 		return;
4670 	case IBMVNIC_CRQ_XPORT_EVENT:
4671 		netif_carrier_off(netdev);
4672 		adapter->crq.active = false;
4673 		/* terminate any thread waiting for a response
4674 		 * from the device
4675 		 */
4676 		if (!completion_done(&adapter->fw_done)) {
4677 			adapter->fw_done_rc = -EIO;
4678 			complete(&adapter->fw_done);
4679 		}
4680 		if (!completion_done(&adapter->stats_done))
4681 			complete(&adapter->stats_done);
4682 		if (test_bit(0, &adapter->resetting))
4683 			adapter->force_reset_recovery = true;
4684 		if (gen_crq->cmd == IBMVNIC_PARTITION_MIGRATED) {
4685 			dev_info(dev, "Migrated, re-enabling adapter\n");
4686 			ibmvnic_reset(adapter, VNIC_RESET_MOBILITY);
4687 		} else if (gen_crq->cmd == IBMVNIC_DEVICE_FAILOVER) {
4688 			dev_info(dev, "Backing device failover detected\n");
4689 			adapter->failover_pending = true;
4690 		} else {
4691 			/* The adapter lost the connection */
4692 			dev_err(dev, "Virtual Adapter failed (rc=%d)\n",
4693 				gen_crq->cmd);
4694 			ibmvnic_reset(adapter, VNIC_RESET_FATAL);
4695 		}
4696 		return;
4697 	case IBMVNIC_CRQ_CMD_RSP:
4698 		break;
4699 	default:
4700 		dev_err(dev, "Got an invalid msg type 0x%02x\n",
4701 			gen_crq->first);
4702 		return;
4703 	}
4704 
4705 	switch (gen_crq->cmd) {
4706 	case VERSION_EXCHANGE_RSP:
4707 		rc = crq->version_exchange_rsp.rc.code;
4708 		if (rc) {
4709 			dev_err(dev, "Error %ld in VERSION_EXCHG_RSP\n", rc);
4710 			break;
4711 		}
4712 		ibmvnic_version =
4713 			    be16_to_cpu(crq->version_exchange_rsp.version);
4714 		dev_info(dev, "Partner protocol version is %d\n",
4715 			 ibmvnic_version);
4716 		send_cap_queries(adapter);
4717 		break;
4718 	case QUERY_CAPABILITY_RSP:
4719 		handle_query_cap_rsp(crq, adapter);
4720 		break;
4721 	case QUERY_MAP_RSP:
4722 		handle_query_map_rsp(crq, adapter);
4723 		break;
4724 	case REQUEST_MAP_RSP:
4725 		adapter->fw_done_rc = crq->request_map_rsp.rc.code;
4726 		complete(&adapter->fw_done);
4727 		break;
4728 	case REQUEST_UNMAP_RSP:
4729 		handle_request_unmap_rsp(crq, adapter);
4730 		break;
4731 	case REQUEST_CAPABILITY_RSP:
4732 		handle_request_cap_rsp(crq, adapter);
4733 		break;
4734 	case LOGIN_RSP:
4735 		netdev_dbg(netdev, "Got Login Response\n");
4736 		handle_login_rsp(crq, adapter);
4737 		break;
4738 	case LOGICAL_LINK_STATE_RSP:
4739 		netdev_dbg(netdev,
4740 			   "Got Logical Link State Response, state: %d rc: %d\n",
4741 			   crq->logical_link_state_rsp.link_state,
4742 			   crq->logical_link_state_rsp.rc.code);
4743 		adapter->logical_link_state =
4744 		    crq->logical_link_state_rsp.link_state;
4745 		adapter->init_done_rc = crq->logical_link_state_rsp.rc.code;
4746 		complete(&adapter->init_done);
4747 		break;
4748 	case LINK_STATE_INDICATION:
4749 		netdev_dbg(netdev, "Got Logical Link State Indication\n");
4750 		adapter->phys_link_state =
4751 		    crq->link_state_indication.phys_link_state;
4752 		adapter->logical_link_state =
4753 		    crq->link_state_indication.logical_link_state;
4754 		if (adapter->phys_link_state && adapter->logical_link_state)
4755 			netif_carrier_on(netdev);
4756 		else
4757 			netif_carrier_off(netdev);
4758 		break;
4759 	case CHANGE_MAC_ADDR_RSP:
4760 		netdev_dbg(netdev, "Got MAC address change Response\n");
4761 		adapter->fw_done_rc = handle_change_mac_rsp(crq, adapter);
4762 		break;
4763 	case ERROR_INDICATION:
4764 		netdev_dbg(netdev, "Got Error Indication\n");
4765 		handle_error_indication(crq, adapter);
4766 		break;
4767 	case REQUEST_STATISTICS_RSP:
4768 		netdev_dbg(netdev, "Got Statistics Response\n");
4769 		complete(&adapter->stats_done);
4770 		break;
4771 	case QUERY_IP_OFFLOAD_RSP:
4772 		netdev_dbg(netdev, "Got Query IP offload Response\n");
4773 		handle_query_ip_offload_rsp(adapter);
4774 		break;
4775 	case MULTICAST_CTRL_RSP:
4776 		netdev_dbg(netdev, "Got multicast control Response\n");
4777 		break;
4778 	case CONTROL_IP_OFFLOAD_RSP:
4779 		netdev_dbg(netdev, "Got Control IP offload Response\n");
4780 		dma_unmap_single(dev, adapter->ip_offload_ctrl_tok,
4781 				 sizeof(adapter->ip_offload_ctrl),
4782 				 DMA_TO_DEVICE);
4783 		complete(&adapter->init_done);
4784 		break;
4785 	case COLLECT_FW_TRACE_RSP:
4786 		netdev_dbg(netdev, "Got Collect firmware trace Response\n");
4787 		complete(&adapter->fw_done);
4788 		break;
4789 	case GET_VPD_SIZE_RSP:
4790 		handle_vpd_size_rsp(crq, adapter);
4791 		break;
4792 	case GET_VPD_RSP:
4793 		handle_vpd_rsp(crq, adapter);
4794 		break;
4795 	case QUERY_PHYS_PARMS_RSP:
4796 		adapter->fw_done_rc = handle_query_phys_parms_rsp(crq, adapter);
4797 		complete(&adapter->fw_done);
4798 		break;
4799 	default:
4800 		netdev_err(netdev, "Got an invalid cmd type 0x%02x\n",
4801 			   gen_crq->cmd);
4802 	}
4803 }
4804 
4805 static irqreturn_t ibmvnic_interrupt(int irq, void *instance)
4806 {
4807 	struct ibmvnic_adapter *adapter = instance;
4808 
4809 	tasklet_schedule(&adapter->tasklet);
4810 	return IRQ_HANDLED;
4811 }
4812 
4813 static void ibmvnic_tasklet(void *data)
4814 {
4815 	struct ibmvnic_adapter *adapter = data;
4816 	struct ibmvnic_crq_queue *queue = &adapter->crq;
4817 	union ibmvnic_crq *crq;
4818 	unsigned long flags;
4819 	bool done = false;
4820 
4821 	spin_lock_irqsave(&queue->lock, flags);
4822 	while (!done) {
4823 		/* Pull all the valid messages off the CRQ */
4824 		while ((crq = ibmvnic_next_crq(adapter)) != NULL) {
4825 			ibmvnic_handle_crq(crq, adapter);
4826 			crq->generic.first = 0;
4827 		}
4828 
4829 		/* remain in tasklet until all
4830 		 * capabilities responses are received
4831 		 */
4832 		if (!adapter->wait_capability)
4833 			done = true;
4834 	}
4835 	/* if capabilities CRQ's were sent in this tasklet, the following
4836 	 * tasklet must wait until all responses are received
4837 	 */
4838 	if (atomic_read(&adapter->running_cap_crqs) != 0)
4839 		adapter->wait_capability = true;
4840 	spin_unlock_irqrestore(&queue->lock, flags);
4841 }
4842 
4843 static int ibmvnic_reenable_crq_queue(struct ibmvnic_adapter *adapter)
4844 {
4845 	struct vio_dev *vdev = adapter->vdev;
4846 	int rc;
4847 
4848 	do {
4849 		rc = plpar_hcall_norets(H_ENABLE_CRQ, vdev->unit_address);
4850 	} while (rc == H_IN_PROGRESS || rc == H_BUSY || H_IS_LONG_BUSY(rc));
4851 
4852 	if (rc)
4853 		dev_err(&vdev->dev, "Error enabling adapter (rc=%d)\n", rc);
4854 
4855 	return rc;
4856 }
4857 
4858 static int ibmvnic_reset_crq(struct ibmvnic_adapter *adapter)
4859 {
4860 	struct ibmvnic_crq_queue *crq = &adapter->crq;
4861 	struct device *dev = &adapter->vdev->dev;
4862 	struct vio_dev *vdev = adapter->vdev;
4863 	int rc;
4864 
4865 	/* Close the CRQ */
4866 	do {
4867 		rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);
4868 	} while (rc == H_BUSY || H_IS_LONG_BUSY(rc));
4869 
4870 	/* Clean out the queue */
4871 	memset(crq->msgs, 0, PAGE_SIZE);
4872 	crq->cur = 0;
4873 	crq->active = false;
4874 
4875 	/* And re-open it again */
4876 	rc = plpar_hcall_norets(H_REG_CRQ, vdev->unit_address,
4877 				crq->msg_token, PAGE_SIZE);
4878 
4879 	if (rc == H_CLOSED)
4880 		/* Adapter is good, but other end is not ready */
4881 		dev_warn(dev, "Partner adapter not ready\n");
4882 	else if (rc != 0)
4883 		dev_warn(dev, "Couldn't register crq (rc=%d)\n", rc);
4884 
4885 	return rc;
4886 }
4887 
4888 static void release_crq_queue(struct ibmvnic_adapter *adapter)
4889 {
4890 	struct ibmvnic_crq_queue *crq = &adapter->crq;
4891 	struct vio_dev *vdev = adapter->vdev;
4892 	long rc;
4893 
4894 	if (!crq->msgs)
4895 		return;
4896 
4897 	netdev_dbg(adapter->netdev, "Releasing CRQ\n");
4898 	free_irq(vdev->irq, adapter);
4899 	tasklet_kill(&adapter->tasklet);
4900 	do {
4901 		rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);
4902 	} while (rc == H_BUSY || H_IS_LONG_BUSY(rc));
4903 
4904 	dma_unmap_single(&vdev->dev, crq->msg_token, PAGE_SIZE,
4905 			 DMA_BIDIRECTIONAL);
4906 	free_page((unsigned long)crq->msgs);
4907 	crq->msgs = NULL;
4908 	crq->active = false;
4909 }
4910 
4911 static int init_crq_queue(struct ibmvnic_adapter *adapter)
4912 {
4913 	struct ibmvnic_crq_queue *crq = &adapter->crq;
4914 	struct device *dev = &adapter->vdev->dev;
4915 	struct vio_dev *vdev = adapter->vdev;
4916 	int rc, retrc = -ENOMEM;
4917 
4918 	if (crq->msgs)
4919 		return 0;
4920 
4921 	crq->msgs = (union ibmvnic_crq *)get_zeroed_page(GFP_KERNEL);
4922 	/* Should we allocate more than one page? */
4923 
4924 	if (!crq->msgs)
4925 		return -ENOMEM;
4926 
4927 	crq->size = PAGE_SIZE / sizeof(*crq->msgs);
4928 	crq->msg_token = dma_map_single(dev, crq->msgs, PAGE_SIZE,
4929 					DMA_BIDIRECTIONAL);
4930 	if (dma_mapping_error(dev, crq->msg_token))
4931 		goto map_failed;
4932 
4933 	rc = plpar_hcall_norets(H_REG_CRQ, vdev->unit_address,
4934 				crq->msg_token, PAGE_SIZE);
4935 
4936 	if (rc == H_RESOURCE)
4937 		/* maybe kexecing and resource is busy. try a reset */
4938 		rc = ibmvnic_reset_crq(adapter);
4939 	retrc = rc;
4940 
4941 	if (rc == H_CLOSED) {
4942 		dev_warn(dev, "Partner adapter not ready\n");
4943 	} else if (rc) {
4944 		dev_warn(dev, "Error %d opening adapter\n", rc);
4945 		goto reg_crq_failed;
4946 	}
4947 
4948 	retrc = 0;
4949 
4950 	tasklet_init(&adapter->tasklet, (void *)ibmvnic_tasklet,
4951 		     (unsigned long)adapter);
4952 
4953 	netdev_dbg(adapter->netdev, "registering irq 0x%x\n", vdev->irq);
4954 	snprintf(crq->name, sizeof(crq->name), "ibmvnic-%x",
4955 		 adapter->vdev->unit_address);
4956 	rc = request_irq(vdev->irq, ibmvnic_interrupt, 0, crq->name, adapter);
4957 	if (rc) {
4958 		dev_err(dev, "Couldn't register irq 0x%x. rc=%d\n",
4959 			vdev->irq, rc);
4960 		goto req_irq_failed;
4961 	}
4962 
4963 	rc = vio_enable_interrupts(vdev);
4964 	if (rc) {
4965 		dev_err(dev, "Error %d enabling interrupts\n", rc);
4966 		goto req_irq_failed;
4967 	}
4968 
4969 	crq->cur = 0;
4970 	spin_lock_init(&crq->lock);
4971 
4972 	return retrc;
4973 
4974 req_irq_failed:
4975 	tasklet_kill(&adapter->tasklet);
4976 	do {
4977 		rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);
4978 	} while (rc == H_BUSY || H_IS_LONG_BUSY(rc));
4979 reg_crq_failed:
4980 	dma_unmap_single(dev, crq->msg_token, PAGE_SIZE, DMA_BIDIRECTIONAL);
4981 map_failed:
4982 	free_page((unsigned long)crq->msgs);
4983 	crq->msgs = NULL;
4984 	return retrc;
4985 }
4986 
4987 static int ibmvnic_reset_init(struct ibmvnic_adapter *adapter)
4988 {
4989 	struct device *dev = &adapter->vdev->dev;
4990 	unsigned long timeout = msecs_to_jiffies(30000);
4991 	u64 old_num_rx_queues, old_num_tx_queues;
4992 	int rc;
4993 
4994 	adapter->from_passive_init = false;
4995 
4996 	old_num_rx_queues = adapter->req_rx_queues;
4997 	old_num_tx_queues = adapter->req_tx_queues;
4998 
4999 	reinit_completion(&adapter->init_done);
5000 	adapter->init_done_rc = 0;
5001 	ibmvnic_send_crq_init(adapter);
5002 	if (!wait_for_completion_timeout(&adapter->init_done, timeout)) {
5003 		dev_err(dev, "Initialization sequence timed out\n");
5004 		return -1;
5005 	}
5006 
5007 	if (adapter->init_done_rc) {
5008 		release_crq_queue(adapter);
5009 		return adapter->init_done_rc;
5010 	}
5011 
5012 	if (adapter->from_passive_init) {
5013 		adapter->state = VNIC_OPEN;
5014 		adapter->from_passive_init = false;
5015 		return -1;
5016 	}
5017 
5018 	if (test_bit(0, &adapter->resetting) && !adapter->wait_for_reset &&
5019 	    adapter->reset_reason != VNIC_RESET_MOBILITY) {
5020 		if (adapter->req_rx_queues != old_num_rx_queues ||
5021 		    adapter->req_tx_queues != old_num_tx_queues) {
5022 			release_sub_crqs(adapter, 0);
5023 			rc = init_sub_crqs(adapter);
5024 		} else {
5025 			rc = reset_sub_crq_queues(adapter);
5026 		}
5027 	} else {
5028 		rc = init_sub_crqs(adapter);
5029 	}
5030 
5031 	if (rc) {
5032 		dev_err(dev, "Initialization of sub crqs failed\n");
5033 		release_crq_queue(adapter);
5034 		return rc;
5035 	}
5036 
5037 	rc = init_sub_crq_irqs(adapter);
5038 	if (rc) {
5039 		dev_err(dev, "Failed to initialize sub crq irqs\n");
5040 		release_crq_queue(adapter);
5041 	}
5042 
5043 	return rc;
5044 }
5045 
5046 static int ibmvnic_init(struct ibmvnic_adapter *adapter)
5047 {
5048 	struct device *dev = &adapter->vdev->dev;
5049 	unsigned long timeout = msecs_to_jiffies(30000);
5050 	int rc;
5051 
5052 	adapter->from_passive_init = false;
5053 
5054 	adapter->init_done_rc = 0;
5055 	ibmvnic_send_crq_init(adapter);
5056 	if (!wait_for_completion_timeout(&adapter->init_done, timeout)) {
5057 		dev_err(dev, "Initialization sequence timed out\n");
5058 		return -1;
5059 	}
5060 
5061 	if (adapter->init_done_rc) {
5062 		release_crq_queue(adapter);
5063 		return adapter->init_done_rc;
5064 	}
5065 
5066 	if (adapter->from_passive_init) {
5067 		adapter->state = VNIC_OPEN;
5068 		adapter->from_passive_init = false;
5069 		return -1;
5070 	}
5071 
5072 	rc = init_sub_crqs(adapter);
5073 	if (rc) {
5074 		dev_err(dev, "Initialization of sub crqs failed\n");
5075 		release_crq_queue(adapter);
5076 		return rc;
5077 	}
5078 
5079 	rc = init_sub_crq_irqs(adapter);
5080 	if (rc) {
5081 		dev_err(dev, "Failed to initialize sub crq irqs\n");
5082 		release_crq_queue(adapter);
5083 	}
5084 
5085 	return rc;
5086 }
5087 
5088 static struct device_attribute dev_attr_failover;
5089 
5090 static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id)
5091 {
5092 	struct ibmvnic_adapter *adapter;
5093 	struct net_device *netdev;
5094 	unsigned char *mac_addr_p;
5095 	int rc;
5096 
5097 	dev_dbg(&dev->dev, "entering ibmvnic_probe for UA 0x%x\n",
5098 		dev->unit_address);
5099 
5100 	mac_addr_p = (unsigned char *)vio_get_attribute(dev,
5101 							VETH_MAC_ADDR, NULL);
5102 	if (!mac_addr_p) {
5103 		dev_err(&dev->dev,
5104 			"(%s:%3.3d) ERROR: Can't find MAC_ADDR attribute\n",
5105 			__FILE__, __LINE__);
5106 		return 0;
5107 	}
5108 
5109 	netdev = alloc_etherdev_mq(sizeof(struct ibmvnic_adapter),
5110 				   IBMVNIC_MAX_QUEUES);
5111 	if (!netdev)
5112 		return -ENOMEM;
5113 
5114 	adapter = netdev_priv(netdev);
5115 	adapter->state = VNIC_PROBING;
5116 	dev_set_drvdata(&dev->dev, netdev);
5117 	adapter->vdev = dev;
5118 	adapter->netdev = netdev;
5119 
5120 	ether_addr_copy(adapter->mac_addr, mac_addr_p);
5121 	ether_addr_copy(netdev->dev_addr, adapter->mac_addr);
5122 	netdev->irq = dev->irq;
5123 	netdev->netdev_ops = &ibmvnic_netdev_ops;
5124 	netdev->ethtool_ops = &ibmvnic_ethtool_ops;
5125 	SET_NETDEV_DEV(netdev, &dev->dev);
5126 
5127 	spin_lock_init(&adapter->stats_lock);
5128 
5129 	INIT_WORK(&adapter->ibmvnic_reset, __ibmvnic_reset);
5130 	INIT_DELAYED_WORK(&adapter->ibmvnic_delayed_reset,
5131 			  __ibmvnic_delayed_reset);
5132 	INIT_LIST_HEAD(&adapter->rwi_list);
5133 	spin_lock_init(&adapter->rwi_lock);
5134 	spin_lock_init(&adapter->state_lock);
5135 	mutex_init(&adapter->fw_lock);
5136 	init_completion(&adapter->init_done);
5137 	init_completion(&adapter->fw_done);
5138 	init_completion(&adapter->reset_done);
5139 	init_completion(&adapter->stats_done);
5140 	clear_bit(0, &adapter->resetting);
5141 
5142 	do {
5143 		rc = init_crq_queue(adapter);
5144 		if (rc) {
5145 			dev_err(&dev->dev, "Couldn't initialize crq. rc=%d\n",
5146 				rc);
5147 			goto ibmvnic_init_fail;
5148 		}
5149 
5150 		rc = ibmvnic_init(adapter);
5151 		if (rc && rc != EAGAIN)
5152 			goto ibmvnic_init_fail;
5153 	} while (rc == EAGAIN);
5154 
5155 	rc = init_stats_buffers(adapter);
5156 	if (rc)
5157 		goto ibmvnic_init_fail;
5158 
5159 	rc = init_stats_token(adapter);
5160 	if (rc)
5161 		goto ibmvnic_stats_fail;
5162 
5163 	netdev->mtu = adapter->req_mtu - ETH_HLEN;
5164 	netdev->min_mtu = adapter->min_mtu - ETH_HLEN;
5165 	netdev->max_mtu = adapter->max_mtu - ETH_HLEN;
5166 
5167 	rc = device_create_file(&dev->dev, &dev_attr_failover);
5168 	if (rc)
5169 		goto ibmvnic_dev_file_err;
5170 
5171 	netif_carrier_off(netdev);
5172 	rc = register_netdev(netdev);
5173 	if (rc) {
5174 		dev_err(&dev->dev, "failed to register netdev rc=%d\n", rc);
5175 		goto ibmvnic_register_fail;
5176 	}
5177 	dev_info(&dev->dev, "ibmvnic registered\n");
5178 
5179 	adapter->state = VNIC_PROBED;
5180 
5181 	adapter->wait_for_reset = false;
5182 
5183 	return 0;
5184 
5185 ibmvnic_register_fail:
5186 	device_remove_file(&dev->dev, &dev_attr_failover);
5187 
5188 ibmvnic_dev_file_err:
5189 	release_stats_token(adapter);
5190 
5191 ibmvnic_stats_fail:
5192 	release_stats_buffers(adapter);
5193 
5194 ibmvnic_init_fail:
5195 	release_sub_crqs(adapter, 1);
5196 	release_crq_queue(adapter);
5197 	mutex_destroy(&adapter->fw_lock);
5198 	free_netdev(netdev);
5199 
5200 	return rc;
5201 }
5202 
5203 static int ibmvnic_remove(struct vio_dev *dev)
5204 {
5205 	struct net_device *netdev = dev_get_drvdata(&dev->dev);
5206 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
5207 	unsigned long flags;
5208 
5209 	spin_lock_irqsave(&adapter->state_lock, flags);
5210 	if (adapter->state == VNIC_RESETTING) {
5211 		spin_unlock_irqrestore(&adapter->state_lock, flags);
5212 		return -EBUSY;
5213 	}
5214 
5215 	adapter->state = VNIC_REMOVING;
5216 	spin_unlock_irqrestore(&adapter->state_lock, flags);
5217 
5218 	flush_work(&adapter->ibmvnic_reset);
5219 	flush_delayed_work(&adapter->ibmvnic_delayed_reset);
5220 
5221 	rtnl_lock();
5222 	unregister_netdevice(netdev);
5223 
5224 	release_resources(adapter);
5225 	release_sub_crqs(adapter, 1);
5226 	release_crq_queue(adapter);
5227 
5228 	release_stats_token(adapter);
5229 	release_stats_buffers(adapter);
5230 
5231 	adapter->state = VNIC_REMOVED;
5232 
5233 	rtnl_unlock();
5234 	mutex_destroy(&adapter->fw_lock);
5235 	device_remove_file(&dev->dev, &dev_attr_failover);
5236 	free_netdev(netdev);
5237 	dev_set_drvdata(&dev->dev, NULL);
5238 
5239 	return 0;
5240 }
5241 
5242 static ssize_t failover_store(struct device *dev, struct device_attribute *attr,
5243 			      const char *buf, size_t count)
5244 {
5245 	struct net_device *netdev = dev_get_drvdata(dev);
5246 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
5247 	unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
5248 	__be64 session_token;
5249 	long rc;
5250 
5251 	if (!sysfs_streq(buf, "1"))
5252 		return -EINVAL;
5253 
5254 	rc = plpar_hcall(H_VIOCTL, retbuf, adapter->vdev->unit_address,
5255 			 H_GET_SESSION_TOKEN, 0, 0, 0);
5256 	if (rc) {
5257 		netdev_err(netdev, "Couldn't retrieve session token, rc %ld\n",
5258 			   rc);
5259 		return -EINVAL;
5260 	}
5261 
5262 	session_token = (__be64)retbuf[0];
5263 	netdev_dbg(netdev, "Initiating client failover, session id %llx\n",
5264 		   be64_to_cpu(session_token));
5265 	rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,
5266 				H_SESSION_ERR_DETECTED, session_token, 0, 0);
5267 	if (rc) {
5268 		netdev_err(netdev, "Client initiated failover failed, rc %ld\n",
5269 			   rc);
5270 		return -EINVAL;
5271 	}
5272 
5273 	return count;
5274 }
5275 
5276 static DEVICE_ATTR_WO(failover);
5277 
5278 static unsigned long ibmvnic_get_desired_dma(struct vio_dev *vdev)
5279 {
5280 	struct net_device *netdev = dev_get_drvdata(&vdev->dev);
5281 	struct ibmvnic_adapter *adapter;
5282 	struct iommu_table *tbl;
5283 	unsigned long ret = 0;
5284 	int i;
5285 
5286 	tbl = get_iommu_table_base(&vdev->dev);
5287 
5288 	/* netdev inits at probe time along with the structures we need below*/
5289 	if (!netdev)
5290 		return IOMMU_PAGE_ALIGN(IBMVNIC_IO_ENTITLEMENT_DEFAULT, tbl);
5291 
5292 	adapter = netdev_priv(netdev);
5293 
5294 	ret += PAGE_SIZE; /* the crq message queue */
5295 	ret += IOMMU_PAGE_ALIGN(sizeof(struct ibmvnic_statistics), tbl);
5296 
5297 	for (i = 0; i < adapter->req_tx_queues + adapter->req_rx_queues; i++)
5298 		ret += 4 * PAGE_SIZE; /* the scrq message queue */
5299 
5300 	for (i = 0; i < be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);
5301 	     i++)
5302 		ret += adapter->rx_pool[i].size *
5303 		    IOMMU_PAGE_ALIGN(adapter->rx_pool[i].buff_size, tbl);
5304 
5305 	return ret;
5306 }
5307 
5308 static int ibmvnic_resume(struct device *dev)
5309 {
5310 	struct net_device *netdev = dev_get_drvdata(dev);
5311 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
5312 
5313 	if (adapter->state != VNIC_OPEN)
5314 		return 0;
5315 
5316 	tasklet_schedule(&adapter->tasklet);
5317 
5318 	return 0;
5319 }
5320 
5321 static const struct vio_device_id ibmvnic_device_table[] = {
5322 	{"network", "IBM,vnic"},
5323 	{"", "" }
5324 };
5325 MODULE_DEVICE_TABLE(vio, ibmvnic_device_table);
5326 
5327 static const struct dev_pm_ops ibmvnic_pm_ops = {
5328 	.resume = ibmvnic_resume
5329 };
5330 
5331 static struct vio_driver ibmvnic_driver = {
5332 	.id_table       = ibmvnic_device_table,
5333 	.probe          = ibmvnic_probe,
5334 	.remove         = ibmvnic_remove,
5335 	.get_desired_dma = ibmvnic_get_desired_dma,
5336 	.name		= ibmvnic_driver_name,
5337 	.pm		= &ibmvnic_pm_ops,
5338 };
5339 
5340 /* module functions */
5341 static int __init ibmvnic_module_init(void)
5342 {
5343 	pr_info("%s: %s %s\n", ibmvnic_driver_name, ibmvnic_driver_string,
5344 		IBMVNIC_DRIVER_VERSION);
5345 
5346 	return vio_register_driver(&ibmvnic_driver);
5347 }
5348 
5349 static void __exit ibmvnic_module_exit(void)
5350 {
5351 	vio_unregister_driver(&ibmvnic_driver);
5352 }
5353 
5354 module_init(ibmvnic_module_init);
5355 module_exit(ibmvnic_module_exit);
5356