100 static int reset_sub_crq_queues(struct ibmvnic_adapter *adapter);
101 static inline void reinit_init_done(struct ibmvnic_adapter *adapter);
102 static void send_query_map(struct ibmvnic_adapter *adapter);
105 static int send_login(struct ibmvnic_adapter *adapter);
106 static void send_query_cap(struct ibmvnic_adapter *adapter);
108 static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter);
112 static int init_crq_queue(struct ibmvnic_adapter *adapter);
113 static int send_query_phys_parms(struct ibmvnic_adapter *adapter);
114 static void ibmvnic_tx_scrq_clean_buffer(struct ibmvnic_adapter *adapter,
116 static void free_long_term_buff(struct ibmvnic_adapter *adapter,
118 static void ibmvnic_disable_irqs(struct ibmvnic_adapter *adapter);
119 static void flush_reset_queue(struct ibmvnic_adapter *adapter);
155 static int send_crq_init_complete(struct ibmvnic_adapter *adapter)  in send_crq_init_complete()  argument
163 	return ibmvnic_send_crq(adapter, &crq);  in send_crq_init_complete()
166 static int send_version_xchg(struct ibmvnic_adapter *adapter)  in send_version_xchg()  argument
175 	return ibmvnic_send_crq(adapter, &crq);  in send_version_xchg()
178 static void ibmvnic_clean_queue_affinity(struct ibmvnic_adapter *adapter,  in ibmvnic_clean_queue_affinity()  argument
187 		netdev_warn(adapter->netdev,  in ibmvnic_clean_queue_affinity()
192 static void ibmvnic_clean_affinity(struct ibmvnic_adapter *adapter)  in ibmvnic_clean_affinity()  argument
199 	rxqs = adapter->rx_scrq;  in ibmvnic_clean_affinity()
200 	txqs = adapter->tx_scrq;  in ibmvnic_clean_affinity()
201 	num_txqs = adapter->num_active_tx_scrqs;  in ibmvnic_clean_affinity()
202 	num_rxqs = adapter->num_active_rx_scrqs;  in ibmvnic_clean_affinity()
204 	netdev_dbg(adapter->netdev, "%s: Cleaning irq affinity hints", __func__);  in ibmvnic_clean_affinity()
207 			ibmvnic_clean_queue_affinity(adapter, txqs[i]);  in ibmvnic_clean_affinity()
211 			ibmvnic_clean_queue_affinity(adapter, rxqs[i]);  in ibmvnic_clean_affinity()
250 static void ibmvnic_set_affinity(struct ibmvnic_adapter *adapter)  in ibmvnic_set_affinity()  argument
252 	struct ibmvnic_sub_crq_queue **rxqs = adapter->rx_scrq;  in ibmvnic_set_affinity()
253 	struct ibmvnic_sub_crq_queue **txqs = adapter->tx_scrq;  in ibmvnic_set_affinity()
255 	int num_rxqs = adapter->num_active_rx_scrqs, i_rxqs = 0;  in ibmvnic_set_affinity()
256 	int num_txqs = adapter->num_active_tx_scrqs, i_txqs = 0;  in ibmvnic_set_affinity()
262 	netdev_dbg(adapter->netdev, "%s: Setting irq affinity hints", __func__);  in ibmvnic_set_affinity()
263 	if (!(adapter->rx_scrq && adapter->tx_scrq)) {  in ibmvnic_set_affinity()
264 		netdev_warn(adapter->netdev,  in ibmvnic_set_affinity()
299 		rc = __netif_set_xps_queue(adapter->netdev,  in ibmvnic_set_affinity()
303 			netdev_warn(adapter->netdev, "%s: Set XPS on queue %d failed, rc = %d.\n",  in ibmvnic_set_affinity()
309 		netdev_warn(adapter->netdev,  in ibmvnic_set_affinity()
312 		ibmvnic_clean_affinity(adapter);  in ibmvnic_set_affinity()
318 	struct ibmvnic_adapter *adapter;  in ibmvnic_cpu_online()  local
320 	adapter = hlist_entry_safe(node, struct ibmvnic_adapter, node);  in ibmvnic_cpu_online()
321 	ibmvnic_set_affinity(adapter);  in ibmvnic_cpu_online()
327 	struct ibmvnic_adapter *adapter;  in ibmvnic_cpu_dead()  local
329 	adapter = hlist_entry_safe(node, struct ibmvnic_adapter, node_dead);  in ibmvnic_cpu_dead()
330 	ibmvnic_set_affinity(adapter);  in ibmvnic_cpu_dead()
336 	struct ibmvnic_adapter *adapter;  in ibmvnic_cpu_down_prep()  local
338 	adapter = hlist_entry_safe(node, struct ibmvnic_adapter, node);  in ibmvnic_cpu_down_prep()
339 	ibmvnic_clean_affinity(adapter);  in ibmvnic_cpu_down_prep()
345 static int ibmvnic_cpu_notif_add(struct ibmvnic_adapter *adapter)  in ibmvnic_cpu_notif_add()  argument
349 	ret = cpuhp_state_add_instance_nocalls(ibmvnic_online, &adapter->node);  in ibmvnic_cpu_notif_add()
353 					       &adapter->node_dead);  in ibmvnic_cpu_notif_add()
356 	cpuhp_state_remove_instance_nocalls(ibmvnic_online, &adapter->node);  in ibmvnic_cpu_notif_add()
360 static void ibmvnic_cpu_notif_remove(struct ibmvnic_adapter *adapter)  in ibmvnic_cpu_notif_remove()  argument
362 	cpuhp_state_remove_instance_nocalls(ibmvnic_online, &adapter->node);  in ibmvnic_cpu_notif_remove()
364 					    &adapter->node_dead);  in ibmvnic_cpu_notif_remove()
390 static int ibmvnic_wait_for_completion(struct ibmvnic_adapter *adapter,  in ibmvnic_wait_for_completion()  argument
398 	netdev = adapter->netdev;  in ibmvnic_wait_for_completion()
402 		if (!adapter->crq.active) {  in ibmvnic_wait_for_completion()
450 static int alloc_long_term_buff(struct ibmvnic_adapter *adapter,  in alloc_long_term_buff()  argument
453 	struct device *dev = &adapter->vdev->dev;  in alloc_long_term_buff()
462 		free_long_term_buff(adapter, ltb);  in alloc_long_term_buff()
477 		ltb->map_id = find_first_zero_bit(adapter->map_ids,  in alloc_long_term_buff()
479 		bitmap_set(adapter->map_ids, ltb->map_id, 1);  in alloc_long_term_buff()
489 	mutex_lock(&adapter->fw_lock);  in alloc_long_term_buff()
490 	adapter->fw_done_rc = 0;  in alloc_long_term_buff()
491 	reinit_completion(&adapter->fw_done);  in alloc_long_term_buff()
493 	rc = send_request_map(adapter, ltb->addr, ltb->size, ltb->map_id);  in alloc_long_term_buff()
499 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);  in alloc_long_term_buff()
506 	if (adapter->fw_done_rc) {  in alloc_long_term_buff()
508 			adapter->fw_done_rc);  in alloc_long_term_buff()
515 	mutex_unlock(&adapter->fw_lock);  in alloc_long_term_buff()
519 static void free_long_term_buff(struct ibmvnic_adapter *adapter,  in free_long_term_buff()  argument
522 	struct device *dev = &adapter->vdev->dev;  in free_long_term_buff()
531 	if (adapter->reset_reason != VNIC_RESET_FAILOVER &&  in free_long_term_buff()
532 	    adapter->reset_reason != VNIC_RESET_MOBILITY &&  in free_long_term_buff()
533 	    adapter->reset_reason != VNIC_RESET_TIMEOUT)  in free_long_term_buff()
534 		send_request_unmap(adapter, ltb->map_id);  in free_long_term_buff()
540 	bitmap_clear(adapter->map_ids, ltb->map_id, 1);  in free_long_term_buff()
552 static void free_ltb_set(struct ibmvnic_adapter *adapter,  in free_ltb_set()  argument
558 		free_long_term_buff(adapter, &ltb_set->ltbs[i]);  in free_ltb_set()
582 static int alloc_ltb_set(struct ibmvnic_adapter *adapter,  in alloc_ltb_set()  argument
586 	struct device *dev = &adapter->vdev->dev;  in alloc_ltb_set()
625 			free_long_term_buff(adapter, &old_set.ltbs[i]);  in alloc_ltb_set()
662 		rc = alloc_long_term_buff(adapter, &new_set.ltbs[i], ltb_size);  in alloc_ltb_set()
746 static void deactivate_rx_pools(struct ibmvnic_adapter *adapter)  in deactivate_rx_pools()  argument
750 	for (i = 0; i < adapter->num_active_rx_pools; i++)  in deactivate_rx_pools()
751 		adapter->rx_pool[i].active = 0;  in deactivate_rx_pools()
754 static void replenish_rx_pool(struct ibmvnic_adapter *adapter,  in replenish_rx_pool()  argument
758 	u64 handle = adapter->rx_scrq[pool->index]->handle;  in replenish_rx_pool()
759 	struct device *dev = &adapter->vdev->dev;  in replenish_rx_pool()
777 	rx_scrq = adapter->rx_scrq[pool->index];  in replenish_rx_pool()
796 			skb = netdev_alloc_skb(adapter->netdev,  in replenish_rx_pool()
800 				adapter->replenish_no_mem++;  in replenish_rx_pool()
844 				send_subcrq_indirect(adapter, handle,  in replenish_rx_pool()
850 			adapter->replenish_add_buff_success += ind_bufp->index;  in replenish_rx_pool()
873 	adapter->replenish_add_buff_failure += ind_bufp->index;  in replenish_rx_pool()
876 	if (lpar_rc == H_CLOSED || adapter->failover_pending) {  in replenish_rx_pool()
882 		deactivate_rx_pools(adapter);  in replenish_rx_pool()
883 		netif_carrier_off(adapter->netdev);  in replenish_rx_pool()
887 static void replenish_pools(struct ibmvnic_adapter *adapter)  in replenish_pools()  argument
891 	adapter->replenish_task_cycles++;  in replenish_pools()
892 	for (i = 0; i < adapter->num_active_rx_pools; i++) {  in replenish_pools()
893 		if (adapter->rx_pool[i].active)  in replenish_pools()
894 			replenish_rx_pool(adapter, &adapter->rx_pool[i]);  in replenish_pools()
897 	netdev_dbg(adapter->netdev, "Replenished %d pools\n", i);  in replenish_pools()
900 static void release_stats_buffers(struct ibmvnic_adapter *adapter)  in release_stats_buffers()  argument
902 	kfree(adapter->tx_stats_buffers);  in release_stats_buffers()
903 	kfree(adapter->rx_stats_buffers);  in release_stats_buffers()
904 	adapter->tx_stats_buffers = NULL;  in release_stats_buffers()
905 	adapter->rx_stats_buffers = NULL;  in release_stats_buffers()
908 static int init_stats_buffers(struct ibmvnic_adapter *adapter)  in init_stats_buffers()  argument
910 	adapter->tx_stats_buffers =  in init_stats_buffers()
914 	if (!adapter->tx_stats_buffers)  in init_stats_buffers()
917 	adapter->rx_stats_buffers =  in init_stats_buffers()
921 	if (!adapter->rx_stats_buffers)  in init_stats_buffers()
927 static void release_stats_token(struct ibmvnic_adapter *adapter)  in release_stats_token()  argument
929 	struct device *dev = &adapter->vdev->dev;  in release_stats_token()
931 	if (!adapter->stats_token)  in release_stats_token()
934 	dma_unmap_single(dev, adapter->stats_token,  in release_stats_token()
937 	adapter->stats_token = 0;  in release_stats_token()
940 static int init_stats_token(struct ibmvnic_adapter *adapter)  in init_stats_token()  argument
942 	struct device *dev = &adapter->vdev->dev;  in init_stats_token()
946 	stok = dma_map_single(dev, &adapter->stats,  in init_stats_token()
955 	adapter->stats_token = stok;  in init_stats_token()
956 	netdev_dbg(adapter->netdev, "Stats token initialized (%llx)\n", stok);  in init_stats_token()
966 static void release_rx_pools(struct ibmvnic_adapter *adapter)  in release_rx_pools()  argument
971 	if (!adapter->rx_pool)  in release_rx_pools()
974 	for (i = 0; i < adapter->num_active_rx_pools; i++) {  in release_rx_pools()
975 		rx_pool = &adapter->rx_pool[i];  in release_rx_pools()
977 		netdev_dbg(adapter->netdev, "Releasing rx_pool[%d]\n", i);  in release_rx_pools()
981 		free_ltb_set(adapter, &rx_pool->ltb_set);  in release_rx_pools()
996 	kfree(adapter->rx_pool);  in release_rx_pools()
997 	adapter->rx_pool = NULL;  in release_rx_pools()
998 	adapter->num_active_rx_pools = 0;  in release_rx_pools()
999 	adapter->prev_rx_pool_size = 0;  in release_rx_pools()
1016 static bool reuse_rx_pools(struct ibmvnic_adapter *adapter)  in reuse_rx_pools()  argument
1022 	if (!adapter->rx_pool)  in reuse_rx_pools()
1025 	old_num_pools = adapter->num_active_rx_pools;  in reuse_rx_pools()
1026 	new_num_pools = adapter->req_rx_queues;  in reuse_rx_pools()
1028 	old_pool_size = adapter->prev_rx_pool_size;  in reuse_rx_pools()
1029 	new_pool_size = adapter->req_rx_add_entries_per_subcrq;  in reuse_rx_pools()
1031 	old_buff_size = adapter->prev_rx_buf_sz;  in reuse_rx_pools()
1032 	new_buff_size = adapter->cur_rx_buf_sz;  in reuse_rx_pools()
1055 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in init_rx_pools()  local
1056 	struct device *dev = &adapter->vdev->dev;  in init_rx_pools()
1063 	pool_size = adapter->req_rx_add_entries_per_subcrq;  in init_rx_pools()
1064 	num_pools = adapter->req_rx_queues;  in init_rx_pools()
1065 	buff_size = adapter->cur_rx_buf_sz;  in init_rx_pools()
1067 	if (reuse_rx_pools(adapter)) {  in init_rx_pools()
1073 	release_rx_pools(adapter);  in init_rx_pools()
1075 	adapter->rx_pool = kcalloc(num_pools,  in init_rx_pools()
1078 	if (!adapter->rx_pool) {  in init_rx_pools()
1086 	adapter->num_active_rx_pools = num_pools;  in init_rx_pools()
1089 		rx_pool = &adapter->rx_pool[i];  in init_rx_pools()
1091 		netdev_dbg(adapter->netdev,  in init_rx_pools()
1117 	adapter->prev_rx_pool_size = pool_size;  in init_rx_pools()
1118 	adapter->prev_rx_buf_sz = adapter->cur_rx_buf_sz;  in init_rx_pools()
1122 		rx_pool = &adapter->rx_pool[i];  in init_rx_pools()
1126 		rc = alloc_ltb_set(adapter, &rx_pool->ltb_set,  in init_rx_pools()
1160 	release_rx_pools(adapter);  in init_rx_pools()
1168 static void release_vpd_data(struct ibmvnic_adapter *adapter)  in release_vpd_data()  argument
1170 	if (!adapter->vpd)  in release_vpd_data()
1173 	kfree(adapter->vpd->buff);  in release_vpd_data()
1174 	kfree(adapter->vpd);  in release_vpd_data()
1176 	adapter->vpd = NULL;  in release_vpd_data()
1179 static void release_one_tx_pool(struct ibmvnic_adapter *adapter,  in release_one_tx_pool()  argument
1184 	free_ltb_set(adapter, &tx_pool->ltb_set);  in release_one_tx_pool()
1193 static void release_tx_pools(struct ibmvnic_adapter *adapter)  in release_tx_pools()  argument
1200 	if (!adapter->tx_pool)  in release_tx_pools()
1203 	for (i = 0; i < adapter->num_active_tx_pools; i++) {  in release_tx_pools()
1204 		release_one_tx_pool(adapter, &adapter->tx_pool[i]);  in release_tx_pools()
1205 		release_one_tx_pool(adapter, &adapter->tso_pool[i]);  in release_tx_pools()
1208 	kfree(adapter->tx_pool);  in release_tx_pools()
1209 	adapter->tx_pool = NULL;  in release_tx_pools()
1210 	kfree(adapter->tso_pool);  in release_tx_pools()
1211 	adapter->tso_pool = NULL;  in release_tx_pools()
1212 	adapter->num_active_tx_pools = 0;  in release_tx_pools()
1213 	adapter->prev_tx_pool_size = 0;  in release_tx_pools()
1259 static bool reuse_tx_pools(struct ibmvnic_adapter *adapter)  in reuse_tx_pools()  argument
1265 	if (!adapter->tx_pool)  in reuse_tx_pools()
1268 	old_num_pools = adapter->num_active_tx_pools;  in reuse_tx_pools()
1269 	new_num_pools = adapter->num_active_tx_scrqs;  in reuse_tx_pools()
1270 	old_pool_size = adapter->prev_tx_pool_size;  in reuse_tx_pools()
1271 	new_pool_size = adapter->req_tx_entries_per_subcrq;  in reuse_tx_pools()
1272 	old_mtu = adapter->prev_mtu;  in reuse_tx_pools()
1273 	new_mtu = adapter->req_mtu;  in reuse_tx_pools()
1296 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in init_tx_pools()  local
1297 	struct device *dev = &adapter->vdev->dev;  in init_tx_pools()
1303 	num_pools = adapter->req_tx_queues;  in init_tx_pools()
1309 	if (reuse_tx_pools(adapter)) {  in init_tx_pools()
1315 	release_tx_pools(adapter);  in init_tx_pools()
1317 	pool_size = adapter->req_tx_entries_per_subcrq;  in init_tx_pools()
1318 	num_pools = adapter->num_active_tx_scrqs;  in init_tx_pools()
1320 	adapter->tx_pool = kcalloc(num_pools,  in init_tx_pools()
1322 	if (!adapter->tx_pool)  in init_tx_pools()
1325 	adapter->tso_pool = kcalloc(num_pools,  in init_tx_pools()
1330 	if (!adapter->tso_pool) {  in init_tx_pools()
1331 		kfree(adapter->tx_pool);  in init_tx_pools()
1332 		adapter->tx_pool = NULL;  in init_tx_pools()
1339 	adapter->num_active_tx_pools = num_pools;  in init_tx_pools()
1341 	buff_size = adapter->req_mtu + VLAN_HLEN;  in init_tx_pools()
1346 			i, adapter->req_tx_entries_per_subcrq, buff_size);  in init_tx_pools()
1348 		rc = init_one_tx_pool(netdev, &adapter->tx_pool[i],  in init_tx_pools()
1353 		rc = init_one_tx_pool(netdev, &adapter->tso_pool[i],  in init_tx_pools()
1360 	adapter->prev_tx_pool_size = pool_size;  in init_tx_pools()
1361 	adapter->prev_mtu = adapter->req_mtu;  in init_tx_pools()
1375 		tx_pool = &adapter->tx_pool[i];  in init_tx_pools()
1380 		rc = alloc_ltb_set(adapter, &tx_pool->ltb_set,  in init_tx_pools()
1391 		tso_pool = &adapter->tso_pool[i];  in init_tx_pools()
1396 		rc = alloc_ltb_set(adapter, &tso_pool->ltb_set,  in init_tx_pools()
1410 	release_tx_pools(adapter);  in init_tx_pools()
1418 static void ibmvnic_napi_enable(struct ibmvnic_adapter *adapter)  in ibmvnic_napi_enable()  argument
1422 	if (adapter->napi_enabled)  in ibmvnic_napi_enable()
1425 	for (i = 0; i < adapter->req_rx_queues; i++)  in ibmvnic_napi_enable()
1426 		napi_enable(&adapter->napi[i]);  in ibmvnic_napi_enable()
1428 	adapter->napi_enabled = true;  in ibmvnic_napi_enable()
1431 static void ibmvnic_napi_disable(struct ibmvnic_adapter *adapter)  in ibmvnic_napi_disable()  argument
1435 	if (!adapter->napi_enabled)  in ibmvnic_napi_disable()
1438 	for (i = 0; i < adapter->req_rx_queues; i++) {  in ibmvnic_napi_disable()
1439 		netdev_dbg(adapter->netdev, "Disabling napi[%d]\n", i);  in ibmvnic_napi_disable()
1440 		napi_disable(&adapter->napi[i]);  in ibmvnic_napi_disable()
1443 	adapter->napi_enabled = false;  in ibmvnic_napi_disable()
1446 static int init_napi(struct ibmvnic_adapter *adapter)  in init_napi()  argument
1450 	adapter->napi = kcalloc(adapter->req_rx_queues,  in init_napi()
1452 	if (!adapter->napi)  in init_napi()
1455 	for (i = 0; i < adapter->req_rx_queues; i++) {  in init_napi()
1456 		netdev_dbg(adapter->netdev, "Adding napi[%d]\n", i);  in init_napi()
1457 		netif_napi_add(adapter->netdev, &adapter->napi[i],  in init_napi()
1461 	adapter->num_active_rx_napi = adapter->req_rx_queues;  in init_napi()
1465 static void release_napi(struct ibmvnic_adapter *adapter)  in release_napi()  argument
1469 	if (!adapter->napi)  in release_napi()
1472 	for (i = 0; i < adapter->num_active_rx_napi; i++) {  in release_napi()
1473 		netdev_dbg(adapter->netdev, "Releasing napi[%d]\n", i);  in release_napi()
1474 		netif_napi_del(&adapter->napi[i]);  in release_napi()
1477 	kfree(adapter->napi);  in release_napi()
1478 	adapter->napi = NULL;  in release_napi()
1479 	adapter->num_active_rx_napi = 0;  in release_napi()
1480 	adapter->napi_enabled = false;  in release_napi()
1511 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_login()  local
1524 		adapter->init_done_rc = 0;  in ibmvnic_login()
1525 		reinit_completion(&adapter->init_done);  in ibmvnic_login()
1526 		rc = send_login(adapter);  in ibmvnic_login()
1530 		if (!wait_for_completion_timeout(&adapter->init_done,  in ibmvnic_login()
1533 			adapter->login_pending = false;  in ibmvnic_login()
1537 		if (adapter->init_done_rc == ABORTED) {  in ibmvnic_login()
1540 			adapter->init_done_rc = 0;  in ibmvnic_login()
1546 		} else if (adapter->init_done_rc == PARTIALSUCCESS) {  in ibmvnic_login()
1548 			release_sub_crqs(adapter, 1);  in ibmvnic_login()
1553 			adapter->init_done_rc = 0;  in ibmvnic_login()
1554 			reinit_completion(&adapter->init_done);  in ibmvnic_login()
1555 			send_query_cap(adapter);  in ibmvnic_login()
1556 			if (!wait_for_completion_timeout(&adapter->init_done,  in ibmvnic_login()
1563 			rc = init_sub_crqs(adapter);  in ibmvnic_login()
1570 			rc = init_sub_crq_irqs(adapter);  in ibmvnic_login()
1577 		} else if (adapter->init_done_rc) {  in ibmvnic_login()
1579 				    adapter->init_done_rc);  in ibmvnic_login()
1591 			adapter->init_done_rc = 0;  in ibmvnic_login()
1592 			release_sub_crqs(adapter, true);  in ibmvnic_login()
1600 				reinit_init_done(adapter);  in ibmvnic_login()
1604 				adapter->failover_pending = false;  in ibmvnic_login()
1605 				release_crq_queue(adapter);  in ibmvnic_login()
1616 				spin_lock_irqsave(&adapter->rwi_lock, flags);  in ibmvnic_login()
1617 				flush_reset_queue(adapter);  in ibmvnic_login()
1618 				spin_unlock_irqrestore(&adapter->rwi_lock,  in ibmvnic_login()
1621 				rc = init_crq_queue(adapter);  in ibmvnic_login()
1628 				rc = ibmvnic_reset_init(adapter, false);  in ibmvnic_login()
1642 	__ibmvnic_set_mac(netdev, adapter->mac_addr);  in ibmvnic_login()
1644 	netdev_dbg(netdev, "[S:%s] Login succeeded\n", adapter_state_to_string(adapter->state));  in ibmvnic_login()
1648 static void release_login_buffer(struct ibmvnic_adapter *adapter)  in release_login_buffer()  argument
1650 	if (!adapter->login_buf)  in release_login_buffer()
1653 	dma_unmap_single(&adapter->vdev->dev, adapter->login_buf_token,  in release_login_buffer()
1654 			 adapter->login_buf_sz, DMA_TO_DEVICE);  in release_login_buffer()
1655 	kfree(adapter->login_buf);  in release_login_buffer()
1656 	adapter->login_buf = NULL;  in release_login_buffer()
1659 static void release_login_rsp_buffer(struct ibmvnic_adapter *adapter)  in release_login_rsp_buffer()  argument
1661 	if (!adapter->login_rsp_buf)  in release_login_rsp_buffer()
1664 	dma_unmap_single(&adapter->vdev->dev, adapter->login_rsp_buf_token,  in release_login_rsp_buffer()
1665 			 adapter->login_rsp_buf_sz, DMA_FROM_DEVICE);  in release_login_rsp_buffer()
1666 	kfree(adapter->login_rsp_buf);  in release_login_rsp_buffer()
1667 	adapter->login_rsp_buf = NULL;  in release_login_rsp_buffer()
1670 static void release_resources(struct ibmvnic_adapter *adapter)  in release_resources()  argument
1672 	release_vpd_data(adapter);  in release_resources()
1674 	release_napi(adapter);  in release_resources()
1675 	release_login_buffer(adapter);  in release_resources()
1676 	release_login_rsp_buffer(adapter);  in release_resources()
1679 static int set_link_state(struct ibmvnic_adapter *adapter, u8 link_state)  in set_link_state()  argument
1681 	struct net_device *netdev = adapter->netdev;  in set_link_state()
1697 		reinit_completion(&adapter->init_done);  in set_link_state()
1698 		rc = ibmvnic_send_crq(adapter, &crq);  in set_link_state()
1704 		if (!wait_for_completion_timeout(&adapter->init_done,  in set_link_state()
1710 		if (adapter->init_done_rc == PARTIALSUCCESS) {  in set_link_state()
1714 		} else if (adapter->init_done_rc) {  in set_link_state()
1716 				    adapter->init_done_rc);  in set_link_state()
1717 			return adapter->init_done_rc;  in set_link_state()
1726 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in set_real_num_queues()  local
1730 		   adapter->req_tx_queues, adapter->req_rx_queues);  in set_real_num_queues()
1732 	rc = netif_set_real_num_tx_queues(netdev, adapter->req_tx_queues);  in set_real_num_queues()
1738 	rc = netif_set_real_num_rx_queues(netdev, adapter->req_rx_queues);  in set_real_num_queues()
1745 static int ibmvnic_get_vpd(struct ibmvnic_adapter *adapter)  in ibmvnic_get_vpd()  argument
1747 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_get_vpd()
1752 	if (adapter->vpd->buff)  in ibmvnic_get_vpd()
1753 		len = adapter->vpd->len;  in ibmvnic_get_vpd()
1755 	mutex_lock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1756 	adapter->fw_done_rc = 0;  in ibmvnic_get_vpd()
1757 	reinit_completion(&adapter->fw_done);  in ibmvnic_get_vpd()
1761 	rc = ibmvnic_send_crq(adapter, &crq);  in ibmvnic_get_vpd()
1763 		mutex_unlock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1767 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);  in ibmvnic_get_vpd()
1770 		mutex_unlock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1773 	mutex_unlock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1775 	if (!adapter->vpd->len)  in ibmvnic_get_vpd()
1778 	if (!adapter->vpd->buff)  in ibmvnic_get_vpd()
1779 		adapter->vpd->buff = kzalloc(adapter->vpd->len, GFP_KERNEL);  in ibmvnic_get_vpd()
1780 	else if (adapter->vpd->len != len)  in ibmvnic_get_vpd()
1781 		adapter->vpd->buff =  in ibmvnic_get_vpd()
1782 			krealloc(adapter->vpd->buff,  in ibmvnic_get_vpd()
1783 				 adapter->vpd->len, GFP_KERNEL);  in ibmvnic_get_vpd()
1785 	if (!adapter->vpd->buff) {  in ibmvnic_get_vpd()
1790 	adapter->vpd->dma_addr =  in ibmvnic_get_vpd()
1791 		dma_map_single(dev, adapter->vpd->buff, adapter->vpd->len,  in ibmvnic_get_vpd()
1793 	if (dma_mapping_error(dev, adapter->vpd->dma_addr)) {  in ibmvnic_get_vpd()
1795 		kfree(adapter->vpd->buff);  in ibmvnic_get_vpd()
1796 		adapter->vpd->buff = NULL;  in ibmvnic_get_vpd()
1800 	mutex_lock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1801 	adapter->fw_done_rc = 0;  in ibmvnic_get_vpd()
1802 	reinit_completion(&adapter->fw_done);  in ibmvnic_get_vpd()
1806 	crq.get_vpd.ioba = cpu_to_be32(adapter->vpd->dma_addr);  in ibmvnic_get_vpd()
1807 	crq.get_vpd.len = cpu_to_be32((u32)adapter->vpd->len);  in ibmvnic_get_vpd()
1808 	rc = ibmvnic_send_crq(adapter, &crq);  in ibmvnic_get_vpd()
1810 		kfree(adapter->vpd->buff);  in ibmvnic_get_vpd()
1811 		adapter->vpd->buff = NULL;  in ibmvnic_get_vpd()
1812 		mutex_unlock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1816 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);  in ibmvnic_get_vpd()
1819 		kfree(adapter->vpd->buff);  in ibmvnic_get_vpd()
1820 		adapter->vpd->buff = NULL;  in ibmvnic_get_vpd()
1821 		mutex_unlock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1825 	mutex_unlock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1829 static int init_resources(struct ibmvnic_adapter *adapter)  in init_resources()  argument
1831 	struct net_device *netdev = adapter->netdev;  in init_resources()
1838 	adapter->vpd = kzalloc(sizeof(*adapter->vpd), GFP_KERNEL);  in init_resources()
1839 	if (!adapter->vpd)  in init_resources()
1843 	rc = ibmvnic_get_vpd(adapter);  in init_resources()
1849 	rc = init_napi(adapter);  in init_resources()
1853 	send_query_map(adapter);  in init_resources()
1865 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in __ibmvnic_open()  local
1866 	enum vnic_state prev_state = adapter->state;  in __ibmvnic_open()
1869 	adapter->state = VNIC_OPENING;  in __ibmvnic_open()
1870 	replenish_pools(adapter);  in __ibmvnic_open()
1871 	ibmvnic_napi_enable(adapter);  in __ibmvnic_open()
1876 	for (i = 0; i < adapter->req_rx_queues; i++) {  in __ibmvnic_open()
1879 			enable_irq(adapter->rx_scrq[i]->irq);  in __ibmvnic_open()
1880 		enable_scrq_irq(adapter, adapter->rx_scrq[i]);  in __ibmvnic_open()
1883 	for (i = 0; i < adapter->req_tx_queues; i++) {  in __ibmvnic_open()
1886 			enable_irq(adapter->tx_scrq[i]->irq);  in __ibmvnic_open()
1887 		enable_scrq_irq(adapter, adapter->tx_scrq[i]);  in __ibmvnic_open()
1894 		if (adapter->reset_reason != VNIC_RESET_NON_FATAL)  in __ibmvnic_open()
1898 	rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_UP);  in __ibmvnic_open()
1900 		ibmvnic_napi_disable(adapter);  in __ibmvnic_open()
1901 		ibmvnic_disable_irqs(adapter);  in __ibmvnic_open()
1905 	adapter->tx_queues_active = true;  in __ibmvnic_open()
1917 		for (i = 0; i < adapter->req_rx_queues; i++)  in __ibmvnic_open()
1918 			napi_schedule(&adapter->napi[i]);  in __ibmvnic_open()
1921 	adapter->state = VNIC_OPEN;  in __ibmvnic_open()
1927 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_open()  local
1943 	if (adapter->failover_pending || (test_bit(0, &adapter->resetting))) {  in ibmvnic_open()
1945 			   adapter_state_to_string(adapter->state),  in ibmvnic_open()
1946 			   adapter->failover_pending);  in ibmvnic_open()
1947 		adapter->state = VNIC_OPEN;  in ibmvnic_open()
1952 	if (adapter->state != VNIC_CLOSED) {  in ibmvnic_open()
1957 		rc = init_resources(adapter);  in ibmvnic_open()
1972 	    (adapter->failover_pending || (test_bit(0, &adapter->resetting)))) {  in ibmvnic_open()
1973 		adapter->state = VNIC_OPEN;  in ibmvnic_open()
1978 		release_resources(adapter);  in ibmvnic_open()
1979 		release_rx_pools(adapter);  in ibmvnic_open()
1980 		release_tx_pools(adapter);  in ibmvnic_open()
1986 static void clean_rx_pools(struct ibmvnic_adapter *adapter)  in clean_rx_pools()  argument
1994 	if (!adapter->rx_pool)  in clean_rx_pools()
1997 	rx_scrqs = adapter->num_active_rx_pools;  in clean_rx_pools()
1998 	rx_entries = adapter->req_rx_add_entries_per_subcrq;  in clean_rx_pools()
2002 		rx_pool = &adapter->rx_pool[i];  in clean_rx_pools()
2006 		netdev_dbg(adapter->netdev, "Cleaning rx_pool[%d]\n", i);  in clean_rx_pools()
2017 static void clean_one_tx_pool(struct ibmvnic_adapter *adapter,  in clean_one_tx_pool()  argument
2038 static void clean_tx_pools(struct ibmvnic_adapter *adapter)  in clean_tx_pools()  argument
2043 	if (!adapter->tx_pool || !adapter->tso_pool)  in clean_tx_pools()
2046 	tx_scrqs = adapter->num_active_tx_pools;  in clean_tx_pools()
2050 		netdev_dbg(adapter->netdev, "Cleaning tx_pool[%d]\n", i);  in clean_tx_pools()
2051 		clean_one_tx_pool(adapter, &adapter->tx_pool[i]);  in clean_tx_pools()
2052 		clean_one_tx_pool(adapter, &adapter->tso_pool[i]);  in clean_tx_pools()
2056 static void ibmvnic_disable_irqs(struct ibmvnic_adapter *adapter)  in ibmvnic_disable_irqs()  argument
2058 	struct net_device *netdev = adapter->netdev;  in ibmvnic_disable_irqs()
2061 	if (adapter->tx_scrq) {  in ibmvnic_disable_irqs()
2062 		for (i = 0; i < adapter->req_tx_queues; i++)  in ibmvnic_disable_irqs()
2063 			if (adapter->tx_scrq[i]->irq) {  in ibmvnic_disable_irqs()
2066 				disable_scrq_irq(adapter, adapter->tx_scrq[i]);  in ibmvnic_disable_irqs()
2067 				disable_irq(adapter->tx_scrq[i]->irq);  in ibmvnic_disable_irqs()
2071 	if (adapter->rx_scrq) {  in ibmvnic_disable_irqs()
2072 		for (i = 0; i < adapter->req_rx_queues; i++) {  in ibmvnic_disable_irqs()
2073 			if (adapter->rx_scrq[i]->irq) {  in ibmvnic_disable_irqs()
2076 				disable_scrq_irq(adapter, adapter->rx_scrq[i]);  in ibmvnic_disable_irqs()
2077 				disable_irq(adapter->rx_scrq[i]->irq);  in ibmvnic_disable_irqs()
2085 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_cleanup()  local
2089 	adapter->tx_queues_active = false;  in ibmvnic_cleanup()
2096 	if (test_bit(0, &adapter->resetting))  in ibmvnic_cleanup()
2101 	ibmvnic_napi_disable(adapter);  in ibmvnic_cleanup()
2102 	ibmvnic_disable_irqs(adapter);  in ibmvnic_cleanup()
2107 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in __ibmvnic_close()  local
2110 	adapter->state = VNIC_CLOSING;  in __ibmvnic_close()
2111 	rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_DN);  in __ibmvnic_close()
2112 	adapter->state = VNIC_CLOSED;  in __ibmvnic_close()
2118 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_close()  local
2122 		   adapter_state_to_string(adapter->state),  in ibmvnic_close()
2123 		   adapter->failover_pending,  in ibmvnic_close()
2124 		   adapter->force_reset_recovery);  in ibmvnic_close()
2129 	if (adapter->failover_pending) {  in ibmvnic_close()
2130 		adapter->state = VNIC_CLOSED;  in ibmvnic_close()
2136 	clean_rx_pools(adapter);  in ibmvnic_close()
2137 	clean_tx_pools(adapter);  in ibmvnic_close()
2294 static void ibmvnic_tx_scrq_clean_buffer(struct ibmvnic_adapter *adapter,  in ibmvnic_tx_scrq_clean_buffer()  argument
2316 			tx_pool = &adapter->tso_pool[queue_num];  in ibmvnic_tx_scrq_clean_buffer()
2319 			tx_pool = &adapter->tx_pool[queue_num];  in ibmvnic_tx_scrq_clean_buffer()
2326 		adapter->netdev->stats.tx_packets--;  in ibmvnic_tx_scrq_clean_buffer()
2327 		adapter->netdev->stats.tx_bytes -= tx_buff->skb->len;  in ibmvnic_tx_scrq_clean_buffer()
2328 		adapter->tx_stats_buffers[queue_num].packets--;  in ibmvnic_tx_scrq_clean_buffer()
2329 		adapter->tx_stats_buffers[queue_num].bytes -=  in ibmvnic_tx_scrq_clean_buffer()
2333 		adapter->netdev->stats.tx_dropped++;  in ibmvnic_tx_scrq_clean_buffer()
2339 	    (adapter->req_tx_entries_per_subcrq / 2) &&  in ibmvnic_tx_scrq_clean_buffer()
2340 	    __netif_subqueue_stopped(adapter->netdev, queue_num)) {  in ibmvnic_tx_scrq_clean_buffer()
2343 		if (adapter->tx_queues_active) {  in ibmvnic_tx_scrq_clean_buffer()
2344 			netif_wake_subqueue(adapter->netdev, queue_num);  in ibmvnic_tx_scrq_clean_buffer()
2345 			netdev_dbg(adapter->netdev, "Started queue %d\n",  in ibmvnic_tx_scrq_clean_buffer()
2353 static int ibmvnic_tx_scrq_flush(struct ibmvnic_adapter *adapter,  in ibmvnic_tx_scrq_flush()  argument
2369 	rc = send_subcrq_indirect(adapter, handle, dma_addr, entries);  in ibmvnic_tx_scrq_flush()
2371 		ibmvnic_tx_scrq_clean_buffer(adapter, tx_scrq);  in ibmvnic_tx_scrq_flush()
2379 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_xmit()  local
2381 	u8 *hdrs = (u8 *)&adapter->tx_rx_desc_req;  in ibmvnic_xmit()
2382 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_xmit()
2410 	if (!adapter->tx_queues_active) {  in ibmvnic_xmit()
2419 	tx_scrq = adapter->tx_scrq[queue_num];  in ibmvnic_xmit()
2427 		ibmvnic_tx_scrq_flush(adapter, tx_scrq);  in ibmvnic_xmit()
2432 		tx_pool = &adapter->tso_pool[queue_num];  in ibmvnic_xmit()
2434 		tx_pool = &adapter->tx_pool[queue_num];  in ibmvnic_xmit()
2442 		ibmvnic_tx_scrq_flush(adapter, tx_scrq);  in ibmvnic_xmit()
2501 	if (adapter->vlan_header_insertion && skb_vlan_tag_present(skb)) {  in ibmvnic_xmit()
2536 		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq);  in ibmvnic_xmit()
2548 		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq);  in ibmvnic_xmit()
2554 					>= adapter->req_tx_entries_per_subcrq) {  in ibmvnic_xmit()
2576 	if (lpar_rc == H_CLOSED || adapter->failover_pending) {  in ibmvnic_xmit()
2590 	adapter->tx_send_failed += tx_send_failed;  in ibmvnic_xmit()
2591 	adapter->tx_map_failed += tx_map_failed;  in ibmvnic_xmit()
2592 	adapter->tx_stats_buffers[queue_num].packets += tx_packets;  in ibmvnic_xmit()
2593 	adapter->tx_stats_buffers[queue_num].bytes += tx_bytes;  in ibmvnic_xmit()
2594 	adapter->tx_stats_buffers[queue_num].dropped_packets += tx_dropped;  in ibmvnic_xmit()
2601 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_set_multi()  local
2610 		if (!adapter->promisc_supported)  in ibmvnic_set_multi()
2619 			ibmvnic_send_crq(adapter, &crq);  in ibmvnic_set_multi()
2626 			ibmvnic_send_crq(adapter, &crq);  in ibmvnic_set_multi()
2636 				ibmvnic_send_crq(adapter, &crq);  in ibmvnic_set_multi()
2644 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in __ibmvnic_set_mac()  local
2658 	mutex_lock(&adapter->fw_lock);  in __ibmvnic_set_mac()
2659 	adapter->fw_done_rc = 0;  in __ibmvnic_set_mac()
2660 	reinit_completion(&adapter->fw_done);  in __ibmvnic_set_mac()
2662 	rc = ibmvnic_send_crq(adapter, &crq);  in __ibmvnic_set_mac()
2665 		mutex_unlock(&adapter->fw_lock);  in __ibmvnic_set_mac()
2669 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);  in __ibmvnic_set_mac()
2671 	if (rc || adapter->fw_done_rc) {  in __ibmvnic_set_mac()
2673 		mutex_unlock(&adapter->fw_lock);  in __ibmvnic_set_mac()
2676 	mutex_unlock(&adapter->fw_lock);  in __ibmvnic_set_mac()
2679 	ether_addr_copy(adapter->mac_addr, netdev->dev_addr);  in __ibmvnic_set_mac()
2685 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_set_mac()  local
2693 	ether_addr_copy(adapter->mac_addr, addr->sa_data);  in ibmvnic_set_mac()
2694 	if (adapter->state != VNIC_PROBED)  in ibmvnic_set_mac()
2728 static inline void reinit_init_done(struct ibmvnic_adapter *adapter)  in reinit_init_done()  argument
2730 	reinit_completion(&adapter->init_done);  in reinit_init_done()
2731 	adapter->init_done_rc = 0;  in reinit_init_done()
2738 static int do_reset(struct ibmvnic_adapter *adapter,  in do_reset()  argument
2741 	struct net_device *netdev = adapter->netdev;  in do_reset()
2746 	netdev_dbg(adapter->netdev,  in do_reset()
2748 		   adapter_state_to_string(adapter->state),  in do_reset()
2749 		   adapter->failover_pending,  in do_reset()
2753 	adapter->reset_reason = rwi->reset_reason;  in do_reset()
2755 	if (!(adapter->reset_reason == VNIC_RESET_CHANGE_PARAM))  in do_reset()
2763 		adapter->failover_pending = false;  in do_reset()
2766 	reset_state = adapter->state;  in do_reset()
2775 	old_num_rx_queues = adapter->req_rx_queues;  in do_reset()
2776 	old_num_tx_queues = adapter->req_tx_queues;  in do_reset()
2777 	old_num_rx_slots = adapter->req_rx_add_entries_per_subcrq;  in do_reset()
2778 	old_num_tx_slots = adapter->req_tx_entries_per_subcrq;  in do_reset()
2783 	    adapter->reset_reason != VNIC_RESET_MOBILITY &&  in do_reset()
2784 	    adapter->reset_reason != VNIC_RESET_FAILOVER) {  in do_reset()
2785 		if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {  in do_reset()
2790 			adapter->state = VNIC_CLOSING;  in do_reset()
2798 			rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_DN);  in do_reset()
2803 			if (adapter->state == VNIC_OPEN) {  in do_reset()
2814 				adapter->state = VNIC_CLOSING;  in do_reset()
2817 			if (adapter->state != VNIC_CLOSING) {  in do_reset()
2824 			adapter->state = VNIC_CLOSED;  in do_reset()
2828 	if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {  in do_reset()
2829 		release_resources(adapter);  in do_reset()
2830 		release_sub_crqs(adapter, 1);  in do_reset()
2831 		release_crq_queue(adapter);  in do_reset()
2834 	if (adapter->reset_reason != VNIC_RESET_NON_FATAL) {  in do_reset()
2838 		adapter->state = VNIC_PROBED;  in do_reset()
2840 		reinit_init_done(adapter);  in do_reset()
2842 		if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {  in do_reset()
2843 			rc = init_crq_queue(adapter);  in do_reset()
2844 		} else if (adapter->reset_reason == VNIC_RESET_MOBILITY) {  in do_reset()
2845 			rc = ibmvnic_reenable_crq_queue(adapter);  in do_reset()
2846 			release_sub_crqs(adapter, 1);  in do_reset()
2848 			rc = ibmvnic_reset_crq(adapter);  in do_reset()
2850 				rc = vio_enable_interrupts(adapter->vdev);  in do_reset()
2852 					netdev_err(adapter->netdev,  in do_reset()
2859 			netdev_err(adapter->netdev,  in do_reset()
2864 		rc = ibmvnic_reset_init(adapter, true);  in do_reset()
2880 		if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {  in do_reset()
2881 			rc = init_resources(adapter);  in do_reset()
2884 		} else if (adapter->req_rx_queues != old_num_rx_queues ||  in do_reset()
2885 		    adapter->req_tx_queues != old_num_tx_queues ||  in do_reset()
2886 		    adapter->req_rx_add_entries_per_subcrq !=  in do_reset()
2888 		    adapter->req_tx_entries_per_subcrq !=  in do_reset()
2890 		    !adapter->rx_pool ||  in do_reset()
2891 		    !adapter->tso_pool ||  in do_reset()
2892 		    !adapter->tx_pool) {  in do_reset()
2893 			release_napi(adapter);  in do_reset()
2894 			release_vpd_data(adapter);  in do_reset()
2896 			rc = init_resources(adapter);  in do_reset()
2917 		ibmvnic_disable_irqs(adapter);  in do_reset()
2919 	adapter->state = VNIC_CLOSED;  in do_reset()
2935 	if (adapter->reset_reason == VNIC_RESET_FAILOVER ||  in do_reset()
2936 	    adapter->reset_reason == VNIC_RESET_MOBILITY)  in do_reset()
2944 		adapter->state = reset_state;  in do_reset()
2946 	if (!(adapter->reset_reason == VNIC_RESET_CHANGE_PARAM))  in do_reset()
2949 	netdev_dbg(adapter->netdev, "[S:%s FOP:%d] Reset done, rc %d\n",  in do_reset()
2950 		   adapter_state_to_string(adapter->state),  in do_reset()
2951 		   adapter->failover_pending, rc);  in do_reset()
2955 static int do_hard_reset(struct ibmvnic_adapter *adapter,  in do_hard_reset()  argument
2958 	struct net_device *netdev = adapter->netdev;  in do_hard_reset()
2961 	netdev_dbg(adapter->netdev, "Hard resetting driver (%s)\n",  in do_hard_reset()
2965 	reset_state = adapter->state;  in do_hard_reset()
2973 	adapter->reset_reason = rwi->reset_reason;  in do_hard_reset()
2976 	release_resources(adapter);  in do_hard_reset()
2977 	release_sub_crqs(adapter, 0);  in do_hard_reset()
2978 	release_crq_queue(adapter);  in do_hard_reset()
2983 	adapter->state = VNIC_PROBED;  in do_hard_reset()
2985 	reinit_init_done(adapter);  in do_hard_reset()
2987 	rc = init_crq_queue(adapter);  in do_hard_reset()
2989 		netdev_err(adapter->netdev,  in do_hard_reset()
2994 	rc = ibmvnic_reset_init(adapter, false);  in do_hard_reset()
3008 	rc = init_resources(adapter);  in do_hard_reset()
3012 	ibmvnic_disable_irqs(adapter);  in do_hard_reset()
3013 	adapter->state = VNIC_CLOSED;  in do_hard_reset()
3028 		adapter->state = reset_state;  in do_hard_reset()
3029 	netdev_dbg(adapter->netdev, "[S:%s FOP:%d] Hard reset done, rc %d\n",  in do_hard_reset()
3030 		   adapter_state_to_string(adapter->state),  in do_hard_reset()
3031 		   adapter->failover_pending, rc);  in do_hard_reset()
3035 static struct ibmvnic_rwi *get_next_rwi(struct ibmvnic_adapter *adapter)  in get_next_rwi()  argument
3040 	spin_lock_irqsave(&adapter->rwi_lock, flags);  in get_next_rwi()
3042 	if (!list_empty(&adapter->rwi_list)) {  in get_next_rwi()
3043 		rwi = list_first_entry(&adapter->rwi_list, struct ibmvnic_rwi,  in get_next_rwi()
3050 	spin_unlock_irqrestore(&adapter->rwi_lock, flags);  in get_next_rwi()
3068 static int do_passive_init(struct ibmvnic_adapter *adapter)  in do_passive_init()  argument
3071 	struct net_device *netdev = adapter->netdev;  in do_passive_init()
3072 	struct device *dev = &adapter->vdev->dev;  in do_passive_init()
3077 	adapter->state = VNIC_PROBING;  in do_passive_init()
3078 	reinit_completion(&adapter->init_done);  in do_passive_init()
3079 	adapter->init_done_rc = 0;  in do_passive_init()
3080 	adapter->crq.active = true;  in do_passive_init()
3082 	rc = send_crq_init_complete(adapter);  in do_passive_init()
3086 	rc = send_version_xchg(adapter);  in do_passive_init()
3088 		netdev_dbg(adapter->netdev, "send_version_xchg failed, rc=%d\n", rc);  in do_passive_init()
3090 	if (!wait_for_completion_timeout(&adapter->init_done, timeout)) {  in do_passive_init()
3096 	rc = init_sub_crqs(adapter);  in do_passive_init()
3102 	rc = init_sub_crq_irqs(adapter);  in do_passive_init()
3108 	netdev->mtu = adapter->req_mtu - ETH_HLEN;  in do_passive_init()
3109 	netdev->min_mtu = adapter->min_mtu - ETH_HLEN;  in do_passive_init()
3110 	netdev->max_mtu = adapter->max_mtu - ETH_HLEN;  in do_passive_init()
3112 	adapter->state = VNIC_PROBED;  in do_passive_init()
3118 	release_sub_crqs(adapter, 1);  in do_passive_init()
3120 	adapter->state = VNIC_DOWN;  in do_passive_init()
3126 	struct ibmvnic_adapter *adapter;  in __ibmvnic_reset()  local
3138 	adapter = container_of(work, struct ibmvnic_adapter, ibmvnic_reset);  in __ibmvnic_reset()
3139 		dev = &adapter->vdev->dev;  in __ibmvnic_reset()
3150 	if (adapter->state == VNIC_PROBING &&  in __ibmvnic_reset()
3151 	    !wait_for_completion_timeout(&adapter->probe_done, timeout)) {  in __ibmvnic_reset()
3154 				   &adapter->ibmvnic_delayed_reset,  in __ibmvnic_reset()
3160 	if (adapter->state == VNIC_REMOVING)  in __ibmvnic_reset()
3188 	spin_lock(&adapter->rwi_lock);  in __ibmvnic_reset()
3189 	if (!list_empty(&adapter->rwi_list)) {  in __ibmvnic_reset()
3190 		if (test_and_set_bit_lock(0, &adapter->resetting)) {  in __ibmvnic_reset()
3192 					   &adapter->ibmvnic_delayed_reset,  in __ibmvnic_reset()
3198 	spin_unlock(&adapter->rwi_lock);  in __ibmvnic_reset()
3203 	rwi = get_next_rwi(adapter);  in __ibmvnic_reset()
3205 		spin_lock_irqsave(&adapter->state_lock, flags);  in __ibmvnic_reset()
3207 		if (adapter->state == VNIC_REMOVING ||  in __ibmvnic_reset()
3208 		    adapter->state == VNIC_REMOVED) {  in __ibmvnic_reset()
3209 			spin_unlock_irqrestore(&adapter->state_lock, flags);  in __ibmvnic_reset()
3216 			reset_state = adapter->state;  in __ibmvnic_reset()
3219 		spin_unlock_irqrestore(&adapter->state_lock, flags);  in __ibmvnic_reset()
3223 			rc = do_passive_init(adapter);  in __ibmvnic_reset()
3226 				netif_carrier_on(adapter->netdev);  in __ibmvnic_reset()
3227 		} else if (adapter->force_reset_recovery) {  in __ibmvnic_reset()
3232 			adapter->failover_pending = false;  in __ibmvnic_reset()
3235 			if (adapter->wait_for_reset) {  in __ibmvnic_reset()
3237 				adapter->force_reset_recovery = false;  in __ibmvnic_reset()
3238 				rc = do_hard_reset(adapter, rwi, reset_state);  in __ibmvnic_reset()
3241 				adapter->force_reset_recovery = false;  in __ibmvnic_reset()
3242 				rc = do_hard_reset(adapter, rwi, reset_state);  in __ibmvnic_reset()
3258 				netdev_dbg(adapter->netdev,  in __ibmvnic_reset()
3260 					   adapter_state_to_string(adapter->state),  in __ibmvnic_reset()
3266 			rc = do_reset(adapter, rwi, reset_state);  in __ibmvnic_reset()
3269 		adapter->last_reset_time = jiffies;  in __ibmvnic_reset()
3272 			netdev_dbg(adapter->netdev, "Reset failed, rc=%d\n", rc);  in __ibmvnic_reset()
3274 		rwi = get_next_rwi(adapter);  in __ibmvnic_reset()
3293 			adapter->force_reset_recovery = true;  in __ibmvnic_reset()
3296 	if (adapter->wait_for_reset) {  in __ibmvnic_reset()
3297 		adapter->reset_done_rc = rc;  in __ibmvnic_reset()
3298 		complete(&adapter->reset_done);  in __ibmvnic_reset()
3301 	clear_bit_unlock(0, &adapter->resetting);  in __ibmvnic_reset()
3303 	netdev_dbg(adapter->netdev,  in __ibmvnic_reset()
3305 		   adapter_state_to_string(adapter->state),  in __ibmvnic_reset()
3306 		   adapter->force_reset_recovery,  in __ibmvnic_reset()
3307 		   adapter->wait_for_reset);  in __ibmvnic_reset()
3312 	struct ibmvnic_adapter *adapter;  in __ibmvnic_delayed_reset()  local
3314 	adapter = container_of(work, struct ibmvnic_adapter,  in __ibmvnic_delayed_reset()
3316 	__ibmvnic_reset(&adapter->ibmvnic_reset);  in __ibmvnic_delayed_reset()
3319 static void flush_reset_queue(struct ibmvnic_adapter *adapter)  in flush_reset_queue()  argument
3323 	if (!list_empty(&adapter->rwi_list)) {  in flush_reset_queue()
3324 		list_for_each_safe(entry, tmp_entry, &adapter->rwi_list) {  in flush_reset_queue()
3331 static int ibmvnic_reset(struct ibmvnic_adapter *adapter,  in ibmvnic_reset()  argument
3334 	struct net_device *netdev = adapter->netdev;  in ibmvnic_reset()
3339 	spin_lock_irqsave(&adapter->rwi_lock, flags);  in ibmvnic_reset()
3346 	if (adapter->state == VNIC_REMOVING ||  in ibmvnic_reset()
3347 	    adapter->state == VNIC_REMOVED ||  in ibmvnic_reset()
3348 	    (adapter->failover_pending && reason != VNIC_RESET_FAILOVER)) {  in ibmvnic_reset()
3354 	list_for_each_entry(tmp, &adapter->rwi_list, list) {  in ibmvnic_reset()
3371 	if (adapter->force_reset_recovery)  in ibmvnic_reset()
3372 		flush_reset_queue(adapter);  in ibmvnic_reset()
3375 	list_add_tail(&rwi->list, &adapter->rwi_list);  in ibmvnic_reset()
3376 	netdev_dbg(adapter->netdev, "Scheduling reset (reason %s)\n",  in ibmvnic_reset()
3378 	queue_work(system_long_wq, &adapter->ibmvnic_reset);  in ibmvnic_reset()
3383 	spin_unlock_irqrestore(&adapter->rwi_lock, flags);  in ibmvnic_reset()
3393 	struct ibmvnic_adapter *adapter = netdev_priv(dev);  in ibmvnic_tx_timeout()  local
3395 	if (test_bit(0, &adapter->resetting)) {  in ibmvnic_tx_timeout()
3396 		netdev_err(adapter->netdev,  in ibmvnic_tx_timeout()
3403 	if (time_before(jiffies, (adapter->last_reset_time + dev->watchdog_timeo))) {  in ibmvnic_tx_timeout()
3407 	ibmvnic_reset(adapter, VNIC_RESET_TIMEOUT);  in ibmvnic_tx_timeout()
3410 static void remove_buff_from_pool(struct ibmvnic_adapter *adapter,  in remove_buff_from_pool()  argument
3413 	struct ibmvnic_rx_pool *pool = &adapter->rx_pool[rx_buff->pool_index];  in remove_buff_from_pool()
3426 	struct ibmvnic_adapter *adapter;  in ibmvnic_poll()  local
3432 	adapter = netdev_priv(netdev);  in ibmvnic_poll()
3433 	scrq_num = (int)(napi - adapter->napi);  in ibmvnic_poll()
3435 	rx_scrq = adapter->rx_scrq[scrq_num];  in ibmvnic_poll()
3446 		if (unlikely(test_bit(0, &adapter->resetting) &&  in ibmvnic_poll()
3447 			     adapter->reset_reason != VNIC_RESET_NON_FATAL)) {  in ibmvnic_poll()
3448 			enable_scrq_irq(adapter, rx_scrq);  in ibmvnic_poll()
3453 		if (!pending_scrq(adapter, rx_scrq))  in ibmvnic_poll()
3455 		next = ibmvnic_next_scrq(adapter, rx_scrq);  in ibmvnic_poll()
3465 			remove_buff_from_pool(adapter, rx_buff);  in ibmvnic_poll()
3470 			remove_buff_from_pool(adapter, rx_buff);  in ibmvnic_poll()
3486 		if (adapter->rx_vlan_header_insertion &&  in ibmvnic_poll()
3493 		remove_buff_from_pool(adapter, rx_buff);  in ibmvnic_poll()
3508 		adapter->rx_stats_buffers[scrq_num].packets++;  in ibmvnic_poll()
3509 		adapter->rx_stats_buffers[scrq_num].bytes += length;  in ibmvnic_poll()
3513 	if (adapter->state != VNIC_CLOSING &&  in ibmvnic_poll()
3514 	    ((atomic_read(&adapter->rx_pool[scrq_num].available) <  in ibmvnic_poll()
3515 	      adapter->req_rx_add_entries_per_subcrq / 2) ||  in ibmvnic_poll()
3517 		replenish_rx_pool(adapter, &adapter->rx_pool[scrq_num]);  in ibmvnic_poll()
3520 			enable_scrq_irq(adapter, rx_scrq);  in ibmvnic_poll()
3521 			if (pending_scrq(adapter, rx_scrq)) {  in ibmvnic_poll()
3523 					disable_scrq_irq(adapter, rx_scrq);  in ibmvnic_poll()
3532 static int wait_for_reset(struct ibmvnic_adapter *adapter)  in wait_for_reset()  argument
3536 	adapter->fallback.mtu = adapter->req_mtu;  in wait_for_reset()
3537 	adapter->fallback.rx_queues = adapter->req_rx_queues;  in wait_for_reset()
3538 	adapter->fallback.tx_queues = adapter->req_tx_queues;  in wait_for_reset()
3539 	adapter->fallback.rx_entries = adapter->req_rx_add_entries_per_subcrq;  in wait_for_reset()
3540 	adapter->fallback.tx_entries = adapter->req_tx_entries_per_subcrq;  in wait_for_reset()
3542 	reinit_completion(&adapter->reset_done);  in wait_for_reset()
3543 	adapter->wait_for_reset = true;  in wait_for_reset()
3544 	rc = ibmvnic_reset(adapter, VNIC_RESET_CHANGE_PARAM);  in wait_for_reset()
3550 	rc = ibmvnic_wait_for_completion(adapter, &adapter->reset_done, 60000);  in wait_for_reset()
3557 	if (adapter->reset_done_rc) {  in wait_for_reset()
3559 		adapter->desired.mtu = adapter->fallback.mtu;  in wait_for_reset()
3560 		adapter->desired.rx_queues = adapter->fallback.rx_queues;  in wait_for_reset()
3561 		adapter->desired.tx_queues = adapter->fallback.tx_queues;  in wait_for_reset()
3562 		adapter->desired.rx_entries = adapter->fallback.rx_entries;  in wait_for_reset()
3563 		adapter->desired.tx_entries = adapter->fallback.tx_entries;  in wait_for_reset()
3565 		reinit_completion(&adapter->reset_done);  in wait_for_reset()
3566 		adapter->wait_for_reset = true;  in wait_for_reset()
3567 		rc = ibmvnic_reset(adapter, VNIC_RESET_CHANGE_PARAM);  in wait_for_reset()
3572 		rc = ibmvnic_wait_for_completion(adapter, &adapter->reset_done,  in wait_for_reset()
3580 	adapter->wait_for_reset = false;  in wait_for_reset()
3587 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_change_mtu()  local
3589 	adapter->desired.mtu = new_mtu + ETH_HLEN;  in ibmvnic_change_mtu()
3591 	return wait_for_reset(adapter);  in ibmvnic_change_mtu()
3628 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_get_link_ksettings()  local
3631 	rc = send_query_phys_parms(adapter);  in ibmvnic_get_link_ksettings()
3633 		adapter->speed = SPEED_UNKNOWN;  in ibmvnic_get_link_ksettings()
3634 		adapter->duplex = DUPLEX_UNKNOWN;  in ibmvnic_get_link_ksettings()
3636 	cmd->base.speed = adapter->speed;  in ibmvnic_get_link_ksettings()
3637 	cmd->base.duplex = adapter->duplex;  in ibmvnic_get_link_ksettings()
3648 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_get_drvinfo()  local
3652 	strscpy(info->fw_version, adapter->fw_version,  in ibmvnic_get_drvinfo()
3658 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_get_msglevel()  local
3660 	return adapter->msg_enable;  in ibmvnic_get_msglevel()
3665 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_set_msglevel()  local
3667 	adapter->msg_enable = data;  in ibmvnic_set_msglevel()
3672 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_get_link()  local
3677 	return adapter->logical_link_state;  in ibmvnic_get_link()
3685 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_get_ringparam()  local
3687 	ring->rx_max_pending = adapter->max_rx_add_entries_per_subcrq;  in ibmvnic_get_ringparam()
3688 	ring->tx_max_pending = adapter->max_tx_entries_per_subcrq;  in ibmvnic_get_ringparam()
3691 	ring->rx_pending = adapter->req_rx_add_entries_per_subcrq;  in ibmvnic_get_ringparam()
3692 	ring->tx_pending = adapter->req_tx_entries_per_subcrq;  in ibmvnic_get_ringparam()
3702 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_set_ringparam()  local
3704 	if (ring->rx_pending > adapter->max_rx_add_entries_per_subcrq  ||  in ibmvnic_set_ringparam()
3705 	    ring->tx_pending > adapter->max_tx_entries_per_subcrq) {  in ibmvnic_set_ringparam()
3708 			   adapter->max_rx_add_entries_per_subcrq);  in ibmvnic_set_ringparam()
3710 			   adapter->max_tx_entries_per_subcrq);  in ibmvnic_set_ringparam()
3714 	adapter->desired.rx_entries = ring->rx_pending;  in ibmvnic_set_ringparam()
3715 	adapter->desired.tx_entries = ring->tx_pending;  in ibmvnic_set_ringparam()
3717 	return wait_for_reset(adapter);  in ibmvnic_set_ringparam()
3723 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_get_channels()  local
3725 	channels->max_rx = adapter->max_rx_queues;  in ibmvnic_get_channels()
3726 	channels->max_tx = adapter->max_tx_queues;  in ibmvnic_get_channels()
3729 	channels->rx_count = adapter->req_rx_queues;  in ibmvnic_get_channels()
3730 	channels->tx_count = adapter->req_tx_queues;  in ibmvnic_get_channels()
3738 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_set_channels()  local
3740 	adapter->desired.rx_queues = channels->rx_count;  in ibmvnic_set_channels()
3741 	adapter->desired.tx_queues = channels->tx_count;  in ibmvnic_set_channels()
3743 	return wait_for_reset(adapter);  in ibmvnic_set_channels()
3748 	struct ibmvnic_adapter *adapter = netdev_priv(dev);  in ibmvnic_get_strings()  local
3757 	for (i = 0; i < adapter->req_tx_queues; i++) {  in ibmvnic_get_strings()
3768 	for (i = 0; i < adapter->req_rx_queues; i++) {  in ibmvnic_get_strings()
3782 	struct ibmvnic_adapter *adapter = netdev_priv(dev);  in ibmvnic_get_sset_count()  local
3787 		       adapter->req_tx_queues * NUM_TX_STATS +  in ibmvnic_get_sset_count()
3788 		       adapter->req_rx_queues * NUM_RX_STATS;  in ibmvnic_get_sset_count()
3797 	struct ibmvnic_adapter *adapter = netdev_priv(dev);  in ibmvnic_get_ethtool_stats()  local
3805 	crq.request_statistics.ioba = cpu_to_be32(adapter->stats_token);  in ibmvnic_get_ethtool_stats()
3810 	reinit_completion(&adapter->stats_done);  in ibmvnic_get_ethtool_stats()
3811 	rc = ibmvnic_send_crq(adapter, &crq);  in ibmvnic_get_ethtool_stats()
3814 	rc = ibmvnic_wait_for_completion(adapter, &adapter->stats_done, 10000);  in ibmvnic_get_ethtool_stats()
3820 				      (adapter, ibmvnic_stats[i].offset));  in ibmvnic_get_ethtool_stats()
3822 	for (j = 0; j < adapter->req_tx_queues; j++) {  in ibmvnic_get_ethtool_stats()
3823 		data[i] = adapter->tx_stats_buffers[j].packets;  in ibmvnic_get_ethtool_stats()
3825 		data[i] = adapter->tx_stats_buffers[j].bytes;  in ibmvnic_get_ethtool_stats()
3827 		data[i] = adapter->tx_stats_buffers[j].dropped_packets;  in ibmvnic_get_ethtool_stats()
3831 	for (j = 0; j < adapter->req_rx_queues; j++) {  in ibmvnic_get_ethtool_stats()
3832 		data[i] = adapter->rx_stats_buffers[j].packets;  in ibmvnic_get_ethtool_stats()
3834 		data[i] = adapter->rx_stats_buffers[j].bytes;  in ibmvnic_get_ethtool_stats()
3836 		data[i] = adapter->rx_stats_buffers[j].interrupts;  in ibmvnic_get_ethtool_stats()
3858 static int reset_one_sub_crq_queue(struct ibmvnic_adapter *adapter,  in reset_one_sub_crq_queue()  argument
3864 		netdev_dbg(adapter->netdev, "Invalid scrq reset.\n");  in reset_one_sub_crq_queue()
3880 		netdev_dbg(adapter->netdev, "Invalid scrq reset\n");  in reset_one_sub_crq_queue()
3884 	rc = h_reg_sub_crq(adapter->vdev->unit_address, scrq->msg_token,  in reset_one_sub_crq_queue()
3889 static int reset_sub_crq_queues(struct ibmvnic_adapter *adapter)  in reset_sub_crq_queues()  argument
3893 	if (!adapter->tx_scrq || !adapter->rx_scrq)  in reset_sub_crq_queues()
3896 	ibmvnic_clean_affinity(adapter);  in reset_sub_crq_queues()
3898 	for (i = 0; i < adapter->req_tx_queues; i++) {  in reset_sub_crq_queues()
3899 		netdev_dbg(adapter->netdev, "Re-setting tx_scrq[%d]\n", i);  in reset_sub_crq_queues()
3900 		rc = reset_one_sub_crq_queue(adapter, adapter->tx_scrq[i]);  in reset_sub_crq_queues()
3905 	for (i = 0; i < adapter->req_rx_queues; i++) {  in reset_sub_crq_queues()
3906 		netdev_dbg(adapter->netdev, "Re-setting rx_scrq[%d]\n", i);  in reset_sub_crq_queues()
3907 		rc = reset_one_sub_crq_queue(adapter, adapter->rx_scrq[i]);  in reset_sub_crq_queues()
3915 static void release_sub_crq_queue(struct ibmvnic_adapter *adapter,  in release_sub_crq_queue()  argument
3919 	struct device *dev = &adapter->vdev->dev;  in release_sub_crq_queue()
3922 	netdev_dbg(adapter->netdev, "Releasing sub-CRQ\n");  in release_sub_crq_queue()
3928 						adapter->vdev->unit_address,  in release_sub_crq_queue()
3933 			netdev_err(adapter->netdev,  in release_sub_crq_queue()
3952 							*adapter)  in init_sub_crq_queue()
3954 	struct device *dev = &adapter->vdev->dev;  in init_sub_crq_queue()
3978 	rc = h_reg_sub_crq(adapter->vdev->unit_address, scrq->msg_token,  in init_sub_crq_queue()
3982 		rc = ibmvnic_reset_crq(adapter);  in init_sub_crq_queue()
3991 	scrq->adapter = adapter;  in init_sub_crq_queue()
4006 	netdev_dbg(adapter->netdev,  in init_sub_crq_queue()
4015 					adapter->vdev->unit_address,  in init_sub_crq_queue()
4031 static void release_sub_crqs(struct ibmvnic_adapter *adapter, bool do_h_free)  in release_sub_crqs()  argument
4035 	ibmvnic_clean_affinity(adapter);  in release_sub_crqs()
4036 	if (adapter->tx_scrq) {  in release_sub_crqs()
4037 		for (i = 0; i < adapter->num_active_tx_scrqs; i++) {  in release_sub_crqs()
4038 			if (!adapter->tx_scrq[i])  in release_sub_crqs()
4041 			netdev_dbg(adapter->netdev, "Releasing tx_scrq[%d]\n",  in release_sub_crqs()
4043 			ibmvnic_tx_scrq_clean_buffer(adapter, adapter->tx_scrq[i]);  in release_sub_crqs()
4044 			if (adapter->tx_scrq[i]->irq) {  in release_sub_crqs()
4045 				free_irq(adapter->tx_scrq[i]->irq,  in release_sub_crqs()
4046 					 adapter->tx_scrq[i]);  in release_sub_crqs()
4047 				irq_dispose_mapping(adapter->tx_scrq[i]->irq);  in release_sub_crqs()
4048 				adapter->tx_scrq[i]->irq = 0;  in release_sub_crqs()
4051 			release_sub_crq_queue(adapter, adapter->tx_scrq[i],  in release_sub_crqs()
4055 		kfree(adapter->tx_scrq);  in release_sub_crqs()
4056 		adapter->tx_scrq = NULL;  in release_sub_crqs()
4057 		adapter->num_active_tx_scrqs = 0;  in release_sub_crqs()
4060 	if (adapter->rx_scrq) {  in release_sub_crqs()
4061 		for (i = 0; i < adapter->num_active_rx_scrqs; i++) {  in release_sub_crqs()
4062 			if (!adapter->rx_scrq[i])  in release_sub_crqs()
4065 			netdev_dbg(adapter->netdev, "Releasing rx_scrq[%d]\n",  in release_sub_crqs()
4067 			if (adapter->rx_scrq[i]->irq) {  in release_sub_crqs()
4068 				free_irq(adapter->rx_scrq[i]->irq,  in release_sub_crqs()
4069 					 adapter->rx_scrq[i]);  in release_sub_crqs()
4070 				irq_dispose_mapping(adapter->rx_scrq[i]->irq);  in release_sub_crqs()
4071 				adapter->rx_scrq[i]->irq = 0;  in release_sub_crqs()
4074 			release_sub_crq_queue(adapter, adapter->rx_scrq[i],  in release_sub_crqs()
4078 		kfree(adapter->rx_scrq);  in release_sub_crqs()
4079 		adapter->rx_scrq = NULL;  in release_sub_crqs()
4080 		adapter->num_active_rx_scrqs = 0;  in release_sub_crqs()
4084 static int disable_scrq_irq(struct ibmvnic_adapter *adapter,  in disable_scrq_irq()  argument
4087 	struct device *dev = &adapter->vdev->dev;  in disable_scrq_irq()
4090 	rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,  in disable_scrq_irq()
4122 static int enable_scrq_irq(struct ibmvnic_adapter *adapter,  in enable_scrq_irq()  argument
4125 	struct device *dev = &adapter->vdev->dev;  in enable_scrq_irq()
4133 	if (test_bit(0, &adapter->resetting) &&  in enable_scrq_irq()
4134 	    adapter->reset_reason == VNIC_RESET_MOBILITY) {  in enable_scrq_irq()
4138 	rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,  in enable_scrq_irq()
4146 static int ibmvnic_complete_tx(struct ibmvnic_adapter *adapter,  in ibmvnic_complete_tx()  argument
4149 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_complete_tx()
4158 	while (pending_scrq(adapter, scrq)) {  in ibmvnic_complete_tx()
4164 		next = ibmvnic_next_scrq(adapter, scrq);  in ibmvnic_complete_tx()
4168 				tx_pool = &adapter->tso_pool[pool];  in ibmvnic_complete_tx()
4171 				tx_pool = &adapter->tx_pool[pool];  in ibmvnic_complete_tx()
4188 				netdev_warn(adapter->netdev,  in ibmvnic_complete_tx()
4199 		txq = netdev_get_tx_queue(adapter->netdev, scrq->pool_index);  in ibmvnic_complete_tx()
4203 		    (adapter->req_tx_entries_per_subcrq / 2) &&  in ibmvnic_complete_tx()
4204 		    __netif_subqueue_stopped(adapter->netdev,  in ibmvnic_complete_tx()
4207 			if (adapter->tx_queues_active) {  in ibmvnic_complete_tx()
4208 				netif_wake_subqueue(adapter->netdev,  in ibmvnic_complete_tx()
4210 				netdev_dbg(adapter->netdev,  in ibmvnic_complete_tx()
4218 	enable_scrq_irq(adapter, scrq);  in ibmvnic_complete_tx()
4220 	if (pending_scrq(adapter, scrq)) {  in ibmvnic_complete_tx()
4221 		disable_scrq_irq(adapter, scrq);  in ibmvnic_complete_tx()
4231 	struct ibmvnic_adapter *adapter = scrq->adapter;  in ibmvnic_interrupt_tx()  local
4233 	disable_scrq_irq(adapter, scrq);  in ibmvnic_interrupt_tx()
4234 	ibmvnic_complete_tx(adapter, scrq);  in ibmvnic_interrupt_tx()
4242 	struct ibmvnic_adapter *adapter = scrq->adapter;  in ibmvnic_interrupt_rx()  local
4247 	if (unlikely(adapter->state != VNIC_OPEN))  in ibmvnic_interrupt_rx()
4250 	adapter->rx_stats_buffers[scrq->scrq_num].interrupts++;  in ibmvnic_interrupt_rx()
4252 	if (napi_schedule_prep(&adapter->napi[scrq->scrq_num])) {  in ibmvnic_interrupt_rx()
4253 		disable_scrq_irq(adapter, scrq);  in ibmvnic_interrupt_rx()
4254 		__napi_schedule(&adapter->napi[scrq->scrq_num]);  in ibmvnic_interrupt_rx()
4260 static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter)  in init_sub_crq_irqs()  argument
4262 	struct device *dev = &adapter->vdev->dev;  in init_sub_crq_irqs()
4267 	for (i = 0; i < adapter->req_tx_queues; i++) {  in init_sub_crq_irqs()
4268 		netdev_dbg(adapter->netdev, "Initializing tx_scrq[%d] irq\n",  in init_sub_crq_irqs()
4270 		scrq = adapter->tx_scrq[i];  in init_sub_crq_irqs()
4280 			 adapter->vdev->unit_address, i);  in init_sub_crq_irqs()
4292 	for (i = 0; i < adapter->req_rx_queues; i++) {  in init_sub_crq_irqs()
4293 		netdev_dbg(adapter->netdev, "Initializing rx_scrq[%d] irq\n",  in init_sub_crq_irqs()
4295 		scrq = adapter->rx_scrq[i];  in init_sub_crq_irqs()
4303 			 adapter->vdev->unit_address, i);  in init_sub_crq_irqs()
4315 	ibmvnic_set_affinity(adapter);  in init_sub_crq_irqs()
4322 		free_irq(adapter->rx_scrq[j]->irq, adapter->rx_scrq[j]);  in init_sub_crq_irqs()
4323 		irq_dispose_mapping(adapter->rx_scrq[j]->irq);  in init_sub_crq_irqs()
4325 	i = adapter->req_tx_queues;  in init_sub_crq_irqs()
4328 		free_irq(adapter->tx_scrq[j]->irq, adapter->tx_scrq[j]);  in init_sub_crq_irqs()
4329 		irq_dispose_mapping(adapter->tx_scrq[j]->irq);  in init_sub_crq_irqs()
4331 	release_sub_crqs(adapter, 1);  in init_sub_crq_irqs()
4335 static int init_sub_crqs(struct ibmvnic_adapter *adapter)  in init_sub_crqs()  argument
4337 	struct device *dev = &adapter->vdev->dev;  in init_sub_crqs()
4344 	total_queues = adapter->req_tx_queues + adapter->req_rx_queues;  in init_sub_crqs()
4351 		allqueues[i] = init_sub_crq_queue(adapter);  in init_sub_crqs()
4361 	    adapter->min_tx_queues + adapter->min_rx_queues) {  in init_sub_crqs()
4368 		netdev_dbg(adapter->netdev, "Reducing number of queues\n");  in init_sub_crqs()
4371 			if (adapter->req_rx_queues > adapter->min_rx_queues)  in init_sub_crqs()
4372 				adapter->req_rx_queues--;  in init_sub_crqs()
4377 			if (adapter->req_tx_queues > adapter->min_tx_queues)  in init_sub_crqs()
4378 				adapter->req_tx_queues--;  in init_sub_crqs()
4385 	adapter->tx_scrq = kcalloc(adapter->req_tx_queues,  in init_sub_crqs()
4386 				   sizeof(*adapter->tx_scrq), GFP_KERNEL);  in init_sub_crqs()
4387 	if (!adapter->tx_scrq)  in init_sub_crqs()
4390 	for (i = 0; i < adapter->req_tx_queues; i++) {  in init_sub_crqs()
4391 		adapter->tx_scrq[i] = allqueues[i];  in init_sub_crqs()
4392 		adapter->tx_scrq[i]->pool_index = i;  in init_sub_crqs()
4393 		adapter->num_active_tx_scrqs++;  in init_sub_crqs()
4396 	adapter->rx_scrq = kcalloc(adapter->req_rx_queues,  in init_sub_crqs()
4397 				   sizeof(*adapter->rx_scrq), GFP_KERNEL);  in init_sub_crqs()
4398 	if (!adapter->rx_scrq)  in init_sub_crqs()
4401 	for (i = 0; i < adapter->req_rx_queues; i++) {  in init_sub_crqs()
4402 		adapter->rx_scrq[i] = allqueues[i + adapter->req_tx_queues];  in init_sub_crqs()
4403 		adapter->rx_scrq[i]->scrq_num = i;  in init_sub_crqs()
4404 		adapter->num_active_rx_scrqs++;  in init_sub_crqs()
4411 	kfree(adapter->tx_scrq);  in init_sub_crqs()
4412 	adapter->tx_scrq = NULL;  in init_sub_crqs()
4415 		release_sub_crq_queue(adapter, allqueues[i], 1);  in init_sub_crqs()
4420 static void send_request_cap(struct ibmvnic_adapter *adapter, int retry)  in send_request_cap()  argument
4422 	struct device *dev = &adapter->vdev->dev;  in send_request_cap()
4432 	if (!(adapter->netdev->flags & IFF_PROMISC) ||  in send_request_cap()
4433 	    adapter->promisc_supported)  in send_request_cap()
4442 		atomic_set(&adapter->running_cap_crqs, cap_reqs);  in send_request_cap()
4444 		if (adapter->min_tx_entries_per_subcrq > entries_page ||  in send_request_cap()
4445 		    adapter->min_rx_add_entries_per_subcrq > entries_page) {  in send_request_cap()
4450 		if (adapter->desired.mtu)  in send_request_cap()
4451 			adapter->req_mtu = adapter->desired.mtu;  in send_request_cap()
4453 			adapter->req_mtu = adapter->netdev->mtu + ETH_HLEN;  in send_request_cap()
4455 		if (!adapter->desired.tx_entries)  in send_request_cap()
4456 			adapter->desired.tx_entries =  in send_request_cap()
4457 					adapter->max_tx_entries_per_subcrq;  in send_request_cap()
4458 		if (!adapter->desired.rx_entries)  in send_request_cap()
4459 			adapter->desired.rx_entries =  in send_request_cap()
4460 					adapter->max_rx_add_entries_per_subcrq;  in send_request_cap()
4463 			      (adapter->req_mtu + IBMVNIC_BUFFER_HLEN);  in send_request_cap()
4465 		if ((adapter->req_mtu + IBMVNIC_BUFFER_HLEN) *  in send_request_cap()
4466 			adapter->desired.tx_entries > IBMVNIC_LTB_SET_SIZE) {  in send_request_cap()
4467 			adapter->desired.tx_entries = max_entries;  in send_request_cap()
4470 		if ((adapter->req_mtu + IBMVNIC_BUFFER_HLEN) *  in send_request_cap()
4471 			adapter->desired.rx_entries > IBMVNIC_LTB_SET_SIZE) {  in send_request_cap()
4472 			adapter->desired.rx_entries = max_entries;  in send_request_cap()
4475 		if (adapter->desired.tx_entries)  in send_request_cap()
4476 			adapter->req_tx_entries_per_subcrq =  in send_request_cap()
4477 					adapter->desired.tx_entries;  in send_request_cap()
4479 			adapter->req_tx_entries_per_subcrq =  in send_request_cap()
4480 					adapter->max_tx_entries_per_subcrq;  in send_request_cap()
4482 		if (adapter->desired.rx_entries)  in send_request_cap()
4483 			adapter->req_rx_add_entries_per_subcrq =  in send_request_cap()
4484 					adapter->desired.rx_entries;  in send_request_cap()
4486 			adapter->req_rx_add_entries_per_subcrq =  in send_request_cap()
4487 					adapter->max_rx_add_entries_per_subcrq;  in send_request_cap()
4489 		if (adapter->desired.tx_queues)  in send_request_cap()
4490 			adapter->req_tx_queues =  in send_request_cap()
4491 					adapter->desired.tx_queues;  in send_request_cap()
4493 			adapter->req_tx_queues =  in send_request_cap()
4494 					adapter->opt_tx_comp_sub_queues;  in send_request_cap()
4496 		if (adapter->desired.rx_queues)  in send_request_cap()
4497 			adapter->req_rx_queues =  in send_request_cap()
4498 					adapter->desired.rx_queues;  in send_request_cap()
4500 			adapter->req_rx_queues =  in send_request_cap()
4501 					adapter->opt_rx_comp_queues;  in send_request_cap()
4503 		adapter->req_rx_add_queues = adapter->max_rx_add_queues;  in send_request_cap()
4505 		atomic_add(cap_reqs, &adapter->running_cap_crqs);  in send_request_cap()
4512 	crq.request_capability.number = cpu_to_be64(adapter->req_tx_queues);  in send_request_cap()
4514 	ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4517 	crq.request_capability.number = cpu_to_be64(adapter->req_rx_queues);  in send_request_cap()
4519 	ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4522 	crq.request_capability.number = cpu_to_be64(adapter->req_rx_add_queues);  in send_request_cap()
4524 	ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4529 	    cpu_to_be64(adapter->req_tx_entries_per_subcrq);  in send_request_cap()
4531 	ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4536 	    cpu_to_be64(adapter->req_rx_add_entries_per_subcrq);  in send_request_cap()
4538 	ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4541 	crq.request_capability.number = cpu_to_be64(adapter->req_mtu);  in send_request_cap()
4543 	ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4545 	if (adapter->netdev->flags & IFF_PROMISC) {  in send_request_cap()
4546 		if (adapter->promisc_supported) {  in send_request_cap()
4551 			ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4558 		ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4567 static int pending_scrq(struct ibmvnic_adapter *adapter,  in pending_scrq()  argument
4583 static union sub_crq *ibmvnic_next_scrq(struct ibmvnic_adapter *adapter,  in ibmvnic_next_scrq()  argument
4607 static union ibmvnic_crq *ibmvnic_next_crq(struct ibmvnic_adapter *adapter)  in ibmvnic_next_crq()  argument
4609 	struct ibmvnic_crq_queue *queue = &adapter->crq;  in ibmvnic_next_crq()
4642 static int send_subcrq_indirect(struct ibmvnic_adapter *adapter,  in send_subcrq_indirect()  argument
4645 	unsigned int ua = adapter->vdev->unit_address;  in send_subcrq_indirect()
4646 	struct device *dev = &adapter->vdev->dev;  in send_subcrq_indirect()
4661 static int ibmvnic_send_crq(struct ibmvnic_adapter *adapter,  in ibmvnic_send_crq()  argument
4664 	unsigned int ua = adapter->vdev->unit_address;  in ibmvnic_send_crq()
4665 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_send_crq()
4669 	netdev_dbg(adapter->netdev, "Sending CRQ: %016lx %016lx\n",  in ibmvnic_send_crq()
4673 	if (!adapter->crq.active &&  in ibmvnic_send_crq()
4698 static int ibmvnic_send_crq_init(struct ibmvnic_adapter *adapter)  in ibmvnic_send_crq_init()  argument
4700 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_send_crq_init()
4708 	netdev_dbg(adapter->netdev, "Sending CRQ init\n");  in ibmvnic_send_crq_init()
4711 		rc = ibmvnic_send_crq(adapter, &crq);  in ibmvnic_send_crq_init()
4733 static int vnic_client_data_len(struct ibmvnic_adapter *adapter)  in vnic_client_data_len()  argument
4744 	len += strlen(adapter->netdev->name) + 1;  in vnic_client_data_len()
4749 static void vnic_add_client_data(struct ibmvnic_adapter *adapter,  in vnic_add_client_data()  argument
4771 	len = strlen(adapter->netdev->name) + 1;  in vnic_add_client_data()
4773 	strscpy(vlcd->name, adapter->netdev->name, len);  in vnic_add_client_data()
4776 static int send_login(struct ibmvnic_adapter *adapter)  in send_login()  argument
4780 	struct device *dev = &adapter->vdev->dev;  in send_login()
4793 	if (!adapter->tx_scrq || !adapter->rx_scrq) {  in send_login()
4794 		netdev_err(adapter->netdev,  in send_login()
4799 	release_login_buffer(adapter);  in send_login()
4800 	release_login_rsp_buffer(adapter);  in send_login()
4802 	client_data_len = vnic_client_data_len(adapter);  in send_login()
4806 	    sizeof(u64) * (adapter->req_tx_queues + adapter->req_rx_queues) +  in send_login()
4821 			  sizeof(u64) * adapter->req_tx_queues +  in send_login()
4822 			  sizeof(u64) * adapter->req_rx_queues +  in send_login()
4823 			  sizeof(u64) * adapter->req_rx_queues +  in send_login()
4837 	adapter->login_buf = login_buffer;  in send_login()
4838 	adapter->login_buf_token = buffer_token;  in send_login()
4839 	adapter->login_buf_sz = buffer_size;  in send_login()
4840 	adapter->login_rsp_buf = login_rsp_buffer;  in send_login()
4841 	adapter->login_rsp_buf_token = rsp_buffer_token;  in send_login()
4842 	adapter->login_rsp_buf_sz = rsp_buffer_size;  in send_login()
4846 	login_buffer->num_txcomp_subcrqs = cpu_to_be32(adapter->req_tx_queues);  in send_login()
4849 	login_buffer->num_rxcomp_subcrqs = cpu_to_be32(adapter->req_rx_queues);  in send_login()
4852 			sizeof(u64) * adapter->req_tx_queues);  in send_login()
4860 				      sizeof(u64) * adapter->req_tx_queues);  in send_login()
4862 	for (i = 0; i < adapter->req_tx_queues; i++) {  in send_login()
4863 		if (adapter->tx_scrq[i]) {  in send_login()
4865 				cpu_to_be64(adapter->tx_scrq[i]->crq_num);  in send_login()
4869 	for (i = 0; i < adapter->req_rx_queues; i++) {  in send_login()
4870 		if (adapter->rx_scrq[i]) {  in send_login()
4872 				cpu_to_be64(adapter->rx_scrq[i]->crq_num);  in send_login()
4878 		((char *)rx_list_p + (sizeof(u64) * adapter->req_rx_queues));  in send_login()
4883 	vnic_add_client_data(adapter, vlcd);  in send_login()
4885 	netdev_dbg(adapter->netdev, "Login Buffer:\n");  in send_login()
4886 	for (i = 0; i < (adapter->login_buf_sz - 1) / 8 + 1; i++) {  in send_login()
4887 		netdev_dbg(adapter->netdev, "%016lx\n",  in send_login()
4888 			   ((unsigned long *)(adapter->login_buf))[i]);  in send_login()
4897 	adapter->login_pending = true;  in send_login()
4898 	rc = ibmvnic_send_crq(adapter, &crq);  in send_login()
4900 		adapter->login_pending = false;  in send_login()
4901 		netdev_err(adapter->netdev, "Failed to send login, rc=%d\n", rc);  in send_login()
4912 	adapter->login_rsp_buf = NULL;  in send_login()
4917 	adapter->login_buf = NULL;  in send_login()
4922 static int send_request_map(struct ibmvnic_adapter *adapter, dma_addr_t addr,  in send_request_map()  argument
4933 	return ibmvnic_send_crq(adapter, &crq);  in send_request_map()
4936 static int send_request_unmap(struct ibmvnic_adapter *adapter, u8 map_id)  in send_request_unmap()  argument
4944 	return ibmvnic_send_crq(adapter, &crq);  in send_request_unmap()
4947 static void send_query_map(struct ibmvnic_adapter *adapter)  in send_query_map()  argument
4954 	ibmvnic_send_crq(adapter, &crq);  in send_query_map()
4958 static void send_query_cap(struct ibmvnic_adapter *adapter)  in send_query_cap()  argument
4969 	atomic_set(&adapter->running_cap_crqs, cap_reqs);  in send_query_cap()
4976 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
4980 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
4984 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
4988 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
4992 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
4996 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5001 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5006 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5011 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5016 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5020 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5024 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5028 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5032 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5036 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5040 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5044 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5048 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5052 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5056 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5060 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5065 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5070 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5075 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5080 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5089 static void send_query_ip_offload(struct ibmvnic_adapter *adapter)  in send_query_ip_offload()  argument
5092 	struct device *dev = &adapter->vdev->dev;  in send_query_ip_offload()
5095 	adapter->ip_offload_tok =  in send_query_ip_offload()
5097 			       &adapter->ip_offload_buf,  in send_query_ip_offload()
5101 	if (dma_mapping_error(dev, adapter->ip_offload_tok)) {  in send_query_ip_offload()
5112 	    cpu_to_be32(adapter->ip_offload_tok);  in send_query_ip_offload()
5114 	ibmvnic_send_crq(adapter, &crq);  in send_query_ip_offload()
5117 static void send_control_ip_offload(struct ibmvnic_adapter *adapter)  in send_control_ip_offload()  argument
5119 	struct ibmvnic_control_ip_offload_buffer *ctrl_buf = &adapter->ip_offload_ctrl;  in send_control_ip_offload()
5120 	struct ibmvnic_query_ip_offload_buffer *buf = &adapter->ip_offload_buf;  in send_control_ip_offload()
5121 	struct device *dev = &adapter->vdev->dev;  in send_control_ip_offload()
5125 	adapter->ip_offload_ctrl_tok =  in send_control_ip_offload()
5128 			       sizeof(adapter->ip_offload_ctrl),  in send_control_ip_offload()
5131 	if (dma_mapping_error(dev, adapter->ip_offload_ctrl_tok)) {  in send_control_ip_offload()
5136 	ctrl_buf->len = cpu_to_be32(sizeof(adapter->ip_offload_ctrl));  in send_control_ip_offload()
5151 	if (adapter->state != VNIC_PROBING) {  in send_control_ip_offload()
5152 		old_hw_features = adapter->netdev->hw_features;  in send_control_ip_offload()
5153 		adapter->netdev->hw_features = 0;  in send_control_ip_offload()
5156 	adapter->netdev->hw_features = NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO;  in send_control_ip_offload()
5159 		adapter->netdev->hw_features |= NETIF_F_IP_CSUM;  in send_control_ip_offload()
5162 		adapter->netdev->hw_features |= NETIF_F_IPV6_CSUM;  in send_control_ip_offload()
5164 	if ((adapter->netdev->features &  in send_control_ip_offload()
5166 		adapter->netdev->hw_features |= NETIF_F_RXCSUM;  in send_control_ip_offload()
5169 		adapter->netdev->hw_features |= NETIF_F_TSO;  in send_control_ip_offload()
5171 		adapter->netdev->hw_features |= NETIF_F_TSO6;  in send_control_ip_offload()
5173 	if (adapter->state == VNIC_PROBING) {  in send_control_ip_offload()
5174 		adapter->netdev->features |= adapter->netdev->hw_features;  in send_control_ip_offload()
5175 	} else if (old_hw_features != adapter->netdev->hw_features) {  in send_control_ip_offload()
5179 		adapter->netdev->features &= adapter->netdev->hw_features;  in send_control_ip_offload()
5181 		tmp = (old_hw_features ^ adapter->netdev->hw_features) &  in send_control_ip_offload()
5182 			adapter->netdev->hw_features;  in send_control_ip_offload()
5183 		adapter->netdev->features |=  in send_control_ip_offload()
5184 				tmp & adapter->netdev->wanted_features;  in send_control_ip_offload()
5191 	    cpu_to_be32(sizeof(adapter->ip_offload_ctrl));  in send_control_ip_offload()
5192 	crq.control_ip_offload.ioba = cpu_to_be32(adapter->ip_offload_ctrl_tok);  in send_control_ip_offload()
5193 	ibmvnic_send_crq(adapter, &crq);  in send_control_ip_offload()
5197 				struct ibmvnic_adapter *adapter)  in handle_vpd_size_rsp()  argument
5199 	struct device *dev = &adapter->vdev->dev;  in handle_vpd_size_rsp()
5204 		complete(&adapter->fw_done);  in handle_vpd_size_rsp()
5208 	adapter->vpd->len = be64_to_cpu(crq->get_vpd_size_rsp.len);  in handle_vpd_size_rsp()
5209 	complete(&adapter->fw_done);  in handle_vpd_size_rsp()
5213 			   struct ibmvnic_adapter *adapter)  in handle_vpd_rsp()  argument
5215 	struct device *dev = &adapter->vdev->dev;  in handle_vpd_rsp()
5219 	memset(adapter->fw_version, 0, 32);  in handle_vpd_rsp()
5221 	dma_unmap_single(dev, adapter->vpd->dma_addr, adapter->vpd->len,  in handle_vpd_rsp()
5233 	substr = strnstr(adapter->vpd->buff, "RM", adapter->vpd->len);  in handle_vpd_rsp()
5240 	if ((substr + 2) < (adapter->vpd->buff + adapter->vpd->len)) {  in handle_vpd_rsp()
5249 	    (adapter->vpd->buff + adapter->vpd->len)) {  in handle_vpd_rsp()
5250 		strncpy((char *)adapter->fw_version, substr + 3, fw_level_len);  in handle_vpd_rsp()
5256 	if (adapter->fw_version[0] == '\0')  in handle_vpd_rsp()
5257 		strscpy((char *)adapter->fw_version, "N/A", sizeof(adapter->fw_version));  in handle_vpd_rsp()
5258 	complete(&adapter->fw_done);  in handle_vpd_rsp()
5261 static void handle_query_ip_offload_rsp(struct ibmvnic_adapter *adapter)  in handle_query_ip_offload_rsp()  argument
5263 	struct device *dev = &adapter->vdev->dev;  in handle_query_ip_offload_rsp()
5264 	struct ibmvnic_query_ip_offload_buffer *buf = &adapter->ip_offload_buf;  in handle_query_ip_offload_rsp()
5267 	dma_unmap_single(dev, adapter->ip_offload_tok,  in handle_query_ip_offload_rsp()
5268 			 sizeof(adapter->ip_offload_buf), DMA_FROM_DEVICE);  in handle_query_ip_offload_rsp()
5270 	netdev_dbg(adapter->netdev, "Query IP Offload Buffer:\n");  in handle_query_ip_offload_rsp()
5271 	for (i = 0; i < (sizeof(adapter->ip_offload_buf) - 1) / 8 + 1; i++)  in handle_query_ip_offload_rsp()
5272 		netdev_dbg(adapter->netdev, "%016lx\n",  in handle_query_ip_offload_rsp()
5275 	netdev_dbg(adapter->netdev, "ipv4_chksum = %d\n", buf->ipv4_chksum);  in handle_query_ip_offload_rsp()
5276 	netdev_dbg(adapter->netdev, "ipv6_chksum = %d\n", buf->ipv6_chksum);  in handle_query_ip_offload_rsp()
5277 	netdev_dbg(adapter->netdev, "tcp_ipv4_chksum = %d\n",  in handle_query_ip_offload_rsp()
5279 	netdev_dbg(adapter->netdev, "tcp_ipv6_chksum = %d\n",  in handle_query_ip_offload_rsp()
5281 	netdev_dbg(adapter->netdev, "udp_ipv4_chksum = %d\n",  in handle_query_ip_offload_rsp()
5283 	netdev_dbg(adapter->netdev, "udp_ipv6_chksum = %d\n",  in handle_query_ip_offload_rsp()
5285 	netdev_dbg(adapter->netdev, "large_tx_ipv4 = %d\n",  in handle_query_ip_offload_rsp()
5287 	netdev_dbg(adapter->netdev, "large_tx_ipv6 = %d\n",  in handle_query_ip_offload_rsp()
5289 	netdev_dbg(adapter->netdev, "large_rx_ipv4 = %d\n",  in handle_query_ip_offload_rsp()
5291 	netdev_dbg(adapter->netdev, "large_rx_ipv6 = %d\n",  in handle_query_ip_offload_rsp()
5293 	netdev_dbg(adapter->netdev, "max_ipv4_hdr_sz = %d\n",  in handle_query_ip_offload_rsp()
5295 	netdev_dbg(adapter->netdev, "max_ipv6_hdr_sz = %d\n",  in handle_query_ip_offload_rsp()
5297 	netdev_dbg(adapter->netdev, "max_tcp_hdr_size = %d\n",  in handle_query_ip_offload_rsp()
5299 	netdev_dbg(adapter->netdev, "max_udp_hdr_size = %d\n",  in handle_query_ip_offload_rsp()
5301 	netdev_dbg(adapter->netdev, "max_large_tx_size = %d\n",  in handle_query_ip_offload_rsp()
5303 	netdev_dbg(adapter->netdev, "max_large_rx_size = %d\n",  in handle_query_ip_offload_rsp()
5305 	netdev_dbg(adapter->netdev, "ipv6_ext_hdr = %d\n",  in handle_query_ip_offload_rsp()
5307 	netdev_dbg(adapter->netdev, "tcp_pseudosum_req = %d\n",  in handle_query_ip_offload_rsp()
5309 	netdev_dbg(adapter->netdev, "num_ipv6_ext_hd = %d\n",  in handle_query_ip_offload_rsp()
5311 	netdev_dbg(adapter->netdev, "off_ipv6_ext_hd = %d\n",  in handle_query_ip_offload_rsp()
5314 	send_control_ip_offload(adapter);  in handle_query_ip_offload_rsp()
5340 				    struct ibmvnic_adapter *adapter)  in handle_error_indication()  argument
5342 	struct device *dev = &adapter->vdev->dev;  in handle_error_indication()
5354 		ibmvnic_reset(adapter, VNIC_RESET_FATAL);  in handle_error_indication()
5356 		ibmvnic_reset(adapter, VNIC_RESET_NON_FATAL);  in handle_error_indication()
5360 				 struct ibmvnic_adapter *adapter)  in handle_change_mac_rsp()  argument
5362 	struct net_device *netdev = adapter->netdev;  in handle_change_mac_rsp()
5363 	struct device *dev = &adapter->vdev->dev;  in handle_change_mac_rsp()
5375 	ether_addr_copy(adapter->mac_addr,  in handle_change_mac_rsp()
5378 	complete(&adapter->fw_done);  in handle_change_mac_rsp()
5383 				   struct ibmvnic_adapter *adapter)  in handle_request_cap_rsp()  argument
5385 	struct device *dev = &adapter->vdev->dev;  in handle_request_cap_rsp()
5389 	atomic_dec(&adapter->running_cap_crqs);  in handle_request_cap_rsp()
5390 	netdev_dbg(adapter->netdev, "Outstanding request-caps: %d\n",  in handle_request_cap_rsp()
5391 		   atomic_read(&adapter->running_cap_crqs));  in handle_request_cap_rsp()
5394 		req_value = &adapter->req_tx_queues;  in handle_request_cap_rsp()
5398 		req_value = &adapter->req_rx_queues;  in handle_request_cap_rsp()
5402 		req_value = &adapter->req_rx_add_queues;  in handle_request_cap_rsp()
5406 		req_value = &adapter->req_tx_entries_per_subcrq;  in handle_request_cap_rsp()
5410 		req_value = &adapter->req_rx_add_entries_per_subcrq;  in handle_request_cap_rsp()
5414 		req_value = &adapter->req_mtu;  in handle_request_cap_rsp()
5418 		req_value = &adapter->promisc;  in handle_request_cap_rsp()
5440 			*req_value = adapter->fallback.mtu;  in handle_request_cap_rsp()
5446 		send_request_cap(adapter, 1);  in handle_request_cap_rsp()
5455 	if (atomic_read(&adapter->running_cap_crqs) == 0)  in handle_request_cap_rsp()
5456 		send_query_ip_offload(adapter);  in handle_request_cap_rsp()
5460 			    struct ibmvnic_adapter *adapter)  in handle_login_rsp()  argument
5462 	struct device *dev = &adapter->vdev->dev;  in handle_login_rsp()
5463 	struct net_device *netdev = adapter->netdev;  in handle_login_rsp()
5464 	struct ibmvnic_login_rsp_buffer *login_rsp = adapter->login_rsp_buf;  in handle_login_rsp()
5465 	struct ibmvnic_login_buffer *login = adapter->login_buf;  in handle_login_rsp()
5477 	if (!adapter->login_pending) {  in handle_login_rsp()
5481 	adapter->login_pending = false;  in handle_login_rsp()
5488 		adapter->init_done_rc = login_rsp_crq->generic.rc.code;  in handle_login_rsp()
5489 		complete(&adapter->init_done);  in handle_login_rsp()
5493 	if (adapter->failover_pending) {  in handle_login_rsp()
5494 		adapter->init_done_rc = -EAGAIN;  in handle_login_rsp()
5496 		complete(&adapter->init_done);  in handle_login_rsp()
5501 	netdev->mtu = adapter->req_mtu - ETH_HLEN;  in handle_login_rsp()
5503 	netdev_dbg(adapter->netdev, "Login Response Buffer:\n");  in handle_login_rsp()
5504 	for (i = 0; i < (adapter->login_rsp_buf_sz - 1) / 8 + 1; i++) {  in handle_login_rsp()
5505 		netdev_dbg(adapter->netdev, "%016lx\n",  in handle_login_rsp()
5506 			   ((unsigned long *)(adapter->login_rsp_buf))[i]);  in handle_login_rsp()
5512 	     adapter->req_rx_add_queues !=  in handle_login_rsp()
5515 		ibmvnic_reset(adapter, VNIC_RESET_FATAL);  in handle_login_rsp()
5531 		ibmvnic_reset(adapter, VNIC_RESET_FATAL);  in handle_login_rsp()
5535 	size_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +  in handle_login_rsp()
5536 		be32_to_cpu(adapter->login_rsp_buf->off_rxadd_buff_size));  in handle_login_rsp()
5540 	adapter->cur_rx_buf_sz = be64_to_cpu(size_array[0]);  in handle_login_rsp()
5542 	num_tx_pools = be32_to_cpu(adapter->login_rsp_buf->num_txsubm_subcrqs);  in handle_login_rsp()
5543 	num_rx_pools = be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);  in handle_login_rsp()
5545 	tx_handle_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +  in handle_login_rsp()
5546 				  be32_to_cpu(adapter->login_rsp_buf->off_txsubm_subcrqs));  in handle_login_rsp()
5547 	rx_handle_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +  in handle_login_rsp()
5548 				  be32_to_cpu(adapter->login_rsp_buf->off_rxadd_subcrqs));  in handle_login_rsp()
5551 		adapter->tx_scrq[i]->handle = tx_handle_array[i];  in handle_login_rsp()
5554 		adapter->rx_scrq[i]->handle = rx_handle_array[i];  in handle_login_rsp()
5556 	adapter->num_active_tx_scrqs = num_tx_pools;  in handle_login_rsp()
5557 	adapter->num_active_rx_scrqs = num_rx_pools;  in handle_login_rsp()
5558 	release_login_rsp_buffer(adapter);  in handle_login_rsp()
5559 	release_login_buffer(adapter);  in handle_login_rsp()
5560 	complete(&adapter->init_done);  in handle_login_rsp()
5566 				     struct ibmvnic_adapter *adapter)  in handle_request_unmap_rsp()  argument
5568 	struct device *dev = &adapter->vdev->dev;  in handle_request_unmap_rsp()
5577 				 struct ibmvnic_adapter *adapter)  in handle_query_map_rsp()  argument
5579 	struct net_device *netdev = adapter->netdev;  in handle_query_map_rsp()
5580 	struct device *dev = &adapter->vdev->dev;  in handle_query_map_rsp()
5595 				 struct ibmvnic_adapter *adapter)  in handle_query_cap_rsp()  argument
5597 	struct net_device *netdev = adapter->netdev;  in handle_query_cap_rsp()
5598 	struct device *dev = &adapter->vdev->dev;  in handle_query_cap_rsp()
5601 	atomic_dec(&adapter->running_cap_crqs);  in handle_query_cap_rsp()
5603 		   atomic_read(&adapter->running_cap_crqs));  in handle_query_cap_rsp()
5612 		adapter->min_tx_queues =  in handle_query_cap_rsp()
5615 			   adapter->min_tx_queues);  in handle_query_cap_rsp()
5618 		adapter->min_rx_queues =  in handle_query_cap_rsp()
5621 			   adapter->min_rx_queues);  in handle_query_cap_rsp()
5624 		adapter->min_rx_add_queues =  in handle_query_cap_rsp()
5627 			   adapter->min_rx_add_queues);  in handle_query_cap_rsp()
5630 		adapter->max_tx_queues =  in handle_query_cap_rsp()
5633 			   adapter->max_tx_queues);  in handle_query_cap_rsp()
5636 		adapter->max_rx_queues =  in handle_query_cap_rsp()
5639 			   adapter->max_rx_queues);  in handle_query_cap_rsp()
5642 		adapter->max_rx_add_queues =  in handle_query_cap_rsp()
5645 			   adapter->max_rx_add_queues);  in handle_query_cap_rsp()
5648 		adapter->min_tx_entries_per_subcrq =  in handle_query_cap_rsp()
5651 			   adapter->min_tx_entries_per_subcrq);  in handle_query_cap_rsp()
5654 		adapter->min_rx_add_entries_per_subcrq =  in handle_query_cap_rsp()
5657 			   adapter->min_rx_add_entries_per_subcrq);  in handle_query_cap_rsp()
5660 		adapter->max_tx_entries_per_subcrq =  in handle_query_cap_rsp()
5663 			   adapter->max_tx_entries_per_subcrq);  in handle_query_cap_rsp()
5666 		adapter->max_rx_add_entries_per_subcrq =  in handle_query_cap_rsp()
5669 			   adapter->max_rx_add_entries_per_subcrq);  in handle_query_cap_rsp()
5672 		adapter->tcp_ip_offload =  in handle_query_cap_rsp()
5675 			   adapter->tcp_ip_offload);  in handle_query_cap_rsp()
5678 		adapter->promisc_supported =  in handle_query_cap_rsp()
5681 			   adapter->promisc_supported);  in handle_query_cap_rsp()
5684 		adapter->min_mtu = be64_to_cpu(crq->query_capability.number);  in handle_query_cap_rsp()
5685 		netdev->min_mtu = adapter->min_mtu - ETH_HLEN;  in handle_query_cap_rsp()
5686 		netdev_dbg(netdev, "min_mtu = %lld\n", adapter->min_mtu);  in handle_query_cap_rsp()
5689 		adapter->max_mtu = be64_to_cpu(crq->query_capability.number);  in handle_query_cap_rsp()
5690 		netdev->max_mtu = adapter->max_mtu - ETH_HLEN;  in handle_query_cap_rsp()
5691 		netdev_dbg(netdev, "max_mtu = %lld\n", adapter->max_mtu);  in handle_query_cap_rsp()
5694 		adapter->max_multicast_filters =  in handle_query_cap_rsp()
5697 			   adapter->max_multicast_filters);  in handle_query_cap_rsp()
5700 		adapter->vlan_header_insertion =  in handle_query_cap_rsp()
5702 		if (adapter->vlan_header_insertion)  in handle_query_cap_rsp()
5705 			   adapter->vlan_header_insertion);  in handle_query_cap_rsp()
5708 		adapter->rx_vlan_header_insertion =  in handle_query_cap_rsp()
5711 			   adapter->rx_vlan_header_insertion);  in handle_query_cap_rsp()
5714 		adapter->max_tx_sg_entries =  in handle_query_cap_rsp()
5717 			   adapter->max_tx_sg_entries);  in handle_query_cap_rsp()
5720 		adapter->rx_sg_supported =  in handle_query_cap_rsp()
5723 			   adapter->rx_sg_supported);  in handle_query_cap_rsp()
5726 		adapter->opt_tx_comp_sub_queues =  in handle_query_cap_rsp()
5729 			   adapter->opt_tx_comp_sub_queues);  in handle_query_cap_rsp()
5732 		adapter->opt_rx_comp_queues =  in handle_query_cap_rsp()
5735 			   adapter->opt_rx_comp_queues);  in handle_query_cap_rsp()
5738 		adapter->opt_rx_bufadd_q_per_rx_comp_q =  in handle_query_cap_rsp()
5741 			   adapter->opt_rx_bufadd_q_per_rx_comp_q);  in handle_query_cap_rsp()
5744 		adapter->opt_tx_entries_per_subcrq =  in handle_query_cap_rsp()
5747 			   adapter->opt_tx_entries_per_subcrq);  in handle_query_cap_rsp()
5750 		adapter->opt_rxba_entries_per_subcrq =  in handle_query_cap_rsp()
5753 			   adapter->opt_rxba_entries_per_subcrq);  in handle_query_cap_rsp()
5756 		adapter->tx_rx_desc_req = crq->query_capability.number;  in handle_query_cap_rsp()
5758 			   adapter->tx_rx_desc_req);  in handle_query_cap_rsp()
5767 	if (atomic_read(&adapter->running_cap_crqs) == 0)  in handle_query_cap_rsp()
5768 		send_request_cap(adapter, 0);  in handle_query_cap_rsp()
5771 static int send_query_phys_parms(struct ibmvnic_adapter *adapter)  in send_query_phys_parms()  argument
5780 	mutex_lock(&adapter->fw_lock);  in send_query_phys_parms()
5781 	adapter->fw_done_rc = 0;  in send_query_phys_parms()
5782 	reinit_completion(&adapter->fw_done);  in send_query_phys_parms()
5784 	rc = ibmvnic_send_crq(adapter, &crq);  in send_query_phys_parms()
5786 		mutex_unlock(&adapter->fw_lock);  in send_query_phys_parms()
5790 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);  in send_query_phys_parms()
5792 		mutex_unlock(&adapter->fw_lock);  in send_query_phys_parms()
5796 	mutex_unlock(&adapter->fw_lock);  in send_query_phys_parms()
5797 	return adapter->fw_done_rc ? -EIO : 0;  in send_query_phys_parms()
5801 				       struct ibmvnic_adapter *adapter)  in handle_query_phys_parms_rsp()  argument
5803 	struct net_device *netdev = adapter->netdev;  in handle_query_phys_parms_rsp()
5814 		adapter->speed = SPEED_10;  in handle_query_phys_parms_rsp()
5817 		adapter->speed = SPEED_100;  in handle_query_phys_parms_rsp()
5820 		adapter->speed = SPEED_1000;  in handle_query_phys_parms_rsp()
5823 		adapter->speed = SPEED_10000;  in handle_query_phys_parms_rsp()
5826 		adapter->speed = SPEED_25000;  in handle_query_phys_parms_rsp()
5829 		adapter->speed = SPEED_40000;  in handle_query_phys_parms_rsp()
5832 		adapter->speed = SPEED_50000;  in handle_query_phys_parms_rsp()
5835 		adapter->speed = SPEED_100000;  in handle_query_phys_parms_rsp()
5838 		adapter->speed = SPEED_200000;  in handle_query_phys_parms_rsp()
5843 		adapter->speed = SPEED_UNKNOWN;  in handle_query_phys_parms_rsp()
5846 		adapter->duplex = DUPLEX_FULL;  in handle_query_phys_parms_rsp()
5848 		adapter->duplex = DUPLEX_HALF;  in handle_query_phys_parms_rsp()
5850 		adapter->duplex = DUPLEX_UNKNOWN;  in handle_query_phys_parms_rsp()
5856 			       struct ibmvnic_adapter *adapter)  in ibmvnic_handle_crq()  argument
5859 	struct net_device *netdev = adapter->netdev;  in ibmvnic_handle_crq()
5860 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_handle_crq()
5872 			adapter->from_passive_init = true;  in ibmvnic_handle_crq()
5876 			adapter->login_pending = false;  in ibmvnic_handle_crq()
5878 			if (adapter->state == VNIC_DOWN)  in ibmvnic_handle_crq()
5879 				rc = ibmvnic_reset(adapter, VNIC_RESET_PASSIVE_INIT);  in ibmvnic_handle_crq()
5881 				rc = ibmvnic_reset(adapter, VNIC_RESET_FAILOVER);  in ibmvnic_handle_crq()
5896 				adapter->failover_pending = false;  in ibmvnic_handle_crq()
5899 			if (!completion_done(&adapter->init_done)) {  in ibmvnic_handle_crq()
5900 				if (!adapter->init_done_rc)  in ibmvnic_handle_crq()
5901 					adapter->init_done_rc = -EAGAIN;  in ibmvnic_handle_crq()
5902 				complete(&adapter->init_done);  in ibmvnic_handle_crq()
5908 			adapter->crq.active = true;  in ibmvnic_handle_crq()
5909 			send_version_xchg(adapter);  in ibmvnic_handle_crq()
5917 		adapter->crq.active = false;  in ibmvnic_handle_crq()
5921 		if (!completion_done(&adapter->fw_done)) {  in ibmvnic_handle_crq()
5922 			adapter->fw_done_rc = -EIO;  in ibmvnic_handle_crq()
5923 			complete(&adapter->fw_done);  in ibmvnic_handle_crq()
5927 		if (!completion_done(&adapter->init_done)) {  in ibmvnic_handle_crq()
5928 			adapter->init_done_rc = -EAGAIN;  in ibmvnic_handle_crq()
5929 			complete(&adapter->init_done);  in ibmvnic_handle_crq()
5932 		if (!completion_done(&adapter->stats_done))  in ibmvnic_handle_crq()
5933 			complete(&adapter->stats_done);  in ibmvnic_handle_crq()
5934 		if (test_bit(0, &adapter->resetting))  in ibmvnic_handle_crq()
5935 			adapter->force_reset_recovery = true;  in ibmvnic_handle_crq()
5938 			ibmvnic_reset(adapter, VNIC_RESET_MOBILITY);  in ibmvnic_handle_crq()
5941 			adapter->failover_pending = true;  in ibmvnic_handle_crq()
5946 			ibmvnic_reset(adapter, VNIC_RESET_FATAL);  in ibmvnic_handle_crq()
5968 		send_query_cap(adapter);  in ibmvnic_handle_crq()
5971 		handle_query_cap_rsp(crq, adapter);  in ibmvnic_handle_crq()
5974 		handle_query_map_rsp(crq, adapter);  in ibmvnic_handle_crq()
5977 		adapter->fw_done_rc = crq->request_map_rsp.rc.code;  in ibmvnic_handle_crq()
5978 		complete(&adapter->fw_done);  in ibmvnic_handle_crq()
5981 		handle_request_unmap_rsp(crq, adapter);  in ibmvnic_handle_crq()
5984 		handle_request_cap_rsp(crq, adapter);  in ibmvnic_handle_crq()
5988 		handle_login_rsp(crq, adapter);  in ibmvnic_handle_crq()
5995 		adapter->logical_link_state =  in ibmvnic_handle_crq()
5997 		adapter->init_done_rc = crq->logical_link_state_rsp.rc.code;  in ibmvnic_handle_crq()
5998 		complete(&adapter->init_done);  in ibmvnic_handle_crq()
6002 		adapter->phys_link_state =  in ibmvnic_handle_crq()
6004 		adapter->logical_link_state =  in ibmvnic_handle_crq()
6006 		if (adapter->phys_link_state && adapter->logical_link_state)  in ibmvnic_handle_crq()
6013 		adapter->fw_done_rc = handle_change_mac_rsp(crq, adapter);  in ibmvnic_handle_crq()
6017 		handle_error_indication(crq, adapter);  in ibmvnic_handle_crq()
6021 		complete(&adapter->stats_done);  in ibmvnic_handle_crq()
6025 		handle_query_ip_offload_rsp(adapter);  in ibmvnic_handle_crq()
6032 		dma_unmap_single(dev, adapter->ip_offload_ctrl_tok,  in ibmvnic_handle_crq()
6033 				 sizeof(adapter->ip_offload_ctrl),  in ibmvnic_handle_crq()
6035 		complete(&adapter->init_done);  in ibmvnic_handle_crq()
6039 		complete(&adapter->fw_done);  in ibmvnic_handle_crq()
6042 		handle_vpd_size_rsp(crq, adapter);  in ibmvnic_handle_crq()
6045 		handle_vpd_rsp(crq, adapter);  in ibmvnic_handle_crq()
6048 		adapter->fw_done_rc = handle_query_phys_parms_rsp(crq, adapter);  in ibmvnic_handle_crq()
6049 		complete(&adapter->fw_done);  in ibmvnic_handle_crq()
6059 	struct ibmvnic_adapter *adapter = instance;  in ibmvnic_interrupt()  local
6061 	tasklet_schedule(&adapter->tasklet);  in ibmvnic_interrupt()
6067 	struct ibmvnic_adapter *adapter = from_tasklet(adapter, t, tasklet);  in ibmvnic_tasklet()  local
6068 	struct ibmvnic_crq_queue *queue = &adapter->crq;  in ibmvnic_tasklet()
6075 	while ((crq = ibmvnic_next_crq(adapter)) != NULL) {  in ibmvnic_tasklet()
6082 		ibmvnic_handle_crq(crq, adapter);  in ibmvnic_tasklet()
6089 static int ibmvnic_reenable_crq_queue(struct ibmvnic_adapter *adapter)  in ibmvnic_reenable_crq_queue()  argument
6091 	struct vio_dev *vdev = adapter->vdev;  in ibmvnic_reenable_crq_queue()
6104 static int ibmvnic_reset_crq(struct ibmvnic_adapter *adapter)  in ibmvnic_reset_crq()  argument
6106 	struct ibmvnic_crq_queue *crq = &adapter->crq;  in ibmvnic_reset_crq()
6107 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_reset_crq()
6108 	struct vio_dev *vdev = adapter->vdev;  in ibmvnic_reset_crq()
6137 static void release_crq_queue(struct ibmvnic_adapter *adapter)  in release_crq_queue()  argument
6139 	struct ibmvnic_crq_queue *crq = &adapter->crq;  in release_crq_queue()
6140 	struct vio_dev *vdev = adapter->vdev;  in release_crq_queue()
6146 	netdev_dbg(adapter->netdev, "Releasing CRQ\n");  in release_crq_queue()
6147 	free_irq(vdev->irq, adapter);  in release_crq_queue()
6148 	tasklet_kill(&adapter->tasklet);  in release_crq_queue()
6160 static int init_crq_queue(struct ibmvnic_adapter *adapter)  in init_crq_queue()  argument
6162 	struct ibmvnic_crq_queue *crq = &adapter->crq;  in init_crq_queue()
6163 	struct device *dev = &adapter->vdev->dev;  in init_crq_queue()
6164 	struct vio_dev *vdev = adapter->vdev;  in init_crq_queue()
6187 		rc = ibmvnic_reset_crq(adapter);  in init_crq_queue()
6199 	tasklet_setup(&adapter->tasklet, (void *)ibmvnic_tasklet);  in init_crq_queue()
6201 	netdev_dbg(adapter->netdev, "registering irq 0x%x\n", vdev->irq);  in init_crq_queue()
6203 		 adapter->vdev->unit_address);  in init_crq_queue()
6204 	rc = request_irq(vdev->irq, ibmvnic_interrupt, 0, crq->name, adapter);  in init_crq_queue()
6221 	tasklet_schedule(&adapter->tasklet);  in init_crq_queue()
6226 	tasklet_kill(&adapter->tasklet);  in init_crq_queue()
6238 static int ibmvnic_reset_init(struct ibmvnic_adapter *adapter, bool reset)  in ibmvnic_reset_init()  argument
6240 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_reset_init()
6242 	u64 old_num_rx_queues = adapter->req_rx_queues;  in ibmvnic_reset_init()
6243 	u64 old_num_tx_queues = adapter->req_tx_queues;  in ibmvnic_reset_init()
6246 	adapter->from_passive_init = false;  in ibmvnic_reset_init()
6248 	rc = ibmvnic_send_crq_init(adapter);  in ibmvnic_reset_init()
6254 	if (!wait_for_completion_timeout(&adapter->init_done, timeout)) {  in ibmvnic_reset_init()
6259 	if (adapter->init_done_rc) {  in ibmvnic_reset_init()
6260 		release_crq_queue(adapter);  in ibmvnic_reset_init()
6261 		dev_err(dev, "CRQ-init failed, %d\n", adapter->init_done_rc);  in ibmvnic_reset_init()
6262 		return adapter->init_done_rc;  in ibmvnic_reset_init()
6265 	if (adapter->from_passive_init) {  in ibmvnic_reset_init()
6266 		adapter->state = VNIC_OPEN;  in ibmvnic_reset_init()
6267 		adapter->from_passive_init = false;  in ibmvnic_reset_init()
6273 	    test_bit(0, &adapter->resetting) && !adapter->wait_for_reset &&  in ibmvnic_reset_init()
6274 	    adapter->reset_reason != VNIC_RESET_MOBILITY) {  in ibmvnic_reset_init()
6275 		if (adapter->req_rx_queues != old_num_rx_queues ||  in ibmvnic_reset_init()
6276 		    adapter->req_tx_queues != old_num_tx_queues) {  in ibmvnic_reset_init()
6277 			release_sub_crqs(adapter, 0);  in ibmvnic_reset_init()
6278 			rc = init_sub_crqs(adapter);  in ibmvnic_reset_init()
6287 			clean_tx_pools(adapter);  in ibmvnic_reset_init()
6289 			rc = reset_sub_crq_queues(adapter);  in ibmvnic_reset_init()
6292 		rc = init_sub_crqs(adapter);  in ibmvnic_reset_init()
6297 		release_crq_queue(adapter);  in ibmvnic_reset_init()
6301 	rc = init_sub_crq_irqs(adapter);  in ibmvnic_reset_init()
6304 		release_crq_queue(adapter);  in ibmvnic_reset_init()
6314 	struct ibmvnic_adapter *adapter;  in ibmvnic_probe()  local
6338 	adapter = netdev_priv(netdev);  in ibmvnic_probe()
6339 	adapter->state = VNIC_PROBING;  in ibmvnic_probe()
6341 	adapter->vdev = dev;  in ibmvnic_probe()
6342 	adapter->netdev = netdev;  in ibmvnic_probe()
6343 	adapter->login_pending = false;  in ibmvnic_probe()
6344 	memset(&adapter->map_ids, 0, sizeof(adapter->map_ids));  in ibmvnic_probe()
6346 	bitmap_set(adapter->map_ids, 0, 1);  in ibmvnic_probe()
6348 	ether_addr_copy(adapter->mac_addr, mac_addr_p);  in ibmvnic_probe()
6349 	eth_hw_addr_set(netdev, adapter->mac_addr);  in ibmvnic_probe()
6355 	INIT_WORK(&adapter->ibmvnic_reset, __ibmvnic_reset);  in ibmvnic_probe()
6356 	INIT_DELAYED_WORK(&adapter->ibmvnic_delayed_reset,  in ibmvnic_probe()
6358 	INIT_LIST_HEAD(&adapter->rwi_list);  in ibmvnic_probe()
6359 	spin_lock_init(&adapter->rwi_lock);  in ibmvnic_probe()
6360 	spin_lock_init(&adapter->state_lock);  in ibmvnic_probe()
6361 	mutex_init(&adapter->fw_lock);  in ibmvnic_probe()
6362 	init_completion(&adapter->probe_done);  in ibmvnic_probe()
6363 	init_completion(&adapter->init_done);  in ibmvnic_probe()
6364 	init_completion(&adapter->fw_done);  in ibmvnic_probe()
6365 	init_completion(&adapter->reset_done);  in ibmvnic_probe()
6366 	init_completion(&adapter->stats_done);  in ibmvnic_probe()
6367 	clear_bit(0, &adapter->resetting);  in ibmvnic_probe()
6368 	adapter->prev_rx_buf_sz = 0;  in ibmvnic_probe()
6369 	adapter->prev_mtu = 0;  in ibmvnic_probe()
6373 		reinit_init_done(adapter);  in ibmvnic_probe()
6378 		adapter->failover_pending = false;  in ibmvnic_probe()
6384 		release_crq_queue(adapter);  in ibmvnic_probe()
6396 		spin_lock_irqsave(&adapter->rwi_lock, flags);  in ibmvnic_probe()
6397 		flush_reset_queue(adapter);  in ibmvnic_probe()
6398 		spin_unlock_irqrestore(&adapter->rwi_lock, flags);  in ibmvnic_probe()
6400 		rc = init_crq_queue(adapter);  in ibmvnic_probe()
6407 		rc = ibmvnic_reset_init(adapter, false);  in ibmvnic_probe()
6418 	rc = init_stats_buffers(adapter);  in ibmvnic_probe()
6422 	rc = init_stats_token(adapter);  in ibmvnic_probe()
6433 		adapter->state = VNIC_PROBED;  in ibmvnic_probe()
6434 		netdev->mtu = adapter->req_mtu - ETH_HLEN;  in ibmvnic_probe()
6435 		netdev->min_mtu = adapter->min_mtu - ETH_HLEN;  in ibmvnic_probe()
6436 		netdev->max_mtu = adapter->max_mtu - ETH_HLEN;  in ibmvnic_probe()
6438 		adapter->state = VNIC_DOWN;  in ibmvnic_probe()
6441 	adapter->wait_for_reset = false;  in ibmvnic_probe()
6442 	adapter->last_reset_time = jiffies;  in ibmvnic_probe()
6451 	rc = ibmvnic_cpu_notif_add(adapter);  in ibmvnic_probe()
6457 	complete(&adapter->probe_done);  in ibmvnic_probe()
6468 	release_stats_token(adapter);  in ibmvnic_probe()
6471 	release_stats_buffers(adapter);  in ibmvnic_probe()
6474 	release_sub_crqs(adapter, 1);  in ibmvnic_probe()
6475 	release_crq_queue(adapter);  in ibmvnic_probe()
6480 	adapter->state = VNIC_REMOVING;  in ibmvnic_probe()
6481 	complete(&adapter->probe_done);  in ibmvnic_probe()
6482 	flush_work(&adapter->ibmvnic_reset);  in ibmvnic_probe()
6483 	flush_delayed_work(&adapter->ibmvnic_delayed_reset);  in ibmvnic_probe()
6485 	flush_reset_queue(adapter);  in ibmvnic_probe()
6487 	mutex_destroy(&adapter->fw_lock);  in ibmvnic_probe()
6496 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_remove()  local
6499 	spin_lock_irqsave(&adapter->state_lock, flags);  in ibmvnic_remove()
6508 	spin_lock(&adapter->rwi_lock);  in ibmvnic_remove()
6509 	adapter->state = VNIC_REMOVING;  in ibmvnic_remove()
6510 	spin_unlock(&adapter->rwi_lock);  in ibmvnic_remove()
6512 	spin_unlock_irqrestore(&adapter->state_lock, flags);  in ibmvnic_remove()
6514 	ibmvnic_cpu_notif_remove(adapter);  in ibmvnic_remove()
6516 	flush_work(&adapter->ibmvnic_reset);  in ibmvnic_remove()
6517 	flush_delayed_work(&adapter->ibmvnic_delayed_reset);  in ibmvnic_remove()
6522 	release_resources(adapter);  in ibmvnic_remove()
6523 	release_rx_pools(adapter);  in ibmvnic_remove()
6524 	release_tx_pools(adapter);  in ibmvnic_remove()
6525 	release_sub_crqs(adapter, 1);  in ibmvnic_remove()
6526 	release_crq_queue(adapter);  in ibmvnic_remove()
6528 	release_stats_token(adapter);  in ibmvnic_remove()
6529 	release_stats_buffers(adapter);  in ibmvnic_remove()
6531 	adapter->state = VNIC_REMOVED;  in ibmvnic_remove()
6534 	mutex_destroy(&adapter->fw_lock);  in ibmvnic_remove()
6544 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in failover_store()  local
6552 	rc = plpar_hcall(H_VIOCTL, retbuf, adapter->vdev->unit_address,  in failover_store()
6563 	rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,  in failover_store()
6576 	ibmvnic_reset(adapter, VNIC_RESET_FAILOVER);  in failover_store()
6585 	struct ibmvnic_adapter *adapter;  in ibmvnic_get_desired_dma()  local
6596 	adapter = netdev_priv(netdev);  in ibmvnic_get_desired_dma()
6601 	for (i = 0; i < adapter->req_tx_queues + adapter->req_rx_queues; i++)  in ibmvnic_get_desired_dma()
6604 	for (i = 0; i < adapter->num_active_rx_pools; i++)  in ibmvnic_get_desired_dma()
6605 		ret += adapter->rx_pool[i].size *  in ibmvnic_get_desired_dma()
6606 		    IOMMU_PAGE_ALIGN(adapter->rx_pool[i].buff_size, tbl);  in ibmvnic_get_desired_dma()
6614 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_resume()  local
6616 	if (adapter->state != VNIC_OPEN)  in ibmvnic_resume()
6619 	tasklet_schedule(&adapter->tasklet);  in ibmvnic_resume()