100 static int reset_sub_crq_queues(struct ibmvnic_adapter *adapter);
101 static inline void reinit_init_done(struct ibmvnic_adapter *adapter);
102 static void send_query_map(struct ibmvnic_adapter *adapter);
105 static int send_login(struct ibmvnic_adapter *adapter);
106 static void send_query_cap(struct ibmvnic_adapter *adapter);
108 static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter);
112 static int init_crq_queue(struct ibmvnic_adapter *adapter);
113 static int send_query_phys_parms(struct ibmvnic_adapter *adapter);
114 static void ibmvnic_tx_scrq_clean_buffer(struct ibmvnic_adapter *adapter,
116 static void free_long_term_buff(struct ibmvnic_adapter *adapter,
118 static void ibmvnic_disable_irqs(struct ibmvnic_adapter *adapter);
119 static void flush_reset_queue(struct ibmvnic_adapter *adapter);
156 static int send_crq_init_complete(struct ibmvnic_adapter *adapter)  in send_crq_init_complete()  argument
164 	return ibmvnic_send_crq(adapter, &crq);  in send_crq_init_complete()
167 static int send_version_xchg(struct ibmvnic_adapter *adapter)  in send_version_xchg()  argument
176 	return ibmvnic_send_crq(adapter, &crq);  in send_version_xchg()
179 static void ibmvnic_clean_queue_affinity(struct ibmvnic_adapter *adapter,  in ibmvnic_clean_queue_affinity()  argument
188 		netdev_warn(adapter->netdev,  in ibmvnic_clean_queue_affinity()
193 static void ibmvnic_clean_affinity(struct ibmvnic_adapter *adapter)  in ibmvnic_clean_affinity()  argument
200 	rxqs = adapter->rx_scrq;  in ibmvnic_clean_affinity()
201 	txqs = adapter->tx_scrq;  in ibmvnic_clean_affinity()
202 	num_txqs = adapter->num_active_tx_scrqs;  in ibmvnic_clean_affinity()
203 	num_rxqs = adapter->num_active_rx_scrqs;  in ibmvnic_clean_affinity()
205 	netdev_dbg(adapter->netdev, "%s: Cleaning irq affinity hints", __func__);  in ibmvnic_clean_affinity()
208 			ibmvnic_clean_queue_affinity(adapter, txqs[i]);  in ibmvnic_clean_affinity()
212 			ibmvnic_clean_queue_affinity(adapter, rxqs[i]);  in ibmvnic_clean_affinity()
251 static void ibmvnic_set_affinity(struct ibmvnic_adapter *adapter)  in ibmvnic_set_affinity()  argument
253 	struct ibmvnic_sub_crq_queue **rxqs = adapter->rx_scrq;  in ibmvnic_set_affinity()
254 	struct ibmvnic_sub_crq_queue **txqs = adapter->tx_scrq;  in ibmvnic_set_affinity()
256 	int num_rxqs = adapter->num_active_rx_scrqs, i_rxqs = 0;  in ibmvnic_set_affinity()
257 	int num_txqs = adapter->num_active_tx_scrqs, i_txqs = 0;  in ibmvnic_set_affinity()
263 	netdev_dbg(adapter->netdev, "%s: Setting irq affinity hints", __func__);  in ibmvnic_set_affinity()
264 	if (!(adapter->rx_scrq && adapter->tx_scrq)) {  in ibmvnic_set_affinity()
265 		netdev_warn(adapter->netdev,  in ibmvnic_set_affinity()
300 		rc = __netif_set_xps_queue(adapter->netdev,  in ibmvnic_set_affinity()
304 			netdev_warn(adapter->netdev, "%s: Set XPS on queue %d failed, rc = %d.\n",  in ibmvnic_set_affinity()
310 		netdev_warn(adapter->netdev,  in ibmvnic_set_affinity()
313 		ibmvnic_clean_affinity(adapter);  in ibmvnic_set_affinity()
319 	struct ibmvnic_adapter *adapter;  in ibmvnic_cpu_online()  local
321 	adapter = hlist_entry_safe(node, struct ibmvnic_adapter, node);  in ibmvnic_cpu_online()
322 	ibmvnic_set_affinity(adapter);  in ibmvnic_cpu_online()
328 	struct ibmvnic_adapter *adapter;  in ibmvnic_cpu_dead()  local
330 	adapter = hlist_entry_safe(node, struct ibmvnic_adapter, node_dead);  in ibmvnic_cpu_dead()
331 	ibmvnic_set_affinity(adapter);  in ibmvnic_cpu_dead()
337 	struct ibmvnic_adapter *adapter;  in ibmvnic_cpu_down_prep()  local
339 	adapter = hlist_entry_safe(node, struct ibmvnic_adapter, node);  in ibmvnic_cpu_down_prep()
340 	ibmvnic_clean_affinity(adapter);  in ibmvnic_cpu_down_prep()
346 static int ibmvnic_cpu_notif_add(struct ibmvnic_adapter *adapter)  in ibmvnic_cpu_notif_add()  argument
350 	ret = cpuhp_state_add_instance_nocalls(ibmvnic_online, &adapter->node);  in ibmvnic_cpu_notif_add()
354 					       &adapter->node_dead);  in ibmvnic_cpu_notif_add()
357 	cpuhp_state_remove_instance_nocalls(ibmvnic_online, &adapter->node);  in ibmvnic_cpu_notif_add()
361 static void ibmvnic_cpu_notif_remove(struct ibmvnic_adapter *adapter)  in ibmvnic_cpu_notif_remove()  argument
363 	cpuhp_state_remove_instance_nocalls(ibmvnic_online, &adapter->node);  in ibmvnic_cpu_notif_remove()
365 					    &adapter->node_dead);  in ibmvnic_cpu_notif_remove()
384  * @adapter: private device data
391 static int ibmvnic_wait_for_completion(struct ibmvnic_adapter *adapter,  in ibmvnic_wait_for_completion()  argument
399 	netdev = adapter->netdev;  in ibmvnic_wait_for_completion()
403 		if (!adapter->crq.active) {  in ibmvnic_wait_for_completion()
433  * @adapter: ibmvnic adapter associated to the LTB
446  * with the VIOS and reuse it on next open. Free LTB when adapter is closed.
451 static int alloc_long_term_buff(struct ibmvnic_adapter *adapter,  in alloc_long_term_buff()  argument
454 	struct device *dev = &adapter->vdev->dev;  in alloc_long_term_buff()
463 		free_long_term_buff(adapter, ltb);  in alloc_long_term_buff()
478 		ltb->map_id = find_first_zero_bit(adapter->map_ids,  in alloc_long_term_buff()
480 		bitmap_set(adapter->map_ids, ltb->map_id, 1);  in alloc_long_term_buff()
490 	mutex_lock(&adapter->fw_lock);  in alloc_long_term_buff()
491 	adapter->fw_done_rc = 0;  in alloc_long_term_buff()
492 	reinit_completion(&adapter->fw_done);  in alloc_long_term_buff()
494 	rc = send_request_map(adapter, ltb->addr, ltb->size, ltb->map_id);  in alloc_long_term_buff()
500 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);  in alloc_long_term_buff()
507 	if (adapter->fw_done_rc) {  in alloc_long_term_buff()
509 			adapter->fw_done_rc);  in alloc_long_term_buff()
516 	mutex_unlock(&adapter->fw_lock);  in alloc_long_term_buff()
520 static void free_long_term_buff(struct ibmvnic_adapter *adapter,  in free_long_term_buff()  argument
523 	struct device *dev = &adapter->vdev->dev;  in free_long_term_buff()
532 	if (adapter->reset_reason != VNIC_RESET_FAILOVER &&  in free_long_term_buff()
533 	    adapter->reset_reason != VNIC_RESET_MOBILITY &&  in free_long_term_buff()
534 	    adapter->reset_reason != VNIC_RESET_TIMEOUT)  in free_long_term_buff()
535 		send_request_unmap(adapter, ltb->map_id);  in free_long_term_buff()
541 	bitmap_clear(adapter->map_ids, ltb->map_id, 1);  in free_long_term_buff()
547  * @adapter: The ibmvnic adapter containing this ltb set
553 static void free_ltb_set(struct ibmvnic_adapter *adapter,  in free_ltb_set()  argument
559 		free_long_term_buff(adapter, &ltb_set->ltbs[i]);  in free_ltb_set()
569  * @adapter: ibmvnic adapter associated to the LTB
583 static int alloc_ltb_set(struct ibmvnic_adapter *adapter,  in alloc_ltb_set()  argument
587 	struct device *dev = &adapter->vdev->dev;  in alloc_ltb_set()
626 			free_long_term_buff(adapter, &old_set.ltbs[i]);  in alloc_ltb_set()
663 		rc = alloc_long_term_buff(adapter, &new_set.ltbs[i], ltb_size);  in alloc_ltb_set()
747 static void deactivate_rx_pools(struct ibmvnic_adapter *adapter)  in deactivate_rx_pools()  argument
751 	for (i = 0; i < adapter->num_active_rx_pools; i++)  in deactivate_rx_pools()
752 		adapter->rx_pool[i].active = 0;  in deactivate_rx_pools()
755 static void replenish_rx_pool(struct ibmvnic_adapter *adapter,  in replenish_rx_pool()  argument
759 	u64 handle = adapter->rx_scrq[pool->index]->handle;  in replenish_rx_pool()
760 	struct device *dev = &adapter->vdev->dev;  in replenish_rx_pool()
778 	rx_scrq = adapter->rx_scrq[pool->index];  in replenish_rx_pool()
797 			skb = netdev_alloc_skb(adapter->netdev,  in replenish_rx_pool()
801 				adapter->replenish_no_mem++;  in replenish_rx_pool()
845 				send_subcrq_indirect(adapter, handle,  in replenish_rx_pool()
851 			adapter->replenish_add_buff_success += ind_bufp->index;  in replenish_rx_pool()
874 	adapter->replenish_add_buff_failure += ind_bufp->index;  in replenish_rx_pool()
877 	if (lpar_rc == H_CLOSED || adapter->failover_pending) {  in replenish_rx_pool()
883 		deactivate_rx_pools(adapter);  in replenish_rx_pool()
884 		netif_carrier_off(adapter->netdev);  in replenish_rx_pool()
888 static void replenish_pools(struct ibmvnic_adapter *adapter)  in replenish_pools()  argument
892 	adapter->replenish_task_cycles++;  in replenish_pools()
893 	for (i = 0; i < adapter->num_active_rx_pools; i++) {  in replenish_pools()
894 		if (adapter->rx_pool[i].active)  in replenish_pools()
895 			replenish_rx_pool(adapter, &adapter->rx_pool[i]);  in replenish_pools()
898 	netdev_dbg(adapter->netdev, "Replenished %d pools\n", i);  in replenish_pools()
901 static void release_stats_buffers(struct ibmvnic_adapter *adapter)  in release_stats_buffers()  argument
903 	kfree(adapter->tx_stats_buffers);  in release_stats_buffers()
904 	kfree(adapter->rx_stats_buffers);  in release_stats_buffers()
905 	adapter->tx_stats_buffers = NULL;  in release_stats_buffers()
906 	adapter->rx_stats_buffers = NULL;  in release_stats_buffers()
909 static int init_stats_buffers(struct ibmvnic_adapter *adapter)  in init_stats_buffers()  argument
911 	adapter->tx_stats_buffers =  in init_stats_buffers()
915 	if (!adapter->tx_stats_buffers)  in init_stats_buffers()
918 	adapter->rx_stats_buffers =  in init_stats_buffers()
922 	if (!adapter->rx_stats_buffers)  in init_stats_buffers()
928 static void release_stats_token(struct ibmvnic_adapter *adapter)  in release_stats_token()  argument
930 	struct device *dev = &adapter->vdev->dev;  in release_stats_token()
932 	if (!adapter->stats_token)  in release_stats_token()
935 	dma_unmap_single(dev, adapter->stats_token,  in release_stats_token()
938 	adapter->stats_token = 0;  in release_stats_token()
941 static int init_stats_token(struct ibmvnic_adapter *adapter)  in init_stats_token()  argument
943 	struct device *dev = &adapter->vdev->dev;  in init_stats_token()
947 	stok = dma_map_single(dev, &adapter->stats,  in init_stats_token()
956 	adapter->stats_token = stok;  in init_stats_token()
957 	netdev_dbg(adapter->netdev, "Stats token initialized (%llx)\n", stok);  in init_stats_token()
962  * release_rx_pools() - Release any rx pools attached to @adapter.
963  * @adapter: ibmvnic adapter
967 static void release_rx_pools(struct ibmvnic_adapter *adapter)  in release_rx_pools()  argument
972 	if (!adapter->rx_pool)  in release_rx_pools()
975 	for (i = 0; i < adapter->num_active_rx_pools; i++) {  in release_rx_pools()
976 		rx_pool = &adapter->rx_pool[i];  in release_rx_pools()
978 		netdev_dbg(adapter->netdev, "Releasing rx_pool[%d]\n", i);  in release_rx_pools()
982 		free_ltb_set(adapter, &rx_pool->ltb_set);  in release_rx_pools()
997 	kfree(adapter->rx_pool);  in release_rx_pools()
998 	adapter->rx_pool = NULL;  in release_rx_pools()
999 	adapter->num_active_rx_pools = 0;  in release_rx_pools()
1000 	adapter->prev_rx_pool_size = 0;  in release_rx_pools()
1005  * @adapter: ibmvnic adapter
1007  * Check if the existing rx pools in the adapter can be reused. The
1017 static bool reuse_rx_pools(struct ibmvnic_adapter *adapter)  in reuse_rx_pools()  argument
1023 	if (!adapter->rx_pool)  in reuse_rx_pools()
1026 	old_num_pools = adapter->num_active_rx_pools;  in reuse_rx_pools()
1027 	new_num_pools = adapter->req_rx_queues;  in reuse_rx_pools()
1029 	old_pool_size = adapter->prev_rx_pool_size;  in reuse_rx_pools()
1030 	new_pool_size = adapter->req_rx_add_entries_per_subcrq;  in reuse_rx_pools()
1032 	old_buff_size = adapter->prev_rx_buf_sz;  in reuse_rx_pools()
1033 	new_buff_size = adapter->cur_rx_buf_sz;  in reuse_rx_pools()
1044  * init_rx_pools(): Initialize the set of receiver pools in the adapter.
1047  * Initialize the set of receiver pools in the ibmvnic adapter associated
1056 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in init_rx_pools()  local
1057 	struct device *dev = &adapter->vdev->dev;  in init_rx_pools()
1064 	pool_size = adapter->req_rx_add_entries_per_subcrq;  in init_rx_pools()
1065 	num_pools = adapter->req_rx_queues;  in init_rx_pools()
1066 	buff_size = adapter->cur_rx_buf_sz;  in init_rx_pools()
1068 	if (reuse_rx_pools(adapter)) {  in init_rx_pools()
1074 	release_rx_pools(adapter);  in init_rx_pools()
1076 	adapter->rx_pool = kcalloc(num_pools,  in init_rx_pools()
1079 	if (!adapter->rx_pool) {  in init_rx_pools()
1087 	adapter->num_active_rx_pools = num_pools;  in init_rx_pools()
1090 		rx_pool = &adapter->rx_pool[i];  in init_rx_pools()
1092 		netdev_dbg(adapter->netdev,  in init_rx_pools()
1118 	adapter->prev_rx_pool_size = pool_size;  in init_rx_pools()
1119 	adapter->prev_rx_buf_sz = adapter->cur_rx_buf_sz;  in init_rx_pools()
1123 		rx_pool = &adapter->rx_pool[i];  in init_rx_pools()
1127 		rc = alloc_ltb_set(adapter, &rx_pool->ltb_set,  in init_rx_pools()
1161 	release_rx_pools(adapter);  in init_rx_pools()
1169 static void release_vpd_data(struct ibmvnic_adapter *adapter)  in release_vpd_data()  argument
1171 	if (!adapter->vpd)  in release_vpd_data()
1174 	kfree(adapter->vpd->buff);  in release_vpd_data()
1175 	kfree(adapter->vpd);  in release_vpd_data()
1177 	adapter->vpd = NULL;  in release_vpd_data()
1180 static void release_one_tx_pool(struct ibmvnic_adapter *adapter,  in release_one_tx_pool()  argument
1185 	free_ltb_set(adapter, &tx_pool->ltb_set);  in release_one_tx_pool()
1189  * release_tx_pools() - Release any tx pools attached to @adapter.
1190  * @adapter: ibmvnic adapter
1194 static void release_tx_pools(struct ibmvnic_adapter *adapter)  in release_tx_pools()  argument
1201 	if (!adapter->tx_pool)  in release_tx_pools()
1204 	for (i = 0; i < adapter->num_active_tx_pools; i++) {  in release_tx_pools()
1205 		release_one_tx_pool(adapter, &adapter->tx_pool[i]);  in release_tx_pools()
1206 		release_one_tx_pool(adapter, &adapter->tso_pool[i]);  in release_tx_pools()
1209 	kfree(adapter->tx_pool);  in release_tx_pools()
1210 	adapter->tx_pool = NULL;  in release_tx_pools()
1211 	kfree(adapter->tso_pool);  in release_tx_pools()
1212 	adapter->tso_pool = NULL;  in release_tx_pools()
1213 	adapter->num_active_tx_pools = 0;  in release_tx_pools()
1214 	adapter->prev_tx_pool_size = 0;  in release_tx_pools()
1249  * @adapter: ibmvnic adapter
1251  * Check if the existing tx pools in the adapter can be reused. The
1260 static bool reuse_tx_pools(struct ibmvnic_adapter *adapter)  in reuse_tx_pools()  argument
1266 	if (!adapter->tx_pool)  in reuse_tx_pools()
1269 	old_num_pools = adapter->num_active_tx_pools;  in reuse_tx_pools()
1270 	new_num_pools = adapter->num_active_tx_scrqs;  in reuse_tx_pools()
1271 	old_pool_size = adapter->prev_tx_pool_size;  in reuse_tx_pools()
1272 	new_pool_size = adapter->req_tx_entries_per_subcrq;  in reuse_tx_pools()
1273 	old_mtu = adapter->prev_mtu;  in reuse_tx_pools()
1274 	new_mtu = adapter->req_mtu;  in reuse_tx_pools()
1285  * init_tx_pools(): Initialize the set of transmit pools in the adapter.
1288  * Initialize the set of transmit pools in the ibmvnic adapter associated
1297 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in init_tx_pools()  local
1298 	struct device *dev = &adapter->vdev->dev;  in init_tx_pools()
1304 	num_pools = adapter->req_tx_queues;  in init_tx_pools()
1310 	if (reuse_tx_pools(adapter)) {  in init_tx_pools()
1316 	release_tx_pools(adapter);  in init_tx_pools()
1318 	pool_size = adapter->req_tx_entries_per_subcrq;  in init_tx_pools()
1319 	num_pools = adapter->num_active_tx_scrqs;  in init_tx_pools()
1321 	adapter->tx_pool = kcalloc(num_pools,  in init_tx_pools()
1323 	if (!adapter->tx_pool)  in init_tx_pools()
1326 	adapter->tso_pool = kcalloc(num_pools,  in init_tx_pools()
1331 	if (!adapter->tso_pool) {  in init_tx_pools()
1332 		kfree(adapter->tx_pool);  in init_tx_pools()
1333 		adapter->tx_pool = NULL;  in init_tx_pools()
1340 	adapter->num_active_tx_pools = num_pools;  in init_tx_pools()
1342 	buff_size = adapter->req_mtu + VLAN_HLEN;  in init_tx_pools()
1347 			i, adapter->req_tx_entries_per_subcrq, buff_size);  in init_tx_pools()
1349 		rc = init_one_tx_pool(netdev, &adapter->tx_pool[i],  in init_tx_pools()
1354 		rc = init_one_tx_pool(netdev, &adapter->tso_pool[i],  in init_tx_pools()
1361 	adapter->prev_tx_pool_size = pool_size;  in init_tx_pools()
1362 	adapter->prev_mtu = adapter->req_mtu;  in init_tx_pools()
1376 		tx_pool = &adapter->tx_pool[i];  in init_tx_pools()
1381 		rc = alloc_ltb_set(adapter, &tx_pool->ltb_set,  in init_tx_pools()
1392 		tso_pool = &adapter->tso_pool[i];  in init_tx_pools()
1397 		rc = alloc_ltb_set(adapter, &tso_pool->ltb_set,  in init_tx_pools()
1411 	release_tx_pools(adapter);  in init_tx_pools()
1419 static void ibmvnic_napi_enable(struct ibmvnic_adapter *adapter)  in ibmvnic_napi_enable()  argument
1423 	if (adapter->napi_enabled)  in ibmvnic_napi_enable()
1426 	for (i = 0; i < adapter->req_rx_queues; i++)  in ibmvnic_napi_enable()
1427 		napi_enable(&adapter->napi[i]);  in ibmvnic_napi_enable()
1429 	adapter->napi_enabled = true;  in ibmvnic_napi_enable()
1432 static void ibmvnic_napi_disable(struct ibmvnic_adapter *adapter)  in ibmvnic_napi_disable()  argument
1436 	if (!adapter->napi_enabled)  in ibmvnic_napi_disable()
1439 	for (i = 0; i < adapter->req_rx_queues; i++) {  in ibmvnic_napi_disable()
1440 		netdev_dbg(adapter->netdev, "Disabling napi[%d]\n", i);  in ibmvnic_napi_disable()
1441 		napi_disable(&adapter->napi[i]);  in ibmvnic_napi_disable()
1444 	adapter->napi_enabled = false;  in ibmvnic_napi_disable()
1447 static int init_napi(struct ibmvnic_adapter *adapter)  in init_napi()  argument
1451 	adapter->napi = kcalloc(adapter->req_rx_queues,  in init_napi()
1453 	if (!adapter->napi)  in init_napi()
1456 	for (i = 0; i < adapter->req_rx_queues; i++) {  in init_napi()
1457 		netdev_dbg(adapter->netdev, "Adding napi[%d]\n", i);  in init_napi()
1458 		netif_napi_add(adapter->netdev, &adapter->napi[i],  in init_napi()
1462 	adapter->num_active_rx_napi = adapter->req_rx_queues;  in init_napi()
1466 static void release_napi(struct ibmvnic_adapter *adapter)  in release_napi()  argument
1470 	if (!adapter->napi)  in release_napi()
1473 	for (i = 0; i < adapter->num_active_rx_napi; i++) {  in release_napi()
1474 		netdev_dbg(adapter->netdev, "Releasing napi[%d]\n", i);  in release_napi()
1475 		netif_napi_del(&adapter->napi[i]);  in release_napi()
1478 	kfree(adapter->napi);  in release_napi()
1479 	adapter->napi = NULL;  in release_napi()
1480 	adapter->num_active_rx_napi = 0;  in release_napi()
1481 	adapter->napi_enabled = false;  in release_napi()
1512 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_login()  local
1525 		adapter->init_done_rc = 0;  in ibmvnic_login()
1526 		reinit_completion(&adapter->init_done);  in ibmvnic_login()
1527 		rc = send_login(adapter);  in ibmvnic_login()
1531 		if (!wait_for_completion_timeout(&adapter->init_done,  in ibmvnic_login()
1534 			adapter->login_pending = false;  in ibmvnic_login()
1538 		if (adapter->init_done_rc == ABORTED) {  in ibmvnic_login()
1541 			adapter->init_done_rc = 0;  in ibmvnic_login()
1547 		} else if (adapter->init_done_rc == PARTIALSUCCESS) {  in ibmvnic_login()
1549 			release_sub_crqs(adapter, 1);  in ibmvnic_login()
1554 			adapter->init_done_rc = 0;  in ibmvnic_login()
1555 			reinit_completion(&adapter->init_done);  in ibmvnic_login()
1556 			send_query_cap(adapter);  in ibmvnic_login()
1557 			if (!wait_for_completion_timeout(&adapter->init_done,  in ibmvnic_login()
1564 			rc = init_sub_crqs(adapter);  in ibmvnic_login()
1571 			rc = init_sub_crq_irqs(adapter);  in ibmvnic_login()
1578 		} else if (adapter->init_done_rc) {  in ibmvnic_login()
1579 			netdev_warn(netdev, "Adapter login failed, init_done_rc = %d\n",  in ibmvnic_login()
1580 				    adapter->init_done_rc);  in ibmvnic_login()
1583 			/* adapter login failed, so free any CRQs or sub-CRQs  in ibmvnic_login()
1592 			adapter->init_done_rc = 0;  in ibmvnic_login()
1593 			release_sub_crqs(adapter, true);  in ibmvnic_login()
1601 				reinit_init_done(adapter);  in ibmvnic_login()
1605 				adapter->failover_pending = false;  in ibmvnic_login()
1606 				release_crq_queue(adapter);  in ibmvnic_login()
1617 				spin_lock_irqsave(&adapter->rwi_lock, flags);  in ibmvnic_login()
1618 				flush_reset_queue(adapter);  in ibmvnic_login()
1619 				spin_unlock_irqrestore(&adapter->rwi_lock,  in ibmvnic_login()
1622 				rc = init_crq_queue(adapter);  in ibmvnic_login()
1629 				rc = ibmvnic_reset_init(adapter, false);  in ibmvnic_login()
1643 	__ibmvnic_set_mac(netdev, adapter->mac_addr);  in ibmvnic_login()
1645 	netdev_dbg(netdev, "[S:%s] Login succeeded\n", adapter_state_to_string(adapter->state));  in ibmvnic_login()
1649 static void release_login_buffer(struct ibmvnic_adapter *adapter)  in release_login_buffer()  argument
1651 	if (!adapter->login_buf)  in release_login_buffer()
1654 	dma_unmap_single(&adapter->vdev->dev, adapter->login_buf_token,  in release_login_buffer()
1655 			 adapter->login_buf_sz, DMA_TO_DEVICE);  in release_login_buffer()
1656 	kfree(adapter->login_buf);  in release_login_buffer()
1657 	adapter->login_buf = NULL;  in release_login_buffer()
1660 static void release_login_rsp_buffer(struct ibmvnic_adapter *adapter)  in release_login_rsp_buffer()  argument
1662 	if (!adapter->login_rsp_buf)  in release_login_rsp_buffer()
1665 	dma_unmap_single(&adapter->vdev->dev, adapter->login_rsp_buf_token,  in release_login_rsp_buffer()
1666 			 adapter->login_rsp_buf_sz, DMA_FROM_DEVICE);  in release_login_rsp_buffer()
1667 	kfree(adapter->login_rsp_buf);  in release_login_rsp_buffer()
1668 	adapter->login_rsp_buf = NULL;  in release_login_rsp_buffer()
1671 static void release_resources(struct ibmvnic_adapter *adapter)  in release_resources()  argument
1673 	release_vpd_data(adapter);  in release_resources()
1675 	release_napi(adapter);  in release_resources()
1676 	release_login_buffer(adapter);  in release_resources()
1677 	release_login_rsp_buffer(adapter);  in release_resources()
1680 static int set_link_state(struct ibmvnic_adapter *adapter, u8 link_state)  in set_link_state()  argument
1682 	struct net_device *netdev = adapter->netdev;  in set_link_state()
1698 		reinit_completion(&adapter->init_done);  in set_link_state()
1699 		rc = ibmvnic_send_crq(adapter, &crq);  in set_link_state()
1705 		if (!wait_for_completion_timeout(&adapter->init_done,  in set_link_state()
1711 		if (adapter->init_done_rc == PARTIALSUCCESS) {  in set_link_state()
1715 		} else if (adapter->init_done_rc) {  in set_link_state()
1717 				    adapter->init_done_rc);  in set_link_state()
1718 			return adapter->init_done_rc;  in set_link_state()
1727 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in set_real_num_queues()  local
1731 		   adapter->req_tx_queues, adapter->req_rx_queues);  in set_real_num_queues()
1733 	rc = netif_set_real_num_tx_queues(netdev, adapter->req_tx_queues);  in set_real_num_queues()
1739 	rc = netif_set_real_num_rx_queues(netdev, adapter->req_rx_queues);  in set_real_num_queues()
1746 static int ibmvnic_get_vpd(struct ibmvnic_adapter *adapter)  in ibmvnic_get_vpd()  argument
1748 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_get_vpd()
1753 	if (adapter->vpd->buff)  in ibmvnic_get_vpd()
1754 		len = adapter->vpd->len;  in ibmvnic_get_vpd()
1756 	mutex_lock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1757 	adapter->fw_done_rc = 0;  in ibmvnic_get_vpd()
1758 	reinit_completion(&adapter->fw_done);  in ibmvnic_get_vpd()
1762 	rc = ibmvnic_send_crq(adapter, &crq);  in ibmvnic_get_vpd()
1764 		mutex_unlock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1768 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);  in ibmvnic_get_vpd()
1771 		mutex_unlock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1774 	mutex_unlock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1776 	if (!adapter->vpd->len)  in ibmvnic_get_vpd()
1779 	if (!adapter->vpd->buff)  in ibmvnic_get_vpd()
1780 		adapter->vpd->buff = kzalloc(adapter->vpd->len, GFP_KERNEL);  in ibmvnic_get_vpd()
1781 	else if (adapter->vpd->len != len)  in ibmvnic_get_vpd()
1782 		adapter->vpd->buff =  in ibmvnic_get_vpd()
1783 			krealloc(adapter->vpd->buff,  in ibmvnic_get_vpd()
1784 				 adapter->vpd->len, GFP_KERNEL);  in ibmvnic_get_vpd()
1786 	if (!adapter->vpd->buff) {  in ibmvnic_get_vpd()
1791 	adapter->vpd->dma_addr =  in ibmvnic_get_vpd()
1792 		dma_map_single(dev, adapter->vpd->buff, adapter->vpd->len,  in ibmvnic_get_vpd()
1794 	if (dma_mapping_error(dev, adapter->vpd->dma_addr)) {  in ibmvnic_get_vpd()
1796 		kfree(adapter->vpd->buff);  in ibmvnic_get_vpd()
1797 		adapter->vpd->buff = NULL;  in ibmvnic_get_vpd()
1801 	mutex_lock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1802 	adapter->fw_done_rc = 0;  in ibmvnic_get_vpd()
1803 	reinit_completion(&adapter->fw_done);  in ibmvnic_get_vpd()
1807 	crq.get_vpd.ioba = cpu_to_be32(adapter->vpd->dma_addr);  in ibmvnic_get_vpd()
1808 	crq.get_vpd.len = cpu_to_be32((u32)adapter->vpd->len);  in ibmvnic_get_vpd()
1809 	rc = ibmvnic_send_crq(adapter, &crq);  in ibmvnic_get_vpd()
1811 		kfree(adapter->vpd->buff);  in ibmvnic_get_vpd()
1812 		adapter->vpd->buff = NULL;  in ibmvnic_get_vpd()
1813 		mutex_unlock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1817 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);  in ibmvnic_get_vpd()
1820 		kfree(adapter->vpd->buff);  in ibmvnic_get_vpd()
1821 		adapter->vpd->buff = NULL;  in ibmvnic_get_vpd()
1822 		mutex_unlock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1826 	mutex_unlock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1830 static int init_resources(struct ibmvnic_adapter *adapter)  in init_resources()  argument
1832 	struct net_device *netdev = adapter->netdev;  in init_resources()
1839 	adapter->vpd = kzalloc(sizeof(*adapter->vpd), GFP_KERNEL);  in init_resources()
1840 	if (!adapter->vpd)  in init_resources()
1844 	rc = ibmvnic_get_vpd(adapter);  in init_resources()
1850 	rc = init_napi(adapter);  in init_resources()
1854 	send_query_map(adapter);  in init_resources()
1866 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in __ibmvnic_open()  local
1867 	enum vnic_state prev_state = adapter->state;  in __ibmvnic_open()
1870 	adapter->state = VNIC_OPENING;  in __ibmvnic_open()
1871 	replenish_pools(adapter);  in __ibmvnic_open()
1872 	ibmvnic_napi_enable(adapter);  in __ibmvnic_open()
1877 	for (i = 0; i < adapter->req_rx_queues; i++) {  in __ibmvnic_open()
1880 			enable_irq(adapter->rx_scrq[i]->irq);  in __ibmvnic_open()
1881 		enable_scrq_irq(adapter, adapter->rx_scrq[i]);  in __ibmvnic_open()
1884 	for (i = 0; i < adapter->req_tx_queues; i++) {  in __ibmvnic_open()
1887 			enable_irq(adapter->tx_scrq[i]->irq);  in __ibmvnic_open()
1888 		enable_scrq_irq(adapter, adapter->tx_scrq[i]);  in __ibmvnic_open()
1895 		if (adapter->reset_reason != VNIC_RESET_NON_FATAL)  in __ibmvnic_open()
1899 	rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_UP);  in __ibmvnic_open()
1901 		ibmvnic_napi_disable(adapter);  in __ibmvnic_open()
1902 		ibmvnic_disable_irqs(adapter);  in __ibmvnic_open()
1906 	adapter->tx_queues_active = true;  in __ibmvnic_open()
1918 		for (i = 0; i < adapter->req_rx_queues; i++)  in __ibmvnic_open()
1919 			napi_schedule(&adapter->napi[i]);  in __ibmvnic_open()
1922 	adapter->state = VNIC_OPEN;  in __ibmvnic_open()
1928 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_open()  local
1937 	 * It should be safe to overwrite the adapter->state here. Since  in ibmvnic_open()
1944 	if (adapter->failover_pending || (test_bit(0, &adapter->resetting))) {  in ibmvnic_open()
1946 			   adapter_state_to_string(adapter->state),  in ibmvnic_open()
1947 			   adapter->failover_pending);  in ibmvnic_open()
1948 		adapter->state = VNIC_OPEN;  in ibmvnic_open()
1953 	if (adapter->state != VNIC_CLOSED) {  in ibmvnic_open()
1958 		rc = init_resources(adapter);  in ibmvnic_open()
1973 	    (adapter->failover_pending || (test_bit(0, &adapter->resetting)))) {  in ibmvnic_open()
1974 		adapter->state = VNIC_OPEN;  in ibmvnic_open()
1979 		release_resources(adapter);  in ibmvnic_open()
1980 		release_rx_pools(adapter);  in ibmvnic_open()
1981 		release_tx_pools(adapter);  in ibmvnic_open()
1987 static void clean_rx_pools(struct ibmvnic_adapter *adapter)  in clean_rx_pools()  argument
1995 	if (!adapter->rx_pool)  in clean_rx_pools()
1998 	rx_scrqs = adapter->num_active_rx_pools;  in clean_rx_pools()
1999 	rx_entries = adapter->req_rx_add_entries_per_subcrq;  in clean_rx_pools()
2003 		rx_pool = &adapter->rx_pool[i];  in clean_rx_pools()
2007 		netdev_dbg(adapter->netdev, "Cleaning rx_pool[%d]\n", i);  in clean_rx_pools()
2018 static void clean_one_tx_pool(struct ibmvnic_adapter *adapter,  in clean_one_tx_pool()  argument
2039 static void clean_tx_pools(struct ibmvnic_adapter *adapter)  in clean_tx_pools()  argument
2044 	if (!adapter->tx_pool || !adapter->tso_pool)  in clean_tx_pools()
2047 	tx_scrqs = adapter->num_active_tx_pools;  in clean_tx_pools()
2051 		netdev_dbg(adapter->netdev, "Cleaning tx_pool[%d]\n", i);  in clean_tx_pools()
2052 		clean_one_tx_pool(adapter, &adapter->tx_pool[i]);  in clean_tx_pools()
2053 		clean_one_tx_pool(adapter, &adapter->tso_pool[i]);  in clean_tx_pools()
2057 static void ibmvnic_disable_irqs(struct ibmvnic_adapter *adapter)  in ibmvnic_disable_irqs()  argument
2059 	struct net_device *netdev = adapter->netdev;  in ibmvnic_disable_irqs()
2062 	if (adapter->tx_scrq) {  in ibmvnic_disable_irqs()
2063 		for (i = 0; i < adapter->req_tx_queues; i++)  in ibmvnic_disable_irqs()
2064 			if (adapter->tx_scrq[i]->irq) {  in ibmvnic_disable_irqs()
2067 				disable_scrq_irq(adapter, adapter->tx_scrq[i]);  in ibmvnic_disable_irqs()
2068 				disable_irq(adapter->tx_scrq[i]->irq);  in ibmvnic_disable_irqs()
2072 	if (adapter->rx_scrq) {  in ibmvnic_disable_irqs()
2073 		for (i = 0; i < adapter->req_rx_queues; i++) {  in ibmvnic_disable_irqs()
2074 			if (adapter->rx_scrq[i]->irq) {  in ibmvnic_disable_irqs()
2077 				disable_scrq_irq(adapter, adapter->rx_scrq[i]);  in ibmvnic_disable_irqs()
2078 				disable_irq(adapter->rx_scrq[i]->irq);  in ibmvnic_disable_irqs()
2086 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_cleanup()  local
2090 	adapter->tx_queues_active = false;  in ibmvnic_cleanup()
2097 	if (test_bit(0, &adapter->resetting))  in ibmvnic_cleanup()
2102 	ibmvnic_napi_disable(adapter);  in ibmvnic_cleanup()
2103 	ibmvnic_disable_irqs(adapter);  in ibmvnic_cleanup()
2108 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in __ibmvnic_close()  local
2111 	adapter->state = VNIC_CLOSING;  in __ibmvnic_close()
2112 	rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_DN);  in __ibmvnic_close()
2113 	adapter->state = VNIC_CLOSED;  in __ibmvnic_close()
2119 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_close()  local
2123 		   adapter_state_to_string(adapter->state),  in ibmvnic_close()
2124 		   adapter->failover_pending,  in ibmvnic_close()
2125 		   adapter->force_reset_recovery);  in ibmvnic_close()
2130 	if (adapter->failover_pending) {  in ibmvnic_close()
2131 		adapter->state = VNIC_CLOSED;  in ibmvnic_close()
2137 	clean_rx_pools(adapter);  in ibmvnic_close()
2138 	clean_tx_pools(adapter);  in ibmvnic_close()
2295 static void ibmvnic_tx_scrq_clean_buffer(struct ibmvnic_adapter *adapter,  in ibmvnic_tx_scrq_clean_buffer()  argument
2317 			tx_pool = &adapter->tso_pool[queue_num];  in ibmvnic_tx_scrq_clean_buffer()
2320 			tx_pool = &adapter->tx_pool[queue_num];  in ibmvnic_tx_scrq_clean_buffer()
2327 		adapter->netdev->stats.tx_packets--;  in ibmvnic_tx_scrq_clean_buffer()
2328 		adapter->netdev->stats.tx_bytes -= tx_buff->skb->len;  in ibmvnic_tx_scrq_clean_buffer()
2329 		adapter->tx_stats_buffers[queue_num].batched_packets--;  in ibmvnic_tx_scrq_clean_buffer()
2330 		adapter->tx_stats_buffers[queue_num].bytes -=  in ibmvnic_tx_scrq_clean_buffer()
2334 		adapter->netdev->stats.tx_dropped++;  in ibmvnic_tx_scrq_clean_buffer()
2340 	    (adapter->req_tx_entries_per_subcrq / 2) &&  in ibmvnic_tx_scrq_clean_buffer()
2341 	    __netif_subqueue_stopped(adapter->netdev, queue_num)) {  in ibmvnic_tx_scrq_clean_buffer()
2344 		if (adapter->tx_queues_active) {  in ibmvnic_tx_scrq_clean_buffer()
2345 			netif_wake_subqueue(adapter->netdev, queue_num);  in ibmvnic_tx_scrq_clean_buffer()
2346 			netdev_dbg(adapter->netdev, "Started queue %d\n",  in ibmvnic_tx_scrq_clean_buffer()
2354 static int send_subcrq_direct(struct ibmvnic_adapter *adapter,  in send_subcrq_direct()  argument
2357 	unsigned int ua = adapter->vdev->unit_address;  in send_subcrq_direct()
2358 	struct device *dev = &adapter->vdev->dev;  in send_subcrq_direct()
2374 static int ibmvnic_tx_scrq_flush(struct ibmvnic_adapter *adapter,  in ibmvnic_tx_scrq_flush()  argument
2393 		rc = send_subcrq_indirect(adapter, handle, dma_addr, entries);  in ibmvnic_tx_scrq_flush()
2395 		rc = send_subcrq_direct(adapter, handle,  in ibmvnic_tx_scrq_flush()
2399 		ibmvnic_tx_scrq_clean_buffer(adapter, tx_scrq);  in ibmvnic_tx_scrq_flush()
2407 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_xmit()  local
2409 	u8 *hdrs = (u8 *)&adapter->tx_rx_desc_req;  in ibmvnic_xmit()
2410 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_xmit()
2441 	if (!adapter->tx_queues_active) {  in ibmvnic_xmit()
2450 	tx_scrq = adapter->tx_scrq[queue_num];  in ibmvnic_xmit()
2458 		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq, true);  in ibmvnic_xmit()
2465 		tx_pool = &adapter->tso_pool[queue_num];  in ibmvnic_xmit()
2467 		tx_pool = &adapter->tx_pool[queue_num];  in ibmvnic_xmit()
2476 		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq, true);  in ibmvnic_xmit()
2563 	if (adapter->vlan_header_insertion && skb_vlan_tag_present(skb)) {  in ibmvnic_xmit()
2596 		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq, false);  in ibmvnic_xmit()
2611 		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq, true);  in ibmvnic_xmit()
2624 		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq, true);  in ibmvnic_xmit()
2633 					>= adapter->req_tx_entries_per_subcrq) {  in ibmvnic_xmit()
2654 	if (lpar_rc == H_CLOSED || adapter->failover_pending) {  in ibmvnic_xmit()
2668 	adapter->tx_send_failed += tx_send_failed;  in ibmvnic_xmit()
2669 	adapter->tx_map_failed += tx_map_failed;  in ibmvnic_xmit()
2670 	adapter->tx_stats_buffers[queue_num].batched_packets += tx_bpackets;  in ibmvnic_xmit()
2671 	adapter->tx_stats_buffers[queue_num].direct_packets += tx_dpackets;  in ibmvnic_xmit()
2672 	adapter->tx_stats_buffers[queue_num].bytes += tx_bytes;  in ibmvnic_xmit()
2673 	adapter->tx_stats_buffers[queue_num].dropped_packets += tx_dropped;  in ibmvnic_xmit()
2680 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_set_multi()  local
2689 		if (!adapter->promisc_supported)  in ibmvnic_set_multi()
2698 			ibmvnic_send_crq(adapter, &crq);  in ibmvnic_set_multi()
2705 			ibmvnic_send_crq(adapter, &crq);  in ibmvnic_set_multi()
2715 				ibmvnic_send_crq(adapter, &crq);  in ibmvnic_set_multi()
2723 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in __ibmvnic_set_mac()  local
2737 	mutex_lock(&adapter->fw_lock);  in __ibmvnic_set_mac()
2738 	adapter->fw_done_rc = 0;  in __ibmvnic_set_mac()
2739 	reinit_completion(&adapter->fw_done);  in __ibmvnic_set_mac()
2741 	rc = ibmvnic_send_crq(adapter, &crq);  in __ibmvnic_set_mac()
2744 		mutex_unlock(&adapter->fw_lock);  in __ibmvnic_set_mac()
2748 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);  in __ibmvnic_set_mac()
2750 	if (rc || adapter->fw_done_rc) {  in __ibmvnic_set_mac()
2752 		mutex_unlock(&adapter->fw_lock);  in __ibmvnic_set_mac()
2755 	mutex_unlock(&adapter->fw_lock);  in __ibmvnic_set_mac()
2758 	ether_addr_copy(adapter->mac_addr, netdev->dev_addr);  in __ibmvnic_set_mac()
2764 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_set_mac()  local
2772 	ether_addr_copy(adapter->mac_addr, addr->sa_data);  in ibmvnic_set_mac()
2773 	if (adapter->state != VNIC_PROBED)  in ibmvnic_set_mac()
2807 static inline void reinit_init_done(struct ibmvnic_adapter *adapter)  in reinit_init_done()  argument
2809 	reinit_completion(&adapter->init_done);  in reinit_init_done()
2810 	adapter->init_done_rc = 0;  in reinit_init_done()
2817 static int do_reset(struct ibmvnic_adapter *adapter,  in do_reset()  argument
2820 	struct net_device *netdev = adapter->netdev;  in do_reset()
2825 	netdev_dbg(adapter->netdev,  in do_reset()
2827 		   adapter_state_to_string(adapter->state),  in do_reset()
2828 		   adapter->failover_pending,  in do_reset()
2832 	adapter->reset_reason = rwi->reset_reason;  in do_reset()
2834 	if (!(adapter->reset_reason == VNIC_RESET_CHANGE_PARAM))  in do_reset()
2842 		adapter->failover_pending = false;  in do_reset()
2845 	reset_state = adapter->state;  in do_reset()
2854 	old_num_rx_queues = adapter->req_rx_queues;  in do_reset()
2855 	old_num_tx_queues = adapter->req_tx_queues;  in do_reset()
2856 	old_num_rx_slots = adapter->req_rx_add_entries_per_subcrq;  in do_reset()
2857 	old_num_tx_slots = adapter->req_tx_entries_per_subcrq;  in do_reset()
2862 	    adapter->reset_reason != VNIC_RESET_MOBILITY &&  in do_reset()
2863 	    adapter->reset_reason != VNIC_RESET_FAILOVER) {  in do_reset()
2864 		if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {  in do_reset()
2869 			adapter->state = VNIC_CLOSING;  in do_reset()
2877 			rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_DN);  in do_reset()
2882 			if (adapter->state == VNIC_OPEN) {  in do_reset()
2885 				 * set the adapter state to OPEN. Update our  in do_reset()
2893 				adapter->state = VNIC_CLOSING;  in do_reset()
2896 			if (adapter->state != VNIC_CLOSING) {  in do_reset()
2897 				/* If someone else changed the adapter state  in do_reset()
2903 			adapter->state = VNIC_CLOSED;  in do_reset()
2907 	if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {  in do_reset()
2908 		release_resources(adapter);  in do_reset()
2909 		release_sub_crqs(adapter, 1);  in do_reset()
2910 		release_crq_queue(adapter);  in do_reset()
2913 	if (adapter->reset_reason != VNIC_RESET_NON_FATAL) {  in do_reset()
2917 		adapter->state = VNIC_PROBED;  in do_reset()
2919 		reinit_init_done(adapter);  in do_reset()
2921 		if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {  in do_reset()
2922 			rc = init_crq_queue(adapter);  in do_reset()
2923 		} else if (adapter->reset_reason == VNIC_RESET_MOBILITY) {  in do_reset()
2924 			rc = ibmvnic_reenable_crq_queue(adapter);  in do_reset()
2925 			release_sub_crqs(adapter, 1);  in do_reset()
2927 			rc = ibmvnic_reset_crq(adapter);  in do_reset()
2929 				rc = vio_enable_interrupts(adapter->vdev);  in do_reset()
2931 					netdev_err(adapter->netdev,  in do_reset()
2938 			netdev_err(adapter->netdev,  in do_reset()
2943 		rc = ibmvnic_reset_init(adapter, true);  in do_reset()
2947 		/* If the adapter was in PROBE or DOWN state prior to the reset,  in do_reset()
2959 		if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {  in do_reset()
2960 			rc = init_resources(adapter);  in do_reset()
2963 		} else if (adapter->req_rx_queues != old_num_rx_queues ||  in do_reset()
2964 		    adapter->req_tx_queues != old_num_tx_queues ||  in do_reset()
2965 		    adapter->req_rx_add_entries_per_subcrq !=  in do_reset()
2967 		    adapter->req_tx_entries_per_subcrq !=  in do_reset()
2969 		    !adapter->rx_pool ||  in do_reset()
2970 		    !adapter->tso_pool ||  in do_reset()
2971 		    !adapter->tx_pool) {  in do_reset()
2972 			release_napi(adapter);  in do_reset()
2973 			release_vpd_data(adapter);  in do_reset()
2975 			rc = init_resources(adapter);  in do_reset()
2996 		ibmvnic_disable_irqs(adapter);  in do_reset()
2998 	adapter->state = VNIC_CLOSED;  in do_reset()
3014 	if (adapter->reset_reason == VNIC_RESET_FAILOVER ||  in do_reset()
3015 	    adapter->reset_reason == VNIC_RESET_MOBILITY)  in do_reset()
3021 	/* restore the adapter state if reset failed */  in do_reset()
3023 		adapter->state = reset_state;  in do_reset()
3025 	if (!(adapter->reset_reason == VNIC_RESET_CHANGE_PARAM))  in do_reset()
3028 	netdev_dbg(adapter->netdev, "[S:%s FOP:%d] Reset done, rc %d\n",  in do_reset()
3029 		   adapter_state_to_string(adapter->state),  in do_reset()
3030 		   adapter->failover_pending, rc);  in do_reset()
3034 static int do_hard_reset(struct ibmvnic_adapter *adapter,  in do_hard_reset()  argument
3037 	struct net_device *netdev = adapter->netdev;  in do_hard_reset()
3040 	netdev_dbg(adapter->netdev, "Hard resetting driver (%s)\n",  in do_hard_reset()
3044 	reset_state = adapter->state;  in do_hard_reset()
3052 	adapter->reset_reason = rwi->reset_reason;  in do_hard_reset()
3055 	release_resources(adapter);  in do_hard_reset()
3056 	release_sub_crqs(adapter, 0);  in do_hard_reset()
3057 	release_crq_queue(adapter);  in do_hard_reset()
3062 	adapter->state = VNIC_PROBED;  in do_hard_reset()
3064 	reinit_init_done(adapter);  in do_hard_reset()
3066 	rc = init_crq_queue(adapter);  in do_hard_reset()
3068 		netdev_err(adapter->netdev,  in do_hard_reset()
3073 	rc = ibmvnic_reset_init(adapter, false);  in do_hard_reset()
3077 	/* If the adapter was in PROBE or DOWN state prior to the reset,  in do_hard_reset()
3087 	rc = init_resources(adapter);  in do_hard_reset()
3091 	ibmvnic_disable_irqs(adapter);  in do_hard_reset()
3092 	adapter->state = VNIC_CLOSED;  in do_hard_reset()
3105 	/* restore adapter state if reset failed */  in do_hard_reset()
3107 		adapter->state = reset_state;  in do_hard_reset()
3108 	netdev_dbg(adapter->netdev, "[S:%s FOP:%d] Hard reset done, rc %d\n",  in do_hard_reset()
3109 		   adapter_state_to_string(adapter->state),  in do_hard_reset()
3110 		   adapter->failover_pending, rc);  in do_hard_reset()
3114 static struct ibmvnic_rwi *get_next_rwi(struct ibmvnic_adapter *adapter)  in get_next_rwi()  argument
3119 	spin_lock_irqsave(&adapter->rwi_lock, flags);  in get_next_rwi()
3121 	if (!list_empty(&adapter->rwi_list)) {  in get_next_rwi()
3122 		rwi = list_first_entry(&adapter->rwi_list, struct ibmvnic_rwi,  in get_next_rwi()
3129 	spin_unlock_irqrestore(&adapter->rwi_lock, flags);  in get_next_rwi()
3135  * @adapter: ibmvnic_adapter struct
3147 static int do_passive_init(struct ibmvnic_adapter *adapter)  in do_passive_init()  argument
3150 	struct net_device *netdev = adapter->netdev;  in do_passive_init()
3151 	struct device *dev = &adapter->vdev->dev;  in do_passive_init()
3156 	adapter->state = VNIC_PROBING;  in do_passive_init()
3157 	reinit_completion(&adapter->init_done);  in do_passive_init()
3158 	adapter->init_done_rc = 0;  in do_passive_init()
3159 	adapter->crq.active = true;  in do_passive_init()
3161 	rc = send_crq_init_complete(adapter);  in do_passive_init()
3165 	rc = send_version_xchg(adapter);  in do_passive_init()
3167 		netdev_dbg(adapter->netdev, "send_version_xchg failed, rc=%d\n", rc);  in do_passive_init()
3169 	if (!wait_for_completion_timeout(&adapter->init_done, timeout)) {  in do_passive_init()
3175 	rc = init_sub_crqs(adapter);  in do_passive_init()
3181 	rc = init_sub_crq_irqs(adapter);  in do_passive_init()
3187 	netdev->mtu = adapter->req_mtu - ETH_HLEN;  in do_passive_init()
3188 	netdev->min_mtu = adapter->min_mtu - ETH_HLEN;  in do_passive_init()
3189 	netdev->max_mtu = adapter->max_mtu - ETH_HLEN;  in do_passive_init()
3191 	adapter->state = VNIC_PROBED;  in do_passive_init()
3197 	release_sub_crqs(adapter, 1);  in do_passive_init()
3199 	adapter->state = VNIC_DOWN;  in do_passive_init()
3205 	struct ibmvnic_adapter *adapter;  in __ibmvnic_reset()  local
3217 	adapter = container_of(work, struct ibmvnic_adapter, ibmvnic_reset);  in __ibmvnic_reset()
3218 		dev = &adapter->vdev->dev;  in __ibmvnic_reset()
3229 	if (adapter->state == VNIC_PROBING &&  in __ibmvnic_reset()
3230 	    !wait_for_completion_timeout(&adapter->probe_done, timeout)) {  in __ibmvnic_reset()
3233 				   &adapter->ibmvnic_delayed_reset,  in __ibmvnic_reset()
3238 	/* adapter is done with probe (i.e state is never VNIC_PROBING now) */  in __ibmvnic_reset()
3239 	if (adapter->state == VNIC_REMOVING)  in __ibmvnic_reset()
3267 	spin_lock(&adapter->rwi_lock);  in __ibmvnic_reset()
3268 	if (!list_empty(&adapter->rwi_list)) {  in __ibmvnic_reset()
3269 		if (test_and_set_bit_lock(0, &adapter->resetting)) {  in __ibmvnic_reset()
3271 					   &adapter->ibmvnic_delayed_reset,  in __ibmvnic_reset()
3277 	spin_unlock(&adapter->rwi_lock);  in __ibmvnic_reset()
3282 	rwi = get_next_rwi(adapter);  in __ibmvnic_reset()
3284 		spin_lock_irqsave(&adapter->state_lock, flags);  in __ibmvnic_reset()
3286 		if (adapter->state == VNIC_REMOVING ||  in __ibmvnic_reset()
3287 		    adapter->state == VNIC_REMOVED) {  in __ibmvnic_reset()
3288 			spin_unlock_irqrestore(&adapter->state_lock, flags);  in __ibmvnic_reset()
3295 			reset_state = adapter->state;  in __ibmvnic_reset()
3298 		spin_unlock_irqrestore(&adapter->state_lock, flags);  in __ibmvnic_reset()
3302 			rc = do_passive_init(adapter);  in __ibmvnic_reset()
3305 				netif_carrier_on(adapter->netdev);  in __ibmvnic_reset()
3306 		} else if (adapter->force_reset_recovery) {  in __ibmvnic_reset()
3311 			adapter->failover_pending = false;  in __ibmvnic_reset()
3314 			if (adapter->wait_for_reset) {  in __ibmvnic_reset()
3316 				adapter->force_reset_recovery = false;  in __ibmvnic_reset()
3317 				rc = do_hard_reset(adapter, rwi, reset_state);  in __ibmvnic_reset()
3320 				adapter->force_reset_recovery = false;  in __ibmvnic_reset()
3321 				rc = do_hard_reset(adapter, rwi, reset_state);  in __ibmvnic_reset()
3334 			 * adapter some time to settle down before retrying.  in __ibmvnic_reset()
3337 				netdev_dbg(adapter->netdev,  in __ibmvnic_reset()
3339 					   adapter_state_to_string(adapter->state),  in __ibmvnic_reset()
3345 			rc = do_reset(adapter, rwi, reset_state);  in __ibmvnic_reset()
3348 		adapter->last_reset_time = jiffies;  in __ibmvnic_reset()
3351 			netdev_dbg(adapter->netdev, "Reset failed, rc=%d\n", rc);  in __ibmvnic_reset()
3353 		rwi = get_next_rwi(adapter);  in __ibmvnic_reset()
3357 		 * the adapter would be in an undefined state. So retry the  in __ibmvnic_reset()
3372 			adapter->force_reset_recovery = true;  in __ibmvnic_reset()
3375 	if (adapter->wait_for_reset) {  in __ibmvnic_reset()
3376 		adapter->reset_done_rc = rc;  in __ibmvnic_reset()
3377 		complete(&adapter->reset_done);  in __ibmvnic_reset()
3380 	clear_bit_unlock(0, &adapter->resetting);  in __ibmvnic_reset()
3382 	netdev_dbg(adapter->netdev,  in __ibmvnic_reset()
3384 		   adapter_state_to_string(adapter->state),  in __ibmvnic_reset()
3385 		   adapter->force_reset_recovery,  in __ibmvnic_reset()
3386 		   adapter->wait_for_reset);  in __ibmvnic_reset()
3391 	struct ibmvnic_adapter *adapter;  in __ibmvnic_delayed_reset()  local
3393 	adapter = container_of(work, struct ibmvnic_adapter,  in __ibmvnic_delayed_reset()
3395 	__ibmvnic_reset(&adapter->ibmvnic_reset);  in __ibmvnic_delayed_reset()
3398 static void flush_reset_queue(struct ibmvnic_adapter *adapter)  in flush_reset_queue()  argument
3402 	if (!list_empty(&adapter->rwi_list)) {  in flush_reset_queue()
3403 		list_for_each_safe(entry, tmp_entry, &adapter->rwi_list) {  in flush_reset_queue()
3410 static int ibmvnic_reset(struct ibmvnic_adapter *adapter,  in ibmvnic_reset()  argument
3413 	struct net_device *netdev = adapter->netdev;  in ibmvnic_reset()
3418 	spin_lock_irqsave(&adapter->rwi_lock, flags);  in ibmvnic_reset()
3425 	if (adapter->state == VNIC_REMOVING ||  in ibmvnic_reset()
3426 	    adapter->state == VNIC_REMOVED ||  in ibmvnic_reset()
3427 	    (adapter->failover_pending && reason != VNIC_RESET_FAILOVER)) {  in ibmvnic_reset()
3429 		netdev_dbg(netdev, "Adapter removing or pending failover, skipping reset\n");  in ibmvnic_reset()
3433 	list_for_each_entry(tmp, &adapter->rwi_list, list) {  in ibmvnic_reset()
3450 	if (adapter->force_reset_recovery)  in ibmvnic_reset()
3451 		flush_reset_queue(adapter);  in ibmvnic_reset()
3454 	list_add_tail(&rwi->list, &adapter->rwi_list);  in ibmvnic_reset()
3455 	netdev_dbg(adapter->netdev, "Scheduling reset (reason %s)\n",  in ibmvnic_reset()
3457 	queue_work(system_long_wq, &adapter->ibmvnic_reset);  in ibmvnic_reset()
3462 	spin_unlock_irqrestore(&adapter->rwi_lock, flags);  in ibmvnic_reset()
3472 	struct ibmvnic_adapter *adapter = netdev_priv(dev);  in ibmvnic_tx_timeout()  local
3474 	if (test_bit(0, &adapter->resetting)) {  in ibmvnic_tx_timeout()
3475 		netdev_err(adapter->netdev,  in ibmvnic_tx_timeout()
3476 			   "Adapter is resetting, skip timeout reset\n");  in ibmvnic_tx_timeout()
3482 	if (time_before(jiffies, (adapter->last_reset_time + dev->watchdog_timeo))) {  in ibmvnic_tx_timeout()
3486 	ibmvnic_reset(adapter, VNIC_RESET_TIMEOUT);  in ibmvnic_tx_timeout()
3489 static void remove_buff_from_pool(struct ibmvnic_adapter *adapter,  in remove_buff_from_pool()  argument
3492 	struct ibmvnic_rx_pool *pool = &adapter->rx_pool[rx_buff->pool_index];  in remove_buff_from_pool()
3505 	struct ibmvnic_adapter *adapter;  in ibmvnic_poll()  local
3511 	adapter = netdev_priv(netdev);  in ibmvnic_poll()
3512 	scrq_num = (int)(napi - adapter->napi);  in ibmvnic_poll()
3514 	rx_scrq = adapter->rx_scrq[scrq_num];  in ibmvnic_poll()
3525 		if (unlikely(test_bit(0, &adapter->resetting) &&  in ibmvnic_poll()
3526 			     adapter->reset_reason != VNIC_RESET_NON_FATAL)) {  in ibmvnic_poll()
3527 			enable_scrq_irq(adapter, rx_scrq);  in ibmvnic_poll()
3532 		if (!pending_scrq(adapter, rx_scrq))  in ibmvnic_poll()
3534 		next = ibmvnic_next_scrq(adapter, rx_scrq);  in ibmvnic_poll()
3544 			remove_buff_from_pool(adapter, rx_buff);  in ibmvnic_poll()
3549 			remove_buff_from_pool(adapter, rx_buff);  in ibmvnic_poll()
3565 		if (adapter->rx_vlan_header_insertion &&  in ibmvnic_poll()
3572 		remove_buff_from_pool(adapter, rx_buff);  in ibmvnic_poll()
3587 		adapter->rx_stats_buffers[scrq_num].packets++;  in ibmvnic_poll()
3588 		adapter->rx_stats_buffers[scrq_num].bytes += length;  in ibmvnic_poll()
3592 	if (adapter->state != VNIC_CLOSING &&  in ibmvnic_poll()
3593 	    ((atomic_read(&adapter->rx_pool[scrq_num].available) <  in ibmvnic_poll()
3594 	      adapter->req_rx_add_entries_per_subcrq / 2) ||  in ibmvnic_poll()
3596 		replenish_rx_pool(adapter, &adapter->rx_pool[scrq_num]);  in ibmvnic_poll()
3599 			enable_scrq_irq(adapter, rx_scrq);  in ibmvnic_poll()
3600 			if (pending_scrq(adapter, rx_scrq)) {  in ibmvnic_poll()
3602 					disable_scrq_irq(adapter, rx_scrq);  in ibmvnic_poll()
3611 static int wait_for_reset(struct ibmvnic_adapter *adapter)  in wait_for_reset()  argument
3615 	adapter->fallback.mtu = adapter->req_mtu;  in wait_for_reset()
3616 	adapter->fallback.rx_queues = adapter->req_rx_queues;  in wait_for_reset()
3617 	adapter->fallback.tx_queues = adapter->req_tx_queues;  in wait_for_reset()
3618 	adapter->fallback.rx_entries = adapter->req_rx_add_entries_per_subcrq;  in wait_for_reset()
3619 	adapter->fallback.tx_entries = adapter->req_tx_entries_per_subcrq;  in wait_for_reset()
3621 	reinit_completion(&adapter->reset_done);  in wait_for_reset()
3622 	adapter->wait_for_reset = true;  in wait_for_reset()
3623 	rc = ibmvnic_reset(adapter, VNIC_RESET_CHANGE_PARAM);  in wait_for_reset()
3629 	rc = ibmvnic_wait_for_completion(adapter, &adapter->reset_done, 60000);  in wait_for_reset()
3636 	if (adapter->reset_done_rc) {  in wait_for_reset()
3638 		adapter->desired.mtu = adapter->fallback.mtu;  in wait_for_reset()
3639 		adapter->desired.rx_queues = adapter->fallback.rx_queues;  in wait_for_reset()
3640 		adapter->desired.tx_queues = adapter->fallback.tx_queues;  in wait_for_reset()
3641 		adapter->desired.rx_entries = adapter->fallback.rx_entries;  in wait_for_reset()
3642 		adapter->desired.tx_entries = adapter->fallback.tx_entries;  in wait_for_reset()
3644 		reinit_completion(&adapter->reset_done);  in wait_for_reset()
3645 		adapter->wait_for_reset = true;  in wait_for_reset()
3646 		rc = ibmvnic_reset(adapter, VNIC_RESET_CHANGE_PARAM);  in wait_for_reset()
3651 		rc = ibmvnic_wait_for_completion(adapter, &adapter->reset_done,  in wait_for_reset()
3659 	adapter->wait_for_reset = false;  in wait_for_reset()
3666 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_change_mtu()  local
3668 	adapter->desired.mtu = new_mtu + ETH_HLEN;  in ibmvnic_change_mtu()
3670 	return wait_for_reset(adapter);  in ibmvnic_change_mtu()
3707 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_get_link_ksettings()  local
3710 	rc = send_query_phys_parms(adapter);  in ibmvnic_get_link_ksettings()
3712 		adapter->speed = SPEED_UNKNOWN;  in ibmvnic_get_link_ksettings()
3713 		adapter->duplex = DUPLEX_UNKNOWN;  in ibmvnic_get_link_ksettings()
3715 	cmd->base.speed = adapter->speed;  in ibmvnic_get_link_ksettings()
3716 	cmd->base.duplex = adapter->duplex;  in ibmvnic_get_link_ksettings()
3727 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_get_drvinfo()  local
3731 	strscpy(info->fw_version, adapter->fw_version,  in ibmvnic_get_drvinfo()
3737 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_get_msglevel()  local
3739 	return adapter->msg_enable;  in ibmvnic_get_msglevel()
3744 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_set_msglevel()  local
3746 	adapter->msg_enable = data;  in ibmvnic_set_msglevel()
3751 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_get_link()  local
3756 	return adapter->logical_link_state;  in ibmvnic_get_link()
3764 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_get_ringparam()  local
3766 	ring->rx_max_pending = adapter->max_rx_add_entries_per_subcrq;  in ibmvnic_get_ringparam()
3767 	ring->tx_max_pending = adapter->max_tx_entries_per_subcrq;  in ibmvnic_get_ringparam()
3770 	ring->rx_pending = adapter->req_rx_add_entries_per_subcrq;  in ibmvnic_get_ringparam()
3771 	ring->tx_pending = adapter->req_tx_entries_per_subcrq;  in ibmvnic_get_ringparam()
3781 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_set_ringparam()  local
3783 	if (ring->rx_pending > adapter->max_rx_add_entries_per_subcrq  ||  in ibmvnic_set_ringparam()
3784 	    ring->tx_pending > adapter->max_tx_entries_per_subcrq) {  in ibmvnic_set_ringparam()
3787 			   adapter->max_rx_add_entries_per_subcrq);  in ibmvnic_set_ringparam()
3789 			   adapter->max_tx_entries_per_subcrq);  in ibmvnic_set_ringparam()
3793 	adapter->desired.rx_entries = ring->rx_pending;  in ibmvnic_set_ringparam()
3794 	adapter->desired.tx_entries = ring->tx_pending;  in ibmvnic_set_ringparam()
3796 	return wait_for_reset(adapter);  in ibmvnic_set_ringparam()
3802 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_get_channels()  local
3804 	channels->max_rx = adapter->max_rx_queues;  in ibmvnic_get_channels()
3805 	channels->max_tx = adapter->max_tx_queues;  in ibmvnic_get_channels()
3808 	channels->rx_count = adapter->req_rx_queues;  in ibmvnic_get_channels()
3809 	channels->tx_count = adapter->req_tx_queues;  in ibmvnic_get_channels()
3817 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_set_channels()  local
3819 	adapter->desired.rx_queues = channels->rx_count;  in ibmvnic_set_channels()
3820 	adapter->desired.tx_queues = channels->tx_count;  in ibmvnic_set_channels()
3822 	return wait_for_reset(adapter);  in ibmvnic_set_channels()
3827 	struct ibmvnic_adapter *adapter = netdev_priv(dev);  in ibmvnic_get_strings()  local
3836 	for (i = 0; i < adapter->req_tx_queues; i++) {  in ibmvnic_get_strings()
3850 	for (i = 0; i < adapter->req_rx_queues; i++) {  in ibmvnic_get_strings()
3864 	struct ibmvnic_adapter *adapter = netdev_priv(dev);  in ibmvnic_get_sset_count()  local
3869 		       adapter->req_tx_queues * NUM_TX_STATS +  in ibmvnic_get_sset_count()
3870 		       adapter->req_rx_queues * NUM_RX_STATS;  in ibmvnic_get_sset_count()
3879 	struct ibmvnic_adapter *adapter = netdev_priv(dev);  in ibmvnic_get_ethtool_stats()  local
3887 	crq.request_statistics.ioba = cpu_to_be32(adapter->stats_token);  in ibmvnic_get_ethtool_stats()
3892 	reinit_completion(&adapter->stats_done);  in ibmvnic_get_ethtool_stats()
3893 	rc = ibmvnic_send_crq(adapter, &crq);  in ibmvnic_get_ethtool_stats()
3896 	rc = ibmvnic_wait_for_completion(adapter, &adapter->stats_done, 10000);  in ibmvnic_get_ethtool_stats()
3902 				      (adapter, ibmvnic_stats[i].offset));  in ibmvnic_get_ethtool_stats()
3904 	for (j = 0; j < adapter->req_tx_queues; j++) {  in ibmvnic_get_ethtool_stats()
3905 		data[i] = adapter->tx_stats_buffers[j].batched_packets;  in ibmvnic_get_ethtool_stats()
3907 		data[i] = adapter->tx_stats_buffers[j].direct_packets;  in ibmvnic_get_ethtool_stats()
3909 		data[i] = adapter->tx_stats_buffers[j].bytes;  in ibmvnic_get_ethtool_stats()
3911 		data[i] = adapter->tx_stats_buffers[j].dropped_packets;  in ibmvnic_get_ethtool_stats()
3915 	for (j = 0; j < adapter->req_rx_queues; j++) {  in ibmvnic_get_ethtool_stats()
3916 		data[i] = adapter->rx_stats_buffers[j].packets;  in ibmvnic_get_ethtool_stats()
3918 		data[i] = adapter->rx_stats_buffers[j].bytes;  in ibmvnic_get_ethtool_stats()
3920 		data[i] = adapter->rx_stats_buffers[j].interrupts;  in ibmvnic_get_ethtool_stats()
3942 static int reset_one_sub_crq_queue(struct ibmvnic_adapter *adapter,  in reset_one_sub_crq_queue()  argument
3948 		netdev_dbg(adapter->netdev, "Invalid scrq reset.\n");  in reset_one_sub_crq_queue()
3964 		netdev_dbg(adapter->netdev, "Invalid scrq reset\n");  in reset_one_sub_crq_queue()
3968 	rc = h_reg_sub_crq(adapter->vdev->unit_address, scrq->msg_token,  in reset_one_sub_crq_queue()
3973 static int reset_sub_crq_queues(struct ibmvnic_adapter *adapter)  in reset_sub_crq_queues()  argument
3977 	if (!adapter->tx_scrq || !adapter->rx_scrq)  in reset_sub_crq_queues()
3980 	ibmvnic_clean_affinity(adapter);  in reset_sub_crq_queues()
3982 	for (i = 0; i < adapter->req_tx_queues; i++) {  in reset_sub_crq_queues()
3983 		netdev_dbg(adapter->netdev, "Re-setting tx_scrq[%d]\n", i);  in reset_sub_crq_queues()
3984 		rc = reset_one_sub_crq_queue(adapter, adapter->tx_scrq[i]);  in reset_sub_crq_queues()
3989 	for (i = 0; i < adapter->req_rx_queues; i++) {  in reset_sub_crq_queues()
3990 		netdev_dbg(adapter->netdev, "Re-setting rx_scrq[%d]\n", i);  in reset_sub_crq_queues()
3991 		rc = reset_one_sub_crq_queue(adapter, adapter->rx_scrq[i]);  in reset_sub_crq_queues()
3999 static void release_sub_crq_queue(struct ibmvnic_adapter *adapter,  in release_sub_crq_queue()  argument
4003 	struct device *dev = &adapter->vdev->dev;  in release_sub_crq_queue()
4006 	netdev_dbg(adapter->netdev, "Releasing sub-CRQ\n");  in release_sub_crq_queue()
4012 						adapter->vdev->unit_address,  in release_sub_crq_queue()
4017 			netdev_err(adapter->netdev,  in release_sub_crq_queue()
4036 							*adapter)  in init_sub_crq_queue()
4038 	struct device *dev = &adapter->vdev->dev;  in init_sub_crq_queue()
4062 	rc = h_reg_sub_crq(adapter->vdev->unit_address, scrq->msg_token,  in init_sub_crq_queue()
4066 		rc = ibmvnic_reset_crq(adapter);  in init_sub_crq_queue()
4069 		dev_warn(dev, "Partner adapter not ready, waiting.\n");  in init_sub_crq_queue()
4075 	scrq->adapter = adapter;  in init_sub_crq_queue()
4090 	netdev_dbg(adapter->netdev,  in init_sub_crq_queue()
4099 					adapter->vdev->unit_address,  in init_sub_crq_queue()
4115 static void release_sub_crqs(struct ibmvnic_adapter *adapter, bool do_h_free)  in release_sub_crqs()  argument
4119 	ibmvnic_clean_affinity(adapter);  in release_sub_crqs()
4120 	if (adapter->tx_scrq) {  in release_sub_crqs()
4121 		for (i = 0; i < adapter->num_active_tx_scrqs; i++) {  in release_sub_crqs()
4122 			if (!adapter->tx_scrq[i])  in release_sub_crqs()
4125 			netdev_dbg(adapter->netdev, "Releasing tx_scrq[%d]\n",  in release_sub_crqs()
4127 			ibmvnic_tx_scrq_clean_buffer(adapter, adapter->tx_scrq[i]);  in release_sub_crqs()
4128 			if (adapter->tx_scrq[i]->irq) {  in release_sub_crqs()
4129 				free_irq(adapter->tx_scrq[i]->irq,  in release_sub_crqs()
4130 					 adapter->tx_scrq[i]);  in release_sub_crqs()
4131 				irq_dispose_mapping(adapter->tx_scrq[i]->irq);  in release_sub_crqs()
4132 				adapter->tx_scrq[i]->irq = 0;  in release_sub_crqs()
4135 			release_sub_crq_queue(adapter, adapter->tx_scrq[i],  in release_sub_crqs()
4139 		kfree(adapter->tx_scrq);  in release_sub_crqs()
4140 		adapter->tx_scrq = NULL;  in release_sub_crqs()
4141 		adapter->num_active_tx_scrqs = 0;  in release_sub_crqs()
4148 	clean_tx_pools(adapter);  in release_sub_crqs()
4150 	if (adapter->rx_scrq) {  in release_sub_crqs()
4151 		for (i = 0; i < adapter->num_active_rx_scrqs; i++) {  in release_sub_crqs()
4152 			if (!adapter->rx_scrq[i])  in release_sub_crqs()
4155 			netdev_dbg(adapter->netdev, "Releasing rx_scrq[%d]\n",  in release_sub_crqs()
4157 			if (adapter->rx_scrq[i]->irq) {  in release_sub_crqs()
4158 				free_irq(adapter->rx_scrq[i]->irq,  in release_sub_crqs()
4159 					 adapter->rx_scrq[i]);  in release_sub_crqs()
4160 				irq_dispose_mapping(adapter->rx_scrq[i]->irq);  in release_sub_crqs()
4161 				adapter->rx_scrq[i]->irq = 0;  in release_sub_crqs()
4164 			release_sub_crq_queue(adapter, adapter->rx_scrq[i],  in release_sub_crqs()
4168 		kfree(adapter->rx_scrq);  in release_sub_crqs()
4169 		adapter->rx_scrq = NULL;  in release_sub_crqs()
4170 		adapter->num_active_rx_scrqs = 0;  in release_sub_crqs()
4174 static int disable_scrq_irq(struct ibmvnic_adapter *adapter,  in disable_scrq_irq()  argument
4177 	struct device *dev = &adapter->vdev->dev;  in disable_scrq_irq()
4180 	rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,  in disable_scrq_irq()
4212 static int enable_scrq_irq(struct ibmvnic_adapter *adapter,  in enable_scrq_irq()  argument
4215 	struct device *dev = &adapter->vdev->dev;  in enable_scrq_irq()
4223 	if (test_bit(0, &adapter->resetting) &&  in enable_scrq_irq()
4224 	    adapter->reset_reason == VNIC_RESET_MOBILITY) {  in enable_scrq_irq()
4228 	rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,  in enable_scrq_irq()
4236 static int ibmvnic_complete_tx(struct ibmvnic_adapter *adapter,  in ibmvnic_complete_tx()  argument
4239 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_complete_tx()
4248 	while (pending_scrq(adapter, scrq)) {  in ibmvnic_complete_tx()
4254 		next = ibmvnic_next_scrq(adapter, scrq);  in ibmvnic_complete_tx()
4258 				tx_pool = &adapter->tso_pool[pool];  in ibmvnic_complete_tx()
4261 				tx_pool = &adapter->tx_pool[pool];  in ibmvnic_complete_tx()
4278 				netdev_warn(adapter->netdev,  in ibmvnic_complete_tx()
4289 		txq = netdev_get_tx_queue(adapter->netdev, scrq->pool_index);  in ibmvnic_complete_tx()
4293 		    (adapter->req_tx_entries_per_subcrq / 2) &&  in ibmvnic_complete_tx()
4294 		    __netif_subqueue_stopped(adapter->netdev,  in ibmvnic_complete_tx()
4297 			if (adapter->tx_queues_active) {  in ibmvnic_complete_tx()
4298 				netif_wake_subqueue(adapter->netdev,  in ibmvnic_complete_tx()
4300 				netdev_dbg(adapter->netdev,  in ibmvnic_complete_tx()
4308 	enable_scrq_irq(adapter, scrq);  in ibmvnic_complete_tx()
4310 	if (pending_scrq(adapter, scrq)) {  in ibmvnic_complete_tx()
4311 		disable_scrq_irq(adapter, scrq);  in ibmvnic_complete_tx()
4321 	struct ibmvnic_adapter *adapter = scrq->adapter;  in ibmvnic_interrupt_tx()  local
4323 	disable_scrq_irq(adapter, scrq);  in ibmvnic_interrupt_tx()
4324 	ibmvnic_complete_tx(adapter, scrq);  in ibmvnic_interrupt_tx()
4332 	struct ibmvnic_adapter *adapter = scrq->adapter;  in ibmvnic_interrupt_rx()  local
4337 	if (unlikely(adapter->state != VNIC_OPEN))  in ibmvnic_interrupt_rx()
4340 	adapter->rx_stats_buffers[scrq->scrq_num].interrupts++;  in ibmvnic_interrupt_rx()
4342 	if (napi_schedule_prep(&adapter->napi[scrq->scrq_num])) {  in ibmvnic_interrupt_rx()
4343 		disable_scrq_irq(adapter, scrq);  in ibmvnic_interrupt_rx()
4344 		__napi_schedule(&adapter->napi[scrq->scrq_num]);  in ibmvnic_interrupt_rx()
4350 static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter)  in init_sub_crq_irqs()  argument
4352 	struct device *dev = &adapter->vdev->dev;  in init_sub_crq_irqs()
4357 	for (i = 0; i < adapter->req_tx_queues; i++) {  in init_sub_crq_irqs()
4358 		netdev_dbg(adapter->netdev, "Initializing tx_scrq[%d] irq\n",  in init_sub_crq_irqs()
4360 		scrq = adapter->tx_scrq[i];  in init_sub_crq_irqs()
4370 			 adapter->vdev->unit_address, i);  in init_sub_crq_irqs()
4382 	for (i = 0; i < adapter->req_rx_queues; i++) {  in init_sub_crq_irqs()
4383 		netdev_dbg(adapter->netdev, "Initializing rx_scrq[%d] irq\n",  in init_sub_crq_irqs()
4385 		scrq = adapter->rx_scrq[i];  in init_sub_crq_irqs()
4393 			 adapter->vdev->unit_address, i);  in init_sub_crq_irqs()
4405 	ibmvnic_set_affinity(adapter);  in init_sub_crq_irqs()
4412 		free_irq(adapter->rx_scrq[j]->irq, adapter->rx_scrq[j]);  in init_sub_crq_irqs()
4413 		irq_dispose_mapping(adapter->rx_scrq[j]->irq);  in init_sub_crq_irqs()
4415 	i = adapter->req_tx_queues;  in init_sub_crq_irqs()
4418 		free_irq(adapter->tx_scrq[j]->irq, adapter->tx_scrq[j]);  in init_sub_crq_irqs()
4419 		irq_dispose_mapping(adapter->tx_scrq[j]->irq);  in init_sub_crq_irqs()
4421 	release_sub_crqs(adapter, 1);  in init_sub_crq_irqs()
4425 static int init_sub_crqs(struct ibmvnic_adapter *adapter)  in init_sub_crqs()  argument
4427 	struct device *dev = &adapter->vdev->dev;  in init_sub_crqs()
4434 	total_queues = adapter->req_tx_queues + adapter->req_rx_queues;  in init_sub_crqs()
4441 		allqueues[i] = init_sub_crq_queue(adapter);  in init_sub_crqs()
4451 	    adapter->min_tx_queues + adapter->min_rx_queues) {  in init_sub_crqs()
4458 		netdev_dbg(adapter->netdev, "Reducing number of queues\n");  in init_sub_crqs()
4461 			if (adapter->req_rx_queues > adapter->min_rx_queues)  in init_sub_crqs()
4462 				adapter->req_rx_queues--;  in init_sub_crqs()
4467 			if (adapter->req_tx_queues > adapter->min_tx_queues)  in init_sub_crqs()
4468 				adapter->req_tx_queues--;  in init_sub_crqs()
4475 	adapter->tx_scrq = kcalloc(adapter->req_tx_queues,  in init_sub_crqs()
4476 				   sizeof(*adapter->tx_scrq), GFP_KERNEL);  in init_sub_crqs()
4477 	if (!adapter->tx_scrq)  in init_sub_crqs()
4480 	for (i = 0; i < adapter->req_tx_queues; i++) {  in init_sub_crqs()
4481 		adapter->tx_scrq[i] = allqueues[i];  in init_sub_crqs()
4482 		adapter->tx_scrq[i]->pool_index = i;  in init_sub_crqs()
4483 		adapter->num_active_tx_scrqs++;  in init_sub_crqs()
4486 	adapter->rx_scrq = kcalloc(adapter->req_rx_queues,  in init_sub_crqs()
4487 				   sizeof(*adapter->rx_scrq), GFP_KERNEL);  in init_sub_crqs()
4488 	if (!adapter->rx_scrq)  in init_sub_crqs()
4491 	for (i = 0; i < adapter->req_rx_queues; i++) {  in init_sub_crqs()
4492 		adapter->rx_scrq[i] = allqueues[i + adapter->req_tx_queues];  in init_sub_crqs()
4493 		adapter->rx_scrq[i]->scrq_num = i;  in init_sub_crqs()
4494 		adapter->num_active_rx_scrqs++;  in init_sub_crqs()
4501 	kfree(adapter->tx_scrq);  in init_sub_crqs()
4502 	adapter->tx_scrq = NULL;  in init_sub_crqs()
4505 		release_sub_crq_queue(adapter, allqueues[i], 1);  in init_sub_crqs()
4510 static void send_request_cap(struct ibmvnic_adapter *adapter, int retry)  in send_request_cap()  argument
4512 	struct device *dev = &adapter->vdev->dev;  in send_request_cap()
4522 	if (!(adapter->netdev->flags & IFF_PROMISC) ||  in send_request_cap()
4523 	    adapter->promisc_supported)  in send_request_cap()
4532 		atomic_set(&adapter->running_cap_crqs, cap_reqs);  in send_request_cap()
4534 		if (adapter->min_tx_entries_per_subcrq > entries_page ||  in send_request_cap()
4535 		    adapter->min_rx_add_entries_per_subcrq > entries_page) {  in send_request_cap()
4540 		if (adapter->desired.mtu)  in send_request_cap()
4541 			adapter->req_mtu = adapter->desired.mtu;  in send_request_cap()
4543 			adapter->req_mtu = adapter->netdev->mtu + ETH_HLEN;  in send_request_cap()
4545 		if (!adapter->desired.tx_entries)  in send_request_cap()
4546 			adapter->desired.tx_entries =  in send_request_cap()
4547 					adapter->max_tx_entries_per_subcrq;  in send_request_cap()
4548 		if (!adapter->desired.rx_entries)  in send_request_cap()
4549 			adapter->desired.rx_entries =  in send_request_cap()
4550 					adapter->max_rx_add_entries_per_subcrq;  in send_request_cap()
4553 			      (adapter->req_mtu + IBMVNIC_BUFFER_HLEN);  in send_request_cap()
4555 		if ((adapter->req_mtu + IBMVNIC_BUFFER_HLEN) *  in send_request_cap()
4556 			adapter->desired.tx_entries > IBMVNIC_LTB_SET_SIZE) {  in send_request_cap()
4557 			adapter->desired.tx_entries = max_entries;  in send_request_cap()
4560 		if ((adapter->req_mtu + IBMVNIC_BUFFER_HLEN) *  in send_request_cap()
4561 			adapter->desired.rx_entries > IBMVNIC_LTB_SET_SIZE) {  in send_request_cap()
4562 			adapter->desired.rx_entries = max_entries;  in send_request_cap()
4565 		if (adapter->desired.tx_entries)  in send_request_cap()
4566 			adapter->req_tx_entries_per_subcrq =  in send_request_cap()
4567 					adapter->desired.tx_entries;  in send_request_cap()
4569 			adapter->req_tx_entries_per_subcrq =  in send_request_cap()
4570 					adapter->max_tx_entries_per_subcrq;  in send_request_cap()
4572 		if (adapter->desired.rx_entries)  in send_request_cap()
4573 			adapter->req_rx_add_entries_per_subcrq =  in send_request_cap()
4574 					adapter->desired.rx_entries;  in send_request_cap()
4576 			adapter->req_rx_add_entries_per_subcrq =  in send_request_cap()
4577 					adapter->max_rx_add_entries_per_subcrq;  in send_request_cap()
4579 		if (adapter->desired.tx_queues)  in send_request_cap()
4580 			adapter->req_tx_queues =  in send_request_cap()
4581 					adapter->desired.tx_queues;  in send_request_cap()
4583 			adapter->req_tx_queues =  in send_request_cap()
4584 					adapter->opt_tx_comp_sub_queues;  in send_request_cap()
4586 		if (adapter->desired.rx_queues)  in send_request_cap()
4587 			adapter->req_rx_queues =  in send_request_cap()
4588 					adapter->desired.rx_queues;  in send_request_cap()
4590 			adapter->req_rx_queues =  in send_request_cap()
4591 					adapter->opt_rx_comp_queues;  in send_request_cap()
4593 		adapter->req_rx_add_queues = adapter->max_rx_add_queues;  in send_request_cap()
4595 		atomic_add(cap_reqs, &adapter->running_cap_crqs);  in send_request_cap()
4602 	crq.request_capability.number = cpu_to_be64(adapter->req_tx_queues);  in send_request_cap()
4604 	ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4607 	crq.request_capability.number = cpu_to_be64(adapter->req_rx_queues);  in send_request_cap()
4609 	ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4612 	crq.request_capability.number = cpu_to_be64(adapter->req_rx_add_queues);  in send_request_cap()
4614 	ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4619 	    cpu_to_be64(adapter->req_tx_entries_per_subcrq);  in send_request_cap()
4621 	ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4626 	    cpu_to_be64(adapter->req_rx_add_entries_per_subcrq);  in send_request_cap()
4628 	ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4631 	crq.request_capability.number = cpu_to_be64(adapter->req_mtu);  in send_request_cap()
4633 	ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4635 	if (adapter->netdev->flags & IFF_PROMISC) {  in send_request_cap()
4636 		if (adapter->promisc_supported) {  in send_request_cap()
4641 			ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4648 		ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4657 static int pending_scrq(struct ibmvnic_adapter *adapter,  in pending_scrq()  argument
4673 static union sub_crq *ibmvnic_next_scrq(struct ibmvnic_adapter *adapter,  in ibmvnic_next_scrq()  argument
4697 static union ibmvnic_crq *ibmvnic_next_crq(struct ibmvnic_adapter *adapter)  in ibmvnic_next_crq()  argument
4699 	struct ibmvnic_crq_queue *queue = &adapter->crq;  in ibmvnic_next_crq()
4718 				     "%s failed: Send request is malformed or adapter failover pending. (rc=%d)\n",  in print_subcrq_error()
4723 				     "%s failed: Backing queue closed. Adapter is down or failover pending. (rc=%d)\n",  in print_subcrq_error()
4732 static int send_subcrq_indirect(struct ibmvnic_adapter *adapter,  in send_subcrq_indirect()  argument
4735 	unsigned int ua = adapter->vdev->unit_address;  in send_subcrq_indirect()
4736 	struct device *dev = &adapter->vdev->dev;  in send_subcrq_indirect()
4751 static int ibmvnic_send_crq(struct ibmvnic_adapter *adapter,  in ibmvnic_send_crq()  argument
4754 	unsigned int ua = adapter->vdev->unit_address;  in ibmvnic_send_crq()
4755 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_send_crq()
4759 	netdev_dbg(adapter->netdev, "Sending CRQ: %016lx %016lx\n",  in ibmvnic_send_crq()
4763 	if (!adapter->crq.active &&  in ibmvnic_send_crq()
4788 static int ibmvnic_send_crq_init(struct ibmvnic_adapter *adapter)  in ibmvnic_send_crq_init()  argument
4790 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_send_crq_init()
4798 	netdev_dbg(adapter->netdev, "Sending CRQ init\n");  in ibmvnic_send_crq_init()
4801 		rc = ibmvnic_send_crq(adapter, &crq);  in ibmvnic_send_crq_init()
4823 static int vnic_client_data_len(struct ibmvnic_adapter *adapter)  in vnic_client_data_len()  argument
4834 	len += strlen(adapter->netdev->name) + 1;  in vnic_client_data_len()
4839 static void vnic_add_client_data(struct ibmvnic_adapter *adapter,  in vnic_add_client_data()  argument
4861 	len = strlen(adapter->netdev->name) + 1;  in vnic_add_client_data()
4863 	strscpy(vlcd->name, adapter->netdev->name, len);  in vnic_add_client_data()
4866 static int send_login(struct ibmvnic_adapter *adapter)  in send_login()  argument
4870 	struct device *dev = &adapter->vdev->dev;  in send_login()
4883 	if (!adapter->tx_scrq || !adapter->rx_scrq) {  in send_login()
4884 		netdev_err(adapter->netdev,  in send_login()
4889 	release_login_buffer(adapter);  in send_login()
4890 	release_login_rsp_buffer(adapter);  in send_login()
4892 	client_data_len = vnic_client_data_len(adapter);  in send_login()
4896 	    sizeof(u64) * (adapter->req_tx_queues + adapter->req_rx_queues) +  in send_login()
4911 			  sizeof(u64) * adapter->req_tx_queues +  in send_login()
4912 			  sizeof(u64) * adapter->req_rx_queues +  in send_login()
4913 			  sizeof(u64) * adapter->req_rx_queues +  in send_login()
4927 	adapter->login_buf = login_buffer;  in send_login()
4928 	adapter->login_buf_token = buffer_token;  in send_login()
4929 	adapter->login_buf_sz = buffer_size;  in send_login()
4930 	adapter->login_rsp_buf = login_rsp_buffer;  in send_login()
4931 	adapter->login_rsp_buf_token = rsp_buffer_token;  in send_login()
4932 	adapter->login_rsp_buf_sz = rsp_buffer_size;  in send_login()
4936 	login_buffer->num_txcomp_subcrqs = cpu_to_be32(adapter->req_tx_queues);  in send_login()
4939 	login_buffer->num_rxcomp_subcrqs = cpu_to_be32(adapter->req_rx_queues);  in send_login()
4942 			sizeof(u64) * adapter->req_tx_queues);  in send_login()
4950 				      sizeof(u64) * adapter->req_tx_queues);  in send_login()
4952 	for (i = 0; i < adapter->req_tx_queues; i++) {  in send_login()
4953 		if (adapter->tx_scrq[i]) {  in send_login()
4955 				cpu_to_be64(adapter->tx_scrq[i]->crq_num);  in send_login()
4959 	for (i = 0; i < adapter->req_rx_queues; i++) {  in send_login()
4960 		if (adapter->rx_scrq[i]) {  in send_login()
4962 				cpu_to_be64(adapter->rx_scrq[i]->crq_num);  in send_login()
4968 		((char *)rx_list_p + (sizeof(u64) * adapter->req_rx_queues));  in send_login()
4973 	vnic_add_client_data(adapter, vlcd);  in send_login()
4975 	netdev_dbg(adapter->netdev, "Login Buffer:\n");  in send_login()
4976 	for (i = 0; i < (adapter->login_buf_sz - 1) / 8 + 1; i++) {  in send_login()
4977 		netdev_dbg(adapter->netdev, "%016lx\n",  in send_login()
4978 			   ((unsigned long *)(adapter->login_buf))[i]);  in send_login()
4987 	adapter->login_pending = true;  in send_login()
4988 	rc = ibmvnic_send_crq(adapter, &crq);  in send_login()
4990 		adapter->login_pending = false;  in send_login()
4991 		netdev_err(adapter->netdev, "Failed to send login, rc=%d\n", rc);  in send_login()
5002 	adapter->login_rsp_buf = NULL;  in send_login()
5007 	adapter->login_buf = NULL;  in send_login()
5012 static int send_request_map(struct ibmvnic_adapter *adapter, dma_addr_t addr,  in send_request_map()  argument
5023 	return ibmvnic_send_crq(adapter, &crq);  in send_request_map()
5026 static int send_request_unmap(struct ibmvnic_adapter *adapter, u8 map_id)  in send_request_unmap()  argument
5034 	return ibmvnic_send_crq(adapter, &crq);  in send_request_unmap()
5037 static void send_query_map(struct ibmvnic_adapter *adapter)  in send_query_map()  argument
5044 	ibmvnic_send_crq(adapter, &crq);  in send_query_map()
5048 static void send_query_cap(struct ibmvnic_adapter *adapter)  in send_query_cap()  argument
5059 	atomic_set(&adapter->running_cap_crqs, cap_reqs);  in send_query_cap()
5066 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5070 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5074 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5078 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5082 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5086 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5091 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5096 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5101 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5106 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5110 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5114 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5118 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5122 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5126 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5130 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5134 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5138 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5142 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5146 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5150 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5155 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5160 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5165 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5170 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5179 static void send_query_ip_offload(struct ibmvnic_adapter *adapter)  in send_query_ip_offload()  argument
5182 	struct device *dev = &adapter->vdev->dev;  in send_query_ip_offload()
5185 	adapter->ip_offload_tok =  in send_query_ip_offload()
5187 			       &adapter->ip_offload_buf,  in send_query_ip_offload()
5191 	if (dma_mapping_error(dev, adapter->ip_offload_tok)) {  in send_query_ip_offload()
5202 	    cpu_to_be32(adapter->ip_offload_tok);  in send_query_ip_offload()
5204 	ibmvnic_send_crq(adapter, &crq);  in send_query_ip_offload()
5207 static void send_control_ip_offload(struct ibmvnic_adapter *adapter)  in send_control_ip_offload()  argument
5209 	struct ibmvnic_control_ip_offload_buffer *ctrl_buf = &adapter->ip_offload_ctrl;  in send_control_ip_offload()
5210 	struct ibmvnic_query_ip_offload_buffer *buf = &adapter->ip_offload_buf;  in send_control_ip_offload()
5211 	struct device *dev = &adapter->vdev->dev;  in send_control_ip_offload()
5215 	adapter->ip_offload_ctrl_tok =  in send_control_ip_offload()
5218 			       sizeof(adapter->ip_offload_ctrl),  in send_control_ip_offload()
5221 	if (dma_mapping_error(dev, adapter->ip_offload_ctrl_tok)) {  in send_control_ip_offload()
5226 	ctrl_buf->len = cpu_to_be32(sizeof(adapter->ip_offload_ctrl));  in send_control_ip_offload()
5241 	if (adapter->state != VNIC_PROBING) {  in send_control_ip_offload()
5242 		old_hw_features = adapter->netdev->hw_features;  in send_control_ip_offload()
5243 		adapter->netdev->hw_features = 0;  in send_control_ip_offload()
5246 	adapter->netdev->hw_features = NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO;  in send_control_ip_offload()
5249 		adapter->netdev->hw_features |= NETIF_F_IP_CSUM;  in send_control_ip_offload()
5252 		adapter->netdev->hw_features |= NETIF_F_IPV6_CSUM;  in send_control_ip_offload()
5254 	if ((adapter->netdev->features &  in send_control_ip_offload()
5256 		adapter->netdev->hw_features |= NETIF_F_RXCSUM;  in send_control_ip_offload()
5259 		adapter->netdev->hw_features |= NETIF_F_TSO;  in send_control_ip_offload()
5261 		adapter->netdev->hw_features |= NETIF_F_TSO6;  in send_control_ip_offload()
5263 	if (adapter->state == VNIC_PROBING) {  in send_control_ip_offload()
5264 		adapter->netdev->features |= adapter->netdev->hw_features;  in send_control_ip_offload()
5265 	} else if (old_hw_features != adapter->netdev->hw_features) {  in send_control_ip_offload()
5269 		adapter->netdev->features &= adapter->netdev->hw_features;  in send_control_ip_offload()
5271 		tmp = (old_hw_features ^ adapter->netdev->hw_features) &  in send_control_ip_offload()
5272 			adapter->netdev->hw_features;  in send_control_ip_offload()
5273 		adapter->netdev->features |=  in send_control_ip_offload()
5274 				tmp & adapter->netdev->wanted_features;  in send_control_ip_offload()
5281 	    cpu_to_be32(sizeof(adapter->ip_offload_ctrl));  in send_control_ip_offload()
5282 	crq.control_ip_offload.ioba = cpu_to_be32(adapter->ip_offload_ctrl_tok);  in send_control_ip_offload()
5283 	ibmvnic_send_crq(adapter, &crq);  in send_control_ip_offload()
5287 				struct ibmvnic_adapter *adapter)  in handle_vpd_size_rsp()  argument
5289 	struct device *dev = &adapter->vdev->dev;  in handle_vpd_size_rsp()
5294 		complete(&adapter->fw_done);  in handle_vpd_size_rsp()
5298 	adapter->vpd->len = be64_to_cpu(crq->get_vpd_size_rsp.len);  in handle_vpd_size_rsp()
5299 	complete(&adapter->fw_done);  in handle_vpd_size_rsp()
5303 			   struct ibmvnic_adapter *adapter)  in handle_vpd_rsp()  argument
5305 	struct device *dev = &adapter->vdev->dev;  in handle_vpd_rsp()
5309 	memset(adapter->fw_version, 0, 32);  in handle_vpd_rsp()
5311 	dma_unmap_single(dev, adapter->vpd->dma_addr, adapter->vpd->len,  in handle_vpd_rsp()
5323 	substr = strnstr(adapter->vpd->buff, "RM", adapter->vpd->len);  in handle_vpd_rsp()
5330 	if ((substr + 2) < (adapter->vpd->buff + adapter->vpd->len)) {  in handle_vpd_rsp()
5337 	/* copy firmware version string from vpd into adapter */  in handle_vpd_rsp()
5339 	    (adapter->vpd->buff + adapter->vpd->len)) {  in handle_vpd_rsp()
5340 		strncpy((char *)adapter->fw_version, substr + 3, fw_level_len);  in handle_vpd_rsp()
5346 	if (adapter->fw_version[0] == '\0')  in handle_vpd_rsp()
5347 		strscpy((char *)adapter->fw_version, "N/A", sizeof(adapter->fw_version));  in handle_vpd_rsp()
5348 	complete(&adapter->fw_done);  in handle_vpd_rsp()
5351 static void handle_query_ip_offload_rsp(struct ibmvnic_adapter *adapter)  in handle_query_ip_offload_rsp()  argument
5353 	struct device *dev = &adapter->vdev->dev;  in handle_query_ip_offload_rsp()
5354 	struct ibmvnic_query_ip_offload_buffer *buf = &adapter->ip_offload_buf;  in handle_query_ip_offload_rsp()
5357 	dma_unmap_single(dev, adapter->ip_offload_tok,  in handle_query_ip_offload_rsp()
5358 			 sizeof(adapter->ip_offload_buf), DMA_FROM_DEVICE);  in handle_query_ip_offload_rsp()
5360 	netdev_dbg(adapter->netdev, "Query IP Offload Buffer:\n");  in handle_query_ip_offload_rsp()
5361 	for (i = 0; i < (sizeof(adapter->ip_offload_buf) - 1) / 8 + 1; i++)  in handle_query_ip_offload_rsp()
5362 		netdev_dbg(adapter->netdev, "%016lx\n",  in handle_query_ip_offload_rsp()
5365 	netdev_dbg(adapter->netdev, "ipv4_chksum = %d\n", buf->ipv4_chksum);  in handle_query_ip_offload_rsp()
5366 	netdev_dbg(adapter->netdev, "ipv6_chksum = %d\n", buf->ipv6_chksum);  in handle_query_ip_offload_rsp()
5367 	netdev_dbg(adapter->netdev, "tcp_ipv4_chksum = %d\n",  in handle_query_ip_offload_rsp()
5369 	netdev_dbg(adapter->netdev, "tcp_ipv6_chksum = %d\n",  in handle_query_ip_offload_rsp()
5371 	netdev_dbg(adapter->netdev, "udp_ipv4_chksum = %d\n",  in handle_query_ip_offload_rsp()
5373 	netdev_dbg(adapter->netdev, "udp_ipv6_chksum = %d\n",  in handle_query_ip_offload_rsp()
5375 	netdev_dbg(adapter->netdev, "large_tx_ipv4 = %d\n",  in handle_query_ip_offload_rsp()
5377 	netdev_dbg(adapter->netdev, "large_tx_ipv6 = %d\n",  in handle_query_ip_offload_rsp()
5379 	netdev_dbg(adapter->netdev, "large_rx_ipv4 = %d\n",  in handle_query_ip_offload_rsp()
5381 	netdev_dbg(adapter->netdev, "large_rx_ipv6 = %d\n",  in handle_query_ip_offload_rsp()
5383 	netdev_dbg(adapter->netdev, "max_ipv4_hdr_sz = %d\n",  in handle_query_ip_offload_rsp()
5385 	netdev_dbg(adapter->netdev, "max_ipv6_hdr_sz = %d\n",  in handle_query_ip_offload_rsp()
5387 	netdev_dbg(adapter->netdev, "max_tcp_hdr_size = %d\n",  in handle_query_ip_offload_rsp()
5389 	netdev_dbg(adapter->netdev, "max_udp_hdr_size = %d\n",  in handle_query_ip_offload_rsp()
5391 	netdev_dbg(adapter->netdev, "max_large_tx_size = %d\n",  in handle_query_ip_offload_rsp()
5393 	netdev_dbg(adapter->netdev, "max_large_rx_size = %d\n",  in handle_query_ip_offload_rsp()
5395 	netdev_dbg(adapter->netdev, "ipv6_ext_hdr = %d\n",  in handle_query_ip_offload_rsp()
5397 	netdev_dbg(adapter->netdev, "tcp_pseudosum_req = %d\n",  in handle_query_ip_offload_rsp()
5399 	netdev_dbg(adapter->netdev, "num_ipv6_ext_hd = %d\n",  in handle_query_ip_offload_rsp()
5401 	netdev_dbg(adapter->netdev, "off_ipv6_ext_hd = %d\n",  in handle_query_ip_offload_rsp()
5404 	send_control_ip_offload(adapter);  in handle_query_ip_offload_rsp()
5411 		return "adapter problem";  in ibmvnic_fw_err_cause()
5430 				    struct ibmvnic_adapter *adapter)  in handle_error_indication()  argument
5432 	struct device *dev = &adapter->vdev->dev;  in handle_error_indication()
5444 		ibmvnic_reset(adapter, VNIC_RESET_FATAL);  in handle_error_indication()
5446 		ibmvnic_reset(adapter, VNIC_RESET_NON_FATAL);  in handle_error_indication()
5450 				 struct ibmvnic_adapter *adapter)  in handle_change_mac_rsp()  argument
5452 	struct net_device *netdev = adapter->netdev;  in handle_change_mac_rsp()
5453 	struct device *dev = &adapter->vdev->dev;  in handle_change_mac_rsp()
5465 	ether_addr_copy(adapter->mac_addr,  in handle_change_mac_rsp()
5468 	complete(&adapter->fw_done);  in handle_change_mac_rsp()
5473 				   struct ibmvnic_adapter *adapter)  in handle_request_cap_rsp()  argument
5475 	struct device *dev = &adapter->vdev->dev;  in handle_request_cap_rsp()
5479 	atomic_dec(&adapter->running_cap_crqs);  in handle_request_cap_rsp()
5480 	netdev_dbg(adapter->netdev, "Outstanding request-caps: %d\n",  in handle_request_cap_rsp()
5481 		   atomic_read(&adapter->running_cap_crqs));  in handle_request_cap_rsp()
5484 		req_value = &adapter->req_tx_queues;  in handle_request_cap_rsp()
5488 		req_value = &adapter->req_rx_queues;  in handle_request_cap_rsp()
5492 		req_value = &adapter->req_rx_add_queues;  in handle_request_cap_rsp()
5496 		req_value = &adapter->req_tx_entries_per_subcrq;  in handle_request_cap_rsp()
5500 		req_value = &adapter->req_rx_add_entries_per_subcrq;  in handle_request_cap_rsp()
5504 		req_value = &adapter->req_mtu;  in handle_request_cap_rsp()
5508 		req_value = &adapter->promisc;  in handle_request_cap_rsp()
5530 			*req_value = adapter->fallback.mtu;  in handle_request_cap_rsp()
5536 		send_request_cap(adapter, 1);  in handle_request_cap_rsp()
5545 	if (atomic_read(&adapter->running_cap_crqs) == 0)  in handle_request_cap_rsp()
5546 		send_query_ip_offload(adapter);  in handle_request_cap_rsp()
5550 			    struct ibmvnic_adapter *adapter)  in handle_login_rsp()  argument
5552 	struct device *dev = &adapter->vdev->dev;  in handle_login_rsp()
5553 	struct net_device *netdev = adapter->netdev;  in handle_login_rsp()
5554 	struct ibmvnic_login_rsp_buffer *login_rsp = adapter->login_rsp_buf;  in handle_login_rsp()
5555 	struct ibmvnic_login_buffer *login = adapter->login_buf;  in handle_login_rsp()
5567 	if (!adapter->login_pending) {  in handle_login_rsp()
5571 	adapter->login_pending = false;  in handle_login_rsp()
5578 		adapter->init_done_rc = login_rsp_crq->generic.rc.code;  in handle_login_rsp()
5579 		complete(&adapter->init_done);  in handle_login_rsp()
5583 	if (adapter->failover_pending) {  in handle_login_rsp()
5584 		adapter->init_done_rc = -EAGAIN;  in handle_login_rsp()
5586 		complete(&adapter->init_done);  in handle_login_rsp()
5591 	netdev->mtu = adapter->req_mtu - ETH_HLEN;  in handle_login_rsp()
5593 	netdev_dbg(adapter->netdev, "Login Response Buffer:\n");  in handle_login_rsp()
5594 	for (i = 0; i < (adapter->login_rsp_buf_sz - 1) / 8 + 1; i++) {  in handle_login_rsp()
5595 		netdev_dbg(adapter->netdev, "%016lx\n",  in handle_login_rsp()
5596 			   ((unsigned long *)(adapter->login_rsp_buf))[i]);  in handle_login_rsp()
5602 	     adapter->req_rx_add_queues !=  in handle_login_rsp()
5605 		ibmvnic_reset(adapter, VNIC_RESET_FATAL);  in handle_login_rsp()
5621 		ibmvnic_reset(adapter, VNIC_RESET_FATAL);  in handle_login_rsp()
5625 	size_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +  in handle_login_rsp()
5626 		be32_to_cpu(adapter->login_rsp_buf->off_rxadd_buff_size));  in handle_login_rsp()
5630 	adapter->cur_rx_buf_sz = be64_to_cpu(size_array[0]);  in handle_login_rsp()
5632 	num_tx_pools = be32_to_cpu(adapter->login_rsp_buf->num_txsubm_subcrqs);  in handle_login_rsp()
5633 	num_rx_pools = be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);  in handle_login_rsp()
5635 	tx_handle_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +  in handle_login_rsp()
5636 				  be32_to_cpu(adapter->login_rsp_buf->off_txsubm_subcrqs));  in handle_login_rsp()
5637 	rx_handle_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +  in handle_login_rsp()
5638 				  be32_to_cpu(adapter->login_rsp_buf->off_rxadd_subcrqs));  in handle_login_rsp()
5641 		adapter->tx_scrq[i]->handle = tx_handle_array[i];  in handle_login_rsp()
5644 		adapter->rx_scrq[i]->handle = rx_handle_array[i];  in handle_login_rsp()
5646 	adapter->num_active_tx_scrqs = num_tx_pools;  in handle_login_rsp()
5647 	adapter->num_active_rx_scrqs = num_rx_pools;  in handle_login_rsp()
5648 	release_login_rsp_buffer(adapter);  in handle_login_rsp()
5649 	release_login_buffer(adapter);  in handle_login_rsp()
5650 	complete(&adapter->init_done);  in handle_login_rsp()
5656 				     struct ibmvnic_adapter *adapter)  in handle_request_unmap_rsp()  argument
5658 	struct device *dev = &adapter->vdev->dev;  in handle_request_unmap_rsp()
5667 				 struct ibmvnic_adapter *adapter)  in handle_query_map_rsp()  argument
5669 	struct net_device *netdev = adapter->netdev;  in handle_query_map_rsp()
5670 	struct device *dev = &adapter->vdev->dev;  in handle_query_map_rsp()
5685 				 struct ibmvnic_adapter *adapter)  in handle_query_cap_rsp()  argument
5687 	struct net_device *netdev = adapter->netdev;  in handle_query_cap_rsp()
5688 	struct device *dev = &adapter->vdev->dev;  in handle_query_cap_rsp()
5691 	atomic_dec(&adapter->running_cap_crqs);  in handle_query_cap_rsp()
5693 		   atomic_read(&adapter->running_cap_crqs));  in handle_query_cap_rsp()
5702 		adapter->min_tx_queues =  in handle_query_cap_rsp()
5705 			   adapter->min_tx_queues);  in handle_query_cap_rsp()
5708 		adapter->min_rx_queues =  in handle_query_cap_rsp()
5711 			   adapter->min_rx_queues);  in handle_query_cap_rsp()
5714 		adapter->min_rx_add_queues =  in handle_query_cap_rsp()
5717 			   adapter->min_rx_add_queues);  in handle_query_cap_rsp()
5720 		adapter->max_tx_queues =  in handle_query_cap_rsp()
5723 			   adapter->max_tx_queues);  in handle_query_cap_rsp()
5726 		adapter->max_rx_queues =  in handle_query_cap_rsp()
5729 			   adapter->max_rx_queues);  in handle_query_cap_rsp()
5732 		adapter->max_rx_add_queues =  in handle_query_cap_rsp()
5735 			   adapter->max_rx_add_queues);  in handle_query_cap_rsp()
5738 		adapter->min_tx_entries_per_subcrq =  in handle_query_cap_rsp()
5741 			   adapter->min_tx_entries_per_subcrq);  in handle_query_cap_rsp()
5744 		adapter->min_rx_add_entries_per_subcrq =  in handle_query_cap_rsp()
5747 			   adapter->min_rx_add_entries_per_subcrq);  in handle_query_cap_rsp()
5750 		adapter->max_tx_entries_per_subcrq =  in handle_query_cap_rsp()
5753 			   adapter->max_tx_entries_per_subcrq);  in handle_query_cap_rsp()
5756 		adapter->max_rx_add_entries_per_subcrq =  in handle_query_cap_rsp()
5759 			   adapter->max_rx_add_entries_per_subcrq);  in handle_query_cap_rsp()
5762 		adapter->tcp_ip_offload =  in handle_query_cap_rsp()
5765 			   adapter->tcp_ip_offload);  in handle_query_cap_rsp()
5768 		adapter->promisc_supported =  in handle_query_cap_rsp()
5771 			   adapter->promisc_supported);  in handle_query_cap_rsp()
5774 		adapter->min_mtu = be64_to_cpu(crq->query_capability.number);  in handle_query_cap_rsp()
5775 		netdev->min_mtu = adapter->min_mtu - ETH_HLEN;  in handle_query_cap_rsp()
5776 		netdev_dbg(netdev, "min_mtu = %lld\n", adapter->min_mtu);  in handle_query_cap_rsp()
5779 		adapter->max_mtu = be64_to_cpu(crq->query_capability.number);  in handle_query_cap_rsp()
5780 		netdev->max_mtu = adapter->max_mtu - ETH_HLEN;  in handle_query_cap_rsp()
5781 		netdev_dbg(netdev, "max_mtu = %lld\n", adapter->max_mtu);  in handle_query_cap_rsp()
5784 		adapter->max_multicast_filters =  in handle_query_cap_rsp()
5787 			   adapter->max_multicast_filters);  in handle_query_cap_rsp()
5790 		adapter->vlan_header_insertion =  in handle_query_cap_rsp()
5792 		if (adapter->vlan_header_insertion)  in handle_query_cap_rsp()
5795 			   adapter->vlan_header_insertion);  in handle_query_cap_rsp()
5798 		adapter->rx_vlan_header_insertion =  in handle_query_cap_rsp()
5801 			   adapter->rx_vlan_header_insertion);  in handle_query_cap_rsp()
5804 		adapter->max_tx_sg_entries =  in handle_query_cap_rsp()
5807 			   adapter->max_tx_sg_entries);  in handle_query_cap_rsp()
5810 		adapter->rx_sg_supported =  in handle_query_cap_rsp()
5813 			   adapter->rx_sg_supported);  in handle_query_cap_rsp()
5816 		adapter->opt_tx_comp_sub_queues =  in handle_query_cap_rsp()
5819 			   adapter->opt_tx_comp_sub_queues);  in handle_query_cap_rsp()
5822 		adapter->opt_rx_comp_queues =  in handle_query_cap_rsp()
5825 			   adapter->opt_rx_comp_queues);  in handle_query_cap_rsp()
5828 		adapter->opt_rx_bufadd_q_per_rx_comp_q =  in handle_query_cap_rsp()
5831 			   adapter->opt_rx_bufadd_q_per_rx_comp_q);  in handle_query_cap_rsp()
5834 		adapter->opt_tx_entries_per_subcrq =  in handle_query_cap_rsp()
5837 			   adapter->opt_tx_entries_per_subcrq);  in handle_query_cap_rsp()
5840 		adapter->opt_rxba_entries_per_subcrq =  in handle_query_cap_rsp()
5843 			   adapter->opt_rxba_entries_per_subcrq);  in handle_query_cap_rsp()
5846 		adapter->tx_rx_desc_req = crq->query_capability.number;  in handle_query_cap_rsp()
5848 			   adapter->tx_rx_desc_req);  in handle_query_cap_rsp()
5857 	if (atomic_read(&adapter->running_cap_crqs) == 0)  in handle_query_cap_rsp()
5858 		send_request_cap(adapter, 0);  in handle_query_cap_rsp()
5861 static int send_query_phys_parms(struct ibmvnic_adapter *adapter)  in send_query_phys_parms()  argument
5870 	mutex_lock(&adapter->fw_lock);  in send_query_phys_parms()
5871 	adapter->fw_done_rc = 0;  in send_query_phys_parms()
5872 	reinit_completion(&adapter->fw_done);  in send_query_phys_parms()
5874 	rc = ibmvnic_send_crq(adapter, &crq);  in send_query_phys_parms()
5876 		mutex_unlock(&adapter->fw_lock);  in send_query_phys_parms()
5880 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);  in send_query_phys_parms()
5882 		mutex_unlock(&adapter->fw_lock);  in send_query_phys_parms()
5886 	mutex_unlock(&adapter->fw_lock);  in send_query_phys_parms()
5887 	return adapter->fw_done_rc ? -EIO : 0;  in send_query_phys_parms()
5891 				       struct ibmvnic_adapter *adapter)  in handle_query_phys_parms_rsp()  argument
5893 	struct net_device *netdev = adapter->netdev;  in handle_query_phys_parms_rsp()
5904 		adapter->speed = SPEED_10;  in handle_query_phys_parms_rsp()
5907 		adapter->speed = SPEED_100;  in handle_query_phys_parms_rsp()
5910 		adapter->speed = SPEED_1000;  in handle_query_phys_parms_rsp()
5913 		adapter->speed = SPEED_10000;  in handle_query_phys_parms_rsp()
5916 		adapter->speed = SPEED_25000;  in handle_query_phys_parms_rsp()
5919 		adapter->speed = SPEED_40000;  in handle_query_phys_parms_rsp()
5922 		adapter->speed = SPEED_50000;  in handle_query_phys_parms_rsp()
5925 		adapter->speed = SPEED_100000;  in handle_query_phys_parms_rsp()
5928 		adapter->speed = SPEED_200000;  in handle_query_phys_parms_rsp()
5933 		adapter->speed = SPEED_UNKNOWN;  in handle_query_phys_parms_rsp()
5936 		adapter->duplex = DUPLEX_FULL;  in handle_query_phys_parms_rsp()
5938 		adapter->duplex = DUPLEX_HALF;  in handle_query_phys_parms_rsp()
5940 		adapter->duplex = DUPLEX_UNKNOWN;  in handle_query_phys_parms_rsp()
5946 			       struct ibmvnic_adapter *adapter)  in ibmvnic_handle_crq()  argument
5949 	struct net_device *netdev = adapter->netdev;  in ibmvnic_handle_crq()
5950 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_handle_crq()
5962 			adapter->from_passive_init = true;  in ibmvnic_handle_crq()
5966 			adapter->login_pending = false;  in ibmvnic_handle_crq()
5968 			if (adapter->state == VNIC_DOWN)  in ibmvnic_handle_crq()
5969 				rc = ibmvnic_reset(adapter, VNIC_RESET_PASSIVE_INIT);  in ibmvnic_handle_crq()
5971 				rc = ibmvnic_reset(adapter, VNIC_RESET_FAILOVER);  in ibmvnic_handle_crq()
5975 				 * reset either because the adapter was still  in ibmvnic_handle_crq()
5980 				 * is already scheduled or the adapter is  in ibmvnic_handle_crq()
5986 				adapter->failover_pending = false;  in ibmvnic_handle_crq()
5989 			if (!completion_done(&adapter->init_done)) {  in ibmvnic_handle_crq()
5990 				if (!adapter->init_done_rc)  in ibmvnic_handle_crq()
5991 					adapter->init_done_rc = -EAGAIN;  in ibmvnic_handle_crq()
5992 				complete(&adapter->init_done);  in ibmvnic_handle_crq()
5998 			adapter->crq.active = true;  in ibmvnic_handle_crq()
5999 			send_version_xchg(adapter);  in ibmvnic_handle_crq()
6007 		adapter->crq.active = false;  in ibmvnic_handle_crq()
6011 		if (!completion_done(&adapter->fw_done)) {  in ibmvnic_handle_crq()
6012 			adapter->fw_done_rc = -EIO;  in ibmvnic_handle_crq()
6013 			complete(&adapter->fw_done);  in ibmvnic_handle_crq()
6017 		if (!completion_done(&adapter->init_done)) {  in ibmvnic_handle_crq()
6018 			adapter->init_done_rc = -EAGAIN;  in ibmvnic_handle_crq()
6019 			complete(&adapter->init_done);  in ibmvnic_handle_crq()
6022 		if (!completion_done(&adapter->stats_done))  in ibmvnic_handle_crq()
6023 			complete(&adapter->stats_done);  in ibmvnic_handle_crq()
6024 		if (test_bit(0, &adapter->resetting))  in ibmvnic_handle_crq()
6025 			adapter->force_reset_recovery = true;  in ibmvnic_handle_crq()
6027 			dev_info(dev, "Migrated, re-enabling adapter\n");  in ibmvnic_handle_crq()
6028 			ibmvnic_reset(adapter, VNIC_RESET_MOBILITY);  in ibmvnic_handle_crq()
6031 			adapter->failover_pending = true;  in ibmvnic_handle_crq()
6033 			/* The adapter lost the connection */  in ibmvnic_handle_crq()
6034 			dev_err(dev, "Virtual Adapter failed (rc=%d)\n",  in ibmvnic_handle_crq()
6036 			ibmvnic_reset(adapter, VNIC_RESET_FATAL);  in ibmvnic_handle_crq()
6058 		send_query_cap(adapter);  in ibmvnic_handle_crq()
6061 		handle_query_cap_rsp(crq, adapter);  in ibmvnic_handle_crq()
6064 		handle_query_map_rsp(crq, adapter);  in ibmvnic_handle_crq()
6067 		adapter->fw_done_rc = crq->request_map_rsp.rc.code;  in ibmvnic_handle_crq()
6068 		complete(&adapter->fw_done);  in ibmvnic_handle_crq()
6071 		handle_request_unmap_rsp(crq, adapter);  in ibmvnic_handle_crq()
6074 		handle_request_cap_rsp(crq, adapter);  in ibmvnic_handle_crq()
6078 		handle_login_rsp(crq, adapter);  in ibmvnic_handle_crq()
6085 		adapter->logical_link_state =  in ibmvnic_handle_crq()
6087 		adapter->init_done_rc = crq->logical_link_state_rsp.rc.code;  in ibmvnic_handle_crq()
6088 		complete(&adapter->init_done);  in ibmvnic_handle_crq()
6092 		adapter->phys_link_state =  in ibmvnic_handle_crq()
6094 		adapter->logical_link_state =  in ibmvnic_handle_crq()
6096 		if (adapter->phys_link_state && adapter->logical_link_state)  in ibmvnic_handle_crq()
6103 		adapter->fw_done_rc = handle_change_mac_rsp(crq, adapter);  in ibmvnic_handle_crq()
6107 		handle_error_indication(crq, adapter);  in ibmvnic_handle_crq()
6111 		complete(&adapter->stats_done);  in ibmvnic_handle_crq()
6115 		handle_query_ip_offload_rsp(adapter);  in ibmvnic_handle_crq()
6122 		dma_unmap_single(dev, adapter->ip_offload_ctrl_tok,  in ibmvnic_handle_crq()
6123 				 sizeof(adapter->ip_offload_ctrl),  in ibmvnic_handle_crq()
6125 		complete(&adapter->init_done);  in ibmvnic_handle_crq()
6129 		complete(&adapter->fw_done);  in ibmvnic_handle_crq()
6132 		handle_vpd_size_rsp(crq, adapter);  in ibmvnic_handle_crq()
6135 		handle_vpd_rsp(crq, adapter);  in ibmvnic_handle_crq()
6138 		adapter->fw_done_rc = handle_query_phys_parms_rsp(crq, adapter);  in ibmvnic_handle_crq()
6139 		complete(&adapter->fw_done);  in ibmvnic_handle_crq()
6149 	struct ibmvnic_adapter *adapter = instance;  in ibmvnic_interrupt()  local
6151 	tasklet_schedule(&adapter->tasklet);  in ibmvnic_interrupt()
6157 	struct ibmvnic_adapter *adapter = from_tasklet(adapter, t, tasklet);  in ibmvnic_tasklet()  local
6158 	struct ibmvnic_crq_queue *queue = &adapter->crq;  in ibmvnic_tasklet()
6165 	while ((crq = ibmvnic_next_crq(adapter)) != NULL) {  in ibmvnic_tasklet()
6172 		ibmvnic_handle_crq(crq, adapter);  in ibmvnic_tasklet()
6179 static int ibmvnic_reenable_crq_queue(struct ibmvnic_adapter *adapter)  in ibmvnic_reenable_crq_queue()  argument
6181 	struct vio_dev *vdev = adapter->vdev;  in ibmvnic_reenable_crq_queue()
6189 		dev_err(&vdev->dev, "Error enabling adapter (rc=%d)\n", rc);  in ibmvnic_reenable_crq_queue()
6194 static int ibmvnic_reset_crq(struct ibmvnic_adapter *adapter)  in ibmvnic_reset_crq()  argument
6196 	struct ibmvnic_crq_queue *crq = &adapter->crq;  in ibmvnic_reset_crq()
6197 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_reset_crq()
6198 	struct vio_dev *vdev = adapter->vdev;  in ibmvnic_reset_crq()
6219 		/* Adapter is good, but other end is not ready */  in ibmvnic_reset_crq()
6220 		dev_warn(dev, "Partner adapter not ready\n");  in ibmvnic_reset_crq()
6227 static void release_crq_queue(struct ibmvnic_adapter *adapter)  in release_crq_queue()  argument
6229 	struct ibmvnic_crq_queue *crq = &adapter->crq;  in release_crq_queue()
6230 	struct vio_dev *vdev = adapter->vdev;  in release_crq_queue()
6236 	netdev_dbg(adapter->netdev, "Releasing CRQ\n");  in release_crq_queue()
6237 	free_irq(vdev->irq, adapter);  in release_crq_queue()
6238 	tasklet_kill(&adapter->tasklet);  in release_crq_queue()
6250 static int init_crq_queue(struct ibmvnic_adapter *adapter)  in init_crq_queue()  argument
6252 	struct ibmvnic_crq_queue *crq = &adapter->crq;  in init_crq_queue()
6253 	struct device *dev = &adapter->vdev->dev;  in init_crq_queue()
6254 	struct vio_dev *vdev = adapter->vdev;  in init_crq_queue()
6277 		rc = ibmvnic_reset_crq(adapter);  in init_crq_queue()
6281 		dev_warn(dev, "Partner adapter not ready\n");  in init_crq_queue()
6283 		dev_warn(dev, "Error %d opening adapter\n", rc);  in init_crq_queue()
6289 	tasklet_setup(&adapter->tasklet, (void *)ibmvnic_tasklet);  in init_crq_queue()
6291 	netdev_dbg(adapter->netdev, "registering irq 0x%x\n", vdev->irq);  in init_crq_queue()
6293 		 adapter->vdev->unit_address);  in init_crq_queue()
6294 	rc = request_irq(vdev->irq, ibmvnic_interrupt, 0, crq->name, adapter);  in init_crq_queue()
6311 	tasklet_schedule(&adapter->tasklet);  in init_crq_queue()
6316 	tasklet_kill(&adapter->tasklet);  in init_crq_queue()
6328 static int ibmvnic_reset_init(struct ibmvnic_adapter *adapter, bool reset)  in ibmvnic_reset_init()  argument
6330 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_reset_init()
6332 	u64 old_num_rx_queues = adapter->req_rx_queues;  in ibmvnic_reset_init()
6333 	u64 old_num_tx_queues = adapter->req_tx_queues;  in ibmvnic_reset_init()
6336 	adapter->from_passive_init = false;  in ibmvnic_reset_init()
6338 	rc = ibmvnic_send_crq_init(adapter);  in ibmvnic_reset_init()
6344 	if (!wait_for_completion_timeout(&adapter->init_done, timeout)) {  in ibmvnic_reset_init()
6349 	if (adapter->init_done_rc) {  in ibmvnic_reset_init()
6350 		release_crq_queue(adapter);  in ibmvnic_reset_init()
6351 		dev_err(dev, "CRQ-init failed, %d\n", adapter->init_done_rc);  in ibmvnic_reset_init()
6352 		return adapter->init_done_rc;  in ibmvnic_reset_init()
6355 	if (adapter->from_passive_init) {  in ibmvnic_reset_init()
6356 		adapter->state = VNIC_OPEN;  in ibmvnic_reset_init()
6357 		adapter->from_passive_init = false;  in ibmvnic_reset_init()
6363 	    test_bit(0, &adapter->resetting) && !adapter->wait_for_reset &&  in ibmvnic_reset_init()
6364 	    adapter->reset_reason != VNIC_RESET_MOBILITY) {  in ibmvnic_reset_init()
6365 		if (adapter->req_rx_queues != old_num_rx_queues ||  in ibmvnic_reset_init()
6366 		    adapter->req_tx_queues != old_num_tx_queues) {  in ibmvnic_reset_init()
6367 			release_sub_crqs(adapter, 0);  in ibmvnic_reset_init()
6368 			rc = init_sub_crqs(adapter);  in ibmvnic_reset_init()
6377 			clean_tx_pools(adapter);  in ibmvnic_reset_init()
6379 			rc = reset_sub_crq_queues(adapter);  in ibmvnic_reset_init()
6382 		rc = init_sub_crqs(adapter);  in ibmvnic_reset_init()
6387 		release_crq_queue(adapter);  in ibmvnic_reset_init()
6391 	rc = init_sub_crq_irqs(adapter);  in ibmvnic_reset_init()
6394 		release_crq_queue(adapter);  in ibmvnic_reset_init()
6404 	struct ibmvnic_adapter *adapter;  in ibmvnic_probe()  local
6428 	adapter = netdev_priv(netdev);  in ibmvnic_probe()
6429 	adapter->state = VNIC_PROBING;  in ibmvnic_probe()
6431 	adapter->vdev = dev;  in ibmvnic_probe()
6432 	adapter->netdev = netdev;  in ibmvnic_probe()
6433 	adapter->login_pending = false;  in ibmvnic_probe()
6434 	memset(&adapter->map_ids, 0, sizeof(adapter->map_ids));  in ibmvnic_probe()
6436 	bitmap_set(adapter->map_ids, 0, 1);  in ibmvnic_probe()
6438 	ether_addr_copy(adapter->mac_addr, mac_addr_p);  in ibmvnic_probe()
6439 	eth_hw_addr_set(netdev, adapter->mac_addr);  in ibmvnic_probe()
6445 	INIT_WORK(&adapter->ibmvnic_reset, __ibmvnic_reset);  in ibmvnic_probe()
6446 	INIT_DELAYED_WORK(&adapter->ibmvnic_delayed_reset,  in ibmvnic_probe()
6448 	INIT_LIST_HEAD(&adapter->rwi_list);  in ibmvnic_probe()
6449 	spin_lock_init(&adapter->rwi_lock);  in ibmvnic_probe()
6450 	spin_lock_init(&adapter->state_lock);  in ibmvnic_probe()
6451 	mutex_init(&adapter->fw_lock);  in ibmvnic_probe()
6452 	init_completion(&adapter->probe_done);  in ibmvnic_probe()
6453 	init_completion(&adapter->init_done);  in ibmvnic_probe()
6454 	init_completion(&adapter->fw_done);  in ibmvnic_probe()
6455 	init_completion(&adapter->reset_done);  in ibmvnic_probe()
6456 	init_completion(&adapter->stats_done);  in ibmvnic_probe()
6457 	clear_bit(0, &adapter->resetting);  in ibmvnic_probe()
6458 	adapter->prev_rx_buf_sz = 0;  in ibmvnic_probe()
6459 	adapter->prev_mtu = 0;  in ibmvnic_probe()
6463 		reinit_init_done(adapter);  in ibmvnic_probe()
6468 		adapter->failover_pending = false;  in ibmvnic_probe()
6474 		release_crq_queue(adapter);  in ibmvnic_probe()
6486 		spin_lock_irqsave(&adapter->rwi_lock, flags);  in ibmvnic_probe()
6487 		flush_reset_queue(adapter);  in ibmvnic_probe()
6488 		spin_unlock_irqrestore(&adapter->rwi_lock, flags);  in ibmvnic_probe()
6490 		rc = init_crq_queue(adapter);  in ibmvnic_probe()
6497 		rc = ibmvnic_reset_init(adapter, false);  in ibmvnic_probe()
6508 	rc = init_stats_buffers(adapter);  in ibmvnic_probe()
6512 	rc = init_stats_token(adapter);  in ibmvnic_probe()
6523 		adapter->state = VNIC_PROBED;  in ibmvnic_probe()
6524 		netdev->mtu = adapter->req_mtu - ETH_HLEN;  in ibmvnic_probe()
6525 		netdev->min_mtu = adapter->min_mtu - ETH_HLEN;  in ibmvnic_probe()
6526 		netdev->max_mtu = adapter->max_mtu - ETH_HLEN;  in ibmvnic_probe()
6528 		adapter->state = VNIC_DOWN;  in ibmvnic_probe()
6531 	adapter->wait_for_reset = false;  in ibmvnic_probe()
6532 	adapter->last_reset_time = jiffies;  in ibmvnic_probe()
6541 	rc = ibmvnic_cpu_notif_add(adapter);  in ibmvnic_probe()
6547 	complete(&adapter->probe_done);  in ibmvnic_probe()
6558 	release_stats_token(adapter);  in ibmvnic_probe()
6561 	release_stats_buffers(adapter);  in ibmvnic_probe()
6564 	release_sub_crqs(adapter, 1);  in ibmvnic_probe()
6565 	release_crq_queue(adapter);  in ibmvnic_probe()
6570 	adapter->state = VNIC_REMOVING;  in ibmvnic_probe()
6571 	complete(&adapter->probe_done);  in ibmvnic_probe()
6572 	flush_work(&adapter->ibmvnic_reset);  in ibmvnic_probe()
6573 	flush_delayed_work(&adapter->ibmvnic_delayed_reset);  in ibmvnic_probe()
6575 	flush_reset_queue(adapter);  in ibmvnic_probe()
6577 	mutex_destroy(&adapter->fw_lock);  in ibmvnic_probe()
6586 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_remove()  local
6589 	spin_lock_irqsave(&adapter->state_lock, flags);  in ibmvnic_remove()
6598 	spin_lock(&adapter->rwi_lock);  in ibmvnic_remove()
6599 	adapter->state = VNIC_REMOVING;  in ibmvnic_remove()
6600 	spin_unlock(&adapter->rwi_lock);  in ibmvnic_remove()
6602 	spin_unlock_irqrestore(&adapter->state_lock, flags);  in ibmvnic_remove()
6604 	ibmvnic_cpu_notif_remove(adapter);  in ibmvnic_remove()
6606 	flush_work(&adapter->ibmvnic_reset);  in ibmvnic_remove()
6607 	flush_delayed_work(&adapter->ibmvnic_delayed_reset);  in ibmvnic_remove()
6612 	release_resources(adapter);  in ibmvnic_remove()
6613 	release_rx_pools(adapter);  in ibmvnic_remove()
6614 	release_tx_pools(adapter);  in ibmvnic_remove()
6615 	release_sub_crqs(adapter, 1);  in ibmvnic_remove()
6616 	release_crq_queue(adapter);  in ibmvnic_remove()
6618 	release_stats_token(adapter);  in ibmvnic_remove()
6619 	release_stats_buffers(adapter);  in ibmvnic_remove()
6621 	adapter->state = VNIC_REMOVED;  in ibmvnic_remove()
6624 	mutex_destroy(&adapter->fw_lock);  in ibmvnic_remove()
6634 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in failover_store()  local
6642 	rc = plpar_hcall(H_VIOCTL, retbuf, adapter->vdev->unit_address,  in failover_store()
6653 	rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,  in failover_store()
6666 	ibmvnic_reset(adapter, VNIC_RESET_FAILOVER);  in failover_store()
6675 	struct ibmvnic_adapter *adapter;  in ibmvnic_get_desired_dma()  local
6686 	adapter = netdev_priv(netdev);  in ibmvnic_get_desired_dma()
6691 	for (i = 0; i < adapter->req_tx_queues + adapter->req_rx_queues; i++)  in ibmvnic_get_desired_dma()
6694 	for (i = 0; i < adapter->num_active_rx_pools; i++)  in ibmvnic_get_desired_dma()
6695 		ret += adapter->rx_pool[i].size *  in ibmvnic_get_desired_dma()
6696 		    IOMMU_PAGE_ALIGN(adapter->rx_pool[i].buff_size, tbl);  in ibmvnic_get_desired_dma()
6704 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_resume()  local
6706 	if (adapter->state != VNIC_OPEN)  in ibmvnic_resume()
6709 	tasklet_schedule(&adapter->tasklet);  in ibmvnic_resume()