195fa0405SHaiyang Zhang /* 295fa0405SHaiyang Zhang * Copyright (c) 2009, Microsoft Corporation. 395fa0405SHaiyang Zhang * 495fa0405SHaiyang Zhang * This program is free software; you can redistribute it and/or modify it 595fa0405SHaiyang Zhang * under the terms and conditions of the GNU General Public License, 695fa0405SHaiyang Zhang * version 2, as published by the Free Software Foundation. 795fa0405SHaiyang Zhang * 895fa0405SHaiyang Zhang * This program is distributed in the hope it will be useful, but WITHOUT 995fa0405SHaiyang Zhang * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 1095fa0405SHaiyang Zhang * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 1195fa0405SHaiyang Zhang * more details. 1295fa0405SHaiyang Zhang * 1395fa0405SHaiyang Zhang * You should have received a copy of the GNU General Public License along with 14adf8d3ffSJeff Kirsher * this program; if not, see <http://www.gnu.org/licenses/>. 1595fa0405SHaiyang Zhang * 1695fa0405SHaiyang Zhang * Authors: 1795fa0405SHaiyang Zhang * Haiyang Zhang <haiyangz@microsoft.com> 1895fa0405SHaiyang Zhang * Hank Janssen <hjanssen@microsoft.com> 1995fa0405SHaiyang Zhang */ 2095fa0405SHaiyang Zhang #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 2195fa0405SHaiyang Zhang 2295fa0405SHaiyang Zhang #include <linux/kernel.h> 2395fa0405SHaiyang Zhang #include <linux/sched.h> 2495fa0405SHaiyang Zhang #include <linux/wait.h> 2595fa0405SHaiyang Zhang #include <linux/mm.h> 2695fa0405SHaiyang Zhang #include <linux/delay.h> 2795fa0405SHaiyang Zhang #include <linux/io.h> 2895fa0405SHaiyang Zhang #include <linux/slab.h> 2995fa0405SHaiyang Zhang #include <linux/netdevice.h> 30f157e78dSHaiyang Zhang #include <linux/if_ether.h> 31d6472302SStephen Rothwell #include <linux/vmalloc.h> 329749fed5Sstephen hemminger #include <linux/rtnetlink.h> 3343bf99ceSstephen hemminger #include <linux/prefetch.h> 34a7f99d0fSStephen Hemminger #include <linux/reciprocal_div.h> 359749fed5Sstephen hemminger 36c25aaf81SKY Srinivasan #include <asm/sync_bitops.h> 3795fa0405SHaiyang Zhang 3895fa0405SHaiyang Zhang #include "hyperv_net.h" 39ec966381SStephen Hemminger #include "netvsc_trace.h" 4095fa0405SHaiyang Zhang 4184bf9cefSKY Srinivasan /* 4284bf9cefSKY Srinivasan * Switch the data path from the synthetic interface to the VF 4384bf9cefSKY Srinivasan * interface. 4484bf9cefSKY Srinivasan */ 450a1275caSVitaly Kuznetsov void netvsc_switch_datapath(struct net_device *ndev, bool vf) 4684bf9cefSKY Srinivasan { 473d541ac5SVitaly Kuznetsov struct net_device_context *net_device_ctx = netdev_priv(ndev); 483d541ac5SVitaly Kuznetsov struct hv_device *dev = net_device_ctx->device_ctx; 4979e8cbe7Sstephen hemminger struct netvsc_device *nv_dev = rtnl_dereference(net_device_ctx->nvdev); 500a1275caSVitaly Kuznetsov struct nvsp_message *init_pkt = &nv_dev->channel_init_pkt; 5184bf9cefSKY Srinivasan 5284bf9cefSKY Srinivasan memset(init_pkt, 0, sizeof(struct nvsp_message)); 5384bf9cefSKY Srinivasan init_pkt->hdr.msg_type = NVSP_MSG4_TYPE_SWITCH_DATA_PATH; 5484bf9cefSKY Srinivasan if (vf) 5584bf9cefSKY Srinivasan init_pkt->msg.v4_msg.active_dp.active_datapath = 5684bf9cefSKY Srinivasan NVSP_DATAPATH_VF; 5784bf9cefSKY Srinivasan else 5884bf9cefSKY Srinivasan init_pkt->msg.v4_msg.active_dp.active_datapath = 5984bf9cefSKY Srinivasan NVSP_DATAPATH_SYNTHETIC; 6084bf9cefSKY Srinivasan 61ec966381SStephen Hemminger trace_nvsp_send(ndev, init_pkt); 62ec966381SStephen Hemminger 6384bf9cefSKY Srinivasan vmbus_sendpacket(dev->channel, init_pkt, 6484bf9cefSKY Srinivasan sizeof(struct nvsp_message), 6584bf9cefSKY Srinivasan (unsigned long)init_pkt, 6684bf9cefSKY Srinivasan VM_PKT_DATA_INBAND, 0); 6784bf9cefSKY Srinivasan } 6884bf9cefSKY Srinivasan 6988098834SVitaly Kuznetsov static struct netvsc_device *alloc_net_device(void) 7095fa0405SHaiyang Zhang { 7195fa0405SHaiyang Zhang struct netvsc_device *net_device; 7295fa0405SHaiyang Zhang 7395fa0405SHaiyang Zhang net_device = kzalloc(sizeof(struct netvsc_device), GFP_KERNEL); 7495fa0405SHaiyang Zhang if (!net_device) 7595fa0405SHaiyang Zhang return NULL; 7695fa0405SHaiyang Zhang 77dc5cd894SHaiyang Zhang init_waitqueue_head(&net_device->wait_drain); 7895fa0405SHaiyang Zhang net_device->destroy = false; 790da6edbdSStephen Hemminger 807c3877f2SHaiyang Zhang net_device->max_pkt = RNDIS_MAX_PKT_DEFAULT; 817c3877f2SHaiyang Zhang net_device->pkt_align = RNDIS_PKT_ALIGN_DEFAULT; 828b532797Sstephen hemminger 83fd612602SStephen Hemminger init_completion(&net_device->channel_init_wait); 84732e4985Sstephen hemminger init_waitqueue_head(&net_device->subchan_open); 858195b139SStephen Hemminger INIT_WORK(&net_device->subchan_work, rndis_set_subchannel); 867c3877f2SHaiyang Zhang 8795fa0405SHaiyang Zhang return net_device; 8895fa0405SHaiyang Zhang } 8995fa0405SHaiyang Zhang 90545a8e79Sstephen hemminger static void free_netvsc_device(struct rcu_head *head) 91f90251c8SHaiyang Zhang { 92545a8e79Sstephen hemminger struct netvsc_device *nvdev 93545a8e79Sstephen hemminger = container_of(head, struct netvsc_device, rcu); 94c0b558e5SHaiyang Zhang int i; 95c0b558e5SHaiyang Zhang 9602400fceSStephen Hemminger kfree(nvdev->extension); 9702400fceSStephen Hemminger vfree(nvdev->recv_buf); 9802400fceSStephen Hemminger vfree(nvdev->send_buf); 9902400fceSStephen Hemminger kfree(nvdev->send_section_map); 10002400fceSStephen Hemminger 101c0b558e5SHaiyang Zhang for (i = 0; i < VRSS_CHANNEL_MAX; i++) 1027426b1a5Sstephen hemminger vfree(nvdev->chan_table[i].mrc.slots); 103c0b558e5SHaiyang Zhang 104f90251c8SHaiyang Zhang kfree(nvdev); 105f90251c8SHaiyang Zhang } 106f90251c8SHaiyang Zhang 107545a8e79Sstephen hemminger static void free_netvsc_device_rcu(struct netvsc_device *nvdev) 108545a8e79Sstephen hemminger { 109545a8e79Sstephen hemminger call_rcu(&nvdev->rcu, free_netvsc_device); 110545a8e79Sstephen hemminger } 11146b4f7f5Sstephen hemminger 1127992894cSMohammed Gamal static void netvsc_revoke_recv_buf(struct hv_device *device, 1133f076effSMohammed Gamal struct netvsc_device *net_device, 1143f076effSMohammed Gamal struct net_device *ndev) 11595fa0405SHaiyang Zhang { 1167992894cSMohammed Gamal struct nvsp_message *revoke_packet; 1177a2a0a84SStephen Hemminger int ret; 11895fa0405SHaiyang Zhang 11995fa0405SHaiyang Zhang /* 12095fa0405SHaiyang Zhang * If we got a section count, it means we received a 12195fa0405SHaiyang Zhang * SendReceiveBufferComplete msg (ie sent 12295fa0405SHaiyang Zhang * NvspMessage1TypeSendReceiveBuffer msg) therefore, we need 12395fa0405SHaiyang Zhang * to send a revoke msg here 12495fa0405SHaiyang Zhang */ 12595fa0405SHaiyang Zhang if (net_device->recv_section_cnt) { 12695fa0405SHaiyang Zhang /* Send the revoke receive buffer */ 12795fa0405SHaiyang Zhang revoke_packet = &net_device->revoke_packet; 12895fa0405SHaiyang Zhang memset(revoke_packet, 0, sizeof(struct nvsp_message)); 12995fa0405SHaiyang Zhang 13095fa0405SHaiyang Zhang revoke_packet->hdr.msg_type = 13195fa0405SHaiyang Zhang NVSP_MSG1_TYPE_REVOKE_RECV_BUF; 13295fa0405SHaiyang Zhang revoke_packet->msg.v1_msg. 13395fa0405SHaiyang Zhang revoke_recv_buf.id = NETVSC_RECEIVE_BUFFER_ID; 13495fa0405SHaiyang Zhang 135ec966381SStephen Hemminger trace_nvsp_send(ndev, revoke_packet); 136ec966381SStephen Hemminger 1373d541ac5SVitaly Kuznetsov ret = vmbus_sendpacket(device->channel, 13895fa0405SHaiyang Zhang revoke_packet, 13995fa0405SHaiyang Zhang sizeof(struct nvsp_message), 14095fa0405SHaiyang Zhang (unsigned long)revoke_packet, 14195fa0405SHaiyang Zhang VM_PKT_DATA_INBAND, 0); 14273e64fa4SK. Y. Srinivasan /* If the failure is because the channel is rescinded; 14373e64fa4SK. Y. Srinivasan * ignore the failure since we cannot send on a rescinded 14473e64fa4SK. Y. Srinivasan * channel. This would allow us to properly cleanup 14573e64fa4SK. Y. Srinivasan * even when the channel is rescinded. 14673e64fa4SK. Y. Srinivasan */ 14773e64fa4SK. Y. Srinivasan if (device->channel->rescind) 14873e64fa4SK. Y. Srinivasan ret = 0; 14995fa0405SHaiyang Zhang /* 15095fa0405SHaiyang Zhang * If we failed here, we might as well return and 15195fa0405SHaiyang Zhang * have a leak rather than continue and a bugchk 15295fa0405SHaiyang Zhang */ 15395fa0405SHaiyang Zhang if (ret != 0) { 15495fa0405SHaiyang Zhang netdev_err(ndev, "unable to send " 15595fa0405SHaiyang Zhang "revoke receive buffer to netvsp\n"); 1567a2a0a84SStephen Hemminger return; 15795fa0405SHaiyang Zhang } 1588b532797Sstephen hemminger net_device->recv_section_cnt = 0; 15995fa0405SHaiyang Zhang } 1607992894cSMohammed Gamal } 1617992894cSMohammed Gamal 1627992894cSMohammed Gamal static void netvsc_revoke_send_buf(struct hv_device *device, 1633f076effSMohammed Gamal struct netvsc_device *net_device, 1643f076effSMohammed Gamal struct net_device *ndev) 1657992894cSMohammed Gamal { 1667992894cSMohammed Gamal struct nvsp_message *revoke_packet; 1677992894cSMohammed Gamal int ret; 16895fa0405SHaiyang Zhang 169c25aaf81SKY Srinivasan /* Deal with the send buffer we may have setup. 170c25aaf81SKY Srinivasan * If we got a send section size, it means we received a 171c51ed182SHaiyang Zhang * NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE msg (ie sent 172c51ed182SHaiyang Zhang * NVSP_MSG1_TYPE_SEND_SEND_BUF msg) therefore, we need 173c25aaf81SKY Srinivasan * to send a revoke msg here 174c25aaf81SKY Srinivasan */ 1758b532797Sstephen hemminger if (net_device->send_section_cnt) { 176c25aaf81SKY Srinivasan /* Send the revoke receive buffer */ 177c25aaf81SKY Srinivasan revoke_packet = &net_device->revoke_packet; 178c25aaf81SKY Srinivasan memset(revoke_packet, 0, sizeof(struct nvsp_message)); 179c25aaf81SKY Srinivasan 180c25aaf81SKY Srinivasan revoke_packet->hdr.msg_type = 181c25aaf81SKY Srinivasan NVSP_MSG1_TYPE_REVOKE_SEND_BUF; 182c51ed182SHaiyang Zhang revoke_packet->msg.v1_msg.revoke_send_buf.id = 183c51ed182SHaiyang Zhang NETVSC_SEND_BUFFER_ID; 184c25aaf81SKY Srinivasan 185ec966381SStephen Hemminger trace_nvsp_send(ndev, revoke_packet); 186ec966381SStephen Hemminger 1873d541ac5SVitaly Kuznetsov ret = vmbus_sendpacket(device->channel, 188c25aaf81SKY Srinivasan revoke_packet, 189c25aaf81SKY Srinivasan sizeof(struct nvsp_message), 190c25aaf81SKY Srinivasan (unsigned long)revoke_packet, 191c25aaf81SKY Srinivasan VM_PKT_DATA_INBAND, 0); 19273e64fa4SK. Y. Srinivasan 19373e64fa4SK. Y. Srinivasan /* If the failure is because the channel is rescinded; 19473e64fa4SK. Y. Srinivasan * ignore the failure since we cannot send on a rescinded 19573e64fa4SK. Y. Srinivasan * channel. This would allow us to properly cleanup 19673e64fa4SK. Y. Srinivasan * even when the channel is rescinded. 19773e64fa4SK. Y. Srinivasan */ 19873e64fa4SK. Y. Srinivasan if (device->channel->rescind) 19973e64fa4SK. Y. Srinivasan ret = 0; 20073e64fa4SK. Y. Srinivasan 201c25aaf81SKY Srinivasan /* If we failed here, we might as well return and 202c25aaf81SKY Srinivasan * have a leak rather than continue and a bugchk 203c25aaf81SKY Srinivasan */ 204c25aaf81SKY Srinivasan if (ret != 0) { 205c25aaf81SKY Srinivasan netdev_err(ndev, "unable to send " 206c25aaf81SKY Srinivasan "revoke send buffer to netvsp\n"); 2077a2a0a84SStephen Hemminger return; 208c25aaf81SKY Srinivasan } 2098b532797Sstephen hemminger net_device->send_section_cnt = 0; 210c25aaf81SKY Srinivasan } 2110cf73780SVitaly Kuznetsov } 2120cf73780SVitaly Kuznetsov 2137992894cSMohammed Gamal static void netvsc_teardown_recv_gpadl(struct hv_device *device, 2143f076effSMohammed Gamal struct netvsc_device *net_device, 2153f076effSMohammed Gamal struct net_device *ndev) 2160cf73780SVitaly Kuznetsov { 2170cf73780SVitaly Kuznetsov int ret; 2180cf73780SVitaly Kuznetsov 2190cf73780SVitaly Kuznetsov if (net_device->recv_buf_gpadl_handle) { 2200cf73780SVitaly Kuznetsov ret = vmbus_teardown_gpadl(device->channel, 2210cf73780SVitaly Kuznetsov net_device->recv_buf_gpadl_handle); 2220cf73780SVitaly Kuznetsov 2230cf73780SVitaly Kuznetsov /* If we failed here, we might as well return and have a leak 2240cf73780SVitaly Kuznetsov * rather than continue and a bugchk 2250cf73780SVitaly Kuznetsov */ 2260cf73780SVitaly Kuznetsov if (ret != 0) { 2270cf73780SVitaly Kuznetsov netdev_err(ndev, 2280cf73780SVitaly Kuznetsov "unable to teardown receive buffer's gpadl\n"); 2290cf73780SVitaly Kuznetsov return; 2300cf73780SVitaly Kuznetsov } 2310cf73780SVitaly Kuznetsov net_device->recv_buf_gpadl_handle = 0; 2320cf73780SVitaly Kuznetsov } 2337992894cSMohammed Gamal } 2347992894cSMohammed Gamal 2357992894cSMohammed Gamal static void netvsc_teardown_send_gpadl(struct hv_device *device, 2363f076effSMohammed Gamal struct netvsc_device *net_device, 2373f076effSMohammed Gamal struct net_device *ndev) 2387992894cSMohammed Gamal { 2397992894cSMohammed Gamal int ret; 2400cf73780SVitaly Kuznetsov 241c25aaf81SKY Srinivasan if (net_device->send_buf_gpadl_handle) { 2423d541ac5SVitaly Kuznetsov ret = vmbus_teardown_gpadl(device->channel, 243c25aaf81SKY Srinivasan net_device->send_buf_gpadl_handle); 244c25aaf81SKY Srinivasan 245c25aaf81SKY Srinivasan /* If we failed here, we might as well return and have a leak 246c25aaf81SKY Srinivasan * rather than continue and a bugchk 247c25aaf81SKY Srinivasan */ 248c25aaf81SKY Srinivasan if (ret != 0) { 249c25aaf81SKY Srinivasan netdev_err(ndev, 250c25aaf81SKY Srinivasan "unable to teardown send buffer's gpadl\n"); 2517a2a0a84SStephen Hemminger return; 252c25aaf81SKY Srinivasan } 2532f18423dSDave Jones net_device->send_buf_gpadl_handle = 0; 254c25aaf81SKY Srinivasan } 25595fa0405SHaiyang Zhang } 25695fa0405SHaiyang Zhang 2577426b1a5Sstephen hemminger int netvsc_alloc_recv_comp_ring(struct netvsc_device *net_device, u32 q_idx) 2587426b1a5Sstephen hemminger { 2597426b1a5Sstephen hemminger struct netvsc_channel *nvchan = &net_device->chan_table[q_idx]; 2607426b1a5Sstephen hemminger int node = cpu_to_node(nvchan->channel->target_cpu); 2617426b1a5Sstephen hemminger size_t size; 2627426b1a5Sstephen hemminger 2637426b1a5Sstephen hemminger size = net_device->recv_completion_cnt * sizeof(struct recv_comp_data); 2647426b1a5Sstephen hemminger nvchan->mrc.slots = vzalloc_node(size, node); 2657426b1a5Sstephen hemminger if (!nvchan->mrc.slots) 2667426b1a5Sstephen hemminger nvchan->mrc.slots = vzalloc(size); 2677426b1a5Sstephen hemminger 2687426b1a5Sstephen hemminger return nvchan->mrc.slots ? 0 : -ENOMEM; 2697426b1a5Sstephen hemminger } 2707426b1a5Sstephen hemminger 27195790837Sstephen hemminger static int netvsc_init_buf(struct hv_device *device, 2728b532797Sstephen hemminger struct netvsc_device *net_device, 2738b532797Sstephen hemminger const struct netvsc_device_info *device_info) 27495fa0405SHaiyang Zhang { 2757426b1a5Sstephen hemminger struct nvsp_1_message_send_receive_buffer_complete *resp; 27695833370Sstephen hemminger struct net_device *ndev = hv_get_drvdata(device); 27795833370Sstephen hemminger struct nvsp_message *init_packet; 2788b532797Sstephen hemminger unsigned int buf_size; 279fdfb70d2Sstephen hemminger size_t map_words; 28095833370Sstephen hemminger int ret = 0; 28195fa0405SHaiyang Zhang 2828b532797Sstephen hemminger /* Get receive buffer area. */ 2830ab09befSAlex Ng buf_size = device_info->recv_sections * device_info->recv_section_size; 2848b532797Sstephen hemminger buf_size = roundup(buf_size, PAGE_SIZE); 2858b532797Sstephen hemminger 28611b2b653SHaiyang Zhang /* Legacy hosts only allow smaller receive buffer */ 28711b2b653SHaiyang Zhang if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_2) 28811b2b653SHaiyang Zhang buf_size = min_t(unsigned int, buf_size, 28911b2b653SHaiyang Zhang NETVSC_RECEIVE_BUFFER_SIZE_LEGACY); 29011b2b653SHaiyang Zhang 2918b532797Sstephen hemminger net_device->recv_buf = vzalloc(buf_size); 29295fa0405SHaiyang Zhang if (!net_device->recv_buf) { 2938b532797Sstephen hemminger netdev_err(ndev, 2948b532797Sstephen hemminger "unable to allocate receive buffer of size %u\n", 2958b532797Sstephen hemminger buf_size); 29695fa0405SHaiyang Zhang ret = -ENOMEM; 29795fa0405SHaiyang Zhang goto cleanup; 29895fa0405SHaiyang Zhang } 29995fa0405SHaiyang Zhang 300c5d24bddSHaiyang Zhang net_device->recv_buf_size = buf_size; 301c5d24bddSHaiyang Zhang 30295fa0405SHaiyang Zhang /* 30395fa0405SHaiyang Zhang * Establish the gpadl handle for this buffer on this 30495fa0405SHaiyang Zhang * channel. Note: This call uses the vmbus connection rather 30595fa0405SHaiyang Zhang * than the channel to establish the gpadl handle. 30695fa0405SHaiyang Zhang */ 30795fa0405SHaiyang Zhang ret = vmbus_establish_gpadl(device->channel, net_device->recv_buf, 3088b532797Sstephen hemminger buf_size, 30995fa0405SHaiyang Zhang &net_device->recv_buf_gpadl_handle); 31095fa0405SHaiyang Zhang if (ret != 0) { 31195fa0405SHaiyang Zhang netdev_err(ndev, 31295fa0405SHaiyang Zhang "unable to establish receive buffer's gpadl\n"); 31395fa0405SHaiyang Zhang goto cleanup; 31495fa0405SHaiyang Zhang } 31595fa0405SHaiyang Zhang 31695fa0405SHaiyang Zhang /* Notify the NetVsp of the gpadl handle */ 31795fa0405SHaiyang Zhang init_packet = &net_device->channel_init_pkt; 31895fa0405SHaiyang Zhang memset(init_packet, 0, sizeof(struct nvsp_message)); 31995fa0405SHaiyang Zhang init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_RECV_BUF; 32095fa0405SHaiyang Zhang init_packet->msg.v1_msg.send_recv_buf. 32195fa0405SHaiyang Zhang gpadl_handle = net_device->recv_buf_gpadl_handle; 32295fa0405SHaiyang Zhang init_packet->msg.v1_msg. 32395fa0405SHaiyang Zhang send_recv_buf.id = NETVSC_RECEIVE_BUFFER_ID; 32495fa0405SHaiyang Zhang 325ec966381SStephen Hemminger trace_nvsp_send(ndev, init_packet); 326ec966381SStephen Hemminger 32795fa0405SHaiyang Zhang /* Send the gpadl notification request */ 32895fa0405SHaiyang Zhang ret = vmbus_sendpacket(device->channel, init_packet, 32995fa0405SHaiyang Zhang sizeof(struct nvsp_message), 33095fa0405SHaiyang Zhang (unsigned long)init_packet, 33195fa0405SHaiyang Zhang VM_PKT_DATA_INBAND, 33295fa0405SHaiyang Zhang VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 33395fa0405SHaiyang Zhang if (ret != 0) { 33495fa0405SHaiyang Zhang netdev_err(ndev, 33595fa0405SHaiyang Zhang "unable to send receive buffer's gpadl to netvsp\n"); 33695fa0405SHaiyang Zhang goto cleanup; 33795fa0405SHaiyang Zhang } 33895fa0405SHaiyang Zhang 3395362855aSVitaly Kuznetsov wait_for_completion(&net_device->channel_init_wait); 34095fa0405SHaiyang Zhang 34195fa0405SHaiyang Zhang /* Check the response */ 3427426b1a5Sstephen hemminger resp = &init_packet->msg.v1_msg.send_recv_buf_complete; 3437426b1a5Sstephen hemminger if (resp->status != NVSP_STAT_SUCCESS) { 3447426b1a5Sstephen hemminger netdev_err(ndev, 3457426b1a5Sstephen hemminger "Unable to complete receive buffer initialization with NetVsp - status %d\n", 3467426b1a5Sstephen hemminger resp->status); 34795fa0405SHaiyang Zhang ret = -EINVAL; 34895fa0405SHaiyang Zhang goto cleanup; 34995fa0405SHaiyang Zhang } 35095fa0405SHaiyang Zhang 35195fa0405SHaiyang Zhang /* Parse the response */ 3527426b1a5Sstephen hemminger netdev_dbg(ndev, "Receive sections: %u sub_allocs: size %u count: %u\n", 3537426b1a5Sstephen hemminger resp->num_sections, resp->sections[0].sub_alloc_size, 3547426b1a5Sstephen hemminger resp->sections[0].num_sub_allocs); 35595fa0405SHaiyang Zhang 3568b532797Sstephen hemminger /* There should only be one section for the entire receive buffer */ 3578b532797Sstephen hemminger if (resp->num_sections != 1 || resp->sections[0].offset != 0) { 35895fa0405SHaiyang Zhang ret = -EINVAL; 35995fa0405SHaiyang Zhang goto cleanup; 36095fa0405SHaiyang Zhang } 36195fa0405SHaiyang Zhang 3628b532797Sstephen hemminger net_device->recv_section_size = resp->sections[0].sub_alloc_size; 3638b532797Sstephen hemminger net_device->recv_section_cnt = resp->sections[0].num_sub_allocs; 3648b532797Sstephen hemminger 3657426b1a5Sstephen hemminger /* Setup receive completion ring */ 3667426b1a5Sstephen hemminger net_device->recv_completion_cnt 3678b532797Sstephen hemminger = round_up(net_device->recv_section_cnt + 1, 3687426b1a5Sstephen hemminger PAGE_SIZE / sizeof(u64)); 3697426b1a5Sstephen hemminger ret = netvsc_alloc_recv_comp_ring(net_device, 0); 3707426b1a5Sstephen hemminger if (ret) 3717426b1a5Sstephen hemminger goto cleanup; 3727426b1a5Sstephen hemminger 3737426b1a5Sstephen hemminger /* Now setup the send buffer. */ 3740ab09befSAlex Ng buf_size = device_info->send_sections * device_info->send_section_size; 3758b532797Sstephen hemminger buf_size = round_up(buf_size, PAGE_SIZE); 3768b532797Sstephen hemminger 3778b532797Sstephen hemminger net_device->send_buf = vzalloc(buf_size); 378c25aaf81SKY Srinivasan if (!net_device->send_buf) { 3798b532797Sstephen hemminger netdev_err(ndev, "unable to allocate send buffer of size %u\n", 3808b532797Sstephen hemminger buf_size); 381c25aaf81SKY Srinivasan ret = -ENOMEM; 382c25aaf81SKY Srinivasan goto cleanup; 383c25aaf81SKY Srinivasan } 384c25aaf81SKY Srinivasan 385c25aaf81SKY Srinivasan /* Establish the gpadl handle for this buffer on this 386c25aaf81SKY Srinivasan * channel. Note: This call uses the vmbus connection rather 387c25aaf81SKY Srinivasan * than the channel to establish the gpadl handle. 388c25aaf81SKY Srinivasan */ 389c25aaf81SKY Srinivasan ret = vmbus_establish_gpadl(device->channel, net_device->send_buf, 3908b532797Sstephen hemminger buf_size, 391c25aaf81SKY Srinivasan &net_device->send_buf_gpadl_handle); 392c25aaf81SKY Srinivasan if (ret != 0) { 393c25aaf81SKY Srinivasan netdev_err(ndev, 394c25aaf81SKY Srinivasan "unable to establish send buffer's gpadl\n"); 395c25aaf81SKY Srinivasan goto cleanup; 396c25aaf81SKY Srinivasan } 397c25aaf81SKY Srinivasan 398c25aaf81SKY Srinivasan /* Notify the NetVsp of the gpadl handle */ 399c25aaf81SKY Srinivasan init_packet = &net_device->channel_init_pkt; 400c25aaf81SKY Srinivasan memset(init_packet, 0, sizeof(struct nvsp_message)); 401c25aaf81SKY Srinivasan init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_SEND_BUF; 402c51ed182SHaiyang Zhang init_packet->msg.v1_msg.send_send_buf.gpadl_handle = 403c25aaf81SKY Srinivasan net_device->send_buf_gpadl_handle; 404c51ed182SHaiyang Zhang init_packet->msg.v1_msg.send_send_buf.id = NETVSC_SEND_BUFFER_ID; 405c25aaf81SKY Srinivasan 406ec966381SStephen Hemminger trace_nvsp_send(ndev, init_packet); 407ec966381SStephen Hemminger 408c25aaf81SKY Srinivasan /* Send the gpadl notification request */ 409c25aaf81SKY Srinivasan ret = vmbus_sendpacket(device->channel, init_packet, 410c25aaf81SKY Srinivasan sizeof(struct nvsp_message), 411c25aaf81SKY Srinivasan (unsigned long)init_packet, 412c25aaf81SKY Srinivasan VM_PKT_DATA_INBAND, 413c25aaf81SKY Srinivasan VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 414c25aaf81SKY Srinivasan if (ret != 0) { 415c25aaf81SKY Srinivasan netdev_err(ndev, 416c25aaf81SKY Srinivasan "unable to send send buffer's gpadl to netvsp\n"); 417c25aaf81SKY Srinivasan goto cleanup; 418c25aaf81SKY Srinivasan } 419c25aaf81SKY Srinivasan 4205362855aSVitaly Kuznetsov wait_for_completion(&net_device->channel_init_wait); 421c25aaf81SKY Srinivasan 422c25aaf81SKY Srinivasan /* Check the response */ 423c25aaf81SKY Srinivasan if (init_packet->msg.v1_msg. 424c25aaf81SKY Srinivasan send_send_buf_complete.status != NVSP_STAT_SUCCESS) { 425c25aaf81SKY Srinivasan netdev_err(ndev, "Unable to complete send buffer " 426c25aaf81SKY Srinivasan "initialization with NetVsp - status %d\n", 427c25aaf81SKY Srinivasan init_packet->msg.v1_msg. 428c51ed182SHaiyang Zhang send_send_buf_complete.status); 429c25aaf81SKY Srinivasan ret = -EINVAL; 430c25aaf81SKY Srinivasan goto cleanup; 431c25aaf81SKY Srinivasan } 432c25aaf81SKY Srinivasan 433c25aaf81SKY Srinivasan /* Parse the response */ 434c25aaf81SKY Srinivasan net_device->send_section_size = init_packet->msg. 435c25aaf81SKY Srinivasan v1_msg.send_send_buf_complete.section_size; 436c25aaf81SKY Srinivasan 4378b532797Sstephen hemminger /* Section count is simply the size divided by the section size. */ 4388b532797Sstephen hemminger net_device->send_section_cnt = buf_size / net_device->send_section_size; 439c25aaf81SKY Srinivasan 44093ba2222SVitaly Kuznetsov netdev_dbg(ndev, "Send section size: %d, Section count:%d\n", 441c25aaf81SKY Srinivasan net_device->send_section_size, net_device->send_section_cnt); 442c25aaf81SKY Srinivasan 443c25aaf81SKY Srinivasan /* Setup state for managing the send buffer. */ 444fdfb70d2Sstephen hemminger map_words = DIV_ROUND_UP(net_device->send_section_cnt, BITS_PER_LONG); 445c25aaf81SKY Srinivasan 446fdfb70d2Sstephen hemminger net_device->send_section_map = kcalloc(map_words, sizeof(ulong), GFP_KERNEL); 447dd1d3f8fSWei Yongjun if (net_device->send_section_map == NULL) { 448dd1d3f8fSWei Yongjun ret = -ENOMEM; 449c25aaf81SKY Srinivasan goto cleanup; 450dd1d3f8fSWei Yongjun } 451c25aaf81SKY Srinivasan 45295fa0405SHaiyang Zhang goto exit; 45395fa0405SHaiyang Zhang 45495fa0405SHaiyang Zhang cleanup: 4553f076effSMohammed Gamal netvsc_revoke_recv_buf(device, net_device, ndev); 4563f076effSMohammed Gamal netvsc_revoke_send_buf(device, net_device, ndev); 4573f076effSMohammed Gamal netvsc_teardown_recv_gpadl(device, net_device, ndev); 4583f076effSMohammed Gamal netvsc_teardown_send_gpadl(device, net_device, ndev); 45995fa0405SHaiyang Zhang 46095fa0405SHaiyang Zhang exit: 46195fa0405SHaiyang Zhang return ret; 46295fa0405SHaiyang Zhang } 46395fa0405SHaiyang Zhang 464f157e78dSHaiyang Zhang /* Negotiate NVSP protocol version */ 465f157e78dSHaiyang Zhang static int negotiate_nvsp_ver(struct hv_device *device, 466f157e78dSHaiyang Zhang struct netvsc_device *net_device, 467f157e78dSHaiyang Zhang struct nvsp_message *init_packet, 468f157e78dSHaiyang Zhang u32 nvsp_ver) 46995fa0405SHaiyang Zhang { 4700a1275caSVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 4717390fe9cSNicholas Mc Guire int ret; 472f157e78dSHaiyang Zhang 473f157e78dSHaiyang Zhang memset(init_packet, 0, sizeof(struct nvsp_message)); 474f157e78dSHaiyang Zhang init_packet->hdr.msg_type = NVSP_MSG_TYPE_INIT; 475f157e78dSHaiyang Zhang init_packet->msg.init_msg.init.min_protocol_ver = nvsp_ver; 476f157e78dSHaiyang Zhang init_packet->msg.init_msg.init.max_protocol_ver = nvsp_ver; 477ec966381SStephen Hemminger trace_nvsp_send(ndev, init_packet); 478ec966381SStephen Hemminger 479f157e78dSHaiyang Zhang /* Send the init request */ 480f157e78dSHaiyang Zhang ret = vmbus_sendpacket(device->channel, init_packet, 481f157e78dSHaiyang Zhang sizeof(struct nvsp_message), 482f157e78dSHaiyang Zhang (unsigned long)init_packet, 483f157e78dSHaiyang Zhang VM_PKT_DATA_INBAND, 484f157e78dSHaiyang Zhang VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 485f157e78dSHaiyang Zhang 486f157e78dSHaiyang Zhang if (ret != 0) 487f157e78dSHaiyang Zhang return ret; 488f157e78dSHaiyang Zhang 4895362855aSVitaly Kuznetsov wait_for_completion(&net_device->channel_init_wait); 490f157e78dSHaiyang Zhang 491f157e78dSHaiyang Zhang if (init_packet->msg.init_msg.init_complete.status != 492f157e78dSHaiyang Zhang NVSP_STAT_SUCCESS) 493f157e78dSHaiyang Zhang return -EINVAL; 494f157e78dSHaiyang Zhang 495a1eabb01SHaiyang Zhang if (nvsp_ver == NVSP_PROTOCOL_VERSION_1) 496f157e78dSHaiyang Zhang return 0; 497f157e78dSHaiyang Zhang 49871790a27SHaiyang Zhang /* NVSPv2 or later: Send NDIS config */ 499f157e78dSHaiyang Zhang memset(init_packet, 0, sizeof(struct nvsp_message)); 500f157e78dSHaiyang Zhang init_packet->hdr.msg_type = NVSP_MSG2_TYPE_SEND_NDIS_CONFIG; 5010a1275caSVitaly Kuznetsov init_packet->msg.v2_msg.send_ndis_config.mtu = ndev->mtu + ETH_HLEN; 5021f5f3a75SHaiyang Zhang init_packet->msg.v2_msg.send_ndis_config.capability.ieee8021q = 1; 503f157e78dSHaiyang Zhang 5047f5d5af0SHaiyang Zhang if (nvsp_ver >= NVSP_PROTOCOL_VERSION_5) { 50571790a27SHaiyang Zhang init_packet->msg.v2_msg.send_ndis_config.capability.sriov = 1; 50671790a27SHaiyang Zhang 5077f5d5af0SHaiyang Zhang /* Teaming bit is needed to receive link speed updates */ 5087f5d5af0SHaiyang Zhang init_packet->msg.v2_msg.send_ndis_config.capability.teaming = 1; 5097f5d5af0SHaiyang Zhang } 5107f5d5af0SHaiyang Zhang 511ec966381SStephen Hemminger trace_nvsp_send(ndev, init_packet); 512ec966381SStephen Hemminger 513f157e78dSHaiyang Zhang ret = vmbus_sendpacket(device->channel, init_packet, 514f157e78dSHaiyang Zhang sizeof(struct nvsp_message), 515f157e78dSHaiyang Zhang (unsigned long)init_packet, 516f157e78dSHaiyang Zhang VM_PKT_DATA_INBAND, 0); 517f157e78dSHaiyang Zhang 518f157e78dSHaiyang Zhang return ret; 519f157e78dSHaiyang Zhang } 520f157e78dSHaiyang Zhang 52195790837Sstephen hemminger static int netvsc_connect_vsp(struct hv_device *device, 5228b532797Sstephen hemminger struct netvsc_device *net_device, 5238b532797Sstephen hemminger const struct netvsc_device_info *device_info) 524f157e78dSHaiyang Zhang { 525ec966381SStephen Hemminger struct net_device *ndev = hv_get_drvdata(device); 5261b17ca04SColin Ian King static const u32 ver_list[] = { 527e5a78fadSStephen Hemminger NVSP_PROTOCOL_VERSION_1, NVSP_PROTOCOL_VERSION_2, 52895790837Sstephen hemminger NVSP_PROTOCOL_VERSION_4, NVSP_PROTOCOL_VERSION_5 52995790837Sstephen hemminger }; 53095790837Sstephen hemminger struct nvsp_message *init_packet; 53195790837Sstephen hemminger int ndis_version, i, ret; 53295fa0405SHaiyang Zhang 53395fa0405SHaiyang Zhang init_packet = &net_device->channel_init_pkt; 53495fa0405SHaiyang Zhang 535f157e78dSHaiyang Zhang /* Negotiate the latest NVSP protocol supported */ 536e5a78fadSStephen Hemminger for (i = ARRAY_SIZE(ver_list) - 1; i >= 0; i--) 537f157e78dSHaiyang Zhang if (negotiate_nvsp_ver(device, net_device, init_packet, 538a1eabb01SHaiyang Zhang ver_list[i]) == 0) { 539a1eabb01SHaiyang Zhang net_device->nvsp_version = ver_list[i]; 540a1eabb01SHaiyang Zhang break; 541a1eabb01SHaiyang Zhang } 542a1eabb01SHaiyang Zhang 543a1eabb01SHaiyang Zhang if (i < 0) { 54495fa0405SHaiyang Zhang ret = -EPROTO; 54595fa0405SHaiyang Zhang goto cleanup; 54695fa0405SHaiyang Zhang } 547f157e78dSHaiyang Zhang 548f157e78dSHaiyang Zhang pr_debug("Negotiated NVSP version:%x\n", net_device->nvsp_version); 549f157e78dSHaiyang Zhang 55095fa0405SHaiyang Zhang /* Send the ndis version */ 55195fa0405SHaiyang Zhang memset(init_packet, 0, sizeof(struct nvsp_message)); 55295fa0405SHaiyang Zhang 553a1eabb01SHaiyang Zhang if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_4) 5541f73db49SKY Srinivasan ndis_version = 0x00060001; 555a1eabb01SHaiyang Zhang else 556a1eabb01SHaiyang Zhang ndis_version = 0x0006001e; 55795fa0405SHaiyang Zhang 55895fa0405SHaiyang Zhang init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_NDIS_VER; 55995fa0405SHaiyang Zhang init_packet->msg.v1_msg. 56095fa0405SHaiyang Zhang send_ndis_ver.ndis_major_ver = 56195fa0405SHaiyang Zhang (ndis_version & 0xFFFF0000) >> 16; 56295fa0405SHaiyang Zhang init_packet->msg.v1_msg. 56395fa0405SHaiyang Zhang send_ndis_ver.ndis_minor_ver = 56495fa0405SHaiyang Zhang ndis_version & 0xFFFF; 56595fa0405SHaiyang Zhang 566ec966381SStephen Hemminger trace_nvsp_send(ndev, init_packet); 567ec966381SStephen Hemminger 56895fa0405SHaiyang Zhang /* Send the init request */ 56995fa0405SHaiyang Zhang ret = vmbus_sendpacket(device->channel, init_packet, 57095fa0405SHaiyang Zhang sizeof(struct nvsp_message), 57195fa0405SHaiyang Zhang (unsigned long)init_packet, 57295fa0405SHaiyang Zhang VM_PKT_DATA_INBAND, 0); 57395fa0405SHaiyang Zhang if (ret != 0) 57495fa0405SHaiyang Zhang goto cleanup; 57595fa0405SHaiyang Zhang 57699d3016dSHaiyang Zhang 5778b532797Sstephen hemminger ret = netvsc_init_buf(device, net_device, device_info); 57895fa0405SHaiyang Zhang 57995fa0405SHaiyang Zhang cleanup: 58095fa0405SHaiyang Zhang return ret; 58195fa0405SHaiyang Zhang } 58295fa0405SHaiyang Zhang 58395fa0405SHaiyang Zhang /* 58495fa0405SHaiyang Zhang * netvsc_device_remove - Callback when the root bus device is removed 58595fa0405SHaiyang Zhang */ 586e08f3ea5SStephen Hemminger void netvsc_device_remove(struct hv_device *device) 58795fa0405SHaiyang Zhang { 5883d541ac5SVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 5893d541ac5SVitaly Kuznetsov struct net_device_context *net_device_ctx = netdev_priv(ndev); 59079e8cbe7Sstephen hemminger struct netvsc_device *net_device 59179e8cbe7Sstephen hemminger = rtnl_dereference(net_device_ctx->nvdev); 59215a863bfSstephen hemminger int i; 59395fa0405SHaiyang Zhang 594a56d99d7SMohammed Gamal /* 595a56d99d7SMohammed Gamal * Revoke receive buffer. If host is pre-Win2016 then tear down 596a56d99d7SMohammed Gamal * receive buffer GPADL. Do the same for send buffer. 597a56d99d7SMohammed Gamal */ 5983f076effSMohammed Gamal netvsc_revoke_recv_buf(device, net_device, ndev); 599a56d99d7SMohammed Gamal if (vmbus_proto_version < VERSION_WIN10) 6003f076effSMohammed Gamal netvsc_teardown_recv_gpadl(device, net_device, ndev); 601a56d99d7SMohammed Gamal 6023f076effSMohammed Gamal netvsc_revoke_send_buf(device, net_device, ndev); 603a56d99d7SMohammed Gamal if (vmbus_proto_version < VERSION_WIN10) 6043f076effSMohammed Gamal netvsc_teardown_send_gpadl(device, net_device, ndev); 60595fa0405SHaiyang Zhang 606545a8e79Sstephen hemminger RCU_INIT_POINTER(net_device_ctx->nvdev, NULL); 60795fa0405SHaiyang Zhang 6088348e046SStephen Hemminger /* And disassociate NAPI context from device */ 6098348e046SStephen Hemminger for (i = 0; i < net_device->num_chn; i++) 6108348e046SStephen Hemminger netif_napi_del(&net_device->chan_table[i].napi); 6118348e046SStephen Hemminger 61295fa0405SHaiyang Zhang /* 61395fa0405SHaiyang Zhang * At this point, no one should be accessing net_device 61495fa0405SHaiyang Zhang * except in here 61595fa0405SHaiyang Zhang */ 61693ba2222SVitaly Kuznetsov netdev_dbg(ndev, "net device safe to remove\n"); 61795fa0405SHaiyang Zhang 61895fa0405SHaiyang Zhang /* Now, we can close the channel safely */ 61995fa0405SHaiyang Zhang vmbus_close(device->channel); 62095fa0405SHaiyang Zhang 621a56d99d7SMohammed Gamal /* 622a56d99d7SMohammed Gamal * If host is Win2016 or higher then we do the GPADL tear down 623a56d99d7SMohammed Gamal * here after VMBus is closed. 624a56d99d7SMohammed Gamal */ 6257992894cSMohammed Gamal if (vmbus_proto_version >= VERSION_WIN10) { 6263f076effSMohammed Gamal netvsc_teardown_recv_gpadl(device, net_device, ndev); 6273f076effSMohammed Gamal netvsc_teardown_send_gpadl(device, net_device, ndev); 6287992894cSMohammed Gamal } 6290cf73780SVitaly Kuznetsov 63095fa0405SHaiyang Zhang /* Release all resources */ 631545a8e79Sstephen hemminger free_netvsc_device_rcu(net_device); 63295fa0405SHaiyang Zhang } 63395fa0405SHaiyang Zhang 63433be96e4SHaiyang Zhang #define RING_AVAIL_PERCENT_HIWATER 20 63533be96e4SHaiyang Zhang #define RING_AVAIL_PERCENT_LOWATER 10 63633be96e4SHaiyang Zhang 63733be96e4SHaiyang Zhang /* 63833be96e4SHaiyang Zhang * Get the percentage of available bytes to write in the ring. 63933be96e4SHaiyang Zhang * The return value is in range from 0 to 100. 64033be96e4SHaiyang Zhang */ 641a7f99d0fSStephen Hemminger static u32 hv_ringbuf_avail_percent(const struct hv_ring_buffer_info *ring_info) 64233be96e4SHaiyang Zhang { 643a7f99d0fSStephen Hemminger u32 avail_write = hv_get_bytes_to_write(ring_info); 64433be96e4SHaiyang Zhang 645a7f99d0fSStephen Hemminger return reciprocal_divide(avail_write * 100, netvsc_ring_reciprocal); 64633be96e4SHaiyang Zhang } 64733be96e4SHaiyang Zhang 648c25aaf81SKY Srinivasan static inline void netvsc_free_send_slot(struct netvsc_device *net_device, 649c25aaf81SKY Srinivasan u32 index) 650c25aaf81SKY Srinivasan { 651c25aaf81SKY Srinivasan sync_change_bit(index, net_device->send_section_map); 652c25aaf81SKY Srinivasan } 653c25aaf81SKY Srinivasan 654bc304dd3SStephen Hemminger static void netvsc_send_tx_complete(struct netvsc_device *net_device, 65525b85ee8SKY Srinivasan struct vmbus_channel *incoming_channel, 65697c1723aSKY Srinivasan struct hv_device *device, 657f9645430Sstephen hemminger const struct vmpacket_descriptor *desc, 658f9645430Sstephen hemminger int budget) 65995fa0405SHaiyang Zhang { 66050698d80Sstephen hemminger struct sk_buff *skb = (struct sk_buff *)(unsigned long)desc->trans_id; 6613d541ac5SVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 66209af87d1SSimon Xiao struct net_device_context *ndev_ctx = netdev_priv(ndev); 663bc304dd3SStephen Hemminger struct vmbus_channel *channel = device->channel; 6645b54dac8SHaiyang Zhang u16 q_idx = 0; 6655b54dac8SHaiyang Zhang int queue_sends; 66633be96e4SHaiyang Zhang 66795fa0405SHaiyang Zhang /* Notify the layer above us */ 668bc304dd3SStephen Hemminger if (likely(skb)) { 669793e3955Sstephen hemminger const struct hv_netvsc_packet *packet 670bc304dd3SStephen Hemminger = (struct hv_netvsc_packet *)skb->cb; 671793e3955Sstephen hemminger u32 send_index = packet->send_buf_index; 672793e3955Sstephen hemminger struct netvsc_stats *tx_stats; 673bc304dd3SStephen Hemminger 674c25aaf81SKY Srinivasan if (send_index != NETVSC_INVALID_INDEX) 675c25aaf81SKY Srinivasan netvsc_free_send_slot(net_device, send_index); 676793e3955Sstephen hemminger q_idx = packet->q_idx; 67725b85ee8SKY Srinivasan channel = incoming_channel; 678bc304dd3SStephen Hemminger 6796c80f3fcSSimon Xiao tx_stats = &net_device->chan_table[q_idx].tx_stats; 680793e3955Sstephen hemminger 681793e3955Sstephen hemminger u64_stats_update_begin(&tx_stats->syncp); 682793e3955Sstephen hemminger tx_stats->packets += packet->total_packets; 683793e3955Sstephen hemminger tx_stats->bytes += packet->total_bytes; 684793e3955Sstephen hemminger u64_stats_update_end(&tx_stats->syncp); 685793e3955Sstephen hemminger 686f9645430Sstephen hemminger napi_consume_skb(skb, budget); 6875b54dac8SHaiyang Zhang } 68895fa0405SHaiyang Zhang 689b8b835a8Sstephen hemminger queue_sends = 690b8b835a8Sstephen hemminger atomic_dec_return(&net_device->chan_table[q_idx].queue_sends); 6911d06825bSHaiyang Zhang 6927b2ee50cSStephen Hemminger if (unlikely(net_device->destroy)) { 6937b2ee50cSStephen Hemminger if (queue_sends == 0) 694dc5cd894SHaiyang Zhang wake_up(&net_device->wait_drain); 6957b2ee50cSStephen Hemminger } else { 6967b2ee50cSStephen Hemminger struct netdev_queue *txq = netdev_get_tx_queue(ndev, q_idx); 697dc5cd894SHaiyang Zhang 6987b2ee50cSStephen Hemminger if (netif_tx_queue_stopped(txq) && 699bc304dd3SStephen Hemminger (hv_ringbuf_avail_percent(&channel->outbound) > RING_AVAIL_PERCENT_HIWATER || 70009af87d1SSimon Xiao queue_sends < 1)) { 7017b2ee50cSStephen Hemminger netif_tx_wake_queue(txq); 70209af87d1SSimon Xiao ndev_ctx->eth_stats.wake_queue++; 70309af87d1SSimon Xiao } 704bc304dd3SStephen Hemminger } 7057b2ee50cSStephen Hemminger } 706bc304dd3SStephen Hemminger 707bc304dd3SStephen Hemminger static void netvsc_send_completion(struct netvsc_device *net_device, 708bc304dd3SStephen Hemminger struct vmbus_channel *incoming_channel, 709bc304dd3SStephen Hemminger struct hv_device *device, 710f9645430Sstephen hemminger const struct vmpacket_descriptor *desc, 711f9645430Sstephen hemminger int budget) 712bc304dd3SStephen Hemminger { 713f3dd3f47Sstephen hemminger struct nvsp_message *nvsp_packet = hv_pkt_data(desc); 714bc304dd3SStephen Hemminger struct net_device *ndev = hv_get_drvdata(device); 715bc304dd3SStephen Hemminger 716bc304dd3SStephen Hemminger switch (nvsp_packet->hdr.msg_type) { 717bc304dd3SStephen Hemminger case NVSP_MSG_TYPE_INIT_COMPLETE: 718bc304dd3SStephen Hemminger case NVSP_MSG1_TYPE_SEND_RECV_BUF_COMPLETE: 719bc304dd3SStephen Hemminger case NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE: 720bc304dd3SStephen Hemminger case NVSP_MSG5_TYPE_SUBCHANNEL: 721bc304dd3SStephen Hemminger /* Copy the response back */ 722bc304dd3SStephen Hemminger memcpy(&net_device->channel_init_pkt, nvsp_packet, 723bc304dd3SStephen Hemminger sizeof(struct nvsp_message)); 724bc304dd3SStephen Hemminger complete(&net_device->channel_init_wait); 725bc304dd3SStephen Hemminger break; 726bc304dd3SStephen Hemminger 727bc304dd3SStephen Hemminger case NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE: 728bc304dd3SStephen Hemminger netvsc_send_tx_complete(net_device, incoming_channel, 729f9645430Sstephen hemminger device, desc, budget); 730bc304dd3SStephen Hemminger break; 731bc304dd3SStephen Hemminger 732bc304dd3SStephen Hemminger default: 733bc304dd3SStephen Hemminger netdev_err(ndev, 734bc304dd3SStephen Hemminger "Unknown send completion type %d received!!\n", 735bc304dd3SStephen Hemminger nvsp_packet->hdr.msg_type); 73695fa0405SHaiyang Zhang } 73795fa0405SHaiyang Zhang } 73895fa0405SHaiyang Zhang 739c25aaf81SKY Srinivasan static u32 netvsc_get_next_send_section(struct netvsc_device *net_device) 740c25aaf81SKY Srinivasan { 741b58a1858Sstephen hemminger unsigned long *map_addr = net_device->send_section_map; 742b58a1858Sstephen hemminger unsigned int i; 743c25aaf81SKY Srinivasan 744fdfb70d2Sstephen hemminger for_each_clear_bit(i, map_addr, net_device->send_section_cnt) { 745b58a1858Sstephen hemminger if (sync_test_and_set_bit(i, map_addr) == 0) 746b58a1858Sstephen hemminger return i; 747c25aaf81SKY Srinivasan } 748b58a1858Sstephen hemminger 749b58a1858Sstephen hemminger return NETVSC_INVALID_INDEX; 750c25aaf81SKY Srinivasan } 751c25aaf81SKY Srinivasan 75226a11262SStephen Hemminger static void netvsc_copy_to_send_buf(struct netvsc_device *net_device, 753c25aaf81SKY Srinivasan unsigned int section_index, 7547c3877f2SHaiyang Zhang u32 pend_size, 75524476760SKY Srinivasan struct hv_netvsc_packet *packet, 756a9f2e2d6SKY Srinivasan struct rndis_message *rndis_msg, 75702b6de01Sstephen hemminger struct hv_page_buffer *pb, 758cfd8afd9SStephen Hemminger bool xmit_more) 759c25aaf81SKY Srinivasan { 760c25aaf81SKY Srinivasan char *start = net_device->send_buf; 7617c3877f2SHaiyang Zhang char *dest = start + (section_index * net_device->send_section_size) 7627c3877f2SHaiyang Zhang + pend_size; 763c25aaf81SKY Srinivasan int i; 7647c3877f2SHaiyang Zhang u32 padding = 0; 765aa0a34beSHaiyang Zhang u32 page_count = packet->cp_partial ? packet->rmsg_pgcnt : 766aa0a34beSHaiyang Zhang packet->page_buf_cnt; 767b85e06f7SStephen Hemminger u32 remain; 7687c3877f2SHaiyang Zhang 7697c3877f2SHaiyang Zhang /* Add padding */ 770b85e06f7SStephen Hemminger remain = packet->total_data_buflen & (net_device->pkt_align - 1); 771cfd8afd9SStephen Hemminger if (xmit_more && remain) { 7727c3877f2SHaiyang Zhang padding = net_device->pkt_align - remain; 77324476760SKY Srinivasan rndis_msg->msg_len += padding; 7747c3877f2SHaiyang Zhang packet->total_data_buflen += padding; 7757c3877f2SHaiyang Zhang } 776c25aaf81SKY Srinivasan 777aa0a34beSHaiyang Zhang for (i = 0; i < page_count; i++) { 77802b6de01Sstephen hemminger char *src = phys_to_virt(pb[i].pfn << PAGE_SHIFT); 77902b6de01Sstephen hemminger u32 offset = pb[i].offset; 78002b6de01Sstephen hemminger u32 len = pb[i].len; 781c25aaf81SKY Srinivasan 782c25aaf81SKY Srinivasan memcpy(dest, (src + offset), len); 783c25aaf81SKY Srinivasan dest += len; 784c25aaf81SKY Srinivasan } 7857c3877f2SHaiyang Zhang 78626a11262SStephen Hemminger if (padding) 7877c3877f2SHaiyang Zhang memset(dest, 0, padding); 788c25aaf81SKY Srinivasan } 789c25aaf81SKY Srinivasan 7903a8963acSStephen Hemminger static inline int netvsc_send_pkt( 7910a1275caSVitaly Kuznetsov struct hv_device *device, 7927c3877f2SHaiyang Zhang struct hv_netvsc_packet *packet, 793a9f2e2d6SKY Srinivasan struct netvsc_device *net_device, 79402b6de01Sstephen hemminger struct hv_page_buffer *pb, 7953a3d9a0aSKY Srinivasan struct sk_buff *skb) 79695fa0405SHaiyang Zhang { 7977c3877f2SHaiyang Zhang struct nvsp_message nvmsg; 798ec966381SStephen Hemminger struct nvsp_1_message_send_rndis_packet *rpkt = 799956a25c9SJoe Perches &nvmsg.msg.v1_msg.send_rndis_pkt; 800956a25c9SJoe Perches struct netvsc_channel * const nvchan = 801956a25c9SJoe Perches &net_device->chan_table[packet->q_idx]; 802b8b835a8Sstephen hemminger struct vmbus_channel *out_channel = nvchan->channel; 8030a1275caSVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 80409af87d1SSimon Xiao struct net_device_context *ndev_ctx = netdev_priv(ndev); 805b8b835a8Sstephen hemminger struct netdev_queue *txq = netdev_get_tx_queue(ndev, packet->q_idx); 8067c3877f2SHaiyang Zhang u64 req_id; 8077c3877f2SHaiyang Zhang int ret; 80882fa3c77SKY Srinivasan u32 ring_avail = hv_ringbuf_avail_percent(&out_channel->outbound); 809c25aaf81SKY Srinivasan 8107c3877f2SHaiyang Zhang nvmsg.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT; 811956a25c9SJoe Perches if (skb) 812956a25c9SJoe Perches rpkt->channel_type = 0; /* 0 is RMC_DATA */ 8137c3877f2SHaiyang Zhang else 814956a25c9SJoe Perches rpkt->channel_type = 1; /* 1 is RMC_CONTROL */ 815956a25c9SJoe Perches 816956a25c9SJoe Perches rpkt->send_buf_section_index = packet->send_buf_index; 817956a25c9SJoe Perches if (packet->send_buf_index == NETVSC_INVALID_INDEX) 818956a25c9SJoe Perches rpkt->send_buf_section_size = 0; 819956a25c9SJoe Perches else 820956a25c9SJoe Perches rpkt->send_buf_section_size = packet->total_data_buflen; 82195fa0405SHaiyang Zhang 8223a3d9a0aSKY Srinivasan req_id = (ulong)skb; 823f1ea3cd7SHaiyang Zhang 824c3582a2cSHaiyang Zhang if (out_channel->rescind) 825c3582a2cSHaiyang Zhang return -ENODEV; 826c3582a2cSHaiyang Zhang 827ec966381SStephen Hemminger trace_nvsp_send_pkt(ndev, out_channel, rpkt); 828ec966381SStephen Hemminger 82995fa0405SHaiyang Zhang if (packet->page_buf_cnt) { 83002b6de01Sstephen hemminger if (packet->cp_partial) 83102b6de01Sstephen hemminger pb += packet->rmsg_pgcnt; 83202b6de01Sstephen hemminger 8335a668d8cSstephen hemminger ret = vmbus_sendpacket_pagebuffer(out_channel, 83402b6de01Sstephen hemminger pb, packet->page_buf_cnt, 8355a668d8cSstephen hemminger &nvmsg, sizeof(nvmsg), 8365a668d8cSstephen hemminger req_id); 83795fa0405SHaiyang Zhang } else { 8385dd0fb9bSstephen hemminger ret = vmbus_sendpacket(out_channel, 8395dd0fb9bSstephen hemminger &nvmsg, sizeof(nvmsg), 8405dd0fb9bSstephen hemminger req_id, VM_PKT_DATA_INBAND, 8413454323cSStephen Hemminger VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 84295fa0405SHaiyang Zhang } 84395fa0405SHaiyang Zhang 8441d06825bSHaiyang Zhang if (ret == 0) { 845b8b835a8Sstephen hemminger atomic_inc_return(&nvchan->queue_sends); 8465b54dac8SHaiyang Zhang 84709af87d1SSimon Xiao if (ring_avail < RING_AVAIL_PERCENT_LOWATER) { 848b8b835a8Sstephen hemminger netif_tx_stop_queue(txq); 84909af87d1SSimon Xiao ndev_ctx->eth_stats.stop_queue++; 85009af87d1SSimon Xiao } 8511d06825bSHaiyang Zhang } else if (ret == -EAGAIN) { 852b8b835a8Sstephen hemminger netif_tx_stop_queue(txq); 85309af87d1SSimon Xiao ndev_ctx->eth_stats.stop_queue++; 854b8b835a8Sstephen hemminger if (atomic_read(&nvchan->queue_sends) < 1) { 855b8b835a8Sstephen hemminger netif_tx_wake_queue(txq); 85609af87d1SSimon Xiao ndev_ctx->eth_stats.wake_queue++; 85733be96e4SHaiyang Zhang ret = -ENOSPC; 85833be96e4SHaiyang Zhang } 8591d06825bSHaiyang Zhang } else { 8604a2176c6Sstephen hemminger netdev_err(ndev, 8614a2176c6Sstephen hemminger "Unable to send packet pages %u len %u, ret %d\n", 8624a2176c6Sstephen hemminger packet->page_buf_cnt, packet->total_data_buflen, 8634a2176c6Sstephen hemminger ret); 8641d06825bSHaiyang Zhang } 86595fa0405SHaiyang Zhang 8667c3877f2SHaiyang Zhang return ret; 8677c3877f2SHaiyang Zhang } 8687c3877f2SHaiyang Zhang 869c85e4924SHaiyang Zhang /* Move packet out of multi send data (msd), and clear msd */ 870c85e4924SHaiyang Zhang static inline void move_pkt_msd(struct hv_netvsc_packet **msd_send, 871c85e4924SHaiyang Zhang struct sk_buff **msd_skb, 872c85e4924SHaiyang Zhang struct multi_send_data *msdp) 873c85e4924SHaiyang Zhang { 874c85e4924SHaiyang Zhang *msd_skb = msdp->skb; 875c85e4924SHaiyang Zhang *msd_send = msdp->pkt; 876c85e4924SHaiyang Zhang msdp->skb = NULL; 877c85e4924SHaiyang Zhang msdp->pkt = NULL; 878c85e4924SHaiyang Zhang msdp->count = 0; 879c85e4924SHaiyang Zhang } 880c85e4924SHaiyang Zhang 8812a926f79Sstephen hemminger /* RCU already held by caller */ 882cfd8afd9SStephen Hemminger int netvsc_send(struct net_device *ndev, 88324476760SKY Srinivasan struct hv_netvsc_packet *packet, 884a9f2e2d6SKY Srinivasan struct rndis_message *rndis_msg, 88502b6de01Sstephen hemminger struct hv_page_buffer *pb, 8863a3d9a0aSKY Srinivasan struct sk_buff *skb) 8877c3877f2SHaiyang Zhang { 888cfd8afd9SStephen Hemminger struct net_device_context *ndev_ctx = netdev_priv(ndev); 8893962981fSstephen hemminger struct netvsc_device *net_device 890867047c4Sstephen hemminger = rcu_dereference_bh(ndev_ctx->nvdev); 8912a926f79Sstephen hemminger struct hv_device *device = ndev_ctx->device_ctx; 8926c4c137eSStephen Hemminger int ret = 0; 893b8b835a8Sstephen hemminger struct netvsc_channel *nvchan; 8947c3877f2SHaiyang Zhang u32 pktlen = packet->total_data_buflen, msd_len = 0; 8957c3877f2SHaiyang Zhang unsigned int section_index = NETVSC_INVALID_INDEX; 8967c3877f2SHaiyang Zhang struct multi_send_data *msdp; 8977c3877f2SHaiyang Zhang struct hv_netvsc_packet *msd_send = NULL, *cur_send = NULL; 898c85e4924SHaiyang Zhang struct sk_buff *msd_skb = NULL; 899cfd8afd9SStephen Hemminger bool try_batch, xmit_more; 9007c3877f2SHaiyang Zhang 901592b4fe8Sstephen hemminger /* If device is rescinded, return error and packet will get dropped. */ 9022a926f79Sstephen hemminger if (unlikely(!net_device || net_device->destroy)) 9037c3877f2SHaiyang Zhang return -ENODEV; 9047c3877f2SHaiyang Zhang 905b8b835a8Sstephen hemminger nvchan = &net_device->chan_table[packet->q_idx]; 9067c3877f2SHaiyang Zhang packet->send_buf_index = NETVSC_INVALID_INDEX; 907aa0a34beSHaiyang Zhang packet->cp_partial = false; 9087c3877f2SHaiyang Zhang 909cf8190e4SHaiyang Zhang /* Send control message directly without accessing msd (Multi-Send 910cf8190e4SHaiyang Zhang * Data) field which may be changed during data packet processing. 911cf8190e4SHaiyang Zhang */ 91212f69661SStephen Hemminger if (!skb) 91312f69661SStephen Hemminger return netvsc_send_pkt(device, packet, net_device, pb, skb); 914cf8190e4SHaiyang Zhang 9157c3877f2SHaiyang Zhang /* batch packets in send buffer if possible */ 916b8b835a8Sstephen hemminger msdp = &nvchan->msd; 9177c3877f2SHaiyang Zhang if (msdp->pkt) 9187c3877f2SHaiyang Zhang msd_len = msdp->pkt->total_data_buflen; 9197c3877f2SHaiyang Zhang 920ebc1dcf6Sstephen hemminger try_batch = msd_len > 0 && msdp->count < net_device->max_pkt; 921aa0a34beSHaiyang Zhang if (try_batch && msd_len + pktlen + net_device->pkt_align < 9227c3877f2SHaiyang Zhang net_device->send_section_size) { 9237c3877f2SHaiyang Zhang section_index = msdp->pkt->send_buf_index; 9247c3877f2SHaiyang Zhang 925aa0a34beSHaiyang Zhang } else if (try_batch && msd_len + packet->rmsg_size < 926aa0a34beSHaiyang Zhang net_device->send_section_size) { 927aa0a34beSHaiyang Zhang section_index = msdp->pkt->send_buf_index; 928aa0a34beSHaiyang Zhang packet->cp_partial = true; 929aa0a34beSHaiyang Zhang 930ebc1dcf6Sstephen hemminger } else if (pktlen + net_device->pkt_align < 9317c3877f2SHaiyang Zhang net_device->send_section_size) { 9327c3877f2SHaiyang Zhang section_index = netvsc_get_next_send_section(net_device); 933cad5c197Sstephen hemminger if (unlikely(section_index == NETVSC_INVALID_INDEX)) { 934cad5c197Sstephen hemminger ++ndev_ctx->eth_stats.tx_send_full; 935cad5c197Sstephen hemminger } else { 936c85e4924SHaiyang Zhang move_pkt_msd(&msd_send, &msd_skb, msdp); 9377c3877f2SHaiyang Zhang msd_len = 0; 9387c3877f2SHaiyang Zhang } 9397c3877f2SHaiyang Zhang } 9407c3877f2SHaiyang Zhang 941cfd8afd9SStephen Hemminger /* Keep aggregating only if stack says more data is coming 942cfd8afd9SStephen Hemminger * and not doing mixed modes send and not flow blocked 943cfd8afd9SStephen Hemminger */ 944cfd8afd9SStephen Hemminger xmit_more = skb->xmit_more && 945cfd8afd9SStephen Hemminger !packet->cp_partial && 946cfd8afd9SStephen Hemminger !netif_xmit_stopped(netdev_get_tx_queue(ndev, packet->q_idx)); 947cfd8afd9SStephen Hemminger 9487c3877f2SHaiyang Zhang if (section_index != NETVSC_INVALID_INDEX) { 9497c3877f2SHaiyang Zhang netvsc_copy_to_send_buf(net_device, 9507c3877f2SHaiyang Zhang section_index, msd_len, 951cfd8afd9SStephen Hemminger packet, rndis_msg, pb, xmit_more); 952b08cc791SKY Srinivasan 953aa0a34beSHaiyang Zhang packet->send_buf_index = section_index; 954aa0a34beSHaiyang Zhang 955aa0a34beSHaiyang Zhang if (packet->cp_partial) { 956aa0a34beSHaiyang Zhang packet->page_buf_cnt -= packet->rmsg_pgcnt; 957aa0a34beSHaiyang Zhang packet->total_data_buflen = msd_len + packet->rmsg_size; 958aa0a34beSHaiyang Zhang } else { 959aa0a34beSHaiyang Zhang packet->page_buf_cnt = 0; 960aa0a34beSHaiyang Zhang packet->total_data_buflen += msd_len; 961aa0a34beSHaiyang Zhang } 9627c3877f2SHaiyang Zhang 963793e3955Sstephen hemminger if (msdp->pkt) { 964793e3955Sstephen hemminger packet->total_packets += msdp->pkt->total_packets; 965793e3955Sstephen hemminger packet->total_bytes += msdp->pkt->total_bytes; 966793e3955Sstephen hemminger } 967793e3955Sstephen hemminger 968c85e4924SHaiyang Zhang if (msdp->skb) 96917db4bceSStephen Hemminger dev_consume_skb_any(msdp->skb); 970ee90b812SHaiyang Zhang 971cfd8afd9SStephen Hemminger if (xmit_more) { 972c85e4924SHaiyang Zhang msdp->skb = skb; 9737c3877f2SHaiyang Zhang msdp->pkt = packet; 9747c3877f2SHaiyang Zhang msdp->count++; 9757c3877f2SHaiyang Zhang } else { 9767c3877f2SHaiyang Zhang cur_send = packet; 977c85e4924SHaiyang Zhang msdp->skb = NULL; 9787c3877f2SHaiyang Zhang msdp->pkt = NULL; 9797c3877f2SHaiyang Zhang msdp->count = 0; 9807c3877f2SHaiyang Zhang } 9817c3877f2SHaiyang Zhang } else { 982c85e4924SHaiyang Zhang move_pkt_msd(&msd_send, &msd_skb, msdp); 9837c3877f2SHaiyang Zhang cur_send = packet; 9847c3877f2SHaiyang Zhang } 9857c3877f2SHaiyang Zhang 9867c3877f2SHaiyang Zhang if (msd_send) { 9876c4c137eSStephen Hemminger int m_ret = netvsc_send_pkt(device, msd_send, net_device, 9880a1275caSVitaly Kuznetsov NULL, msd_skb); 9897c3877f2SHaiyang Zhang 9907c3877f2SHaiyang Zhang if (m_ret != 0) { 9917c3877f2SHaiyang Zhang netvsc_free_send_slot(net_device, 9927c3877f2SHaiyang Zhang msd_send->send_buf_index); 993c85e4924SHaiyang Zhang dev_kfree_skb_any(msd_skb); 9947c3877f2SHaiyang Zhang } 9957c3877f2SHaiyang Zhang } 9967c3877f2SHaiyang Zhang 9977c3877f2SHaiyang Zhang if (cur_send) 9980a1275caSVitaly Kuznetsov ret = netvsc_send_pkt(device, cur_send, net_device, pb, skb); 9997c3877f2SHaiyang Zhang 10007aab5159SJerry Snitselaar if (ret != 0 && section_index != NETVSC_INVALID_INDEX) 1001d953ca4dSHaiyang Zhang netvsc_free_send_slot(net_device, section_index); 1002d953ca4dSHaiyang Zhang 100395fa0405SHaiyang Zhang return ret; 100495fa0405SHaiyang Zhang } 100595fa0405SHaiyang Zhang 10067426b1a5Sstephen hemminger /* Send pending recv completions */ 1007cad5c197Sstephen hemminger static int send_recv_completions(struct net_device *ndev, 1008cad5c197Sstephen hemminger struct netvsc_device *nvdev, 1009cad5c197Sstephen hemminger struct netvsc_channel *nvchan) 101095fa0405SHaiyang Zhang { 10117426b1a5Sstephen hemminger struct multi_recv_comp *mrc = &nvchan->mrc; 10127426b1a5Sstephen hemminger struct recv_comp_msg { 10137426b1a5Sstephen hemminger struct nvsp_message_header hdr; 10147426b1a5Sstephen hemminger u32 status; 10157426b1a5Sstephen hemminger } __packed; 10167426b1a5Sstephen hemminger struct recv_comp_msg msg = { 10177426b1a5Sstephen hemminger .hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE, 10187426b1a5Sstephen hemminger }; 101995fa0405SHaiyang Zhang int ret; 102095fa0405SHaiyang Zhang 10217426b1a5Sstephen hemminger while (mrc->first != mrc->next) { 10227426b1a5Sstephen hemminger const struct recv_comp_data *rcd 10237426b1a5Sstephen hemminger = mrc->slots + mrc->first; 102495fa0405SHaiyang Zhang 10257426b1a5Sstephen hemminger msg.status = rcd->status; 10267426b1a5Sstephen hemminger ret = vmbus_sendpacket(nvchan->channel, &msg, sizeof(msg), 10277426b1a5Sstephen hemminger rcd->tid, VM_PKT_COMP, 0); 1028cad5c197Sstephen hemminger if (unlikely(ret)) { 1029cad5c197Sstephen hemminger struct net_device_context *ndev_ctx = netdev_priv(ndev); 1030cad5c197Sstephen hemminger 1031cad5c197Sstephen hemminger ++ndev_ctx->eth_stats.rx_comp_busy; 1032c0b558e5SHaiyang Zhang return ret; 1033cad5c197Sstephen hemminger } 10347426b1a5Sstephen hemminger 10357426b1a5Sstephen hemminger if (++mrc->first == nvdev->recv_completion_cnt) 10367426b1a5Sstephen hemminger mrc->first = 0; 103795fa0405SHaiyang Zhang } 1038c0b558e5SHaiyang Zhang 10397426b1a5Sstephen hemminger /* receive completion ring has been emptied */ 10407426b1a5Sstephen hemminger if (unlikely(nvdev->destroy)) 10417426b1a5Sstephen hemminger wake_up(&nvdev->wait_drain); 10427426b1a5Sstephen hemminger 10437426b1a5Sstephen hemminger return 0; 10447426b1a5Sstephen hemminger } 10457426b1a5Sstephen hemminger 10467426b1a5Sstephen hemminger /* Count how many receive completions are outstanding */ 10477426b1a5Sstephen hemminger static void recv_comp_slot_avail(const struct netvsc_device *nvdev, 10487426b1a5Sstephen hemminger const struct multi_recv_comp *mrc, 1049c0b558e5SHaiyang Zhang u32 *filled, u32 *avail) 1050c0b558e5SHaiyang Zhang { 10517426b1a5Sstephen hemminger u32 count = nvdev->recv_completion_cnt; 1052c0b558e5SHaiyang Zhang 10537426b1a5Sstephen hemminger if (mrc->next >= mrc->first) 10547426b1a5Sstephen hemminger *filled = mrc->next - mrc->first; 10557426b1a5Sstephen hemminger else 10567426b1a5Sstephen hemminger *filled = (count - mrc->first) + mrc->next; 1057c0b558e5SHaiyang Zhang 10587426b1a5Sstephen hemminger *avail = count - *filled - 1; 105995fa0405SHaiyang Zhang } 1060c0b558e5SHaiyang Zhang 10617426b1a5Sstephen hemminger /* Add receive complete to ring to send to host. */ 10627426b1a5Sstephen hemminger static void enq_receive_complete(struct net_device *ndev, 10637426b1a5Sstephen hemminger struct netvsc_device *nvdev, u16 q_idx, 10647426b1a5Sstephen hemminger u64 tid, u32 status) 1065c0b558e5SHaiyang Zhang { 10667426b1a5Sstephen hemminger struct netvsc_channel *nvchan = &nvdev->chan_table[q_idx]; 10677426b1a5Sstephen hemminger struct multi_recv_comp *mrc = &nvchan->mrc; 10687426b1a5Sstephen hemminger struct recv_comp_data *rcd; 1069c0b558e5SHaiyang Zhang u32 filled, avail; 1070c0b558e5SHaiyang Zhang 10717426b1a5Sstephen hemminger recv_comp_slot_avail(nvdev, mrc, &filled, &avail); 1072c0b558e5SHaiyang Zhang 10737426b1a5Sstephen hemminger if (unlikely(filled > NAPI_POLL_WEIGHT)) { 1074cad5c197Sstephen hemminger send_recv_completions(ndev, nvdev, nvchan); 10757426b1a5Sstephen hemminger recv_comp_slot_avail(nvdev, mrc, &filled, &avail); 1076c0b558e5SHaiyang Zhang } 1077c0b558e5SHaiyang Zhang 10787426b1a5Sstephen hemminger if (unlikely(!avail)) { 10797426b1a5Sstephen hemminger netdev_err(ndev, "Recv_comp full buf q:%hd, tid:%llx\n", 10807426b1a5Sstephen hemminger q_idx, tid); 10817426b1a5Sstephen hemminger return; 1082c0b558e5SHaiyang Zhang } 1083c0b558e5SHaiyang Zhang 10847426b1a5Sstephen hemminger rcd = mrc->slots + mrc->next; 10857426b1a5Sstephen hemminger rcd->tid = tid; 10867426b1a5Sstephen hemminger rcd->status = status; 1087c0b558e5SHaiyang Zhang 10887426b1a5Sstephen hemminger if (++mrc->next == nvdev->recv_completion_cnt) 10897426b1a5Sstephen hemminger mrc->next = 0; 109095fa0405SHaiyang Zhang } 109195fa0405SHaiyang Zhang 109215a863bfSstephen hemminger static int netvsc_receive(struct net_device *ndev, 1093dc54a08cSstephen hemminger struct netvsc_device *net_device, 1094dc54a08cSstephen hemminger struct net_device_context *net_device_ctx, 109597c1723aSKY Srinivasan struct hv_device *device, 1096dc54a08cSstephen hemminger struct vmbus_channel *channel, 1097f3dd3f47Sstephen hemminger const struct vmpacket_descriptor *desc, 1098dc54a08cSstephen hemminger struct nvsp_message *nvsp) 109995fa0405SHaiyang Zhang { 1100f3dd3f47Sstephen hemminger const struct vmtransfer_page_packet_header *vmxferpage_packet 1101f3dd3f47Sstephen hemminger = container_of(desc, const struct vmtransfer_page_packet_header, d); 110215a863bfSstephen hemminger u16 q_idx = channel->offermsg.offer.sub_channel_index; 1103dc54a08cSstephen hemminger char *recv_buf = net_device->recv_buf; 11044baab261SHaiyang Zhang u32 status = NVSP_STAT_SUCCESS; 110545326342SHaiyang Zhang int i; 110645326342SHaiyang Zhang int count = 0; 110795fa0405SHaiyang Zhang 110895fa0405SHaiyang Zhang /* Make sure this is a valid nvsp packet */ 1109dc54a08cSstephen hemminger if (unlikely(nvsp->hdr.msg_type != NVSP_MSG1_TYPE_SEND_RNDIS_PKT)) { 1110dc54a08cSstephen hemminger netif_err(net_device_ctx, rx_err, ndev, 1111dc54a08cSstephen hemminger "Unknown nvsp packet type received %u\n", 1112dc54a08cSstephen hemminger nvsp->hdr.msg_type); 111315a863bfSstephen hemminger return 0; 111495fa0405SHaiyang Zhang } 111595fa0405SHaiyang Zhang 1116dc54a08cSstephen hemminger if (unlikely(vmxferpage_packet->xfer_pageset_id != NETVSC_RECEIVE_BUFFER_ID)) { 1117dc54a08cSstephen hemminger netif_err(net_device_ctx, rx_err, ndev, 1118dc54a08cSstephen hemminger "Invalid xfer page set id - expecting %x got %x\n", 1119dc54a08cSstephen hemminger NETVSC_RECEIVE_BUFFER_ID, 112095fa0405SHaiyang Zhang vmxferpage_packet->xfer_pageset_id); 112115a863bfSstephen hemminger return 0; 112295fa0405SHaiyang Zhang } 112395fa0405SHaiyang Zhang 11244baab261SHaiyang Zhang count = vmxferpage_packet->range_cnt; 112595fa0405SHaiyang Zhang 112695fa0405SHaiyang Zhang /* Each range represents 1 RNDIS pkt that contains 1 ethernet frame */ 11274baab261SHaiyang Zhang for (i = 0; i < count; i++) { 1128c5d24bddSHaiyang Zhang u32 offset = vmxferpage_packet->ranges[i].byte_offset; 1129dc54a08cSstephen hemminger u32 buflen = vmxferpage_packet->ranges[i].byte_count; 1130c5d24bddSHaiyang Zhang void *data; 11315c71dadbSHaiyang Zhang int ret; 113295fa0405SHaiyang Zhang 1133c5d24bddSHaiyang Zhang if (unlikely(offset + buflen > net_device->recv_buf_size)) { 1134c5d24bddSHaiyang Zhang status = NVSP_STAT_FAIL; 1135c5d24bddSHaiyang Zhang netif_err(net_device_ctx, rx_err, ndev, 1136c5d24bddSHaiyang Zhang "Packet offset:%u + len:%u too big\n", 1137c5d24bddSHaiyang Zhang offset, buflen); 1138c5d24bddSHaiyang Zhang 1139c5d24bddSHaiyang Zhang continue; 1140c5d24bddSHaiyang Zhang } 1141c5d24bddSHaiyang Zhang 1142c5d24bddSHaiyang Zhang data = recv_buf + offset; 1143c5d24bddSHaiyang Zhang 1144ec966381SStephen Hemminger trace_rndis_recv(ndev, q_idx, data); 1145ec966381SStephen Hemminger 114695fa0405SHaiyang Zhang /* Pass it to the upper layer */ 11475c71dadbSHaiyang Zhang ret = rndis_filter_receive(ndev, net_device, 1148dc54a08cSstephen hemminger channel, data, buflen); 11495c71dadbSHaiyang Zhang 11505c71dadbSHaiyang Zhang if (unlikely(ret != NVSP_STAT_SUCCESS)) 11515c71dadbSHaiyang Zhang status = NVSP_STAT_FAIL; 115295fa0405SHaiyang Zhang } 115395fa0405SHaiyang Zhang 11547426b1a5Sstephen hemminger enq_receive_complete(ndev, net_device, q_idx, 11557426b1a5Sstephen hemminger vmxferpage_packet->d.trans_id, status); 115615a863bfSstephen hemminger 115715a863bfSstephen hemminger return count; 115895fa0405SHaiyang Zhang } 115995fa0405SHaiyang Zhang 11605b54dac8SHaiyang Zhang static void netvsc_send_table(struct hv_device *hdev, 116171790a27SHaiyang Zhang struct nvsp_message *nvmsg) 11625b54dac8SHaiyang Zhang { 11630a1275caSVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(hdev); 11647ce10124Sstephen hemminger struct net_device_context *net_device_ctx = netdev_priv(ndev); 11655b54dac8SHaiyang Zhang int i; 11665b54dac8SHaiyang Zhang u32 count, *tab; 11675b54dac8SHaiyang Zhang 11685b54dac8SHaiyang Zhang count = nvmsg->msg.v5_msg.send_table.count; 11695b54dac8SHaiyang Zhang if (count != VRSS_SEND_TAB_SIZE) { 11705b54dac8SHaiyang Zhang netdev_err(ndev, "Received wrong send-table size:%u\n", count); 11715b54dac8SHaiyang Zhang return; 11725b54dac8SHaiyang Zhang } 11735b54dac8SHaiyang Zhang 11745b54dac8SHaiyang Zhang tab = (u32 *)((unsigned long)&nvmsg->msg.v5_msg.send_table + 11755b54dac8SHaiyang Zhang nvmsg->msg.v5_msg.send_table.offset); 11765b54dac8SHaiyang Zhang 11775b54dac8SHaiyang Zhang for (i = 0; i < count; i++) 117839e91cfbSHaiyang Zhang net_device_ctx->tx_table[i] = tab[i]; 11795b54dac8SHaiyang Zhang } 11805b54dac8SHaiyang Zhang 1181f9a7da91SVitaly Kuznetsov static void netvsc_send_vf(struct net_device_context *net_device_ctx, 118271790a27SHaiyang Zhang struct nvsp_message *nvmsg) 118371790a27SHaiyang Zhang { 1184f9a7da91SVitaly Kuznetsov net_device_ctx->vf_alloc = nvmsg->msg.v4_msg.vf_assoc.allocated; 1185f9a7da91SVitaly Kuznetsov net_device_ctx->vf_serial = nvmsg->msg.v4_msg.vf_assoc.serial; 118671790a27SHaiyang Zhang } 118771790a27SHaiyang Zhang 118871790a27SHaiyang Zhang static inline void netvsc_receive_inband(struct hv_device *hdev, 1189f9a7da91SVitaly Kuznetsov struct net_device_context *net_device_ctx, 119071790a27SHaiyang Zhang struct nvsp_message *nvmsg) 119171790a27SHaiyang Zhang { 119271790a27SHaiyang Zhang switch (nvmsg->hdr.msg_type) { 119371790a27SHaiyang Zhang case NVSP_MSG5_TYPE_SEND_INDIRECTION_TABLE: 119471790a27SHaiyang Zhang netvsc_send_table(hdev, nvmsg); 119571790a27SHaiyang Zhang break; 119671790a27SHaiyang Zhang 119771790a27SHaiyang Zhang case NVSP_MSG4_TYPE_SEND_VF_ASSOCIATION: 1198f9a7da91SVitaly Kuznetsov netvsc_send_vf(net_device_ctx, nvmsg); 119971790a27SHaiyang Zhang break; 120071790a27SHaiyang Zhang } 120171790a27SHaiyang Zhang } 120271790a27SHaiyang Zhang 120315a863bfSstephen hemminger static int netvsc_process_raw_pkt(struct hv_device *device, 120499a50bb1SK. Y. Srinivasan struct vmbus_channel *channel, 120599a50bb1SK. Y. Srinivasan struct netvsc_device *net_device, 120699a50bb1SK. Y. Srinivasan struct net_device *ndev, 1207f9645430Sstephen hemminger const struct vmpacket_descriptor *desc, 1208f9645430Sstephen hemminger int budget) 120999a50bb1SK. Y. Srinivasan { 1210f9a7da91SVitaly Kuznetsov struct net_device_context *net_device_ctx = netdev_priv(ndev); 1211f3dd3f47Sstephen hemminger struct nvsp_message *nvmsg = hv_pkt_data(desc); 121299a50bb1SK. Y. Srinivasan 1213ec966381SStephen Hemminger trace_nvsp_recv(ndev, channel, nvmsg); 1214ec966381SStephen Hemminger 121599a50bb1SK. Y. Srinivasan switch (desc->type) { 121699a50bb1SK. Y. Srinivasan case VM_PKT_COMP: 1217f9645430Sstephen hemminger netvsc_send_completion(net_device, channel, device, 1218f9645430Sstephen hemminger desc, budget); 121999a50bb1SK. Y. Srinivasan break; 122099a50bb1SK. Y. Srinivasan 122199a50bb1SK. Y. Srinivasan case VM_PKT_DATA_USING_XFER_PAGES: 122215a863bfSstephen hemminger return netvsc_receive(ndev, net_device, net_device_ctx, 1223f3dd3f47Sstephen hemminger device, channel, desc, nvmsg); 122499a50bb1SK. Y. Srinivasan break; 122599a50bb1SK. Y. Srinivasan 122699a50bb1SK. Y. Srinivasan case VM_PKT_DATA_INBAND: 1227f9a7da91SVitaly Kuznetsov netvsc_receive_inband(device, net_device_ctx, nvmsg); 122899a50bb1SK. Y. Srinivasan break; 122999a50bb1SK. Y. Srinivasan 123099a50bb1SK. Y. Srinivasan default: 123199a50bb1SK. Y. Srinivasan netdev_err(ndev, "unhandled packet type %d, tid %llx\n", 1232f4f1c23dSstephen hemminger desc->type, desc->trans_id); 123399a50bb1SK. Y. Srinivasan break; 123499a50bb1SK. Y. Srinivasan } 123515a863bfSstephen hemminger 123615a863bfSstephen hemminger return 0; 123715a863bfSstephen hemminger } 123815a863bfSstephen hemminger 123915a863bfSstephen hemminger static struct hv_device *netvsc_channel_to_device(struct vmbus_channel *channel) 124015a863bfSstephen hemminger { 124115a863bfSstephen hemminger struct vmbus_channel *primary = channel->primary_channel; 124215a863bfSstephen hemminger 124315a863bfSstephen hemminger return primary ? primary->device_obj : channel->device_obj; 124415a863bfSstephen hemminger } 124515a863bfSstephen hemminger 1246262b7f14Sstephen hemminger /* Network processing softirq 1247262b7f14Sstephen hemminger * Process data in incoming ring buffer from host 1248262b7f14Sstephen hemminger * Stops when ring is empty or budget is met or exceeded. 1249262b7f14Sstephen hemminger */ 125015a863bfSstephen hemminger int netvsc_poll(struct napi_struct *napi, int budget) 125115a863bfSstephen hemminger { 125215a863bfSstephen hemminger struct netvsc_channel *nvchan 125315a863bfSstephen hemminger = container_of(napi, struct netvsc_channel, napi); 125435fbbccfSstephen hemminger struct netvsc_device *net_device = nvchan->net_device; 125515a863bfSstephen hemminger struct vmbus_channel *channel = nvchan->channel; 125615a863bfSstephen hemminger struct hv_device *device = netvsc_channel_to_device(channel); 125715a863bfSstephen hemminger struct net_device *ndev = hv_get_drvdata(device); 125815a863bfSstephen hemminger int work_done = 0; 125915a863bfSstephen hemminger 1260f4f1c23dSstephen hemminger /* If starting a new interval */ 1261f4f1c23dSstephen hemminger if (!nvchan->desc) 1262f4f1c23dSstephen hemminger nvchan->desc = hv_pkt_iter_first(channel); 126315a863bfSstephen hemminger 1264f4f1c23dSstephen hemminger while (nvchan->desc && work_done < budget) { 1265f4f1c23dSstephen hemminger work_done += netvsc_process_raw_pkt(device, channel, net_device, 1266f9645430Sstephen hemminger ndev, nvchan->desc, budget); 1267f4f1c23dSstephen hemminger nvchan->desc = hv_pkt_iter_next(channel, nvchan->desc); 126815a863bfSstephen hemminger } 126915a863bfSstephen hemminger 12707426b1a5Sstephen hemminger /* If send of pending receive completions suceeded 1271f4e40363Sstephen hemminger * and did not exhaust NAPI budget this time 1272f4f1c23dSstephen hemminger * and not doing busy poll 1273f4e40363Sstephen hemminger * then re-enable host interrupts 1274f4e40363Sstephen hemminger * and reschedule if ring is not empty. 1275262b7f14Sstephen hemminger */ 1276cad5c197Sstephen hemminger if (send_recv_completions(ndev, net_device, nvchan) == 0 && 12777426b1a5Sstephen hemminger work_done < budget && 127815a863bfSstephen hemminger napi_complete_done(napi, work_done) && 1279d64e38aeSStephen Hemminger hv_end_read(&channel->inbound) && 1280d64e38aeSStephen Hemminger napi_schedule_prep(napi)) { 12817426b1a5Sstephen hemminger hv_begin_read(&channel->inbound); 1282d64e38aeSStephen Hemminger __napi_schedule(napi); 12837426b1a5Sstephen hemminger } 1284f4f1c23dSstephen hemminger 1285f4f1c23dSstephen hemminger /* Driver may overshoot since multiple packets per descriptor */ 1286f4f1c23dSstephen hemminger return min(work_done, budget); 128799a50bb1SK. Y. Srinivasan } 128899a50bb1SK. Y. Srinivasan 1289262b7f14Sstephen hemminger /* Call back when data is available in host ring buffer. 1290262b7f14Sstephen hemminger * Processing is deferred until network softirq (NAPI) 1291262b7f14Sstephen hemminger */ 12925b54dac8SHaiyang Zhang void netvsc_channel_cb(void *context) 129395fa0405SHaiyang Zhang { 12946de38af6Sstephen hemminger struct netvsc_channel *nvchan = context; 129543bf99ceSstephen hemminger struct vmbus_channel *channel = nvchan->channel; 129643bf99ceSstephen hemminger struct hv_ring_buffer_info *rbi = &channel->inbound; 129743bf99ceSstephen hemminger 129843bf99ceSstephen hemminger /* preload first vmpacket descriptor */ 129943bf99ceSstephen hemminger prefetch(hv_get_ring_buffer(rbi) + rbi->priv_read_index); 130095fa0405SHaiyang Zhang 1301f4f1c23dSstephen hemminger if (napi_schedule_prep(&nvchan->napi)) { 130215a863bfSstephen hemminger /* disable interupts from host */ 130343bf99ceSstephen hemminger hv_begin_read(rbi); 13040d6dd357Sstephen hemminger 130568633edaSStephen Hemminger __napi_schedule_irqoff(&nvchan->napi); 1306f4f1c23dSstephen hemminger } 130795fa0405SHaiyang Zhang } 130895fa0405SHaiyang Zhang 130995fa0405SHaiyang Zhang /* 131095fa0405SHaiyang Zhang * netvsc_device_add - Callback when the device belonging to this 131195fa0405SHaiyang Zhang * driver is added 131295fa0405SHaiyang Zhang */ 13139749fed5Sstephen hemminger struct netvsc_device *netvsc_device_add(struct hv_device *device, 13142c7f83caSstephen hemminger const struct netvsc_device_info *device_info) 131595fa0405SHaiyang Zhang { 131688098834SVitaly Kuznetsov int i, ret = 0; 131795fa0405SHaiyang Zhang struct netvsc_device *net_device; 131888098834SVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 131988098834SVitaly Kuznetsov struct net_device_context *net_device_ctx = netdev_priv(ndev); 132095fa0405SHaiyang Zhang 132188098834SVitaly Kuznetsov net_device = alloc_net_device(); 1322b1c84927SDan Carpenter if (!net_device) 13239749fed5Sstephen hemminger return ERR_PTR(-ENOMEM); 132495fa0405SHaiyang Zhang 13256b0cbe31SHaiyang Zhang for (i = 0; i < VRSS_SEND_TAB_SIZE; i++) 13266b0cbe31SHaiyang Zhang net_device_ctx->tx_table[i] = 0; 13276b0cbe31SHaiyang Zhang 132815a863bfSstephen hemminger /* Because the device uses NAPI, all the interrupt batching and 132915a863bfSstephen hemminger * control is done via Net softirq, not the channel handling 133015a863bfSstephen hemminger */ 133115a863bfSstephen hemminger set_channel_read_mode(device->channel, HV_CALL_ISR); 133215a863bfSstephen hemminger 1333bffb1842SK. Y. Srinivasan /* If we're reopening the device we may have multiple queues, fill the 1334bffb1842SK. Y. Srinivasan * chn_table with the default channel to use it before subchannels are 1335bffb1842SK. Y. Srinivasan * opened. 1336bffb1842SK. Y. Srinivasan * Initialize the channel state before we open; 1337bffb1842SK. Y. Srinivasan * we can be interrupted as soon as we open the channel. 1338bffb1842SK. Y. Srinivasan */ 1339bffb1842SK. Y. Srinivasan 1340bffb1842SK. Y. Srinivasan for (i = 0; i < VRSS_CHANNEL_MAX; i++) { 1341bffb1842SK. Y. Srinivasan struct netvsc_channel *nvchan = &net_device->chan_table[i]; 1342bffb1842SK. Y. Srinivasan 1343bffb1842SK. Y. Srinivasan nvchan->channel = device->channel; 134435fbbccfSstephen hemminger nvchan->net_device = net_device; 13454a0dee1fSFlorian Fainelli u64_stats_init(&nvchan->tx_stats.syncp); 13464a0dee1fSFlorian Fainelli u64_stats_init(&nvchan->rx_stats.syncp); 1347bffb1842SK. Y. Srinivasan } 1348bffb1842SK. Y. Srinivasan 13492be0f264Sstephen hemminger /* Enable NAPI handler before init callbacks */ 13502be0f264Sstephen hemminger netif_napi_add(ndev, &net_device->chan_table[0].napi, 13512be0f264Sstephen hemminger netvsc_poll, NAPI_POLL_WEIGHT); 13522be0f264Sstephen hemminger 135395fa0405SHaiyang Zhang /* Open the channel */ 1354a7f99d0fSStephen Hemminger ret = vmbus_open(device->channel, netvsc_ring_bytes, 1355a7f99d0fSStephen Hemminger netvsc_ring_bytes, NULL, 0, 1356a7f99d0fSStephen Hemminger netvsc_channel_cb, net_device->chan_table); 135795fa0405SHaiyang Zhang 135895fa0405SHaiyang Zhang if (ret != 0) { 135995fa0405SHaiyang Zhang netdev_err(ndev, "unable to open channel: %d\n", ret); 136095fa0405SHaiyang Zhang goto cleanup; 136195fa0405SHaiyang Zhang } 136295fa0405SHaiyang Zhang 136395fa0405SHaiyang Zhang /* Channel is opened */ 136493ba2222SVitaly Kuznetsov netdev_dbg(ndev, "hv_netvsc channel opened successfully\n"); 136595fa0405SHaiyang Zhang 136615a863bfSstephen hemminger napi_enable(&net_device->chan_table[0].napi); 136788098834SVitaly Kuznetsov 136895fa0405SHaiyang Zhang /* Connect with the NetVsp */ 13698b532797Sstephen hemminger ret = netvsc_connect_vsp(device, net_device, device_info); 137095fa0405SHaiyang Zhang if (ret != 0) { 137195fa0405SHaiyang Zhang netdev_err(ndev, 137295fa0405SHaiyang Zhang "unable to connect to NetVSP - %d\n", ret); 137395fa0405SHaiyang Zhang goto close; 137495fa0405SHaiyang Zhang } 137595fa0405SHaiyang Zhang 137612f69661SStephen Hemminger /* Writing nvdev pointer unlocks netvsc_send(), make sure chn_table is 137712f69661SStephen Hemminger * populated. 137812f69661SStephen Hemminger */ 137912f69661SStephen Hemminger rcu_assign_pointer(net_device_ctx->nvdev, net_device); 138012f69661SStephen Hemminger 13819749fed5Sstephen hemminger return net_device; 138295fa0405SHaiyang Zhang 138395fa0405SHaiyang Zhang close: 138449393347Sstephen hemminger RCU_INIT_POINTER(net_device_ctx->nvdev, NULL); 138549393347Sstephen hemminger napi_disable(&net_device->chan_table[0].napi); 138615a863bfSstephen hemminger 138795fa0405SHaiyang Zhang /* Now, we can close the channel safely */ 138895fa0405SHaiyang Zhang vmbus_close(device->channel); 138995fa0405SHaiyang Zhang 139095fa0405SHaiyang Zhang cleanup: 1391fcfb4a00SStephen Hemminger netif_napi_del(&net_device->chan_table[0].napi); 1392545a8e79Sstephen hemminger free_netvsc_device(&net_device->rcu); 139395fa0405SHaiyang Zhang 13949749fed5Sstephen hemminger return ERR_PTR(ret); 139595fa0405SHaiyang Zhang } 1396