195fa0405SHaiyang Zhang /* 295fa0405SHaiyang Zhang * Copyright (c) 2009, Microsoft Corporation. 395fa0405SHaiyang Zhang * 495fa0405SHaiyang Zhang * This program is free software; you can redistribute it and/or modify it 595fa0405SHaiyang Zhang * under the terms and conditions of the GNU General Public License, 695fa0405SHaiyang Zhang * version 2, as published by the Free Software Foundation. 795fa0405SHaiyang Zhang * 895fa0405SHaiyang Zhang * This program is distributed in the hope it will be useful, but WITHOUT 995fa0405SHaiyang Zhang * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 1095fa0405SHaiyang Zhang * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 1195fa0405SHaiyang Zhang * more details. 1295fa0405SHaiyang Zhang * 1395fa0405SHaiyang Zhang * You should have received a copy of the GNU General Public License along with 14adf8d3ffSJeff Kirsher * this program; if not, see <http://www.gnu.org/licenses/>. 1595fa0405SHaiyang Zhang * 1695fa0405SHaiyang Zhang * Authors: 1795fa0405SHaiyang Zhang * Haiyang Zhang <haiyangz@microsoft.com> 1895fa0405SHaiyang Zhang * Hank Janssen <hjanssen@microsoft.com> 1995fa0405SHaiyang Zhang */ 2095fa0405SHaiyang Zhang #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 2195fa0405SHaiyang Zhang 2295fa0405SHaiyang Zhang #include <linux/kernel.h> 2395fa0405SHaiyang Zhang #include <linux/sched.h> 2495fa0405SHaiyang Zhang #include <linux/wait.h> 2595fa0405SHaiyang Zhang #include <linux/mm.h> 2695fa0405SHaiyang Zhang #include <linux/delay.h> 2795fa0405SHaiyang Zhang #include <linux/io.h> 2895fa0405SHaiyang Zhang #include <linux/slab.h> 2995fa0405SHaiyang Zhang #include <linux/netdevice.h> 30f157e78dSHaiyang Zhang #include <linux/if_ether.h> 31d6472302SStephen Rothwell #include <linux/vmalloc.h> 329749fed5Sstephen hemminger #include <linux/rtnetlink.h> 3343bf99ceSstephen hemminger #include <linux/prefetch.h> 349749fed5Sstephen hemminger 35c25aaf81SKY Srinivasan #include <asm/sync_bitops.h> 3695fa0405SHaiyang Zhang 3795fa0405SHaiyang Zhang #include "hyperv_net.h" 3895fa0405SHaiyang Zhang 3984bf9cefSKY Srinivasan /* 4084bf9cefSKY Srinivasan * Switch the data path from the synthetic interface to the VF 4184bf9cefSKY Srinivasan * interface. 4284bf9cefSKY Srinivasan */ 430a1275caSVitaly Kuznetsov void netvsc_switch_datapath(struct net_device *ndev, bool vf) 4484bf9cefSKY Srinivasan { 453d541ac5SVitaly Kuznetsov struct net_device_context *net_device_ctx = netdev_priv(ndev); 463d541ac5SVitaly Kuznetsov struct hv_device *dev = net_device_ctx->device_ctx; 4779e8cbe7Sstephen hemminger struct netvsc_device *nv_dev = rtnl_dereference(net_device_ctx->nvdev); 480a1275caSVitaly Kuznetsov struct nvsp_message *init_pkt = &nv_dev->channel_init_pkt; 4984bf9cefSKY Srinivasan 5084bf9cefSKY Srinivasan memset(init_pkt, 0, sizeof(struct nvsp_message)); 5184bf9cefSKY Srinivasan init_pkt->hdr.msg_type = NVSP_MSG4_TYPE_SWITCH_DATA_PATH; 5284bf9cefSKY Srinivasan if (vf) 5384bf9cefSKY Srinivasan init_pkt->msg.v4_msg.active_dp.active_datapath = 5484bf9cefSKY Srinivasan NVSP_DATAPATH_VF; 5584bf9cefSKY Srinivasan else 5684bf9cefSKY Srinivasan init_pkt->msg.v4_msg.active_dp.active_datapath = 5784bf9cefSKY Srinivasan NVSP_DATAPATH_SYNTHETIC; 5884bf9cefSKY Srinivasan 5984bf9cefSKY Srinivasan vmbus_sendpacket(dev->channel, init_pkt, 6084bf9cefSKY Srinivasan sizeof(struct nvsp_message), 6184bf9cefSKY Srinivasan (unsigned long)init_pkt, 6284bf9cefSKY Srinivasan VM_PKT_DATA_INBAND, 0); 6384bf9cefSKY Srinivasan } 6484bf9cefSKY Srinivasan 6588098834SVitaly Kuznetsov static struct netvsc_device *alloc_net_device(void) 6695fa0405SHaiyang Zhang { 6795fa0405SHaiyang Zhang struct netvsc_device *net_device; 6895fa0405SHaiyang Zhang 6995fa0405SHaiyang Zhang net_device = kzalloc(sizeof(struct netvsc_device), GFP_KERNEL); 7095fa0405SHaiyang Zhang if (!net_device) 7195fa0405SHaiyang Zhang return NULL; 7295fa0405SHaiyang Zhang 73dc5cd894SHaiyang Zhang init_waitqueue_head(&net_device->wait_drain); 7495fa0405SHaiyang Zhang net_device->destroy = false; 7584bf9cefSKY Srinivasan atomic_set(&net_device->open_cnt, 0); 767c3877f2SHaiyang Zhang net_device->max_pkt = RNDIS_MAX_PKT_DEFAULT; 777c3877f2SHaiyang Zhang net_device->pkt_align = RNDIS_PKT_ALIGN_DEFAULT; 788b532797Sstephen hemminger 79fd612602SStephen Hemminger init_completion(&net_device->channel_init_wait); 80732e4985Sstephen hemminger init_waitqueue_head(&net_device->subchan_open); 818195b139SStephen Hemminger INIT_WORK(&net_device->subchan_work, rndis_set_subchannel); 827c3877f2SHaiyang Zhang 8395fa0405SHaiyang Zhang return net_device; 8495fa0405SHaiyang Zhang } 8595fa0405SHaiyang Zhang 86545a8e79Sstephen hemminger static void free_netvsc_device(struct rcu_head *head) 87f90251c8SHaiyang Zhang { 88545a8e79Sstephen hemminger struct netvsc_device *nvdev 89545a8e79Sstephen hemminger = container_of(head, struct netvsc_device, rcu); 90c0b558e5SHaiyang Zhang int i; 91c0b558e5SHaiyang Zhang 92c0b558e5SHaiyang Zhang for (i = 0; i < VRSS_CHANNEL_MAX; i++) 937426b1a5Sstephen hemminger vfree(nvdev->chan_table[i].mrc.slots); 94c0b558e5SHaiyang Zhang 95f90251c8SHaiyang Zhang kfree(nvdev); 96f90251c8SHaiyang Zhang } 97f90251c8SHaiyang Zhang 98545a8e79Sstephen hemminger static void free_netvsc_device_rcu(struct netvsc_device *nvdev) 99545a8e79Sstephen hemminger { 100545a8e79Sstephen hemminger call_rcu(&nvdev->rcu, free_netvsc_device); 101545a8e79Sstephen hemminger } 10246b4f7f5Sstephen hemminger 1037a2a0a84SStephen Hemminger static void netvsc_destroy_buf(struct hv_device *device) 10495fa0405SHaiyang Zhang { 10595fa0405SHaiyang Zhang struct nvsp_message *revoke_packet; 1063d541ac5SVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 1073962981fSstephen hemminger struct net_device_context *ndc = netdev_priv(ndev); 1083962981fSstephen hemminger struct netvsc_device *net_device = rtnl_dereference(ndc->nvdev); 1097a2a0a84SStephen Hemminger int ret; 11095fa0405SHaiyang Zhang 11195fa0405SHaiyang Zhang /* 11295fa0405SHaiyang Zhang * If we got a section count, it means we received a 11395fa0405SHaiyang Zhang * SendReceiveBufferComplete msg (ie sent 11495fa0405SHaiyang Zhang * NvspMessage1TypeSendReceiveBuffer msg) therefore, we need 11595fa0405SHaiyang Zhang * to send a revoke msg here 11695fa0405SHaiyang Zhang */ 11795fa0405SHaiyang Zhang if (net_device->recv_section_cnt) { 11895fa0405SHaiyang Zhang /* Send the revoke receive buffer */ 11995fa0405SHaiyang Zhang revoke_packet = &net_device->revoke_packet; 12095fa0405SHaiyang Zhang memset(revoke_packet, 0, sizeof(struct nvsp_message)); 12195fa0405SHaiyang Zhang 12295fa0405SHaiyang Zhang revoke_packet->hdr.msg_type = 12395fa0405SHaiyang Zhang NVSP_MSG1_TYPE_REVOKE_RECV_BUF; 12495fa0405SHaiyang Zhang revoke_packet->msg.v1_msg. 12595fa0405SHaiyang Zhang revoke_recv_buf.id = NETVSC_RECEIVE_BUFFER_ID; 12695fa0405SHaiyang Zhang 1273d541ac5SVitaly Kuznetsov ret = vmbus_sendpacket(device->channel, 12895fa0405SHaiyang Zhang revoke_packet, 12995fa0405SHaiyang Zhang sizeof(struct nvsp_message), 13095fa0405SHaiyang Zhang (unsigned long)revoke_packet, 13195fa0405SHaiyang Zhang VM_PKT_DATA_INBAND, 0); 13273e64fa4SK. Y. Srinivasan /* If the failure is because the channel is rescinded; 13373e64fa4SK. Y. Srinivasan * ignore the failure since we cannot send on a rescinded 13473e64fa4SK. Y. Srinivasan * channel. This would allow us to properly cleanup 13573e64fa4SK. Y. Srinivasan * even when the channel is rescinded. 13673e64fa4SK. Y. Srinivasan */ 13773e64fa4SK. Y. Srinivasan if (device->channel->rescind) 13873e64fa4SK. Y. Srinivasan ret = 0; 13995fa0405SHaiyang Zhang /* 14095fa0405SHaiyang Zhang * If we failed here, we might as well return and 14195fa0405SHaiyang Zhang * have a leak rather than continue and a bugchk 14295fa0405SHaiyang Zhang */ 14395fa0405SHaiyang Zhang if (ret != 0) { 14495fa0405SHaiyang Zhang netdev_err(ndev, "unable to send " 14595fa0405SHaiyang Zhang "revoke receive buffer to netvsp\n"); 1467a2a0a84SStephen Hemminger return; 14795fa0405SHaiyang Zhang } 1488b532797Sstephen hemminger net_device->recv_section_cnt = 0; 14995fa0405SHaiyang Zhang } 15095fa0405SHaiyang Zhang 15195fa0405SHaiyang Zhang /* Teardown the gpadl on the vsp end */ 15295fa0405SHaiyang Zhang if (net_device->recv_buf_gpadl_handle) { 1533d541ac5SVitaly Kuznetsov ret = vmbus_teardown_gpadl(device->channel, 15495fa0405SHaiyang Zhang net_device->recv_buf_gpadl_handle); 15595fa0405SHaiyang Zhang 15695fa0405SHaiyang Zhang /* If we failed here, we might as well return and have a leak 15795fa0405SHaiyang Zhang * rather than continue and a bugchk 15895fa0405SHaiyang Zhang */ 15995fa0405SHaiyang Zhang if (ret != 0) { 16095fa0405SHaiyang Zhang netdev_err(ndev, 16195fa0405SHaiyang Zhang "unable to teardown receive buffer's gpadl\n"); 1627a2a0a84SStephen Hemminger return; 16395fa0405SHaiyang Zhang } 16495fa0405SHaiyang Zhang net_device->recv_buf_gpadl_handle = 0; 16595fa0405SHaiyang Zhang } 16695fa0405SHaiyang Zhang 16795fa0405SHaiyang Zhang if (net_device->recv_buf) { 16895fa0405SHaiyang Zhang /* Free up the receive buffer */ 169b679ef73SHaiyang Zhang vfree(net_device->recv_buf); 17095fa0405SHaiyang Zhang net_device->recv_buf = NULL; 17195fa0405SHaiyang Zhang } 17295fa0405SHaiyang Zhang 173c25aaf81SKY Srinivasan /* Deal with the send buffer we may have setup. 174c25aaf81SKY Srinivasan * If we got a send section size, it means we received a 175c51ed182SHaiyang Zhang * NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE msg (ie sent 176c51ed182SHaiyang Zhang * NVSP_MSG1_TYPE_SEND_SEND_BUF msg) therefore, we need 177c25aaf81SKY Srinivasan * to send a revoke msg here 178c25aaf81SKY Srinivasan */ 1798b532797Sstephen hemminger if (net_device->send_section_cnt) { 180c25aaf81SKY Srinivasan /* Send the revoke receive buffer */ 181c25aaf81SKY Srinivasan revoke_packet = &net_device->revoke_packet; 182c25aaf81SKY Srinivasan memset(revoke_packet, 0, sizeof(struct nvsp_message)); 183c25aaf81SKY Srinivasan 184c25aaf81SKY Srinivasan revoke_packet->hdr.msg_type = 185c25aaf81SKY Srinivasan NVSP_MSG1_TYPE_REVOKE_SEND_BUF; 186c51ed182SHaiyang Zhang revoke_packet->msg.v1_msg.revoke_send_buf.id = 187c51ed182SHaiyang Zhang NETVSC_SEND_BUFFER_ID; 188c25aaf81SKY Srinivasan 1893d541ac5SVitaly Kuznetsov ret = vmbus_sendpacket(device->channel, 190c25aaf81SKY Srinivasan revoke_packet, 191c25aaf81SKY Srinivasan sizeof(struct nvsp_message), 192c25aaf81SKY Srinivasan (unsigned long)revoke_packet, 193c25aaf81SKY Srinivasan VM_PKT_DATA_INBAND, 0); 19473e64fa4SK. Y. Srinivasan 19573e64fa4SK. Y. Srinivasan /* If the failure is because the channel is rescinded; 19673e64fa4SK. Y. Srinivasan * ignore the failure since we cannot send on a rescinded 19773e64fa4SK. Y. Srinivasan * channel. This would allow us to properly cleanup 19873e64fa4SK. Y. Srinivasan * even when the channel is rescinded. 19973e64fa4SK. Y. Srinivasan */ 20073e64fa4SK. Y. Srinivasan if (device->channel->rescind) 20173e64fa4SK. Y. Srinivasan ret = 0; 20273e64fa4SK. Y. Srinivasan 203c25aaf81SKY Srinivasan /* If we failed here, we might as well return and 204c25aaf81SKY Srinivasan * have a leak rather than continue and a bugchk 205c25aaf81SKY Srinivasan */ 206c25aaf81SKY Srinivasan if (ret != 0) { 207c25aaf81SKY Srinivasan netdev_err(ndev, "unable to send " 208c25aaf81SKY Srinivasan "revoke send buffer to netvsp\n"); 2097a2a0a84SStephen Hemminger return; 210c25aaf81SKY Srinivasan } 2118b532797Sstephen hemminger net_device->send_section_cnt = 0; 212c25aaf81SKY Srinivasan } 213c25aaf81SKY Srinivasan /* Teardown the gpadl on the vsp end */ 214c25aaf81SKY Srinivasan if (net_device->send_buf_gpadl_handle) { 2153d541ac5SVitaly Kuznetsov ret = vmbus_teardown_gpadl(device->channel, 216c25aaf81SKY Srinivasan net_device->send_buf_gpadl_handle); 217c25aaf81SKY Srinivasan 218c25aaf81SKY Srinivasan /* If we failed here, we might as well return and have a leak 219c25aaf81SKY Srinivasan * rather than continue and a bugchk 220c25aaf81SKY Srinivasan */ 221c25aaf81SKY Srinivasan if (ret != 0) { 222c25aaf81SKY Srinivasan netdev_err(ndev, 223c25aaf81SKY Srinivasan "unable to teardown send buffer's gpadl\n"); 2247a2a0a84SStephen Hemminger return; 225c25aaf81SKY Srinivasan } 2262f18423dSDave Jones net_device->send_buf_gpadl_handle = 0; 227c25aaf81SKY Srinivasan } 228c25aaf81SKY Srinivasan if (net_device->send_buf) { 229c51ed182SHaiyang Zhang /* Free up the send buffer */ 23006b47aacSKY Srinivasan vfree(net_device->send_buf); 231c25aaf81SKY Srinivasan net_device->send_buf = NULL; 232c25aaf81SKY Srinivasan } 233c25aaf81SKY Srinivasan kfree(net_device->send_section_map); 23495fa0405SHaiyang Zhang } 23595fa0405SHaiyang Zhang 2367426b1a5Sstephen hemminger int netvsc_alloc_recv_comp_ring(struct netvsc_device *net_device, u32 q_idx) 2377426b1a5Sstephen hemminger { 2387426b1a5Sstephen hemminger struct netvsc_channel *nvchan = &net_device->chan_table[q_idx]; 2397426b1a5Sstephen hemminger int node = cpu_to_node(nvchan->channel->target_cpu); 2407426b1a5Sstephen hemminger size_t size; 2417426b1a5Sstephen hemminger 2427426b1a5Sstephen hemminger size = net_device->recv_completion_cnt * sizeof(struct recv_comp_data); 2437426b1a5Sstephen hemminger nvchan->mrc.slots = vzalloc_node(size, node); 2447426b1a5Sstephen hemminger if (!nvchan->mrc.slots) 2457426b1a5Sstephen hemminger nvchan->mrc.slots = vzalloc(size); 2467426b1a5Sstephen hemminger 2477426b1a5Sstephen hemminger return nvchan->mrc.slots ? 0 : -ENOMEM; 2487426b1a5Sstephen hemminger } 2497426b1a5Sstephen hemminger 25095790837Sstephen hemminger static int netvsc_init_buf(struct hv_device *device, 2518b532797Sstephen hemminger struct netvsc_device *net_device, 2528b532797Sstephen hemminger const struct netvsc_device_info *device_info) 25395fa0405SHaiyang Zhang { 2547426b1a5Sstephen hemminger struct nvsp_1_message_send_receive_buffer_complete *resp; 25595833370Sstephen hemminger struct net_device *ndev = hv_get_drvdata(device); 25695833370Sstephen hemminger struct nvsp_message *init_packet; 2578b532797Sstephen hemminger unsigned int buf_size; 258fdfb70d2Sstephen hemminger size_t map_words; 25995833370Sstephen hemminger int ret = 0; 26095fa0405SHaiyang Zhang 2618b532797Sstephen hemminger /* Get receive buffer area. */ 2620ab09befSAlex Ng buf_size = device_info->recv_sections * device_info->recv_section_size; 2638b532797Sstephen hemminger buf_size = roundup(buf_size, PAGE_SIZE); 2648b532797Sstephen hemminger 2658b532797Sstephen hemminger net_device->recv_buf = vzalloc(buf_size); 26695fa0405SHaiyang Zhang if (!net_device->recv_buf) { 2678b532797Sstephen hemminger netdev_err(ndev, 2688b532797Sstephen hemminger "unable to allocate receive buffer of size %u\n", 2698b532797Sstephen hemminger buf_size); 27095fa0405SHaiyang Zhang ret = -ENOMEM; 27195fa0405SHaiyang Zhang goto cleanup; 27295fa0405SHaiyang Zhang } 27395fa0405SHaiyang Zhang 27495fa0405SHaiyang Zhang /* 27595fa0405SHaiyang Zhang * Establish the gpadl handle for this buffer on this 27695fa0405SHaiyang Zhang * channel. Note: This call uses the vmbus connection rather 27795fa0405SHaiyang Zhang * than the channel to establish the gpadl handle. 27895fa0405SHaiyang Zhang */ 27995fa0405SHaiyang Zhang ret = vmbus_establish_gpadl(device->channel, net_device->recv_buf, 2808b532797Sstephen hemminger buf_size, 28195fa0405SHaiyang Zhang &net_device->recv_buf_gpadl_handle); 28295fa0405SHaiyang Zhang if (ret != 0) { 28395fa0405SHaiyang Zhang netdev_err(ndev, 28495fa0405SHaiyang Zhang "unable to establish receive buffer's gpadl\n"); 28595fa0405SHaiyang Zhang goto cleanup; 28695fa0405SHaiyang Zhang } 28795fa0405SHaiyang Zhang 28895fa0405SHaiyang Zhang /* Notify the NetVsp of the gpadl handle */ 28995fa0405SHaiyang Zhang init_packet = &net_device->channel_init_pkt; 29095fa0405SHaiyang Zhang memset(init_packet, 0, sizeof(struct nvsp_message)); 29195fa0405SHaiyang Zhang init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_RECV_BUF; 29295fa0405SHaiyang Zhang init_packet->msg.v1_msg.send_recv_buf. 29395fa0405SHaiyang Zhang gpadl_handle = net_device->recv_buf_gpadl_handle; 29495fa0405SHaiyang Zhang init_packet->msg.v1_msg. 29595fa0405SHaiyang Zhang send_recv_buf.id = NETVSC_RECEIVE_BUFFER_ID; 29695fa0405SHaiyang Zhang 29795fa0405SHaiyang Zhang /* Send the gpadl notification request */ 29895fa0405SHaiyang Zhang ret = vmbus_sendpacket(device->channel, init_packet, 29995fa0405SHaiyang Zhang sizeof(struct nvsp_message), 30095fa0405SHaiyang Zhang (unsigned long)init_packet, 30195fa0405SHaiyang Zhang VM_PKT_DATA_INBAND, 30295fa0405SHaiyang Zhang VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 30395fa0405SHaiyang Zhang if (ret != 0) { 30495fa0405SHaiyang Zhang netdev_err(ndev, 30595fa0405SHaiyang Zhang "unable to send receive buffer's gpadl to netvsp\n"); 30695fa0405SHaiyang Zhang goto cleanup; 30795fa0405SHaiyang Zhang } 30895fa0405SHaiyang Zhang 3095362855aSVitaly Kuznetsov wait_for_completion(&net_device->channel_init_wait); 31095fa0405SHaiyang Zhang 31195fa0405SHaiyang Zhang /* Check the response */ 3127426b1a5Sstephen hemminger resp = &init_packet->msg.v1_msg.send_recv_buf_complete; 3137426b1a5Sstephen hemminger if (resp->status != NVSP_STAT_SUCCESS) { 3147426b1a5Sstephen hemminger netdev_err(ndev, 3157426b1a5Sstephen hemminger "Unable to complete receive buffer initialization with NetVsp - status %d\n", 3167426b1a5Sstephen hemminger resp->status); 31795fa0405SHaiyang Zhang ret = -EINVAL; 31895fa0405SHaiyang Zhang goto cleanup; 31995fa0405SHaiyang Zhang } 32095fa0405SHaiyang Zhang 32195fa0405SHaiyang Zhang /* Parse the response */ 3227426b1a5Sstephen hemminger netdev_dbg(ndev, "Receive sections: %u sub_allocs: size %u count: %u\n", 3237426b1a5Sstephen hemminger resp->num_sections, resp->sections[0].sub_alloc_size, 3247426b1a5Sstephen hemminger resp->sections[0].num_sub_allocs); 32595fa0405SHaiyang Zhang 3268b532797Sstephen hemminger /* There should only be one section for the entire receive buffer */ 3278b532797Sstephen hemminger if (resp->num_sections != 1 || resp->sections[0].offset != 0) { 32895fa0405SHaiyang Zhang ret = -EINVAL; 32995fa0405SHaiyang Zhang goto cleanup; 33095fa0405SHaiyang Zhang } 33195fa0405SHaiyang Zhang 3328b532797Sstephen hemminger net_device->recv_section_size = resp->sections[0].sub_alloc_size; 3338b532797Sstephen hemminger net_device->recv_section_cnt = resp->sections[0].num_sub_allocs; 3348b532797Sstephen hemminger 3357426b1a5Sstephen hemminger /* Setup receive completion ring */ 3367426b1a5Sstephen hemminger net_device->recv_completion_cnt 3378b532797Sstephen hemminger = round_up(net_device->recv_section_cnt + 1, 3387426b1a5Sstephen hemminger PAGE_SIZE / sizeof(u64)); 3397426b1a5Sstephen hemminger ret = netvsc_alloc_recv_comp_ring(net_device, 0); 3407426b1a5Sstephen hemminger if (ret) 3417426b1a5Sstephen hemminger goto cleanup; 3427426b1a5Sstephen hemminger 3437426b1a5Sstephen hemminger /* Now setup the send buffer. */ 3440ab09befSAlex Ng buf_size = device_info->send_sections * device_info->send_section_size; 3458b532797Sstephen hemminger buf_size = round_up(buf_size, PAGE_SIZE); 3468b532797Sstephen hemminger 3478b532797Sstephen hemminger net_device->send_buf = vzalloc(buf_size); 348c25aaf81SKY Srinivasan if (!net_device->send_buf) { 3498b532797Sstephen hemminger netdev_err(ndev, "unable to allocate send buffer of size %u\n", 3508b532797Sstephen hemminger buf_size); 351c25aaf81SKY Srinivasan ret = -ENOMEM; 352c25aaf81SKY Srinivasan goto cleanup; 353c25aaf81SKY Srinivasan } 354c25aaf81SKY Srinivasan 355c25aaf81SKY Srinivasan /* Establish the gpadl handle for this buffer on this 356c25aaf81SKY Srinivasan * channel. Note: This call uses the vmbus connection rather 357c25aaf81SKY Srinivasan * than the channel to establish the gpadl handle. 358c25aaf81SKY Srinivasan */ 359c25aaf81SKY Srinivasan ret = vmbus_establish_gpadl(device->channel, net_device->send_buf, 3608b532797Sstephen hemminger buf_size, 361c25aaf81SKY Srinivasan &net_device->send_buf_gpadl_handle); 362c25aaf81SKY Srinivasan if (ret != 0) { 363c25aaf81SKY Srinivasan netdev_err(ndev, 364c25aaf81SKY Srinivasan "unable to establish send buffer's gpadl\n"); 365c25aaf81SKY Srinivasan goto cleanup; 366c25aaf81SKY Srinivasan } 367c25aaf81SKY Srinivasan 368c25aaf81SKY Srinivasan /* Notify the NetVsp of the gpadl handle */ 369c25aaf81SKY Srinivasan init_packet = &net_device->channel_init_pkt; 370c25aaf81SKY Srinivasan memset(init_packet, 0, sizeof(struct nvsp_message)); 371c25aaf81SKY Srinivasan init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_SEND_BUF; 372c51ed182SHaiyang Zhang init_packet->msg.v1_msg.send_send_buf.gpadl_handle = 373c25aaf81SKY Srinivasan net_device->send_buf_gpadl_handle; 374c51ed182SHaiyang Zhang init_packet->msg.v1_msg.send_send_buf.id = NETVSC_SEND_BUFFER_ID; 375c25aaf81SKY Srinivasan 376c25aaf81SKY Srinivasan /* Send the gpadl notification request */ 377c25aaf81SKY Srinivasan ret = vmbus_sendpacket(device->channel, init_packet, 378c25aaf81SKY Srinivasan sizeof(struct nvsp_message), 379c25aaf81SKY Srinivasan (unsigned long)init_packet, 380c25aaf81SKY Srinivasan VM_PKT_DATA_INBAND, 381c25aaf81SKY Srinivasan VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 382c25aaf81SKY Srinivasan if (ret != 0) { 383c25aaf81SKY Srinivasan netdev_err(ndev, 384c25aaf81SKY Srinivasan "unable to send send buffer's gpadl to netvsp\n"); 385c25aaf81SKY Srinivasan goto cleanup; 386c25aaf81SKY Srinivasan } 387c25aaf81SKY Srinivasan 3885362855aSVitaly Kuznetsov wait_for_completion(&net_device->channel_init_wait); 389c25aaf81SKY Srinivasan 390c25aaf81SKY Srinivasan /* Check the response */ 391c25aaf81SKY Srinivasan if (init_packet->msg.v1_msg. 392c25aaf81SKY Srinivasan send_send_buf_complete.status != NVSP_STAT_SUCCESS) { 393c25aaf81SKY Srinivasan netdev_err(ndev, "Unable to complete send buffer " 394c25aaf81SKY Srinivasan "initialization with NetVsp - status %d\n", 395c25aaf81SKY Srinivasan init_packet->msg.v1_msg. 396c51ed182SHaiyang Zhang send_send_buf_complete.status); 397c25aaf81SKY Srinivasan ret = -EINVAL; 398c25aaf81SKY Srinivasan goto cleanup; 399c25aaf81SKY Srinivasan } 400c25aaf81SKY Srinivasan 401c25aaf81SKY Srinivasan /* Parse the response */ 402c25aaf81SKY Srinivasan net_device->send_section_size = init_packet->msg. 403c25aaf81SKY Srinivasan v1_msg.send_send_buf_complete.section_size; 404c25aaf81SKY Srinivasan 4058b532797Sstephen hemminger /* Section count is simply the size divided by the section size. */ 4068b532797Sstephen hemminger net_device->send_section_cnt = buf_size / net_device->send_section_size; 407c25aaf81SKY Srinivasan 40893ba2222SVitaly Kuznetsov netdev_dbg(ndev, "Send section size: %d, Section count:%d\n", 409c25aaf81SKY Srinivasan net_device->send_section_size, net_device->send_section_cnt); 410c25aaf81SKY Srinivasan 411c25aaf81SKY Srinivasan /* Setup state for managing the send buffer. */ 412fdfb70d2Sstephen hemminger map_words = DIV_ROUND_UP(net_device->send_section_cnt, BITS_PER_LONG); 413c25aaf81SKY Srinivasan 414fdfb70d2Sstephen hemminger net_device->send_section_map = kcalloc(map_words, sizeof(ulong), GFP_KERNEL); 415dd1d3f8fSWei Yongjun if (net_device->send_section_map == NULL) { 416dd1d3f8fSWei Yongjun ret = -ENOMEM; 417c25aaf81SKY Srinivasan goto cleanup; 418dd1d3f8fSWei Yongjun } 419c25aaf81SKY Srinivasan 42095fa0405SHaiyang Zhang goto exit; 42195fa0405SHaiyang Zhang 42295fa0405SHaiyang Zhang cleanup: 4233d541ac5SVitaly Kuznetsov netvsc_destroy_buf(device); 42495fa0405SHaiyang Zhang 42595fa0405SHaiyang Zhang exit: 42695fa0405SHaiyang Zhang return ret; 42795fa0405SHaiyang Zhang } 42895fa0405SHaiyang Zhang 429f157e78dSHaiyang Zhang /* Negotiate NVSP protocol version */ 430f157e78dSHaiyang Zhang static int negotiate_nvsp_ver(struct hv_device *device, 431f157e78dSHaiyang Zhang struct netvsc_device *net_device, 432f157e78dSHaiyang Zhang struct nvsp_message *init_packet, 433f157e78dSHaiyang Zhang u32 nvsp_ver) 43495fa0405SHaiyang Zhang { 4350a1275caSVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 4367390fe9cSNicholas Mc Guire int ret; 437f157e78dSHaiyang Zhang 438f157e78dSHaiyang Zhang memset(init_packet, 0, sizeof(struct nvsp_message)); 439f157e78dSHaiyang Zhang init_packet->hdr.msg_type = NVSP_MSG_TYPE_INIT; 440f157e78dSHaiyang Zhang init_packet->msg.init_msg.init.min_protocol_ver = nvsp_ver; 441f157e78dSHaiyang Zhang init_packet->msg.init_msg.init.max_protocol_ver = nvsp_ver; 442f157e78dSHaiyang Zhang 443f157e78dSHaiyang Zhang /* Send the init request */ 444f157e78dSHaiyang Zhang ret = vmbus_sendpacket(device->channel, init_packet, 445f157e78dSHaiyang Zhang sizeof(struct nvsp_message), 446f157e78dSHaiyang Zhang (unsigned long)init_packet, 447f157e78dSHaiyang Zhang VM_PKT_DATA_INBAND, 448f157e78dSHaiyang Zhang VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 449f157e78dSHaiyang Zhang 450f157e78dSHaiyang Zhang if (ret != 0) 451f157e78dSHaiyang Zhang return ret; 452f157e78dSHaiyang Zhang 4535362855aSVitaly Kuznetsov wait_for_completion(&net_device->channel_init_wait); 454f157e78dSHaiyang Zhang 455f157e78dSHaiyang Zhang if (init_packet->msg.init_msg.init_complete.status != 456f157e78dSHaiyang Zhang NVSP_STAT_SUCCESS) 457f157e78dSHaiyang Zhang return -EINVAL; 458f157e78dSHaiyang Zhang 459a1eabb01SHaiyang Zhang if (nvsp_ver == NVSP_PROTOCOL_VERSION_1) 460f157e78dSHaiyang Zhang return 0; 461f157e78dSHaiyang Zhang 46271790a27SHaiyang Zhang /* NVSPv2 or later: Send NDIS config */ 463f157e78dSHaiyang Zhang memset(init_packet, 0, sizeof(struct nvsp_message)); 464f157e78dSHaiyang Zhang init_packet->hdr.msg_type = NVSP_MSG2_TYPE_SEND_NDIS_CONFIG; 4650a1275caSVitaly Kuznetsov init_packet->msg.v2_msg.send_ndis_config.mtu = ndev->mtu + ETH_HLEN; 4661f5f3a75SHaiyang Zhang init_packet->msg.v2_msg.send_ndis_config.capability.ieee8021q = 1; 467f157e78dSHaiyang Zhang 4687f5d5af0SHaiyang Zhang if (nvsp_ver >= NVSP_PROTOCOL_VERSION_5) { 46971790a27SHaiyang Zhang init_packet->msg.v2_msg.send_ndis_config.capability.sriov = 1; 47071790a27SHaiyang Zhang 4717f5d5af0SHaiyang Zhang /* Teaming bit is needed to receive link speed updates */ 4727f5d5af0SHaiyang Zhang init_packet->msg.v2_msg.send_ndis_config.capability.teaming = 1; 4737f5d5af0SHaiyang Zhang } 4747f5d5af0SHaiyang Zhang 475f157e78dSHaiyang Zhang ret = vmbus_sendpacket(device->channel, init_packet, 476f157e78dSHaiyang Zhang sizeof(struct nvsp_message), 477f157e78dSHaiyang Zhang (unsigned long)init_packet, 478f157e78dSHaiyang Zhang VM_PKT_DATA_INBAND, 0); 479f157e78dSHaiyang Zhang 480f157e78dSHaiyang Zhang return ret; 481f157e78dSHaiyang Zhang } 482f157e78dSHaiyang Zhang 48395790837Sstephen hemminger static int netvsc_connect_vsp(struct hv_device *device, 4848b532797Sstephen hemminger struct netvsc_device *net_device, 4858b532797Sstephen hemminger const struct netvsc_device_info *device_info) 486f157e78dSHaiyang Zhang { 4871b17ca04SColin Ian King static const u32 ver_list[] = { 488e5a78fadSStephen Hemminger NVSP_PROTOCOL_VERSION_1, NVSP_PROTOCOL_VERSION_2, 48995790837Sstephen hemminger NVSP_PROTOCOL_VERSION_4, NVSP_PROTOCOL_VERSION_5 49095790837Sstephen hemminger }; 49195790837Sstephen hemminger struct nvsp_message *init_packet; 49295790837Sstephen hemminger int ndis_version, i, ret; 49395fa0405SHaiyang Zhang 49495fa0405SHaiyang Zhang init_packet = &net_device->channel_init_pkt; 49595fa0405SHaiyang Zhang 496f157e78dSHaiyang Zhang /* Negotiate the latest NVSP protocol supported */ 497e5a78fadSStephen Hemminger for (i = ARRAY_SIZE(ver_list) - 1; i >= 0; i--) 498f157e78dSHaiyang Zhang if (negotiate_nvsp_ver(device, net_device, init_packet, 499a1eabb01SHaiyang Zhang ver_list[i]) == 0) { 500a1eabb01SHaiyang Zhang net_device->nvsp_version = ver_list[i]; 501a1eabb01SHaiyang Zhang break; 502a1eabb01SHaiyang Zhang } 503a1eabb01SHaiyang Zhang 504a1eabb01SHaiyang Zhang if (i < 0) { 50595fa0405SHaiyang Zhang ret = -EPROTO; 50695fa0405SHaiyang Zhang goto cleanup; 50795fa0405SHaiyang Zhang } 508f157e78dSHaiyang Zhang 509f157e78dSHaiyang Zhang pr_debug("Negotiated NVSP version:%x\n", net_device->nvsp_version); 510f157e78dSHaiyang Zhang 51195fa0405SHaiyang Zhang /* Send the ndis version */ 51295fa0405SHaiyang Zhang memset(init_packet, 0, sizeof(struct nvsp_message)); 51395fa0405SHaiyang Zhang 514a1eabb01SHaiyang Zhang if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_4) 5151f73db49SKY Srinivasan ndis_version = 0x00060001; 516a1eabb01SHaiyang Zhang else 517a1eabb01SHaiyang Zhang ndis_version = 0x0006001e; 51895fa0405SHaiyang Zhang 51995fa0405SHaiyang Zhang init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_NDIS_VER; 52095fa0405SHaiyang Zhang init_packet->msg.v1_msg. 52195fa0405SHaiyang Zhang send_ndis_ver.ndis_major_ver = 52295fa0405SHaiyang Zhang (ndis_version & 0xFFFF0000) >> 16; 52395fa0405SHaiyang Zhang init_packet->msg.v1_msg. 52495fa0405SHaiyang Zhang send_ndis_ver.ndis_minor_ver = 52595fa0405SHaiyang Zhang ndis_version & 0xFFFF; 52695fa0405SHaiyang Zhang 52795fa0405SHaiyang Zhang /* Send the init request */ 52895fa0405SHaiyang Zhang ret = vmbus_sendpacket(device->channel, init_packet, 52995fa0405SHaiyang Zhang sizeof(struct nvsp_message), 53095fa0405SHaiyang Zhang (unsigned long)init_packet, 53195fa0405SHaiyang Zhang VM_PKT_DATA_INBAND, 0); 53295fa0405SHaiyang Zhang if (ret != 0) 53395fa0405SHaiyang Zhang goto cleanup; 53495fa0405SHaiyang Zhang 53599d3016dSHaiyang Zhang 5368b532797Sstephen hemminger ret = netvsc_init_buf(device, net_device, device_info); 53795fa0405SHaiyang Zhang 53895fa0405SHaiyang Zhang cleanup: 53995fa0405SHaiyang Zhang return ret; 54095fa0405SHaiyang Zhang } 54195fa0405SHaiyang Zhang 5423d541ac5SVitaly Kuznetsov static void netvsc_disconnect_vsp(struct hv_device *device) 54395fa0405SHaiyang Zhang { 5443d541ac5SVitaly Kuznetsov netvsc_destroy_buf(device); 54595fa0405SHaiyang Zhang } 54695fa0405SHaiyang Zhang 54795fa0405SHaiyang Zhang /* 54895fa0405SHaiyang Zhang * netvsc_device_remove - Callback when the root bus device is removed 54995fa0405SHaiyang Zhang */ 550e08f3ea5SStephen Hemminger void netvsc_device_remove(struct hv_device *device) 55195fa0405SHaiyang Zhang { 5523d541ac5SVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 5533d541ac5SVitaly Kuznetsov struct net_device_context *net_device_ctx = netdev_priv(ndev); 55479e8cbe7Sstephen hemminger struct netvsc_device *net_device 55579e8cbe7Sstephen hemminger = rtnl_dereference(net_device_ctx->nvdev); 55615a863bfSstephen hemminger int i; 55795fa0405SHaiyang Zhang 5588195b139SStephen Hemminger cancel_work_sync(&net_device->subchan_work); 5598195b139SStephen Hemminger 5603d541ac5SVitaly Kuznetsov netvsc_disconnect_vsp(device); 56195fa0405SHaiyang Zhang 562545a8e79Sstephen hemminger RCU_INIT_POINTER(net_device_ctx->nvdev, NULL); 56395fa0405SHaiyang Zhang 56495fa0405SHaiyang Zhang /* 56595fa0405SHaiyang Zhang * At this point, no one should be accessing net_device 56695fa0405SHaiyang Zhang * except in here 56795fa0405SHaiyang Zhang */ 56893ba2222SVitaly Kuznetsov netdev_dbg(ndev, "net device safe to remove\n"); 56995fa0405SHaiyang Zhang 57095fa0405SHaiyang Zhang /* Now, we can close the channel safely */ 57195fa0405SHaiyang Zhang vmbus_close(device->channel); 57295fa0405SHaiyang Zhang 57376bb5db5Sstephen hemminger /* And dissassociate NAPI context from device */ 57479cd874cSstephen hemminger for (i = 0; i < net_device->num_chn; i++) 57576bb5db5Sstephen hemminger netif_napi_del(&net_device->chan_table[i].napi); 57615a863bfSstephen hemminger 57795fa0405SHaiyang Zhang /* Release all resources */ 578545a8e79Sstephen hemminger free_netvsc_device_rcu(net_device); 57995fa0405SHaiyang Zhang } 58095fa0405SHaiyang Zhang 58133be96e4SHaiyang Zhang #define RING_AVAIL_PERCENT_HIWATER 20 58233be96e4SHaiyang Zhang #define RING_AVAIL_PERCENT_LOWATER 10 58333be96e4SHaiyang Zhang 58433be96e4SHaiyang Zhang /* 58533be96e4SHaiyang Zhang * Get the percentage of available bytes to write in the ring. 58633be96e4SHaiyang Zhang * The return value is in range from 0 to 100. 58733be96e4SHaiyang Zhang */ 58833be96e4SHaiyang Zhang static inline u32 hv_ringbuf_avail_percent( 58933be96e4SHaiyang Zhang struct hv_ring_buffer_info *ring_info) 59033be96e4SHaiyang Zhang { 59133be96e4SHaiyang Zhang u32 avail_read, avail_write; 59233be96e4SHaiyang Zhang 59333be96e4SHaiyang Zhang hv_get_ringbuffer_availbytes(ring_info, &avail_read, &avail_write); 59433be96e4SHaiyang Zhang 59533be96e4SHaiyang Zhang return avail_write * 100 / ring_info->ring_datasize; 59633be96e4SHaiyang Zhang } 59733be96e4SHaiyang Zhang 598c25aaf81SKY Srinivasan static inline void netvsc_free_send_slot(struct netvsc_device *net_device, 599c25aaf81SKY Srinivasan u32 index) 600c25aaf81SKY Srinivasan { 601c25aaf81SKY Srinivasan sync_change_bit(index, net_device->send_section_map); 602c25aaf81SKY Srinivasan } 603c25aaf81SKY Srinivasan 604bc304dd3SStephen Hemminger static void netvsc_send_tx_complete(struct netvsc_device *net_device, 60525b85ee8SKY Srinivasan struct vmbus_channel *incoming_channel, 60697c1723aSKY Srinivasan struct hv_device *device, 607f9645430Sstephen hemminger const struct vmpacket_descriptor *desc, 608f9645430Sstephen hemminger int budget) 60995fa0405SHaiyang Zhang { 61050698d80Sstephen hemminger struct sk_buff *skb = (struct sk_buff *)(unsigned long)desc->trans_id; 6113d541ac5SVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 61209af87d1SSimon Xiao struct net_device_context *ndev_ctx = netdev_priv(ndev); 613bc304dd3SStephen Hemminger struct vmbus_channel *channel = device->channel; 6145b54dac8SHaiyang Zhang u16 q_idx = 0; 6155b54dac8SHaiyang Zhang int queue_sends; 61633be96e4SHaiyang Zhang 61795fa0405SHaiyang Zhang /* Notify the layer above us */ 618bc304dd3SStephen Hemminger if (likely(skb)) { 619793e3955Sstephen hemminger const struct hv_netvsc_packet *packet 620bc304dd3SStephen Hemminger = (struct hv_netvsc_packet *)skb->cb; 621793e3955Sstephen hemminger u32 send_index = packet->send_buf_index; 622793e3955Sstephen hemminger struct netvsc_stats *tx_stats; 623bc304dd3SStephen Hemminger 624c25aaf81SKY Srinivasan if (send_index != NETVSC_INVALID_INDEX) 625c25aaf81SKY Srinivasan netvsc_free_send_slot(net_device, send_index); 626793e3955Sstephen hemminger q_idx = packet->q_idx; 62725b85ee8SKY Srinivasan channel = incoming_channel; 628bc304dd3SStephen Hemminger 6296c80f3fcSSimon Xiao tx_stats = &net_device->chan_table[q_idx].tx_stats; 630793e3955Sstephen hemminger 631793e3955Sstephen hemminger u64_stats_update_begin(&tx_stats->syncp); 632793e3955Sstephen hemminger tx_stats->packets += packet->total_packets; 633793e3955Sstephen hemminger tx_stats->bytes += packet->total_bytes; 634793e3955Sstephen hemminger u64_stats_update_end(&tx_stats->syncp); 635793e3955Sstephen hemminger 636f9645430Sstephen hemminger napi_consume_skb(skb, budget); 6375b54dac8SHaiyang Zhang } 63895fa0405SHaiyang Zhang 639b8b835a8Sstephen hemminger queue_sends = 640b8b835a8Sstephen hemminger atomic_dec_return(&net_device->chan_table[q_idx].queue_sends); 6411d06825bSHaiyang Zhang 64246b4f7f5Sstephen hemminger if (net_device->destroy && queue_sends == 0) 643dc5cd894SHaiyang Zhang wake_up(&net_device->wait_drain); 644dc5cd894SHaiyang Zhang 6455b54dac8SHaiyang Zhang if (netif_tx_queue_stopped(netdev_get_tx_queue(ndev, q_idx)) && 646bc304dd3SStephen Hemminger (hv_ringbuf_avail_percent(&channel->outbound) > RING_AVAIL_PERCENT_HIWATER || 64709af87d1SSimon Xiao queue_sends < 1)) { 648796cc88cSStephen Hemminger netif_tx_wake_queue(netdev_get_tx_queue(ndev, q_idx)); 64909af87d1SSimon Xiao ndev_ctx->eth_stats.wake_queue++; 65009af87d1SSimon Xiao } 651bc304dd3SStephen Hemminger } 652bc304dd3SStephen Hemminger 653bc304dd3SStephen Hemminger static void netvsc_send_completion(struct netvsc_device *net_device, 654bc304dd3SStephen Hemminger struct vmbus_channel *incoming_channel, 655bc304dd3SStephen Hemminger struct hv_device *device, 656f9645430Sstephen hemminger const struct vmpacket_descriptor *desc, 657f9645430Sstephen hemminger int budget) 658bc304dd3SStephen Hemminger { 659f3dd3f47Sstephen hemminger struct nvsp_message *nvsp_packet = hv_pkt_data(desc); 660bc304dd3SStephen Hemminger struct net_device *ndev = hv_get_drvdata(device); 661bc304dd3SStephen Hemminger 662bc304dd3SStephen Hemminger switch (nvsp_packet->hdr.msg_type) { 663bc304dd3SStephen Hemminger case NVSP_MSG_TYPE_INIT_COMPLETE: 664bc304dd3SStephen Hemminger case NVSP_MSG1_TYPE_SEND_RECV_BUF_COMPLETE: 665bc304dd3SStephen Hemminger case NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE: 666bc304dd3SStephen Hemminger case NVSP_MSG5_TYPE_SUBCHANNEL: 667bc304dd3SStephen Hemminger /* Copy the response back */ 668bc304dd3SStephen Hemminger memcpy(&net_device->channel_init_pkt, nvsp_packet, 669bc304dd3SStephen Hemminger sizeof(struct nvsp_message)); 670bc304dd3SStephen Hemminger complete(&net_device->channel_init_wait); 671bc304dd3SStephen Hemminger break; 672bc304dd3SStephen Hemminger 673bc304dd3SStephen Hemminger case NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE: 674bc304dd3SStephen Hemminger netvsc_send_tx_complete(net_device, incoming_channel, 675f9645430Sstephen hemminger device, desc, budget); 676bc304dd3SStephen Hemminger break; 677bc304dd3SStephen Hemminger 678bc304dd3SStephen Hemminger default: 679bc304dd3SStephen Hemminger netdev_err(ndev, 680bc304dd3SStephen Hemminger "Unknown send completion type %d received!!\n", 681bc304dd3SStephen Hemminger nvsp_packet->hdr.msg_type); 68295fa0405SHaiyang Zhang } 68395fa0405SHaiyang Zhang } 68495fa0405SHaiyang Zhang 685c25aaf81SKY Srinivasan static u32 netvsc_get_next_send_section(struct netvsc_device *net_device) 686c25aaf81SKY Srinivasan { 687b58a1858Sstephen hemminger unsigned long *map_addr = net_device->send_section_map; 688b58a1858Sstephen hemminger unsigned int i; 689c25aaf81SKY Srinivasan 690fdfb70d2Sstephen hemminger for_each_clear_bit(i, map_addr, net_device->send_section_cnt) { 691b58a1858Sstephen hemminger if (sync_test_and_set_bit(i, map_addr) == 0) 692b58a1858Sstephen hemminger return i; 693c25aaf81SKY Srinivasan } 694b58a1858Sstephen hemminger 695b58a1858Sstephen hemminger return NETVSC_INVALID_INDEX; 696c25aaf81SKY Srinivasan } 697c25aaf81SKY Srinivasan 698da19fcd0SLad, Prabhakar static u32 netvsc_copy_to_send_buf(struct netvsc_device *net_device, 699c25aaf81SKY Srinivasan unsigned int section_index, 7007c3877f2SHaiyang Zhang u32 pend_size, 70124476760SKY Srinivasan struct hv_netvsc_packet *packet, 702a9f2e2d6SKY Srinivasan struct rndis_message *rndis_msg, 70302b6de01Sstephen hemminger struct hv_page_buffer *pb, 704694a9fb0SKY Srinivasan struct sk_buff *skb) 705c25aaf81SKY Srinivasan { 706c25aaf81SKY Srinivasan char *start = net_device->send_buf; 7077c3877f2SHaiyang Zhang char *dest = start + (section_index * net_device->send_section_size) 7087c3877f2SHaiyang Zhang + pend_size; 709c25aaf81SKY Srinivasan int i; 710c25aaf81SKY Srinivasan u32 msg_size = 0; 7117c3877f2SHaiyang Zhang u32 padding = 0; 7127c3877f2SHaiyang Zhang u32 remain = packet->total_data_buflen % net_device->pkt_align; 713aa0a34beSHaiyang Zhang u32 page_count = packet->cp_partial ? packet->rmsg_pgcnt : 714aa0a34beSHaiyang Zhang packet->page_buf_cnt; 7157c3877f2SHaiyang Zhang 7167c3877f2SHaiyang Zhang /* Add padding */ 717ebc1dcf6Sstephen hemminger if (skb->xmit_more && remain && !packet->cp_partial) { 7187c3877f2SHaiyang Zhang padding = net_device->pkt_align - remain; 71924476760SKY Srinivasan rndis_msg->msg_len += padding; 7207c3877f2SHaiyang Zhang packet->total_data_buflen += padding; 7217c3877f2SHaiyang Zhang } 722c25aaf81SKY Srinivasan 723aa0a34beSHaiyang Zhang for (i = 0; i < page_count; i++) { 72402b6de01Sstephen hemminger char *src = phys_to_virt(pb[i].pfn << PAGE_SHIFT); 72502b6de01Sstephen hemminger u32 offset = pb[i].offset; 72602b6de01Sstephen hemminger u32 len = pb[i].len; 727c25aaf81SKY Srinivasan 728c25aaf81SKY Srinivasan memcpy(dest, (src + offset), len); 729c25aaf81SKY Srinivasan msg_size += len; 730c25aaf81SKY Srinivasan dest += len; 731c25aaf81SKY Srinivasan } 7327c3877f2SHaiyang Zhang 7337c3877f2SHaiyang Zhang if (padding) { 7347c3877f2SHaiyang Zhang memset(dest, 0, padding); 7357c3877f2SHaiyang Zhang msg_size += padding; 7367c3877f2SHaiyang Zhang } 7377c3877f2SHaiyang Zhang 738c25aaf81SKY Srinivasan return msg_size; 739c25aaf81SKY Srinivasan } 740c25aaf81SKY Srinivasan 7413a8963acSStephen Hemminger static inline int netvsc_send_pkt( 7420a1275caSVitaly Kuznetsov struct hv_device *device, 7437c3877f2SHaiyang Zhang struct hv_netvsc_packet *packet, 744a9f2e2d6SKY Srinivasan struct netvsc_device *net_device, 74502b6de01Sstephen hemminger struct hv_page_buffer *pb, 7463a3d9a0aSKY Srinivasan struct sk_buff *skb) 74795fa0405SHaiyang Zhang { 7487c3877f2SHaiyang Zhang struct nvsp_message nvmsg; 749956a25c9SJoe Perches struct nvsp_1_message_send_rndis_packet * const rpkt = 750956a25c9SJoe Perches &nvmsg.msg.v1_msg.send_rndis_pkt; 751956a25c9SJoe Perches struct netvsc_channel * const nvchan = 752956a25c9SJoe Perches &net_device->chan_table[packet->q_idx]; 753b8b835a8Sstephen hemminger struct vmbus_channel *out_channel = nvchan->channel; 7540a1275caSVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 75509af87d1SSimon Xiao struct net_device_context *ndev_ctx = netdev_priv(ndev); 756b8b835a8Sstephen hemminger struct netdev_queue *txq = netdev_get_tx_queue(ndev, packet->q_idx); 7577c3877f2SHaiyang Zhang u64 req_id; 7587c3877f2SHaiyang Zhang int ret; 75982fa3c77SKY Srinivasan u32 ring_avail = hv_ringbuf_avail_percent(&out_channel->outbound); 760c25aaf81SKY Srinivasan 7617c3877f2SHaiyang Zhang nvmsg.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT; 762956a25c9SJoe Perches if (skb) 763956a25c9SJoe Perches rpkt->channel_type = 0; /* 0 is RMC_DATA */ 7647c3877f2SHaiyang Zhang else 765956a25c9SJoe Perches rpkt->channel_type = 1; /* 1 is RMC_CONTROL */ 766956a25c9SJoe Perches 767956a25c9SJoe Perches rpkt->send_buf_section_index = packet->send_buf_index; 768956a25c9SJoe Perches if (packet->send_buf_index == NETVSC_INVALID_INDEX) 769956a25c9SJoe Perches rpkt->send_buf_section_size = 0; 770956a25c9SJoe Perches else 771956a25c9SJoe Perches rpkt->send_buf_section_size = packet->total_data_buflen; 77295fa0405SHaiyang Zhang 7733a3d9a0aSKY Srinivasan req_id = (ulong)skb; 774f1ea3cd7SHaiyang Zhang 775c3582a2cSHaiyang Zhang if (out_channel->rescind) 776c3582a2cSHaiyang Zhang return -ENODEV; 777c3582a2cSHaiyang Zhang 77895fa0405SHaiyang Zhang if (packet->page_buf_cnt) { 77902b6de01Sstephen hemminger if (packet->cp_partial) 78002b6de01Sstephen hemminger pb += packet->rmsg_pgcnt; 78102b6de01Sstephen hemminger 7825a668d8cSstephen hemminger ret = vmbus_sendpacket_pagebuffer(out_channel, 78302b6de01Sstephen hemminger pb, packet->page_buf_cnt, 7845a668d8cSstephen hemminger &nvmsg, sizeof(nvmsg), 7855a668d8cSstephen hemminger req_id); 78695fa0405SHaiyang Zhang } else { 7875dd0fb9bSstephen hemminger ret = vmbus_sendpacket(out_channel, 7885dd0fb9bSstephen hemminger &nvmsg, sizeof(nvmsg), 7895dd0fb9bSstephen hemminger req_id, VM_PKT_DATA_INBAND, 7903454323cSStephen Hemminger VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 79195fa0405SHaiyang Zhang } 79295fa0405SHaiyang Zhang 7931d06825bSHaiyang Zhang if (ret == 0) { 794b8b835a8Sstephen hemminger atomic_inc_return(&nvchan->queue_sends); 7955b54dac8SHaiyang Zhang 79609af87d1SSimon Xiao if (ring_avail < RING_AVAIL_PERCENT_LOWATER) { 797b8b835a8Sstephen hemminger netif_tx_stop_queue(txq); 79809af87d1SSimon Xiao ndev_ctx->eth_stats.stop_queue++; 79909af87d1SSimon Xiao } 8001d06825bSHaiyang Zhang } else if (ret == -EAGAIN) { 801b8b835a8Sstephen hemminger netif_tx_stop_queue(txq); 80209af87d1SSimon Xiao ndev_ctx->eth_stats.stop_queue++; 803b8b835a8Sstephen hemminger if (atomic_read(&nvchan->queue_sends) < 1) { 804b8b835a8Sstephen hemminger netif_tx_wake_queue(txq); 80509af87d1SSimon Xiao ndev_ctx->eth_stats.wake_queue++; 80633be96e4SHaiyang Zhang ret = -ENOSPC; 80733be96e4SHaiyang Zhang } 8081d06825bSHaiyang Zhang } else { 8094a2176c6Sstephen hemminger netdev_err(ndev, 8104a2176c6Sstephen hemminger "Unable to send packet pages %u len %u, ret %d\n", 8114a2176c6Sstephen hemminger packet->page_buf_cnt, packet->total_data_buflen, 8124a2176c6Sstephen hemminger ret); 8131d06825bSHaiyang Zhang } 81495fa0405SHaiyang Zhang 8157c3877f2SHaiyang Zhang return ret; 8167c3877f2SHaiyang Zhang } 8177c3877f2SHaiyang Zhang 818c85e4924SHaiyang Zhang /* Move packet out of multi send data (msd), and clear msd */ 819c85e4924SHaiyang Zhang static inline void move_pkt_msd(struct hv_netvsc_packet **msd_send, 820c85e4924SHaiyang Zhang struct sk_buff **msd_skb, 821c85e4924SHaiyang Zhang struct multi_send_data *msdp) 822c85e4924SHaiyang Zhang { 823c85e4924SHaiyang Zhang *msd_skb = msdp->skb; 824c85e4924SHaiyang Zhang *msd_send = msdp->pkt; 825c85e4924SHaiyang Zhang msdp->skb = NULL; 826c85e4924SHaiyang Zhang msdp->pkt = NULL; 827c85e4924SHaiyang Zhang msdp->count = 0; 828c85e4924SHaiyang Zhang } 829c85e4924SHaiyang Zhang 8302a926f79Sstephen hemminger /* RCU already held by caller */ 8312a926f79Sstephen hemminger int netvsc_send(struct net_device_context *ndev_ctx, 83224476760SKY Srinivasan struct hv_netvsc_packet *packet, 833a9f2e2d6SKY Srinivasan struct rndis_message *rndis_msg, 83402b6de01Sstephen hemminger struct hv_page_buffer *pb, 8353a3d9a0aSKY Srinivasan struct sk_buff *skb) 8367c3877f2SHaiyang Zhang { 8373962981fSstephen hemminger struct netvsc_device *net_device 838867047c4Sstephen hemminger = rcu_dereference_bh(ndev_ctx->nvdev); 8392a926f79Sstephen hemminger struct hv_device *device = ndev_ctx->device_ctx; 8406c4c137eSStephen Hemminger int ret = 0; 841b8b835a8Sstephen hemminger struct netvsc_channel *nvchan; 8427c3877f2SHaiyang Zhang u32 pktlen = packet->total_data_buflen, msd_len = 0; 8437c3877f2SHaiyang Zhang unsigned int section_index = NETVSC_INVALID_INDEX; 8447c3877f2SHaiyang Zhang struct multi_send_data *msdp; 8457c3877f2SHaiyang Zhang struct hv_netvsc_packet *msd_send = NULL, *cur_send = NULL; 846c85e4924SHaiyang Zhang struct sk_buff *msd_skb = NULL; 847aa0a34beSHaiyang Zhang bool try_batch; 848bde79be5SKY Srinivasan bool xmit_more = (skb != NULL) ? skb->xmit_more : false; 8497c3877f2SHaiyang Zhang 850592b4fe8Sstephen hemminger /* If device is rescinded, return error and packet will get dropped. */ 8512a926f79Sstephen hemminger if (unlikely(!net_device || net_device->destroy)) 8527c3877f2SHaiyang Zhang return -ENODEV; 8537c3877f2SHaiyang Zhang 854e8f0a89cSVitaly Kuznetsov /* We may race with netvsc_connect_vsp()/netvsc_init_buf() and get 855e8f0a89cSVitaly Kuznetsov * here before the negotiation with the host is finished and 856e8f0a89cSVitaly Kuznetsov * send_section_map may not be allocated yet. 857e8f0a89cSVitaly Kuznetsov */ 8582d694d2aSstephen hemminger if (unlikely(!net_device->send_section_map)) 859e8f0a89cSVitaly Kuznetsov return -EAGAIN; 860e8f0a89cSVitaly Kuznetsov 861b8b835a8Sstephen hemminger nvchan = &net_device->chan_table[packet->q_idx]; 8627c3877f2SHaiyang Zhang packet->send_buf_index = NETVSC_INVALID_INDEX; 863aa0a34beSHaiyang Zhang packet->cp_partial = false; 8647c3877f2SHaiyang Zhang 865cf8190e4SHaiyang Zhang /* Send control message directly without accessing msd (Multi-Send 866cf8190e4SHaiyang Zhang * Data) field which may be changed during data packet processing. 867cf8190e4SHaiyang Zhang */ 868cf8190e4SHaiyang Zhang if (!skb) { 869cf8190e4SHaiyang Zhang cur_send = packet; 870cf8190e4SHaiyang Zhang goto send_now; 871cf8190e4SHaiyang Zhang } 872cf8190e4SHaiyang Zhang 8737c3877f2SHaiyang Zhang /* batch packets in send buffer if possible */ 874b8b835a8Sstephen hemminger msdp = &nvchan->msd; 8757c3877f2SHaiyang Zhang if (msdp->pkt) 8767c3877f2SHaiyang Zhang msd_len = msdp->pkt->total_data_buflen; 8777c3877f2SHaiyang Zhang 878ebc1dcf6Sstephen hemminger try_batch = msd_len > 0 && msdp->count < net_device->max_pkt; 879aa0a34beSHaiyang Zhang if (try_batch && msd_len + pktlen + net_device->pkt_align < 8807c3877f2SHaiyang Zhang net_device->send_section_size) { 8817c3877f2SHaiyang Zhang section_index = msdp->pkt->send_buf_index; 8827c3877f2SHaiyang Zhang 883aa0a34beSHaiyang Zhang } else if (try_batch && msd_len + packet->rmsg_size < 884aa0a34beSHaiyang Zhang net_device->send_section_size) { 885aa0a34beSHaiyang Zhang section_index = msdp->pkt->send_buf_index; 886aa0a34beSHaiyang Zhang packet->cp_partial = true; 887aa0a34beSHaiyang Zhang 888ebc1dcf6Sstephen hemminger } else if (pktlen + net_device->pkt_align < 8897c3877f2SHaiyang Zhang net_device->send_section_size) { 8907c3877f2SHaiyang Zhang section_index = netvsc_get_next_send_section(net_device); 891cad5c197Sstephen hemminger if (unlikely(section_index == NETVSC_INVALID_INDEX)) { 892cad5c197Sstephen hemminger ++ndev_ctx->eth_stats.tx_send_full; 893cad5c197Sstephen hemminger } else { 894c85e4924SHaiyang Zhang move_pkt_msd(&msd_send, &msd_skb, msdp); 8957c3877f2SHaiyang Zhang msd_len = 0; 8967c3877f2SHaiyang Zhang } 8977c3877f2SHaiyang Zhang } 8987c3877f2SHaiyang Zhang 8997c3877f2SHaiyang Zhang if (section_index != NETVSC_INVALID_INDEX) { 9007c3877f2SHaiyang Zhang netvsc_copy_to_send_buf(net_device, 9017c3877f2SHaiyang Zhang section_index, msd_len, 902694a9fb0SKY Srinivasan packet, rndis_msg, pb, skb); 903b08cc791SKY Srinivasan 904aa0a34beSHaiyang Zhang packet->send_buf_index = section_index; 905aa0a34beSHaiyang Zhang 906aa0a34beSHaiyang Zhang if (packet->cp_partial) { 907aa0a34beSHaiyang Zhang packet->page_buf_cnt -= packet->rmsg_pgcnt; 908aa0a34beSHaiyang Zhang packet->total_data_buflen = msd_len + packet->rmsg_size; 909aa0a34beSHaiyang Zhang } else { 910aa0a34beSHaiyang Zhang packet->page_buf_cnt = 0; 911aa0a34beSHaiyang Zhang packet->total_data_buflen += msd_len; 912aa0a34beSHaiyang Zhang } 9137c3877f2SHaiyang Zhang 914793e3955Sstephen hemminger if (msdp->pkt) { 915793e3955Sstephen hemminger packet->total_packets += msdp->pkt->total_packets; 916793e3955Sstephen hemminger packet->total_bytes += msdp->pkt->total_bytes; 917793e3955Sstephen hemminger } 918793e3955Sstephen hemminger 919c85e4924SHaiyang Zhang if (msdp->skb) 92017db4bceSStephen Hemminger dev_consume_skb_any(msdp->skb); 921ee90b812SHaiyang Zhang 922bde79be5SKY Srinivasan if (xmit_more && !packet->cp_partial) { 923c85e4924SHaiyang Zhang msdp->skb = skb; 9247c3877f2SHaiyang Zhang msdp->pkt = packet; 9257c3877f2SHaiyang Zhang msdp->count++; 9267c3877f2SHaiyang Zhang } else { 9277c3877f2SHaiyang Zhang cur_send = packet; 928c85e4924SHaiyang Zhang msdp->skb = NULL; 9297c3877f2SHaiyang Zhang msdp->pkt = NULL; 9307c3877f2SHaiyang Zhang msdp->count = 0; 9317c3877f2SHaiyang Zhang } 9327c3877f2SHaiyang Zhang } else { 933c85e4924SHaiyang Zhang move_pkt_msd(&msd_send, &msd_skb, msdp); 9347c3877f2SHaiyang Zhang cur_send = packet; 9357c3877f2SHaiyang Zhang } 9367c3877f2SHaiyang Zhang 9377c3877f2SHaiyang Zhang if (msd_send) { 9386c4c137eSStephen Hemminger int m_ret = netvsc_send_pkt(device, msd_send, net_device, 9390a1275caSVitaly Kuznetsov NULL, msd_skb); 9407c3877f2SHaiyang Zhang 9417c3877f2SHaiyang Zhang if (m_ret != 0) { 9427c3877f2SHaiyang Zhang netvsc_free_send_slot(net_device, 9437c3877f2SHaiyang Zhang msd_send->send_buf_index); 944c85e4924SHaiyang Zhang dev_kfree_skb_any(msd_skb); 9457c3877f2SHaiyang Zhang } 9467c3877f2SHaiyang Zhang } 9477c3877f2SHaiyang Zhang 948cf8190e4SHaiyang Zhang send_now: 9497c3877f2SHaiyang Zhang if (cur_send) 9500a1275caSVitaly Kuznetsov ret = netvsc_send_pkt(device, cur_send, net_device, pb, skb); 9517c3877f2SHaiyang Zhang 9527aab5159SJerry Snitselaar if (ret != 0 && section_index != NETVSC_INVALID_INDEX) 953d953ca4dSHaiyang Zhang netvsc_free_send_slot(net_device, section_index); 954d953ca4dSHaiyang Zhang 95595fa0405SHaiyang Zhang return ret; 95695fa0405SHaiyang Zhang } 95795fa0405SHaiyang Zhang 9587426b1a5Sstephen hemminger /* Send pending recv completions */ 959cad5c197Sstephen hemminger static int send_recv_completions(struct net_device *ndev, 960cad5c197Sstephen hemminger struct netvsc_device *nvdev, 961cad5c197Sstephen hemminger struct netvsc_channel *nvchan) 96295fa0405SHaiyang Zhang { 9637426b1a5Sstephen hemminger struct multi_recv_comp *mrc = &nvchan->mrc; 9647426b1a5Sstephen hemminger struct recv_comp_msg { 9657426b1a5Sstephen hemminger struct nvsp_message_header hdr; 9667426b1a5Sstephen hemminger u32 status; 9677426b1a5Sstephen hemminger } __packed; 9687426b1a5Sstephen hemminger struct recv_comp_msg msg = { 9697426b1a5Sstephen hemminger .hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE, 9707426b1a5Sstephen hemminger }; 97195fa0405SHaiyang Zhang int ret; 97295fa0405SHaiyang Zhang 9737426b1a5Sstephen hemminger while (mrc->first != mrc->next) { 9747426b1a5Sstephen hemminger const struct recv_comp_data *rcd 9757426b1a5Sstephen hemminger = mrc->slots + mrc->first; 97695fa0405SHaiyang Zhang 9777426b1a5Sstephen hemminger msg.status = rcd->status; 9787426b1a5Sstephen hemminger ret = vmbus_sendpacket(nvchan->channel, &msg, sizeof(msg), 9797426b1a5Sstephen hemminger rcd->tid, VM_PKT_COMP, 0); 980cad5c197Sstephen hemminger if (unlikely(ret)) { 981cad5c197Sstephen hemminger struct net_device_context *ndev_ctx = netdev_priv(ndev); 982cad5c197Sstephen hemminger 983cad5c197Sstephen hemminger ++ndev_ctx->eth_stats.rx_comp_busy; 984c0b558e5SHaiyang Zhang return ret; 985cad5c197Sstephen hemminger } 9867426b1a5Sstephen hemminger 9877426b1a5Sstephen hemminger if (++mrc->first == nvdev->recv_completion_cnt) 9887426b1a5Sstephen hemminger mrc->first = 0; 98995fa0405SHaiyang Zhang } 990c0b558e5SHaiyang Zhang 9917426b1a5Sstephen hemminger /* receive completion ring has been emptied */ 9927426b1a5Sstephen hemminger if (unlikely(nvdev->destroy)) 9937426b1a5Sstephen hemminger wake_up(&nvdev->wait_drain); 9947426b1a5Sstephen hemminger 9957426b1a5Sstephen hemminger return 0; 9967426b1a5Sstephen hemminger } 9977426b1a5Sstephen hemminger 9987426b1a5Sstephen hemminger /* Count how many receive completions are outstanding */ 9997426b1a5Sstephen hemminger static void recv_comp_slot_avail(const struct netvsc_device *nvdev, 10007426b1a5Sstephen hemminger const struct multi_recv_comp *mrc, 1001c0b558e5SHaiyang Zhang u32 *filled, u32 *avail) 1002c0b558e5SHaiyang Zhang { 10037426b1a5Sstephen hemminger u32 count = nvdev->recv_completion_cnt; 1004c0b558e5SHaiyang Zhang 10057426b1a5Sstephen hemminger if (mrc->next >= mrc->first) 10067426b1a5Sstephen hemminger *filled = mrc->next - mrc->first; 10077426b1a5Sstephen hemminger else 10087426b1a5Sstephen hemminger *filled = (count - mrc->first) + mrc->next; 1009c0b558e5SHaiyang Zhang 10107426b1a5Sstephen hemminger *avail = count - *filled - 1; 101195fa0405SHaiyang Zhang } 1012c0b558e5SHaiyang Zhang 10137426b1a5Sstephen hemminger /* Add receive complete to ring to send to host. */ 10147426b1a5Sstephen hemminger static void enq_receive_complete(struct net_device *ndev, 10157426b1a5Sstephen hemminger struct netvsc_device *nvdev, u16 q_idx, 10167426b1a5Sstephen hemminger u64 tid, u32 status) 1017c0b558e5SHaiyang Zhang { 10187426b1a5Sstephen hemminger struct netvsc_channel *nvchan = &nvdev->chan_table[q_idx]; 10197426b1a5Sstephen hemminger struct multi_recv_comp *mrc = &nvchan->mrc; 10207426b1a5Sstephen hemminger struct recv_comp_data *rcd; 1021c0b558e5SHaiyang Zhang u32 filled, avail; 1022c0b558e5SHaiyang Zhang 10237426b1a5Sstephen hemminger recv_comp_slot_avail(nvdev, mrc, &filled, &avail); 1024c0b558e5SHaiyang Zhang 10257426b1a5Sstephen hemminger if (unlikely(filled > NAPI_POLL_WEIGHT)) { 1026cad5c197Sstephen hemminger send_recv_completions(ndev, nvdev, nvchan); 10277426b1a5Sstephen hemminger recv_comp_slot_avail(nvdev, mrc, &filled, &avail); 1028c0b558e5SHaiyang Zhang } 1029c0b558e5SHaiyang Zhang 10307426b1a5Sstephen hemminger if (unlikely(!avail)) { 10317426b1a5Sstephen hemminger netdev_err(ndev, "Recv_comp full buf q:%hd, tid:%llx\n", 10327426b1a5Sstephen hemminger q_idx, tid); 10337426b1a5Sstephen hemminger return; 1034c0b558e5SHaiyang Zhang } 1035c0b558e5SHaiyang Zhang 10367426b1a5Sstephen hemminger rcd = mrc->slots + mrc->next; 10377426b1a5Sstephen hemminger rcd->tid = tid; 10387426b1a5Sstephen hemminger rcd->status = status; 1039c0b558e5SHaiyang Zhang 10407426b1a5Sstephen hemminger if (++mrc->next == nvdev->recv_completion_cnt) 10417426b1a5Sstephen hemminger mrc->next = 0; 104295fa0405SHaiyang Zhang } 104395fa0405SHaiyang Zhang 104415a863bfSstephen hemminger static int netvsc_receive(struct net_device *ndev, 1045dc54a08cSstephen hemminger struct netvsc_device *net_device, 1046dc54a08cSstephen hemminger struct net_device_context *net_device_ctx, 104797c1723aSKY Srinivasan struct hv_device *device, 1048dc54a08cSstephen hemminger struct vmbus_channel *channel, 1049f3dd3f47Sstephen hemminger const struct vmpacket_descriptor *desc, 1050dc54a08cSstephen hemminger struct nvsp_message *nvsp) 105195fa0405SHaiyang Zhang { 1052f3dd3f47Sstephen hemminger const struct vmtransfer_page_packet_header *vmxferpage_packet 1053f3dd3f47Sstephen hemminger = container_of(desc, const struct vmtransfer_page_packet_header, d); 105415a863bfSstephen hemminger u16 q_idx = channel->offermsg.offer.sub_channel_index; 1055dc54a08cSstephen hemminger char *recv_buf = net_device->recv_buf; 10564baab261SHaiyang Zhang u32 status = NVSP_STAT_SUCCESS; 105745326342SHaiyang Zhang int i; 105845326342SHaiyang Zhang int count = 0; 105995fa0405SHaiyang Zhang 106095fa0405SHaiyang Zhang /* Make sure this is a valid nvsp packet */ 1061dc54a08cSstephen hemminger if (unlikely(nvsp->hdr.msg_type != NVSP_MSG1_TYPE_SEND_RNDIS_PKT)) { 1062dc54a08cSstephen hemminger netif_err(net_device_ctx, rx_err, ndev, 1063dc54a08cSstephen hemminger "Unknown nvsp packet type received %u\n", 1064dc54a08cSstephen hemminger nvsp->hdr.msg_type); 106515a863bfSstephen hemminger return 0; 106695fa0405SHaiyang Zhang } 106795fa0405SHaiyang Zhang 1068dc54a08cSstephen hemminger if (unlikely(vmxferpage_packet->xfer_pageset_id != NETVSC_RECEIVE_BUFFER_ID)) { 1069dc54a08cSstephen hemminger netif_err(net_device_ctx, rx_err, ndev, 1070dc54a08cSstephen hemminger "Invalid xfer page set id - expecting %x got %x\n", 1071dc54a08cSstephen hemminger NETVSC_RECEIVE_BUFFER_ID, 107295fa0405SHaiyang Zhang vmxferpage_packet->xfer_pageset_id); 107315a863bfSstephen hemminger return 0; 107495fa0405SHaiyang Zhang } 107595fa0405SHaiyang Zhang 10764baab261SHaiyang Zhang count = vmxferpage_packet->range_cnt; 107795fa0405SHaiyang Zhang 107895fa0405SHaiyang Zhang /* Each range represents 1 RNDIS pkt that contains 1 ethernet frame */ 10794baab261SHaiyang Zhang for (i = 0; i < count; i++) { 1080dc54a08cSstephen hemminger void *data = recv_buf 1081dc54a08cSstephen hemminger + vmxferpage_packet->ranges[i].byte_offset; 1082dc54a08cSstephen hemminger u32 buflen = vmxferpage_packet->ranges[i].byte_count; 108395fa0405SHaiyang Zhang 108495fa0405SHaiyang Zhang /* Pass it to the upper layer */ 1085dc54a08cSstephen hemminger status = rndis_filter_receive(ndev, net_device, device, 1086dc54a08cSstephen hemminger channel, data, buflen); 108795fa0405SHaiyang Zhang } 108895fa0405SHaiyang Zhang 10897426b1a5Sstephen hemminger enq_receive_complete(ndev, net_device, q_idx, 10907426b1a5Sstephen hemminger vmxferpage_packet->d.trans_id, status); 109115a863bfSstephen hemminger 109215a863bfSstephen hemminger return count; 109395fa0405SHaiyang Zhang } 109495fa0405SHaiyang Zhang 10955b54dac8SHaiyang Zhang static void netvsc_send_table(struct hv_device *hdev, 109671790a27SHaiyang Zhang struct nvsp_message *nvmsg) 10975b54dac8SHaiyang Zhang { 10980a1275caSVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(hdev); 10997ce10124Sstephen hemminger struct net_device_context *net_device_ctx = netdev_priv(ndev); 11005b54dac8SHaiyang Zhang int i; 11015b54dac8SHaiyang Zhang u32 count, *tab; 11025b54dac8SHaiyang Zhang 11035b54dac8SHaiyang Zhang count = nvmsg->msg.v5_msg.send_table.count; 11045b54dac8SHaiyang Zhang if (count != VRSS_SEND_TAB_SIZE) { 11055b54dac8SHaiyang Zhang netdev_err(ndev, "Received wrong send-table size:%u\n", count); 11065b54dac8SHaiyang Zhang return; 11075b54dac8SHaiyang Zhang } 11085b54dac8SHaiyang Zhang 11095b54dac8SHaiyang Zhang tab = (u32 *)((unsigned long)&nvmsg->msg.v5_msg.send_table + 11105b54dac8SHaiyang Zhang nvmsg->msg.v5_msg.send_table.offset); 11115b54dac8SHaiyang Zhang 11125b54dac8SHaiyang Zhang for (i = 0; i < count; i++) 111339e91cfbSHaiyang Zhang net_device_ctx->tx_table[i] = tab[i]; 11145b54dac8SHaiyang Zhang } 11155b54dac8SHaiyang Zhang 1116f9a7da91SVitaly Kuznetsov static void netvsc_send_vf(struct net_device_context *net_device_ctx, 111771790a27SHaiyang Zhang struct nvsp_message *nvmsg) 111871790a27SHaiyang Zhang { 1119f9a7da91SVitaly Kuznetsov net_device_ctx->vf_alloc = nvmsg->msg.v4_msg.vf_assoc.allocated; 1120f9a7da91SVitaly Kuznetsov net_device_ctx->vf_serial = nvmsg->msg.v4_msg.vf_assoc.serial; 112171790a27SHaiyang Zhang } 112271790a27SHaiyang Zhang 112371790a27SHaiyang Zhang static inline void netvsc_receive_inband(struct hv_device *hdev, 1124f9a7da91SVitaly Kuznetsov struct net_device_context *net_device_ctx, 112571790a27SHaiyang Zhang struct nvsp_message *nvmsg) 112671790a27SHaiyang Zhang { 112771790a27SHaiyang Zhang switch (nvmsg->hdr.msg_type) { 112871790a27SHaiyang Zhang case NVSP_MSG5_TYPE_SEND_INDIRECTION_TABLE: 112971790a27SHaiyang Zhang netvsc_send_table(hdev, nvmsg); 113071790a27SHaiyang Zhang break; 113171790a27SHaiyang Zhang 113271790a27SHaiyang Zhang case NVSP_MSG4_TYPE_SEND_VF_ASSOCIATION: 1133f9a7da91SVitaly Kuznetsov netvsc_send_vf(net_device_ctx, nvmsg); 113471790a27SHaiyang Zhang break; 113571790a27SHaiyang Zhang } 113671790a27SHaiyang Zhang } 113771790a27SHaiyang Zhang 113815a863bfSstephen hemminger static int netvsc_process_raw_pkt(struct hv_device *device, 113999a50bb1SK. Y. Srinivasan struct vmbus_channel *channel, 114099a50bb1SK. Y. Srinivasan struct netvsc_device *net_device, 114199a50bb1SK. Y. Srinivasan struct net_device *ndev, 1142f9645430Sstephen hemminger const struct vmpacket_descriptor *desc, 1143f9645430Sstephen hemminger int budget) 114499a50bb1SK. Y. Srinivasan { 1145f9a7da91SVitaly Kuznetsov struct net_device_context *net_device_ctx = netdev_priv(ndev); 1146f3dd3f47Sstephen hemminger struct nvsp_message *nvmsg = hv_pkt_data(desc); 114799a50bb1SK. Y. Srinivasan 114899a50bb1SK. Y. Srinivasan switch (desc->type) { 114999a50bb1SK. Y. Srinivasan case VM_PKT_COMP: 1150f9645430Sstephen hemminger netvsc_send_completion(net_device, channel, device, 1151f9645430Sstephen hemminger desc, budget); 115299a50bb1SK. Y. Srinivasan break; 115399a50bb1SK. Y. Srinivasan 115499a50bb1SK. Y. Srinivasan case VM_PKT_DATA_USING_XFER_PAGES: 115515a863bfSstephen hemminger return netvsc_receive(ndev, net_device, net_device_ctx, 1156f3dd3f47Sstephen hemminger device, channel, desc, nvmsg); 115799a50bb1SK. Y. Srinivasan break; 115899a50bb1SK. Y. Srinivasan 115999a50bb1SK. Y. Srinivasan case VM_PKT_DATA_INBAND: 1160f9a7da91SVitaly Kuznetsov netvsc_receive_inband(device, net_device_ctx, nvmsg); 116199a50bb1SK. Y. Srinivasan break; 116299a50bb1SK. Y. Srinivasan 116399a50bb1SK. Y. Srinivasan default: 116499a50bb1SK. Y. Srinivasan netdev_err(ndev, "unhandled packet type %d, tid %llx\n", 1165f4f1c23dSstephen hemminger desc->type, desc->trans_id); 116699a50bb1SK. Y. Srinivasan break; 116799a50bb1SK. Y. Srinivasan } 116815a863bfSstephen hemminger 116915a863bfSstephen hemminger return 0; 117015a863bfSstephen hemminger } 117115a863bfSstephen hemminger 117215a863bfSstephen hemminger static struct hv_device *netvsc_channel_to_device(struct vmbus_channel *channel) 117315a863bfSstephen hemminger { 117415a863bfSstephen hemminger struct vmbus_channel *primary = channel->primary_channel; 117515a863bfSstephen hemminger 117615a863bfSstephen hemminger return primary ? primary->device_obj : channel->device_obj; 117715a863bfSstephen hemminger } 117815a863bfSstephen hemminger 1179262b7f14Sstephen hemminger /* Network processing softirq 1180262b7f14Sstephen hemminger * Process data in incoming ring buffer from host 1181262b7f14Sstephen hemminger * Stops when ring is empty or budget is met or exceeded. 1182262b7f14Sstephen hemminger */ 118315a863bfSstephen hemminger int netvsc_poll(struct napi_struct *napi, int budget) 118415a863bfSstephen hemminger { 118515a863bfSstephen hemminger struct netvsc_channel *nvchan 118615a863bfSstephen hemminger = container_of(napi, struct netvsc_channel, napi); 118735fbbccfSstephen hemminger struct netvsc_device *net_device = nvchan->net_device; 118815a863bfSstephen hemminger struct vmbus_channel *channel = nvchan->channel; 118915a863bfSstephen hemminger struct hv_device *device = netvsc_channel_to_device(channel); 119015a863bfSstephen hemminger struct net_device *ndev = hv_get_drvdata(device); 119115a863bfSstephen hemminger int work_done = 0; 119215a863bfSstephen hemminger 1193f4f1c23dSstephen hemminger /* If starting a new interval */ 1194f4f1c23dSstephen hemminger if (!nvchan->desc) 1195f4f1c23dSstephen hemminger nvchan->desc = hv_pkt_iter_first(channel); 119615a863bfSstephen hemminger 1197f4f1c23dSstephen hemminger while (nvchan->desc && work_done < budget) { 1198f4f1c23dSstephen hemminger work_done += netvsc_process_raw_pkt(device, channel, net_device, 1199f9645430Sstephen hemminger ndev, nvchan->desc, budget); 1200f4f1c23dSstephen hemminger nvchan->desc = hv_pkt_iter_next(channel, nvchan->desc); 120115a863bfSstephen hemminger } 120215a863bfSstephen hemminger 12037426b1a5Sstephen hemminger /* If send of pending receive completions suceeded 1204f4e40363Sstephen hemminger * and did not exhaust NAPI budget this time 1205f4f1c23dSstephen hemminger * and not doing busy poll 1206f4e40363Sstephen hemminger * then re-enable host interrupts 1207f4e40363Sstephen hemminger * and reschedule if ring is not empty. 1208262b7f14Sstephen hemminger */ 1209cad5c197Sstephen hemminger if (send_recv_completions(ndev, net_device, nvchan) == 0 && 12107426b1a5Sstephen hemminger work_done < budget && 121115a863bfSstephen hemminger napi_complete_done(napi, work_done) && 12127426b1a5Sstephen hemminger hv_end_read(&channel->inbound)) { 12137426b1a5Sstephen hemminger hv_begin_read(&channel->inbound); 121415a863bfSstephen hemminger napi_reschedule(napi); 12157426b1a5Sstephen hemminger } 1216f4f1c23dSstephen hemminger 1217f4f1c23dSstephen hemminger /* Driver may overshoot since multiple packets per descriptor */ 1218f4f1c23dSstephen hemminger return min(work_done, budget); 121999a50bb1SK. Y. Srinivasan } 122099a50bb1SK. Y. Srinivasan 1221262b7f14Sstephen hemminger /* Call back when data is available in host ring buffer. 1222262b7f14Sstephen hemminger * Processing is deferred until network softirq (NAPI) 1223262b7f14Sstephen hemminger */ 12245b54dac8SHaiyang Zhang void netvsc_channel_cb(void *context) 122595fa0405SHaiyang Zhang { 12266de38af6Sstephen hemminger struct netvsc_channel *nvchan = context; 122743bf99ceSstephen hemminger struct vmbus_channel *channel = nvchan->channel; 122843bf99ceSstephen hemminger struct hv_ring_buffer_info *rbi = &channel->inbound; 122943bf99ceSstephen hemminger 123043bf99ceSstephen hemminger /* preload first vmpacket descriptor */ 123143bf99ceSstephen hemminger prefetch(hv_get_ring_buffer(rbi) + rbi->priv_read_index); 123295fa0405SHaiyang Zhang 1233f4f1c23dSstephen hemminger if (napi_schedule_prep(&nvchan->napi)) { 123415a863bfSstephen hemminger /* disable interupts from host */ 123543bf99ceSstephen hemminger hv_begin_read(rbi); 12360d6dd357Sstephen hemminger 1237f4f1c23dSstephen hemminger __napi_schedule(&nvchan->napi); 1238f4f1c23dSstephen hemminger } 123995fa0405SHaiyang Zhang } 124095fa0405SHaiyang Zhang 124195fa0405SHaiyang Zhang /* 124295fa0405SHaiyang Zhang * netvsc_device_add - Callback when the device belonging to this 124395fa0405SHaiyang Zhang * driver is added 124495fa0405SHaiyang Zhang */ 12459749fed5Sstephen hemminger struct netvsc_device *netvsc_device_add(struct hv_device *device, 12462c7f83caSstephen hemminger const struct netvsc_device_info *device_info) 124795fa0405SHaiyang Zhang { 124888098834SVitaly Kuznetsov int i, ret = 0; 12492c7f83caSstephen hemminger int ring_size = device_info->ring_size; 125095fa0405SHaiyang Zhang struct netvsc_device *net_device; 125188098834SVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 125288098834SVitaly Kuznetsov struct net_device_context *net_device_ctx = netdev_priv(ndev); 125395fa0405SHaiyang Zhang 125488098834SVitaly Kuznetsov net_device = alloc_net_device(); 1255b1c84927SDan Carpenter if (!net_device) 12569749fed5Sstephen hemminger return ERR_PTR(-ENOMEM); 125795fa0405SHaiyang Zhang 12586b0cbe31SHaiyang Zhang for (i = 0; i < VRSS_SEND_TAB_SIZE; i++) 12596b0cbe31SHaiyang Zhang net_device_ctx->tx_table[i] = 0; 12606b0cbe31SHaiyang Zhang 12615b54dac8SHaiyang Zhang net_device->ring_size = ring_size; 12625b54dac8SHaiyang Zhang 126315a863bfSstephen hemminger /* Because the device uses NAPI, all the interrupt batching and 126415a863bfSstephen hemminger * control is done via Net softirq, not the channel handling 126515a863bfSstephen hemminger */ 126615a863bfSstephen hemminger set_channel_read_mode(device->channel, HV_CALL_ISR); 126715a863bfSstephen hemminger 1268bffb1842SK. Y. Srinivasan /* If we're reopening the device we may have multiple queues, fill the 1269bffb1842SK. Y. Srinivasan * chn_table with the default channel to use it before subchannels are 1270bffb1842SK. Y. Srinivasan * opened. 1271bffb1842SK. Y. Srinivasan * Initialize the channel state before we open; 1272bffb1842SK. Y. Srinivasan * we can be interrupted as soon as we open the channel. 1273bffb1842SK. Y. Srinivasan */ 1274bffb1842SK. Y. Srinivasan 1275bffb1842SK. Y. Srinivasan for (i = 0; i < VRSS_CHANNEL_MAX; i++) { 1276bffb1842SK. Y. Srinivasan struct netvsc_channel *nvchan = &net_device->chan_table[i]; 1277bffb1842SK. Y. Srinivasan 1278bffb1842SK. Y. Srinivasan nvchan->channel = device->channel; 127935fbbccfSstephen hemminger nvchan->net_device = net_device; 12804a0dee1fSFlorian Fainelli u64_stats_init(&nvchan->tx_stats.syncp); 12814a0dee1fSFlorian Fainelli u64_stats_init(&nvchan->rx_stats.syncp); 1282bffb1842SK. Y. Srinivasan } 1283bffb1842SK. Y. Srinivasan 12842be0f264Sstephen hemminger /* Enable NAPI handler before init callbacks */ 12852be0f264Sstephen hemminger netif_napi_add(ndev, &net_device->chan_table[0].napi, 12862be0f264Sstephen hemminger netvsc_poll, NAPI_POLL_WEIGHT); 12872be0f264Sstephen hemminger 128895fa0405SHaiyang Zhang /* Open the channel */ 128995fa0405SHaiyang Zhang ret = vmbus_open(device->channel, ring_size * PAGE_SIZE, 129095fa0405SHaiyang Zhang ring_size * PAGE_SIZE, NULL, 0, 12916de38af6Sstephen hemminger netvsc_channel_cb, 12926de38af6Sstephen hemminger net_device->chan_table); 129395fa0405SHaiyang Zhang 129495fa0405SHaiyang Zhang if (ret != 0) { 12952be0f264Sstephen hemminger netif_napi_del(&net_device->chan_table[0].napi); 129695fa0405SHaiyang Zhang netdev_err(ndev, "unable to open channel: %d\n", ret); 129795fa0405SHaiyang Zhang goto cleanup; 129895fa0405SHaiyang Zhang } 129995fa0405SHaiyang Zhang 130095fa0405SHaiyang Zhang /* Channel is opened */ 130193ba2222SVitaly Kuznetsov netdev_dbg(ndev, "hv_netvsc channel opened successfully\n"); 130295fa0405SHaiyang Zhang 130315a863bfSstephen hemminger napi_enable(&net_device->chan_table[0].napi); 130488098834SVitaly Kuznetsov 130588098834SVitaly Kuznetsov /* Writing nvdev pointer unlocks netvsc_send(), make sure chn_table is 130688098834SVitaly Kuznetsov * populated. 130788098834SVitaly Kuznetsov */ 1308545a8e79Sstephen hemminger rcu_assign_pointer(net_device_ctx->nvdev, net_device); 13095b54dac8SHaiyang Zhang 131095fa0405SHaiyang Zhang /* Connect with the NetVsp */ 13118b532797Sstephen hemminger ret = netvsc_connect_vsp(device, net_device, device_info); 131295fa0405SHaiyang Zhang if (ret != 0) { 131395fa0405SHaiyang Zhang netdev_err(ndev, 131495fa0405SHaiyang Zhang "unable to connect to NetVSP - %d\n", ret); 131595fa0405SHaiyang Zhang goto close; 131695fa0405SHaiyang Zhang } 131795fa0405SHaiyang Zhang 13189749fed5Sstephen hemminger return net_device; 131995fa0405SHaiyang Zhang 132095fa0405SHaiyang Zhang close: 132149393347Sstephen hemminger RCU_INIT_POINTER(net_device_ctx->nvdev, NULL); 132249393347Sstephen hemminger napi_disable(&net_device->chan_table[0].napi); 132315a863bfSstephen hemminger 132495fa0405SHaiyang Zhang /* Now, we can close the channel safely */ 132595fa0405SHaiyang Zhang vmbus_close(device->channel); 132695fa0405SHaiyang Zhang 132795fa0405SHaiyang Zhang cleanup: 1328545a8e79Sstephen hemminger free_netvsc_device(&net_device->rcu); 132995fa0405SHaiyang Zhang 13309749fed5Sstephen hemminger return ERR_PTR(ret); 133195fa0405SHaiyang Zhang } 1332