195fa0405SHaiyang Zhang /* 295fa0405SHaiyang Zhang * Copyright (c) 2009, Microsoft Corporation. 395fa0405SHaiyang Zhang * 495fa0405SHaiyang Zhang * This program is free software; you can redistribute it and/or modify it 595fa0405SHaiyang Zhang * under the terms and conditions of the GNU General Public License, 695fa0405SHaiyang Zhang * version 2, as published by the Free Software Foundation. 795fa0405SHaiyang Zhang * 895fa0405SHaiyang Zhang * This program is distributed in the hope it will be useful, but WITHOUT 995fa0405SHaiyang Zhang * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 1095fa0405SHaiyang Zhang * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 1195fa0405SHaiyang Zhang * more details. 1295fa0405SHaiyang Zhang * 1395fa0405SHaiyang Zhang * You should have received a copy of the GNU General Public License along with 14adf8d3ffSJeff Kirsher * this program; if not, see <http://www.gnu.org/licenses/>. 1595fa0405SHaiyang Zhang * 1695fa0405SHaiyang Zhang * Authors: 1795fa0405SHaiyang Zhang * Haiyang Zhang <haiyangz@microsoft.com> 1895fa0405SHaiyang Zhang * Hank Janssen <hjanssen@microsoft.com> 1995fa0405SHaiyang Zhang */ 2095fa0405SHaiyang Zhang #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 2195fa0405SHaiyang Zhang 2295fa0405SHaiyang Zhang #include <linux/kernel.h> 2395fa0405SHaiyang Zhang #include <linux/sched.h> 2495fa0405SHaiyang Zhang #include <linux/wait.h> 2595fa0405SHaiyang Zhang #include <linux/mm.h> 2695fa0405SHaiyang Zhang #include <linux/delay.h> 2795fa0405SHaiyang Zhang #include <linux/io.h> 2895fa0405SHaiyang Zhang #include <linux/slab.h> 2995fa0405SHaiyang Zhang #include <linux/netdevice.h> 30f157e78dSHaiyang Zhang #include <linux/if_ether.h> 31d6472302SStephen Rothwell #include <linux/vmalloc.h> 32c25aaf81SKY Srinivasan #include <asm/sync_bitops.h> 3395fa0405SHaiyang Zhang 3495fa0405SHaiyang Zhang #include "hyperv_net.h" 3595fa0405SHaiyang Zhang 3684bf9cefSKY Srinivasan /* 3784bf9cefSKY Srinivasan * Switch the data path from the synthetic interface to the VF 3884bf9cefSKY Srinivasan * interface. 3984bf9cefSKY Srinivasan */ 400a1275caSVitaly Kuznetsov void netvsc_switch_datapath(struct net_device *ndev, bool vf) 4184bf9cefSKY Srinivasan { 423d541ac5SVitaly Kuznetsov struct net_device_context *net_device_ctx = netdev_priv(ndev); 433d541ac5SVitaly Kuznetsov struct hv_device *dev = net_device_ctx->device_ctx; 440a1275caSVitaly Kuznetsov struct netvsc_device *nv_dev = net_device_ctx->nvdev; 450a1275caSVitaly Kuznetsov struct nvsp_message *init_pkt = &nv_dev->channel_init_pkt; 4684bf9cefSKY Srinivasan 4784bf9cefSKY Srinivasan memset(init_pkt, 0, sizeof(struct nvsp_message)); 4884bf9cefSKY Srinivasan init_pkt->hdr.msg_type = NVSP_MSG4_TYPE_SWITCH_DATA_PATH; 4984bf9cefSKY Srinivasan if (vf) 5084bf9cefSKY Srinivasan init_pkt->msg.v4_msg.active_dp.active_datapath = 5184bf9cefSKY Srinivasan NVSP_DATAPATH_VF; 5284bf9cefSKY Srinivasan else 5384bf9cefSKY Srinivasan init_pkt->msg.v4_msg.active_dp.active_datapath = 5484bf9cefSKY Srinivasan NVSP_DATAPATH_SYNTHETIC; 5584bf9cefSKY Srinivasan 5684bf9cefSKY Srinivasan vmbus_sendpacket(dev->channel, init_pkt, 5784bf9cefSKY Srinivasan sizeof(struct nvsp_message), 5884bf9cefSKY Srinivasan (unsigned long)init_pkt, 5984bf9cefSKY Srinivasan VM_PKT_DATA_INBAND, 0); 6053fa1a6fSHaiyang Zhang 6153fa1a6fSHaiyang Zhang net_device_ctx->datapath = vf; 6284bf9cefSKY Srinivasan } 6384bf9cefSKY Srinivasan 6488098834SVitaly Kuznetsov static struct netvsc_device *alloc_net_device(void) 6595fa0405SHaiyang Zhang { 6695fa0405SHaiyang Zhang struct netvsc_device *net_device; 6795fa0405SHaiyang Zhang 6895fa0405SHaiyang Zhang net_device = kzalloc(sizeof(struct netvsc_device), GFP_KERNEL); 6995fa0405SHaiyang Zhang if (!net_device) 7095fa0405SHaiyang Zhang return NULL; 7195fa0405SHaiyang Zhang 72b8b835a8Sstephen hemminger net_device->chan_table[0].mrc.buf 73b8b835a8Sstephen hemminger = vzalloc(NETVSC_RECVSLOT_MAX * sizeof(struct recv_comp_data)); 74c0b558e5SHaiyang Zhang 75dc5cd894SHaiyang Zhang init_waitqueue_head(&net_device->wait_drain); 7695fa0405SHaiyang Zhang net_device->destroy = false; 7784bf9cefSKY Srinivasan atomic_set(&net_device->open_cnt, 0); 787c3877f2SHaiyang Zhang net_device->max_pkt = RNDIS_MAX_PKT_DEFAULT; 797c3877f2SHaiyang Zhang net_device->pkt_align = RNDIS_PKT_ALIGN_DEFAULT; 80fd612602SStephen Hemminger init_completion(&net_device->channel_init_wait); 817c3877f2SHaiyang Zhang 8295fa0405SHaiyang Zhang return net_device; 8395fa0405SHaiyang Zhang } 8495fa0405SHaiyang Zhang 85545a8e79Sstephen hemminger static void free_netvsc_device(struct rcu_head *head) 86f90251c8SHaiyang Zhang { 87545a8e79Sstephen hemminger struct netvsc_device *nvdev 88545a8e79Sstephen hemminger = container_of(head, struct netvsc_device, rcu); 89c0b558e5SHaiyang Zhang int i; 90c0b558e5SHaiyang Zhang 91c0b558e5SHaiyang Zhang for (i = 0; i < VRSS_CHANNEL_MAX; i++) 92b8b835a8Sstephen hemminger vfree(nvdev->chan_table[i].mrc.buf); 93c0b558e5SHaiyang Zhang 94f90251c8SHaiyang Zhang kfree(nvdev); 95f90251c8SHaiyang Zhang } 96f90251c8SHaiyang Zhang 97545a8e79Sstephen hemminger static void free_netvsc_device_rcu(struct netvsc_device *nvdev) 98545a8e79Sstephen hemminger { 99545a8e79Sstephen hemminger call_rcu(&nvdev->rcu, free_netvsc_device); 100545a8e79Sstephen hemminger } 10146b4f7f5Sstephen hemminger 1027a2a0a84SStephen Hemminger static void netvsc_destroy_buf(struct hv_device *device) 10395fa0405SHaiyang Zhang { 10495fa0405SHaiyang Zhang struct nvsp_message *revoke_packet; 1053d541ac5SVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 1062625466dSVitaly Kuznetsov struct netvsc_device *net_device = net_device_to_netvsc_device(ndev); 1077a2a0a84SStephen Hemminger int ret; 10895fa0405SHaiyang Zhang 10995fa0405SHaiyang Zhang /* 11095fa0405SHaiyang Zhang * If we got a section count, it means we received a 11195fa0405SHaiyang Zhang * SendReceiveBufferComplete msg (ie sent 11295fa0405SHaiyang Zhang * NvspMessage1TypeSendReceiveBuffer msg) therefore, we need 11395fa0405SHaiyang Zhang * to send a revoke msg here 11495fa0405SHaiyang Zhang */ 11595fa0405SHaiyang Zhang if (net_device->recv_section_cnt) { 11695fa0405SHaiyang Zhang /* Send the revoke receive buffer */ 11795fa0405SHaiyang Zhang revoke_packet = &net_device->revoke_packet; 11895fa0405SHaiyang Zhang memset(revoke_packet, 0, sizeof(struct nvsp_message)); 11995fa0405SHaiyang Zhang 12095fa0405SHaiyang Zhang revoke_packet->hdr.msg_type = 12195fa0405SHaiyang Zhang NVSP_MSG1_TYPE_REVOKE_RECV_BUF; 12295fa0405SHaiyang Zhang revoke_packet->msg.v1_msg. 12395fa0405SHaiyang Zhang revoke_recv_buf.id = NETVSC_RECEIVE_BUFFER_ID; 12495fa0405SHaiyang Zhang 1253d541ac5SVitaly Kuznetsov ret = vmbus_sendpacket(device->channel, 12695fa0405SHaiyang Zhang revoke_packet, 12795fa0405SHaiyang Zhang sizeof(struct nvsp_message), 12895fa0405SHaiyang Zhang (unsigned long)revoke_packet, 12995fa0405SHaiyang Zhang VM_PKT_DATA_INBAND, 0); 13073e64fa4SK. Y. Srinivasan /* If the failure is because the channel is rescinded; 13173e64fa4SK. Y. Srinivasan * ignore the failure since we cannot send on a rescinded 13273e64fa4SK. Y. Srinivasan * channel. This would allow us to properly cleanup 13373e64fa4SK. Y. Srinivasan * even when the channel is rescinded. 13473e64fa4SK. Y. Srinivasan */ 13573e64fa4SK. Y. Srinivasan if (device->channel->rescind) 13673e64fa4SK. Y. Srinivasan ret = 0; 13795fa0405SHaiyang Zhang /* 13895fa0405SHaiyang Zhang * If we failed here, we might as well return and 13995fa0405SHaiyang Zhang * have a leak rather than continue and a bugchk 14095fa0405SHaiyang Zhang */ 14195fa0405SHaiyang Zhang if (ret != 0) { 14295fa0405SHaiyang Zhang netdev_err(ndev, "unable to send " 14395fa0405SHaiyang Zhang "revoke receive buffer to netvsp\n"); 1447a2a0a84SStephen Hemminger return; 14595fa0405SHaiyang Zhang } 14695fa0405SHaiyang Zhang } 14795fa0405SHaiyang Zhang 14895fa0405SHaiyang Zhang /* Teardown the gpadl on the vsp end */ 14995fa0405SHaiyang Zhang if (net_device->recv_buf_gpadl_handle) { 1503d541ac5SVitaly Kuznetsov ret = vmbus_teardown_gpadl(device->channel, 15195fa0405SHaiyang Zhang net_device->recv_buf_gpadl_handle); 15295fa0405SHaiyang Zhang 15395fa0405SHaiyang Zhang /* If we failed here, we might as well return and have a leak 15495fa0405SHaiyang Zhang * rather than continue and a bugchk 15595fa0405SHaiyang Zhang */ 15695fa0405SHaiyang Zhang if (ret != 0) { 15795fa0405SHaiyang Zhang netdev_err(ndev, 15895fa0405SHaiyang Zhang "unable to teardown receive buffer's gpadl\n"); 1597a2a0a84SStephen Hemminger return; 16095fa0405SHaiyang Zhang } 16195fa0405SHaiyang Zhang net_device->recv_buf_gpadl_handle = 0; 16295fa0405SHaiyang Zhang } 16395fa0405SHaiyang Zhang 16495fa0405SHaiyang Zhang if (net_device->recv_buf) { 16595fa0405SHaiyang Zhang /* Free up the receive buffer */ 166b679ef73SHaiyang Zhang vfree(net_device->recv_buf); 16795fa0405SHaiyang Zhang net_device->recv_buf = NULL; 16895fa0405SHaiyang Zhang } 16995fa0405SHaiyang Zhang 17095fa0405SHaiyang Zhang if (net_device->recv_section) { 17195fa0405SHaiyang Zhang net_device->recv_section_cnt = 0; 17295fa0405SHaiyang Zhang kfree(net_device->recv_section); 17395fa0405SHaiyang Zhang net_device->recv_section = NULL; 17495fa0405SHaiyang Zhang } 17595fa0405SHaiyang Zhang 176c25aaf81SKY Srinivasan /* Deal with the send buffer we may have setup. 177c25aaf81SKY Srinivasan * If we got a send section size, it means we received a 178c51ed182SHaiyang Zhang * NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE msg (ie sent 179c51ed182SHaiyang Zhang * NVSP_MSG1_TYPE_SEND_SEND_BUF msg) therefore, we need 180c25aaf81SKY Srinivasan * to send a revoke msg here 181c25aaf81SKY Srinivasan */ 182c25aaf81SKY Srinivasan if (net_device->send_section_size) { 183c25aaf81SKY Srinivasan /* Send the revoke receive buffer */ 184c25aaf81SKY Srinivasan revoke_packet = &net_device->revoke_packet; 185c25aaf81SKY Srinivasan memset(revoke_packet, 0, sizeof(struct nvsp_message)); 186c25aaf81SKY Srinivasan 187c25aaf81SKY Srinivasan revoke_packet->hdr.msg_type = 188c25aaf81SKY Srinivasan NVSP_MSG1_TYPE_REVOKE_SEND_BUF; 189c51ed182SHaiyang Zhang revoke_packet->msg.v1_msg.revoke_send_buf.id = 190c51ed182SHaiyang Zhang NETVSC_SEND_BUFFER_ID; 191c25aaf81SKY Srinivasan 1923d541ac5SVitaly Kuznetsov ret = vmbus_sendpacket(device->channel, 193c25aaf81SKY Srinivasan revoke_packet, 194c25aaf81SKY Srinivasan sizeof(struct nvsp_message), 195c25aaf81SKY Srinivasan (unsigned long)revoke_packet, 196c25aaf81SKY Srinivasan VM_PKT_DATA_INBAND, 0); 19773e64fa4SK. Y. Srinivasan 19873e64fa4SK. Y. Srinivasan /* If the failure is because the channel is rescinded; 19973e64fa4SK. Y. Srinivasan * ignore the failure since we cannot send on a rescinded 20073e64fa4SK. Y. Srinivasan * channel. This would allow us to properly cleanup 20173e64fa4SK. Y. Srinivasan * even when the channel is rescinded. 20273e64fa4SK. Y. Srinivasan */ 20373e64fa4SK. Y. Srinivasan if (device->channel->rescind) 20473e64fa4SK. Y. Srinivasan ret = 0; 20573e64fa4SK. Y. Srinivasan 206c25aaf81SKY Srinivasan /* If we failed here, we might as well return and 207c25aaf81SKY Srinivasan * have a leak rather than continue and a bugchk 208c25aaf81SKY Srinivasan */ 209c25aaf81SKY Srinivasan if (ret != 0) { 210c25aaf81SKY Srinivasan netdev_err(ndev, "unable to send " 211c25aaf81SKY Srinivasan "revoke send buffer to netvsp\n"); 2127a2a0a84SStephen Hemminger return; 213c25aaf81SKY Srinivasan } 214c25aaf81SKY Srinivasan } 215c25aaf81SKY Srinivasan /* Teardown the gpadl on the vsp end */ 216c25aaf81SKY Srinivasan if (net_device->send_buf_gpadl_handle) { 2173d541ac5SVitaly Kuznetsov ret = vmbus_teardown_gpadl(device->channel, 218c25aaf81SKY Srinivasan net_device->send_buf_gpadl_handle); 219c25aaf81SKY Srinivasan 220c25aaf81SKY Srinivasan /* If we failed here, we might as well return and have a leak 221c25aaf81SKY Srinivasan * rather than continue and a bugchk 222c25aaf81SKY Srinivasan */ 223c25aaf81SKY Srinivasan if (ret != 0) { 224c25aaf81SKY Srinivasan netdev_err(ndev, 225c25aaf81SKY Srinivasan "unable to teardown send buffer's gpadl\n"); 2267a2a0a84SStephen Hemminger return; 227c25aaf81SKY Srinivasan } 2282f18423dSDave Jones net_device->send_buf_gpadl_handle = 0; 229c25aaf81SKY Srinivasan } 230c25aaf81SKY Srinivasan if (net_device->send_buf) { 231c51ed182SHaiyang Zhang /* Free up the send buffer */ 23206b47aacSKY Srinivasan vfree(net_device->send_buf); 233c25aaf81SKY Srinivasan net_device->send_buf = NULL; 234c25aaf81SKY Srinivasan } 235c25aaf81SKY Srinivasan kfree(net_device->send_section_map); 23695fa0405SHaiyang Zhang } 23795fa0405SHaiyang Zhang 23895790837Sstephen hemminger static int netvsc_init_buf(struct hv_device *device, 23995790837Sstephen hemminger struct netvsc_device *net_device) 24095fa0405SHaiyang Zhang { 24195fa0405SHaiyang Zhang int ret = 0; 24295fa0405SHaiyang Zhang struct nvsp_message *init_packet; 24395fa0405SHaiyang Zhang struct net_device *ndev; 244fdfb70d2Sstephen hemminger size_t map_words; 2450a726c2bSK. Y. Srinivasan int node; 24695fa0405SHaiyang Zhang 2470a1275caSVitaly Kuznetsov ndev = hv_get_drvdata(device); 24895fa0405SHaiyang Zhang 2490a726c2bSK. Y. Srinivasan node = cpu_to_node(device->channel->target_cpu); 2500a726c2bSK. Y. Srinivasan net_device->recv_buf = vzalloc_node(net_device->recv_buf_size, node); 2510a726c2bSK. Y. Srinivasan if (!net_device->recv_buf) 252b679ef73SHaiyang Zhang net_device->recv_buf = vzalloc(net_device->recv_buf_size); 2530a726c2bSK. Y. Srinivasan 25495fa0405SHaiyang Zhang if (!net_device->recv_buf) { 25595fa0405SHaiyang Zhang netdev_err(ndev, "unable to allocate receive " 25695fa0405SHaiyang Zhang "buffer of size %d\n", net_device->recv_buf_size); 25795fa0405SHaiyang Zhang ret = -ENOMEM; 25895fa0405SHaiyang Zhang goto cleanup; 25995fa0405SHaiyang Zhang } 26095fa0405SHaiyang Zhang 26195fa0405SHaiyang Zhang /* 26295fa0405SHaiyang Zhang * Establish the gpadl handle for this buffer on this 26395fa0405SHaiyang Zhang * channel. Note: This call uses the vmbus connection rather 26495fa0405SHaiyang Zhang * than the channel to establish the gpadl handle. 26595fa0405SHaiyang Zhang */ 26695fa0405SHaiyang Zhang ret = vmbus_establish_gpadl(device->channel, net_device->recv_buf, 26795fa0405SHaiyang Zhang net_device->recv_buf_size, 26895fa0405SHaiyang Zhang &net_device->recv_buf_gpadl_handle); 26995fa0405SHaiyang Zhang if (ret != 0) { 27095fa0405SHaiyang Zhang netdev_err(ndev, 27195fa0405SHaiyang Zhang "unable to establish receive buffer's gpadl\n"); 27295fa0405SHaiyang Zhang goto cleanup; 27395fa0405SHaiyang Zhang } 27495fa0405SHaiyang Zhang 27595fa0405SHaiyang Zhang /* Notify the NetVsp of the gpadl handle */ 27695fa0405SHaiyang Zhang init_packet = &net_device->channel_init_pkt; 27795fa0405SHaiyang Zhang memset(init_packet, 0, sizeof(struct nvsp_message)); 27895fa0405SHaiyang Zhang init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_RECV_BUF; 27995fa0405SHaiyang Zhang init_packet->msg.v1_msg.send_recv_buf. 28095fa0405SHaiyang Zhang gpadl_handle = net_device->recv_buf_gpadl_handle; 28195fa0405SHaiyang Zhang init_packet->msg.v1_msg. 28295fa0405SHaiyang Zhang send_recv_buf.id = NETVSC_RECEIVE_BUFFER_ID; 28395fa0405SHaiyang Zhang 28495fa0405SHaiyang Zhang /* Send the gpadl notification request */ 28595fa0405SHaiyang Zhang ret = vmbus_sendpacket(device->channel, init_packet, 28695fa0405SHaiyang Zhang sizeof(struct nvsp_message), 28795fa0405SHaiyang Zhang (unsigned long)init_packet, 28895fa0405SHaiyang Zhang VM_PKT_DATA_INBAND, 28995fa0405SHaiyang Zhang VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 29095fa0405SHaiyang Zhang if (ret != 0) { 29195fa0405SHaiyang Zhang netdev_err(ndev, 29295fa0405SHaiyang Zhang "unable to send receive buffer's gpadl to netvsp\n"); 29395fa0405SHaiyang Zhang goto cleanup; 29495fa0405SHaiyang Zhang } 29595fa0405SHaiyang Zhang 2965362855aSVitaly Kuznetsov wait_for_completion(&net_device->channel_init_wait); 29795fa0405SHaiyang Zhang 29895fa0405SHaiyang Zhang /* Check the response */ 29995fa0405SHaiyang Zhang if (init_packet->msg.v1_msg. 30095fa0405SHaiyang Zhang send_recv_buf_complete.status != NVSP_STAT_SUCCESS) { 30195fa0405SHaiyang Zhang netdev_err(ndev, "Unable to complete receive buffer " 30295fa0405SHaiyang Zhang "initialization with NetVsp - status %d\n", 30395fa0405SHaiyang Zhang init_packet->msg.v1_msg. 30495fa0405SHaiyang Zhang send_recv_buf_complete.status); 30595fa0405SHaiyang Zhang ret = -EINVAL; 30695fa0405SHaiyang Zhang goto cleanup; 30795fa0405SHaiyang Zhang } 30895fa0405SHaiyang Zhang 30995fa0405SHaiyang Zhang /* Parse the response */ 31095fa0405SHaiyang Zhang 31195fa0405SHaiyang Zhang net_device->recv_section_cnt = init_packet->msg. 31295fa0405SHaiyang Zhang v1_msg.send_recv_buf_complete.num_sections; 31395fa0405SHaiyang Zhang 314c1813200SHaiyang Zhang net_device->recv_section = kmemdup( 315c1813200SHaiyang Zhang init_packet->msg.v1_msg.send_recv_buf_complete.sections, 316c1813200SHaiyang Zhang net_device->recv_section_cnt * 317c1813200SHaiyang Zhang sizeof(struct nvsp_1_receive_buffer_section), 31895fa0405SHaiyang Zhang GFP_KERNEL); 31995fa0405SHaiyang Zhang if (net_device->recv_section == NULL) { 32095fa0405SHaiyang Zhang ret = -EINVAL; 32195fa0405SHaiyang Zhang goto cleanup; 32295fa0405SHaiyang Zhang } 32395fa0405SHaiyang Zhang 32495fa0405SHaiyang Zhang /* 32595fa0405SHaiyang Zhang * For 1st release, there should only be 1 section that represents the 32695fa0405SHaiyang Zhang * entire receive buffer 32795fa0405SHaiyang Zhang */ 32895fa0405SHaiyang Zhang if (net_device->recv_section_cnt != 1 || 32995fa0405SHaiyang Zhang net_device->recv_section->offset != 0) { 33095fa0405SHaiyang Zhang ret = -EINVAL; 33195fa0405SHaiyang Zhang goto cleanup; 33295fa0405SHaiyang Zhang } 33395fa0405SHaiyang Zhang 334c25aaf81SKY Srinivasan /* Now setup the send buffer. 335c25aaf81SKY Srinivasan */ 3365defde59SK. Y. Srinivasan net_device->send_buf = vzalloc_node(net_device->send_buf_size, node); 3375defde59SK. Y. Srinivasan if (!net_device->send_buf) 33806b47aacSKY Srinivasan net_device->send_buf = vzalloc(net_device->send_buf_size); 339c25aaf81SKY Srinivasan if (!net_device->send_buf) { 340c25aaf81SKY Srinivasan netdev_err(ndev, "unable to allocate send " 341c25aaf81SKY Srinivasan "buffer of size %d\n", net_device->send_buf_size); 342c25aaf81SKY Srinivasan ret = -ENOMEM; 343c25aaf81SKY Srinivasan goto cleanup; 344c25aaf81SKY Srinivasan } 345c25aaf81SKY Srinivasan 346c25aaf81SKY Srinivasan /* Establish the gpadl handle for this buffer on this 347c25aaf81SKY Srinivasan * channel. Note: This call uses the vmbus connection rather 348c25aaf81SKY Srinivasan * than the channel to establish the gpadl handle. 349c25aaf81SKY Srinivasan */ 350c25aaf81SKY Srinivasan ret = vmbus_establish_gpadl(device->channel, net_device->send_buf, 351c25aaf81SKY Srinivasan net_device->send_buf_size, 352c25aaf81SKY Srinivasan &net_device->send_buf_gpadl_handle); 353c25aaf81SKY Srinivasan if (ret != 0) { 354c25aaf81SKY Srinivasan netdev_err(ndev, 355c25aaf81SKY Srinivasan "unable to establish send buffer's gpadl\n"); 356c25aaf81SKY Srinivasan goto cleanup; 357c25aaf81SKY Srinivasan } 358c25aaf81SKY Srinivasan 359c25aaf81SKY Srinivasan /* Notify the NetVsp of the gpadl handle */ 360c25aaf81SKY Srinivasan init_packet = &net_device->channel_init_pkt; 361c25aaf81SKY Srinivasan memset(init_packet, 0, sizeof(struct nvsp_message)); 362c25aaf81SKY Srinivasan init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_SEND_BUF; 363c51ed182SHaiyang Zhang init_packet->msg.v1_msg.send_send_buf.gpadl_handle = 364c25aaf81SKY Srinivasan net_device->send_buf_gpadl_handle; 365c51ed182SHaiyang Zhang init_packet->msg.v1_msg.send_send_buf.id = NETVSC_SEND_BUFFER_ID; 366c25aaf81SKY Srinivasan 367c25aaf81SKY Srinivasan /* Send the gpadl notification request */ 368c25aaf81SKY Srinivasan ret = vmbus_sendpacket(device->channel, init_packet, 369c25aaf81SKY Srinivasan sizeof(struct nvsp_message), 370c25aaf81SKY Srinivasan (unsigned long)init_packet, 371c25aaf81SKY Srinivasan VM_PKT_DATA_INBAND, 372c25aaf81SKY Srinivasan VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 373c25aaf81SKY Srinivasan if (ret != 0) { 374c25aaf81SKY Srinivasan netdev_err(ndev, 375c25aaf81SKY Srinivasan "unable to send send buffer's gpadl to netvsp\n"); 376c25aaf81SKY Srinivasan goto cleanup; 377c25aaf81SKY Srinivasan } 378c25aaf81SKY Srinivasan 3795362855aSVitaly Kuznetsov wait_for_completion(&net_device->channel_init_wait); 380c25aaf81SKY Srinivasan 381c25aaf81SKY Srinivasan /* Check the response */ 382c25aaf81SKY Srinivasan if (init_packet->msg.v1_msg. 383c25aaf81SKY Srinivasan send_send_buf_complete.status != NVSP_STAT_SUCCESS) { 384c25aaf81SKY Srinivasan netdev_err(ndev, "Unable to complete send buffer " 385c25aaf81SKY Srinivasan "initialization with NetVsp - status %d\n", 386c25aaf81SKY Srinivasan init_packet->msg.v1_msg. 387c51ed182SHaiyang Zhang send_send_buf_complete.status); 388c25aaf81SKY Srinivasan ret = -EINVAL; 389c25aaf81SKY Srinivasan goto cleanup; 390c25aaf81SKY Srinivasan } 391c25aaf81SKY Srinivasan 392c25aaf81SKY Srinivasan /* Parse the response */ 393c25aaf81SKY Srinivasan net_device->send_section_size = init_packet->msg. 394c25aaf81SKY Srinivasan v1_msg.send_send_buf_complete.section_size; 395c25aaf81SKY Srinivasan 396c25aaf81SKY Srinivasan /* Section count is simply the size divided by the section size. 397c25aaf81SKY Srinivasan */ 398c25aaf81SKY Srinivasan net_device->send_section_cnt = 399c25aaf81SKY Srinivasan net_device->send_buf_size / net_device->send_section_size; 400c25aaf81SKY Srinivasan 40193ba2222SVitaly Kuznetsov netdev_dbg(ndev, "Send section size: %d, Section count:%d\n", 402c25aaf81SKY Srinivasan net_device->send_section_size, net_device->send_section_cnt); 403c25aaf81SKY Srinivasan 404c25aaf81SKY Srinivasan /* Setup state for managing the send buffer. */ 405fdfb70d2Sstephen hemminger map_words = DIV_ROUND_UP(net_device->send_section_cnt, BITS_PER_LONG); 406c25aaf81SKY Srinivasan 407fdfb70d2Sstephen hemminger net_device->send_section_map = kcalloc(map_words, sizeof(ulong), GFP_KERNEL); 408dd1d3f8fSWei Yongjun if (net_device->send_section_map == NULL) { 409dd1d3f8fSWei Yongjun ret = -ENOMEM; 410c25aaf81SKY Srinivasan goto cleanup; 411dd1d3f8fSWei Yongjun } 412c25aaf81SKY Srinivasan 41395fa0405SHaiyang Zhang goto exit; 41495fa0405SHaiyang Zhang 41595fa0405SHaiyang Zhang cleanup: 4163d541ac5SVitaly Kuznetsov netvsc_destroy_buf(device); 41795fa0405SHaiyang Zhang 41895fa0405SHaiyang Zhang exit: 41995fa0405SHaiyang Zhang return ret; 42095fa0405SHaiyang Zhang } 42195fa0405SHaiyang Zhang 422f157e78dSHaiyang Zhang /* Negotiate NVSP protocol version */ 423f157e78dSHaiyang Zhang static int negotiate_nvsp_ver(struct hv_device *device, 424f157e78dSHaiyang Zhang struct netvsc_device *net_device, 425f157e78dSHaiyang Zhang struct nvsp_message *init_packet, 426f157e78dSHaiyang Zhang u32 nvsp_ver) 42795fa0405SHaiyang Zhang { 4280a1275caSVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 4297390fe9cSNicholas Mc Guire int ret; 430f157e78dSHaiyang Zhang 431f157e78dSHaiyang Zhang memset(init_packet, 0, sizeof(struct nvsp_message)); 432f157e78dSHaiyang Zhang init_packet->hdr.msg_type = NVSP_MSG_TYPE_INIT; 433f157e78dSHaiyang Zhang init_packet->msg.init_msg.init.min_protocol_ver = nvsp_ver; 434f157e78dSHaiyang Zhang init_packet->msg.init_msg.init.max_protocol_ver = nvsp_ver; 435f157e78dSHaiyang Zhang 436f157e78dSHaiyang Zhang /* Send the init request */ 437f157e78dSHaiyang Zhang ret = vmbus_sendpacket(device->channel, init_packet, 438f157e78dSHaiyang Zhang sizeof(struct nvsp_message), 439f157e78dSHaiyang Zhang (unsigned long)init_packet, 440f157e78dSHaiyang Zhang VM_PKT_DATA_INBAND, 441f157e78dSHaiyang Zhang VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 442f157e78dSHaiyang Zhang 443f157e78dSHaiyang Zhang if (ret != 0) 444f157e78dSHaiyang Zhang return ret; 445f157e78dSHaiyang Zhang 4465362855aSVitaly Kuznetsov wait_for_completion(&net_device->channel_init_wait); 447f157e78dSHaiyang Zhang 448f157e78dSHaiyang Zhang if (init_packet->msg.init_msg.init_complete.status != 449f157e78dSHaiyang Zhang NVSP_STAT_SUCCESS) 450f157e78dSHaiyang Zhang return -EINVAL; 451f157e78dSHaiyang Zhang 452a1eabb01SHaiyang Zhang if (nvsp_ver == NVSP_PROTOCOL_VERSION_1) 453f157e78dSHaiyang Zhang return 0; 454f157e78dSHaiyang Zhang 45571790a27SHaiyang Zhang /* NVSPv2 or later: Send NDIS config */ 456f157e78dSHaiyang Zhang memset(init_packet, 0, sizeof(struct nvsp_message)); 457f157e78dSHaiyang Zhang init_packet->hdr.msg_type = NVSP_MSG2_TYPE_SEND_NDIS_CONFIG; 4580a1275caSVitaly Kuznetsov init_packet->msg.v2_msg.send_ndis_config.mtu = ndev->mtu + ETH_HLEN; 4591f5f3a75SHaiyang Zhang init_packet->msg.v2_msg.send_ndis_config.capability.ieee8021q = 1; 460f157e78dSHaiyang Zhang 4617f5d5af0SHaiyang Zhang if (nvsp_ver >= NVSP_PROTOCOL_VERSION_5) { 46271790a27SHaiyang Zhang init_packet->msg.v2_msg.send_ndis_config.capability.sriov = 1; 46371790a27SHaiyang Zhang 4647f5d5af0SHaiyang Zhang /* Teaming bit is needed to receive link speed updates */ 4657f5d5af0SHaiyang Zhang init_packet->msg.v2_msg.send_ndis_config.capability.teaming = 1; 4667f5d5af0SHaiyang Zhang } 4677f5d5af0SHaiyang Zhang 468f157e78dSHaiyang Zhang ret = vmbus_sendpacket(device->channel, init_packet, 469f157e78dSHaiyang Zhang sizeof(struct nvsp_message), 470f157e78dSHaiyang Zhang (unsigned long)init_packet, 471f157e78dSHaiyang Zhang VM_PKT_DATA_INBAND, 0); 472f157e78dSHaiyang Zhang 473f157e78dSHaiyang Zhang return ret; 474f157e78dSHaiyang Zhang } 475f157e78dSHaiyang Zhang 47695790837Sstephen hemminger static int netvsc_connect_vsp(struct hv_device *device, 47795790837Sstephen hemminger struct netvsc_device *net_device) 478f157e78dSHaiyang Zhang { 479e5a78fadSStephen Hemminger const u32 ver_list[] = { 480e5a78fadSStephen Hemminger NVSP_PROTOCOL_VERSION_1, NVSP_PROTOCOL_VERSION_2, 48195790837Sstephen hemminger NVSP_PROTOCOL_VERSION_4, NVSP_PROTOCOL_VERSION_5 48295790837Sstephen hemminger }; 48395790837Sstephen hemminger struct nvsp_message *init_packet; 48495790837Sstephen hemminger int ndis_version, i, ret; 48595fa0405SHaiyang Zhang 48695fa0405SHaiyang Zhang init_packet = &net_device->channel_init_pkt; 48795fa0405SHaiyang Zhang 488f157e78dSHaiyang Zhang /* Negotiate the latest NVSP protocol supported */ 489e5a78fadSStephen Hemminger for (i = ARRAY_SIZE(ver_list) - 1; i >= 0; i--) 490f157e78dSHaiyang Zhang if (negotiate_nvsp_ver(device, net_device, init_packet, 491a1eabb01SHaiyang Zhang ver_list[i]) == 0) { 492a1eabb01SHaiyang Zhang net_device->nvsp_version = ver_list[i]; 493a1eabb01SHaiyang Zhang break; 494a1eabb01SHaiyang Zhang } 495a1eabb01SHaiyang Zhang 496a1eabb01SHaiyang Zhang if (i < 0) { 49795fa0405SHaiyang Zhang ret = -EPROTO; 49895fa0405SHaiyang Zhang goto cleanup; 49995fa0405SHaiyang Zhang } 500f157e78dSHaiyang Zhang 501f157e78dSHaiyang Zhang pr_debug("Negotiated NVSP version:%x\n", net_device->nvsp_version); 502f157e78dSHaiyang Zhang 50395fa0405SHaiyang Zhang /* Send the ndis version */ 50495fa0405SHaiyang Zhang memset(init_packet, 0, sizeof(struct nvsp_message)); 50595fa0405SHaiyang Zhang 506a1eabb01SHaiyang Zhang if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_4) 5071f73db49SKY Srinivasan ndis_version = 0x00060001; 508a1eabb01SHaiyang Zhang else 509a1eabb01SHaiyang Zhang ndis_version = 0x0006001e; 51095fa0405SHaiyang Zhang 51195fa0405SHaiyang Zhang init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_NDIS_VER; 51295fa0405SHaiyang Zhang init_packet->msg.v1_msg. 51395fa0405SHaiyang Zhang send_ndis_ver.ndis_major_ver = 51495fa0405SHaiyang Zhang (ndis_version & 0xFFFF0000) >> 16; 51595fa0405SHaiyang Zhang init_packet->msg.v1_msg. 51695fa0405SHaiyang Zhang send_ndis_ver.ndis_minor_ver = 51795fa0405SHaiyang Zhang ndis_version & 0xFFFF; 51895fa0405SHaiyang Zhang 51995fa0405SHaiyang Zhang /* Send the init request */ 52095fa0405SHaiyang Zhang ret = vmbus_sendpacket(device->channel, init_packet, 52195fa0405SHaiyang Zhang sizeof(struct nvsp_message), 52295fa0405SHaiyang Zhang (unsigned long)init_packet, 52395fa0405SHaiyang Zhang VM_PKT_DATA_INBAND, 0); 52495fa0405SHaiyang Zhang if (ret != 0) 52595fa0405SHaiyang Zhang goto cleanup; 52695fa0405SHaiyang Zhang 52795fa0405SHaiyang Zhang /* Post the big receive buffer to NetVSP */ 52899d3016dSHaiyang Zhang if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_2) 52999d3016dSHaiyang Zhang net_device->recv_buf_size = NETVSC_RECEIVE_BUFFER_SIZE_LEGACY; 53099d3016dSHaiyang Zhang else 53199d3016dSHaiyang Zhang net_device->recv_buf_size = NETVSC_RECEIVE_BUFFER_SIZE; 532c25aaf81SKY Srinivasan net_device->send_buf_size = NETVSC_SEND_BUFFER_SIZE; 53399d3016dSHaiyang Zhang 53495790837Sstephen hemminger ret = netvsc_init_buf(device, net_device); 53595fa0405SHaiyang Zhang 53695fa0405SHaiyang Zhang cleanup: 53795fa0405SHaiyang Zhang return ret; 53895fa0405SHaiyang Zhang } 53995fa0405SHaiyang Zhang 5403d541ac5SVitaly Kuznetsov static void netvsc_disconnect_vsp(struct hv_device *device) 54195fa0405SHaiyang Zhang { 5423d541ac5SVitaly Kuznetsov netvsc_destroy_buf(device); 54395fa0405SHaiyang Zhang } 54495fa0405SHaiyang Zhang 54595fa0405SHaiyang Zhang /* 54695fa0405SHaiyang Zhang * netvsc_device_remove - Callback when the root bus device is removed 54795fa0405SHaiyang Zhang */ 548e08f3ea5SStephen Hemminger void netvsc_device_remove(struct hv_device *device) 54995fa0405SHaiyang Zhang { 5503d541ac5SVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 5513d541ac5SVitaly Kuznetsov struct net_device_context *net_device_ctx = netdev_priv(ndev); 5523d541ac5SVitaly Kuznetsov struct netvsc_device *net_device = net_device_ctx->nvdev; 55315a863bfSstephen hemminger int i; 55495fa0405SHaiyang Zhang 5553d541ac5SVitaly Kuznetsov netvsc_disconnect_vsp(device); 55695fa0405SHaiyang Zhang 557545a8e79Sstephen hemminger RCU_INIT_POINTER(net_device_ctx->nvdev, NULL); 55895fa0405SHaiyang Zhang 55995fa0405SHaiyang Zhang /* 56095fa0405SHaiyang Zhang * At this point, no one should be accessing net_device 56195fa0405SHaiyang Zhang * except in here 56295fa0405SHaiyang Zhang */ 56393ba2222SVitaly Kuznetsov netdev_dbg(ndev, "net device safe to remove\n"); 56495fa0405SHaiyang Zhang 56595fa0405SHaiyang Zhang /* Now, we can close the channel safely */ 56695fa0405SHaiyang Zhang vmbus_close(device->channel); 56795fa0405SHaiyang Zhang 56876bb5db5Sstephen hemminger /* And dissassociate NAPI context from device */ 56979cd874cSstephen hemminger for (i = 0; i < net_device->num_chn; i++) 57076bb5db5Sstephen hemminger netif_napi_del(&net_device->chan_table[i].napi); 57115a863bfSstephen hemminger 57295fa0405SHaiyang Zhang /* Release all resources */ 573545a8e79Sstephen hemminger free_netvsc_device_rcu(net_device); 57495fa0405SHaiyang Zhang } 57595fa0405SHaiyang Zhang 57633be96e4SHaiyang Zhang #define RING_AVAIL_PERCENT_HIWATER 20 57733be96e4SHaiyang Zhang #define RING_AVAIL_PERCENT_LOWATER 10 57833be96e4SHaiyang Zhang 57933be96e4SHaiyang Zhang /* 58033be96e4SHaiyang Zhang * Get the percentage of available bytes to write in the ring. 58133be96e4SHaiyang Zhang * The return value is in range from 0 to 100. 58233be96e4SHaiyang Zhang */ 58333be96e4SHaiyang Zhang static inline u32 hv_ringbuf_avail_percent( 58433be96e4SHaiyang Zhang struct hv_ring_buffer_info *ring_info) 58533be96e4SHaiyang Zhang { 58633be96e4SHaiyang Zhang u32 avail_read, avail_write; 58733be96e4SHaiyang Zhang 58833be96e4SHaiyang Zhang hv_get_ringbuffer_availbytes(ring_info, &avail_read, &avail_write); 58933be96e4SHaiyang Zhang 59033be96e4SHaiyang Zhang return avail_write * 100 / ring_info->ring_datasize; 59133be96e4SHaiyang Zhang } 59233be96e4SHaiyang Zhang 593c25aaf81SKY Srinivasan static inline void netvsc_free_send_slot(struct netvsc_device *net_device, 594c25aaf81SKY Srinivasan u32 index) 595c25aaf81SKY Srinivasan { 596c25aaf81SKY Srinivasan sync_change_bit(index, net_device->send_section_map); 597c25aaf81SKY Srinivasan } 598c25aaf81SKY Srinivasan 599bc304dd3SStephen Hemminger static void netvsc_send_tx_complete(struct netvsc_device *net_device, 60025b85ee8SKY Srinivasan struct vmbus_channel *incoming_channel, 60197c1723aSKY Srinivasan struct hv_device *device, 602f9645430Sstephen hemminger const struct vmpacket_descriptor *desc, 603f9645430Sstephen hemminger int budget) 60495fa0405SHaiyang Zhang { 60550698d80Sstephen hemminger struct sk_buff *skb = (struct sk_buff *)(unsigned long)desc->trans_id; 6063d541ac5SVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 607bc304dd3SStephen Hemminger struct vmbus_channel *channel = device->channel; 6085b54dac8SHaiyang Zhang u16 q_idx = 0; 6095b54dac8SHaiyang Zhang int queue_sends; 61033be96e4SHaiyang Zhang 61195fa0405SHaiyang Zhang /* Notify the layer above us */ 612bc304dd3SStephen Hemminger if (likely(skb)) { 613793e3955Sstephen hemminger const struct hv_netvsc_packet *packet 614bc304dd3SStephen Hemminger = (struct hv_netvsc_packet *)skb->cb; 615793e3955Sstephen hemminger u32 send_index = packet->send_buf_index; 616793e3955Sstephen hemminger struct netvsc_stats *tx_stats; 617bc304dd3SStephen Hemminger 618c25aaf81SKY Srinivasan if (send_index != NETVSC_INVALID_INDEX) 619c25aaf81SKY Srinivasan netvsc_free_send_slot(net_device, send_index); 620793e3955Sstephen hemminger q_idx = packet->q_idx; 62125b85ee8SKY Srinivasan channel = incoming_channel; 622bc304dd3SStephen Hemminger 6236c80f3fcSSimon Xiao tx_stats = &net_device->chan_table[q_idx].tx_stats; 624793e3955Sstephen hemminger 625793e3955Sstephen hemminger u64_stats_update_begin(&tx_stats->syncp); 626793e3955Sstephen hemminger tx_stats->packets += packet->total_packets; 627793e3955Sstephen hemminger tx_stats->bytes += packet->total_bytes; 628793e3955Sstephen hemminger u64_stats_update_end(&tx_stats->syncp); 629793e3955Sstephen hemminger 630f9645430Sstephen hemminger napi_consume_skb(skb, budget); 6315b54dac8SHaiyang Zhang } 63295fa0405SHaiyang Zhang 633b8b835a8Sstephen hemminger queue_sends = 634b8b835a8Sstephen hemminger atomic_dec_return(&net_device->chan_table[q_idx].queue_sends); 6351d06825bSHaiyang Zhang 63646b4f7f5Sstephen hemminger if (net_device->destroy && queue_sends == 0) 637dc5cd894SHaiyang Zhang wake_up(&net_device->wait_drain); 638dc5cd894SHaiyang Zhang 6395b54dac8SHaiyang Zhang if (netif_tx_queue_stopped(netdev_get_tx_queue(ndev, q_idx)) && 640bc304dd3SStephen Hemminger (hv_ringbuf_avail_percent(&channel->outbound) > RING_AVAIL_PERCENT_HIWATER || 641bc304dd3SStephen Hemminger queue_sends < 1)) 642796cc88cSStephen Hemminger netif_tx_wake_queue(netdev_get_tx_queue(ndev, q_idx)); 643bc304dd3SStephen Hemminger } 644bc304dd3SStephen Hemminger 645bc304dd3SStephen Hemminger static void netvsc_send_completion(struct netvsc_device *net_device, 646bc304dd3SStephen Hemminger struct vmbus_channel *incoming_channel, 647bc304dd3SStephen Hemminger struct hv_device *device, 648f9645430Sstephen hemminger const struct vmpacket_descriptor *desc, 649f9645430Sstephen hemminger int budget) 650bc304dd3SStephen Hemminger { 651f3dd3f47Sstephen hemminger struct nvsp_message *nvsp_packet = hv_pkt_data(desc); 652bc304dd3SStephen Hemminger struct net_device *ndev = hv_get_drvdata(device); 653bc304dd3SStephen Hemminger 654bc304dd3SStephen Hemminger switch (nvsp_packet->hdr.msg_type) { 655bc304dd3SStephen Hemminger case NVSP_MSG_TYPE_INIT_COMPLETE: 656bc304dd3SStephen Hemminger case NVSP_MSG1_TYPE_SEND_RECV_BUF_COMPLETE: 657bc304dd3SStephen Hemminger case NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE: 658bc304dd3SStephen Hemminger case NVSP_MSG5_TYPE_SUBCHANNEL: 659bc304dd3SStephen Hemminger /* Copy the response back */ 660bc304dd3SStephen Hemminger memcpy(&net_device->channel_init_pkt, nvsp_packet, 661bc304dd3SStephen Hemminger sizeof(struct nvsp_message)); 662bc304dd3SStephen Hemminger complete(&net_device->channel_init_wait); 663bc304dd3SStephen Hemminger break; 664bc304dd3SStephen Hemminger 665bc304dd3SStephen Hemminger case NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE: 666bc304dd3SStephen Hemminger netvsc_send_tx_complete(net_device, incoming_channel, 667f9645430Sstephen hemminger device, desc, budget); 668bc304dd3SStephen Hemminger break; 669bc304dd3SStephen Hemminger 670bc304dd3SStephen Hemminger default: 671bc304dd3SStephen Hemminger netdev_err(ndev, 672bc304dd3SStephen Hemminger "Unknown send completion type %d received!!\n", 673bc304dd3SStephen Hemminger nvsp_packet->hdr.msg_type); 67495fa0405SHaiyang Zhang } 67595fa0405SHaiyang Zhang } 67695fa0405SHaiyang Zhang 677c25aaf81SKY Srinivasan static u32 netvsc_get_next_send_section(struct netvsc_device *net_device) 678c25aaf81SKY Srinivasan { 679b58a1858Sstephen hemminger unsigned long *map_addr = net_device->send_section_map; 680b58a1858Sstephen hemminger unsigned int i; 681c25aaf81SKY Srinivasan 682fdfb70d2Sstephen hemminger for_each_clear_bit(i, map_addr, net_device->send_section_cnt) { 683b58a1858Sstephen hemminger if (sync_test_and_set_bit(i, map_addr) == 0) 684b58a1858Sstephen hemminger return i; 685c25aaf81SKY Srinivasan } 686b58a1858Sstephen hemminger 687b58a1858Sstephen hemminger return NETVSC_INVALID_INDEX; 688c25aaf81SKY Srinivasan } 689c25aaf81SKY Srinivasan 690da19fcd0SLad, Prabhakar static u32 netvsc_copy_to_send_buf(struct netvsc_device *net_device, 691c25aaf81SKY Srinivasan unsigned int section_index, 6927c3877f2SHaiyang Zhang u32 pend_size, 69324476760SKY Srinivasan struct hv_netvsc_packet *packet, 694a9f2e2d6SKY Srinivasan struct rndis_message *rndis_msg, 695694a9fb0SKY Srinivasan struct hv_page_buffer **pb, 696694a9fb0SKY Srinivasan struct sk_buff *skb) 697c25aaf81SKY Srinivasan { 698c25aaf81SKY Srinivasan char *start = net_device->send_buf; 6997c3877f2SHaiyang Zhang char *dest = start + (section_index * net_device->send_section_size) 7007c3877f2SHaiyang Zhang + pend_size; 701c25aaf81SKY Srinivasan int i; 702c25aaf81SKY Srinivasan u32 msg_size = 0; 7037c3877f2SHaiyang Zhang u32 padding = 0; 7047c3877f2SHaiyang Zhang u32 remain = packet->total_data_buflen % net_device->pkt_align; 705aa0a34beSHaiyang Zhang u32 page_count = packet->cp_partial ? packet->rmsg_pgcnt : 706aa0a34beSHaiyang Zhang packet->page_buf_cnt; 7077c3877f2SHaiyang Zhang 7087c3877f2SHaiyang Zhang /* Add padding */ 709ebc1dcf6Sstephen hemminger if (skb->xmit_more && remain && !packet->cp_partial) { 7107c3877f2SHaiyang Zhang padding = net_device->pkt_align - remain; 71124476760SKY Srinivasan rndis_msg->msg_len += padding; 7127c3877f2SHaiyang Zhang packet->total_data_buflen += padding; 7137c3877f2SHaiyang Zhang } 714c25aaf81SKY Srinivasan 715aa0a34beSHaiyang Zhang for (i = 0; i < page_count; i++) { 716a9f2e2d6SKY Srinivasan char *src = phys_to_virt((*pb)[i].pfn << PAGE_SHIFT); 717a9f2e2d6SKY Srinivasan u32 offset = (*pb)[i].offset; 718a9f2e2d6SKY Srinivasan u32 len = (*pb)[i].len; 719c25aaf81SKY Srinivasan 720c25aaf81SKY Srinivasan memcpy(dest, (src + offset), len); 721c25aaf81SKY Srinivasan msg_size += len; 722c25aaf81SKY Srinivasan dest += len; 723c25aaf81SKY Srinivasan } 7247c3877f2SHaiyang Zhang 7257c3877f2SHaiyang Zhang if (padding) { 7267c3877f2SHaiyang Zhang memset(dest, 0, padding); 7277c3877f2SHaiyang Zhang msg_size += padding; 7287c3877f2SHaiyang Zhang } 7297c3877f2SHaiyang Zhang 730c25aaf81SKY Srinivasan return msg_size; 731c25aaf81SKY Srinivasan } 732c25aaf81SKY Srinivasan 7333a8963acSStephen Hemminger static inline int netvsc_send_pkt( 7340a1275caSVitaly Kuznetsov struct hv_device *device, 7357c3877f2SHaiyang Zhang struct hv_netvsc_packet *packet, 736a9f2e2d6SKY Srinivasan struct netvsc_device *net_device, 7373a3d9a0aSKY Srinivasan struct hv_page_buffer **pb, 7383a3d9a0aSKY Srinivasan struct sk_buff *skb) 73995fa0405SHaiyang Zhang { 7407c3877f2SHaiyang Zhang struct nvsp_message nvmsg; 741b8b835a8Sstephen hemminger struct netvsc_channel *nvchan 742b8b835a8Sstephen hemminger = &net_device->chan_table[packet->q_idx]; 743b8b835a8Sstephen hemminger struct vmbus_channel *out_channel = nvchan->channel; 7440a1275caSVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 745b8b835a8Sstephen hemminger struct netdev_queue *txq = netdev_get_tx_queue(ndev, packet->q_idx); 7467c3877f2SHaiyang Zhang u64 req_id; 7477c3877f2SHaiyang Zhang int ret; 748aa0a34beSHaiyang Zhang struct hv_page_buffer *pgbuf; 74982fa3c77SKY Srinivasan u32 ring_avail = hv_ringbuf_avail_percent(&out_channel->outbound); 750c25aaf81SKY Srinivasan 7517c3877f2SHaiyang Zhang nvmsg.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT; 752694a9fb0SKY Srinivasan if (skb != NULL) { 75395fa0405SHaiyang Zhang /* 0 is RMC_DATA; */ 7547c3877f2SHaiyang Zhang nvmsg.msg.v1_msg.send_rndis_pkt.channel_type = 0; 75595fa0405SHaiyang Zhang } else { 75695fa0405SHaiyang Zhang /* 1 is RMC_CONTROL; */ 7577c3877f2SHaiyang Zhang nvmsg.msg.v1_msg.send_rndis_pkt.channel_type = 1; 75895fa0405SHaiyang Zhang } 75995fa0405SHaiyang Zhang 7607c3877f2SHaiyang Zhang nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_index = 7617c3877f2SHaiyang Zhang packet->send_buf_index; 7627c3877f2SHaiyang Zhang if (packet->send_buf_index == NETVSC_INVALID_INDEX) 7637c3877f2SHaiyang Zhang nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_size = 0; 7647c3877f2SHaiyang Zhang else 7657c3877f2SHaiyang Zhang nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_size = 7667c3877f2SHaiyang Zhang packet->total_data_buflen; 76795fa0405SHaiyang Zhang 7683a3d9a0aSKY Srinivasan req_id = (ulong)skb; 769f1ea3cd7SHaiyang Zhang 770c3582a2cSHaiyang Zhang if (out_channel->rescind) 771c3582a2cSHaiyang Zhang return -ENODEV; 772c3582a2cSHaiyang Zhang 77395fa0405SHaiyang Zhang if (packet->page_buf_cnt) { 774a9f2e2d6SKY Srinivasan pgbuf = packet->cp_partial ? (*pb) + 775a9f2e2d6SKY Srinivasan packet->rmsg_pgcnt : (*pb); 77682fa3c77SKY Srinivasan ret = vmbus_sendpacket_pagebuffer_ctl(out_channel, 777aa0a34beSHaiyang Zhang pgbuf, 77895fa0405SHaiyang Zhang packet->page_buf_cnt, 7797c3877f2SHaiyang Zhang &nvmsg, 78095fa0405SHaiyang Zhang sizeof(struct nvsp_message), 78182fa3c77SKY Srinivasan req_id, 7823454323cSStephen Hemminger VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 78395fa0405SHaiyang Zhang } else { 78482fa3c77SKY Srinivasan ret = vmbus_sendpacket_ctl(out_channel, &nvmsg, 78595fa0405SHaiyang Zhang sizeof(struct nvsp_message), 786f1ea3cd7SHaiyang Zhang req_id, 78795fa0405SHaiyang Zhang VM_PKT_DATA_INBAND, 7883454323cSStephen Hemminger VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 78995fa0405SHaiyang Zhang } 79095fa0405SHaiyang Zhang 7911d06825bSHaiyang Zhang if (ret == 0) { 792b8b835a8Sstephen hemminger atomic_inc_return(&nvchan->queue_sends); 7935b54dac8SHaiyang Zhang 79446b4f7f5Sstephen hemminger if (ring_avail < RING_AVAIL_PERCENT_LOWATER) 795b8b835a8Sstephen hemminger netif_tx_stop_queue(txq); 7961d06825bSHaiyang Zhang } else if (ret == -EAGAIN) { 797b8b835a8Sstephen hemminger netif_tx_stop_queue(txq); 798b8b835a8Sstephen hemminger if (atomic_read(&nvchan->queue_sends) < 1) { 799b8b835a8Sstephen hemminger netif_tx_wake_queue(txq); 80033be96e4SHaiyang Zhang ret = -ENOSPC; 80133be96e4SHaiyang Zhang } 8021d06825bSHaiyang Zhang } else { 80395fa0405SHaiyang Zhang netdev_err(ndev, "Unable to send packet %p ret %d\n", 80495fa0405SHaiyang Zhang packet, ret); 8051d06825bSHaiyang Zhang } 80695fa0405SHaiyang Zhang 8077c3877f2SHaiyang Zhang return ret; 8087c3877f2SHaiyang Zhang } 8097c3877f2SHaiyang Zhang 810c85e4924SHaiyang Zhang /* Move packet out of multi send data (msd), and clear msd */ 811c85e4924SHaiyang Zhang static inline void move_pkt_msd(struct hv_netvsc_packet **msd_send, 812c85e4924SHaiyang Zhang struct sk_buff **msd_skb, 813c85e4924SHaiyang Zhang struct multi_send_data *msdp) 814c85e4924SHaiyang Zhang { 815c85e4924SHaiyang Zhang *msd_skb = msdp->skb; 816c85e4924SHaiyang Zhang *msd_send = msdp->pkt; 817c85e4924SHaiyang Zhang msdp->skb = NULL; 818c85e4924SHaiyang Zhang msdp->pkt = NULL; 819c85e4924SHaiyang Zhang msdp->count = 0; 820c85e4924SHaiyang Zhang } 821c85e4924SHaiyang Zhang 8227c3877f2SHaiyang Zhang int netvsc_send(struct hv_device *device, 82324476760SKY Srinivasan struct hv_netvsc_packet *packet, 824a9f2e2d6SKY Srinivasan struct rndis_message *rndis_msg, 8253a3d9a0aSKY Srinivasan struct hv_page_buffer **pb, 8263a3d9a0aSKY Srinivasan struct sk_buff *skb) 8277c3877f2SHaiyang Zhang { 828592b4fe8Sstephen hemminger struct netvsc_device *net_device = hv_device_to_netvsc_device(device); 8296c4c137eSStephen Hemminger int ret = 0; 830b8b835a8Sstephen hemminger struct netvsc_channel *nvchan; 8317c3877f2SHaiyang Zhang u32 pktlen = packet->total_data_buflen, msd_len = 0; 8327c3877f2SHaiyang Zhang unsigned int section_index = NETVSC_INVALID_INDEX; 8337c3877f2SHaiyang Zhang struct multi_send_data *msdp; 8347c3877f2SHaiyang Zhang struct hv_netvsc_packet *msd_send = NULL, *cur_send = NULL; 835c85e4924SHaiyang Zhang struct sk_buff *msd_skb = NULL; 836aa0a34beSHaiyang Zhang bool try_batch; 837bde79be5SKY Srinivasan bool xmit_more = (skb != NULL) ? skb->xmit_more : false; 8387c3877f2SHaiyang Zhang 839592b4fe8Sstephen hemminger /* If device is rescinded, return error and packet will get dropped. */ 840592b4fe8Sstephen hemminger if (unlikely(net_device->destroy)) 8417c3877f2SHaiyang Zhang return -ENODEV; 8427c3877f2SHaiyang Zhang 843e8f0a89cSVitaly Kuznetsov /* We may race with netvsc_connect_vsp()/netvsc_init_buf() and get 844e8f0a89cSVitaly Kuznetsov * here before the negotiation with the host is finished and 845e8f0a89cSVitaly Kuznetsov * send_section_map may not be allocated yet. 846e8f0a89cSVitaly Kuznetsov */ 8472d694d2aSstephen hemminger if (unlikely(!net_device->send_section_map)) 848e8f0a89cSVitaly Kuznetsov return -EAGAIN; 849e8f0a89cSVitaly Kuznetsov 850b8b835a8Sstephen hemminger nvchan = &net_device->chan_table[packet->q_idx]; 8517c3877f2SHaiyang Zhang packet->send_buf_index = NETVSC_INVALID_INDEX; 852aa0a34beSHaiyang Zhang packet->cp_partial = false; 8537c3877f2SHaiyang Zhang 854cf8190e4SHaiyang Zhang /* Send control message directly without accessing msd (Multi-Send 855cf8190e4SHaiyang Zhang * Data) field which may be changed during data packet processing. 856cf8190e4SHaiyang Zhang */ 857cf8190e4SHaiyang Zhang if (!skb) { 858cf8190e4SHaiyang Zhang cur_send = packet; 859cf8190e4SHaiyang Zhang goto send_now; 860cf8190e4SHaiyang Zhang } 861cf8190e4SHaiyang Zhang 8627c3877f2SHaiyang Zhang /* batch packets in send buffer if possible */ 863b8b835a8Sstephen hemminger msdp = &nvchan->msd; 8647c3877f2SHaiyang Zhang if (msdp->pkt) 8657c3877f2SHaiyang Zhang msd_len = msdp->pkt->total_data_buflen; 8667c3877f2SHaiyang Zhang 867ebc1dcf6Sstephen hemminger try_batch = msd_len > 0 && msdp->count < net_device->max_pkt; 868aa0a34beSHaiyang Zhang if (try_batch && msd_len + pktlen + net_device->pkt_align < 8697c3877f2SHaiyang Zhang net_device->send_section_size) { 8707c3877f2SHaiyang Zhang section_index = msdp->pkt->send_buf_index; 8717c3877f2SHaiyang Zhang 872aa0a34beSHaiyang Zhang } else if (try_batch && msd_len + packet->rmsg_size < 873aa0a34beSHaiyang Zhang net_device->send_section_size) { 874aa0a34beSHaiyang Zhang section_index = msdp->pkt->send_buf_index; 875aa0a34beSHaiyang Zhang packet->cp_partial = true; 876aa0a34beSHaiyang Zhang 877ebc1dcf6Sstephen hemminger } else if (pktlen + net_device->pkt_align < 8787c3877f2SHaiyang Zhang net_device->send_section_size) { 8797c3877f2SHaiyang Zhang section_index = netvsc_get_next_send_section(net_device); 8807c3877f2SHaiyang Zhang if (section_index != NETVSC_INVALID_INDEX) { 881c85e4924SHaiyang Zhang move_pkt_msd(&msd_send, &msd_skb, msdp); 8827c3877f2SHaiyang Zhang msd_len = 0; 8837c3877f2SHaiyang Zhang } 8847c3877f2SHaiyang Zhang } 8857c3877f2SHaiyang Zhang 8867c3877f2SHaiyang Zhang if (section_index != NETVSC_INVALID_INDEX) { 8877c3877f2SHaiyang Zhang netvsc_copy_to_send_buf(net_device, 8887c3877f2SHaiyang Zhang section_index, msd_len, 889694a9fb0SKY Srinivasan packet, rndis_msg, pb, skb); 890b08cc791SKY Srinivasan 891aa0a34beSHaiyang Zhang packet->send_buf_index = section_index; 892aa0a34beSHaiyang Zhang 893aa0a34beSHaiyang Zhang if (packet->cp_partial) { 894aa0a34beSHaiyang Zhang packet->page_buf_cnt -= packet->rmsg_pgcnt; 895aa0a34beSHaiyang Zhang packet->total_data_buflen = msd_len + packet->rmsg_size; 896aa0a34beSHaiyang Zhang } else { 897aa0a34beSHaiyang Zhang packet->page_buf_cnt = 0; 898aa0a34beSHaiyang Zhang packet->total_data_buflen += msd_len; 899aa0a34beSHaiyang Zhang } 9007c3877f2SHaiyang Zhang 901793e3955Sstephen hemminger if (msdp->pkt) { 902793e3955Sstephen hemminger packet->total_packets += msdp->pkt->total_packets; 903793e3955Sstephen hemminger packet->total_bytes += msdp->pkt->total_bytes; 904793e3955Sstephen hemminger } 905793e3955Sstephen hemminger 906c85e4924SHaiyang Zhang if (msdp->skb) 90717db4bceSStephen Hemminger dev_consume_skb_any(msdp->skb); 908ee90b812SHaiyang Zhang 909bde79be5SKY Srinivasan if (xmit_more && !packet->cp_partial) { 910c85e4924SHaiyang Zhang msdp->skb = skb; 9117c3877f2SHaiyang Zhang msdp->pkt = packet; 9127c3877f2SHaiyang Zhang msdp->count++; 9137c3877f2SHaiyang Zhang } else { 9147c3877f2SHaiyang Zhang cur_send = packet; 915c85e4924SHaiyang Zhang msdp->skb = NULL; 9167c3877f2SHaiyang Zhang msdp->pkt = NULL; 9177c3877f2SHaiyang Zhang msdp->count = 0; 9187c3877f2SHaiyang Zhang } 9197c3877f2SHaiyang Zhang } else { 920c85e4924SHaiyang Zhang move_pkt_msd(&msd_send, &msd_skb, msdp); 9217c3877f2SHaiyang Zhang cur_send = packet; 9227c3877f2SHaiyang Zhang } 9237c3877f2SHaiyang Zhang 9247c3877f2SHaiyang Zhang if (msd_send) { 9256c4c137eSStephen Hemminger int m_ret = netvsc_send_pkt(device, msd_send, net_device, 9260a1275caSVitaly Kuznetsov NULL, msd_skb); 9277c3877f2SHaiyang Zhang 9287c3877f2SHaiyang Zhang if (m_ret != 0) { 9297c3877f2SHaiyang Zhang netvsc_free_send_slot(net_device, 9307c3877f2SHaiyang Zhang msd_send->send_buf_index); 931c85e4924SHaiyang Zhang dev_kfree_skb_any(msd_skb); 9327c3877f2SHaiyang Zhang } 9337c3877f2SHaiyang Zhang } 9347c3877f2SHaiyang Zhang 935cf8190e4SHaiyang Zhang send_now: 9367c3877f2SHaiyang Zhang if (cur_send) 9370a1275caSVitaly Kuznetsov ret = netvsc_send_pkt(device, cur_send, net_device, pb, skb); 9387c3877f2SHaiyang Zhang 9397aab5159SJerry Snitselaar if (ret != 0 && section_index != NETVSC_INVALID_INDEX) 940d953ca4dSHaiyang Zhang netvsc_free_send_slot(net_device, section_index); 941d953ca4dSHaiyang Zhang 94295fa0405SHaiyang Zhang return ret; 94395fa0405SHaiyang Zhang } 94495fa0405SHaiyang Zhang 945c0b558e5SHaiyang Zhang static int netvsc_send_recv_completion(struct vmbus_channel *channel, 94663f6921dSHaiyang Zhang u64 transaction_id, u32 status) 94795fa0405SHaiyang Zhang { 94895fa0405SHaiyang Zhang struct nvsp_message recvcompMessage; 94995fa0405SHaiyang Zhang int ret; 95095fa0405SHaiyang Zhang 95195fa0405SHaiyang Zhang recvcompMessage.hdr.msg_type = 95295fa0405SHaiyang Zhang NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE; 95395fa0405SHaiyang Zhang 95463f6921dSHaiyang Zhang recvcompMessage.msg.v1_msg.send_rndis_pkt_complete.status = status; 95595fa0405SHaiyang Zhang 95695fa0405SHaiyang Zhang /* Send the completion */ 9575b54dac8SHaiyang Zhang ret = vmbus_sendpacket(channel, &recvcompMessage, 958c0b558e5SHaiyang Zhang sizeof(struct nvsp_message_header) + sizeof(u32), 959c0b558e5SHaiyang Zhang transaction_id, VM_PKT_COMP, 0); 96095fa0405SHaiyang Zhang 961c0b558e5SHaiyang Zhang return ret; 96295fa0405SHaiyang Zhang } 963c0b558e5SHaiyang Zhang 964c0b558e5SHaiyang Zhang static inline void count_recv_comp_slot(struct netvsc_device *nvdev, u16 q_idx, 965c0b558e5SHaiyang Zhang u32 *filled, u32 *avail) 966c0b558e5SHaiyang Zhang { 967b8b835a8Sstephen hemminger struct multi_recv_comp *mrc = &nvdev->chan_table[q_idx].mrc; 968b8b835a8Sstephen hemminger u32 first = mrc->first; 969b8b835a8Sstephen hemminger u32 next = mrc->next; 970c0b558e5SHaiyang Zhang 971c0b558e5SHaiyang Zhang *filled = (first > next) ? NETVSC_RECVSLOT_MAX - first + next : 972c0b558e5SHaiyang Zhang next - first; 973c0b558e5SHaiyang Zhang 974c0b558e5SHaiyang Zhang *avail = NETVSC_RECVSLOT_MAX - *filled - 1; 97595fa0405SHaiyang Zhang } 976c0b558e5SHaiyang Zhang 977c0b558e5SHaiyang Zhang /* Read the first filled slot, no change to index */ 978c0b558e5SHaiyang Zhang static inline struct recv_comp_data *read_recv_comp_slot(struct netvsc_device 979c0b558e5SHaiyang Zhang *nvdev, u16 q_idx) 980c0b558e5SHaiyang Zhang { 981b8b835a8Sstephen hemminger struct multi_recv_comp *mrc = &nvdev->chan_table[q_idx].mrc; 982c0b558e5SHaiyang Zhang u32 filled, avail; 983c0b558e5SHaiyang Zhang 984b8b835a8Sstephen hemminger if (unlikely(!mrc->buf)) 985c0b558e5SHaiyang Zhang return NULL; 986c0b558e5SHaiyang Zhang 987c0b558e5SHaiyang Zhang count_recv_comp_slot(nvdev, q_idx, &filled, &avail); 988c0b558e5SHaiyang Zhang if (!filled) 989c0b558e5SHaiyang Zhang return NULL; 990c0b558e5SHaiyang Zhang 991b8b835a8Sstephen hemminger return mrc->buf + mrc->first * sizeof(struct recv_comp_data); 992c0b558e5SHaiyang Zhang } 993c0b558e5SHaiyang Zhang 994c0b558e5SHaiyang Zhang /* Put the first filled slot back to available pool */ 995c0b558e5SHaiyang Zhang static inline void put_recv_comp_slot(struct netvsc_device *nvdev, u16 q_idx) 996c0b558e5SHaiyang Zhang { 997b8b835a8Sstephen hemminger struct multi_recv_comp *mrc = &nvdev->chan_table[q_idx].mrc; 998c0b558e5SHaiyang Zhang int num_recv; 999c0b558e5SHaiyang Zhang 1000b8b835a8Sstephen hemminger mrc->first = (mrc->first + 1) % NETVSC_RECVSLOT_MAX; 1001c0b558e5SHaiyang Zhang 1002c0b558e5SHaiyang Zhang num_recv = atomic_dec_return(&nvdev->num_outstanding_recvs); 1003c0b558e5SHaiyang Zhang 1004c0b558e5SHaiyang Zhang if (nvdev->destroy && num_recv == 0) 1005c0b558e5SHaiyang Zhang wake_up(&nvdev->wait_drain); 1006c0b558e5SHaiyang Zhang } 1007c0b558e5SHaiyang Zhang 1008c0b558e5SHaiyang Zhang /* Check and send pending recv completions */ 1009c0b558e5SHaiyang Zhang static void netvsc_chk_recv_comp(struct netvsc_device *nvdev, 1010c0b558e5SHaiyang Zhang struct vmbus_channel *channel, u16 q_idx) 1011c0b558e5SHaiyang Zhang { 1012c0b558e5SHaiyang Zhang struct recv_comp_data *rcd; 1013c0b558e5SHaiyang Zhang int ret; 1014c0b558e5SHaiyang Zhang 1015c0b558e5SHaiyang Zhang while (true) { 1016c0b558e5SHaiyang Zhang rcd = read_recv_comp_slot(nvdev, q_idx); 1017c0b558e5SHaiyang Zhang if (!rcd) 1018c0b558e5SHaiyang Zhang break; 1019c0b558e5SHaiyang Zhang 1020c0b558e5SHaiyang Zhang ret = netvsc_send_recv_completion(channel, rcd->tid, 1021c0b558e5SHaiyang Zhang rcd->status); 1022c0b558e5SHaiyang Zhang if (ret) 1023c0b558e5SHaiyang Zhang break; 1024c0b558e5SHaiyang Zhang 1025c0b558e5SHaiyang Zhang put_recv_comp_slot(nvdev, q_idx); 1026c0b558e5SHaiyang Zhang } 1027c0b558e5SHaiyang Zhang } 1028c0b558e5SHaiyang Zhang 1029c0b558e5SHaiyang Zhang #define NETVSC_RCD_WATERMARK 80 1030c0b558e5SHaiyang Zhang 1031c0b558e5SHaiyang Zhang /* Get next available slot */ 1032c0b558e5SHaiyang Zhang static inline struct recv_comp_data *get_recv_comp_slot( 1033c0b558e5SHaiyang Zhang struct netvsc_device *nvdev, struct vmbus_channel *channel, u16 q_idx) 1034c0b558e5SHaiyang Zhang { 1035b8b835a8Sstephen hemminger struct multi_recv_comp *mrc = &nvdev->chan_table[q_idx].mrc; 1036c0b558e5SHaiyang Zhang u32 filled, avail, next; 1037c0b558e5SHaiyang Zhang struct recv_comp_data *rcd; 1038c0b558e5SHaiyang Zhang 1039b8b835a8Sstephen hemminger if (unlikely(!nvdev->recv_section)) 1040c0b558e5SHaiyang Zhang return NULL; 1041c0b558e5SHaiyang Zhang 1042b8b835a8Sstephen hemminger if (unlikely(!mrc->buf)) 1043c0b558e5SHaiyang Zhang return NULL; 1044c0b558e5SHaiyang Zhang 1045c0b558e5SHaiyang Zhang if (atomic_read(&nvdev->num_outstanding_recvs) > 1046c0b558e5SHaiyang Zhang nvdev->recv_section->num_sub_allocs * NETVSC_RCD_WATERMARK / 100) 1047c0b558e5SHaiyang Zhang netvsc_chk_recv_comp(nvdev, channel, q_idx); 1048c0b558e5SHaiyang Zhang 1049c0b558e5SHaiyang Zhang count_recv_comp_slot(nvdev, q_idx, &filled, &avail); 1050c0b558e5SHaiyang Zhang if (!avail) 1051c0b558e5SHaiyang Zhang return NULL; 1052c0b558e5SHaiyang Zhang 1053b8b835a8Sstephen hemminger next = mrc->next; 1054b8b835a8Sstephen hemminger rcd = mrc->buf + next * sizeof(struct recv_comp_data); 1055b8b835a8Sstephen hemminger mrc->next = (next + 1) % NETVSC_RECVSLOT_MAX; 1056c0b558e5SHaiyang Zhang 1057c0b558e5SHaiyang Zhang atomic_inc(&nvdev->num_outstanding_recvs); 1058c0b558e5SHaiyang Zhang 1059c0b558e5SHaiyang Zhang return rcd; 106095fa0405SHaiyang Zhang } 106195fa0405SHaiyang Zhang 106215a863bfSstephen hemminger static int netvsc_receive(struct net_device *ndev, 1063dc54a08cSstephen hemminger struct netvsc_device *net_device, 1064dc54a08cSstephen hemminger struct net_device_context *net_device_ctx, 106597c1723aSKY Srinivasan struct hv_device *device, 1066dc54a08cSstephen hemminger struct vmbus_channel *channel, 1067f3dd3f47Sstephen hemminger const struct vmpacket_descriptor *desc, 1068dc54a08cSstephen hemminger struct nvsp_message *nvsp) 106995fa0405SHaiyang Zhang { 1070f3dd3f47Sstephen hemminger const struct vmtransfer_page_packet_header *vmxferpage_packet 1071f3dd3f47Sstephen hemminger = container_of(desc, const struct vmtransfer_page_packet_header, d); 107215a863bfSstephen hemminger u16 q_idx = channel->offermsg.offer.sub_channel_index; 1073dc54a08cSstephen hemminger char *recv_buf = net_device->recv_buf; 10744baab261SHaiyang Zhang u32 status = NVSP_STAT_SUCCESS; 107545326342SHaiyang Zhang int i; 107645326342SHaiyang Zhang int count = 0; 1077c0b558e5SHaiyang Zhang int ret; 107895fa0405SHaiyang Zhang 107995fa0405SHaiyang Zhang /* Make sure this is a valid nvsp packet */ 1080dc54a08cSstephen hemminger if (unlikely(nvsp->hdr.msg_type != NVSP_MSG1_TYPE_SEND_RNDIS_PKT)) { 1081dc54a08cSstephen hemminger netif_err(net_device_ctx, rx_err, ndev, 1082dc54a08cSstephen hemminger "Unknown nvsp packet type received %u\n", 1083dc54a08cSstephen hemminger nvsp->hdr.msg_type); 108415a863bfSstephen hemminger return 0; 108595fa0405SHaiyang Zhang } 108695fa0405SHaiyang Zhang 1087dc54a08cSstephen hemminger if (unlikely(vmxferpage_packet->xfer_pageset_id != NETVSC_RECEIVE_BUFFER_ID)) { 1088dc54a08cSstephen hemminger netif_err(net_device_ctx, rx_err, ndev, 1089dc54a08cSstephen hemminger "Invalid xfer page set id - expecting %x got %x\n", 1090dc54a08cSstephen hemminger NETVSC_RECEIVE_BUFFER_ID, 109195fa0405SHaiyang Zhang vmxferpage_packet->xfer_pageset_id); 109215a863bfSstephen hemminger return 0; 109395fa0405SHaiyang Zhang } 109495fa0405SHaiyang Zhang 10954baab261SHaiyang Zhang count = vmxferpage_packet->range_cnt; 109695fa0405SHaiyang Zhang 109795fa0405SHaiyang Zhang /* Each range represents 1 RNDIS pkt that contains 1 ethernet frame */ 10984baab261SHaiyang Zhang for (i = 0; i < count; i++) { 1099dc54a08cSstephen hemminger void *data = recv_buf 1100dc54a08cSstephen hemminger + vmxferpage_packet->ranges[i].byte_offset; 1101dc54a08cSstephen hemminger u32 buflen = vmxferpage_packet->ranges[i].byte_count; 110295fa0405SHaiyang Zhang 110395fa0405SHaiyang Zhang /* Pass it to the upper layer */ 1104dc54a08cSstephen hemminger status = rndis_filter_receive(ndev, net_device, device, 1105dc54a08cSstephen hemminger channel, data, buflen); 110695fa0405SHaiyang Zhang } 110795fa0405SHaiyang Zhang 110815a863bfSstephen hemminger if (net_device->chan_table[q_idx].mrc.buf) { 110915a863bfSstephen hemminger struct recv_comp_data *rcd; 111015a863bfSstephen hemminger 111115a863bfSstephen hemminger rcd = get_recv_comp_slot(net_device, channel, q_idx); 111215a863bfSstephen hemminger if (rcd) { 111315a863bfSstephen hemminger rcd->tid = vmxferpage_packet->d.trans_id; 111415a863bfSstephen hemminger rcd->status = status; 111515a863bfSstephen hemminger } else { 111615a863bfSstephen hemminger netdev_err(ndev, "Recv_comp full buf q:%hd, tid:%llx\n", 111715a863bfSstephen hemminger q_idx, vmxferpage_packet->d.trans_id); 111815a863bfSstephen hemminger } 111915a863bfSstephen hemminger } else { 1120c0b558e5SHaiyang Zhang ret = netvsc_send_recv_completion(channel, 1121c0b558e5SHaiyang Zhang vmxferpage_packet->d.trans_id, 1122c0b558e5SHaiyang Zhang status); 1123c0b558e5SHaiyang Zhang if (ret) 1124c0b558e5SHaiyang Zhang netdev_err(ndev, "Recv_comp q:%hd, tid:%llx, err:%d\n", 1125c0b558e5SHaiyang Zhang q_idx, vmxferpage_packet->d.trans_id, ret); 1126c0b558e5SHaiyang Zhang } 112715a863bfSstephen hemminger return count; 112895fa0405SHaiyang Zhang } 112995fa0405SHaiyang Zhang 11305b54dac8SHaiyang Zhang static void netvsc_send_table(struct hv_device *hdev, 113171790a27SHaiyang Zhang struct nvsp_message *nvmsg) 11325b54dac8SHaiyang Zhang { 11330a1275caSVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(hdev); 11347ce10124Sstephen hemminger struct net_device_context *net_device_ctx = netdev_priv(ndev); 11355b54dac8SHaiyang Zhang int i; 11365b54dac8SHaiyang Zhang u32 count, *tab; 11375b54dac8SHaiyang Zhang 11385b54dac8SHaiyang Zhang count = nvmsg->msg.v5_msg.send_table.count; 11395b54dac8SHaiyang Zhang if (count != VRSS_SEND_TAB_SIZE) { 11405b54dac8SHaiyang Zhang netdev_err(ndev, "Received wrong send-table size:%u\n", count); 11415b54dac8SHaiyang Zhang return; 11425b54dac8SHaiyang Zhang } 11435b54dac8SHaiyang Zhang 11445b54dac8SHaiyang Zhang tab = (u32 *)((unsigned long)&nvmsg->msg.v5_msg.send_table + 11455b54dac8SHaiyang Zhang nvmsg->msg.v5_msg.send_table.offset); 11465b54dac8SHaiyang Zhang 11475b54dac8SHaiyang Zhang for (i = 0; i < count; i++) 11487ce10124Sstephen hemminger net_device_ctx->tx_send_table[i] = tab[i]; 11495b54dac8SHaiyang Zhang } 11505b54dac8SHaiyang Zhang 1151f9a7da91SVitaly Kuznetsov static void netvsc_send_vf(struct net_device_context *net_device_ctx, 115271790a27SHaiyang Zhang struct nvsp_message *nvmsg) 115371790a27SHaiyang Zhang { 1154f9a7da91SVitaly Kuznetsov net_device_ctx->vf_alloc = nvmsg->msg.v4_msg.vf_assoc.allocated; 1155f9a7da91SVitaly Kuznetsov net_device_ctx->vf_serial = nvmsg->msg.v4_msg.vf_assoc.serial; 115671790a27SHaiyang Zhang } 115771790a27SHaiyang Zhang 115871790a27SHaiyang Zhang static inline void netvsc_receive_inband(struct hv_device *hdev, 1159f9a7da91SVitaly Kuznetsov struct net_device_context *net_device_ctx, 116071790a27SHaiyang Zhang struct nvsp_message *nvmsg) 116171790a27SHaiyang Zhang { 116271790a27SHaiyang Zhang switch (nvmsg->hdr.msg_type) { 116371790a27SHaiyang Zhang case NVSP_MSG5_TYPE_SEND_INDIRECTION_TABLE: 116471790a27SHaiyang Zhang netvsc_send_table(hdev, nvmsg); 116571790a27SHaiyang Zhang break; 116671790a27SHaiyang Zhang 116771790a27SHaiyang Zhang case NVSP_MSG4_TYPE_SEND_VF_ASSOCIATION: 1168f9a7da91SVitaly Kuznetsov netvsc_send_vf(net_device_ctx, nvmsg); 116971790a27SHaiyang Zhang break; 117071790a27SHaiyang Zhang } 117171790a27SHaiyang Zhang } 117271790a27SHaiyang Zhang 117315a863bfSstephen hemminger static int netvsc_process_raw_pkt(struct hv_device *device, 117499a50bb1SK. Y. Srinivasan struct vmbus_channel *channel, 117599a50bb1SK. Y. Srinivasan struct netvsc_device *net_device, 117699a50bb1SK. Y. Srinivasan struct net_device *ndev, 1177f9645430Sstephen hemminger const struct vmpacket_descriptor *desc, 1178f9645430Sstephen hemminger int budget) 117999a50bb1SK. Y. Srinivasan { 1180f9a7da91SVitaly Kuznetsov struct net_device_context *net_device_ctx = netdev_priv(ndev); 1181f3dd3f47Sstephen hemminger struct nvsp_message *nvmsg = hv_pkt_data(desc); 118299a50bb1SK. Y. Srinivasan 118399a50bb1SK. Y. Srinivasan switch (desc->type) { 118499a50bb1SK. Y. Srinivasan case VM_PKT_COMP: 1185f9645430Sstephen hemminger netvsc_send_completion(net_device, channel, device, 1186f9645430Sstephen hemminger desc, budget); 118799a50bb1SK. Y. Srinivasan break; 118899a50bb1SK. Y. Srinivasan 118999a50bb1SK. Y. Srinivasan case VM_PKT_DATA_USING_XFER_PAGES: 119015a863bfSstephen hemminger return netvsc_receive(ndev, net_device, net_device_ctx, 1191f3dd3f47Sstephen hemminger device, channel, desc, nvmsg); 119299a50bb1SK. Y. Srinivasan break; 119399a50bb1SK. Y. Srinivasan 119499a50bb1SK. Y. Srinivasan case VM_PKT_DATA_INBAND: 1195f9a7da91SVitaly Kuznetsov netvsc_receive_inband(device, net_device_ctx, nvmsg); 119699a50bb1SK. Y. Srinivasan break; 119799a50bb1SK. Y. Srinivasan 119899a50bb1SK. Y. Srinivasan default: 119999a50bb1SK. Y. Srinivasan netdev_err(ndev, "unhandled packet type %d, tid %llx\n", 1200f4f1c23dSstephen hemminger desc->type, desc->trans_id); 120199a50bb1SK. Y. Srinivasan break; 120299a50bb1SK. Y. Srinivasan } 120315a863bfSstephen hemminger 120415a863bfSstephen hemminger return 0; 120515a863bfSstephen hemminger } 120615a863bfSstephen hemminger 120715a863bfSstephen hemminger static struct hv_device *netvsc_channel_to_device(struct vmbus_channel *channel) 120815a863bfSstephen hemminger { 120915a863bfSstephen hemminger struct vmbus_channel *primary = channel->primary_channel; 121015a863bfSstephen hemminger 121115a863bfSstephen hemminger return primary ? primary->device_obj : channel->device_obj; 121215a863bfSstephen hemminger } 121315a863bfSstephen hemminger 1214262b7f14Sstephen hemminger /* Network processing softirq 1215262b7f14Sstephen hemminger * Process data in incoming ring buffer from host 1216262b7f14Sstephen hemminger * Stops when ring is empty or budget is met or exceeded. 1217262b7f14Sstephen hemminger */ 121815a863bfSstephen hemminger int netvsc_poll(struct napi_struct *napi, int budget) 121915a863bfSstephen hemminger { 122015a863bfSstephen hemminger struct netvsc_channel *nvchan 122115a863bfSstephen hemminger = container_of(napi, struct netvsc_channel, napi); 122215a863bfSstephen hemminger struct vmbus_channel *channel = nvchan->channel; 122315a863bfSstephen hemminger struct hv_device *device = netvsc_channel_to_device(channel); 122415a863bfSstephen hemminger u16 q_idx = channel->offermsg.offer.sub_channel_index; 122515a863bfSstephen hemminger struct net_device *ndev = hv_get_drvdata(device); 122615a863bfSstephen hemminger struct netvsc_device *net_device = net_device_to_netvsc_device(ndev); 122715a863bfSstephen hemminger int work_done = 0; 122815a863bfSstephen hemminger 1229f4f1c23dSstephen hemminger /* If starting a new interval */ 1230f4f1c23dSstephen hemminger if (!nvchan->desc) 1231f4f1c23dSstephen hemminger nvchan->desc = hv_pkt_iter_first(channel); 123215a863bfSstephen hemminger 1233f4f1c23dSstephen hemminger while (nvchan->desc && work_done < budget) { 1234f4f1c23dSstephen hemminger work_done += netvsc_process_raw_pkt(device, channel, net_device, 1235f9645430Sstephen hemminger ndev, nvchan->desc, budget); 1236f4f1c23dSstephen hemminger nvchan->desc = hv_pkt_iter_next(channel, nvchan->desc); 123715a863bfSstephen hemminger } 123815a863bfSstephen hemminger 1239f4f1c23dSstephen hemminger /* If receive ring was exhausted 1240f4f1c23dSstephen hemminger * and not doing busy poll 1241262b7f14Sstephen hemminger * then re-enable host interrupts 1242262b7f14Sstephen hemminger * and reschedule if ring is not empty. 1243262b7f14Sstephen hemminger */ 124415a863bfSstephen hemminger if (work_done < budget && 124515a863bfSstephen hemminger napi_complete_done(napi, work_done) && 124615a863bfSstephen hemminger hv_end_read(&channel->inbound) != 0) 124715a863bfSstephen hemminger napi_reschedule(napi); 124815a863bfSstephen hemminger 124915a863bfSstephen hemminger netvsc_chk_recv_comp(net_device, channel, q_idx); 1250f4f1c23dSstephen hemminger 1251f4f1c23dSstephen hemminger /* Driver may overshoot since multiple packets per descriptor */ 1252f4f1c23dSstephen hemminger return min(work_done, budget); 125399a50bb1SK. Y. Srinivasan } 125499a50bb1SK. Y. Srinivasan 1255262b7f14Sstephen hemminger /* Call back when data is available in host ring buffer. 1256262b7f14Sstephen hemminger * Processing is deferred until network softirq (NAPI) 1257262b7f14Sstephen hemminger */ 12585b54dac8SHaiyang Zhang void netvsc_channel_cb(void *context) 125995fa0405SHaiyang Zhang { 12606de38af6Sstephen hemminger struct netvsc_channel *nvchan = context; 126195fa0405SHaiyang Zhang 1262f4f1c23dSstephen hemminger if (napi_schedule_prep(&nvchan->napi)) { 126315a863bfSstephen hemminger /* disable interupts from host */ 12646de38af6Sstephen hemminger hv_begin_read(&nvchan->channel->inbound); 12650d6dd357Sstephen hemminger 1266f4f1c23dSstephen hemminger __napi_schedule(&nvchan->napi); 1267f4f1c23dSstephen hemminger } 126895fa0405SHaiyang Zhang } 126995fa0405SHaiyang Zhang 127095fa0405SHaiyang Zhang /* 127195fa0405SHaiyang Zhang * netvsc_device_add - Callback when the device belonging to this 127295fa0405SHaiyang Zhang * driver is added 127395fa0405SHaiyang Zhang */ 12742c7f83caSstephen hemminger int netvsc_device_add(struct hv_device *device, 12752c7f83caSstephen hemminger const struct netvsc_device_info *device_info) 127695fa0405SHaiyang Zhang { 127788098834SVitaly Kuznetsov int i, ret = 0; 12782c7f83caSstephen hemminger int ring_size = device_info->ring_size; 127995fa0405SHaiyang Zhang struct netvsc_device *net_device; 128088098834SVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 128188098834SVitaly Kuznetsov struct net_device_context *net_device_ctx = netdev_priv(ndev); 128295fa0405SHaiyang Zhang 128388098834SVitaly Kuznetsov net_device = alloc_net_device(); 1284b1c84927SDan Carpenter if (!net_device) 1285b1c84927SDan Carpenter return -ENOMEM; 128695fa0405SHaiyang Zhang 12875b54dac8SHaiyang Zhang net_device->ring_size = ring_size; 12885b54dac8SHaiyang Zhang 128915a863bfSstephen hemminger /* Because the device uses NAPI, all the interrupt batching and 129015a863bfSstephen hemminger * control is done via Net softirq, not the channel handling 129115a863bfSstephen hemminger */ 129215a863bfSstephen hemminger set_channel_read_mode(device->channel, HV_CALL_ISR); 129315a863bfSstephen hemminger 1294bffb1842SK. Y. Srinivasan /* If we're reopening the device we may have multiple queues, fill the 1295bffb1842SK. Y. Srinivasan * chn_table with the default channel to use it before subchannels are 1296bffb1842SK. Y. Srinivasan * opened. 1297bffb1842SK. Y. Srinivasan * Initialize the channel state before we open; 1298bffb1842SK. Y. Srinivasan * we can be interrupted as soon as we open the channel. 1299bffb1842SK. Y. Srinivasan */ 1300bffb1842SK. Y. Srinivasan 1301bffb1842SK. Y. Srinivasan for (i = 0; i < VRSS_CHANNEL_MAX; i++) { 1302bffb1842SK. Y. Srinivasan struct netvsc_channel *nvchan = &net_device->chan_table[i]; 1303bffb1842SK. Y. Srinivasan 1304bffb1842SK. Y. Srinivasan nvchan->channel = device->channel; 1305bffb1842SK. Y. Srinivasan } 1306bffb1842SK. Y. Srinivasan 13072be0f264Sstephen hemminger /* Enable NAPI handler before init callbacks */ 13082be0f264Sstephen hemminger netif_napi_add(ndev, &net_device->chan_table[0].napi, 13092be0f264Sstephen hemminger netvsc_poll, NAPI_POLL_WEIGHT); 13102be0f264Sstephen hemminger 131195fa0405SHaiyang Zhang /* Open the channel */ 131295fa0405SHaiyang Zhang ret = vmbus_open(device->channel, ring_size * PAGE_SIZE, 131395fa0405SHaiyang Zhang ring_size * PAGE_SIZE, NULL, 0, 13146de38af6Sstephen hemminger netvsc_channel_cb, 13156de38af6Sstephen hemminger net_device->chan_table); 131695fa0405SHaiyang Zhang 131795fa0405SHaiyang Zhang if (ret != 0) { 13182be0f264Sstephen hemminger netif_napi_del(&net_device->chan_table[0].napi); 131995fa0405SHaiyang Zhang netdev_err(ndev, "unable to open channel: %d\n", ret); 132095fa0405SHaiyang Zhang goto cleanup; 132195fa0405SHaiyang Zhang } 132295fa0405SHaiyang Zhang 132395fa0405SHaiyang Zhang /* Channel is opened */ 132493ba2222SVitaly Kuznetsov netdev_dbg(ndev, "hv_netvsc channel opened successfully\n"); 132595fa0405SHaiyang Zhang 132615a863bfSstephen hemminger napi_enable(&net_device->chan_table[0].napi); 132788098834SVitaly Kuznetsov 132888098834SVitaly Kuznetsov /* Writing nvdev pointer unlocks netvsc_send(), make sure chn_table is 132988098834SVitaly Kuznetsov * populated. 133088098834SVitaly Kuznetsov */ 1331545a8e79Sstephen hemminger rcu_assign_pointer(net_device_ctx->nvdev, net_device); 13325b54dac8SHaiyang Zhang 133395fa0405SHaiyang Zhang /* Connect with the NetVsp */ 133495790837Sstephen hemminger ret = netvsc_connect_vsp(device, net_device); 133595fa0405SHaiyang Zhang if (ret != 0) { 133695fa0405SHaiyang Zhang netdev_err(ndev, 133795fa0405SHaiyang Zhang "unable to connect to NetVSP - %d\n", ret); 133895fa0405SHaiyang Zhang goto close; 133995fa0405SHaiyang Zhang } 134095fa0405SHaiyang Zhang 134195fa0405SHaiyang Zhang return ret; 134295fa0405SHaiyang Zhang 134395fa0405SHaiyang Zhang close: 134476bb5db5Sstephen hemminger netif_napi_del(&net_device->chan_table[0].napi); 134515a863bfSstephen hemminger 134695fa0405SHaiyang Zhang /* Now, we can close the channel safely */ 134795fa0405SHaiyang Zhang vmbus_close(device->channel); 134895fa0405SHaiyang Zhang 134995fa0405SHaiyang Zhang cleanup: 1350545a8e79Sstephen hemminger free_netvsc_device(&net_device->rcu); 135195fa0405SHaiyang Zhang 135295fa0405SHaiyang Zhang return ret; 135395790837Sstephen hemminger 135495fa0405SHaiyang Zhang } 1355