195fa0405SHaiyang Zhang /* 295fa0405SHaiyang Zhang * Copyright (c) 2009, Microsoft Corporation. 395fa0405SHaiyang Zhang * 495fa0405SHaiyang Zhang * This program is free software; you can redistribute it and/or modify it 595fa0405SHaiyang Zhang * under the terms and conditions of the GNU General Public License, 695fa0405SHaiyang Zhang * version 2, as published by the Free Software Foundation. 795fa0405SHaiyang Zhang * 895fa0405SHaiyang Zhang * This program is distributed in the hope it will be useful, but WITHOUT 995fa0405SHaiyang Zhang * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 1095fa0405SHaiyang Zhang * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 1195fa0405SHaiyang Zhang * more details. 1295fa0405SHaiyang Zhang * 1395fa0405SHaiyang Zhang * You should have received a copy of the GNU General Public License along with 14adf8d3ffSJeff Kirsher * this program; if not, see <http://www.gnu.org/licenses/>. 1595fa0405SHaiyang Zhang * 1695fa0405SHaiyang Zhang * Authors: 1795fa0405SHaiyang Zhang * Haiyang Zhang <haiyangz@microsoft.com> 1895fa0405SHaiyang Zhang * Hank Janssen <hjanssen@microsoft.com> 1995fa0405SHaiyang Zhang */ 2095fa0405SHaiyang Zhang #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 2195fa0405SHaiyang Zhang 2295fa0405SHaiyang Zhang #include <linux/kernel.h> 2395fa0405SHaiyang Zhang #include <linux/sched.h> 2495fa0405SHaiyang Zhang #include <linux/wait.h> 2595fa0405SHaiyang Zhang #include <linux/mm.h> 2695fa0405SHaiyang Zhang #include <linux/delay.h> 2795fa0405SHaiyang Zhang #include <linux/io.h> 2895fa0405SHaiyang Zhang #include <linux/slab.h> 2995fa0405SHaiyang Zhang #include <linux/netdevice.h> 30f157e78dSHaiyang Zhang #include <linux/if_ether.h> 31d6472302SStephen Rothwell #include <linux/vmalloc.h> 32c25aaf81SKY Srinivasan #include <asm/sync_bitops.h> 3395fa0405SHaiyang Zhang 3495fa0405SHaiyang Zhang #include "hyperv_net.h" 3595fa0405SHaiyang Zhang 3684bf9cefSKY Srinivasan /* 3784bf9cefSKY Srinivasan * Switch the data path from the synthetic interface to the VF 3884bf9cefSKY Srinivasan * interface. 3984bf9cefSKY Srinivasan */ 400a1275caSVitaly Kuznetsov void netvsc_switch_datapath(struct net_device *ndev, bool vf) 4184bf9cefSKY Srinivasan { 423d541ac5SVitaly Kuznetsov struct net_device_context *net_device_ctx = netdev_priv(ndev); 433d541ac5SVitaly Kuznetsov struct hv_device *dev = net_device_ctx->device_ctx; 440a1275caSVitaly Kuznetsov struct netvsc_device *nv_dev = net_device_ctx->nvdev; 450a1275caSVitaly Kuznetsov struct nvsp_message *init_pkt = &nv_dev->channel_init_pkt; 4684bf9cefSKY Srinivasan 4784bf9cefSKY Srinivasan memset(init_pkt, 0, sizeof(struct nvsp_message)); 4884bf9cefSKY Srinivasan init_pkt->hdr.msg_type = NVSP_MSG4_TYPE_SWITCH_DATA_PATH; 4984bf9cefSKY Srinivasan if (vf) 5084bf9cefSKY Srinivasan init_pkt->msg.v4_msg.active_dp.active_datapath = 5184bf9cefSKY Srinivasan NVSP_DATAPATH_VF; 5284bf9cefSKY Srinivasan else 5384bf9cefSKY Srinivasan init_pkt->msg.v4_msg.active_dp.active_datapath = 5484bf9cefSKY Srinivasan NVSP_DATAPATH_SYNTHETIC; 5584bf9cefSKY Srinivasan 5684bf9cefSKY Srinivasan vmbus_sendpacket(dev->channel, init_pkt, 5784bf9cefSKY Srinivasan sizeof(struct nvsp_message), 5884bf9cefSKY Srinivasan (unsigned long)init_pkt, 5984bf9cefSKY Srinivasan VM_PKT_DATA_INBAND, 0); 6084bf9cefSKY Srinivasan } 6184bf9cefSKY Srinivasan 6288098834SVitaly Kuznetsov static struct netvsc_device *alloc_net_device(void) 6395fa0405SHaiyang Zhang { 6495fa0405SHaiyang Zhang struct netvsc_device *net_device; 6595fa0405SHaiyang Zhang 6695fa0405SHaiyang Zhang net_device = kzalloc(sizeof(struct netvsc_device), GFP_KERNEL); 6795fa0405SHaiyang Zhang if (!net_device) 6895fa0405SHaiyang Zhang return NULL; 6995fa0405SHaiyang Zhang 70b8b835a8Sstephen hemminger net_device->chan_table[0].mrc.buf 71b8b835a8Sstephen hemminger = vzalloc(NETVSC_RECVSLOT_MAX * sizeof(struct recv_comp_data)); 72c0b558e5SHaiyang Zhang 73dc5cd894SHaiyang Zhang init_waitqueue_head(&net_device->wait_drain); 7495fa0405SHaiyang Zhang net_device->destroy = false; 7584bf9cefSKY Srinivasan atomic_set(&net_device->open_cnt, 0); 767c3877f2SHaiyang Zhang net_device->max_pkt = RNDIS_MAX_PKT_DEFAULT; 777c3877f2SHaiyang Zhang net_device->pkt_align = RNDIS_PKT_ALIGN_DEFAULT; 78fd612602SStephen Hemminger init_completion(&net_device->channel_init_wait); 797c3877f2SHaiyang Zhang 8095fa0405SHaiyang Zhang return net_device; 8195fa0405SHaiyang Zhang } 8295fa0405SHaiyang Zhang 83f90251c8SHaiyang Zhang static void free_netvsc_device(struct netvsc_device *nvdev) 84f90251c8SHaiyang Zhang { 85c0b558e5SHaiyang Zhang int i; 86c0b558e5SHaiyang Zhang 87c0b558e5SHaiyang Zhang for (i = 0; i < VRSS_CHANNEL_MAX; i++) 88b8b835a8Sstephen hemminger vfree(nvdev->chan_table[i].mrc.buf); 89c0b558e5SHaiyang Zhang 90f90251c8SHaiyang Zhang kfree(nvdev); 91f90251c8SHaiyang Zhang } 92f90251c8SHaiyang Zhang 9346b4f7f5Sstephen hemminger 9446b4f7f5Sstephen hemminger static inline bool netvsc_channel_idle(const struct netvsc_device *net_device, 9546b4f7f5Sstephen hemminger u16 q_idx) 9646b4f7f5Sstephen hemminger { 9746b4f7f5Sstephen hemminger const struct netvsc_channel *nvchan = &net_device->chan_table[q_idx]; 9846b4f7f5Sstephen hemminger 9946b4f7f5Sstephen hemminger return atomic_read(&net_device->num_outstanding_recvs) == 0 && 10046b4f7f5Sstephen hemminger atomic_read(&nvchan->queue_sends) == 0; 10146b4f7f5Sstephen hemminger } 10246b4f7f5Sstephen hemminger 10395fa0405SHaiyang Zhang static struct netvsc_device *get_outbound_net_device(struct hv_device *device) 10495fa0405SHaiyang Zhang { 1052625466dSVitaly Kuznetsov struct netvsc_device *net_device = hv_device_to_netvsc_device(device); 10695fa0405SHaiyang Zhang 10795fa0405SHaiyang Zhang if (net_device && net_device->destroy) 10895fa0405SHaiyang Zhang net_device = NULL; 10995fa0405SHaiyang Zhang 11095fa0405SHaiyang Zhang return net_device; 11195fa0405SHaiyang Zhang } 11295fa0405SHaiyang Zhang 1137a2a0a84SStephen Hemminger static void netvsc_destroy_buf(struct hv_device *device) 11495fa0405SHaiyang Zhang { 11595fa0405SHaiyang Zhang struct nvsp_message *revoke_packet; 1163d541ac5SVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 1172625466dSVitaly Kuznetsov struct netvsc_device *net_device = net_device_to_netvsc_device(ndev); 1187a2a0a84SStephen Hemminger int ret; 11995fa0405SHaiyang Zhang 12095fa0405SHaiyang Zhang /* 12195fa0405SHaiyang Zhang * If we got a section count, it means we received a 12295fa0405SHaiyang Zhang * SendReceiveBufferComplete msg (ie sent 12395fa0405SHaiyang Zhang * NvspMessage1TypeSendReceiveBuffer msg) therefore, we need 12495fa0405SHaiyang Zhang * to send a revoke msg here 12595fa0405SHaiyang Zhang */ 12695fa0405SHaiyang Zhang if (net_device->recv_section_cnt) { 12795fa0405SHaiyang Zhang /* Send the revoke receive buffer */ 12895fa0405SHaiyang Zhang revoke_packet = &net_device->revoke_packet; 12995fa0405SHaiyang Zhang memset(revoke_packet, 0, sizeof(struct nvsp_message)); 13095fa0405SHaiyang Zhang 13195fa0405SHaiyang Zhang revoke_packet->hdr.msg_type = 13295fa0405SHaiyang Zhang NVSP_MSG1_TYPE_REVOKE_RECV_BUF; 13395fa0405SHaiyang Zhang revoke_packet->msg.v1_msg. 13495fa0405SHaiyang Zhang revoke_recv_buf.id = NETVSC_RECEIVE_BUFFER_ID; 13595fa0405SHaiyang Zhang 1363d541ac5SVitaly Kuznetsov ret = vmbus_sendpacket(device->channel, 13795fa0405SHaiyang Zhang revoke_packet, 13895fa0405SHaiyang Zhang sizeof(struct nvsp_message), 13995fa0405SHaiyang Zhang (unsigned long)revoke_packet, 14095fa0405SHaiyang Zhang VM_PKT_DATA_INBAND, 0); 14195fa0405SHaiyang Zhang /* 14295fa0405SHaiyang Zhang * If we failed here, we might as well return and 14395fa0405SHaiyang Zhang * have a leak rather than continue and a bugchk 14495fa0405SHaiyang Zhang */ 14595fa0405SHaiyang Zhang if (ret != 0) { 14695fa0405SHaiyang Zhang netdev_err(ndev, "unable to send " 14795fa0405SHaiyang Zhang "revoke receive buffer to netvsp\n"); 1487a2a0a84SStephen Hemminger return; 14995fa0405SHaiyang Zhang } 15095fa0405SHaiyang Zhang } 15195fa0405SHaiyang Zhang 15295fa0405SHaiyang Zhang /* Teardown the gpadl on the vsp end */ 15395fa0405SHaiyang Zhang if (net_device->recv_buf_gpadl_handle) { 1543d541ac5SVitaly Kuznetsov ret = vmbus_teardown_gpadl(device->channel, 15595fa0405SHaiyang Zhang net_device->recv_buf_gpadl_handle); 15695fa0405SHaiyang Zhang 15795fa0405SHaiyang Zhang /* If we failed here, we might as well return and have a leak 15895fa0405SHaiyang Zhang * rather than continue and a bugchk 15995fa0405SHaiyang Zhang */ 16095fa0405SHaiyang Zhang if (ret != 0) { 16195fa0405SHaiyang Zhang netdev_err(ndev, 16295fa0405SHaiyang Zhang "unable to teardown receive buffer's gpadl\n"); 1637a2a0a84SStephen Hemminger return; 16495fa0405SHaiyang Zhang } 16595fa0405SHaiyang Zhang net_device->recv_buf_gpadl_handle = 0; 16695fa0405SHaiyang Zhang } 16795fa0405SHaiyang Zhang 16895fa0405SHaiyang Zhang if (net_device->recv_buf) { 16995fa0405SHaiyang Zhang /* Free up the receive buffer */ 170b679ef73SHaiyang Zhang vfree(net_device->recv_buf); 17195fa0405SHaiyang Zhang net_device->recv_buf = NULL; 17295fa0405SHaiyang Zhang } 17395fa0405SHaiyang Zhang 17495fa0405SHaiyang Zhang if (net_device->recv_section) { 17595fa0405SHaiyang Zhang net_device->recv_section_cnt = 0; 17695fa0405SHaiyang Zhang kfree(net_device->recv_section); 17795fa0405SHaiyang Zhang net_device->recv_section = NULL; 17895fa0405SHaiyang Zhang } 17995fa0405SHaiyang Zhang 180c25aaf81SKY Srinivasan /* Deal with the send buffer we may have setup. 181c25aaf81SKY Srinivasan * If we got a send section size, it means we received a 182c51ed182SHaiyang Zhang * NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE msg (ie sent 183c51ed182SHaiyang Zhang * NVSP_MSG1_TYPE_SEND_SEND_BUF msg) therefore, we need 184c25aaf81SKY Srinivasan * to send a revoke msg here 185c25aaf81SKY Srinivasan */ 186c25aaf81SKY Srinivasan if (net_device->send_section_size) { 187c25aaf81SKY Srinivasan /* Send the revoke receive buffer */ 188c25aaf81SKY Srinivasan revoke_packet = &net_device->revoke_packet; 189c25aaf81SKY Srinivasan memset(revoke_packet, 0, sizeof(struct nvsp_message)); 190c25aaf81SKY Srinivasan 191c25aaf81SKY Srinivasan revoke_packet->hdr.msg_type = 192c25aaf81SKY Srinivasan NVSP_MSG1_TYPE_REVOKE_SEND_BUF; 193c51ed182SHaiyang Zhang revoke_packet->msg.v1_msg.revoke_send_buf.id = 194c51ed182SHaiyang Zhang NETVSC_SEND_BUFFER_ID; 195c25aaf81SKY Srinivasan 1963d541ac5SVitaly Kuznetsov ret = vmbus_sendpacket(device->channel, 197c25aaf81SKY Srinivasan revoke_packet, 198c25aaf81SKY Srinivasan sizeof(struct nvsp_message), 199c25aaf81SKY Srinivasan (unsigned long)revoke_packet, 200c25aaf81SKY Srinivasan VM_PKT_DATA_INBAND, 0); 201c25aaf81SKY Srinivasan /* If we failed here, we might as well return and 202c25aaf81SKY Srinivasan * have a leak rather than continue and a bugchk 203c25aaf81SKY Srinivasan */ 204c25aaf81SKY Srinivasan if (ret != 0) { 205c25aaf81SKY Srinivasan netdev_err(ndev, "unable to send " 206c25aaf81SKY Srinivasan "revoke send buffer to netvsp\n"); 2077a2a0a84SStephen Hemminger return; 208c25aaf81SKY Srinivasan } 209c25aaf81SKY Srinivasan } 210c25aaf81SKY Srinivasan /* Teardown the gpadl on the vsp end */ 211c25aaf81SKY Srinivasan if (net_device->send_buf_gpadl_handle) { 2123d541ac5SVitaly Kuznetsov ret = vmbus_teardown_gpadl(device->channel, 213c25aaf81SKY Srinivasan net_device->send_buf_gpadl_handle); 214c25aaf81SKY Srinivasan 215c25aaf81SKY Srinivasan /* If we failed here, we might as well return and have a leak 216c25aaf81SKY Srinivasan * rather than continue and a bugchk 217c25aaf81SKY Srinivasan */ 218c25aaf81SKY Srinivasan if (ret != 0) { 219c25aaf81SKY Srinivasan netdev_err(ndev, 220c25aaf81SKY Srinivasan "unable to teardown send buffer's gpadl\n"); 2217a2a0a84SStephen Hemminger return; 222c25aaf81SKY Srinivasan } 2232f18423dSDave Jones net_device->send_buf_gpadl_handle = 0; 224c25aaf81SKY Srinivasan } 225c25aaf81SKY Srinivasan if (net_device->send_buf) { 226c51ed182SHaiyang Zhang /* Free up the send buffer */ 22706b47aacSKY Srinivasan vfree(net_device->send_buf); 228c25aaf81SKY Srinivasan net_device->send_buf = NULL; 229c25aaf81SKY Srinivasan } 230c25aaf81SKY Srinivasan kfree(net_device->send_section_map); 23195fa0405SHaiyang Zhang } 23295fa0405SHaiyang Zhang 233c25aaf81SKY Srinivasan static int netvsc_init_buf(struct hv_device *device) 23495fa0405SHaiyang Zhang { 23595fa0405SHaiyang Zhang int ret = 0; 23695fa0405SHaiyang Zhang struct netvsc_device *net_device; 23795fa0405SHaiyang Zhang struct nvsp_message *init_packet; 23895fa0405SHaiyang Zhang struct net_device *ndev; 2390a726c2bSK. Y. Srinivasan int node; 24095fa0405SHaiyang Zhang 24195fa0405SHaiyang Zhang net_device = get_outbound_net_device(device); 24295fa0405SHaiyang Zhang if (!net_device) 24395fa0405SHaiyang Zhang return -ENODEV; 2440a1275caSVitaly Kuznetsov ndev = hv_get_drvdata(device); 24595fa0405SHaiyang Zhang 2460a726c2bSK. Y. Srinivasan node = cpu_to_node(device->channel->target_cpu); 2470a726c2bSK. Y. Srinivasan net_device->recv_buf = vzalloc_node(net_device->recv_buf_size, node); 2480a726c2bSK. Y. Srinivasan if (!net_device->recv_buf) 249b679ef73SHaiyang Zhang net_device->recv_buf = vzalloc(net_device->recv_buf_size); 2500a726c2bSK. Y. Srinivasan 25195fa0405SHaiyang Zhang if (!net_device->recv_buf) { 25295fa0405SHaiyang Zhang netdev_err(ndev, "unable to allocate receive " 25395fa0405SHaiyang Zhang "buffer of size %d\n", net_device->recv_buf_size); 25495fa0405SHaiyang Zhang ret = -ENOMEM; 25595fa0405SHaiyang Zhang goto cleanup; 25695fa0405SHaiyang Zhang } 25795fa0405SHaiyang Zhang 25895fa0405SHaiyang Zhang /* 25995fa0405SHaiyang Zhang * Establish the gpadl handle for this buffer on this 26095fa0405SHaiyang Zhang * channel. Note: This call uses the vmbus connection rather 26195fa0405SHaiyang Zhang * than the channel to establish the gpadl handle. 26295fa0405SHaiyang Zhang */ 26395fa0405SHaiyang Zhang ret = vmbus_establish_gpadl(device->channel, net_device->recv_buf, 26495fa0405SHaiyang Zhang net_device->recv_buf_size, 26595fa0405SHaiyang Zhang &net_device->recv_buf_gpadl_handle); 26695fa0405SHaiyang Zhang if (ret != 0) { 26795fa0405SHaiyang Zhang netdev_err(ndev, 26895fa0405SHaiyang Zhang "unable to establish receive buffer's gpadl\n"); 26995fa0405SHaiyang Zhang goto cleanup; 27095fa0405SHaiyang Zhang } 27195fa0405SHaiyang Zhang 27295fa0405SHaiyang Zhang /* Notify the NetVsp of the gpadl handle */ 27395fa0405SHaiyang Zhang init_packet = &net_device->channel_init_pkt; 27495fa0405SHaiyang Zhang 27595fa0405SHaiyang Zhang memset(init_packet, 0, sizeof(struct nvsp_message)); 27695fa0405SHaiyang Zhang 27795fa0405SHaiyang Zhang init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_RECV_BUF; 27895fa0405SHaiyang Zhang init_packet->msg.v1_msg.send_recv_buf. 27995fa0405SHaiyang Zhang gpadl_handle = net_device->recv_buf_gpadl_handle; 28095fa0405SHaiyang Zhang init_packet->msg.v1_msg. 28195fa0405SHaiyang Zhang send_recv_buf.id = NETVSC_RECEIVE_BUFFER_ID; 28295fa0405SHaiyang Zhang 28395fa0405SHaiyang Zhang /* Send the gpadl notification request */ 28495fa0405SHaiyang Zhang ret = vmbus_sendpacket(device->channel, init_packet, 28595fa0405SHaiyang Zhang sizeof(struct nvsp_message), 28695fa0405SHaiyang Zhang (unsigned long)init_packet, 28795fa0405SHaiyang Zhang VM_PKT_DATA_INBAND, 28895fa0405SHaiyang Zhang VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 28995fa0405SHaiyang Zhang if (ret != 0) { 29095fa0405SHaiyang Zhang netdev_err(ndev, 29195fa0405SHaiyang Zhang "unable to send receive buffer's gpadl to netvsp\n"); 29295fa0405SHaiyang Zhang goto cleanup; 29395fa0405SHaiyang Zhang } 29495fa0405SHaiyang Zhang 2955362855aSVitaly Kuznetsov wait_for_completion(&net_device->channel_init_wait); 29695fa0405SHaiyang Zhang 29795fa0405SHaiyang Zhang /* Check the response */ 29895fa0405SHaiyang Zhang if (init_packet->msg.v1_msg. 29995fa0405SHaiyang Zhang send_recv_buf_complete.status != NVSP_STAT_SUCCESS) { 30095fa0405SHaiyang Zhang netdev_err(ndev, "Unable to complete receive buffer " 30195fa0405SHaiyang Zhang "initialization with NetVsp - status %d\n", 30295fa0405SHaiyang Zhang init_packet->msg.v1_msg. 30395fa0405SHaiyang Zhang send_recv_buf_complete.status); 30495fa0405SHaiyang Zhang ret = -EINVAL; 30595fa0405SHaiyang Zhang goto cleanup; 30695fa0405SHaiyang Zhang } 30795fa0405SHaiyang Zhang 30895fa0405SHaiyang Zhang /* Parse the response */ 30995fa0405SHaiyang Zhang 31095fa0405SHaiyang Zhang net_device->recv_section_cnt = init_packet->msg. 31195fa0405SHaiyang Zhang v1_msg.send_recv_buf_complete.num_sections; 31295fa0405SHaiyang Zhang 313c1813200SHaiyang Zhang net_device->recv_section = kmemdup( 314c1813200SHaiyang Zhang init_packet->msg.v1_msg.send_recv_buf_complete.sections, 315c1813200SHaiyang Zhang net_device->recv_section_cnt * 316c1813200SHaiyang Zhang sizeof(struct nvsp_1_receive_buffer_section), 31795fa0405SHaiyang Zhang GFP_KERNEL); 31895fa0405SHaiyang Zhang if (net_device->recv_section == NULL) { 31995fa0405SHaiyang Zhang ret = -EINVAL; 32095fa0405SHaiyang Zhang goto cleanup; 32195fa0405SHaiyang Zhang } 32295fa0405SHaiyang Zhang 32395fa0405SHaiyang Zhang /* 32495fa0405SHaiyang Zhang * For 1st release, there should only be 1 section that represents the 32595fa0405SHaiyang Zhang * entire receive buffer 32695fa0405SHaiyang Zhang */ 32795fa0405SHaiyang Zhang if (net_device->recv_section_cnt != 1 || 32895fa0405SHaiyang Zhang net_device->recv_section->offset != 0) { 32995fa0405SHaiyang Zhang ret = -EINVAL; 33095fa0405SHaiyang Zhang goto cleanup; 33195fa0405SHaiyang Zhang } 33295fa0405SHaiyang Zhang 333c25aaf81SKY Srinivasan /* Now setup the send buffer. 334c25aaf81SKY Srinivasan */ 3355defde59SK. Y. Srinivasan net_device->send_buf = vzalloc_node(net_device->send_buf_size, node); 3365defde59SK. Y. Srinivasan if (!net_device->send_buf) 33706b47aacSKY Srinivasan net_device->send_buf = vzalloc(net_device->send_buf_size); 338c25aaf81SKY Srinivasan if (!net_device->send_buf) { 339c25aaf81SKY Srinivasan netdev_err(ndev, "unable to allocate send " 340c25aaf81SKY Srinivasan "buffer of size %d\n", net_device->send_buf_size); 341c25aaf81SKY Srinivasan ret = -ENOMEM; 342c25aaf81SKY Srinivasan goto cleanup; 343c25aaf81SKY Srinivasan } 344c25aaf81SKY Srinivasan 345c25aaf81SKY Srinivasan /* Establish the gpadl handle for this buffer on this 346c25aaf81SKY Srinivasan * channel. Note: This call uses the vmbus connection rather 347c25aaf81SKY Srinivasan * than the channel to establish the gpadl handle. 348c25aaf81SKY Srinivasan */ 349c25aaf81SKY Srinivasan ret = vmbus_establish_gpadl(device->channel, net_device->send_buf, 350c25aaf81SKY Srinivasan net_device->send_buf_size, 351c25aaf81SKY Srinivasan &net_device->send_buf_gpadl_handle); 352c25aaf81SKY Srinivasan if (ret != 0) { 353c25aaf81SKY Srinivasan netdev_err(ndev, 354c25aaf81SKY Srinivasan "unable to establish send buffer's gpadl\n"); 355c25aaf81SKY Srinivasan goto cleanup; 356c25aaf81SKY Srinivasan } 357c25aaf81SKY Srinivasan 358c25aaf81SKY Srinivasan /* Notify the NetVsp of the gpadl handle */ 359c25aaf81SKY Srinivasan init_packet = &net_device->channel_init_pkt; 360c25aaf81SKY Srinivasan memset(init_packet, 0, sizeof(struct nvsp_message)); 361c25aaf81SKY Srinivasan init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_SEND_BUF; 362c51ed182SHaiyang Zhang init_packet->msg.v1_msg.send_send_buf.gpadl_handle = 363c25aaf81SKY Srinivasan net_device->send_buf_gpadl_handle; 364c51ed182SHaiyang Zhang init_packet->msg.v1_msg.send_send_buf.id = NETVSC_SEND_BUFFER_ID; 365c25aaf81SKY Srinivasan 366c25aaf81SKY Srinivasan /* Send the gpadl notification request */ 367c25aaf81SKY Srinivasan ret = vmbus_sendpacket(device->channel, init_packet, 368c25aaf81SKY Srinivasan sizeof(struct nvsp_message), 369c25aaf81SKY Srinivasan (unsigned long)init_packet, 370c25aaf81SKY Srinivasan VM_PKT_DATA_INBAND, 371c25aaf81SKY Srinivasan VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 372c25aaf81SKY Srinivasan if (ret != 0) { 373c25aaf81SKY Srinivasan netdev_err(ndev, 374c25aaf81SKY Srinivasan "unable to send send buffer's gpadl to netvsp\n"); 375c25aaf81SKY Srinivasan goto cleanup; 376c25aaf81SKY Srinivasan } 377c25aaf81SKY Srinivasan 3785362855aSVitaly Kuznetsov wait_for_completion(&net_device->channel_init_wait); 379c25aaf81SKY Srinivasan 380c25aaf81SKY Srinivasan /* Check the response */ 381c25aaf81SKY Srinivasan if (init_packet->msg.v1_msg. 382c25aaf81SKY Srinivasan send_send_buf_complete.status != NVSP_STAT_SUCCESS) { 383c25aaf81SKY Srinivasan netdev_err(ndev, "Unable to complete send buffer " 384c25aaf81SKY Srinivasan "initialization with NetVsp - status %d\n", 385c25aaf81SKY Srinivasan init_packet->msg.v1_msg. 386c51ed182SHaiyang Zhang send_send_buf_complete.status); 387c25aaf81SKY Srinivasan ret = -EINVAL; 388c25aaf81SKY Srinivasan goto cleanup; 389c25aaf81SKY Srinivasan } 390c25aaf81SKY Srinivasan 391c25aaf81SKY Srinivasan /* Parse the response */ 392c25aaf81SKY Srinivasan net_device->send_section_size = init_packet->msg. 393c25aaf81SKY Srinivasan v1_msg.send_send_buf_complete.section_size; 394c25aaf81SKY Srinivasan 395c25aaf81SKY Srinivasan /* Section count is simply the size divided by the section size. 396c25aaf81SKY Srinivasan */ 397c25aaf81SKY Srinivasan net_device->send_section_cnt = 398c25aaf81SKY Srinivasan net_device->send_buf_size / net_device->send_section_size; 399c25aaf81SKY Srinivasan 40093ba2222SVitaly Kuznetsov netdev_dbg(ndev, "Send section size: %d, Section count:%d\n", 401c25aaf81SKY Srinivasan net_device->send_section_size, net_device->send_section_cnt); 402c25aaf81SKY Srinivasan 403c25aaf81SKY Srinivasan /* Setup state for managing the send buffer. */ 404c25aaf81SKY Srinivasan net_device->map_words = DIV_ROUND_UP(net_device->send_section_cnt, 405c25aaf81SKY Srinivasan BITS_PER_LONG); 406c25aaf81SKY Srinivasan 407e53a9c2aSStephen Hemminger net_device->send_section_map = kcalloc(net_device->map_words, 408e53a9c2aSStephen Hemminger sizeof(ulong), GFP_KERNEL); 409dd1d3f8fSWei Yongjun if (net_device->send_section_map == NULL) { 410dd1d3f8fSWei Yongjun ret = -ENOMEM; 411c25aaf81SKY Srinivasan goto cleanup; 412dd1d3f8fSWei Yongjun } 413c25aaf81SKY Srinivasan 41495fa0405SHaiyang Zhang goto exit; 41595fa0405SHaiyang Zhang 41695fa0405SHaiyang Zhang cleanup: 4173d541ac5SVitaly Kuznetsov netvsc_destroy_buf(device); 41895fa0405SHaiyang Zhang 41995fa0405SHaiyang Zhang exit: 42095fa0405SHaiyang Zhang return ret; 42195fa0405SHaiyang Zhang } 42295fa0405SHaiyang Zhang 423f157e78dSHaiyang Zhang /* Negotiate NVSP protocol version */ 424f157e78dSHaiyang Zhang static int negotiate_nvsp_ver(struct hv_device *device, 425f157e78dSHaiyang Zhang struct netvsc_device *net_device, 426f157e78dSHaiyang Zhang struct nvsp_message *init_packet, 427f157e78dSHaiyang Zhang u32 nvsp_ver) 42895fa0405SHaiyang Zhang { 4290a1275caSVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 4307390fe9cSNicholas Mc Guire int ret; 431f157e78dSHaiyang Zhang 432f157e78dSHaiyang Zhang memset(init_packet, 0, sizeof(struct nvsp_message)); 433f157e78dSHaiyang Zhang init_packet->hdr.msg_type = NVSP_MSG_TYPE_INIT; 434f157e78dSHaiyang Zhang init_packet->msg.init_msg.init.min_protocol_ver = nvsp_ver; 435f157e78dSHaiyang Zhang init_packet->msg.init_msg.init.max_protocol_ver = nvsp_ver; 436f157e78dSHaiyang Zhang 437f157e78dSHaiyang Zhang /* Send the init request */ 438f157e78dSHaiyang Zhang ret = vmbus_sendpacket(device->channel, init_packet, 439f157e78dSHaiyang Zhang sizeof(struct nvsp_message), 440f157e78dSHaiyang Zhang (unsigned long)init_packet, 441f157e78dSHaiyang Zhang VM_PKT_DATA_INBAND, 442f157e78dSHaiyang Zhang VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 443f157e78dSHaiyang Zhang 444f157e78dSHaiyang Zhang if (ret != 0) 445f157e78dSHaiyang Zhang return ret; 446f157e78dSHaiyang Zhang 4475362855aSVitaly Kuznetsov wait_for_completion(&net_device->channel_init_wait); 448f157e78dSHaiyang Zhang 449f157e78dSHaiyang Zhang if (init_packet->msg.init_msg.init_complete.status != 450f157e78dSHaiyang Zhang NVSP_STAT_SUCCESS) 451f157e78dSHaiyang Zhang return -EINVAL; 452f157e78dSHaiyang Zhang 453a1eabb01SHaiyang Zhang if (nvsp_ver == NVSP_PROTOCOL_VERSION_1) 454f157e78dSHaiyang Zhang return 0; 455f157e78dSHaiyang Zhang 45671790a27SHaiyang Zhang /* NVSPv2 or later: Send NDIS config */ 457f157e78dSHaiyang Zhang memset(init_packet, 0, sizeof(struct nvsp_message)); 458f157e78dSHaiyang Zhang init_packet->hdr.msg_type = NVSP_MSG2_TYPE_SEND_NDIS_CONFIG; 4590a1275caSVitaly Kuznetsov init_packet->msg.v2_msg.send_ndis_config.mtu = ndev->mtu + ETH_HLEN; 4601f5f3a75SHaiyang Zhang init_packet->msg.v2_msg.send_ndis_config.capability.ieee8021q = 1; 461f157e78dSHaiyang Zhang 4627f5d5af0SHaiyang Zhang if (nvsp_ver >= NVSP_PROTOCOL_VERSION_5) { 46371790a27SHaiyang Zhang init_packet->msg.v2_msg.send_ndis_config.capability.sriov = 1; 46471790a27SHaiyang Zhang 4657f5d5af0SHaiyang Zhang /* Teaming bit is needed to receive link speed updates */ 4667f5d5af0SHaiyang Zhang init_packet->msg.v2_msg.send_ndis_config.capability.teaming = 1; 4677f5d5af0SHaiyang Zhang } 4687f5d5af0SHaiyang Zhang 469f157e78dSHaiyang Zhang ret = vmbus_sendpacket(device->channel, init_packet, 470f157e78dSHaiyang Zhang sizeof(struct nvsp_message), 471f157e78dSHaiyang Zhang (unsigned long)init_packet, 472f157e78dSHaiyang Zhang VM_PKT_DATA_INBAND, 0); 473f157e78dSHaiyang Zhang 474f157e78dSHaiyang Zhang return ret; 475f157e78dSHaiyang Zhang } 476f157e78dSHaiyang Zhang 477f157e78dSHaiyang Zhang static int netvsc_connect_vsp(struct hv_device *device) 478f157e78dSHaiyang Zhang { 479f157e78dSHaiyang Zhang int ret; 48095fa0405SHaiyang Zhang struct netvsc_device *net_device; 48195fa0405SHaiyang Zhang struct nvsp_message *init_packet; 48295fa0405SHaiyang Zhang int ndis_version; 483e5a78fadSStephen Hemminger const u32 ver_list[] = { 484e5a78fadSStephen Hemminger NVSP_PROTOCOL_VERSION_1, NVSP_PROTOCOL_VERSION_2, 485a1eabb01SHaiyang Zhang NVSP_PROTOCOL_VERSION_4, NVSP_PROTOCOL_VERSION_5 }; 486e5a78fadSStephen Hemminger int i; 48795fa0405SHaiyang Zhang 48895fa0405SHaiyang Zhang net_device = get_outbound_net_device(device); 48995fa0405SHaiyang Zhang if (!net_device) 49095fa0405SHaiyang Zhang return -ENODEV; 49195fa0405SHaiyang Zhang 49295fa0405SHaiyang Zhang init_packet = &net_device->channel_init_pkt; 49395fa0405SHaiyang Zhang 494f157e78dSHaiyang Zhang /* Negotiate the latest NVSP protocol supported */ 495e5a78fadSStephen Hemminger for (i = ARRAY_SIZE(ver_list) - 1; i >= 0; i--) 496f157e78dSHaiyang Zhang if (negotiate_nvsp_ver(device, net_device, init_packet, 497a1eabb01SHaiyang Zhang ver_list[i]) == 0) { 498a1eabb01SHaiyang Zhang net_device->nvsp_version = ver_list[i]; 499a1eabb01SHaiyang Zhang break; 500a1eabb01SHaiyang Zhang } 501a1eabb01SHaiyang Zhang 502a1eabb01SHaiyang Zhang if (i < 0) { 50395fa0405SHaiyang Zhang ret = -EPROTO; 50495fa0405SHaiyang Zhang goto cleanup; 50595fa0405SHaiyang Zhang } 506f157e78dSHaiyang Zhang 507f157e78dSHaiyang Zhang pr_debug("Negotiated NVSP version:%x\n", net_device->nvsp_version); 508f157e78dSHaiyang Zhang 50995fa0405SHaiyang Zhang /* Send the ndis version */ 51095fa0405SHaiyang Zhang memset(init_packet, 0, sizeof(struct nvsp_message)); 51195fa0405SHaiyang Zhang 512a1eabb01SHaiyang Zhang if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_4) 5131f73db49SKY Srinivasan ndis_version = 0x00060001; 514a1eabb01SHaiyang Zhang else 515a1eabb01SHaiyang Zhang ndis_version = 0x0006001e; 51695fa0405SHaiyang Zhang 51795fa0405SHaiyang Zhang init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_NDIS_VER; 51895fa0405SHaiyang Zhang init_packet->msg.v1_msg. 51995fa0405SHaiyang Zhang send_ndis_ver.ndis_major_ver = 52095fa0405SHaiyang Zhang (ndis_version & 0xFFFF0000) >> 16; 52195fa0405SHaiyang Zhang init_packet->msg.v1_msg. 52295fa0405SHaiyang Zhang send_ndis_ver.ndis_minor_ver = 52395fa0405SHaiyang Zhang ndis_version & 0xFFFF; 52495fa0405SHaiyang Zhang 52595fa0405SHaiyang Zhang /* Send the init request */ 52695fa0405SHaiyang Zhang ret = vmbus_sendpacket(device->channel, init_packet, 52795fa0405SHaiyang Zhang sizeof(struct nvsp_message), 52895fa0405SHaiyang Zhang (unsigned long)init_packet, 52995fa0405SHaiyang Zhang VM_PKT_DATA_INBAND, 0); 53095fa0405SHaiyang Zhang if (ret != 0) 53195fa0405SHaiyang Zhang goto cleanup; 53295fa0405SHaiyang Zhang 53395fa0405SHaiyang Zhang /* Post the big receive buffer to NetVSP */ 53499d3016dSHaiyang Zhang if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_2) 53599d3016dSHaiyang Zhang net_device->recv_buf_size = NETVSC_RECEIVE_BUFFER_SIZE_LEGACY; 53699d3016dSHaiyang Zhang else 53799d3016dSHaiyang Zhang net_device->recv_buf_size = NETVSC_RECEIVE_BUFFER_SIZE; 538c25aaf81SKY Srinivasan net_device->send_buf_size = NETVSC_SEND_BUFFER_SIZE; 53999d3016dSHaiyang Zhang 540c25aaf81SKY Srinivasan ret = netvsc_init_buf(device); 54195fa0405SHaiyang Zhang 54295fa0405SHaiyang Zhang cleanup: 54395fa0405SHaiyang Zhang return ret; 54495fa0405SHaiyang Zhang } 54595fa0405SHaiyang Zhang 5463d541ac5SVitaly Kuznetsov static void netvsc_disconnect_vsp(struct hv_device *device) 54795fa0405SHaiyang Zhang { 5483d541ac5SVitaly Kuznetsov netvsc_destroy_buf(device); 54995fa0405SHaiyang Zhang } 55095fa0405SHaiyang Zhang 55195fa0405SHaiyang Zhang /* 55295fa0405SHaiyang Zhang * netvsc_device_remove - Callback when the root bus device is removed 55395fa0405SHaiyang Zhang */ 554e08f3ea5SStephen Hemminger void netvsc_device_remove(struct hv_device *device) 55595fa0405SHaiyang Zhang { 5563d541ac5SVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 5573d541ac5SVitaly Kuznetsov struct net_device_context *net_device_ctx = netdev_priv(ndev); 5583d541ac5SVitaly Kuznetsov struct netvsc_device *net_device = net_device_ctx->nvdev; 55995fa0405SHaiyang Zhang 5603d541ac5SVitaly Kuznetsov netvsc_disconnect_vsp(device); 56195fa0405SHaiyang Zhang 5623d541ac5SVitaly Kuznetsov net_device_ctx->nvdev = NULL; 56395fa0405SHaiyang Zhang 56495fa0405SHaiyang Zhang /* 56595fa0405SHaiyang Zhang * At this point, no one should be accessing net_device 56695fa0405SHaiyang Zhang * except in here 56795fa0405SHaiyang Zhang */ 56893ba2222SVitaly Kuznetsov netdev_dbg(ndev, "net device safe to remove\n"); 56995fa0405SHaiyang Zhang 57095fa0405SHaiyang Zhang /* Now, we can close the channel safely */ 57195fa0405SHaiyang Zhang vmbus_close(device->channel); 57295fa0405SHaiyang Zhang 57395fa0405SHaiyang Zhang /* Release all resources */ 574f90251c8SHaiyang Zhang free_netvsc_device(net_device); 57595fa0405SHaiyang Zhang } 57695fa0405SHaiyang Zhang 57733be96e4SHaiyang Zhang #define RING_AVAIL_PERCENT_HIWATER 20 57833be96e4SHaiyang Zhang #define RING_AVAIL_PERCENT_LOWATER 10 57933be96e4SHaiyang Zhang 58033be96e4SHaiyang Zhang /* 58133be96e4SHaiyang Zhang * Get the percentage of available bytes to write in the ring. 58233be96e4SHaiyang Zhang * The return value is in range from 0 to 100. 58333be96e4SHaiyang Zhang */ 58433be96e4SHaiyang Zhang static inline u32 hv_ringbuf_avail_percent( 58533be96e4SHaiyang Zhang struct hv_ring_buffer_info *ring_info) 58633be96e4SHaiyang Zhang { 58733be96e4SHaiyang Zhang u32 avail_read, avail_write; 58833be96e4SHaiyang Zhang 58933be96e4SHaiyang Zhang hv_get_ringbuffer_availbytes(ring_info, &avail_read, &avail_write); 59033be96e4SHaiyang Zhang 59133be96e4SHaiyang Zhang return avail_write * 100 / ring_info->ring_datasize; 59233be96e4SHaiyang Zhang } 59333be96e4SHaiyang Zhang 594c25aaf81SKY Srinivasan static inline void netvsc_free_send_slot(struct netvsc_device *net_device, 595c25aaf81SKY Srinivasan u32 index) 596c25aaf81SKY Srinivasan { 597c25aaf81SKY Srinivasan sync_change_bit(index, net_device->send_section_map); 598c25aaf81SKY Srinivasan } 599c25aaf81SKY Srinivasan 600bc304dd3SStephen Hemminger static void netvsc_send_tx_complete(struct netvsc_device *net_device, 60125b85ee8SKY Srinivasan struct vmbus_channel *incoming_channel, 60297c1723aSKY Srinivasan struct hv_device *device, 60395fa0405SHaiyang Zhang struct vmpacket_descriptor *packet) 60495fa0405SHaiyang Zhang { 605bc304dd3SStephen Hemminger struct sk_buff *skb = (struct sk_buff *)(unsigned long)packet->trans_id; 6063d541ac5SVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 6073d541ac5SVitaly Kuznetsov struct net_device_context *net_device_ctx = netdev_priv(ndev); 608bc304dd3SStephen Hemminger struct vmbus_channel *channel = device->channel; 6095b54dac8SHaiyang Zhang u16 q_idx = 0; 6105b54dac8SHaiyang Zhang int queue_sends; 61133be96e4SHaiyang Zhang 61295fa0405SHaiyang Zhang /* Notify the layer above us */ 613bc304dd3SStephen Hemminger if (likely(skb)) { 614bc304dd3SStephen Hemminger struct hv_netvsc_packet *nvsc_packet 615bc304dd3SStephen Hemminger = (struct hv_netvsc_packet *)skb->cb; 616bc304dd3SStephen Hemminger u32 send_index = nvsc_packet->send_buf_index; 617bc304dd3SStephen Hemminger 618c25aaf81SKY Srinivasan if (send_index != NETVSC_INVALID_INDEX) 619c25aaf81SKY Srinivasan netvsc_free_send_slot(net_device, send_index); 6205b54dac8SHaiyang Zhang q_idx = nvsc_packet->q_idx; 62125b85ee8SKY Srinivasan channel = incoming_channel; 622bc304dd3SStephen Hemminger 62317db4bceSStephen Hemminger dev_consume_skb_any(skb); 6245b54dac8SHaiyang Zhang } 62595fa0405SHaiyang Zhang 626b8b835a8Sstephen hemminger queue_sends = 627b8b835a8Sstephen hemminger atomic_dec_return(&net_device->chan_table[q_idx].queue_sends); 6281d06825bSHaiyang Zhang 62946b4f7f5Sstephen hemminger if (net_device->destroy && queue_sends == 0) 630dc5cd894SHaiyang Zhang wake_up(&net_device->wait_drain); 631dc5cd894SHaiyang Zhang 6325b54dac8SHaiyang Zhang if (netif_tx_queue_stopped(netdev_get_tx_queue(ndev, q_idx)) && 6333d541ac5SVitaly Kuznetsov !net_device_ctx->start_remove && 634bc304dd3SStephen Hemminger (hv_ringbuf_avail_percent(&channel->outbound) > RING_AVAIL_PERCENT_HIWATER || 635bc304dd3SStephen Hemminger queue_sends < 1)) 636796cc88cSStephen Hemminger netif_tx_wake_queue(netdev_get_tx_queue(ndev, q_idx)); 637bc304dd3SStephen Hemminger } 638bc304dd3SStephen Hemminger 639bc304dd3SStephen Hemminger static void netvsc_send_completion(struct netvsc_device *net_device, 640bc304dd3SStephen Hemminger struct vmbus_channel *incoming_channel, 641bc304dd3SStephen Hemminger struct hv_device *device, 642bc304dd3SStephen Hemminger struct vmpacket_descriptor *packet) 643bc304dd3SStephen Hemminger { 644bc304dd3SStephen Hemminger struct nvsp_message *nvsp_packet; 645bc304dd3SStephen Hemminger struct net_device *ndev = hv_get_drvdata(device); 646bc304dd3SStephen Hemminger 647bc304dd3SStephen Hemminger nvsp_packet = (struct nvsp_message *)((unsigned long)packet + 648bc304dd3SStephen Hemminger (packet->offset8 << 3)); 649bc304dd3SStephen Hemminger 650bc304dd3SStephen Hemminger switch (nvsp_packet->hdr.msg_type) { 651bc304dd3SStephen Hemminger case NVSP_MSG_TYPE_INIT_COMPLETE: 652bc304dd3SStephen Hemminger case NVSP_MSG1_TYPE_SEND_RECV_BUF_COMPLETE: 653bc304dd3SStephen Hemminger case NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE: 654bc304dd3SStephen Hemminger case NVSP_MSG5_TYPE_SUBCHANNEL: 655bc304dd3SStephen Hemminger /* Copy the response back */ 656bc304dd3SStephen Hemminger memcpy(&net_device->channel_init_pkt, nvsp_packet, 657bc304dd3SStephen Hemminger sizeof(struct nvsp_message)); 658bc304dd3SStephen Hemminger complete(&net_device->channel_init_wait); 659bc304dd3SStephen Hemminger break; 660bc304dd3SStephen Hemminger 661bc304dd3SStephen Hemminger case NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE: 662bc304dd3SStephen Hemminger netvsc_send_tx_complete(net_device, incoming_channel, 663bc304dd3SStephen Hemminger device, packet); 664bc304dd3SStephen Hemminger break; 665bc304dd3SStephen Hemminger 666bc304dd3SStephen Hemminger default: 667bc304dd3SStephen Hemminger netdev_err(ndev, 668bc304dd3SStephen Hemminger "Unknown send completion type %d received!!\n", 669bc304dd3SStephen Hemminger nvsp_packet->hdr.msg_type); 67095fa0405SHaiyang Zhang } 67195fa0405SHaiyang Zhang } 67295fa0405SHaiyang Zhang 673c25aaf81SKY Srinivasan static u32 netvsc_get_next_send_section(struct netvsc_device *net_device) 674c25aaf81SKY Srinivasan { 675c25aaf81SKY Srinivasan unsigned long index; 676c25aaf81SKY Srinivasan u32 max_words = net_device->map_words; 677c25aaf81SKY Srinivasan unsigned long *map_addr = (unsigned long *)net_device->send_section_map; 678c25aaf81SKY Srinivasan u32 section_cnt = net_device->send_section_cnt; 679c25aaf81SKY Srinivasan int ret_val = NETVSC_INVALID_INDEX; 680c25aaf81SKY Srinivasan int i; 681c25aaf81SKY Srinivasan int prev_val; 682c25aaf81SKY Srinivasan 683c25aaf81SKY Srinivasan for (i = 0; i < max_words; i++) { 684c25aaf81SKY Srinivasan if (!~(map_addr[i])) 685c25aaf81SKY Srinivasan continue; 686c25aaf81SKY Srinivasan index = ffz(map_addr[i]); 687c25aaf81SKY Srinivasan prev_val = sync_test_and_set_bit(index, &map_addr[i]); 688c25aaf81SKY Srinivasan if (prev_val) 689c25aaf81SKY Srinivasan continue; 690c25aaf81SKY Srinivasan if ((index + (i * BITS_PER_LONG)) >= section_cnt) 691c25aaf81SKY Srinivasan break; 692c25aaf81SKY Srinivasan ret_val = (index + (i * BITS_PER_LONG)); 693c25aaf81SKY Srinivasan break; 694c25aaf81SKY Srinivasan } 695c25aaf81SKY Srinivasan return ret_val; 696c25aaf81SKY Srinivasan } 697c25aaf81SKY Srinivasan 698da19fcd0SLad, Prabhakar static u32 netvsc_copy_to_send_buf(struct netvsc_device *net_device, 699c25aaf81SKY Srinivasan unsigned int section_index, 7007c3877f2SHaiyang Zhang u32 pend_size, 70124476760SKY Srinivasan struct hv_netvsc_packet *packet, 702a9f2e2d6SKY Srinivasan struct rndis_message *rndis_msg, 703694a9fb0SKY Srinivasan struct hv_page_buffer **pb, 704694a9fb0SKY Srinivasan struct sk_buff *skb) 705c25aaf81SKY Srinivasan { 706c25aaf81SKY Srinivasan char *start = net_device->send_buf; 7077c3877f2SHaiyang Zhang char *dest = start + (section_index * net_device->send_section_size) 7087c3877f2SHaiyang Zhang + pend_size; 709c25aaf81SKY Srinivasan int i; 710694a9fb0SKY Srinivasan bool is_data_pkt = (skb != NULL) ? true : false; 711bde79be5SKY Srinivasan bool xmit_more = (skb != NULL) ? skb->xmit_more : false; 712c25aaf81SKY Srinivasan u32 msg_size = 0; 7137c3877f2SHaiyang Zhang u32 padding = 0; 7147c3877f2SHaiyang Zhang u32 remain = packet->total_data_buflen % net_device->pkt_align; 715aa0a34beSHaiyang Zhang u32 page_count = packet->cp_partial ? packet->rmsg_pgcnt : 716aa0a34beSHaiyang Zhang packet->page_buf_cnt; 7177c3877f2SHaiyang Zhang 7187c3877f2SHaiyang Zhang /* Add padding */ 719bde79be5SKY Srinivasan if (is_data_pkt && xmit_more && remain && 720aa0a34beSHaiyang Zhang !packet->cp_partial) { 7217c3877f2SHaiyang Zhang padding = net_device->pkt_align - remain; 72224476760SKY Srinivasan rndis_msg->msg_len += padding; 7237c3877f2SHaiyang Zhang packet->total_data_buflen += padding; 7247c3877f2SHaiyang Zhang } 725c25aaf81SKY Srinivasan 726aa0a34beSHaiyang Zhang for (i = 0; i < page_count; i++) { 727a9f2e2d6SKY Srinivasan char *src = phys_to_virt((*pb)[i].pfn << PAGE_SHIFT); 728a9f2e2d6SKY Srinivasan u32 offset = (*pb)[i].offset; 729a9f2e2d6SKY Srinivasan u32 len = (*pb)[i].len; 730c25aaf81SKY Srinivasan 731c25aaf81SKY Srinivasan memcpy(dest, (src + offset), len); 732c25aaf81SKY Srinivasan msg_size += len; 733c25aaf81SKY Srinivasan dest += len; 734c25aaf81SKY Srinivasan } 7357c3877f2SHaiyang Zhang 7367c3877f2SHaiyang Zhang if (padding) { 7377c3877f2SHaiyang Zhang memset(dest, 0, padding); 7387c3877f2SHaiyang Zhang msg_size += padding; 7397c3877f2SHaiyang Zhang } 7407c3877f2SHaiyang Zhang 741c25aaf81SKY Srinivasan return msg_size; 742c25aaf81SKY Srinivasan } 743c25aaf81SKY Srinivasan 7443a8963acSStephen Hemminger static inline int netvsc_send_pkt( 7450a1275caSVitaly Kuznetsov struct hv_device *device, 7467c3877f2SHaiyang Zhang struct hv_netvsc_packet *packet, 747a9f2e2d6SKY Srinivasan struct netvsc_device *net_device, 7483a3d9a0aSKY Srinivasan struct hv_page_buffer **pb, 7493a3d9a0aSKY Srinivasan struct sk_buff *skb) 75095fa0405SHaiyang Zhang { 7517c3877f2SHaiyang Zhang struct nvsp_message nvmsg; 752b8b835a8Sstephen hemminger struct netvsc_channel *nvchan 753b8b835a8Sstephen hemminger = &net_device->chan_table[packet->q_idx]; 754b8b835a8Sstephen hemminger struct vmbus_channel *out_channel = nvchan->channel; 7550a1275caSVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 756b8b835a8Sstephen hemminger struct netdev_queue *txq = netdev_get_tx_queue(ndev, packet->q_idx); 7577c3877f2SHaiyang Zhang u64 req_id; 7587c3877f2SHaiyang Zhang int ret; 759aa0a34beSHaiyang Zhang struct hv_page_buffer *pgbuf; 76082fa3c77SKY Srinivasan u32 ring_avail = hv_ringbuf_avail_percent(&out_channel->outbound); 761bde79be5SKY Srinivasan bool xmit_more = (skb != NULL) ? skb->xmit_more : false; 762c25aaf81SKY Srinivasan 7637c3877f2SHaiyang Zhang nvmsg.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT; 764694a9fb0SKY Srinivasan if (skb != NULL) { 76595fa0405SHaiyang Zhang /* 0 is RMC_DATA; */ 7667c3877f2SHaiyang Zhang nvmsg.msg.v1_msg.send_rndis_pkt.channel_type = 0; 76795fa0405SHaiyang Zhang } else { 76895fa0405SHaiyang Zhang /* 1 is RMC_CONTROL; */ 7697c3877f2SHaiyang Zhang nvmsg.msg.v1_msg.send_rndis_pkt.channel_type = 1; 77095fa0405SHaiyang Zhang } 77195fa0405SHaiyang Zhang 7727c3877f2SHaiyang Zhang nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_index = 7737c3877f2SHaiyang Zhang packet->send_buf_index; 7747c3877f2SHaiyang Zhang if (packet->send_buf_index == NETVSC_INVALID_INDEX) 7757c3877f2SHaiyang Zhang nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_size = 0; 7767c3877f2SHaiyang Zhang else 7777c3877f2SHaiyang Zhang nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_size = 7787c3877f2SHaiyang Zhang packet->total_data_buflen; 77995fa0405SHaiyang Zhang 7803a3d9a0aSKY Srinivasan req_id = (ulong)skb; 781f1ea3cd7SHaiyang Zhang 782c3582a2cSHaiyang Zhang if (out_channel->rescind) 783c3582a2cSHaiyang Zhang return -ENODEV; 784c3582a2cSHaiyang Zhang 78582fa3c77SKY Srinivasan /* 78682fa3c77SKY Srinivasan * It is possible that once we successfully place this packet 78782fa3c77SKY Srinivasan * on the ringbuffer, we may stop the queue. In that case, we want 78882fa3c77SKY Srinivasan * to notify the host independent of the xmit_more flag. We don't 78982fa3c77SKY Srinivasan * need to be precise here; in the worst case we may signal the host 79082fa3c77SKY Srinivasan * unnecessarily. 79182fa3c77SKY Srinivasan */ 79282fa3c77SKY Srinivasan if (ring_avail < (RING_AVAIL_PERCENT_LOWATER + 1)) 793bde79be5SKY Srinivasan xmit_more = false; 79482fa3c77SKY Srinivasan 79595fa0405SHaiyang Zhang if (packet->page_buf_cnt) { 796a9f2e2d6SKY Srinivasan pgbuf = packet->cp_partial ? (*pb) + 797a9f2e2d6SKY Srinivasan packet->rmsg_pgcnt : (*pb); 79882fa3c77SKY Srinivasan ret = vmbus_sendpacket_pagebuffer_ctl(out_channel, 799aa0a34beSHaiyang Zhang pgbuf, 80095fa0405SHaiyang Zhang packet->page_buf_cnt, 8017c3877f2SHaiyang Zhang &nvmsg, 80295fa0405SHaiyang Zhang sizeof(struct nvsp_message), 80382fa3c77SKY Srinivasan req_id, 80482fa3c77SKY Srinivasan VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED, 805bde79be5SKY Srinivasan !xmit_more); 80695fa0405SHaiyang Zhang } else { 80782fa3c77SKY Srinivasan ret = vmbus_sendpacket_ctl(out_channel, &nvmsg, 80895fa0405SHaiyang Zhang sizeof(struct nvsp_message), 809f1ea3cd7SHaiyang Zhang req_id, 81095fa0405SHaiyang Zhang VM_PKT_DATA_INBAND, 81182fa3c77SKY Srinivasan VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED, 812bde79be5SKY Srinivasan !xmit_more); 81395fa0405SHaiyang Zhang } 81495fa0405SHaiyang Zhang 8151d06825bSHaiyang Zhang if (ret == 0) { 816b8b835a8Sstephen hemminger atomic_inc_return(&nvchan->queue_sends); 8175b54dac8SHaiyang Zhang 81846b4f7f5Sstephen hemminger if (ring_avail < RING_AVAIL_PERCENT_LOWATER) 819b8b835a8Sstephen hemminger netif_tx_stop_queue(txq); 8201d06825bSHaiyang Zhang } else if (ret == -EAGAIN) { 821b8b835a8Sstephen hemminger netif_tx_stop_queue(txq); 822b8b835a8Sstephen hemminger if (atomic_read(&nvchan->queue_sends) < 1) { 823b8b835a8Sstephen hemminger netif_tx_wake_queue(txq); 82433be96e4SHaiyang Zhang ret = -ENOSPC; 82533be96e4SHaiyang Zhang } 8261d06825bSHaiyang Zhang } else { 82795fa0405SHaiyang Zhang netdev_err(ndev, "Unable to send packet %p ret %d\n", 82895fa0405SHaiyang Zhang packet, ret); 8291d06825bSHaiyang Zhang } 83095fa0405SHaiyang Zhang 8317c3877f2SHaiyang Zhang return ret; 8327c3877f2SHaiyang Zhang } 8337c3877f2SHaiyang Zhang 834c85e4924SHaiyang Zhang /* Move packet out of multi send data (msd), and clear msd */ 835c85e4924SHaiyang Zhang static inline void move_pkt_msd(struct hv_netvsc_packet **msd_send, 836c85e4924SHaiyang Zhang struct sk_buff **msd_skb, 837c85e4924SHaiyang Zhang struct multi_send_data *msdp) 838c85e4924SHaiyang Zhang { 839c85e4924SHaiyang Zhang *msd_skb = msdp->skb; 840c85e4924SHaiyang Zhang *msd_send = msdp->pkt; 841c85e4924SHaiyang Zhang msdp->skb = NULL; 842c85e4924SHaiyang Zhang msdp->pkt = NULL; 843c85e4924SHaiyang Zhang msdp->count = 0; 844c85e4924SHaiyang Zhang } 845c85e4924SHaiyang Zhang 8467c3877f2SHaiyang Zhang int netvsc_send(struct hv_device *device, 84724476760SKY Srinivasan struct hv_netvsc_packet *packet, 848a9f2e2d6SKY Srinivasan struct rndis_message *rndis_msg, 8493a3d9a0aSKY Srinivasan struct hv_page_buffer **pb, 8503a3d9a0aSKY Srinivasan struct sk_buff *skb) 8517c3877f2SHaiyang Zhang { 8527c3877f2SHaiyang Zhang struct netvsc_device *net_device; 8536c4c137eSStephen Hemminger int ret = 0; 854b8b835a8Sstephen hemminger struct netvsc_channel *nvchan; 8557c3877f2SHaiyang Zhang u32 pktlen = packet->total_data_buflen, msd_len = 0; 8567c3877f2SHaiyang Zhang unsigned int section_index = NETVSC_INVALID_INDEX; 8577c3877f2SHaiyang Zhang struct multi_send_data *msdp; 8587c3877f2SHaiyang Zhang struct hv_netvsc_packet *msd_send = NULL, *cur_send = NULL; 859c85e4924SHaiyang Zhang struct sk_buff *msd_skb = NULL; 860aa0a34beSHaiyang Zhang bool try_batch; 861bde79be5SKY Srinivasan bool xmit_more = (skb != NULL) ? skb->xmit_more : false; 8627c3877f2SHaiyang Zhang 8637c3877f2SHaiyang Zhang net_device = get_outbound_net_device(device); 8647c3877f2SHaiyang Zhang if (!net_device) 8657c3877f2SHaiyang Zhang return -ENODEV; 8667c3877f2SHaiyang Zhang 867e8f0a89cSVitaly Kuznetsov /* We may race with netvsc_connect_vsp()/netvsc_init_buf() and get 868e8f0a89cSVitaly Kuznetsov * here before the negotiation with the host is finished and 869e8f0a89cSVitaly Kuznetsov * send_section_map may not be allocated yet. 870e8f0a89cSVitaly Kuznetsov */ 871e8f0a89cSVitaly Kuznetsov if (!net_device->send_section_map) 872e8f0a89cSVitaly Kuznetsov return -EAGAIN; 873e8f0a89cSVitaly Kuznetsov 874b8b835a8Sstephen hemminger nvchan = &net_device->chan_table[packet->q_idx]; 8757c3877f2SHaiyang Zhang packet->send_buf_index = NETVSC_INVALID_INDEX; 876aa0a34beSHaiyang Zhang packet->cp_partial = false; 8777c3877f2SHaiyang Zhang 878cf8190e4SHaiyang Zhang /* Send control message directly without accessing msd (Multi-Send 879cf8190e4SHaiyang Zhang * Data) field which may be changed during data packet processing. 880cf8190e4SHaiyang Zhang */ 881cf8190e4SHaiyang Zhang if (!skb) { 882cf8190e4SHaiyang Zhang cur_send = packet; 883cf8190e4SHaiyang Zhang goto send_now; 884cf8190e4SHaiyang Zhang } 885cf8190e4SHaiyang Zhang 8867c3877f2SHaiyang Zhang /* batch packets in send buffer if possible */ 887b8b835a8Sstephen hemminger msdp = &nvchan->msd; 8887c3877f2SHaiyang Zhang if (msdp->pkt) 8897c3877f2SHaiyang Zhang msd_len = msdp->pkt->total_data_buflen; 8907c3877f2SHaiyang Zhang 891694a9fb0SKY Srinivasan try_batch = (skb != NULL) && msd_len > 0 && msdp->count < 892aa0a34beSHaiyang Zhang net_device->max_pkt; 893aa0a34beSHaiyang Zhang 894aa0a34beSHaiyang Zhang if (try_batch && msd_len + pktlen + net_device->pkt_align < 8957c3877f2SHaiyang Zhang net_device->send_section_size) { 8967c3877f2SHaiyang Zhang section_index = msdp->pkt->send_buf_index; 8977c3877f2SHaiyang Zhang 898aa0a34beSHaiyang Zhang } else if (try_batch && msd_len + packet->rmsg_size < 899aa0a34beSHaiyang Zhang net_device->send_section_size) { 900aa0a34beSHaiyang Zhang section_index = msdp->pkt->send_buf_index; 901aa0a34beSHaiyang Zhang packet->cp_partial = true; 902aa0a34beSHaiyang Zhang 903694a9fb0SKY Srinivasan } else if ((skb != NULL) && pktlen + net_device->pkt_align < 9047c3877f2SHaiyang Zhang net_device->send_section_size) { 9057c3877f2SHaiyang Zhang section_index = netvsc_get_next_send_section(net_device); 9067c3877f2SHaiyang Zhang if (section_index != NETVSC_INVALID_INDEX) { 907c85e4924SHaiyang Zhang move_pkt_msd(&msd_send, &msd_skb, msdp); 9087c3877f2SHaiyang Zhang msd_len = 0; 9097c3877f2SHaiyang Zhang } 9107c3877f2SHaiyang Zhang } 9117c3877f2SHaiyang Zhang 9127c3877f2SHaiyang Zhang if (section_index != NETVSC_INVALID_INDEX) { 9137c3877f2SHaiyang Zhang netvsc_copy_to_send_buf(net_device, 9147c3877f2SHaiyang Zhang section_index, msd_len, 915694a9fb0SKY Srinivasan packet, rndis_msg, pb, skb); 916b08cc791SKY Srinivasan 917aa0a34beSHaiyang Zhang packet->send_buf_index = section_index; 918aa0a34beSHaiyang Zhang 919aa0a34beSHaiyang Zhang if (packet->cp_partial) { 920aa0a34beSHaiyang Zhang packet->page_buf_cnt -= packet->rmsg_pgcnt; 921aa0a34beSHaiyang Zhang packet->total_data_buflen = msd_len + packet->rmsg_size; 922aa0a34beSHaiyang Zhang } else { 923aa0a34beSHaiyang Zhang packet->page_buf_cnt = 0; 924aa0a34beSHaiyang Zhang packet->total_data_buflen += msd_len; 925aa0a34beSHaiyang Zhang } 9267c3877f2SHaiyang Zhang 927c85e4924SHaiyang Zhang if (msdp->skb) 92817db4bceSStephen Hemminger dev_consume_skb_any(msdp->skb); 929ee90b812SHaiyang Zhang 930bde79be5SKY Srinivasan if (xmit_more && !packet->cp_partial) { 931c85e4924SHaiyang Zhang msdp->skb = skb; 9327c3877f2SHaiyang Zhang msdp->pkt = packet; 9337c3877f2SHaiyang Zhang msdp->count++; 9347c3877f2SHaiyang Zhang } else { 9357c3877f2SHaiyang Zhang cur_send = packet; 936c85e4924SHaiyang Zhang msdp->skb = NULL; 9377c3877f2SHaiyang Zhang msdp->pkt = NULL; 9387c3877f2SHaiyang Zhang msdp->count = 0; 9397c3877f2SHaiyang Zhang } 9407c3877f2SHaiyang Zhang } else { 941c85e4924SHaiyang Zhang move_pkt_msd(&msd_send, &msd_skb, msdp); 9427c3877f2SHaiyang Zhang cur_send = packet; 9437c3877f2SHaiyang Zhang } 9447c3877f2SHaiyang Zhang 9457c3877f2SHaiyang Zhang if (msd_send) { 9466c4c137eSStephen Hemminger int m_ret = netvsc_send_pkt(device, msd_send, net_device, 9470a1275caSVitaly Kuznetsov NULL, msd_skb); 9487c3877f2SHaiyang Zhang 9497c3877f2SHaiyang Zhang if (m_ret != 0) { 9507c3877f2SHaiyang Zhang netvsc_free_send_slot(net_device, 9517c3877f2SHaiyang Zhang msd_send->send_buf_index); 952c85e4924SHaiyang Zhang dev_kfree_skb_any(msd_skb); 9537c3877f2SHaiyang Zhang } 9547c3877f2SHaiyang Zhang } 9557c3877f2SHaiyang Zhang 956cf8190e4SHaiyang Zhang send_now: 9577c3877f2SHaiyang Zhang if (cur_send) 9580a1275caSVitaly Kuznetsov ret = netvsc_send_pkt(device, cur_send, net_device, pb, skb); 9597c3877f2SHaiyang Zhang 9607aab5159SJerry Snitselaar if (ret != 0 && section_index != NETVSC_INVALID_INDEX) 961d953ca4dSHaiyang Zhang netvsc_free_send_slot(net_device, section_index); 962d953ca4dSHaiyang Zhang 96395fa0405SHaiyang Zhang return ret; 96495fa0405SHaiyang Zhang } 96595fa0405SHaiyang Zhang 966c0b558e5SHaiyang Zhang static int netvsc_send_recv_completion(struct vmbus_channel *channel, 96763f6921dSHaiyang Zhang u64 transaction_id, u32 status) 96895fa0405SHaiyang Zhang { 96995fa0405SHaiyang Zhang struct nvsp_message recvcompMessage; 97095fa0405SHaiyang Zhang int ret; 97195fa0405SHaiyang Zhang 97295fa0405SHaiyang Zhang recvcompMessage.hdr.msg_type = 97395fa0405SHaiyang Zhang NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE; 97495fa0405SHaiyang Zhang 97563f6921dSHaiyang Zhang recvcompMessage.msg.v1_msg.send_rndis_pkt_complete.status = status; 97695fa0405SHaiyang Zhang 97795fa0405SHaiyang Zhang /* Send the completion */ 9785b54dac8SHaiyang Zhang ret = vmbus_sendpacket(channel, &recvcompMessage, 979c0b558e5SHaiyang Zhang sizeof(struct nvsp_message_header) + sizeof(u32), 980c0b558e5SHaiyang Zhang transaction_id, VM_PKT_COMP, 0); 98195fa0405SHaiyang Zhang 982c0b558e5SHaiyang Zhang return ret; 98395fa0405SHaiyang Zhang } 984c0b558e5SHaiyang Zhang 985c0b558e5SHaiyang Zhang static inline void count_recv_comp_slot(struct netvsc_device *nvdev, u16 q_idx, 986c0b558e5SHaiyang Zhang u32 *filled, u32 *avail) 987c0b558e5SHaiyang Zhang { 988b8b835a8Sstephen hemminger struct multi_recv_comp *mrc = &nvdev->chan_table[q_idx].mrc; 989b8b835a8Sstephen hemminger u32 first = mrc->first; 990b8b835a8Sstephen hemminger u32 next = mrc->next; 991c0b558e5SHaiyang Zhang 992c0b558e5SHaiyang Zhang *filled = (first > next) ? NETVSC_RECVSLOT_MAX - first + next : 993c0b558e5SHaiyang Zhang next - first; 994c0b558e5SHaiyang Zhang 995c0b558e5SHaiyang Zhang *avail = NETVSC_RECVSLOT_MAX - *filled - 1; 99695fa0405SHaiyang Zhang } 997c0b558e5SHaiyang Zhang 998c0b558e5SHaiyang Zhang /* Read the first filled slot, no change to index */ 999c0b558e5SHaiyang Zhang static inline struct recv_comp_data *read_recv_comp_slot(struct netvsc_device 1000c0b558e5SHaiyang Zhang *nvdev, u16 q_idx) 1001c0b558e5SHaiyang Zhang { 1002b8b835a8Sstephen hemminger struct multi_recv_comp *mrc = &nvdev->chan_table[q_idx].mrc; 1003c0b558e5SHaiyang Zhang u32 filled, avail; 1004c0b558e5SHaiyang Zhang 1005b8b835a8Sstephen hemminger if (unlikely(!mrc->buf)) 1006c0b558e5SHaiyang Zhang return NULL; 1007c0b558e5SHaiyang Zhang 1008c0b558e5SHaiyang Zhang count_recv_comp_slot(nvdev, q_idx, &filled, &avail); 1009c0b558e5SHaiyang Zhang if (!filled) 1010c0b558e5SHaiyang Zhang return NULL; 1011c0b558e5SHaiyang Zhang 1012b8b835a8Sstephen hemminger return mrc->buf + mrc->first * sizeof(struct recv_comp_data); 1013c0b558e5SHaiyang Zhang } 1014c0b558e5SHaiyang Zhang 1015c0b558e5SHaiyang Zhang /* Put the first filled slot back to available pool */ 1016c0b558e5SHaiyang Zhang static inline void put_recv_comp_slot(struct netvsc_device *nvdev, u16 q_idx) 1017c0b558e5SHaiyang Zhang { 1018b8b835a8Sstephen hemminger struct multi_recv_comp *mrc = &nvdev->chan_table[q_idx].mrc; 1019c0b558e5SHaiyang Zhang int num_recv; 1020c0b558e5SHaiyang Zhang 1021b8b835a8Sstephen hemminger mrc->first = (mrc->first + 1) % NETVSC_RECVSLOT_MAX; 1022c0b558e5SHaiyang Zhang 1023c0b558e5SHaiyang Zhang num_recv = atomic_dec_return(&nvdev->num_outstanding_recvs); 1024c0b558e5SHaiyang Zhang 1025c0b558e5SHaiyang Zhang if (nvdev->destroy && num_recv == 0) 1026c0b558e5SHaiyang Zhang wake_up(&nvdev->wait_drain); 1027c0b558e5SHaiyang Zhang } 1028c0b558e5SHaiyang Zhang 1029c0b558e5SHaiyang Zhang /* Check and send pending recv completions */ 1030c0b558e5SHaiyang Zhang static void netvsc_chk_recv_comp(struct netvsc_device *nvdev, 1031c0b558e5SHaiyang Zhang struct vmbus_channel *channel, u16 q_idx) 1032c0b558e5SHaiyang Zhang { 1033c0b558e5SHaiyang Zhang struct recv_comp_data *rcd; 1034c0b558e5SHaiyang Zhang int ret; 1035c0b558e5SHaiyang Zhang 1036c0b558e5SHaiyang Zhang while (true) { 1037c0b558e5SHaiyang Zhang rcd = read_recv_comp_slot(nvdev, q_idx); 1038c0b558e5SHaiyang Zhang if (!rcd) 1039c0b558e5SHaiyang Zhang break; 1040c0b558e5SHaiyang Zhang 1041c0b558e5SHaiyang Zhang ret = netvsc_send_recv_completion(channel, rcd->tid, 1042c0b558e5SHaiyang Zhang rcd->status); 1043c0b558e5SHaiyang Zhang if (ret) 1044c0b558e5SHaiyang Zhang break; 1045c0b558e5SHaiyang Zhang 1046c0b558e5SHaiyang Zhang put_recv_comp_slot(nvdev, q_idx); 1047c0b558e5SHaiyang Zhang } 1048c0b558e5SHaiyang Zhang } 1049c0b558e5SHaiyang Zhang 1050c0b558e5SHaiyang Zhang #define NETVSC_RCD_WATERMARK 80 1051c0b558e5SHaiyang Zhang 1052c0b558e5SHaiyang Zhang /* Get next available slot */ 1053c0b558e5SHaiyang Zhang static inline struct recv_comp_data *get_recv_comp_slot( 1054c0b558e5SHaiyang Zhang struct netvsc_device *nvdev, struct vmbus_channel *channel, u16 q_idx) 1055c0b558e5SHaiyang Zhang { 1056b8b835a8Sstephen hemminger struct multi_recv_comp *mrc = &nvdev->chan_table[q_idx].mrc; 1057c0b558e5SHaiyang Zhang u32 filled, avail, next; 1058c0b558e5SHaiyang Zhang struct recv_comp_data *rcd; 1059c0b558e5SHaiyang Zhang 1060b8b835a8Sstephen hemminger if (unlikely(!nvdev->recv_section)) 1061c0b558e5SHaiyang Zhang return NULL; 1062c0b558e5SHaiyang Zhang 1063b8b835a8Sstephen hemminger if (unlikely(!mrc->buf)) 1064c0b558e5SHaiyang Zhang return NULL; 1065c0b558e5SHaiyang Zhang 1066c0b558e5SHaiyang Zhang if (atomic_read(&nvdev->num_outstanding_recvs) > 1067c0b558e5SHaiyang Zhang nvdev->recv_section->num_sub_allocs * NETVSC_RCD_WATERMARK / 100) 1068c0b558e5SHaiyang Zhang netvsc_chk_recv_comp(nvdev, channel, q_idx); 1069c0b558e5SHaiyang Zhang 1070c0b558e5SHaiyang Zhang count_recv_comp_slot(nvdev, q_idx, &filled, &avail); 1071c0b558e5SHaiyang Zhang if (!avail) 1072c0b558e5SHaiyang Zhang return NULL; 1073c0b558e5SHaiyang Zhang 1074b8b835a8Sstephen hemminger next = mrc->next; 1075b8b835a8Sstephen hemminger rcd = mrc->buf + next * sizeof(struct recv_comp_data); 1076b8b835a8Sstephen hemminger mrc->next = (next + 1) % NETVSC_RECVSLOT_MAX; 1077c0b558e5SHaiyang Zhang 1078c0b558e5SHaiyang Zhang atomic_inc(&nvdev->num_outstanding_recvs); 1079c0b558e5SHaiyang Zhang 1080c0b558e5SHaiyang Zhang return rcd; 108195fa0405SHaiyang Zhang } 108295fa0405SHaiyang Zhang 1083dc54a08cSstephen hemminger static void netvsc_receive(struct net_device *ndev, 1084dc54a08cSstephen hemminger struct netvsc_device *net_device, 1085dc54a08cSstephen hemminger struct net_device_context *net_device_ctx, 108697c1723aSKY Srinivasan struct hv_device *device, 1087dc54a08cSstephen hemminger struct vmbus_channel *channel, 1088dc54a08cSstephen hemminger struct vmtransfer_page_packet_header *vmxferpage_packet, 1089dc54a08cSstephen hemminger struct nvsp_message *nvsp) 109095fa0405SHaiyang Zhang { 1091dc54a08cSstephen hemminger char *recv_buf = net_device->recv_buf; 10924baab261SHaiyang Zhang u32 status = NVSP_STAT_SUCCESS; 109345326342SHaiyang Zhang int i; 109445326342SHaiyang Zhang int count = 0; 1095c0b558e5SHaiyang Zhang int ret; 1096c0b558e5SHaiyang Zhang struct recv_comp_data *rcd; 1097c0b558e5SHaiyang Zhang u16 q_idx = channel->offermsg.offer.sub_channel_index; 109895fa0405SHaiyang Zhang 109995fa0405SHaiyang Zhang /* Make sure this is a valid nvsp packet */ 1100dc54a08cSstephen hemminger if (unlikely(nvsp->hdr.msg_type != NVSP_MSG1_TYPE_SEND_RNDIS_PKT)) { 1101dc54a08cSstephen hemminger netif_err(net_device_ctx, rx_err, ndev, 1102dc54a08cSstephen hemminger "Unknown nvsp packet type received %u\n", 1103dc54a08cSstephen hemminger nvsp->hdr.msg_type); 110495fa0405SHaiyang Zhang return; 110595fa0405SHaiyang Zhang } 110695fa0405SHaiyang Zhang 1107dc54a08cSstephen hemminger if (unlikely(vmxferpage_packet->xfer_pageset_id != NETVSC_RECEIVE_BUFFER_ID)) { 1108dc54a08cSstephen hemminger netif_err(net_device_ctx, rx_err, ndev, 1109dc54a08cSstephen hemminger "Invalid xfer page set id - expecting %x got %x\n", 1110dc54a08cSstephen hemminger NETVSC_RECEIVE_BUFFER_ID, 111195fa0405SHaiyang Zhang vmxferpage_packet->xfer_pageset_id); 111295fa0405SHaiyang Zhang return; 111395fa0405SHaiyang Zhang } 111495fa0405SHaiyang Zhang 11154baab261SHaiyang Zhang count = vmxferpage_packet->range_cnt; 111695fa0405SHaiyang Zhang 111795fa0405SHaiyang Zhang /* Each range represents 1 RNDIS pkt that contains 1 ethernet frame */ 11184baab261SHaiyang Zhang for (i = 0; i < count; i++) { 1119dc54a08cSstephen hemminger void *data = recv_buf 1120dc54a08cSstephen hemminger + vmxferpage_packet->ranges[i].byte_offset; 1121dc54a08cSstephen hemminger u32 buflen = vmxferpage_packet->ranges[i].byte_count; 112295fa0405SHaiyang Zhang 112395fa0405SHaiyang Zhang /* Pass it to the upper layer */ 1124dc54a08cSstephen hemminger status = rndis_filter_receive(ndev, net_device, device, 1125dc54a08cSstephen hemminger channel, data, buflen); 112695fa0405SHaiyang Zhang } 112795fa0405SHaiyang Zhang 1128b8b835a8Sstephen hemminger if (!net_device->chan_table[q_idx].mrc.buf) { 1129c0b558e5SHaiyang Zhang ret = netvsc_send_recv_completion(channel, 1130c0b558e5SHaiyang Zhang vmxferpage_packet->d.trans_id, 1131c0b558e5SHaiyang Zhang status); 1132c0b558e5SHaiyang Zhang if (ret) 1133c0b558e5SHaiyang Zhang netdev_err(ndev, "Recv_comp q:%hd, tid:%llx, err:%d\n", 1134c0b558e5SHaiyang Zhang q_idx, vmxferpage_packet->d.trans_id, ret); 1135c0b558e5SHaiyang Zhang return; 1136c0b558e5SHaiyang Zhang } 1137c0b558e5SHaiyang Zhang 1138c0b558e5SHaiyang Zhang rcd = get_recv_comp_slot(net_device, channel, q_idx); 1139c0b558e5SHaiyang Zhang 1140c0b558e5SHaiyang Zhang if (!rcd) { 1141c0b558e5SHaiyang Zhang netdev_err(ndev, "Recv_comp full buf q:%hd, tid:%llx\n", 1142c0b558e5SHaiyang Zhang q_idx, vmxferpage_packet->d.trans_id); 1143c0b558e5SHaiyang Zhang return; 1144c0b558e5SHaiyang Zhang } 1145c0b558e5SHaiyang Zhang 1146c0b558e5SHaiyang Zhang rcd->tid = vmxferpage_packet->d.trans_id; 1147c0b558e5SHaiyang Zhang rcd->status = status; 114895fa0405SHaiyang Zhang } 114995fa0405SHaiyang Zhang 11505b54dac8SHaiyang Zhang static void netvsc_send_table(struct hv_device *hdev, 115171790a27SHaiyang Zhang struct nvsp_message *nvmsg) 11525b54dac8SHaiyang Zhang { 11535b54dac8SHaiyang Zhang struct netvsc_device *nvscdev; 11540a1275caSVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(hdev); 11555b54dac8SHaiyang Zhang int i; 11565b54dac8SHaiyang Zhang u32 count, *tab; 11575b54dac8SHaiyang Zhang 11585b54dac8SHaiyang Zhang nvscdev = get_outbound_net_device(hdev); 11595b54dac8SHaiyang Zhang if (!nvscdev) 11605b54dac8SHaiyang Zhang return; 11615b54dac8SHaiyang Zhang 11625b54dac8SHaiyang Zhang count = nvmsg->msg.v5_msg.send_table.count; 11635b54dac8SHaiyang Zhang if (count != VRSS_SEND_TAB_SIZE) { 11645b54dac8SHaiyang Zhang netdev_err(ndev, "Received wrong send-table size:%u\n", count); 11655b54dac8SHaiyang Zhang return; 11665b54dac8SHaiyang Zhang } 11675b54dac8SHaiyang Zhang 11685b54dac8SHaiyang Zhang tab = (u32 *)((unsigned long)&nvmsg->msg.v5_msg.send_table + 11695b54dac8SHaiyang Zhang nvmsg->msg.v5_msg.send_table.offset); 11705b54dac8SHaiyang Zhang 11715b54dac8SHaiyang Zhang for (i = 0; i < count; i++) 11725b54dac8SHaiyang Zhang nvscdev->send_table[i] = tab[i]; 11735b54dac8SHaiyang Zhang } 11745b54dac8SHaiyang Zhang 1175f9a7da91SVitaly Kuznetsov static void netvsc_send_vf(struct net_device_context *net_device_ctx, 117671790a27SHaiyang Zhang struct nvsp_message *nvmsg) 117771790a27SHaiyang Zhang { 1178f9a7da91SVitaly Kuznetsov net_device_ctx->vf_alloc = nvmsg->msg.v4_msg.vf_assoc.allocated; 1179f9a7da91SVitaly Kuznetsov net_device_ctx->vf_serial = nvmsg->msg.v4_msg.vf_assoc.serial; 118071790a27SHaiyang Zhang } 118171790a27SHaiyang Zhang 118271790a27SHaiyang Zhang static inline void netvsc_receive_inband(struct hv_device *hdev, 1183f9a7da91SVitaly Kuznetsov struct net_device_context *net_device_ctx, 118471790a27SHaiyang Zhang struct nvsp_message *nvmsg) 118571790a27SHaiyang Zhang { 118671790a27SHaiyang Zhang switch (nvmsg->hdr.msg_type) { 118771790a27SHaiyang Zhang case NVSP_MSG5_TYPE_SEND_INDIRECTION_TABLE: 118871790a27SHaiyang Zhang netvsc_send_table(hdev, nvmsg); 118971790a27SHaiyang Zhang break; 119071790a27SHaiyang Zhang 119171790a27SHaiyang Zhang case NVSP_MSG4_TYPE_SEND_VF_ASSOCIATION: 1192f9a7da91SVitaly Kuznetsov netvsc_send_vf(net_device_ctx, nvmsg); 119371790a27SHaiyang Zhang break; 119471790a27SHaiyang Zhang } 119571790a27SHaiyang Zhang } 119671790a27SHaiyang Zhang 119799a50bb1SK. Y. Srinivasan static void netvsc_process_raw_pkt(struct hv_device *device, 119899a50bb1SK. Y. Srinivasan struct vmbus_channel *channel, 119999a50bb1SK. Y. Srinivasan struct netvsc_device *net_device, 120099a50bb1SK. Y. Srinivasan struct net_device *ndev, 120199a50bb1SK. Y. Srinivasan u64 request_id, 120299a50bb1SK. Y. Srinivasan struct vmpacket_descriptor *desc) 120399a50bb1SK. Y. Srinivasan { 1204f9a7da91SVitaly Kuznetsov struct net_device_context *net_device_ctx = netdev_priv(ndev); 1205dc54a08cSstephen hemminger struct nvsp_message *nvmsg 1206dc54a08cSstephen hemminger = (struct nvsp_message *)((unsigned long)desc 1207dc54a08cSstephen hemminger + (desc->offset8 << 3)); 120899a50bb1SK. Y. Srinivasan 120999a50bb1SK. Y. Srinivasan switch (desc->type) { 121099a50bb1SK. Y. Srinivasan case VM_PKT_COMP: 121199a50bb1SK. Y. Srinivasan netvsc_send_completion(net_device, channel, device, desc); 121299a50bb1SK. Y. Srinivasan break; 121399a50bb1SK. Y. Srinivasan 121499a50bb1SK. Y. Srinivasan case VM_PKT_DATA_USING_XFER_PAGES: 1215dc54a08cSstephen hemminger netvsc_receive(ndev, net_device, net_device_ctx, 1216dc54a08cSstephen hemminger device, channel, 1217dc54a08cSstephen hemminger (struct vmtransfer_page_packet_header *)desc, 1218dc54a08cSstephen hemminger nvmsg); 121999a50bb1SK. Y. Srinivasan break; 122099a50bb1SK. Y. Srinivasan 122199a50bb1SK. Y. Srinivasan case VM_PKT_DATA_INBAND: 1222f9a7da91SVitaly Kuznetsov netvsc_receive_inband(device, net_device_ctx, nvmsg); 122399a50bb1SK. Y. Srinivasan break; 122499a50bb1SK. Y. Srinivasan 122599a50bb1SK. Y. Srinivasan default: 122699a50bb1SK. Y. Srinivasan netdev_err(ndev, "unhandled packet type %d, tid %llx\n", 122799a50bb1SK. Y. Srinivasan desc->type, request_id); 122899a50bb1SK. Y. Srinivasan break; 122999a50bb1SK. Y. Srinivasan } 123099a50bb1SK. Y. Srinivasan } 123199a50bb1SK. Y. Srinivasan 12325b54dac8SHaiyang Zhang void netvsc_channel_cb(void *context) 123395fa0405SHaiyang Zhang { 12340b307ebdSstephen hemminger struct vmbus_channel *channel = context; 1235c0b558e5SHaiyang Zhang u16 q_idx = channel->offermsg.offer.sub_channel_index; 12365b54dac8SHaiyang Zhang struct hv_device *device; 123795fa0405SHaiyang Zhang struct netvsc_device *net_device; 123895fa0405SHaiyang Zhang struct vmpacket_descriptor *desc; 123995fa0405SHaiyang Zhang struct net_device *ndev; 124099a50bb1SK. Y. Srinivasan bool need_to_commit = false; 124195fa0405SHaiyang Zhang 12425b54dac8SHaiyang Zhang if (channel->primary_channel != NULL) 12435b54dac8SHaiyang Zhang device = channel->primary_channel->device_obj; 12445b54dac8SHaiyang Zhang else 12455b54dac8SHaiyang Zhang device = channel->device_obj; 12465b54dac8SHaiyang Zhang 124746b4f7f5Sstephen hemminger ndev = hv_get_drvdata(device); 124846b4f7f5Sstephen hemminger if (unlikely(!ndev)) 1249ee0c4c39SKY Srinivasan return; 125095fa0405SHaiyang Zhang 125146b4f7f5Sstephen hemminger net_device = net_device_to_netvsc_device(ndev); 125246b4f7f5Sstephen hemminger if (unlikely(net_device->destroy) && 125346b4f7f5Sstephen hemminger netvsc_channel_idle(net_device, q_idx)) 125446b4f7f5Sstephen hemminger return; 12550b307ebdSstephen hemminger 12560b307ebdSstephen hemminger while ((desc = get_next_pkt_raw(channel)) != NULL) { 12570b307ebdSstephen hemminger netvsc_process_raw_pkt(device, channel, net_device, 12580b307ebdSstephen hemminger ndev, desc->trans_id, desc); 125999a50bb1SK. Y. Srinivasan 126099a50bb1SK. Y. Srinivasan put_pkt_raw(channel, desc); 126199a50bb1SK. Y. Srinivasan need_to_commit = true; 126299a50bb1SK. Y. Srinivasan } 12630b307ebdSstephen hemminger 12640b307ebdSstephen hemminger if (need_to_commit) 126599a50bb1SK. Y. Srinivasan commit_rd_index(channel); 1266c0b558e5SHaiyang Zhang 1267c0b558e5SHaiyang Zhang netvsc_chk_recv_comp(net_device, channel, q_idx); 126895fa0405SHaiyang Zhang } 126995fa0405SHaiyang Zhang 127095fa0405SHaiyang Zhang /* 127195fa0405SHaiyang Zhang * netvsc_device_add - Callback when the device belonging to this 127295fa0405SHaiyang Zhang * driver is added 127395fa0405SHaiyang Zhang */ 12742c7f83caSstephen hemminger int netvsc_device_add(struct hv_device *device, 12752c7f83caSstephen hemminger const struct netvsc_device_info *device_info) 127695fa0405SHaiyang Zhang { 127788098834SVitaly Kuznetsov int i, ret = 0; 12782c7f83caSstephen hemminger int ring_size = device_info->ring_size; 127995fa0405SHaiyang Zhang struct netvsc_device *net_device; 128088098834SVitaly Kuznetsov struct net_device *ndev = hv_get_drvdata(device); 128188098834SVitaly Kuznetsov struct net_device_context *net_device_ctx = netdev_priv(ndev); 128295fa0405SHaiyang Zhang 128388098834SVitaly Kuznetsov net_device = alloc_net_device(); 1284b1c84927SDan Carpenter if (!net_device) 1285b1c84927SDan Carpenter return -ENOMEM; 128695fa0405SHaiyang Zhang 12875b54dac8SHaiyang Zhang net_device->ring_size = ring_size; 12885b54dac8SHaiyang Zhang 128995fa0405SHaiyang Zhang /* Open the channel */ 129095fa0405SHaiyang Zhang ret = vmbus_open(device->channel, ring_size * PAGE_SIZE, 129195fa0405SHaiyang Zhang ring_size * PAGE_SIZE, NULL, 0, 12925b54dac8SHaiyang Zhang netvsc_channel_cb, device->channel); 129395fa0405SHaiyang Zhang 129495fa0405SHaiyang Zhang if (ret != 0) { 129595fa0405SHaiyang Zhang netdev_err(ndev, "unable to open channel: %d\n", ret); 129695fa0405SHaiyang Zhang goto cleanup; 129795fa0405SHaiyang Zhang } 129895fa0405SHaiyang Zhang 129995fa0405SHaiyang Zhang /* Channel is opened */ 130093ba2222SVitaly Kuznetsov netdev_dbg(ndev, "hv_netvsc channel opened successfully\n"); 130195fa0405SHaiyang Zhang 130288098834SVitaly Kuznetsov /* If we're reopening the device we may have multiple queues, fill the 130388098834SVitaly Kuznetsov * chn_table with the default channel to use it before subchannels are 130488098834SVitaly Kuznetsov * opened. 130588098834SVitaly Kuznetsov */ 130688098834SVitaly Kuznetsov for (i = 0; i < VRSS_CHANNEL_MAX; i++) 1307b8b835a8Sstephen hemminger net_device->chan_table[i].channel = device->channel; 130888098834SVitaly Kuznetsov 130988098834SVitaly Kuznetsov /* Writing nvdev pointer unlocks netvsc_send(), make sure chn_table is 131088098834SVitaly Kuznetsov * populated. 131188098834SVitaly Kuznetsov */ 131288098834SVitaly Kuznetsov wmb(); 131388098834SVitaly Kuznetsov 131488098834SVitaly Kuznetsov net_device_ctx->nvdev = net_device; 13155b54dac8SHaiyang Zhang 131695fa0405SHaiyang Zhang /* Connect with the NetVsp */ 131795fa0405SHaiyang Zhang ret = netvsc_connect_vsp(device); 131895fa0405SHaiyang Zhang if (ret != 0) { 131995fa0405SHaiyang Zhang netdev_err(ndev, 132095fa0405SHaiyang Zhang "unable to connect to NetVSP - %d\n", ret); 132195fa0405SHaiyang Zhang goto close; 132295fa0405SHaiyang Zhang } 132395fa0405SHaiyang Zhang 132495fa0405SHaiyang Zhang return ret; 132595fa0405SHaiyang Zhang 132695fa0405SHaiyang Zhang close: 132795fa0405SHaiyang Zhang /* Now, we can close the channel safely */ 132895fa0405SHaiyang Zhang vmbus_close(device->channel); 132995fa0405SHaiyang Zhang 133095fa0405SHaiyang Zhang cleanup: 1331f90251c8SHaiyang Zhang free_netvsc_device(net_device); 133295fa0405SHaiyang Zhang 133395fa0405SHaiyang Zhang return ret; 133495fa0405SHaiyang Zhang } 1335