1 /* 2 * Copyright (c) 2009, Microsoft Corporation. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms and conditions of the GNU General Public License, 6 * version 2, as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 * more details. 12 * 13 * You should have received a copy of the GNU General Public License along with 14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple 15 * Place - Suite 330, Boston, MA 02111-1307 USA. 16 * 17 * Authors: 18 * Haiyang Zhang <haiyangz@microsoft.com> 19 * Hank Janssen <hjanssen@microsoft.com> 20 */ 21 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 22 23 #include <linux/kernel.h> 24 #include <linux/interrupt.h> 25 #include <linux/sched.h> 26 #include <linux/wait.h> 27 #include <linux/mm.h> 28 #include <linux/slab.h> 29 #include <linux/list.h> 30 #include <linux/module.h> 31 #include <linux/completion.h> 32 #include <linux/delay.h> 33 #include <linux/hyperv.h> 34 #include <asm/mshyperv.h> 35 36 #include "hyperv_vmbus.h" 37 38 static void init_vp_index(struct vmbus_channel *channel, u16 dev_type); 39 40 static const struct vmbus_device vmbus_devs[] = { 41 /* IDE */ 42 { .dev_type = HV_IDE, 43 HV_IDE_GUID, 44 .perf_device = true, 45 }, 46 47 /* SCSI */ 48 { .dev_type = HV_SCSI, 49 HV_SCSI_GUID, 50 .perf_device = true, 51 }, 52 53 /* Fibre Channel */ 54 { .dev_type = HV_FC, 55 HV_SYNTHFC_GUID, 56 .perf_device = true, 57 }, 58 59 /* Synthetic NIC */ 60 { .dev_type = HV_NIC, 61 HV_NIC_GUID, 62 .perf_device = true, 63 }, 64 65 /* Network Direct */ 66 { .dev_type = HV_ND, 67 HV_ND_GUID, 68 .perf_device = true, 69 }, 70 71 /* PCIE */ 72 { .dev_type = HV_PCIE, 73 HV_PCIE_GUID, 74 .perf_device = false, 75 }, 76 77 /* Synthetic Frame Buffer */ 78 { .dev_type = HV_FB, 79 HV_SYNTHVID_GUID, 80 .perf_device = false, 81 }, 82 83 /* Synthetic Keyboard */ 84 { .dev_type = HV_KBD, 85 HV_KBD_GUID, 86 .perf_device = false, 87 }, 88 89 /* Synthetic MOUSE */ 90 { .dev_type = HV_MOUSE, 91 HV_MOUSE_GUID, 92 .perf_device = false, 93 }, 94 95 /* KVP */ 96 { .dev_type = HV_KVP, 97 HV_KVP_GUID, 98 .perf_device = false, 99 }, 100 101 /* Time Synch */ 102 { .dev_type = HV_TS, 103 HV_TS_GUID, 104 .perf_device = false, 105 }, 106 107 /* Heartbeat */ 108 { .dev_type = HV_HB, 109 HV_HEART_BEAT_GUID, 110 .perf_device = false, 111 }, 112 113 /* Shutdown */ 114 { .dev_type = HV_SHUTDOWN, 115 HV_SHUTDOWN_GUID, 116 .perf_device = false, 117 }, 118 119 /* File copy */ 120 { .dev_type = HV_FCOPY, 121 HV_FCOPY_GUID, 122 .perf_device = false, 123 }, 124 125 /* Backup */ 126 { .dev_type = HV_BACKUP, 127 HV_VSS_GUID, 128 .perf_device = false, 129 }, 130 131 /* Dynamic Memory */ 132 { .dev_type = HV_DM, 133 HV_DM_GUID, 134 .perf_device = false, 135 }, 136 137 /* Unknown GUID */ 138 { .dev_type = HV_UNKNOWN, 139 .perf_device = false, 140 }, 141 }; 142 143 static const struct { 144 uuid_le guid; 145 } vmbus_unsupported_devs[] = { 146 { HV_AVMA1_GUID }, 147 { HV_AVMA2_GUID }, 148 { HV_RDV_GUID }, 149 }; 150 151 /* 152 * The rescinded channel may be blocked waiting for a response from the host; 153 * take care of that. 154 */ 155 static void vmbus_rescind_cleanup(struct vmbus_channel *channel) 156 { 157 struct vmbus_channel_msginfo *msginfo; 158 unsigned long flags; 159 160 161 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 162 channel->rescind = true; 163 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, 164 msglistentry) { 165 166 if (msginfo->waiting_channel == channel) { 167 complete(&msginfo->waitevent); 168 break; 169 } 170 } 171 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 172 } 173 174 static bool is_unsupported_vmbus_devs(const uuid_le *guid) 175 { 176 int i; 177 178 for (i = 0; i < ARRAY_SIZE(vmbus_unsupported_devs); i++) 179 if (!uuid_le_cmp(*guid, vmbus_unsupported_devs[i].guid)) 180 return true; 181 return false; 182 } 183 184 static u16 hv_get_dev_type(const struct vmbus_channel *channel) 185 { 186 const uuid_le *guid = &channel->offermsg.offer.if_type; 187 u16 i; 188 189 if (is_hvsock_channel(channel) || is_unsupported_vmbus_devs(guid)) 190 return HV_UNKNOWN; 191 192 for (i = HV_IDE; i < HV_UNKNOWN; i++) { 193 if (!uuid_le_cmp(*guid, vmbus_devs[i].guid)) 194 return i; 195 } 196 pr_info("Unknown GUID: %pUl\n", guid); 197 return i; 198 } 199 200 /** 201 * vmbus_prep_negotiate_resp() - Create default response for Negotiate message 202 * @icmsghdrp: Pointer to msg header structure 203 * @buf: Raw buffer channel data 204 * @fw_version: The framework versions we can support. 205 * @fw_vercnt: The size of @fw_version. 206 * @srv_version: The service versions we can support. 207 * @srv_vercnt: The size of @srv_version. 208 * @nego_fw_version: The selected framework version. 209 * @nego_srv_version: The selected service version. 210 * 211 * Note: Versions are given in decreasing order. 212 * 213 * Set up and fill in default negotiate response message. 214 * Mainly used by Hyper-V drivers. 215 */ 216 bool vmbus_prep_negotiate_resp(struct icmsg_hdr *icmsghdrp, 217 u8 *buf, const int *fw_version, int fw_vercnt, 218 const int *srv_version, int srv_vercnt, 219 int *nego_fw_version, int *nego_srv_version) 220 { 221 int icframe_major, icframe_minor; 222 int icmsg_major, icmsg_minor; 223 int fw_major, fw_minor; 224 int srv_major, srv_minor; 225 int i, j; 226 bool found_match = false; 227 struct icmsg_negotiate *negop; 228 229 icmsghdrp->icmsgsize = 0x10; 230 negop = (struct icmsg_negotiate *)&buf[ 231 sizeof(struct vmbuspipe_hdr) + 232 sizeof(struct icmsg_hdr)]; 233 234 icframe_major = negop->icframe_vercnt; 235 icframe_minor = 0; 236 237 icmsg_major = negop->icmsg_vercnt; 238 icmsg_minor = 0; 239 240 /* 241 * Select the framework version number we will 242 * support. 243 */ 244 245 for (i = 0; i < fw_vercnt; i++) { 246 fw_major = (fw_version[i] >> 16); 247 fw_minor = (fw_version[i] & 0xFFFF); 248 249 for (j = 0; j < negop->icframe_vercnt; j++) { 250 if ((negop->icversion_data[j].major == fw_major) && 251 (negop->icversion_data[j].minor == fw_minor)) { 252 icframe_major = negop->icversion_data[j].major; 253 icframe_minor = negop->icversion_data[j].minor; 254 found_match = true; 255 break; 256 } 257 } 258 259 if (found_match) 260 break; 261 } 262 263 if (!found_match) 264 goto fw_error; 265 266 found_match = false; 267 268 for (i = 0; i < srv_vercnt; i++) { 269 srv_major = (srv_version[i] >> 16); 270 srv_minor = (srv_version[i] & 0xFFFF); 271 272 for (j = negop->icframe_vercnt; 273 (j < negop->icframe_vercnt + negop->icmsg_vercnt); 274 j++) { 275 276 if ((negop->icversion_data[j].major == srv_major) && 277 (negop->icversion_data[j].minor == srv_minor)) { 278 279 icmsg_major = negop->icversion_data[j].major; 280 icmsg_minor = negop->icversion_data[j].minor; 281 found_match = true; 282 break; 283 } 284 } 285 286 if (found_match) 287 break; 288 } 289 290 /* 291 * Respond with the framework and service 292 * version numbers we can support. 293 */ 294 295 fw_error: 296 if (!found_match) { 297 negop->icframe_vercnt = 0; 298 negop->icmsg_vercnt = 0; 299 } else { 300 negop->icframe_vercnt = 1; 301 negop->icmsg_vercnt = 1; 302 } 303 304 if (nego_fw_version) 305 *nego_fw_version = (icframe_major << 16) | icframe_minor; 306 307 if (nego_srv_version) 308 *nego_srv_version = (icmsg_major << 16) | icmsg_minor; 309 310 negop->icversion_data[0].major = icframe_major; 311 negop->icversion_data[0].minor = icframe_minor; 312 negop->icversion_data[1].major = icmsg_major; 313 negop->icversion_data[1].minor = icmsg_minor; 314 return found_match; 315 } 316 317 EXPORT_SYMBOL_GPL(vmbus_prep_negotiate_resp); 318 319 /* 320 * alloc_channel - Allocate and initialize a vmbus channel object 321 */ 322 static struct vmbus_channel *alloc_channel(void) 323 { 324 struct vmbus_channel *channel; 325 326 channel = kzalloc(sizeof(*channel), GFP_ATOMIC); 327 if (!channel) 328 return NULL; 329 330 spin_lock_init(&channel->lock); 331 init_completion(&channel->rescind_event); 332 333 INIT_LIST_HEAD(&channel->sc_list); 334 INIT_LIST_HEAD(&channel->percpu_list); 335 336 tasklet_init(&channel->callback_event, 337 vmbus_on_event, (unsigned long)channel); 338 339 return channel; 340 } 341 342 /* 343 * free_channel - Release the resources used by the vmbus channel object 344 */ 345 static void free_channel(struct vmbus_channel *channel) 346 { 347 tasklet_kill(&channel->callback_event); 348 349 kobject_put(&channel->kobj); 350 } 351 352 static void percpu_channel_enq(void *arg) 353 { 354 struct vmbus_channel *channel = arg; 355 struct hv_per_cpu_context *hv_cpu 356 = this_cpu_ptr(hv_context.cpu_context); 357 358 list_add_tail_rcu(&channel->percpu_list, &hv_cpu->chan_list); 359 } 360 361 static void percpu_channel_deq(void *arg) 362 { 363 struct vmbus_channel *channel = arg; 364 365 list_del_rcu(&channel->percpu_list); 366 } 367 368 369 static void vmbus_release_relid(u32 relid) 370 { 371 struct vmbus_channel_relid_released msg; 372 int ret; 373 374 memset(&msg, 0, sizeof(struct vmbus_channel_relid_released)); 375 msg.child_relid = relid; 376 msg.header.msgtype = CHANNELMSG_RELID_RELEASED; 377 ret = vmbus_post_msg(&msg, sizeof(struct vmbus_channel_relid_released), 378 true); 379 380 trace_vmbus_release_relid(&msg, ret); 381 } 382 383 void hv_process_channel_removal(struct vmbus_channel *channel) 384 { 385 struct vmbus_channel *primary_channel; 386 unsigned long flags; 387 388 BUG_ON(!mutex_is_locked(&vmbus_connection.channel_mutex)); 389 BUG_ON(!channel->rescind); 390 391 if (channel->target_cpu != get_cpu()) { 392 put_cpu(); 393 smp_call_function_single(channel->target_cpu, 394 percpu_channel_deq, channel, true); 395 } else { 396 percpu_channel_deq(channel); 397 put_cpu(); 398 } 399 400 if (channel->primary_channel == NULL) { 401 list_del(&channel->listentry); 402 403 primary_channel = channel; 404 } else { 405 primary_channel = channel->primary_channel; 406 spin_lock_irqsave(&primary_channel->lock, flags); 407 list_del(&channel->sc_list); 408 primary_channel->num_sc--; 409 spin_unlock_irqrestore(&primary_channel->lock, flags); 410 } 411 412 /* 413 * We need to free the bit for init_vp_index() to work in the case 414 * of sub-channel, when we reload drivers like hv_netvsc. 415 */ 416 if (channel->affinity_policy == HV_LOCALIZED) 417 cpumask_clear_cpu(channel->target_cpu, 418 &primary_channel->alloced_cpus_in_node); 419 420 vmbus_release_relid(channel->offermsg.child_relid); 421 422 free_channel(channel); 423 } 424 425 void vmbus_free_channels(void) 426 { 427 struct vmbus_channel *channel, *tmp; 428 429 list_for_each_entry_safe(channel, tmp, &vmbus_connection.chn_list, 430 listentry) { 431 /* hv_process_channel_removal() needs this */ 432 channel->rescind = true; 433 434 vmbus_device_unregister(channel->device_obj); 435 } 436 } 437 438 /* 439 * vmbus_process_offer - Process the offer by creating a channel/device 440 * associated with this offer 441 */ 442 static void vmbus_process_offer(struct vmbus_channel *newchannel) 443 { 444 struct vmbus_channel *channel; 445 bool fnew = true; 446 unsigned long flags; 447 u16 dev_type; 448 int ret; 449 450 /* Make sure this is a new offer */ 451 mutex_lock(&vmbus_connection.channel_mutex); 452 453 /* 454 * Now that we have acquired the channel_mutex, 455 * we can release the potentially racing rescind thread. 456 */ 457 atomic_dec(&vmbus_connection.offer_in_progress); 458 459 list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) { 460 if (!uuid_le_cmp(channel->offermsg.offer.if_type, 461 newchannel->offermsg.offer.if_type) && 462 !uuid_le_cmp(channel->offermsg.offer.if_instance, 463 newchannel->offermsg.offer.if_instance)) { 464 fnew = false; 465 break; 466 } 467 } 468 469 if (fnew) 470 list_add_tail(&newchannel->listentry, 471 &vmbus_connection.chn_list); 472 473 mutex_unlock(&vmbus_connection.channel_mutex); 474 475 if (!fnew) { 476 /* 477 * Check to see if this is a sub-channel. 478 */ 479 if (newchannel->offermsg.offer.sub_channel_index != 0) { 480 /* 481 * Process the sub-channel. 482 */ 483 newchannel->primary_channel = channel; 484 spin_lock_irqsave(&channel->lock, flags); 485 list_add_tail(&newchannel->sc_list, &channel->sc_list); 486 channel->num_sc++; 487 spin_unlock_irqrestore(&channel->lock, flags); 488 } else { 489 goto err_free_chan; 490 } 491 } 492 493 dev_type = hv_get_dev_type(newchannel); 494 495 init_vp_index(newchannel, dev_type); 496 497 if (newchannel->target_cpu != get_cpu()) { 498 put_cpu(); 499 smp_call_function_single(newchannel->target_cpu, 500 percpu_channel_enq, 501 newchannel, true); 502 } else { 503 percpu_channel_enq(newchannel); 504 put_cpu(); 505 } 506 507 /* 508 * This state is used to indicate a successful open 509 * so that when we do close the channel normally, we 510 * can cleanup properly 511 */ 512 newchannel->state = CHANNEL_OPEN_STATE; 513 514 if (!fnew) { 515 struct hv_device *dev 516 = newchannel->primary_channel->device_obj; 517 518 if (vmbus_add_channel_kobj(dev, newchannel)) 519 goto err_free_chan; 520 521 if (channel->sc_creation_callback != NULL) 522 channel->sc_creation_callback(newchannel); 523 newchannel->probe_done = true; 524 return; 525 } 526 527 /* 528 * Start the process of binding this offer to the driver 529 * We need to set the DeviceObject field before calling 530 * vmbus_child_dev_add() 531 */ 532 newchannel->device_obj = vmbus_device_create( 533 &newchannel->offermsg.offer.if_type, 534 &newchannel->offermsg.offer.if_instance, 535 newchannel); 536 if (!newchannel->device_obj) 537 goto err_deq_chan; 538 539 newchannel->device_obj->device_id = dev_type; 540 /* 541 * Add the new device to the bus. This will kick off device-driver 542 * binding which eventually invokes the device driver's AddDevice() 543 * method. 544 */ 545 ret = vmbus_device_register(newchannel->device_obj); 546 547 if (ret != 0) { 548 pr_err("unable to add child device object (relid %d)\n", 549 newchannel->offermsg.child_relid); 550 kfree(newchannel->device_obj); 551 goto err_deq_chan; 552 } 553 554 newchannel->probe_done = true; 555 return; 556 557 err_deq_chan: 558 mutex_lock(&vmbus_connection.channel_mutex); 559 list_del(&newchannel->listentry); 560 mutex_unlock(&vmbus_connection.channel_mutex); 561 562 if (newchannel->target_cpu != get_cpu()) { 563 put_cpu(); 564 smp_call_function_single(newchannel->target_cpu, 565 percpu_channel_deq, newchannel, true); 566 } else { 567 percpu_channel_deq(newchannel); 568 put_cpu(); 569 } 570 571 vmbus_release_relid(newchannel->offermsg.child_relid); 572 573 err_free_chan: 574 free_channel(newchannel); 575 } 576 577 /* 578 * We use this state to statically distribute the channel interrupt load. 579 */ 580 static int next_numa_node_id; 581 582 /* 583 * Starting with Win8, we can statically distribute the incoming 584 * channel interrupt load by binding a channel to VCPU. 585 * We distribute the interrupt loads to one or more NUMA nodes based on 586 * the channel's affinity_policy. 587 * 588 * For pre-win8 hosts or non-performance critical channels we assign the 589 * first CPU in the first NUMA node. 590 */ 591 static void init_vp_index(struct vmbus_channel *channel, u16 dev_type) 592 { 593 u32 cur_cpu; 594 bool perf_chn = vmbus_devs[dev_type].perf_device; 595 struct vmbus_channel *primary = channel->primary_channel; 596 int next_node; 597 cpumask_var_t available_mask; 598 struct cpumask *alloced_mask; 599 600 if ((vmbus_proto_version == VERSION_WS2008) || 601 (vmbus_proto_version == VERSION_WIN7) || (!perf_chn) || 602 !alloc_cpumask_var(&available_mask, GFP_KERNEL)) { 603 /* 604 * Prior to win8, all channel interrupts are 605 * delivered on cpu 0. 606 * Also if the channel is not a performance critical 607 * channel, bind it to cpu 0. 608 * In case alloc_cpumask_var() fails, bind it to cpu 0. 609 */ 610 channel->numa_node = 0; 611 channel->target_cpu = 0; 612 channel->target_vp = hv_cpu_number_to_vp_number(0); 613 return; 614 } 615 616 /* 617 * Based on the channel affinity policy, we will assign the NUMA 618 * nodes. 619 */ 620 621 if ((channel->affinity_policy == HV_BALANCED) || (!primary)) { 622 while (true) { 623 next_node = next_numa_node_id++; 624 if (next_node == nr_node_ids) { 625 next_node = next_numa_node_id = 0; 626 continue; 627 } 628 if (cpumask_empty(cpumask_of_node(next_node))) 629 continue; 630 break; 631 } 632 channel->numa_node = next_node; 633 primary = channel; 634 } 635 alloced_mask = &hv_context.hv_numa_map[primary->numa_node]; 636 637 if (cpumask_weight(alloced_mask) == 638 cpumask_weight(cpumask_of_node(primary->numa_node))) { 639 /* 640 * We have cycled through all the CPUs in the node; 641 * reset the alloced map. 642 */ 643 cpumask_clear(alloced_mask); 644 } 645 646 cpumask_xor(available_mask, alloced_mask, 647 cpumask_of_node(primary->numa_node)); 648 649 cur_cpu = -1; 650 651 if (primary->affinity_policy == HV_LOCALIZED) { 652 /* 653 * Normally Hyper-V host doesn't create more subchannels 654 * than there are VCPUs on the node but it is possible when not 655 * all present VCPUs on the node are initialized by guest. 656 * Clear the alloced_cpus_in_node to start over. 657 */ 658 if (cpumask_equal(&primary->alloced_cpus_in_node, 659 cpumask_of_node(primary->numa_node))) 660 cpumask_clear(&primary->alloced_cpus_in_node); 661 } 662 663 while (true) { 664 cur_cpu = cpumask_next(cur_cpu, available_mask); 665 if (cur_cpu >= nr_cpu_ids) { 666 cur_cpu = -1; 667 cpumask_copy(available_mask, 668 cpumask_of_node(primary->numa_node)); 669 continue; 670 } 671 672 if (primary->affinity_policy == HV_LOCALIZED) { 673 /* 674 * NOTE: in the case of sub-channel, we clear the 675 * sub-channel related bit(s) in 676 * primary->alloced_cpus_in_node in 677 * hv_process_channel_removal(), so when we 678 * reload drivers like hv_netvsc in SMP guest, here 679 * we're able to re-allocate 680 * bit from primary->alloced_cpus_in_node. 681 */ 682 if (!cpumask_test_cpu(cur_cpu, 683 &primary->alloced_cpus_in_node)) { 684 cpumask_set_cpu(cur_cpu, 685 &primary->alloced_cpus_in_node); 686 cpumask_set_cpu(cur_cpu, alloced_mask); 687 break; 688 } 689 } else { 690 cpumask_set_cpu(cur_cpu, alloced_mask); 691 break; 692 } 693 } 694 695 channel->target_cpu = cur_cpu; 696 channel->target_vp = hv_cpu_number_to_vp_number(cur_cpu); 697 698 free_cpumask_var(available_mask); 699 } 700 701 static void vmbus_wait_for_unload(void) 702 { 703 int cpu; 704 void *page_addr; 705 struct hv_message *msg; 706 struct vmbus_channel_message_header *hdr; 707 u32 message_type; 708 709 /* 710 * CHANNELMSG_UNLOAD_RESPONSE is always delivered to the CPU which was 711 * used for initial contact or to CPU0 depending on host version. When 712 * we're crashing on a different CPU let's hope that IRQ handler on 713 * the cpu which receives CHANNELMSG_UNLOAD_RESPONSE is still 714 * functional and vmbus_unload_response() will complete 715 * vmbus_connection.unload_event. If not, the last thing we can do is 716 * read message pages for all CPUs directly. 717 */ 718 while (1) { 719 if (completion_done(&vmbus_connection.unload_event)) 720 break; 721 722 for_each_online_cpu(cpu) { 723 struct hv_per_cpu_context *hv_cpu 724 = per_cpu_ptr(hv_context.cpu_context, cpu); 725 726 page_addr = hv_cpu->synic_message_page; 727 msg = (struct hv_message *)page_addr 728 + VMBUS_MESSAGE_SINT; 729 730 message_type = READ_ONCE(msg->header.message_type); 731 if (message_type == HVMSG_NONE) 732 continue; 733 734 hdr = (struct vmbus_channel_message_header *) 735 msg->u.payload; 736 737 if (hdr->msgtype == CHANNELMSG_UNLOAD_RESPONSE) 738 complete(&vmbus_connection.unload_event); 739 740 vmbus_signal_eom(msg, message_type); 741 } 742 743 mdelay(10); 744 } 745 746 /* 747 * We're crashing and already got the UNLOAD_RESPONSE, cleanup all 748 * maybe-pending messages on all CPUs to be able to receive new 749 * messages after we reconnect. 750 */ 751 for_each_online_cpu(cpu) { 752 struct hv_per_cpu_context *hv_cpu 753 = per_cpu_ptr(hv_context.cpu_context, cpu); 754 755 page_addr = hv_cpu->synic_message_page; 756 msg = (struct hv_message *)page_addr + VMBUS_MESSAGE_SINT; 757 msg->header.message_type = HVMSG_NONE; 758 } 759 } 760 761 /* 762 * vmbus_unload_response - Handler for the unload response. 763 */ 764 static void vmbus_unload_response(struct vmbus_channel_message_header *hdr) 765 { 766 /* 767 * This is a global event; just wakeup the waiting thread. 768 * Once we successfully unload, we can cleanup the monitor state. 769 */ 770 complete(&vmbus_connection.unload_event); 771 } 772 773 void vmbus_initiate_unload(bool crash) 774 { 775 struct vmbus_channel_message_header hdr; 776 777 /* Pre-Win2012R2 hosts don't support reconnect */ 778 if (vmbus_proto_version < VERSION_WIN8_1) 779 return; 780 781 init_completion(&vmbus_connection.unload_event); 782 memset(&hdr, 0, sizeof(struct vmbus_channel_message_header)); 783 hdr.msgtype = CHANNELMSG_UNLOAD; 784 vmbus_post_msg(&hdr, sizeof(struct vmbus_channel_message_header), 785 !crash); 786 787 /* 788 * vmbus_initiate_unload() is also called on crash and the crash can be 789 * happening in an interrupt context, where scheduling is impossible. 790 */ 791 if (!crash) 792 wait_for_completion(&vmbus_connection.unload_event); 793 else 794 vmbus_wait_for_unload(); 795 } 796 797 /* 798 * vmbus_onoffer - Handler for channel offers from vmbus in parent partition. 799 * 800 */ 801 static void vmbus_onoffer(struct vmbus_channel_message_header *hdr) 802 { 803 struct vmbus_channel_offer_channel *offer; 804 struct vmbus_channel *newchannel; 805 806 offer = (struct vmbus_channel_offer_channel *)hdr; 807 808 trace_vmbus_onoffer(offer); 809 810 /* Allocate the channel object and save this offer. */ 811 newchannel = alloc_channel(); 812 if (!newchannel) { 813 vmbus_release_relid(offer->child_relid); 814 atomic_dec(&vmbus_connection.offer_in_progress); 815 pr_err("Unable to allocate channel object\n"); 816 return; 817 } 818 819 /* 820 * Setup state for signalling the host. 821 */ 822 newchannel->sig_event = VMBUS_EVENT_CONNECTION_ID; 823 824 if (vmbus_proto_version != VERSION_WS2008) { 825 newchannel->is_dedicated_interrupt = 826 (offer->is_dedicated_interrupt != 0); 827 newchannel->sig_event = offer->connection_id; 828 } 829 830 memcpy(&newchannel->offermsg, offer, 831 sizeof(struct vmbus_channel_offer_channel)); 832 newchannel->monitor_grp = (u8)offer->monitorid / 32; 833 newchannel->monitor_bit = (u8)offer->monitorid % 32; 834 835 vmbus_process_offer(newchannel); 836 } 837 838 /* 839 * vmbus_onoffer_rescind - Rescind offer handler. 840 * 841 * We queue a work item to process this offer synchronously 842 */ 843 static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr) 844 { 845 struct vmbus_channel_rescind_offer *rescind; 846 struct vmbus_channel *channel; 847 struct device *dev; 848 849 rescind = (struct vmbus_channel_rescind_offer *)hdr; 850 851 trace_vmbus_onoffer_rescind(rescind); 852 853 /* 854 * The offer msg and the corresponding rescind msg 855 * from the host are guranteed to be ordered - 856 * offer comes in first and then the rescind. 857 * Since we process these events in work elements, 858 * and with preemption, we may end up processing 859 * the events out of order. Given that we handle these 860 * work elements on the same CPU, this is possible only 861 * in the case of preemption. In any case wait here 862 * until the offer processing has moved beyond the 863 * point where the channel is discoverable. 864 */ 865 866 while (atomic_read(&vmbus_connection.offer_in_progress) != 0) { 867 /* 868 * We wait here until any channel offer is currently 869 * being processed. 870 */ 871 msleep(1); 872 } 873 874 mutex_lock(&vmbus_connection.channel_mutex); 875 channel = relid2channel(rescind->child_relid); 876 mutex_unlock(&vmbus_connection.channel_mutex); 877 878 if (channel == NULL) { 879 /* 880 * We failed in processing the offer message; 881 * we would have cleaned up the relid in that 882 * failure path. 883 */ 884 return; 885 } 886 887 /* 888 * Before setting channel->rescind in vmbus_rescind_cleanup(), we 889 * should make sure the channel callback is not running any more. 890 */ 891 vmbus_reset_channel_cb(channel); 892 893 /* 894 * Now wait for offer handling to complete. 895 */ 896 vmbus_rescind_cleanup(channel); 897 while (READ_ONCE(channel->probe_done) == false) { 898 /* 899 * We wait here until any channel offer is currently 900 * being processed. 901 */ 902 msleep(1); 903 } 904 905 /* 906 * At this point, the rescind handling can proceed safely. 907 */ 908 909 if (channel->device_obj) { 910 if (channel->chn_rescind_callback) { 911 channel->chn_rescind_callback(channel); 912 return; 913 } 914 /* 915 * We will have to unregister this device from the 916 * driver core. 917 */ 918 dev = get_device(&channel->device_obj->device); 919 if (dev) { 920 vmbus_device_unregister(channel->device_obj); 921 put_device(dev); 922 } 923 } 924 if (channel->primary_channel != NULL) { 925 /* 926 * Sub-channel is being rescinded. Following is the channel 927 * close sequence when initiated from the driveri (refer to 928 * vmbus_close() for details): 929 * 1. Close all sub-channels first 930 * 2. Then close the primary channel. 931 */ 932 mutex_lock(&vmbus_connection.channel_mutex); 933 if (channel->state == CHANNEL_OPEN_STATE) { 934 /* 935 * The channel is currently not open; 936 * it is safe for us to cleanup the channel. 937 */ 938 hv_process_channel_removal(channel); 939 } else { 940 complete(&channel->rescind_event); 941 } 942 mutex_unlock(&vmbus_connection.channel_mutex); 943 } 944 } 945 946 void vmbus_hvsock_device_unregister(struct vmbus_channel *channel) 947 { 948 BUG_ON(!is_hvsock_channel(channel)); 949 950 /* We always get a rescind msg when a connection is closed. */ 951 while (!READ_ONCE(channel->probe_done) || !READ_ONCE(channel->rescind)) 952 msleep(1); 953 954 vmbus_device_unregister(channel->device_obj); 955 } 956 EXPORT_SYMBOL_GPL(vmbus_hvsock_device_unregister); 957 958 959 /* 960 * vmbus_onoffers_delivered - 961 * This is invoked when all offers have been delivered. 962 * 963 * Nothing to do here. 964 */ 965 static void vmbus_onoffers_delivered( 966 struct vmbus_channel_message_header *hdr) 967 { 968 } 969 970 /* 971 * vmbus_onopen_result - Open result handler. 972 * 973 * This is invoked when we received a response to our channel open request. 974 * Find the matching request, copy the response and signal the requesting 975 * thread. 976 */ 977 static void vmbus_onopen_result(struct vmbus_channel_message_header *hdr) 978 { 979 struct vmbus_channel_open_result *result; 980 struct vmbus_channel_msginfo *msginfo; 981 struct vmbus_channel_message_header *requestheader; 982 struct vmbus_channel_open_channel *openmsg; 983 unsigned long flags; 984 985 result = (struct vmbus_channel_open_result *)hdr; 986 987 trace_vmbus_onopen_result(result); 988 989 /* 990 * Find the open msg, copy the result and signal/unblock the wait event 991 */ 992 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 993 994 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, 995 msglistentry) { 996 requestheader = 997 (struct vmbus_channel_message_header *)msginfo->msg; 998 999 if (requestheader->msgtype == CHANNELMSG_OPENCHANNEL) { 1000 openmsg = 1001 (struct vmbus_channel_open_channel *)msginfo->msg; 1002 if (openmsg->child_relid == result->child_relid && 1003 openmsg->openid == result->openid) { 1004 memcpy(&msginfo->response.open_result, 1005 result, 1006 sizeof( 1007 struct vmbus_channel_open_result)); 1008 complete(&msginfo->waitevent); 1009 break; 1010 } 1011 } 1012 } 1013 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 1014 } 1015 1016 /* 1017 * vmbus_ongpadl_created - GPADL created handler. 1018 * 1019 * This is invoked when we received a response to our gpadl create request. 1020 * Find the matching request, copy the response and signal the requesting 1021 * thread. 1022 */ 1023 static void vmbus_ongpadl_created(struct vmbus_channel_message_header *hdr) 1024 { 1025 struct vmbus_channel_gpadl_created *gpadlcreated; 1026 struct vmbus_channel_msginfo *msginfo; 1027 struct vmbus_channel_message_header *requestheader; 1028 struct vmbus_channel_gpadl_header *gpadlheader; 1029 unsigned long flags; 1030 1031 gpadlcreated = (struct vmbus_channel_gpadl_created *)hdr; 1032 1033 trace_vmbus_ongpadl_created(gpadlcreated); 1034 1035 /* 1036 * Find the establish msg, copy the result and signal/unblock the wait 1037 * event 1038 */ 1039 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 1040 1041 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, 1042 msglistentry) { 1043 requestheader = 1044 (struct vmbus_channel_message_header *)msginfo->msg; 1045 1046 if (requestheader->msgtype == CHANNELMSG_GPADL_HEADER) { 1047 gpadlheader = 1048 (struct vmbus_channel_gpadl_header *)requestheader; 1049 1050 if ((gpadlcreated->child_relid == 1051 gpadlheader->child_relid) && 1052 (gpadlcreated->gpadl == gpadlheader->gpadl)) { 1053 memcpy(&msginfo->response.gpadl_created, 1054 gpadlcreated, 1055 sizeof( 1056 struct vmbus_channel_gpadl_created)); 1057 complete(&msginfo->waitevent); 1058 break; 1059 } 1060 } 1061 } 1062 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 1063 } 1064 1065 /* 1066 * vmbus_ongpadl_torndown - GPADL torndown handler. 1067 * 1068 * This is invoked when we received a response to our gpadl teardown request. 1069 * Find the matching request, copy the response and signal the requesting 1070 * thread. 1071 */ 1072 static void vmbus_ongpadl_torndown( 1073 struct vmbus_channel_message_header *hdr) 1074 { 1075 struct vmbus_channel_gpadl_torndown *gpadl_torndown; 1076 struct vmbus_channel_msginfo *msginfo; 1077 struct vmbus_channel_message_header *requestheader; 1078 struct vmbus_channel_gpadl_teardown *gpadl_teardown; 1079 unsigned long flags; 1080 1081 gpadl_torndown = (struct vmbus_channel_gpadl_torndown *)hdr; 1082 1083 trace_vmbus_ongpadl_torndown(gpadl_torndown); 1084 1085 /* 1086 * Find the open msg, copy the result and signal/unblock the wait event 1087 */ 1088 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 1089 1090 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, 1091 msglistentry) { 1092 requestheader = 1093 (struct vmbus_channel_message_header *)msginfo->msg; 1094 1095 if (requestheader->msgtype == CHANNELMSG_GPADL_TEARDOWN) { 1096 gpadl_teardown = 1097 (struct vmbus_channel_gpadl_teardown *)requestheader; 1098 1099 if (gpadl_torndown->gpadl == gpadl_teardown->gpadl) { 1100 memcpy(&msginfo->response.gpadl_torndown, 1101 gpadl_torndown, 1102 sizeof( 1103 struct vmbus_channel_gpadl_torndown)); 1104 complete(&msginfo->waitevent); 1105 break; 1106 } 1107 } 1108 } 1109 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 1110 } 1111 1112 /* 1113 * vmbus_onversion_response - Version response handler 1114 * 1115 * This is invoked when we received a response to our initiate contact request. 1116 * Find the matching request, copy the response and signal the requesting 1117 * thread. 1118 */ 1119 static void vmbus_onversion_response( 1120 struct vmbus_channel_message_header *hdr) 1121 { 1122 struct vmbus_channel_msginfo *msginfo; 1123 struct vmbus_channel_message_header *requestheader; 1124 struct vmbus_channel_version_response *version_response; 1125 unsigned long flags; 1126 1127 version_response = (struct vmbus_channel_version_response *)hdr; 1128 1129 trace_vmbus_onversion_response(version_response); 1130 1131 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 1132 1133 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, 1134 msglistentry) { 1135 requestheader = 1136 (struct vmbus_channel_message_header *)msginfo->msg; 1137 1138 if (requestheader->msgtype == 1139 CHANNELMSG_INITIATE_CONTACT) { 1140 memcpy(&msginfo->response.version_response, 1141 version_response, 1142 sizeof(struct vmbus_channel_version_response)); 1143 complete(&msginfo->waitevent); 1144 } 1145 } 1146 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 1147 } 1148 1149 /* Channel message dispatch table */ 1150 const struct vmbus_channel_message_table_entry 1151 channel_message_table[CHANNELMSG_COUNT] = { 1152 { CHANNELMSG_INVALID, 0, NULL }, 1153 { CHANNELMSG_OFFERCHANNEL, 0, vmbus_onoffer }, 1154 { CHANNELMSG_RESCIND_CHANNELOFFER, 0, vmbus_onoffer_rescind }, 1155 { CHANNELMSG_REQUESTOFFERS, 0, NULL }, 1156 { CHANNELMSG_ALLOFFERS_DELIVERED, 1, vmbus_onoffers_delivered }, 1157 { CHANNELMSG_OPENCHANNEL, 0, NULL }, 1158 { CHANNELMSG_OPENCHANNEL_RESULT, 1, vmbus_onopen_result }, 1159 { CHANNELMSG_CLOSECHANNEL, 0, NULL }, 1160 { CHANNELMSG_GPADL_HEADER, 0, NULL }, 1161 { CHANNELMSG_GPADL_BODY, 0, NULL }, 1162 { CHANNELMSG_GPADL_CREATED, 1, vmbus_ongpadl_created }, 1163 { CHANNELMSG_GPADL_TEARDOWN, 0, NULL }, 1164 { CHANNELMSG_GPADL_TORNDOWN, 1, vmbus_ongpadl_torndown }, 1165 { CHANNELMSG_RELID_RELEASED, 0, NULL }, 1166 { CHANNELMSG_INITIATE_CONTACT, 0, NULL }, 1167 { CHANNELMSG_VERSION_RESPONSE, 1, vmbus_onversion_response }, 1168 { CHANNELMSG_UNLOAD, 0, NULL }, 1169 { CHANNELMSG_UNLOAD_RESPONSE, 1, vmbus_unload_response }, 1170 { CHANNELMSG_18, 0, NULL }, 1171 { CHANNELMSG_19, 0, NULL }, 1172 { CHANNELMSG_20, 0, NULL }, 1173 { CHANNELMSG_TL_CONNECT_REQUEST, 0, NULL }, 1174 }; 1175 1176 /* 1177 * vmbus_onmessage - Handler for channel protocol messages. 1178 * 1179 * This is invoked in the vmbus worker thread context. 1180 */ 1181 void vmbus_onmessage(void *context) 1182 { 1183 struct hv_message *msg = context; 1184 struct vmbus_channel_message_header *hdr; 1185 int size; 1186 1187 hdr = (struct vmbus_channel_message_header *)msg->u.payload; 1188 size = msg->header.payload_size; 1189 1190 trace_vmbus_on_message(hdr); 1191 1192 if (hdr->msgtype >= CHANNELMSG_COUNT) { 1193 pr_err("Received invalid channel message type %d size %d\n", 1194 hdr->msgtype, size); 1195 print_hex_dump_bytes("", DUMP_PREFIX_NONE, 1196 (unsigned char *)msg->u.payload, size); 1197 return; 1198 } 1199 1200 if (channel_message_table[hdr->msgtype].message_handler) 1201 channel_message_table[hdr->msgtype].message_handler(hdr); 1202 else 1203 pr_err("Unhandled channel message type %d\n", hdr->msgtype); 1204 } 1205 1206 /* 1207 * vmbus_request_offers - Send a request to get all our pending offers. 1208 */ 1209 int vmbus_request_offers(void) 1210 { 1211 struct vmbus_channel_message_header *msg; 1212 struct vmbus_channel_msginfo *msginfo; 1213 int ret; 1214 1215 msginfo = kmalloc(sizeof(*msginfo) + 1216 sizeof(struct vmbus_channel_message_header), 1217 GFP_KERNEL); 1218 if (!msginfo) 1219 return -ENOMEM; 1220 1221 msg = (struct vmbus_channel_message_header *)msginfo->msg; 1222 1223 msg->msgtype = CHANNELMSG_REQUESTOFFERS; 1224 1225 ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_message_header), 1226 true); 1227 1228 trace_vmbus_request_offers(ret); 1229 1230 if (ret != 0) { 1231 pr_err("Unable to request offers - %d\n", ret); 1232 1233 goto cleanup; 1234 } 1235 1236 cleanup: 1237 kfree(msginfo); 1238 1239 return ret; 1240 } 1241 1242 /* 1243 * Retrieve the (sub) channel on which to send an outgoing request. 1244 * When a primary channel has multiple sub-channels, we try to 1245 * distribute the load equally amongst all available channels. 1246 */ 1247 struct vmbus_channel *vmbus_get_outgoing_channel(struct vmbus_channel *primary) 1248 { 1249 struct list_head *cur, *tmp; 1250 int cur_cpu; 1251 struct vmbus_channel *cur_channel; 1252 struct vmbus_channel *outgoing_channel = primary; 1253 int next_channel; 1254 int i = 1; 1255 1256 if (list_empty(&primary->sc_list)) 1257 return outgoing_channel; 1258 1259 next_channel = primary->next_oc++; 1260 1261 if (next_channel > (primary->num_sc)) { 1262 primary->next_oc = 0; 1263 return outgoing_channel; 1264 } 1265 1266 cur_cpu = hv_cpu_number_to_vp_number(smp_processor_id()); 1267 list_for_each_safe(cur, tmp, &primary->sc_list) { 1268 cur_channel = list_entry(cur, struct vmbus_channel, sc_list); 1269 if (cur_channel->state != CHANNEL_OPENED_STATE) 1270 continue; 1271 1272 if (cur_channel->target_vp == cur_cpu) 1273 return cur_channel; 1274 1275 if (i == next_channel) 1276 return cur_channel; 1277 1278 i++; 1279 } 1280 1281 return outgoing_channel; 1282 } 1283 EXPORT_SYMBOL_GPL(vmbus_get_outgoing_channel); 1284 1285 static void invoke_sc_cb(struct vmbus_channel *primary_channel) 1286 { 1287 struct list_head *cur, *tmp; 1288 struct vmbus_channel *cur_channel; 1289 1290 if (primary_channel->sc_creation_callback == NULL) 1291 return; 1292 1293 list_for_each_safe(cur, tmp, &primary_channel->sc_list) { 1294 cur_channel = list_entry(cur, struct vmbus_channel, sc_list); 1295 1296 primary_channel->sc_creation_callback(cur_channel); 1297 } 1298 } 1299 1300 void vmbus_set_sc_create_callback(struct vmbus_channel *primary_channel, 1301 void (*sc_cr_cb)(struct vmbus_channel *new_sc)) 1302 { 1303 primary_channel->sc_creation_callback = sc_cr_cb; 1304 } 1305 EXPORT_SYMBOL_GPL(vmbus_set_sc_create_callback); 1306 1307 bool vmbus_are_subchannels_present(struct vmbus_channel *primary) 1308 { 1309 bool ret; 1310 1311 ret = !list_empty(&primary->sc_list); 1312 1313 if (ret) { 1314 /* 1315 * Invoke the callback on sub-channel creation. 1316 * This will present a uniform interface to the 1317 * clients. 1318 */ 1319 invoke_sc_cb(primary); 1320 } 1321 1322 return ret; 1323 } 1324 EXPORT_SYMBOL_GPL(vmbus_are_subchannels_present); 1325 1326 void vmbus_set_chn_rescind_callback(struct vmbus_channel *channel, 1327 void (*chn_rescind_cb)(struct vmbus_channel *)) 1328 { 1329 channel->chn_rescind_callback = chn_rescind_cb; 1330 } 1331 EXPORT_SYMBOL_GPL(vmbus_set_chn_rescind_callback); 1332