1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * 4 * Copyright (c) 2009, Microsoft Corporation. 5 * 6 * Authors: 7 * Haiyang Zhang <haiyangz@microsoft.com> 8 * Hank Janssen <hjanssen@microsoft.com> 9 */ 10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 11 12 #include <linux/kernel.h> 13 #include <linux/sched.h> 14 #include <linux/wait.h> 15 #include <linux/delay.h> 16 #include <linux/mm.h> 17 #include <linux/module.h> 18 #include <linux/slab.h> 19 #include <linux/vmalloc.h> 20 #include <linux/hyperv.h> 21 #include <linux/export.h> 22 #include <linux/io.h> 23 #include <linux/set_memory.h> 24 #include <asm/mshyperv.h> 25 26 #include "hyperv_vmbus.h" 27 28 29 struct vmbus_connection vmbus_connection = { 30 .conn_state = DISCONNECTED, 31 .unload_event = COMPLETION_INITIALIZER( 32 vmbus_connection.unload_event), 33 .next_gpadl_handle = ATOMIC_INIT(0xE1E10), 34 35 .ready_for_suspend_event = COMPLETION_INITIALIZER( 36 vmbus_connection.ready_for_suspend_event), 37 .ready_for_resume_event = COMPLETION_INITIALIZER( 38 vmbus_connection.ready_for_resume_event), 39 }; 40 EXPORT_SYMBOL_GPL(vmbus_connection); 41 42 /* 43 * Negotiated protocol version with the host. 44 */ 45 __u32 vmbus_proto_version; 46 EXPORT_SYMBOL_GPL(vmbus_proto_version); 47 48 /* 49 * Table of VMBus versions listed from newest to oldest. 50 * VERSION_WIN7 and VERSION_WS2008 are no longer supported in 51 * Linux guests and are not listed. 52 */ 53 static __u32 vmbus_versions[] = { 54 VERSION_WIN10_V5_3, 55 VERSION_WIN10_V5_2, 56 VERSION_WIN10_V5_1, 57 VERSION_WIN10_V5, 58 VERSION_WIN10_V4_1, 59 VERSION_WIN10, 60 VERSION_WIN8_1, 61 VERSION_WIN8 62 }; 63 64 /* 65 * Maximal VMBus protocol version guests can negotiate. Useful to cap the 66 * VMBus version for testing and debugging purpose. 67 */ 68 static uint max_version = VERSION_WIN10_V5_3; 69 70 module_param(max_version, uint, S_IRUGO); 71 MODULE_PARM_DESC(max_version, 72 "Maximal VMBus protocol version which can be negotiated"); 73 74 int vmbus_negotiate_version(struct vmbus_channel_msginfo *msginfo, u32 version) 75 { 76 int ret = 0; 77 struct vmbus_channel_initiate_contact *msg; 78 unsigned long flags; 79 80 init_completion(&msginfo->waitevent); 81 82 msg = (struct vmbus_channel_initiate_contact *)msginfo->msg; 83 84 memset(msg, 0, sizeof(*msg)); 85 msg->header.msgtype = CHANNELMSG_INITIATE_CONTACT; 86 msg->vmbus_version_requested = version; 87 88 /* 89 * VMBus protocol 5.0 (VERSION_WIN10_V5) and higher require that we must 90 * use VMBUS_MESSAGE_CONNECTION_ID_4 for the Initiate Contact Message, 91 * and for subsequent messages, we must use the Message Connection ID 92 * field in the host-returned Version Response Message. And, with 93 * VERSION_WIN10_V5 and higher, we don't use msg->interrupt_page, but we 94 * tell the host explicitly that we still use VMBUS_MESSAGE_SINT(2) for 95 * compatibility. 96 * 97 * On old hosts, we should always use VMBUS_MESSAGE_CONNECTION_ID (1). 98 */ 99 if (version >= VERSION_WIN10_V5) { 100 msg->msg_sint = VMBUS_MESSAGE_SINT; 101 vmbus_connection.msg_conn_id = VMBUS_MESSAGE_CONNECTION_ID_4; 102 } else { 103 msg->interrupt_page = virt_to_phys(vmbus_connection.int_page); 104 vmbus_connection.msg_conn_id = VMBUS_MESSAGE_CONNECTION_ID; 105 } 106 107 msg->monitor_page1 = vmbus_connection.monitor_pages_pa[0]; 108 msg->monitor_page2 = vmbus_connection.monitor_pages_pa[1]; 109 110 msg->target_vcpu = hv_cpu_number_to_vp_number(VMBUS_CONNECT_CPU); 111 112 /* 113 * Add to list before we send the request since we may 114 * receive the response before returning from this routine 115 */ 116 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 117 list_add_tail(&msginfo->msglistentry, 118 &vmbus_connection.chn_msg_list); 119 120 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 121 122 ret = vmbus_post_msg(msg, 123 sizeof(struct vmbus_channel_initiate_contact), 124 true); 125 126 trace_vmbus_negotiate_version(msg, ret); 127 128 if (ret != 0) { 129 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 130 list_del(&msginfo->msglistentry); 131 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, 132 flags); 133 return ret; 134 } 135 136 /* Wait for the connection response */ 137 wait_for_completion(&msginfo->waitevent); 138 139 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 140 list_del(&msginfo->msglistentry); 141 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 142 143 /* Check if successful */ 144 if (msginfo->response.version_response.version_supported) { 145 vmbus_connection.conn_state = CONNECTED; 146 147 if (version >= VERSION_WIN10_V5) 148 vmbus_connection.msg_conn_id = 149 msginfo->response.version_response.msg_conn_id; 150 } else { 151 return -ECONNREFUSED; 152 } 153 154 return ret; 155 } 156 157 /* 158 * vmbus_connect - Sends a connect request on the partition service connection 159 */ 160 int vmbus_connect(void) 161 { 162 struct vmbus_channel_msginfo *msginfo = NULL; 163 int i, ret = 0; 164 __u32 version; 165 166 /* Initialize the vmbus connection */ 167 vmbus_connection.conn_state = CONNECTING; 168 vmbus_connection.work_queue = create_workqueue("hv_vmbus_con"); 169 if (!vmbus_connection.work_queue) { 170 ret = -ENOMEM; 171 goto cleanup; 172 } 173 174 vmbus_connection.rescind_work_queue = 175 create_workqueue("hv_vmbus_rescind"); 176 if (!vmbus_connection.rescind_work_queue) { 177 ret = -ENOMEM; 178 goto cleanup; 179 } 180 vmbus_connection.ignore_any_offer_msg = false; 181 182 vmbus_connection.handle_primary_chan_wq = 183 create_workqueue("hv_pri_chan"); 184 if (!vmbus_connection.handle_primary_chan_wq) { 185 ret = -ENOMEM; 186 goto cleanup; 187 } 188 189 vmbus_connection.handle_sub_chan_wq = 190 create_workqueue("hv_sub_chan"); 191 if (!vmbus_connection.handle_sub_chan_wq) { 192 ret = -ENOMEM; 193 goto cleanup; 194 } 195 196 INIT_LIST_HEAD(&vmbus_connection.chn_msg_list); 197 spin_lock_init(&vmbus_connection.channelmsg_lock); 198 199 INIT_LIST_HEAD(&vmbus_connection.chn_list); 200 mutex_init(&vmbus_connection.channel_mutex); 201 202 /* 203 * Setup the vmbus event connection for channel interrupt 204 * abstraction stuff 205 */ 206 vmbus_connection.int_page = 207 (void *)hv_alloc_hyperv_zeroed_page(); 208 if (vmbus_connection.int_page == NULL) { 209 ret = -ENOMEM; 210 goto cleanup; 211 } 212 213 vmbus_connection.recv_int_page = vmbus_connection.int_page; 214 vmbus_connection.send_int_page = 215 (void *)((unsigned long)vmbus_connection.int_page + 216 (HV_HYP_PAGE_SIZE >> 1)); 217 218 /* 219 * Setup the monitor notification facility. The 1st page for 220 * parent->child and the 2nd page for child->parent 221 */ 222 vmbus_connection.monitor_pages[0] = (void *)hv_alloc_hyperv_zeroed_page(); 223 vmbus_connection.monitor_pages[1] = (void *)hv_alloc_hyperv_zeroed_page(); 224 if ((vmbus_connection.monitor_pages[0] == NULL) || 225 (vmbus_connection.monitor_pages[1] == NULL)) { 226 ret = -ENOMEM; 227 goto cleanup; 228 } 229 230 vmbus_connection.monitor_pages_original[0] 231 = vmbus_connection.monitor_pages[0]; 232 vmbus_connection.monitor_pages_original[1] 233 = vmbus_connection.monitor_pages[1]; 234 vmbus_connection.monitor_pages_pa[0] 235 = virt_to_phys(vmbus_connection.monitor_pages[0]); 236 vmbus_connection.monitor_pages_pa[1] 237 = virt_to_phys(vmbus_connection.monitor_pages[1]); 238 239 if (hv_is_isolation_supported()) { 240 ret = set_memory_decrypted((unsigned long) 241 vmbus_connection.monitor_pages[0], 242 1); 243 ret |= set_memory_decrypted((unsigned long) 244 vmbus_connection.monitor_pages[1], 245 1); 246 if (ret) 247 goto cleanup; 248 249 /* 250 * Isolation VM with AMD SNP needs to access monitor page via 251 * address space above shared gpa boundary. 252 */ 253 if (hv_isolation_type_snp()) { 254 vmbus_connection.monitor_pages_pa[0] += 255 ms_hyperv.shared_gpa_boundary; 256 vmbus_connection.monitor_pages_pa[1] += 257 ms_hyperv.shared_gpa_boundary; 258 259 vmbus_connection.monitor_pages[0] 260 = memremap(vmbus_connection.monitor_pages_pa[0], 261 HV_HYP_PAGE_SIZE, 262 MEMREMAP_WB); 263 if (!vmbus_connection.monitor_pages[0]) { 264 ret = -ENOMEM; 265 goto cleanup; 266 } 267 268 vmbus_connection.monitor_pages[1] 269 = memremap(vmbus_connection.monitor_pages_pa[1], 270 HV_HYP_PAGE_SIZE, 271 MEMREMAP_WB); 272 if (!vmbus_connection.monitor_pages[1]) { 273 ret = -ENOMEM; 274 goto cleanup; 275 } 276 } 277 278 /* 279 * Set memory host visibility hvcall smears memory 280 * and so zero monitor pages here. 281 */ 282 memset(vmbus_connection.monitor_pages[0], 0x00, 283 HV_HYP_PAGE_SIZE); 284 memset(vmbus_connection.monitor_pages[1], 0x00, 285 HV_HYP_PAGE_SIZE); 286 287 } 288 289 msginfo = kzalloc(sizeof(*msginfo) + 290 sizeof(struct vmbus_channel_initiate_contact), 291 GFP_KERNEL); 292 if (msginfo == NULL) { 293 ret = -ENOMEM; 294 goto cleanup; 295 } 296 297 /* 298 * Negotiate a compatible VMBUS version number with the 299 * host. We start with the highest number we can support 300 * and work our way down until we negotiate a compatible 301 * version. 302 */ 303 304 for (i = 0; ; i++) { 305 if (i == ARRAY_SIZE(vmbus_versions)) { 306 ret = -EDOM; 307 goto cleanup; 308 } 309 310 version = vmbus_versions[i]; 311 if (version > max_version) 312 continue; 313 314 ret = vmbus_negotiate_version(msginfo, version); 315 if (ret == -ETIMEDOUT) 316 goto cleanup; 317 318 if (vmbus_connection.conn_state == CONNECTED) 319 break; 320 } 321 322 if (hv_is_isolation_supported() && version < VERSION_WIN10_V5_2) { 323 pr_err("Invalid VMBus version %d.%d (expected >= %d.%d) from the host supporting isolation\n", 324 version >> 16, version & 0xFFFF, VERSION_WIN10_V5_2 >> 16, VERSION_WIN10_V5_2 & 0xFFFF); 325 ret = -EINVAL; 326 goto cleanup; 327 } 328 329 vmbus_proto_version = version; 330 pr_info("Vmbus version:%d.%d\n", 331 version >> 16, version & 0xFFFF); 332 333 vmbus_connection.channels = kcalloc(MAX_CHANNEL_RELIDS, 334 sizeof(struct vmbus_channel *), 335 GFP_KERNEL); 336 if (vmbus_connection.channels == NULL) { 337 ret = -ENOMEM; 338 goto cleanup; 339 } 340 341 kfree(msginfo); 342 return 0; 343 344 cleanup: 345 pr_err("Unable to connect to host\n"); 346 347 vmbus_connection.conn_state = DISCONNECTED; 348 vmbus_disconnect(); 349 350 kfree(msginfo); 351 352 return ret; 353 } 354 355 void vmbus_disconnect(void) 356 { 357 /* 358 * First send the unload request to the host. 359 */ 360 vmbus_initiate_unload(false); 361 362 if (vmbus_connection.handle_sub_chan_wq) 363 destroy_workqueue(vmbus_connection.handle_sub_chan_wq); 364 365 if (vmbus_connection.handle_primary_chan_wq) 366 destroy_workqueue(vmbus_connection.handle_primary_chan_wq); 367 368 if (vmbus_connection.rescind_work_queue) 369 destroy_workqueue(vmbus_connection.rescind_work_queue); 370 371 if (vmbus_connection.work_queue) 372 destroy_workqueue(vmbus_connection.work_queue); 373 374 if (vmbus_connection.int_page) { 375 hv_free_hyperv_page((unsigned long)vmbus_connection.int_page); 376 vmbus_connection.int_page = NULL; 377 } 378 379 if (hv_is_isolation_supported()) { 380 /* 381 * memunmap() checks input address is ioremap address or not 382 * inside. It doesn't unmap any thing in the non-SNP CVM and 383 * so not check CVM type here. 384 */ 385 memunmap(vmbus_connection.monitor_pages[0]); 386 memunmap(vmbus_connection.monitor_pages[1]); 387 388 set_memory_encrypted((unsigned long) 389 vmbus_connection.monitor_pages_original[0], 390 1); 391 set_memory_encrypted((unsigned long) 392 vmbus_connection.monitor_pages_original[1], 393 1); 394 } 395 396 hv_free_hyperv_page((unsigned long) 397 vmbus_connection.monitor_pages_original[0]); 398 hv_free_hyperv_page((unsigned long) 399 vmbus_connection.monitor_pages_original[1]); 400 vmbus_connection.monitor_pages_original[0] = 401 vmbus_connection.monitor_pages[0] = NULL; 402 vmbus_connection.monitor_pages_original[1] = 403 vmbus_connection.monitor_pages[1] = NULL; 404 } 405 406 /* 407 * relid2channel - Get the channel object given its 408 * child relative id (ie channel id) 409 */ 410 struct vmbus_channel *relid2channel(u32 relid) 411 { 412 if (WARN_ON(relid >= MAX_CHANNEL_RELIDS)) 413 return NULL; 414 return READ_ONCE(vmbus_connection.channels[relid]); 415 } 416 417 /* 418 * vmbus_on_event - Process a channel event notification 419 * 420 * For batched channels (default) optimize host to guest signaling 421 * by ensuring: 422 * 1. While reading the channel, we disable interrupts from host. 423 * 2. Ensure that we process all posted messages from the host 424 * before returning from this callback. 425 * 3. Once we return, enable signaling from the host. Once this 426 * state is set we check to see if additional packets are 427 * available to read. In this case we repeat the process. 428 * If this tasklet has been running for a long time 429 * then reschedule ourselves. 430 */ 431 void vmbus_on_event(unsigned long data) 432 { 433 struct vmbus_channel *channel = (void *) data; 434 unsigned long time_limit = jiffies + 2; 435 436 trace_vmbus_on_event(channel); 437 438 hv_debug_delay_test(channel, INTERRUPT_DELAY); 439 do { 440 void (*callback_fn)(void *); 441 442 /* A channel once created is persistent even when 443 * there is no driver handling the device. An 444 * unloading driver sets the onchannel_callback to NULL. 445 */ 446 callback_fn = READ_ONCE(channel->onchannel_callback); 447 if (unlikely(callback_fn == NULL)) 448 return; 449 450 (*callback_fn)(channel->channel_callback_context); 451 452 if (channel->callback_mode != HV_CALL_BATCHED) 453 return; 454 455 if (likely(hv_end_read(&channel->inbound) == 0)) 456 return; 457 458 hv_begin_read(&channel->inbound); 459 } while (likely(time_before(jiffies, time_limit))); 460 461 /* The time limit (2 jiffies) has been reached */ 462 tasklet_schedule(&channel->callback_event); 463 } 464 465 /* 466 * vmbus_post_msg - Send a msg on the vmbus's message connection 467 */ 468 int vmbus_post_msg(void *buffer, size_t buflen, bool can_sleep) 469 { 470 struct vmbus_channel_message_header *hdr; 471 union hv_connection_id conn_id; 472 int ret = 0; 473 int retries = 0; 474 u32 usec = 1; 475 476 conn_id.asu32 = 0; 477 conn_id.u.id = vmbus_connection.msg_conn_id; 478 479 /* 480 * hv_post_message() can have transient failures because of 481 * insufficient resources. Retry the operation a couple of 482 * times before giving up. 483 */ 484 while (retries < 100) { 485 ret = hv_post_message(conn_id, 1, buffer, buflen); 486 487 switch (ret) { 488 case HV_STATUS_INVALID_CONNECTION_ID: 489 /* 490 * See vmbus_negotiate_version(): VMBus protocol 5.0 491 * and higher require that we must use 492 * VMBUS_MESSAGE_CONNECTION_ID_4 for the Initiate 493 * Contact message, but on old hosts that only 494 * support VMBus protocol 4.0 or lower, here we get 495 * HV_STATUS_INVALID_CONNECTION_ID and we should 496 * return an error immediately without retrying. 497 */ 498 hdr = buffer; 499 if (hdr->msgtype == CHANNELMSG_INITIATE_CONTACT) 500 return -EINVAL; 501 /* 502 * We could get this if we send messages too 503 * frequently. 504 */ 505 ret = -EAGAIN; 506 break; 507 case HV_STATUS_INSUFFICIENT_MEMORY: 508 case HV_STATUS_INSUFFICIENT_BUFFERS: 509 ret = -ENOBUFS; 510 break; 511 case HV_STATUS_SUCCESS: 512 return ret; 513 default: 514 pr_err("hv_post_msg() failed; error code:%d\n", ret); 515 return -EINVAL; 516 } 517 518 retries++; 519 if (can_sleep && usec > 1000) 520 msleep(usec / 1000); 521 else if (usec < MAX_UDELAY_MS * 1000) 522 udelay(usec); 523 else 524 mdelay(usec / 1000); 525 526 if (retries < 22) 527 usec *= 2; 528 } 529 return ret; 530 } 531 532 /* 533 * vmbus_set_event - Send an event notification to the parent 534 */ 535 void vmbus_set_event(struct vmbus_channel *channel) 536 { 537 u32 child_relid = channel->offermsg.child_relid; 538 539 if (!channel->is_dedicated_interrupt) 540 vmbus_send_interrupt(child_relid); 541 542 ++channel->sig_events; 543 544 if (hv_isolation_type_snp()) 545 hv_ghcb_hypercall(HVCALL_SIGNAL_EVENT, &channel->sig_event, 546 NULL, sizeof(channel->sig_event)); 547 else 548 hv_do_fast_hypercall8(HVCALL_SIGNAL_EVENT, channel->sig_event); 549 } 550 EXPORT_SYMBOL_GPL(vmbus_set_event); 551