1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2012, Microsoft Corporation. 4 * 5 * Author: 6 * K. Y. Srinivasan <kys@microsoft.com> 7 */ 8 9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 10 11 #include <linux/kernel.h> 12 #include <linux/jiffies.h> 13 #include <linux/mman.h> 14 #include <linux/delay.h> 15 #include <linux/init.h> 16 #include <linux/module.h> 17 #include <linux/slab.h> 18 #include <linux/kthread.h> 19 #include <linux/completion.h> 20 #include <linux/memory_hotplug.h> 21 #include <linux/memory.h> 22 #include <linux/notifier.h> 23 #include <linux/percpu_counter.h> 24 #include <linux/page_reporting.h> 25 26 #include <linux/hyperv.h> 27 #include <asm/hyperv-tlfs.h> 28 29 #include <asm/mshyperv.h> 30 31 #define CREATE_TRACE_POINTS 32 #include "hv_trace_balloon.h" 33 34 /* 35 * We begin with definitions supporting the Dynamic Memory protocol 36 * with the host. 37 * 38 * Begin protocol definitions. 39 */ 40 41 42 43 /* 44 * Protocol versions. The low word is the minor version, the high word the major 45 * version. 46 * 47 * History: 48 * Initial version 1.0 49 * Changed to 0.1 on 2009/03/25 50 * Changes to 0.2 on 2009/05/14 51 * Changes to 0.3 on 2009/12/03 52 * Changed to 1.0 on 2011/04/05 53 */ 54 55 #define DYNMEM_MAKE_VERSION(Major, Minor) ((__u32)(((Major) << 16) | (Minor))) 56 #define DYNMEM_MAJOR_VERSION(Version) ((__u32)(Version) >> 16) 57 #define DYNMEM_MINOR_VERSION(Version) ((__u32)(Version) & 0xff) 58 59 enum { 60 DYNMEM_PROTOCOL_VERSION_1 = DYNMEM_MAKE_VERSION(0, 3), 61 DYNMEM_PROTOCOL_VERSION_2 = DYNMEM_MAKE_VERSION(1, 0), 62 DYNMEM_PROTOCOL_VERSION_3 = DYNMEM_MAKE_VERSION(2, 0), 63 64 DYNMEM_PROTOCOL_VERSION_WIN7 = DYNMEM_PROTOCOL_VERSION_1, 65 DYNMEM_PROTOCOL_VERSION_WIN8 = DYNMEM_PROTOCOL_VERSION_2, 66 DYNMEM_PROTOCOL_VERSION_WIN10 = DYNMEM_PROTOCOL_VERSION_3, 67 68 DYNMEM_PROTOCOL_VERSION_CURRENT = DYNMEM_PROTOCOL_VERSION_WIN10 69 }; 70 71 72 73 /* 74 * Message Types 75 */ 76 77 enum dm_message_type { 78 /* 79 * Version 0.3 80 */ 81 DM_ERROR = 0, 82 DM_VERSION_REQUEST = 1, 83 DM_VERSION_RESPONSE = 2, 84 DM_CAPABILITIES_REPORT = 3, 85 DM_CAPABILITIES_RESPONSE = 4, 86 DM_STATUS_REPORT = 5, 87 DM_BALLOON_REQUEST = 6, 88 DM_BALLOON_RESPONSE = 7, 89 DM_UNBALLOON_REQUEST = 8, 90 DM_UNBALLOON_RESPONSE = 9, 91 DM_MEM_HOT_ADD_REQUEST = 10, 92 DM_MEM_HOT_ADD_RESPONSE = 11, 93 DM_VERSION_03_MAX = 11, 94 /* 95 * Version 1.0. 96 */ 97 DM_INFO_MESSAGE = 12, 98 DM_VERSION_1_MAX = 12 99 }; 100 101 102 /* 103 * Structures defining the dynamic memory management 104 * protocol. 105 */ 106 107 union dm_version { 108 struct { 109 __u16 minor_version; 110 __u16 major_version; 111 }; 112 __u32 version; 113 } __packed; 114 115 116 union dm_caps { 117 struct { 118 __u64 balloon:1; 119 __u64 hot_add:1; 120 /* 121 * To support guests that may have alignment 122 * limitations on hot-add, the guest can specify 123 * its alignment requirements; a value of n 124 * represents an alignment of 2^n in mega bytes. 125 */ 126 __u64 hot_add_alignment:4; 127 __u64 reservedz:58; 128 } cap_bits; 129 __u64 caps; 130 } __packed; 131 132 union dm_mem_page_range { 133 struct { 134 /* 135 * The PFN number of the first page in the range. 136 * 40 bits is the architectural limit of a PFN 137 * number for AMD64. 138 */ 139 __u64 start_page:40; 140 /* 141 * The number of pages in the range. 142 */ 143 __u64 page_cnt:24; 144 } finfo; 145 __u64 page_range; 146 } __packed; 147 148 149 150 /* 151 * The header for all dynamic memory messages: 152 * 153 * type: Type of the message. 154 * size: Size of the message in bytes; including the header. 155 * trans_id: The guest is responsible for manufacturing this ID. 156 */ 157 158 struct dm_header { 159 __u16 type; 160 __u16 size; 161 __u32 trans_id; 162 } __packed; 163 164 /* 165 * A generic message format for dynamic memory. 166 * Specific message formats are defined later in the file. 167 */ 168 169 struct dm_message { 170 struct dm_header hdr; 171 __u8 data[]; /* enclosed message */ 172 } __packed; 173 174 175 /* 176 * Specific message types supporting the dynamic memory protocol. 177 */ 178 179 /* 180 * Version negotiation message. Sent from the guest to the host. 181 * The guest is free to try different versions until the host 182 * accepts the version. 183 * 184 * dm_version: The protocol version requested. 185 * is_last_attempt: If TRUE, this is the last version guest will request. 186 * reservedz: Reserved field, set to zero. 187 */ 188 189 struct dm_version_request { 190 struct dm_header hdr; 191 union dm_version version; 192 __u32 is_last_attempt:1; 193 __u32 reservedz:31; 194 } __packed; 195 196 /* 197 * Version response message; Host to Guest and indicates 198 * if the host has accepted the version sent by the guest. 199 * 200 * is_accepted: If TRUE, host has accepted the version and the guest 201 * should proceed to the next stage of the protocol. FALSE indicates that 202 * guest should re-try with a different version. 203 * 204 * reservedz: Reserved field, set to zero. 205 */ 206 207 struct dm_version_response { 208 struct dm_header hdr; 209 __u64 is_accepted:1; 210 __u64 reservedz:63; 211 } __packed; 212 213 /* 214 * Message reporting capabilities. This is sent from the guest to the 215 * host. 216 */ 217 218 struct dm_capabilities { 219 struct dm_header hdr; 220 union dm_caps caps; 221 __u64 min_page_cnt; 222 __u64 max_page_number; 223 } __packed; 224 225 /* 226 * Response to the capabilities message. This is sent from the host to the 227 * guest. This message notifies if the host has accepted the guest's 228 * capabilities. If the host has not accepted, the guest must shutdown 229 * the service. 230 * 231 * is_accepted: Indicates if the host has accepted guest's capabilities. 232 * reservedz: Must be 0. 233 */ 234 235 struct dm_capabilities_resp_msg { 236 struct dm_header hdr; 237 __u64 is_accepted:1; 238 __u64 reservedz:63; 239 } __packed; 240 241 /* 242 * This message is used to report memory pressure from the guest. 243 * This message is not part of any transaction and there is no 244 * response to this message. 245 * 246 * num_avail: Available memory in pages. 247 * num_committed: Committed memory in pages. 248 * page_file_size: The accumulated size of all page files 249 * in the system in pages. 250 * zero_free: The nunber of zero and free pages. 251 * page_file_writes: The writes to the page file in pages. 252 * io_diff: An indicator of file cache efficiency or page file activity, 253 * calculated as File Cache Page Fault Count - Page Read Count. 254 * This value is in pages. 255 * 256 * Some of these metrics are Windows specific and fortunately 257 * the algorithm on the host side that computes the guest memory 258 * pressure only uses num_committed value. 259 */ 260 261 struct dm_status { 262 struct dm_header hdr; 263 __u64 num_avail; 264 __u64 num_committed; 265 __u64 page_file_size; 266 __u64 zero_free; 267 __u32 page_file_writes; 268 __u32 io_diff; 269 } __packed; 270 271 272 /* 273 * Message to ask the guest to allocate memory - balloon up message. 274 * This message is sent from the host to the guest. The guest may not be 275 * able to allocate as much memory as requested. 276 * 277 * num_pages: number of pages to allocate. 278 */ 279 280 struct dm_balloon { 281 struct dm_header hdr; 282 __u32 num_pages; 283 __u32 reservedz; 284 } __packed; 285 286 287 /* 288 * Balloon response message; this message is sent from the guest 289 * to the host in response to the balloon message. 290 * 291 * reservedz: Reserved; must be set to zero. 292 * more_pages: If FALSE, this is the last message of the transaction. 293 * if TRUE there will atleast one more message from the guest. 294 * 295 * range_count: The number of ranges in the range array. 296 * 297 * range_array: An array of page ranges returned to the host. 298 * 299 */ 300 301 struct dm_balloon_response { 302 struct dm_header hdr; 303 __u32 reservedz; 304 __u32 more_pages:1; 305 __u32 range_count:31; 306 union dm_mem_page_range range_array[]; 307 } __packed; 308 309 /* 310 * Un-balloon message; this message is sent from the host 311 * to the guest to give guest more memory. 312 * 313 * more_pages: If FALSE, this is the last message of the transaction. 314 * if TRUE there will atleast one more message from the guest. 315 * 316 * reservedz: Reserved; must be set to zero. 317 * 318 * range_count: The number of ranges in the range array. 319 * 320 * range_array: An array of page ranges returned to the host. 321 * 322 */ 323 324 struct dm_unballoon_request { 325 struct dm_header hdr; 326 __u32 more_pages:1; 327 __u32 reservedz:31; 328 __u32 range_count; 329 union dm_mem_page_range range_array[]; 330 } __packed; 331 332 /* 333 * Un-balloon response message; this message is sent from the guest 334 * to the host in response to an unballoon request. 335 * 336 */ 337 338 struct dm_unballoon_response { 339 struct dm_header hdr; 340 } __packed; 341 342 343 /* 344 * Hot add request message. Message sent from the host to the guest. 345 * 346 * mem_range: Memory range to hot add. 347 * 348 */ 349 350 struct dm_hot_add { 351 struct dm_header hdr; 352 union dm_mem_page_range range; 353 } __packed; 354 355 /* 356 * Hot add response message. 357 * This message is sent by the guest to report the status of a hot add request. 358 * If page_count is less than the requested page count, then the host should 359 * assume all further hot add requests will fail, since this indicates that 360 * the guest has hit an upper physical memory barrier. 361 * 362 * Hot adds may also fail due to low resources; in this case, the guest must 363 * not complete this message until the hot add can succeed, and the host must 364 * not send a new hot add request until the response is sent. 365 * If VSC fails to hot add memory DYNMEM_NUMBER_OF_UNSUCCESSFUL_HOTADD_ATTEMPTS 366 * times it fails the request. 367 * 368 * 369 * page_count: number of pages that were successfully hot added. 370 * 371 * result: result of the operation 1: success, 0: failure. 372 * 373 */ 374 375 struct dm_hot_add_response { 376 struct dm_header hdr; 377 __u32 page_count; 378 __u32 result; 379 } __packed; 380 381 /* 382 * Types of information sent from host to the guest. 383 */ 384 385 enum dm_info_type { 386 INFO_TYPE_MAX_PAGE_CNT = 0, 387 MAX_INFO_TYPE 388 }; 389 390 391 /* 392 * Header for the information message. 393 */ 394 395 struct dm_info_header { 396 enum dm_info_type type; 397 __u32 data_size; 398 } __packed; 399 400 /* 401 * This message is sent from the host to the guest to pass 402 * some relevant information (win8 addition). 403 * 404 * reserved: no used. 405 * info_size: size of the information blob. 406 * info: information blob. 407 */ 408 409 struct dm_info_msg { 410 struct dm_header hdr; 411 __u32 reserved; 412 __u32 info_size; 413 __u8 info[]; 414 }; 415 416 /* 417 * End protocol definitions. 418 */ 419 420 /* 421 * State to manage hot adding memory into the guest. 422 * The range start_pfn : end_pfn specifies the range 423 * that the host has asked us to hot add. The range 424 * start_pfn : ha_end_pfn specifies the range that we have 425 * currently hot added. We hot add in multiples of 128M 426 * chunks; it is possible that we may not be able to bring 427 * online all the pages in the region. The range 428 * covered_start_pfn:covered_end_pfn defines the pages that can 429 * be brough online. 430 */ 431 432 struct hv_hotadd_state { 433 struct list_head list; 434 unsigned long start_pfn; 435 unsigned long covered_start_pfn; 436 unsigned long covered_end_pfn; 437 unsigned long ha_end_pfn; 438 unsigned long end_pfn; 439 /* 440 * A list of gaps. 441 */ 442 struct list_head gap_list; 443 }; 444 445 struct hv_hotadd_gap { 446 struct list_head list; 447 unsigned long start_pfn; 448 unsigned long end_pfn; 449 }; 450 451 struct balloon_state { 452 __u32 num_pages; 453 struct work_struct wrk; 454 }; 455 456 struct hot_add_wrk { 457 union dm_mem_page_range ha_page_range; 458 union dm_mem_page_range ha_region_range; 459 struct work_struct wrk; 460 }; 461 462 static bool allow_hibernation; 463 static bool hot_add = true; 464 static bool do_hot_add; 465 /* 466 * Delay reporting memory pressure by 467 * the specified number of seconds. 468 */ 469 static uint pressure_report_delay = 45; 470 471 /* 472 * The last time we posted a pressure report to host. 473 */ 474 static unsigned long last_post_time; 475 476 module_param(hot_add, bool, (S_IRUGO | S_IWUSR)); 477 MODULE_PARM_DESC(hot_add, "If set attempt memory hot_add"); 478 479 module_param(pressure_report_delay, uint, (S_IRUGO | S_IWUSR)); 480 MODULE_PARM_DESC(pressure_report_delay, "Delay in secs in reporting pressure"); 481 static atomic_t trans_id = ATOMIC_INIT(0); 482 483 static int dm_ring_size = 20 * 1024; 484 485 /* 486 * Driver specific state. 487 */ 488 489 enum hv_dm_state { 490 DM_INITIALIZING = 0, 491 DM_INITIALIZED, 492 DM_BALLOON_UP, 493 DM_BALLOON_DOWN, 494 DM_HOT_ADD, 495 DM_INIT_ERROR 496 }; 497 498 499 static __u8 recv_buffer[HV_HYP_PAGE_SIZE]; 500 static __u8 balloon_up_send_buffer[HV_HYP_PAGE_SIZE]; 501 #define PAGES_IN_2M (2 * 1024 * 1024 / PAGE_SIZE) 502 #define HA_CHUNK (128 * 1024 * 1024 / PAGE_SIZE) 503 504 struct hv_dynmem_device { 505 struct hv_device *dev; 506 enum hv_dm_state state; 507 struct completion host_event; 508 struct completion config_event; 509 510 /* 511 * Number of pages we have currently ballooned out. 512 */ 513 unsigned int num_pages_ballooned; 514 unsigned int num_pages_onlined; 515 unsigned int num_pages_added; 516 517 /* 518 * State to manage the ballooning (up) operation. 519 */ 520 struct balloon_state balloon_wrk; 521 522 /* 523 * State to execute the "hot-add" operation. 524 */ 525 struct hot_add_wrk ha_wrk; 526 527 /* 528 * This state tracks if the host has specified a hot-add 529 * region. 530 */ 531 bool host_specified_ha_region; 532 533 /* 534 * State to synchronize hot-add. 535 */ 536 struct completion ol_waitevent; 537 /* 538 * This thread handles hot-add 539 * requests from the host as well as notifying 540 * the host with regards to memory pressure in 541 * the guest. 542 */ 543 struct task_struct *thread; 544 545 /* 546 * Protects ha_region_list, num_pages_onlined counter and individual 547 * regions from ha_region_list. 548 */ 549 spinlock_t ha_lock; 550 551 /* 552 * A list of hot-add regions. 553 */ 554 struct list_head ha_region_list; 555 556 /* 557 * We start with the highest version we can support 558 * and downgrade based on the host; we save here the 559 * next version to try. 560 */ 561 __u32 next_version; 562 563 /* 564 * The negotiated version agreed by host. 565 */ 566 __u32 version; 567 568 struct page_reporting_dev_info pr_dev_info; 569 }; 570 571 static struct hv_dynmem_device dm_device; 572 573 static void post_status(struct hv_dynmem_device *dm); 574 575 #ifdef CONFIG_MEMORY_HOTPLUG 576 static inline bool has_pfn_is_backed(struct hv_hotadd_state *has, 577 unsigned long pfn) 578 { 579 struct hv_hotadd_gap *gap; 580 581 /* The page is not backed. */ 582 if ((pfn < has->covered_start_pfn) || (pfn >= has->covered_end_pfn)) 583 return false; 584 585 /* Check for gaps. */ 586 list_for_each_entry(gap, &has->gap_list, list) { 587 if ((pfn >= gap->start_pfn) && (pfn < gap->end_pfn)) 588 return false; 589 } 590 591 return true; 592 } 593 594 static unsigned long hv_page_offline_check(unsigned long start_pfn, 595 unsigned long nr_pages) 596 { 597 unsigned long pfn = start_pfn, count = 0; 598 struct hv_hotadd_state *has; 599 bool found; 600 601 while (pfn < start_pfn + nr_pages) { 602 /* 603 * Search for HAS which covers the pfn and when we find one 604 * count how many consequitive PFNs are covered. 605 */ 606 found = false; 607 list_for_each_entry(has, &dm_device.ha_region_list, list) { 608 while ((pfn >= has->start_pfn) && 609 (pfn < has->end_pfn) && 610 (pfn < start_pfn + nr_pages)) { 611 found = true; 612 if (has_pfn_is_backed(has, pfn)) 613 count++; 614 pfn++; 615 } 616 } 617 618 /* 619 * This PFN is not in any HAS (e.g. we're offlining a region 620 * which was present at boot), no need to account for it. Go 621 * to the next one. 622 */ 623 if (!found) 624 pfn++; 625 } 626 627 return count; 628 } 629 630 static int hv_memory_notifier(struct notifier_block *nb, unsigned long val, 631 void *v) 632 { 633 struct memory_notify *mem = (struct memory_notify *)v; 634 unsigned long flags, pfn_count; 635 636 switch (val) { 637 case MEM_ONLINE: 638 case MEM_CANCEL_ONLINE: 639 complete(&dm_device.ol_waitevent); 640 break; 641 642 case MEM_OFFLINE: 643 spin_lock_irqsave(&dm_device.ha_lock, flags); 644 pfn_count = hv_page_offline_check(mem->start_pfn, 645 mem->nr_pages); 646 if (pfn_count <= dm_device.num_pages_onlined) { 647 dm_device.num_pages_onlined -= pfn_count; 648 } else { 649 /* 650 * We're offlining more pages than we managed to online. 651 * This is unexpected. In any case don't let 652 * num_pages_onlined wrap around zero. 653 */ 654 WARN_ON_ONCE(1); 655 dm_device.num_pages_onlined = 0; 656 } 657 spin_unlock_irqrestore(&dm_device.ha_lock, flags); 658 break; 659 case MEM_GOING_ONLINE: 660 case MEM_GOING_OFFLINE: 661 case MEM_CANCEL_OFFLINE: 662 break; 663 } 664 return NOTIFY_OK; 665 } 666 667 static struct notifier_block hv_memory_nb = { 668 .notifier_call = hv_memory_notifier, 669 .priority = 0 670 }; 671 672 /* Check if the particular page is backed and can be onlined and online it. */ 673 static void hv_page_online_one(struct hv_hotadd_state *has, struct page *pg) 674 { 675 if (!has_pfn_is_backed(has, page_to_pfn(pg))) { 676 if (!PageOffline(pg)) 677 __SetPageOffline(pg); 678 return; 679 } 680 if (PageOffline(pg)) 681 __ClearPageOffline(pg); 682 683 /* This frame is currently backed; online the page. */ 684 generic_online_page(pg, 0); 685 686 lockdep_assert_held(&dm_device.ha_lock); 687 dm_device.num_pages_onlined++; 688 } 689 690 static void hv_bring_pgs_online(struct hv_hotadd_state *has, 691 unsigned long start_pfn, unsigned long size) 692 { 693 int i; 694 695 pr_debug("Online %lu pages starting at pfn 0x%lx\n", size, start_pfn); 696 for (i = 0; i < size; i++) 697 hv_page_online_one(has, pfn_to_page(start_pfn + i)); 698 } 699 700 static void hv_mem_hot_add(unsigned long start, unsigned long size, 701 unsigned long pfn_count, 702 struct hv_hotadd_state *has) 703 { 704 int ret = 0; 705 int i, nid; 706 unsigned long start_pfn; 707 unsigned long processed_pfn; 708 unsigned long total_pfn = pfn_count; 709 unsigned long flags; 710 711 for (i = 0; i < (size/HA_CHUNK); i++) { 712 start_pfn = start + (i * HA_CHUNK); 713 714 spin_lock_irqsave(&dm_device.ha_lock, flags); 715 has->ha_end_pfn += HA_CHUNK; 716 717 if (total_pfn > HA_CHUNK) { 718 processed_pfn = HA_CHUNK; 719 total_pfn -= HA_CHUNK; 720 } else { 721 processed_pfn = total_pfn; 722 total_pfn = 0; 723 } 724 725 has->covered_end_pfn += processed_pfn; 726 spin_unlock_irqrestore(&dm_device.ha_lock, flags); 727 728 reinit_completion(&dm_device.ol_waitevent); 729 730 nid = memory_add_physaddr_to_nid(PFN_PHYS(start_pfn)); 731 ret = add_memory(nid, PFN_PHYS((start_pfn)), 732 (HA_CHUNK << PAGE_SHIFT), MHP_MERGE_RESOURCE); 733 734 if (ret) { 735 pr_err("hot_add memory failed error is %d\n", ret); 736 if (ret == -EEXIST) { 737 /* 738 * This error indicates that the error 739 * is not a transient failure. This is the 740 * case where the guest's physical address map 741 * precludes hot adding memory. Stop all further 742 * memory hot-add. 743 */ 744 do_hot_add = false; 745 } 746 spin_lock_irqsave(&dm_device.ha_lock, flags); 747 has->ha_end_pfn -= HA_CHUNK; 748 has->covered_end_pfn -= processed_pfn; 749 spin_unlock_irqrestore(&dm_device.ha_lock, flags); 750 break; 751 } 752 753 /* 754 * Wait for memory to get onlined. If the kernel onlined the 755 * memory when adding it, this will return directly. Otherwise, 756 * it will wait for user space to online the memory. This helps 757 * to avoid adding memory faster than it is getting onlined. As 758 * adding succeeded, it is ok to proceed even if the memory was 759 * not onlined in time. 760 */ 761 wait_for_completion_timeout(&dm_device.ol_waitevent, 5 * HZ); 762 post_status(&dm_device); 763 } 764 } 765 766 static void hv_online_page(struct page *pg, unsigned int order) 767 { 768 struct hv_hotadd_state *has; 769 unsigned long flags; 770 unsigned long pfn = page_to_pfn(pg); 771 772 spin_lock_irqsave(&dm_device.ha_lock, flags); 773 list_for_each_entry(has, &dm_device.ha_region_list, list) { 774 /* The page belongs to a different HAS. */ 775 if ((pfn < has->start_pfn) || 776 (pfn + (1UL << order) > has->end_pfn)) 777 continue; 778 779 hv_bring_pgs_online(has, pfn, 1UL << order); 780 break; 781 } 782 spin_unlock_irqrestore(&dm_device.ha_lock, flags); 783 } 784 785 static int pfn_covered(unsigned long start_pfn, unsigned long pfn_cnt) 786 { 787 struct hv_hotadd_state *has; 788 struct hv_hotadd_gap *gap; 789 unsigned long residual, new_inc; 790 int ret = 0; 791 unsigned long flags; 792 793 spin_lock_irqsave(&dm_device.ha_lock, flags); 794 list_for_each_entry(has, &dm_device.ha_region_list, list) { 795 /* 796 * If the pfn range we are dealing with is not in the current 797 * "hot add block", move on. 798 */ 799 if (start_pfn < has->start_pfn || start_pfn >= has->end_pfn) 800 continue; 801 802 /* 803 * If the current start pfn is not where the covered_end 804 * is, create a gap and update covered_end_pfn. 805 */ 806 if (has->covered_end_pfn != start_pfn) { 807 gap = kzalloc(sizeof(struct hv_hotadd_gap), GFP_ATOMIC); 808 if (!gap) { 809 ret = -ENOMEM; 810 break; 811 } 812 813 INIT_LIST_HEAD(&gap->list); 814 gap->start_pfn = has->covered_end_pfn; 815 gap->end_pfn = start_pfn; 816 list_add_tail(&gap->list, &has->gap_list); 817 818 has->covered_end_pfn = start_pfn; 819 } 820 821 /* 822 * If the current hot add-request extends beyond 823 * our current limit; extend it. 824 */ 825 if ((start_pfn + pfn_cnt) > has->end_pfn) { 826 residual = (start_pfn + pfn_cnt - has->end_pfn); 827 /* 828 * Extend the region by multiples of HA_CHUNK. 829 */ 830 new_inc = (residual / HA_CHUNK) * HA_CHUNK; 831 if (residual % HA_CHUNK) 832 new_inc += HA_CHUNK; 833 834 has->end_pfn += new_inc; 835 } 836 837 ret = 1; 838 break; 839 } 840 spin_unlock_irqrestore(&dm_device.ha_lock, flags); 841 842 return ret; 843 } 844 845 static unsigned long handle_pg_range(unsigned long pg_start, 846 unsigned long pg_count) 847 { 848 unsigned long start_pfn = pg_start; 849 unsigned long pfn_cnt = pg_count; 850 unsigned long size; 851 struct hv_hotadd_state *has; 852 unsigned long pgs_ol = 0; 853 unsigned long old_covered_state; 854 unsigned long res = 0, flags; 855 856 pr_debug("Hot adding %lu pages starting at pfn 0x%lx.\n", pg_count, 857 pg_start); 858 859 spin_lock_irqsave(&dm_device.ha_lock, flags); 860 list_for_each_entry(has, &dm_device.ha_region_list, list) { 861 /* 862 * If the pfn range we are dealing with is not in the current 863 * "hot add block", move on. 864 */ 865 if (start_pfn < has->start_pfn || start_pfn >= has->end_pfn) 866 continue; 867 868 old_covered_state = has->covered_end_pfn; 869 870 if (start_pfn < has->ha_end_pfn) { 871 /* 872 * This is the case where we are backing pages 873 * in an already hot added region. Bring 874 * these pages online first. 875 */ 876 pgs_ol = has->ha_end_pfn - start_pfn; 877 if (pgs_ol > pfn_cnt) 878 pgs_ol = pfn_cnt; 879 880 has->covered_end_pfn += pgs_ol; 881 pfn_cnt -= pgs_ol; 882 /* 883 * Check if the corresponding memory block is already 884 * online. It is possible to observe struct pages still 885 * being uninitialized here so check section instead. 886 * In case the section is online we need to bring the 887 * rest of pfns (which were not backed previously) 888 * online too. 889 */ 890 if (start_pfn > has->start_pfn && 891 online_section_nr(pfn_to_section_nr(start_pfn))) 892 hv_bring_pgs_online(has, start_pfn, pgs_ol); 893 894 } 895 896 if ((has->ha_end_pfn < has->end_pfn) && (pfn_cnt > 0)) { 897 /* 898 * We have some residual hot add range 899 * that needs to be hot added; hot add 900 * it now. Hot add a multiple of 901 * of HA_CHUNK that fully covers the pages 902 * we have. 903 */ 904 size = (has->end_pfn - has->ha_end_pfn); 905 if (pfn_cnt <= size) { 906 size = ((pfn_cnt / HA_CHUNK) * HA_CHUNK); 907 if (pfn_cnt % HA_CHUNK) 908 size += HA_CHUNK; 909 } else { 910 pfn_cnt = size; 911 } 912 spin_unlock_irqrestore(&dm_device.ha_lock, flags); 913 hv_mem_hot_add(has->ha_end_pfn, size, pfn_cnt, has); 914 spin_lock_irqsave(&dm_device.ha_lock, flags); 915 } 916 /* 917 * If we managed to online any pages that were given to us, 918 * we declare success. 919 */ 920 res = has->covered_end_pfn - old_covered_state; 921 break; 922 } 923 spin_unlock_irqrestore(&dm_device.ha_lock, flags); 924 925 return res; 926 } 927 928 static unsigned long process_hot_add(unsigned long pg_start, 929 unsigned long pfn_cnt, 930 unsigned long rg_start, 931 unsigned long rg_size) 932 { 933 struct hv_hotadd_state *ha_region = NULL; 934 int covered; 935 unsigned long flags; 936 937 if (pfn_cnt == 0) 938 return 0; 939 940 if (!dm_device.host_specified_ha_region) { 941 covered = pfn_covered(pg_start, pfn_cnt); 942 if (covered < 0) 943 return 0; 944 945 if (covered) 946 goto do_pg_range; 947 } 948 949 /* 950 * If the host has specified a hot-add range; deal with it first. 951 */ 952 953 if (rg_size != 0) { 954 ha_region = kzalloc(sizeof(struct hv_hotadd_state), GFP_KERNEL); 955 if (!ha_region) 956 return 0; 957 958 INIT_LIST_HEAD(&ha_region->list); 959 INIT_LIST_HEAD(&ha_region->gap_list); 960 961 ha_region->start_pfn = rg_start; 962 ha_region->ha_end_pfn = rg_start; 963 ha_region->covered_start_pfn = pg_start; 964 ha_region->covered_end_pfn = pg_start; 965 ha_region->end_pfn = rg_start + rg_size; 966 967 spin_lock_irqsave(&dm_device.ha_lock, flags); 968 list_add_tail(&ha_region->list, &dm_device.ha_region_list); 969 spin_unlock_irqrestore(&dm_device.ha_lock, flags); 970 } 971 972 do_pg_range: 973 /* 974 * Process the page range specified; bringing them 975 * online if possible. 976 */ 977 return handle_pg_range(pg_start, pfn_cnt); 978 } 979 980 #endif 981 982 static void hot_add_req(struct work_struct *dummy) 983 { 984 struct dm_hot_add_response resp; 985 #ifdef CONFIG_MEMORY_HOTPLUG 986 unsigned long pg_start, pfn_cnt; 987 unsigned long rg_start, rg_sz; 988 #endif 989 struct hv_dynmem_device *dm = &dm_device; 990 991 memset(&resp, 0, sizeof(struct dm_hot_add_response)); 992 resp.hdr.type = DM_MEM_HOT_ADD_RESPONSE; 993 resp.hdr.size = sizeof(struct dm_hot_add_response); 994 995 #ifdef CONFIG_MEMORY_HOTPLUG 996 pg_start = dm->ha_wrk.ha_page_range.finfo.start_page; 997 pfn_cnt = dm->ha_wrk.ha_page_range.finfo.page_cnt; 998 999 rg_start = dm->ha_wrk.ha_region_range.finfo.start_page; 1000 rg_sz = dm->ha_wrk.ha_region_range.finfo.page_cnt; 1001 1002 if ((rg_start == 0) && (!dm->host_specified_ha_region)) { 1003 unsigned long region_size; 1004 unsigned long region_start; 1005 1006 /* 1007 * The host has not specified the hot-add region. 1008 * Based on the hot-add page range being specified, 1009 * compute a hot-add region that can cover the pages 1010 * that need to be hot-added while ensuring the alignment 1011 * and size requirements of Linux as it relates to hot-add. 1012 */ 1013 region_start = pg_start; 1014 region_size = (pfn_cnt / HA_CHUNK) * HA_CHUNK; 1015 if (pfn_cnt % HA_CHUNK) 1016 region_size += HA_CHUNK; 1017 1018 region_start = (pg_start / HA_CHUNK) * HA_CHUNK; 1019 1020 rg_start = region_start; 1021 rg_sz = region_size; 1022 } 1023 1024 if (do_hot_add) 1025 resp.page_count = process_hot_add(pg_start, pfn_cnt, 1026 rg_start, rg_sz); 1027 1028 dm->num_pages_added += resp.page_count; 1029 #endif 1030 /* 1031 * The result field of the response structure has the 1032 * following semantics: 1033 * 1034 * 1. If all or some pages hot-added: Guest should return success. 1035 * 1036 * 2. If no pages could be hot-added: 1037 * 1038 * If the guest returns success, then the host 1039 * will not attempt any further hot-add operations. This 1040 * signifies a permanent failure. 1041 * 1042 * If the guest returns failure, then this failure will be 1043 * treated as a transient failure and the host may retry the 1044 * hot-add operation after some delay. 1045 */ 1046 if (resp.page_count > 0) 1047 resp.result = 1; 1048 else if (!do_hot_add) 1049 resp.result = 1; 1050 else 1051 resp.result = 0; 1052 1053 if (!do_hot_add || resp.page_count == 0) { 1054 if (!allow_hibernation) 1055 pr_err("Memory hot add failed\n"); 1056 else 1057 pr_info("Ignore hot-add request!\n"); 1058 } 1059 1060 dm->state = DM_INITIALIZED; 1061 resp.hdr.trans_id = atomic_inc_return(&trans_id); 1062 vmbus_sendpacket(dm->dev->channel, &resp, 1063 sizeof(struct dm_hot_add_response), 1064 (unsigned long)NULL, 1065 VM_PKT_DATA_INBAND, 0); 1066 } 1067 1068 static void process_info(struct hv_dynmem_device *dm, struct dm_info_msg *msg) 1069 { 1070 struct dm_info_header *info_hdr; 1071 1072 info_hdr = (struct dm_info_header *)msg->info; 1073 1074 switch (info_hdr->type) { 1075 case INFO_TYPE_MAX_PAGE_CNT: 1076 if (info_hdr->data_size == sizeof(__u64)) { 1077 __u64 *max_page_count = (__u64 *)&info_hdr[1]; 1078 1079 pr_info("Max. dynamic memory size: %llu MB\n", 1080 (*max_page_count) >> (20 - HV_HYP_PAGE_SHIFT)); 1081 } 1082 1083 break; 1084 default: 1085 pr_warn("Received Unknown type: %d\n", info_hdr->type); 1086 } 1087 } 1088 1089 static unsigned long compute_balloon_floor(void) 1090 { 1091 unsigned long min_pages; 1092 unsigned long nr_pages = totalram_pages(); 1093 #define MB2PAGES(mb) ((mb) << (20 - PAGE_SHIFT)) 1094 /* Simple continuous piecewiese linear function: 1095 * max MiB -> min MiB gradient 1096 * 0 0 1097 * 16 16 1098 * 32 24 1099 * 128 72 (1/2) 1100 * 512 168 (1/4) 1101 * 2048 360 (1/8) 1102 * 8192 744 (1/16) 1103 * 32768 1512 (1/32) 1104 */ 1105 if (nr_pages < MB2PAGES(128)) 1106 min_pages = MB2PAGES(8) + (nr_pages >> 1); 1107 else if (nr_pages < MB2PAGES(512)) 1108 min_pages = MB2PAGES(40) + (nr_pages >> 2); 1109 else if (nr_pages < MB2PAGES(2048)) 1110 min_pages = MB2PAGES(104) + (nr_pages >> 3); 1111 else if (nr_pages < MB2PAGES(8192)) 1112 min_pages = MB2PAGES(232) + (nr_pages >> 4); 1113 else 1114 min_pages = MB2PAGES(488) + (nr_pages >> 5); 1115 #undef MB2PAGES 1116 return min_pages; 1117 } 1118 1119 /* 1120 * Post our status as it relates memory pressure to the 1121 * host. Host expects the guests to post this status 1122 * periodically at 1 second intervals. 1123 * 1124 * The metrics specified in this protocol are very Windows 1125 * specific and so we cook up numbers here to convey our memory 1126 * pressure. 1127 */ 1128 1129 static void post_status(struct hv_dynmem_device *dm) 1130 { 1131 struct dm_status status; 1132 unsigned long now = jiffies; 1133 unsigned long last_post = last_post_time; 1134 1135 if (pressure_report_delay > 0) { 1136 --pressure_report_delay; 1137 return; 1138 } 1139 1140 if (!time_after(now, (last_post_time + HZ))) 1141 return; 1142 1143 memset(&status, 0, sizeof(struct dm_status)); 1144 status.hdr.type = DM_STATUS_REPORT; 1145 status.hdr.size = sizeof(struct dm_status); 1146 status.hdr.trans_id = atomic_inc_return(&trans_id); 1147 1148 /* 1149 * The host expects the guest to report free and committed memory. 1150 * Furthermore, the host expects the pressure information to include 1151 * the ballooned out pages. For a given amount of memory that we are 1152 * managing we need to compute a floor below which we should not 1153 * balloon. Compute this and add it to the pressure report. 1154 * We also need to report all offline pages (num_pages_added - 1155 * num_pages_onlined) as committed to the host, otherwise it can try 1156 * asking us to balloon them out. 1157 */ 1158 status.num_avail = si_mem_available(); 1159 status.num_committed = vm_memory_committed() + 1160 dm->num_pages_ballooned + 1161 (dm->num_pages_added > dm->num_pages_onlined ? 1162 dm->num_pages_added - dm->num_pages_onlined : 0) + 1163 compute_balloon_floor(); 1164 1165 trace_balloon_status(status.num_avail, status.num_committed, 1166 vm_memory_committed(), dm->num_pages_ballooned, 1167 dm->num_pages_added, dm->num_pages_onlined); 1168 /* 1169 * If our transaction ID is no longer current, just don't 1170 * send the status. This can happen if we were interrupted 1171 * after we picked our transaction ID. 1172 */ 1173 if (status.hdr.trans_id != atomic_read(&trans_id)) 1174 return; 1175 1176 /* 1177 * If the last post time that we sampled has changed, 1178 * we have raced, don't post the status. 1179 */ 1180 if (last_post != last_post_time) 1181 return; 1182 1183 last_post_time = jiffies; 1184 vmbus_sendpacket(dm->dev->channel, &status, 1185 sizeof(struct dm_status), 1186 (unsigned long)NULL, 1187 VM_PKT_DATA_INBAND, 0); 1188 1189 } 1190 1191 static void free_balloon_pages(struct hv_dynmem_device *dm, 1192 union dm_mem_page_range *range_array) 1193 { 1194 int num_pages = range_array->finfo.page_cnt; 1195 __u64 start_frame = range_array->finfo.start_page; 1196 struct page *pg; 1197 int i; 1198 1199 for (i = 0; i < num_pages; i++) { 1200 pg = pfn_to_page(i + start_frame); 1201 __ClearPageOffline(pg); 1202 __free_page(pg); 1203 dm->num_pages_ballooned--; 1204 adjust_managed_page_count(pg, 1); 1205 } 1206 } 1207 1208 1209 1210 static unsigned int alloc_balloon_pages(struct hv_dynmem_device *dm, 1211 unsigned int num_pages, 1212 struct dm_balloon_response *bl_resp, 1213 int alloc_unit) 1214 { 1215 unsigned int i, j; 1216 struct page *pg; 1217 1218 for (i = 0; i < num_pages / alloc_unit; i++) { 1219 if (bl_resp->hdr.size + sizeof(union dm_mem_page_range) > 1220 HV_HYP_PAGE_SIZE) 1221 return i * alloc_unit; 1222 1223 /* 1224 * We execute this code in a thread context. Furthermore, 1225 * we don't want the kernel to try too hard. 1226 */ 1227 pg = alloc_pages(GFP_HIGHUSER | __GFP_NORETRY | 1228 __GFP_NOMEMALLOC | __GFP_NOWARN, 1229 get_order(alloc_unit << PAGE_SHIFT)); 1230 1231 if (!pg) 1232 return i * alloc_unit; 1233 1234 dm->num_pages_ballooned += alloc_unit; 1235 1236 /* 1237 * If we allocatted 2M pages; split them so we 1238 * can free them in any order we get. 1239 */ 1240 1241 if (alloc_unit != 1) 1242 split_page(pg, get_order(alloc_unit << PAGE_SHIFT)); 1243 1244 /* mark all pages offline */ 1245 for (j = 0; j < alloc_unit; j++) { 1246 __SetPageOffline(pg + j); 1247 adjust_managed_page_count(pg + j, -1); 1248 } 1249 1250 bl_resp->range_count++; 1251 bl_resp->range_array[i].finfo.start_page = 1252 page_to_pfn(pg); 1253 bl_resp->range_array[i].finfo.page_cnt = alloc_unit; 1254 bl_resp->hdr.size += sizeof(union dm_mem_page_range); 1255 1256 } 1257 1258 return i * alloc_unit; 1259 } 1260 1261 static void balloon_up(struct work_struct *dummy) 1262 { 1263 unsigned int num_pages = dm_device.balloon_wrk.num_pages; 1264 unsigned int num_ballooned = 0; 1265 struct dm_balloon_response *bl_resp; 1266 int alloc_unit; 1267 int ret; 1268 bool done = false; 1269 int i; 1270 long avail_pages; 1271 unsigned long floor; 1272 1273 /* 1274 * We will attempt 2M allocations. However, if we fail to 1275 * allocate 2M chunks, we will go back to PAGE_SIZE allocations. 1276 */ 1277 alloc_unit = PAGES_IN_2M; 1278 1279 avail_pages = si_mem_available(); 1280 floor = compute_balloon_floor(); 1281 1282 /* Refuse to balloon below the floor. */ 1283 if (avail_pages < num_pages || avail_pages - num_pages < floor) { 1284 pr_info("Balloon request will be partially fulfilled. %s\n", 1285 avail_pages < num_pages ? "Not enough memory." : 1286 "Balloon floor reached."); 1287 1288 num_pages = avail_pages > floor ? (avail_pages - floor) : 0; 1289 } 1290 1291 while (!done) { 1292 memset(balloon_up_send_buffer, 0, HV_HYP_PAGE_SIZE); 1293 bl_resp = (struct dm_balloon_response *)balloon_up_send_buffer; 1294 bl_resp->hdr.type = DM_BALLOON_RESPONSE; 1295 bl_resp->hdr.size = sizeof(struct dm_balloon_response); 1296 bl_resp->more_pages = 1; 1297 1298 num_pages -= num_ballooned; 1299 num_ballooned = alloc_balloon_pages(&dm_device, num_pages, 1300 bl_resp, alloc_unit); 1301 1302 if (alloc_unit != 1 && num_ballooned == 0) { 1303 alloc_unit = 1; 1304 continue; 1305 } 1306 1307 if (num_ballooned == 0 || num_ballooned == num_pages) { 1308 pr_debug("Ballooned %u out of %u requested pages.\n", 1309 num_pages, dm_device.balloon_wrk.num_pages); 1310 1311 bl_resp->more_pages = 0; 1312 done = true; 1313 dm_device.state = DM_INITIALIZED; 1314 } 1315 1316 /* 1317 * We are pushing a lot of data through the channel; 1318 * deal with transient failures caused because of the 1319 * lack of space in the ring buffer. 1320 */ 1321 1322 do { 1323 bl_resp->hdr.trans_id = atomic_inc_return(&trans_id); 1324 ret = vmbus_sendpacket(dm_device.dev->channel, 1325 bl_resp, 1326 bl_resp->hdr.size, 1327 (unsigned long)NULL, 1328 VM_PKT_DATA_INBAND, 0); 1329 1330 if (ret == -EAGAIN) 1331 msleep(20); 1332 post_status(&dm_device); 1333 } while (ret == -EAGAIN); 1334 1335 if (ret) { 1336 /* 1337 * Free up the memory we allocatted. 1338 */ 1339 pr_err("Balloon response failed\n"); 1340 1341 for (i = 0; i < bl_resp->range_count; i++) 1342 free_balloon_pages(&dm_device, 1343 &bl_resp->range_array[i]); 1344 1345 done = true; 1346 } 1347 } 1348 1349 } 1350 1351 static void balloon_down(struct hv_dynmem_device *dm, 1352 struct dm_unballoon_request *req) 1353 { 1354 union dm_mem_page_range *range_array = req->range_array; 1355 int range_count = req->range_count; 1356 struct dm_unballoon_response resp; 1357 int i; 1358 unsigned int prev_pages_ballooned = dm->num_pages_ballooned; 1359 1360 for (i = 0; i < range_count; i++) { 1361 free_balloon_pages(dm, &range_array[i]); 1362 complete(&dm_device.config_event); 1363 } 1364 1365 pr_debug("Freed %u ballooned pages.\n", 1366 prev_pages_ballooned - dm->num_pages_ballooned); 1367 1368 if (req->more_pages == 1) 1369 return; 1370 1371 memset(&resp, 0, sizeof(struct dm_unballoon_response)); 1372 resp.hdr.type = DM_UNBALLOON_RESPONSE; 1373 resp.hdr.trans_id = atomic_inc_return(&trans_id); 1374 resp.hdr.size = sizeof(struct dm_unballoon_response); 1375 1376 vmbus_sendpacket(dm_device.dev->channel, &resp, 1377 sizeof(struct dm_unballoon_response), 1378 (unsigned long)NULL, 1379 VM_PKT_DATA_INBAND, 0); 1380 1381 dm->state = DM_INITIALIZED; 1382 } 1383 1384 static void balloon_onchannelcallback(void *context); 1385 1386 static int dm_thread_func(void *dm_dev) 1387 { 1388 struct hv_dynmem_device *dm = dm_dev; 1389 1390 while (!kthread_should_stop()) { 1391 wait_for_completion_interruptible_timeout( 1392 &dm_device.config_event, 1*HZ); 1393 /* 1394 * The host expects us to post information on the memory 1395 * pressure every second. 1396 */ 1397 reinit_completion(&dm_device.config_event); 1398 post_status(dm); 1399 } 1400 1401 return 0; 1402 } 1403 1404 1405 static void version_resp(struct hv_dynmem_device *dm, 1406 struct dm_version_response *vresp) 1407 { 1408 struct dm_version_request version_req; 1409 int ret; 1410 1411 if (vresp->is_accepted) { 1412 /* 1413 * We are done; wakeup the 1414 * context waiting for version 1415 * negotiation. 1416 */ 1417 complete(&dm->host_event); 1418 return; 1419 } 1420 /* 1421 * If there are more versions to try, continue 1422 * with negotiations; if not 1423 * shutdown the service since we are not able 1424 * to negotiate a suitable version number 1425 * with the host. 1426 */ 1427 if (dm->next_version == 0) 1428 goto version_error; 1429 1430 memset(&version_req, 0, sizeof(struct dm_version_request)); 1431 version_req.hdr.type = DM_VERSION_REQUEST; 1432 version_req.hdr.size = sizeof(struct dm_version_request); 1433 version_req.hdr.trans_id = atomic_inc_return(&trans_id); 1434 version_req.version.version = dm->next_version; 1435 dm->version = version_req.version.version; 1436 1437 /* 1438 * Set the next version to try in case current version fails. 1439 * Win7 protocol ought to be the last one to try. 1440 */ 1441 switch (version_req.version.version) { 1442 case DYNMEM_PROTOCOL_VERSION_WIN8: 1443 dm->next_version = DYNMEM_PROTOCOL_VERSION_WIN7; 1444 version_req.is_last_attempt = 0; 1445 break; 1446 default: 1447 dm->next_version = 0; 1448 version_req.is_last_attempt = 1; 1449 } 1450 1451 ret = vmbus_sendpacket(dm->dev->channel, &version_req, 1452 sizeof(struct dm_version_request), 1453 (unsigned long)NULL, 1454 VM_PKT_DATA_INBAND, 0); 1455 1456 if (ret) 1457 goto version_error; 1458 1459 return; 1460 1461 version_error: 1462 dm->state = DM_INIT_ERROR; 1463 complete(&dm->host_event); 1464 } 1465 1466 static void cap_resp(struct hv_dynmem_device *dm, 1467 struct dm_capabilities_resp_msg *cap_resp) 1468 { 1469 if (!cap_resp->is_accepted) { 1470 pr_err("Capabilities not accepted by host\n"); 1471 dm->state = DM_INIT_ERROR; 1472 } 1473 complete(&dm->host_event); 1474 } 1475 1476 static void balloon_onchannelcallback(void *context) 1477 { 1478 struct hv_device *dev = context; 1479 u32 recvlen; 1480 u64 requestid; 1481 struct dm_message *dm_msg; 1482 struct dm_header *dm_hdr; 1483 struct hv_dynmem_device *dm = hv_get_drvdata(dev); 1484 struct dm_balloon *bal_msg; 1485 struct dm_hot_add *ha_msg; 1486 union dm_mem_page_range *ha_pg_range; 1487 union dm_mem_page_range *ha_region; 1488 1489 memset(recv_buffer, 0, sizeof(recv_buffer)); 1490 vmbus_recvpacket(dev->channel, recv_buffer, 1491 HV_HYP_PAGE_SIZE, &recvlen, &requestid); 1492 1493 if (recvlen > 0) { 1494 dm_msg = (struct dm_message *)recv_buffer; 1495 dm_hdr = &dm_msg->hdr; 1496 1497 switch (dm_hdr->type) { 1498 case DM_VERSION_RESPONSE: 1499 version_resp(dm, 1500 (struct dm_version_response *)dm_msg); 1501 break; 1502 1503 case DM_CAPABILITIES_RESPONSE: 1504 cap_resp(dm, 1505 (struct dm_capabilities_resp_msg *)dm_msg); 1506 break; 1507 1508 case DM_BALLOON_REQUEST: 1509 if (allow_hibernation) { 1510 pr_info("Ignore balloon-up request!\n"); 1511 break; 1512 } 1513 1514 if (dm->state == DM_BALLOON_UP) 1515 pr_warn("Currently ballooning\n"); 1516 bal_msg = (struct dm_balloon *)recv_buffer; 1517 dm->state = DM_BALLOON_UP; 1518 dm_device.balloon_wrk.num_pages = bal_msg->num_pages; 1519 schedule_work(&dm_device.balloon_wrk.wrk); 1520 break; 1521 1522 case DM_UNBALLOON_REQUEST: 1523 if (allow_hibernation) { 1524 pr_info("Ignore balloon-down request!\n"); 1525 break; 1526 } 1527 1528 dm->state = DM_BALLOON_DOWN; 1529 balloon_down(dm, 1530 (struct dm_unballoon_request *)recv_buffer); 1531 break; 1532 1533 case DM_MEM_HOT_ADD_REQUEST: 1534 if (dm->state == DM_HOT_ADD) 1535 pr_warn("Currently hot-adding\n"); 1536 dm->state = DM_HOT_ADD; 1537 ha_msg = (struct dm_hot_add *)recv_buffer; 1538 if (ha_msg->hdr.size == sizeof(struct dm_hot_add)) { 1539 /* 1540 * This is a normal hot-add request specifying 1541 * hot-add memory. 1542 */ 1543 dm->host_specified_ha_region = false; 1544 ha_pg_range = &ha_msg->range; 1545 dm->ha_wrk.ha_page_range = *ha_pg_range; 1546 dm->ha_wrk.ha_region_range.page_range = 0; 1547 } else { 1548 /* 1549 * Host is specifying that we first hot-add 1550 * a region and then partially populate this 1551 * region. 1552 */ 1553 dm->host_specified_ha_region = true; 1554 ha_pg_range = &ha_msg->range; 1555 ha_region = &ha_pg_range[1]; 1556 dm->ha_wrk.ha_page_range = *ha_pg_range; 1557 dm->ha_wrk.ha_region_range = *ha_region; 1558 } 1559 schedule_work(&dm_device.ha_wrk.wrk); 1560 break; 1561 1562 case DM_INFO_MESSAGE: 1563 process_info(dm, (struct dm_info_msg *)dm_msg); 1564 break; 1565 1566 default: 1567 pr_warn("Unhandled message: type: %d\n", dm_hdr->type); 1568 1569 } 1570 } 1571 1572 } 1573 1574 /* Hyper-V only supports reporting 2MB pages or higher */ 1575 #define HV_MIN_PAGE_REPORTING_ORDER 9 1576 #define HV_MIN_PAGE_REPORTING_LEN (HV_HYP_PAGE_SIZE << HV_MIN_PAGE_REPORTING_ORDER) 1577 static int hv_free_page_report(struct page_reporting_dev_info *pr_dev_info, 1578 struct scatterlist *sgl, unsigned int nents) 1579 { 1580 unsigned long flags; 1581 struct hv_memory_hint *hint; 1582 int i; 1583 u64 status; 1584 struct scatterlist *sg; 1585 1586 WARN_ON_ONCE(nents > HV_MEMORY_HINT_MAX_GPA_PAGE_RANGES); 1587 WARN_ON_ONCE(sgl->length < HV_MIN_PAGE_REPORTING_LEN); 1588 local_irq_save(flags); 1589 hint = *(struct hv_memory_hint **)this_cpu_ptr(hyperv_pcpu_input_arg); 1590 if (!hint) { 1591 local_irq_restore(flags); 1592 return -ENOSPC; 1593 } 1594 1595 hint->type = HV_EXT_MEMORY_HEAT_HINT_TYPE_COLD_DISCARD; 1596 hint->reserved = 0; 1597 for_each_sg(sgl, sg, nents, i) { 1598 union hv_gpa_page_range *range; 1599 1600 range = &hint->ranges[i]; 1601 range->address_space = 0; 1602 /* page reporting only reports 2MB pages or higher */ 1603 range->page.largepage = 1; 1604 range->page.additional_pages = 1605 (sg->length / HV_MIN_PAGE_REPORTING_LEN) - 1; 1606 range->page_size = HV_GPA_PAGE_RANGE_PAGE_SIZE_2MB; 1607 range->base_large_pfn = 1608 page_to_hvpfn(sg_page(sg)) >> HV_MIN_PAGE_REPORTING_ORDER; 1609 } 1610 1611 status = hv_do_rep_hypercall(HV_EXT_CALL_MEMORY_HEAT_HINT, nents, 0, 1612 hint, NULL); 1613 local_irq_restore(flags); 1614 if ((status & HV_HYPERCALL_RESULT_MASK) != HV_STATUS_SUCCESS) { 1615 pr_err("Cold memory discard hypercall failed with status %llx\n", 1616 status); 1617 return -EINVAL; 1618 } 1619 1620 return 0; 1621 } 1622 1623 static void enable_page_reporting(void) 1624 { 1625 int ret; 1626 1627 /* Essentially, validating 'PAGE_REPORTING_MIN_ORDER' is big enough. */ 1628 if (pageblock_order < HV_MIN_PAGE_REPORTING_ORDER) { 1629 pr_debug("Cold memory discard is only supported on 2MB pages and above\n"); 1630 return; 1631 } 1632 1633 if (!hv_query_ext_cap(HV_EXT_CAPABILITY_MEMORY_COLD_DISCARD_HINT)) { 1634 pr_debug("Cold memory discard hint not supported by Hyper-V\n"); 1635 return; 1636 } 1637 1638 BUILD_BUG_ON(PAGE_REPORTING_CAPACITY > HV_MEMORY_HINT_MAX_GPA_PAGE_RANGES); 1639 dm_device.pr_dev_info.report = hv_free_page_report; 1640 ret = page_reporting_register(&dm_device.pr_dev_info); 1641 if (ret < 0) { 1642 dm_device.pr_dev_info.report = NULL; 1643 pr_err("Failed to enable cold memory discard: %d\n", ret); 1644 } else { 1645 pr_info("Cold memory discard hint enabled\n"); 1646 } 1647 } 1648 1649 static void disable_page_reporting(void) 1650 { 1651 if (dm_device.pr_dev_info.report) { 1652 page_reporting_unregister(&dm_device.pr_dev_info); 1653 dm_device.pr_dev_info.report = NULL; 1654 } 1655 } 1656 1657 static int balloon_connect_vsp(struct hv_device *dev) 1658 { 1659 struct dm_version_request version_req; 1660 struct dm_capabilities cap_msg; 1661 unsigned long t; 1662 int ret; 1663 1664 ret = vmbus_open(dev->channel, dm_ring_size, dm_ring_size, NULL, 0, 1665 balloon_onchannelcallback, dev); 1666 if (ret) 1667 return ret; 1668 1669 /* 1670 * Initiate the hand shake with the host and negotiate 1671 * a version that the host can support. We start with the 1672 * highest version number and go down if the host cannot 1673 * support it. 1674 */ 1675 memset(&version_req, 0, sizeof(struct dm_version_request)); 1676 version_req.hdr.type = DM_VERSION_REQUEST; 1677 version_req.hdr.size = sizeof(struct dm_version_request); 1678 version_req.hdr.trans_id = atomic_inc_return(&trans_id); 1679 version_req.version.version = DYNMEM_PROTOCOL_VERSION_WIN10; 1680 version_req.is_last_attempt = 0; 1681 dm_device.version = version_req.version.version; 1682 1683 ret = vmbus_sendpacket(dev->channel, &version_req, 1684 sizeof(struct dm_version_request), 1685 (unsigned long)NULL, VM_PKT_DATA_INBAND, 0); 1686 if (ret) 1687 goto out; 1688 1689 t = wait_for_completion_timeout(&dm_device.host_event, 5*HZ); 1690 if (t == 0) { 1691 ret = -ETIMEDOUT; 1692 goto out; 1693 } 1694 1695 /* 1696 * If we could not negotiate a compatible version with the host 1697 * fail the probe function. 1698 */ 1699 if (dm_device.state == DM_INIT_ERROR) { 1700 ret = -EPROTO; 1701 goto out; 1702 } 1703 1704 pr_info("Using Dynamic Memory protocol version %u.%u\n", 1705 DYNMEM_MAJOR_VERSION(dm_device.version), 1706 DYNMEM_MINOR_VERSION(dm_device.version)); 1707 1708 /* 1709 * Now submit our capabilities to the host. 1710 */ 1711 memset(&cap_msg, 0, sizeof(struct dm_capabilities)); 1712 cap_msg.hdr.type = DM_CAPABILITIES_REPORT; 1713 cap_msg.hdr.size = sizeof(struct dm_capabilities); 1714 cap_msg.hdr.trans_id = atomic_inc_return(&trans_id); 1715 1716 /* 1717 * When hibernation (i.e. virtual ACPI S4 state) is enabled, the host 1718 * currently still requires the bits to be set, so we have to add code 1719 * to fail the host's hot-add and balloon up/down requests, if any. 1720 */ 1721 cap_msg.caps.cap_bits.balloon = 1; 1722 cap_msg.caps.cap_bits.hot_add = 1; 1723 1724 /* 1725 * Specify our alignment requirements as it relates 1726 * memory hot-add. Specify 128MB alignment. 1727 */ 1728 cap_msg.caps.cap_bits.hot_add_alignment = 7; 1729 1730 /* 1731 * Currently the host does not use these 1732 * values and we set them to what is done in the 1733 * Windows driver. 1734 */ 1735 cap_msg.min_page_cnt = 0; 1736 cap_msg.max_page_number = -1; 1737 1738 ret = vmbus_sendpacket(dev->channel, &cap_msg, 1739 sizeof(struct dm_capabilities), 1740 (unsigned long)NULL, VM_PKT_DATA_INBAND, 0); 1741 if (ret) 1742 goto out; 1743 1744 t = wait_for_completion_timeout(&dm_device.host_event, 5*HZ); 1745 if (t == 0) { 1746 ret = -ETIMEDOUT; 1747 goto out; 1748 } 1749 1750 /* 1751 * If the host does not like our capabilities, 1752 * fail the probe function. 1753 */ 1754 if (dm_device.state == DM_INIT_ERROR) { 1755 ret = -EPROTO; 1756 goto out; 1757 } 1758 1759 return 0; 1760 out: 1761 vmbus_close(dev->channel); 1762 return ret; 1763 } 1764 1765 static int balloon_probe(struct hv_device *dev, 1766 const struct hv_vmbus_device_id *dev_id) 1767 { 1768 int ret; 1769 1770 allow_hibernation = hv_is_hibernation_supported(); 1771 if (allow_hibernation) 1772 hot_add = false; 1773 1774 #ifdef CONFIG_MEMORY_HOTPLUG 1775 do_hot_add = hot_add; 1776 #else 1777 do_hot_add = false; 1778 #endif 1779 dm_device.dev = dev; 1780 dm_device.state = DM_INITIALIZING; 1781 dm_device.next_version = DYNMEM_PROTOCOL_VERSION_WIN8; 1782 init_completion(&dm_device.host_event); 1783 init_completion(&dm_device.config_event); 1784 INIT_LIST_HEAD(&dm_device.ha_region_list); 1785 spin_lock_init(&dm_device.ha_lock); 1786 INIT_WORK(&dm_device.balloon_wrk.wrk, balloon_up); 1787 INIT_WORK(&dm_device.ha_wrk.wrk, hot_add_req); 1788 dm_device.host_specified_ha_region = false; 1789 1790 #ifdef CONFIG_MEMORY_HOTPLUG 1791 set_online_page_callback(&hv_online_page); 1792 init_completion(&dm_device.ol_waitevent); 1793 register_memory_notifier(&hv_memory_nb); 1794 #endif 1795 1796 hv_set_drvdata(dev, &dm_device); 1797 1798 ret = balloon_connect_vsp(dev); 1799 if (ret != 0) 1800 return ret; 1801 1802 enable_page_reporting(); 1803 dm_device.state = DM_INITIALIZED; 1804 1805 dm_device.thread = 1806 kthread_run(dm_thread_func, &dm_device, "hv_balloon"); 1807 if (IS_ERR(dm_device.thread)) { 1808 ret = PTR_ERR(dm_device.thread); 1809 goto probe_error; 1810 } 1811 1812 return 0; 1813 1814 probe_error: 1815 dm_device.state = DM_INIT_ERROR; 1816 dm_device.thread = NULL; 1817 disable_page_reporting(); 1818 vmbus_close(dev->channel); 1819 #ifdef CONFIG_MEMORY_HOTPLUG 1820 unregister_memory_notifier(&hv_memory_nb); 1821 restore_online_page_callback(&hv_online_page); 1822 #endif 1823 return ret; 1824 } 1825 1826 static int balloon_remove(struct hv_device *dev) 1827 { 1828 struct hv_dynmem_device *dm = hv_get_drvdata(dev); 1829 struct hv_hotadd_state *has, *tmp; 1830 struct hv_hotadd_gap *gap, *tmp_gap; 1831 unsigned long flags; 1832 1833 if (dm->num_pages_ballooned != 0) 1834 pr_warn("Ballooned pages: %d\n", dm->num_pages_ballooned); 1835 1836 cancel_work_sync(&dm->balloon_wrk.wrk); 1837 cancel_work_sync(&dm->ha_wrk.wrk); 1838 1839 kthread_stop(dm->thread); 1840 disable_page_reporting(); 1841 vmbus_close(dev->channel); 1842 #ifdef CONFIG_MEMORY_HOTPLUG 1843 unregister_memory_notifier(&hv_memory_nb); 1844 restore_online_page_callback(&hv_online_page); 1845 #endif 1846 spin_lock_irqsave(&dm_device.ha_lock, flags); 1847 list_for_each_entry_safe(has, tmp, &dm->ha_region_list, list) { 1848 list_for_each_entry_safe(gap, tmp_gap, &has->gap_list, list) { 1849 list_del(&gap->list); 1850 kfree(gap); 1851 } 1852 list_del(&has->list); 1853 kfree(has); 1854 } 1855 spin_unlock_irqrestore(&dm_device.ha_lock, flags); 1856 1857 return 0; 1858 } 1859 1860 static int balloon_suspend(struct hv_device *hv_dev) 1861 { 1862 struct hv_dynmem_device *dm = hv_get_drvdata(hv_dev); 1863 1864 tasklet_disable(&hv_dev->channel->callback_event); 1865 1866 cancel_work_sync(&dm->balloon_wrk.wrk); 1867 cancel_work_sync(&dm->ha_wrk.wrk); 1868 1869 if (dm->thread) { 1870 kthread_stop(dm->thread); 1871 dm->thread = NULL; 1872 vmbus_close(hv_dev->channel); 1873 } 1874 1875 tasklet_enable(&hv_dev->channel->callback_event); 1876 1877 return 0; 1878 1879 } 1880 1881 static int balloon_resume(struct hv_device *dev) 1882 { 1883 int ret; 1884 1885 dm_device.state = DM_INITIALIZING; 1886 1887 ret = balloon_connect_vsp(dev); 1888 1889 if (ret != 0) 1890 goto out; 1891 1892 dm_device.thread = 1893 kthread_run(dm_thread_func, &dm_device, "hv_balloon"); 1894 if (IS_ERR(dm_device.thread)) { 1895 ret = PTR_ERR(dm_device.thread); 1896 dm_device.thread = NULL; 1897 goto close_channel; 1898 } 1899 1900 dm_device.state = DM_INITIALIZED; 1901 return 0; 1902 close_channel: 1903 vmbus_close(dev->channel); 1904 out: 1905 dm_device.state = DM_INIT_ERROR; 1906 #ifdef CONFIG_MEMORY_HOTPLUG 1907 unregister_memory_notifier(&hv_memory_nb); 1908 restore_online_page_callback(&hv_online_page); 1909 #endif 1910 return ret; 1911 } 1912 1913 static const struct hv_vmbus_device_id id_table[] = { 1914 /* Dynamic Memory Class ID */ 1915 /* 525074DC-8985-46e2-8057-A307DC18A502 */ 1916 { HV_DM_GUID, }, 1917 { }, 1918 }; 1919 1920 MODULE_DEVICE_TABLE(vmbus, id_table); 1921 1922 static struct hv_driver balloon_drv = { 1923 .name = "hv_balloon", 1924 .id_table = id_table, 1925 .probe = balloon_probe, 1926 .remove = balloon_remove, 1927 .suspend = balloon_suspend, 1928 .resume = balloon_resume, 1929 .driver = { 1930 .probe_type = PROBE_PREFER_ASYNCHRONOUS, 1931 }, 1932 }; 1933 1934 static int __init init_balloon_drv(void) 1935 { 1936 1937 return vmbus_driver_register(&balloon_drv); 1938 } 1939 1940 module_init(init_balloon_drv); 1941 1942 MODULE_DESCRIPTION("Hyper-V Balloon"); 1943 MODULE_LICENSE("GPL"); 1944