1f33ecfdeSHaren Myneni // SPDX-License-Identifier: GPL-2.0-or-later 2f33ecfdeSHaren Myneni /* 3f33ecfdeSHaren Myneni * Copyright 2020-21 IBM Corp. 4f33ecfdeSHaren Myneni */ 5f33ecfdeSHaren Myneni 6f33ecfdeSHaren Myneni #define pr_fmt(fmt) "vas: " fmt 7f33ecfdeSHaren Myneni 8f33ecfdeSHaren Myneni #include <linux/module.h> 9f33ecfdeSHaren Myneni #include <linux/kernel.h> 10f33ecfdeSHaren Myneni #include <linux/export.h> 11f33ecfdeSHaren Myneni #include <linux/types.h> 12f33ecfdeSHaren Myneni #include <linux/delay.h> 13b22f2d88SHaren Myneni #include <linux/slab.h> 146d0aaf5eSHaren Myneni #include <linux/interrupt.h> 15c736fb97SMichael Ellerman #include <linux/irqdomain.h> 16ca77d488SHaren Myneni #include <asm/machdep.h> 17f33ecfdeSHaren Myneni #include <asm/hvcall.h> 18f33ecfdeSHaren Myneni #include <asm/plpar_wrappers.h> 1946d60bdbSChristophe Leroy #include <asm/firmware.h> 20f33ecfdeSHaren Myneni #include <asm/vas.h> 21f33ecfdeSHaren Myneni #include "vas.h" 22f33ecfdeSHaren Myneni 23f33ecfdeSHaren Myneni #define VAS_INVALID_WIN_ADDRESS 0xFFFFFFFFFFFFFFFFul 24f33ecfdeSHaren Myneni #define VAS_DEFAULT_DOMAIN_ID 0xFFFFFFFFFFFFFFFFul 25f33ecfdeSHaren Myneni /* The hypervisor allows one credit per window right now */ 26f33ecfdeSHaren Myneni #define DEF_WIN_CREDS 1 27f33ecfdeSHaren Myneni 28ca77d488SHaren Myneni static struct vas_all_caps caps_all; 29ca77d488SHaren Myneni static bool copypaste_feat; 30278fe1ccSHaren Myneni static struct hv_vas_cop_feat_caps hv_cop_caps; 31ca77d488SHaren Myneni 32ca77d488SHaren Myneni static struct vas_caps vascaps[VAS_MAX_FEAT_TYPE]; 33b22f2d88SHaren Myneni static DEFINE_MUTEX(vas_pseries_mutex); 3437e67648SHaren Myneni static bool migration_in_progress; 35ca77d488SHaren Myneni 36f33ecfdeSHaren Myneni static long hcall_return_busy_check(long rc) 37f33ecfdeSHaren Myneni { 38f33ecfdeSHaren Myneni /* Check if we are stalled for some time */ 39f33ecfdeSHaren Myneni if (H_IS_LONG_BUSY(rc)) { 40f33ecfdeSHaren Myneni msleep(get_longbusy_msecs(rc)); 41f33ecfdeSHaren Myneni rc = H_BUSY; 42f33ecfdeSHaren Myneni } else if (rc == H_BUSY) { 43f33ecfdeSHaren Myneni cond_resched(); 44f33ecfdeSHaren Myneni } 45f33ecfdeSHaren Myneni 46f33ecfdeSHaren Myneni return rc; 47f33ecfdeSHaren Myneni } 48f33ecfdeSHaren Myneni 49f33ecfdeSHaren Myneni /* 50f33ecfdeSHaren Myneni * Allocate VAS window hcall 51f33ecfdeSHaren Myneni */ 52f33ecfdeSHaren Myneni static int h_allocate_vas_window(struct pseries_vas_window *win, u64 *domain, 53f33ecfdeSHaren Myneni u8 wintype, u16 credits) 54f33ecfdeSHaren Myneni { 55f33ecfdeSHaren Myneni long retbuf[PLPAR_HCALL9_BUFSIZE] = {0}; 56f33ecfdeSHaren Myneni long rc; 57f33ecfdeSHaren Myneni 58f33ecfdeSHaren Myneni do { 59f33ecfdeSHaren Myneni rc = plpar_hcall9(H_ALLOCATE_VAS_WINDOW, retbuf, wintype, 60f33ecfdeSHaren Myneni credits, domain[0], domain[1], domain[2], 61f33ecfdeSHaren Myneni domain[3], domain[4], domain[5]); 62f33ecfdeSHaren Myneni 63f33ecfdeSHaren Myneni rc = hcall_return_busy_check(rc); 64f33ecfdeSHaren Myneni } while (rc == H_BUSY); 65f33ecfdeSHaren Myneni 66f33ecfdeSHaren Myneni if (rc == H_SUCCESS) { 67f33ecfdeSHaren Myneni if (win->win_addr == VAS_INVALID_WIN_ADDRESS) { 68f33ecfdeSHaren Myneni pr_err("H_ALLOCATE_VAS_WINDOW: COPY/PASTE is not supported\n"); 69f33ecfdeSHaren Myneni return -ENOTSUPP; 70f33ecfdeSHaren Myneni } 71f33ecfdeSHaren Myneni win->vas_win.winid = retbuf[0]; 72f33ecfdeSHaren Myneni win->win_addr = retbuf[1]; 73f33ecfdeSHaren Myneni win->complete_irq = retbuf[2]; 74f33ecfdeSHaren Myneni win->fault_irq = retbuf[3]; 75f33ecfdeSHaren Myneni return 0; 76f33ecfdeSHaren Myneni } 77f33ecfdeSHaren Myneni 78f33ecfdeSHaren Myneni pr_err("H_ALLOCATE_VAS_WINDOW error: %ld, wintype: %u, credits: %u\n", 79f33ecfdeSHaren Myneni rc, wintype, credits); 80f33ecfdeSHaren Myneni 81f33ecfdeSHaren Myneni return -EIO; 82f33ecfdeSHaren Myneni } 83f33ecfdeSHaren Myneni 84f33ecfdeSHaren Myneni /* 85f33ecfdeSHaren Myneni * Deallocate VAS window hcall. 86f33ecfdeSHaren Myneni */ 87f33ecfdeSHaren Myneni static int h_deallocate_vas_window(u64 winid) 88f33ecfdeSHaren Myneni { 89f33ecfdeSHaren Myneni long rc; 90f33ecfdeSHaren Myneni 91f33ecfdeSHaren Myneni do { 92f33ecfdeSHaren Myneni rc = plpar_hcall_norets(H_DEALLOCATE_VAS_WINDOW, winid); 93f33ecfdeSHaren Myneni 94f33ecfdeSHaren Myneni rc = hcall_return_busy_check(rc); 95f33ecfdeSHaren Myneni } while (rc == H_BUSY); 96f33ecfdeSHaren Myneni 97f33ecfdeSHaren Myneni if (rc == H_SUCCESS) 98f33ecfdeSHaren Myneni return 0; 99f33ecfdeSHaren Myneni 100f33ecfdeSHaren Myneni pr_err("H_DEALLOCATE_VAS_WINDOW error: %ld, winid: %llu\n", 101f33ecfdeSHaren Myneni rc, winid); 102f33ecfdeSHaren Myneni return -EIO; 103f33ecfdeSHaren Myneni } 104f33ecfdeSHaren Myneni 105f33ecfdeSHaren Myneni /* 106f33ecfdeSHaren Myneni * Modify VAS window. 107f33ecfdeSHaren Myneni * After the window is opened with allocate window hcall, configure it 108f33ecfdeSHaren Myneni * with flags and LPAR PID before using. 109f33ecfdeSHaren Myneni */ 110f33ecfdeSHaren Myneni static int h_modify_vas_window(struct pseries_vas_window *win) 111f33ecfdeSHaren Myneni { 112f33ecfdeSHaren Myneni long rc; 113f33ecfdeSHaren Myneni 114f33ecfdeSHaren Myneni /* 115f33ecfdeSHaren Myneni * AMR value is not supported in Linux VAS implementation. 116f33ecfdeSHaren Myneni * The hypervisor ignores it if 0 is passed. 117f33ecfdeSHaren Myneni */ 118f33ecfdeSHaren Myneni do { 119f33ecfdeSHaren Myneni rc = plpar_hcall_norets(H_MODIFY_VAS_WINDOW, 120976410cdSHaren Myneni win->vas_win.winid, win->pid, 0, 121f33ecfdeSHaren Myneni VAS_MOD_WIN_FLAGS, 0); 122f33ecfdeSHaren Myneni 123f33ecfdeSHaren Myneni rc = hcall_return_busy_check(rc); 124f33ecfdeSHaren Myneni } while (rc == H_BUSY); 125f33ecfdeSHaren Myneni 126f33ecfdeSHaren Myneni if (rc == H_SUCCESS) 127f33ecfdeSHaren Myneni return 0; 128f33ecfdeSHaren Myneni 129976410cdSHaren Myneni pr_err("H_MODIFY_VAS_WINDOW error: %ld, winid %u pid %u\n", 130976410cdSHaren Myneni rc, win->vas_win.winid, win->pid); 131f33ecfdeSHaren Myneni return -EIO; 132f33ecfdeSHaren Myneni } 133f33ecfdeSHaren Myneni 134f33ecfdeSHaren Myneni /* 135f33ecfdeSHaren Myneni * This hcall is used to determine the capabilities from the hypervisor. 136f33ecfdeSHaren Myneni * @hcall: H_QUERY_VAS_CAPABILITIES or H_QUERY_NX_CAPABILITIES 137f33ecfdeSHaren Myneni * @query_type: If 0 is passed, the hypervisor returns the overall 138f33ecfdeSHaren Myneni * capabilities which provides all feature(s) that are 139f33ecfdeSHaren Myneni * available. Then query the hypervisor to get the 140f33ecfdeSHaren Myneni * corresponding capabilities for the specific feature. 141f33ecfdeSHaren Myneni * Example: H_QUERY_VAS_CAPABILITIES provides VAS GZIP QoS 142f33ecfdeSHaren Myneni * and VAS GZIP Default capabilities. 143f33ecfdeSHaren Myneni * H_QUERY_NX_CAPABILITIES provides NX GZIP 144f33ecfdeSHaren Myneni * capabilities. 145f33ecfdeSHaren Myneni * @result: Return buffer to save capabilities. 146f33ecfdeSHaren Myneni */ 147f33ecfdeSHaren Myneni int h_query_vas_capabilities(const u64 hcall, u8 query_type, u64 result) 148f33ecfdeSHaren Myneni { 149f33ecfdeSHaren Myneni long rc; 150f33ecfdeSHaren Myneni 151f33ecfdeSHaren Myneni rc = plpar_hcall_norets(hcall, query_type, result); 152f33ecfdeSHaren Myneni 153f33ecfdeSHaren Myneni if (rc == H_SUCCESS) 154f33ecfdeSHaren Myneni return 0; 155f33ecfdeSHaren Myneni 1560a006aceSNicholas Piggin /* H_FUNCTION means HV does not support VAS so don't print an error */ 1570a006aceSNicholas Piggin if (rc != H_FUNCTION) { 1580a006aceSNicholas Piggin pr_err("%s error %ld, query_type %u, result buffer 0x%llx\n", 1590a006aceSNicholas Piggin (hcall == H_QUERY_VAS_CAPABILITIES) ? 1600a006aceSNicholas Piggin "H_QUERY_VAS_CAPABILITIES" : 1610a006aceSNicholas Piggin "H_QUERY_NX_CAPABILITIES", 1620a006aceSNicholas Piggin rc, query_type, result); 1630a006aceSNicholas Piggin } 1640a006aceSNicholas Piggin 165f33ecfdeSHaren Myneni return -EIO; 166f33ecfdeSHaren Myneni } 167b22f2d88SHaren Myneni EXPORT_SYMBOL_GPL(h_query_vas_capabilities); 168b22f2d88SHaren Myneni 169b22f2d88SHaren Myneni /* 1706d0aaf5eSHaren Myneni * hcall to get fault CRB from the hypervisor. 1716d0aaf5eSHaren Myneni */ 1726d0aaf5eSHaren Myneni static int h_get_nx_fault(u32 winid, u64 buffer) 1736d0aaf5eSHaren Myneni { 1746d0aaf5eSHaren Myneni long rc; 1756d0aaf5eSHaren Myneni 1766d0aaf5eSHaren Myneni rc = plpar_hcall_norets(H_GET_NX_FAULT, winid, buffer); 1776d0aaf5eSHaren Myneni 1786d0aaf5eSHaren Myneni if (rc == H_SUCCESS) 1796d0aaf5eSHaren Myneni return 0; 1806d0aaf5eSHaren Myneni 1816d0aaf5eSHaren Myneni pr_err("H_GET_NX_FAULT error: %ld, winid %u, buffer 0x%llx\n", 1826d0aaf5eSHaren Myneni rc, winid, buffer); 1836d0aaf5eSHaren Myneni return -EIO; 1846d0aaf5eSHaren Myneni 1856d0aaf5eSHaren Myneni } 1866d0aaf5eSHaren Myneni 1876d0aaf5eSHaren Myneni /* 1886d0aaf5eSHaren Myneni * Handle the fault interrupt. 1896d0aaf5eSHaren Myneni * When the fault interrupt is received for each window, query the 1906d0aaf5eSHaren Myneni * hypervisor to get the fault CRB on the specific fault. Then 1916d0aaf5eSHaren Myneni * process the CRB by updating CSB or send signal if the user space 1926d0aaf5eSHaren Myneni * CSB is invalid. 1936d0aaf5eSHaren Myneni * Note: The hypervisor forwards an interrupt for each fault request. 1946d0aaf5eSHaren Myneni * So one fault CRB to process for each H_GET_NX_FAULT hcall. 1956d0aaf5eSHaren Myneni */ 1964cb26607SCédric Le Goater static irqreturn_t pseries_vas_fault_thread_fn(int irq, void *data) 1976d0aaf5eSHaren Myneni { 1986d0aaf5eSHaren Myneni struct pseries_vas_window *txwin = data; 1996d0aaf5eSHaren Myneni struct coprocessor_request_block crb; 2006d0aaf5eSHaren Myneni struct vas_user_win_ref *tsk_ref; 2016d0aaf5eSHaren Myneni int rc; 2026d0aaf5eSHaren Myneni 2036d0aaf5eSHaren Myneni rc = h_get_nx_fault(txwin->vas_win.winid, (u64)virt_to_phys(&crb)); 2046d0aaf5eSHaren Myneni if (!rc) { 2056d0aaf5eSHaren Myneni tsk_ref = &txwin->vas_win.task_ref; 2066d0aaf5eSHaren Myneni vas_dump_crb(&crb); 2076d0aaf5eSHaren Myneni vas_update_csb(&crb, tsk_ref); 2086d0aaf5eSHaren Myneni } 2096d0aaf5eSHaren Myneni 2106d0aaf5eSHaren Myneni return IRQ_HANDLED; 2116d0aaf5eSHaren Myneni } 2126d0aaf5eSHaren Myneni 2136d0aaf5eSHaren Myneni /* 214b22f2d88SHaren Myneni * Allocate window and setup IRQ mapping. 215b22f2d88SHaren Myneni */ 216b22f2d88SHaren Myneni static int allocate_setup_window(struct pseries_vas_window *txwin, 217b22f2d88SHaren Myneni u64 *domain, u8 wintype) 218b22f2d88SHaren Myneni { 219b22f2d88SHaren Myneni int rc; 220b22f2d88SHaren Myneni 221b22f2d88SHaren Myneni rc = h_allocate_vas_window(txwin, domain, wintype, DEF_WIN_CREDS); 222b22f2d88SHaren Myneni if (rc) 223b22f2d88SHaren Myneni return rc; 2246d0aaf5eSHaren Myneni /* 2256d0aaf5eSHaren Myneni * On PowerVM, the hypervisor setup and forwards the fault 2266d0aaf5eSHaren Myneni * interrupt per window. So the IRQ setup and fault handling 2276d0aaf5eSHaren Myneni * will be done for each open window separately. 2286d0aaf5eSHaren Myneni */ 2296d0aaf5eSHaren Myneni txwin->fault_virq = irq_create_mapping(NULL, txwin->fault_irq); 2306d0aaf5eSHaren Myneni if (!txwin->fault_virq) { 2316d0aaf5eSHaren Myneni pr_err("Failed irq mapping %d\n", txwin->fault_irq); 2326d0aaf5eSHaren Myneni rc = -EINVAL; 2336d0aaf5eSHaren Myneni goto out_win; 2346d0aaf5eSHaren Myneni } 2356d0aaf5eSHaren Myneni 2366d0aaf5eSHaren Myneni txwin->name = kasprintf(GFP_KERNEL, "vas-win-%d", 2376d0aaf5eSHaren Myneni txwin->vas_win.winid); 2386d0aaf5eSHaren Myneni if (!txwin->name) { 2396d0aaf5eSHaren Myneni rc = -ENOMEM; 2406d0aaf5eSHaren Myneni goto out_irq; 2416d0aaf5eSHaren Myneni } 2426d0aaf5eSHaren Myneni 2436d0aaf5eSHaren Myneni rc = request_threaded_irq(txwin->fault_virq, NULL, 2446d0aaf5eSHaren Myneni pseries_vas_fault_thread_fn, IRQF_ONESHOT, 2456d0aaf5eSHaren Myneni txwin->name, txwin); 2466d0aaf5eSHaren Myneni if (rc) { 2476d0aaf5eSHaren Myneni pr_err("VAS-Window[%d]: Request IRQ(%u) failed with %d\n", 2486d0aaf5eSHaren Myneni txwin->vas_win.winid, txwin->fault_virq, rc); 2496d0aaf5eSHaren Myneni goto out_free; 2506d0aaf5eSHaren Myneni } 251b22f2d88SHaren Myneni 252b22f2d88SHaren Myneni txwin->vas_win.wcreds_max = DEF_WIN_CREDS; 253b22f2d88SHaren Myneni 254b22f2d88SHaren Myneni return 0; 2556d0aaf5eSHaren Myneni out_free: 2566d0aaf5eSHaren Myneni kfree(txwin->name); 2576d0aaf5eSHaren Myneni out_irq: 2586d0aaf5eSHaren Myneni irq_dispose_mapping(txwin->fault_virq); 2596d0aaf5eSHaren Myneni out_win: 2606d0aaf5eSHaren Myneni h_deallocate_vas_window(txwin->vas_win.winid); 2616d0aaf5eSHaren Myneni return rc; 2626d0aaf5eSHaren Myneni } 2636d0aaf5eSHaren Myneni 2646d0aaf5eSHaren Myneni static inline void free_irq_setup(struct pseries_vas_window *txwin) 2656d0aaf5eSHaren Myneni { 2666d0aaf5eSHaren Myneni free_irq(txwin->fault_virq, txwin); 2676d0aaf5eSHaren Myneni kfree(txwin->name); 2686d0aaf5eSHaren Myneni irq_dispose_mapping(txwin->fault_virq); 269b22f2d88SHaren Myneni } 270b22f2d88SHaren Myneni 271b22f2d88SHaren Myneni static struct vas_window *vas_allocate_window(int vas_id, u64 flags, 272b22f2d88SHaren Myneni enum vas_cop_type cop_type) 273b22f2d88SHaren Myneni { 274b22f2d88SHaren Myneni long domain[PLPAR_HCALL9_BUFSIZE] = {VAS_DEFAULT_DOMAIN_ID}; 275b22f2d88SHaren Myneni struct vas_cop_feat_caps *cop_feat_caps; 276b22f2d88SHaren Myneni struct vas_caps *caps; 277b22f2d88SHaren Myneni struct pseries_vas_window *txwin; 278b22f2d88SHaren Myneni int rc; 279b22f2d88SHaren Myneni 280b22f2d88SHaren Myneni txwin = kzalloc(sizeof(*txwin), GFP_KERNEL); 281b22f2d88SHaren Myneni if (!txwin) 282b22f2d88SHaren Myneni return ERR_PTR(-ENOMEM); 283b22f2d88SHaren Myneni 284b22f2d88SHaren Myneni /* 285b22f2d88SHaren Myneni * A VAS window can have many credits which means that many 286b22f2d88SHaren Myneni * requests can be issued simultaneously. But the hypervisor 287b22f2d88SHaren Myneni * restricts one credit per window. 288b22f2d88SHaren Myneni * The hypervisor introduces 2 different types of credits: 289b22f2d88SHaren Myneni * Default credit type (Uses normal priority FIFO): 290b22f2d88SHaren Myneni * A limited number of credits are assigned to partitions 291b22f2d88SHaren Myneni * based on processor entitlement. But these credits may be 292b22f2d88SHaren Myneni * over-committed on a system depends on whether the CPUs 293b22f2d88SHaren Myneni * are in shared or dedicated modes - that is, more requests 294b22f2d88SHaren Myneni * may be issued across the system than NX can service at 295b22f2d88SHaren Myneni * once which can result in paste command failure (RMA_busy). 296b22f2d88SHaren Myneni * Then the process has to resend requests or fall-back to 297b22f2d88SHaren Myneni * SW compression. 298b22f2d88SHaren Myneni * Quality of Service (QoS) credit type (Uses high priority FIFO): 299b22f2d88SHaren Myneni * To avoid NX HW contention, the system admins can assign 300b22f2d88SHaren Myneni * QoS credits for each LPAR so that this partition is 301b22f2d88SHaren Myneni * guaranteed access to NX resources. These credits are 302b22f2d88SHaren Myneni * assigned to partitions via the HMC. 303b22f2d88SHaren Myneni * Refer PAPR for more information. 304b22f2d88SHaren Myneni * 305b22f2d88SHaren Myneni * Allocate window with QoS credits if user requested. Otherwise 306b22f2d88SHaren Myneni * default credits are used. 307b22f2d88SHaren Myneni */ 308b22f2d88SHaren Myneni if (flags & VAS_TX_WIN_FLAG_QOS_CREDIT) 309b22f2d88SHaren Myneni caps = &vascaps[VAS_GZIP_QOS_FEAT_TYPE]; 310b22f2d88SHaren Myneni else 311b22f2d88SHaren Myneni caps = &vascaps[VAS_GZIP_DEF_FEAT_TYPE]; 312b22f2d88SHaren Myneni 313b22f2d88SHaren Myneni cop_feat_caps = &caps->caps; 314b22f2d88SHaren Myneni 31540562fe4SHaren Myneni if (atomic_inc_return(&cop_feat_caps->nr_used_credits) > 31640562fe4SHaren Myneni atomic_read(&cop_feat_caps->nr_total_credits)) { 317b22f2d88SHaren Myneni pr_err("Credits are not available to allocate window\n"); 318b22f2d88SHaren Myneni rc = -EINVAL; 319b22f2d88SHaren Myneni goto out; 320b22f2d88SHaren Myneni } 321b22f2d88SHaren Myneni 322b22f2d88SHaren Myneni if (vas_id == -1) { 323b22f2d88SHaren Myneni /* 324b22f2d88SHaren Myneni * The user space is requesting to allocate a window on 325b22f2d88SHaren Myneni * a VAS instance where the process is executing. 326b22f2d88SHaren Myneni * On PowerVM, domain values are passed to the hypervisor 327b22f2d88SHaren Myneni * to select VAS instance. Useful if the process is 328b22f2d88SHaren Myneni * affinity to NUMA node. 329b22f2d88SHaren Myneni * The hypervisor selects VAS instance if 330b22f2d88SHaren Myneni * VAS_DEFAULT_DOMAIN_ID (-1) is passed for domain values. 331b22f2d88SHaren Myneni * The h_allocate_vas_window hcall is defined to take a 332b22f2d88SHaren Myneni * domain values as specified by h_home_node_associativity, 333b22f2d88SHaren Myneni * So no unpacking needs to be done. 334b22f2d88SHaren Myneni */ 335b22f2d88SHaren Myneni rc = plpar_hcall9(H_HOME_NODE_ASSOCIATIVITY, domain, 336b22f2d88SHaren Myneni VPHN_FLAG_VCPU, smp_processor_id()); 337b22f2d88SHaren Myneni if (rc != H_SUCCESS) { 338b22f2d88SHaren Myneni pr_err("H_HOME_NODE_ASSOCIATIVITY error: %d\n", rc); 339b22f2d88SHaren Myneni goto out; 340b22f2d88SHaren Myneni } 341b22f2d88SHaren Myneni } 342b22f2d88SHaren Myneni 343976410cdSHaren Myneni txwin->pid = mfspr(SPRN_PID); 344976410cdSHaren Myneni 345b22f2d88SHaren Myneni /* 346b22f2d88SHaren Myneni * Allocate / Deallocate window hcalls and setup / free IRQs 347b22f2d88SHaren Myneni * have to be protected with mutex. 348b22f2d88SHaren Myneni * Open VAS window: Allocate window hcall and setup IRQ 349b22f2d88SHaren Myneni * Close VAS window: Deallocate window hcall and free IRQ 350b22f2d88SHaren Myneni * The hypervisor waits until all NX requests are 351b22f2d88SHaren Myneni * completed before closing the window. So expects OS 352b22f2d88SHaren Myneni * to handle NX faults, means IRQ can be freed only 353b22f2d88SHaren Myneni * after the deallocate window hcall is returned. 354b22f2d88SHaren Myneni * So once the window is closed with deallocate hcall before 355b22f2d88SHaren Myneni * the IRQ is freed, it can be assigned to new allocate 356b22f2d88SHaren Myneni * hcall with the same fault IRQ by the hypervisor. It can 357b22f2d88SHaren Myneni * result in setup IRQ fail for the new window since the 358b22f2d88SHaren Myneni * same fault IRQ is not freed by the OS before. 359b22f2d88SHaren Myneni */ 360b22f2d88SHaren Myneni mutex_lock(&vas_pseries_mutex); 36137e67648SHaren Myneni if (migration_in_progress) 36237e67648SHaren Myneni rc = -EBUSY; 36337e67648SHaren Myneni else 364b22f2d88SHaren Myneni rc = allocate_setup_window(txwin, (u64 *)&domain[0], 365b22f2d88SHaren Myneni cop_feat_caps->win_type); 366b22f2d88SHaren Myneni mutex_unlock(&vas_pseries_mutex); 367b22f2d88SHaren Myneni if (rc) 368b22f2d88SHaren Myneni goto out; 369b22f2d88SHaren Myneni 370b22f2d88SHaren Myneni /* 371b22f2d88SHaren Myneni * Modify window and it is ready to use. 372b22f2d88SHaren Myneni */ 373b22f2d88SHaren Myneni rc = h_modify_vas_window(txwin); 374b22f2d88SHaren Myneni if (!rc) 375b22f2d88SHaren Myneni rc = get_vas_user_win_ref(&txwin->vas_win.task_ref); 376b22f2d88SHaren Myneni if (rc) 377b22f2d88SHaren Myneni goto out_free; 378b22f2d88SHaren Myneni 379b22f2d88SHaren Myneni txwin->win_type = cop_feat_caps->win_type; 380b22f2d88SHaren Myneni mutex_lock(&vas_pseries_mutex); 3818ef7b9e1SHaren Myneni /* 3828ef7b9e1SHaren Myneni * Possible to lose the acquired credit with DLPAR core 3838ef7b9e1SHaren Myneni * removal after the window is opened. So if there are any 3848ef7b9e1SHaren Myneni * closed windows (means with lost credits), do not give new 3858ef7b9e1SHaren Myneni * window to user space. New windows will be opened only 3868ef7b9e1SHaren Myneni * after the existing windows are reopened when credits are 3878ef7b9e1SHaren Myneni * available. 3888ef7b9e1SHaren Myneni */ 3898ef7b9e1SHaren Myneni if (!caps->nr_close_wins) { 390b22f2d88SHaren Myneni list_add(&txwin->win_list, &caps->list); 3918ef7b9e1SHaren Myneni caps->nr_open_windows++; 3928ef7b9e1SHaren Myneni mutex_unlock(&vas_pseries_mutex); 3938ef7b9e1SHaren Myneni vas_user_win_add_mm_context(&txwin->vas_win.task_ref); 3948ef7b9e1SHaren Myneni return &txwin->vas_win; 3958ef7b9e1SHaren Myneni } 396b22f2d88SHaren Myneni mutex_unlock(&vas_pseries_mutex); 397b22f2d88SHaren Myneni 3988ef7b9e1SHaren Myneni put_vas_user_win_ref(&txwin->vas_win.task_ref); 3998ef7b9e1SHaren Myneni rc = -EBUSY; 4008ef7b9e1SHaren Myneni pr_err("No credit is available to allocate window\n"); 401b22f2d88SHaren Myneni 402b22f2d88SHaren Myneni out_free: 4036d0aaf5eSHaren Myneni /* 4046d0aaf5eSHaren Myneni * Window is not operational. Free IRQ before closing 4056d0aaf5eSHaren Myneni * window so that do not have to hold mutex. 4066d0aaf5eSHaren Myneni */ 4076d0aaf5eSHaren Myneni free_irq_setup(txwin); 408b22f2d88SHaren Myneni h_deallocate_vas_window(txwin->vas_win.winid); 409b22f2d88SHaren Myneni out: 41040562fe4SHaren Myneni atomic_dec(&cop_feat_caps->nr_used_credits); 411b22f2d88SHaren Myneni kfree(txwin); 412b22f2d88SHaren Myneni return ERR_PTR(rc); 413b22f2d88SHaren Myneni } 414b22f2d88SHaren Myneni 415b22f2d88SHaren Myneni static u64 vas_paste_address(struct vas_window *vwin) 416b22f2d88SHaren Myneni { 417b22f2d88SHaren Myneni struct pseries_vas_window *win; 418b22f2d88SHaren Myneni 419b22f2d88SHaren Myneni win = container_of(vwin, struct pseries_vas_window, vas_win); 420b22f2d88SHaren Myneni return win->win_addr; 421b22f2d88SHaren Myneni } 422b22f2d88SHaren Myneni 423b22f2d88SHaren Myneni static int deallocate_free_window(struct pseries_vas_window *win) 424b22f2d88SHaren Myneni { 425b22f2d88SHaren Myneni int rc = 0; 426b22f2d88SHaren Myneni 4276d0aaf5eSHaren Myneni /* 4286d0aaf5eSHaren Myneni * The hypervisor waits for all requests including faults 4296d0aaf5eSHaren Myneni * are processed before closing the window - Means all 4306d0aaf5eSHaren Myneni * credits have to be returned. In the case of fault 4316d0aaf5eSHaren Myneni * request, a credit is returned after OS issues 4326d0aaf5eSHaren Myneni * H_GET_NX_FAULT hcall. 4336d0aaf5eSHaren Myneni * So free IRQ after executing H_DEALLOCATE_VAS_WINDOW 4346d0aaf5eSHaren Myneni * hcall. 4356d0aaf5eSHaren Myneni */ 436b22f2d88SHaren Myneni rc = h_deallocate_vas_window(win->vas_win.winid); 4376d0aaf5eSHaren Myneni if (!rc) 4386d0aaf5eSHaren Myneni free_irq_setup(win); 439b22f2d88SHaren Myneni 440b22f2d88SHaren Myneni return rc; 441b22f2d88SHaren Myneni } 442b22f2d88SHaren Myneni 443b22f2d88SHaren Myneni static int vas_deallocate_window(struct vas_window *vwin) 444b22f2d88SHaren Myneni { 445b22f2d88SHaren Myneni struct pseries_vas_window *win; 446b22f2d88SHaren Myneni struct vas_cop_feat_caps *caps; 447b22f2d88SHaren Myneni int rc = 0; 448b22f2d88SHaren Myneni 449b22f2d88SHaren Myneni if (!vwin) 450b22f2d88SHaren Myneni return -EINVAL; 451b22f2d88SHaren Myneni 452b22f2d88SHaren Myneni win = container_of(vwin, struct pseries_vas_window, vas_win); 453b22f2d88SHaren Myneni 454b22f2d88SHaren Myneni /* Should not happen */ 455b22f2d88SHaren Myneni if (win->win_type >= VAS_MAX_FEAT_TYPE) { 456b22f2d88SHaren Myneni pr_err("Window (%u): Invalid window type %u\n", 457b22f2d88SHaren Myneni vwin->winid, win->win_type); 458b22f2d88SHaren Myneni return -EINVAL; 459b22f2d88SHaren Myneni } 460b22f2d88SHaren Myneni 461b22f2d88SHaren Myneni caps = &vascaps[win->win_type].caps; 462b22f2d88SHaren Myneni mutex_lock(&vas_pseries_mutex); 4638ef7b9e1SHaren Myneni /* 4648ef7b9e1SHaren Myneni * VAS window is already closed in the hypervisor when 465716d7a2eSHaren Myneni * lost the credit or with migration. So just remove the entry 466716d7a2eSHaren Myneni * from the list, remove task references and free vas_window 4678ef7b9e1SHaren Myneni * struct. 4688ef7b9e1SHaren Myneni */ 469716d7a2eSHaren Myneni if (!(win->vas_win.status & VAS_WIN_NO_CRED_CLOSE) && 470716d7a2eSHaren Myneni !(win->vas_win.status & VAS_WIN_MIGRATE_CLOSE)) { 471b22f2d88SHaren Myneni rc = deallocate_free_window(win); 472b22f2d88SHaren Myneni if (rc) { 473b22f2d88SHaren Myneni mutex_unlock(&vas_pseries_mutex); 474b22f2d88SHaren Myneni return rc; 475b22f2d88SHaren Myneni } 4768ef7b9e1SHaren Myneni } else 4778ef7b9e1SHaren Myneni vascaps[win->win_type].nr_close_wins--; 478b22f2d88SHaren Myneni 479b22f2d88SHaren Myneni list_del(&win->win_list); 48040562fe4SHaren Myneni atomic_dec(&caps->nr_used_credits); 4818ef7b9e1SHaren Myneni vascaps[win->win_type].nr_open_windows--; 482b22f2d88SHaren Myneni mutex_unlock(&vas_pseries_mutex); 483b22f2d88SHaren Myneni 484b22f2d88SHaren Myneni put_vas_user_win_ref(&vwin->task_ref); 485b22f2d88SHaren Myneni mm_context_remove_vas_window(vwin->task_ref.mm); 486b22f2d88SHaren Myneni 487b22f2d88SHaren Myneni kfree(win); 488b22f2d88SHaren Myneni return 0; 489b22f2d88SHaren Myneni } 490b22f2d88SHaren Myneni 491b22f2d88SHaren Myneni static const struct vas_user_win_ops vops_pseries = { 492b22f2d88SHaren Myneni .open_win = vas_allocate_window, /* Open and configure window */ 493b22f2d88SHaren Myneni .paste_addr = vas_paste_address, /* To do copy/paste */ 494b22f2d88SHaren Myneni .close_win = vas_deallocate_window, /* Close window */ 495b22f2d88SHaren Myneni }; 496b22f2d88SHaren Myneni 497b22f2d88SHaren Myneni /* 498b22f2d88SHaren Myneni * Supporting only nx-gzip coprocessor type now, but this API code 499b22f2d88SHaren Myneni * extended to other coprocessor types later. 500b22f2d88SHaren Myneni */ 501b22f2d88SHaren Myneni int vas_register_api_pseries(struct module *mod, enum vas_cop_type cop_type, 502b22f2d88SHaren Myneni const char *name) 503b22f2d88SHaren Myneni { 504b22f2d88SHaren Myneni int rc; 505b22f2d88SHaren Myneni 506b22f2d88SHaren Myneni if (!copypaste_feat) 507b22f2d88SHaren Myneni return -ENOTSUPP; 508b22f2d88SHaren Myneni 509b22f2d88SHaren Myneni rc = vas_register_coproc_api(mod, cop_type, name, &vops_pseries); 510b22f2d88SHaren Myneni 511b22f2d88SHaren Myneni return rc; 512b22f2d88SHaren Myneni } 513b22f2d88SHaren Myneni EXPORT_SYMBOL_GPL(vas_register_api_pseries); 514b22f2d88SHaren Myneni 515b22f2d88SHaren Myneni void vas_unregister_api_pseries(void) 516b22f2d88SHaren Myneni { 517b22f2d88SHaren Myneni vas_unregister_coproc_api(); 518b22f2d88SHaren Myneni } 519b22f2d88SHaren Myneni EXPORT_SYMBOL_GPL(vas_unregister_api_pseries); 520ca77d488SHaren Myneni 521ca77d488SHaren Myneni /* 522ca77d488SHaren Myneni * Get the specific capabilities based on the feature type. 523ca77d488SHaren Myneni * Right now supports GZIP default and GZIP QoS capabilities. 524ca77d488SHaren Myneni */ 525e14ff96dSNick Child static int __init get_vas_capabilities(u8 feat, enum vas_cop_feat_type type, 526ca77d488SHaren Myneni struct hv_vas_cop_feat_caps *hv_caps) 527ca77d488SHaren Myneni { 528ca77d488SHaren Myneni struct vas_cop_feat_caps *caps; 529ca77d488SHaren Myneni struct vas_caps *vcaps; 530ca77d488SHaren Myneni int rc = 0; 531ca77d488SHaren Myneni 532ca77d488SHaren Myneni vcaps = &vascaps[type]; 533ca77d488SHaren Myneni memset(vcaps, 0, sizeof(*vcaps)); 534ca77d488SHaren Myneni INIT_LIST_HEAD(&vcaps->list); 535ca77d488SHaren Myneni 5368ef7b9e1SHaren Myneni vcaps->feat = feat; 537ca77d488SHaren Myneni caps = &vcaps->caps; 538ca77d488SHaren Myneni 539ca77d488SHaren Myneni rc = h_query_vas_capabilities(H_QUERY_VAS_CAPABILITIES, feat, 540ca77d488SHaren Myneni (u64)virt_to_phys(hv_caps)); 541ca77d488SHaren Myneni if (rc) 542ca77d488SHaren Myneni return rc; 543ca77d488SHaren Myneni 544ca77d488SHaren Myneni caps->user_mode = hv_caps->user_mode; 545ca77d488SHaren Myneni if (!(caps->user_mode & VAS_COPY_PASTE_USER_MODE)) { 546ca77d488SHaren Myneni pr_err("User space COPY/PASTE is not supported\n"); 547ca77d488SHaren Myneni return -ENOTSUPP; 548ca77d488SHaren Myneni } 549ca77d488SHaren Myneni 550ca77d488SHaren Myneni caps->descriptor = be64_to_cpu(hv_caps->descriptor); 551ca77d488SHaren Myneni caps->win_type = hv_caps->win_type; 552ca77d488SHaren Myneni if (caps->win_type >= VAS_MAX_FEAT_TYPE) { 553ca77d488SHaren Myneni pr_err("Unsupported window type %u\n", caps->win_type); 554ca77d488SHaren Myneni return -EINVAL; 555ca77d488SHaren Myneni } 556ca77d488SHaren Myneni caps->max_lpar_creds = be16_to_cpu(hv_caps->max_lpar_creds); 557ca77d488SHaren Myneni caps->max_win_creds = be16_to_cpu(hv_caps->max_win_creds); 55840562fe4SHaren Myneni atomic_set(&caps->nr_total_credits, 559ca77d488SHaren Myneni be16_to_cpu(hv_caps->target_lpar_creds)); 560ca77d488SHaren Myneni if (feat == VAS_GZIP_DEF_FEAT) { 561ca77d488SHaren Myneni caps->def_lpar_creds = be16_to_cpu(hv_caps->def_lpar_creds); 562ca77d488SHaren Myneni 563ca77d488SHaren Myneni if (caps->max_win_creds < DEF_WIN_CREDS) { 564ca77d488SHaren Myneni pr_err("Window creds(%u) > max allowed window creds(%u)\n", 565ca77d488SHaren Myneni DEF_WIN_CREDS, caps->max_win_creds); 566ca77d488SHaren Myneni return -EINVAL; 567ca77d488SHaren Myneni } 568ca77d488SHaren Myneni } 569ca77d488SHaren Myneni 570b903737bSHaren Myneni rc = sysfs_add_vas_caps(caps); 571b903737bSHaren Myneni if (rc) 572b903737bSHaren Myneni return rc; 573b903737bSHaren Myneni 574ca77d488SHaren Myneni copypaste_feat = true; 575ca77d488SHaren Myneni 576ca77d488SHaren Myneni return 0; 577ca77d488SHaren Myneni } 578ca77d488SHaren Myneni 5798ef7b9e1SHaren Myneni /* 580c656cfe5SHaren Myneni * VAS windows can be closed due to lost credits when the core is 581c656cfe5SHaren Myneni * removed. So reopen them if credits are available due to DLPAR 582c656cfe5SHaren Myneni * core add and set the window active status. When NX sees the page 583c656cfe5SHaren Myneni * fault on the unmapped paste address, the kernel handles the fault 584c656cfe5SHaren Myneni * by setting the remapping to new paste address if the window is 585c656cfe5SHaren Myneni * active. 586c656cfe5SHaren Myneni */ 587716d7a2eSHaren Myneni static int reconfig_open_windows(struct vas_caps *vcaps, int creds, 588716d7a2eSHaren Myneni bool migrate) 589c656cfe5SHaren Myneni { 590c656cfe5SHaren Myneni long domain[PLPAR_HCALL9_BUFSIZE] = {VAS_DEFAULT_DOMAIN_ID}; 591c656cfe5SHaren Myneni struct vas_cop_feat_caps *caps = &vcaps->caps; 592c656cfe5SHaren Myneni struct pseries_vas_window *win = NULL, *tmp; 593c656cfe5SHaren Myneni int rc, mv_ents = 0; 594716d7a2eSHaren Myneni int flag; 595c656cfe5SHaren Myneni 596c656cfe5SHaren Myneni /* 597c656cfe5SHaren Myneni * Nothing to do if there are no closed windows. 598c656cfe5SHaren Myneni */ 599c656cfe5SHaren Myneni if (!vcaps->nr_close_wins) 600c656cfe5SHaren Myneni return 0; 601c656cfe5SHaren Myneni 602c656cfe5SHaren Myneni /* 603c656cfe5SHaren Myneni * For the core removal, the hypervisor reduces the credits 604c656cfe5SHaren Myneni * assigned to the LPAR and the kernel closes VAS windows 605c656cfe5SHaren Myneni * in the hypervisor depends on reduced credits. The kernel 606c656cfe5SHaren Myneni * uses LIFO (the last windows that are opened will be closed 607c656cfe5SHaren Myneni * first) and expects to open in the same order when credits 608c656cfe5SHaren Myneni * are available. 609c656cfe5SHaren Myneni * For example, 40 windows are closed when the LPAR lost 2 cores 610c656cfe5SHaren Myneni * (dedicated). If 1 core is added, this LPAR can have 20 more 611c656cfe5SHaren Myneni * credits. It means the kernel can reopen 20 windows. So move 612c656cfe5SHaren Myneni * 20 entries in the VAS windows lost and reopen next 20 windows. 613716d7a2eSHaren Myneni * For partition migration, reopen all windows that are closed 614716d7a2eSHaren Myneni * during resume. 615c656cfe5SHaren Myneni */ 616716d7a2eSHaren Myneni if ((vcaps->nr_close_wins > creds) && !migrate) 617c656cfe5SHaren Myneni mv_ents = vcaps->nr_close_wins - creds; 618c656cfe5SHaren Myneni 619c656cfe5SHaren Myneni list_for_each_entry_safe(win, tmp, &vcaps->list, win_list) { 620c656cfe5SHaren Myneni if (!mv_ents) 621c656cfe5SHaren Myneni break; 622c656cfe5SHaren Myneni 623c656cfe5SHaren Myneni mv_ents--; 624c656cfe5SHaren Myneni } 625c656cfe5SHaren Myneni 626716d7a2eSHaren Myneni /* 627716d7a2eSHaren Myneni * Open windows if they are closed only with migration or 628716d7a2eSHaren Myneni * DLPAR (lost credit) before. 629716d7a2eSHaren Myneni */ 630716d7a2eSHaren Myneni if (migrate) 631716d7a2eSHaren Myneni flag = VAS_WIN_MIGRATE_CLOSE; 632716d7a2eSHaren Myneni else 633716d7a2eSHaren Myneni flag = VAS_WIN_NO_CRED_CLOSE; 634716d7a2eSHaren Myneni 635c656cfe5SHaren Myneni list_for_each_entry_safe_from(win, tmp, &vcaps->list, win_list) { 636c656cfe5SHaren Myneni /* 637716d7a2eSHaren Myneni * This window is closed with DLPAR and migration events. 638716d7a2eSHaren Myneni * So reopen the window with the last event. 639716d7a2eSHaren Myneni * The user space is not suspended with the current 640716d7a2eSHaren Myneni * migration notifier. So the user space can issue DLPAR 641716d7a2eSHaren Myneni * CPU hotplug while migration in progress. In this case 642716d7a2eSHaren Myneni * this window will be opened with the last event. 643c656cfe5SHaren Myneni */ 644716d7a2eSHaren Myneni if ((win->vas_win.status & VAS_WIN_NO_CRED_CLOSE) && 645716d7a2eSHaren Myneni (win->vas_win.status & VAS_WIN_MIGRATE_CLOSE)) { 646716d7a2eSHaren Myneni win->vas_win.status &= ~flag; 647716d7a2eSHaren Myneni continue; 648716d7a2eSHaren Myneni } 649716d7a2eSHaren Myneni 650716d7a2eSHaren Myneni /* 651716d7a2eSHaren Myneni * Nothing to do on this window if it is not closed 652716d7a2eSHaren Myneni * with this flag 653716d7a2eSHaren Myneni */ 654716d7a2eSHaren Myneni if (!(win->vas_win.status & flag)) 655c656cfe5SHaren Myneni continue; 656c656cfe5SHaren Myneni 657c656cfe5SHaren Myneni rc = allocate_setup_window(win, (u64 *)&domain[0], 658c656cfe5SHaren Myneni caps->win_type); 659c656cfe5SHaren Myneni if (rc) 660c656cfe5SHaren Myneni return rc; 661c656cfe5SHaren Myneni 662c656cfe5SHaren Myneni rc = h_modify_vas_window(win); 663c656cfe5SHaren Myneni if (rc) 664c656cfe5SHaren Myneni goto out; 665c656cfe5SHaren Myneni 666c656cfe5SHaren Myneni mutex_lock(&win->vas_win.task_ref.mmap_mutex); 667c656cfe5SHaren Myneni /* 668c656cfe5SHaren Myneni * Set window status to active 669c656cfe5SHaren Myneni */ 670716d7a2eSHaren Myneni win->vas_win.status &= ~flag; 671c656cfe5SHaren Myneni mutex_unlock(&win->vas_win.task_ref.mmap_mutex); 672c656cfe5SHaren Myneni win->win_type = caps->win_type; 673c656cfe5SHaren Myneni if (!--vcaps->nr_close_wins) 674c656cfe5SHaren Myneni break; 675c656cfe5SHaren Myneni } 676c656cfe5SHaren Myneni 677c656cfe5SHaren Myneni return 0; 678c656cfe5SHaren Myneni out: 679c656cfe5SHaren Myneni /* 680c656cfe5SHaren Myneni * Window modify HCALL failed. So close the window to the 681c656cfe5SHaren Myneni * hypervisor and return. 682c656cfe5SHaren Myneni */ 683c656cfe5SHaren Myneni free_irq_setup(win); 684c656cfe5SHaren Myneni h_deallocate_vas_window(win->vas_win.winid); 685c656cfe5SHaren Myneni return rc; 686c656cfe5SHaren Myneni } 687c656cfe5SHaren Myneni 688c656cfe5SHaren Myneni /* 6898ef7b9e1SHaren Myneni * The hypervisor reduces the available credits if the LPAR lost core. It 6908ef7b9e1SHaren Myneni * means the excessive windows should not be active and the user space 6918ef7b9e1SHaren Myneni * should not be using these windows to send compression requests to NX. 6928ef7b9e1SHaren Myneni * So the kernel closes the excessive windows and unmap the paste address 6938ef7b9e1SHaren Myneni * such that the user space receives paste instruction failure. Then up to 6948ef7b9e1SHaren Myneni * the user space to fall back to SW compression and manage with the 6958ef7b9e1SHaren Myneni * existing windows. 6968ef7b9e1SHaren Myneni */ 697716d7a2eSHaren Myneni static int reconfig_close_windows(struct vas_caps *vcap, int excess_creds, 698716d7a2eSHaren Myneni bool migrate) 6998ef7b9e1SHaren Myneni { 7008ef7b9e1SHaren Myneni struct pseries_vas_window *win, *tmp; 7018ef7b9e1SHaren Myneni struct vas_user_win_ref *task_ref; 7028ef7b9e1SHaren Myneni struct vm_area_struct *vma; 703716d7a2eSHaren Myneni int rc = 0, flag; 704716d7a2eSHaren Myneni 705716d7a2eSHaren Myneni if (migrate) 706716d7a2eSHaren Myneni flag = VAS_WIN_MIGRATE_CLOSE; 707716d7a2eSHaren Myneni else 708716d7a2eSHaren Myneni flag = VAS_WIN_NO_CRED_CLOSE; 7098ef7b9e1SHaren Myneni 7108ef7b9e1SHaren Myneni list_for_each_entry_safe(win, tmp, &vcap->list, win_list) { 7118ef7b9e1SHaren Myneni /* 7128ef7b9e1SHaren Myneni * This window is already closed due to lost credit 713716d7a2eSHaren Myneni * or for migration before. Go for next window. 714716d7a2eSHaren Myneni * For migration, nothing to do since this window 715716d7a2eSHaren Myneni * closed for DLPAR and will be reopened even on 716716d7a2eSHaren Myneni * the destination system with other DLPAR operation. 7178ef7b9e1SHaren Myneni */ 718716d7a2eSHaren Myneni if ((win->vas_win.status & VAS_WIN_MIGRATE_CLOSE) || 719716d7a2eSHaren Myneni (win->vas_win.status & VAS_WIN_NO_CRED_CLOSE)) { 720716d7a2eSHaren Myneni win->vas_win.status |= flag; 7218ef7b9e1SHaren Myneni continue; 722716d7a2eSHaren Myneni } 7238ef7b9e1SHaren Myneni 7248ef7b9e1SHaren Myneni task_ref = &win->vas_win.task_ref; 7258ef7b9e1SHaren Myneni mutex_lock(&task_ref->mmap_mutex); 7268ef7b9e1SHaren Myneni vma = task_ref->vma; 7278ef7b9e1SHaren Myneni /* 7288ef7b9e1SHaren Myneni * Number of available credits are reduced, So select 7298ef7b9e1SHaren Myneni * and close windows. 7308ef7b9e1SHaren Myneni */ 731716d7a2eSHaren Myneni win->vas_win.status |= flag; 7328ef7b9e1SHaren Myneni 7338ef7b9e1SHaren Myneni mmap_write_lock(task_ref->mm); 7348ef7b9e1SHaren Myneni /* 7358ef7b9e1SHaren Myneni * vma is set in the original mapping. But this mapping 7368ef7b9e1SHaren Myneni * is done with mmap() after the window is opened with ioctl. 7378ef7b9e1SHaren Myneni * so we may not see the original mapping if the core remove 7388ef7b9e1SHaren Myneni * is done before the original mmap() and after the ioctl. 7398ef7b9e1SHaren Myneni */ 7408ef7b9e1SHaren Myneni if (vma) 7418ef7b9e1SHaren Myneni zap_page_range(vma, vma->vm_start, 7428ef7b9e1SHaren Myneni vma->vm_end - vma->vm_start); 7438ef7b9e1SHaren Myneni 7448ef7b9e1SHaren Myneni mmap_write_unlock(task_ref->mm); 7458ef7b9e1SHaren Myneni mutex_unlock(&task_ref->mmap_mutex); 7468ef7b9e1SHaren Myneni /* 7478ef7b9e1SHaren Myneni * Close VAS window in the hypervisor, but do not 7488ef7b9e1SHaren Myneni * free vas_window struct since it may be reused 7498ef7b9e1SHaren Myneni * when the credit is available later (DLPAR with 7508ef7b9e1SHaren Myneni * adding cores). This struct will be used 7518ef7b9e1SHaren Myneni * later when the process issued with close(FD). 7528ef7b9e1SHaren Myneni */ 7538ef7b9e1SHaren Myneni rc = deallocate_free_window(win); 754716d7a2eSHaren Myneni /* 755716d7a2eSHaren Myneni * This failure is from the hypervisor. 756716d7a2eSHaren Myneni * No way to stop migration for these failures. 757716d7a2eSHaren Myneni * So ignore error and continue closing other windows. 758716d7a2eSHaren Myneni */ 759716d7a2eSHaren Myneni if (rc && !migrate) 7608ef7b9e1SHaren Myneni return rc; 7618ef7b9e1SHaren Myneni 7628ef7b9e1SHaren Myneni vcap->nr_close_wins++; 7638ef7b9e1SHaren Myneni 764716d7a2eSHaren Myneni /* 765716d7a2eSHaren Myneni * For migration, do not depend on lpar_creds in case if 766716d7a2eSHaren Myneni * mismatch with the hypervisor value (should not happen). 767716d7a2eSHaren Myneni * So close all active windows in the list and will be 768716d7a2eSHaren Myneni * reopened windows based on the new lpar_creds on the 769716d7a2eSHaren Myneni * destination system during resume. 770716d7a2eSHaren Myneni */ 771716d7a2eSHaren Myneni if (!migrate && !--excess_creds) 7728ef7b9e1SHaren Myneni break; 7738ef7b9e1SHaren Myneni } 7748ef7b9e1SHaren Myneni 7758ef7b9e1SHaren Myneni return 0; 7768ef7b9e1SHaren Myneni } 7778ef7b9e1SHaren Myneni 7788ef7b9e1SHaren Myneni /* 7798ef7b9e1SHaren Myneni * Get new VAS capabilities when the core add/removal configuration 7808ef7b9e1SHaren Myneni * changes. Reconfig window configurations based on the credits 7818ef7b9e1SHaren Myneni * availability from this new capabilities. 7828ef7b9e1SHaren Myneni */ 78357831bfbSHaren Myneni int vas_reconfig_capabilties(u8 type, int new_nr_creds) 7848ef7b9e1SHaren Myneni { 7858ef7b9e1SHaren Myneni struct vas_cop_feat_caps *caps; 78657831bfbSHaren Myneni int old_nr_creds; 7878ef7b9e1SHaren Myneni struct vas_caps *vcaps; 7888ef7b9e1SHaren Myneni int rc = 0, nr_active_wins; 7898ef7b9e1SHaren Myneni 7908ef7b9e1SHaren Myneni if (type >= VAS_MAX_FEAT_TYPE) { 7918ef7b9e1SHaren Myneni pr_err("Invalid credit type %d\n", type); 7928ef7b9e1SHaren Myneni return -EINVAL; 7938ef7b9e1SHaren Myneni } 7948ef7b9e1SHaren Myneni 7958ef7b9e1SHaren Myneni vcaps = &vascaps[type]; 7968ef7b9e1SHaren Myneni caps = &vcaps->caps; 7978ef7b9e1SHaren Myneni 7988ef7b9e1SHaren Myneni mutex_lock(&vas_pseries_mutex); 7998ef7b9e1SHaren Myneni 8008ef7b9e1SHaren Myneni old_nr_creds = atomic_read(&caps->nr_total_credits); 8018ef7b9e1SHaren Myneni 8028ef7b9e1SHaren Myneni atomic_set(&caps->nr_total_credits, new_nr_creds); 8038ef7b9e1SHaren Myneni /* 8048ef7b9e1SHaren Myneni * The total number of available credits may be decreased or 8051fd02f66SJulia Lawall * increased with DLPAR operation. Means some windows have to be 8068ef7b9e1SHaren Myneni * closed / reopened. Hold the vas_pseries_mutex so that the 807*901a30cfSJason Wang * user space can not open new windows. 8088ef7b9e1SHaren Myneni */ 809c656cfe5SHaren Myneni if (old_nr_creds < new_nr_creds) { 810c656cfe5SHaren Myneni /* 811c656cfe5SHaren Myneni * If the existing target credits is less than the new 812c656cfe5SHaren Myneni * target, reopen windows if they are closed due to 813c656cfe5SHaren Myneni * the previous DLPAR (core removal). 814c656cfe5SHaren Myneni */ 815716d7a2eSHaren Myneni rc = reconfig_open_windows(vcaps, new_nr_creds - old_nr_creds, 816716d7a2eSHaren Myneni false); 817c656cfe5SHaren Myneni } else { 8188ef7b9e1SHaren Myneni /* 8198ef7b9e1SHaren Myneni * # active windows is more than new LPAR available 8208ef7b9e1SHaren Myneni * credits. So close the excessive windows. 8218ef7b9e1SHaren Myneni * On pseries, each window will have 1 credit. 8228ef7b9e1SHaren Myneni */ 8238ef7b9e1SHaren Myneni nr_active_wins = vcaps->nr_open_windows - vcaps->nr_close_wins; 8248ef7b9e1SHaren Myneni if (nr_active_wins > new_nr_creds) 8258ef7b9e1SHaren Myneni rc = reconfig_close_windows(vcaps, 826716d7a2eSHaren Myneni nr_active_wins - new_nr_creds, 827716d7a2eSHaren Myneni false); 8288ef7b9e1SHaren Myneni } 8298ef7b9e1SHaren Myneni 8308ef7b9e1SHaren Myneni mutex_unlock(&vas_pseries_mutex); 8318ef7b9e1SHaren Myneni return rc; 8328ef7b9e1SHaren Myneni } 8338ef7b9e1SHaren Myneni /* 8348ef7b9e1SHaren Myneni * Total number of default credits available (target_credits) 8358ef7b9e1SHaren Myneni * in LPAR depends on number of cores configured. It varies based on 8368ef7b9e1SHaren Myneni * whether processors are in shared mode or dedicated mode. 8378ef7b9e1SHaren Myneni * Get the notifier when CPU configuration is changed with DLPAR 8388ef7b9e1SHaren Myneni * operation so that get the new target_credits (vas default capabilities) 8398ef7b9e1SHaren Myneni * and then update the existing windows usage if needed. 8408ef7b9e1SHaren Myneni */ 8418ef7b9e1SHaren Myneni static int pseries_vas_notifier(struct notifier_block *nb, 8428ef7b9e1SHaren Myneni unsigned long action, void *data) 8438ef7b9e1SHaren Myneni { 8448ef7b9e1SHaren Myneni struct of_reconfig_data *rd = data; 8458ef7b9e1SHaren Myneni struct device_node *dn = rd->dn; 8468ef7b9e1SHaren Myneni const __be32 *intserv = NULL; 84757831bfbSHaren Myneni int new_nr_creds, len, rc = 0; 8488ef7b9e1SHaren Myneni 8498ef7b9e1SHaren Myneni if ((action == OF_RECONFIG_ATTACH_NODE) || 8508ef7b9e1SHaren Myneni (action == OF_RECONFIG_DETACH_NODE)) 8518ef7b9e1SHaren Myneni intserv = of_get_property(dn, "ibm,ppc-interrupt-server#s", 8528ef7b9e1SHaren Myneni &len); 8538ef7b9e1SHaren Myneni /* 8548ef7b9e1SHaren Myneni * Processor config is not changed 8558ef7b9e1SHaren Myneni */ 8568ef7b9e1SHaren Myneni if (!intserv) 8578ef7b9e1SHaren Myneni return NOTIFY_OK; 8588ef7b9e1SHaren Myneni 85957831bfbSHaren Myneni rc = h_query_vas_capabilities(H_QUERY_VAS_CAPABILITIES, 86057831bfbSHaren Myneni vascaps[VAS_GZIP_DEF_FEAT_TYPE].feat, 86157831bfbSHaren Myneni (u64)virt_to_phys(&hv_cop_caps)); 86257831bfbSHaren Myneni if (!rc) { 86357831bfbSHaren Myneni new_nr_creds = be16_to_cpu(hv_cop_caps.target_lpar_creds); 86457831bfbSHaren Myneni rc = vas_reconfig_capabilties(VAS_GZIP_DEF_FEAT_TYPE, 86557831bfbSHaren Myneni new_nr_creds); 86657831bfbSHaren Myneni } 86757831bfbSHaren Myneni 8688ef7b9e1SHaren Myneni if (rc) 8698ef7b9e1SHaren Myneni pr_err("Failed reconfig VAS capabilities with DLPAR\n"); 8708ef7b9e1SHaren Myneni 8718ef7b9e1SHaren Myneni return rc; 8728ef7b9e1SHaren Myneni } 8738ef7b9e1SHaren Myneni 8748ef7b9e1SHaren Myneni static struct notifier_block pseries_vas_nb = { 8758ef7b9e1SHaren Myneni .notifier_call = pseries_vas_notifier, 8768ef7b9e1SHaren Myneni }; 8778ef7b9e1SHaren Myneni 87837e67648SHaren Myneni /* 87937e67648SHaren Myneni * For LPM, all windows have to be closed on the source partition 88037e67648SHaren Myneni * before migration and reopen them on the destination partition 88137e67648SHaren Myneni * after migration. So closing windows during suspend and 88237e67648SHaren Myneni * reopen them during resume. 88337e67648SHaren Myneni */ 88437e67648SHaren Myneni int vas_migration_handler(int action) 88537e67648SHaren Myneni { 88637e67648SHaren Myneni struct vas_cop_feat_caps *caps; 88737e67648SHaren Myneni int old_nr_creds, new_nr_creds = 0; 88837e67648SHaren Myneni struct vas_caps *vcaps; 88937e67648SHaren Myneni int i, rc = 0; 89037e67648SHaren Myneni 89137e67648SHaren Myneni /* 89237e67648SHaren Myneni * NX-GZIP is not enabled. Nothing to do for migration. 89337e67648SHaren Myneni */ 89437e67648SHaren Myneni if (!copypaste_feat) 89537e67648SHaren Myneni return rc; 89637e67648SHaren Myneni 89737e67648SHaren Myneni mutex_lock(&vas_pseries_mutex); 89837e67648SHaren Myneni 89937e67648SHaren Myneni if (action == VAS_SUSPEND) 90037e67648SHaren Myneni migration_in_progress = true; 90137e67648SHaren Myneni else 90237e67648SHaren Myneni migration_in_progress = false; 90337e67648SHaren Myneni 90437e67648SHaren Myneni for (i = 0; i < VAS_MAX_FEAT_TYPE; i++) { 90537e67648SHaren Myneni vcaps = &vascaps[i]; 90637e67648SHaren Myneni caps = &vcaps->caps; 90737e67648SHaren Myneni old_nr_creds = atomic_read(&caps->nr_total_credits); 90837e67648SHaren Myneni 90937e67648SHaren Myneni rc = h_query_vas_capabilities(H_QUERY_VAS_CAPABILITIES, 91037e67648SHaren Myneni vcaps->feat, 91137e67648SHaren Myneni (u64)virt_to_phys(&hv_cop_caps)); 91237e67648SHaren Myneni if (!rc) { 91337e67648SHaren Myneni new_nr_creds = be16_to_cpu(hv_cop_caps.target_lpar_creds); 91437e67648SHaren Myneni /* 91537e67648SHaren Myneni * Should not happen. But incase print messages, close 91637e67648SHaren Myneni * all windows in the list during suspend and reopen 91737e67648SHaren Myneni * windows based on new lpar_creds on the destination 91837e67648SHaren Myneni * system. 91937e67648SHaren Myneni */ 92037e67648SHaren Myneni if (old_nr_creds != new_nr_creds) { 92137e67648SHaren Myneni pr_err("Target credits mismatch with the hypervisor\n"); 92237e67648SHaren Myneni pr_err("state(%d): lpar creds: %d HV lpar creds: %d\n", 92337e67648SHaren Myneni action, old_nr_creds, new_nr_creds); 92437e67648SHaren Myneni pr_err("Used creds: %d, Active creds: %d\n", 92537e67648SHaren Myneni atomic_read(&caps->nr_used_credits), 92637e67648SHaren Myneni vcaps->nr_open_windows - vcaps->nr_close_wins); 92737e67648SHaren Myneni } 92837e67648SHaren Myneni } else { 92937e67648SHaren Myneni pr_err("state(%d): Get VAS capabilities failed with %d\n", 93037e67648SHaren Myneni action, rc); 93137e67648SHaren Myneni /* 93237e67648SHaren Myneni * We can not stop migration with the current lpm 93337e67648SHaren Myneni * implementation. So continue closing all windows in 93437e67648SHaren Myneni * the list (during suspend) and return without 93537e67648SHaren Myneni * opening windows (during resume) if VAS capabilities 93637e67648SHaren Myneni * HCALL failed. 93737e67648SHaren Myneni */ 93837e67648SHaren Myneni if (action == VAS_RESUME) 93937e67648SHaren Myneni goto out; 94037e67648SHaren Myneni } 94137e67648SHaren Myneni 94237e67648SHaren Myneni switch (action) { 94337e67648SHaren Myneni case VAS_SUSPEND: 94437e67648SHaren Myneni rc = reconfig_close_windows(vcaps, vcaps->nr_open_windows, 94537e67648SHaren Myneni true); 94637e67648SHaren Myneni break; 94737e67648SHaren Myneni case VAS_RESUME: 94837e67648SHaren Myneni atomic_set(&caps->nr_total_credits, new_nr_creds); 94937e67648SHaren Myneni rc = reconfig_open_windows(vcaps, new_nr_creds, true); 95037e67648SHaren Myneni break; 95137e67648SHaren Myneni default: 95237e67648SHaren Myneni /* should not happen */ 95337e67648SHaren Myneni pr_err("Invalid migration action %d\n", action); 95437e67648SHaren Myneni rc = -EINVAL; 95537e67648SHaren Myneni goto out; 95637e67648SHaren Myneni } 95737e67648SHaren Myneni 95837e67648SHaren Myneni /* 95937e67648SHaren Myneni * Ignore errors during suspend and return for resume. 96037e67648SHaren Myneni */ 96137e67648SHaren Myneni if (rc && (action == VAS_RESUME)) 96237e67648SHaren Myneni goto out; 96337e67648SHaren Myneni } 96437e67648SHaren Myneni 96537e67648SHaren Myneni out: 96637e67648SHaren Myneni mutex_unlock(&vas_pseries_mutex); 96737e67648SHaren Myneni return rc; 96837e67648SHaren Myneni } 96937e67648SHaren Myneni 970ca77d488SHaren Myneni static int __init pseries_vas_init(void) 971ca77d488SHaren Myneni { 972ca77d488SHaren Myneni struct hv_vas_all_caps *hv_caps; 973278fe1ccSHaren Myneni int rc = 0; 974ca77d488SHaren Myneni 975ca77d488SHaren Myneni /* 976ca77d488SHaren Myneni * Linux supports user space COPY/PASTE only with Radix 977ca77d488SHaren Myneni */ 978ca77d488SHaren Myneni if (!radix_enabled()) { 979ca77d488SHaren Myneni pr_err("API is supported only with radix page tables\n"); 980ca77d488SHaren Myneni return -ENOTSUPP; 981ca77d488SHaren Myneni } 982ca77d488SHaren Myneni 983ca77d488SHaren Myneni hv_caps = kmalloc(sizeof(*hv_caps), GFP_KERNEL); 984ca77d488SHaren Myneni if (!hv_caps) 985ca77d488SHaren Myneni return -ENOMEM; 986ca77d488SHaren Myneni /* 987ca77d488SHaren Myneni * Get VAS overall capabilities by passing 0 to feature type. 988ca77d488SHaren Myneni */ 989ca77d488SHaren Myneni rc = h_query_vas_capabilities(H_QUERY_VAS_CAPABILITIES, 0, 990ca77d488SHaren Myneni (u64)virt_to_phys(hv_caps)); 991ca77d488SHaren Myneni if (rc) 992ca77d488SHaren Myneni goto out; 993ca77d488SHaren Myneni 994ca77d488SHaren Myneni caps_all.descriptor = be64_to_cpu(hv_caps->descriptor); 995ca77d488SHaren Myneni caps_all.feat_type = be64_to_cpu(hv_caps->feat_type); 996ca77d488SHaren Myneni 997b903737bSHaren Myneni sysfs_pseries_vas_init(&caps_all); 998b903737bSHaren Myneni 999ca77d488SHaren Myneni /* 1000ca77d488SHaren Myneni * QOS capabilities available 1001ca77d488SHaren Myneni */ 1002ca77d488SHaren Myneni if (caps_all.feat_type & VAS_GZIP_QOS_FEAT_BIT) { 1003ca77d488SHaren Myneni rc = get_vas_capabilities(VAS_GZIP_QOS_FEAT, 1004278fe1ccSHaren Myneni VAS_GZIP_QOS_FEAT_TYPE, &hv_cop_caps); 1005ca77d488SHaren Myneni 1006ca77d488SHaren Myneni if (rc) 1007278fe1ccSHaren Myneni goto out; 1008ca77d488SHaren Myneni } 1009ca77d488SHaren Myneni /* 1010ca77d488SHaren Myneni * Default capabilities available 1011ca77d488SHaren Myneni */ 1012278fe1ccSHaren Myneni if (caps_all.feat_type & VAS_GZIP_DEF_FEAT_BIT) 1013ca77d488SHaren Myneni rc = get_vas_capabilities(VAS_GZIP_DEF_FEAT, 1014278fe1ccSHaren Myneni VAS_GZIP_DEF_FEAT_TYPE, &hv_cop_caps); 1015ca77d488SHaren Myneni 1016278fe1ccSHaren Myneni if (!rc && copypaste_feat) { 1017278fe1ccSHaren Myneni if (firmware_has_feature(FW_FEATURE_LPAR)) 10188ef7b9e1SHaren Myneni of_reconfig_notifier_register(&pseries_vas_nb); 10198ef7b9e1SHaren Myneni 1020ca77d488SHaren Myneni pr_info("GZIP feature is available\n"); 1021278fe1ccSHaren Myneni } else { 1022278fe1ccSHaren Myneni /* 1023278fe1ccSHaren Myneni * Should not happen, but only when get default 1024278fe1ccSHaren Myneni * capabilities HCALL failed. So disable copy paste 1025278fe1ccSHaren Myneni * feature. 1026278fe1ccSHaren Myneni */ 1027278fe1ccSHaren Myneni copypaste_feat = false; 1028278fe1ccSHaren Myneni } 1029ca77d488SHaren Myneni 1030ca77d488SHaren Myneni out: 1031ca77d488SHaren Myneni kfree(hv_caps); 1032ca77d488SHaren Myneni return rc; 1033ca77d488SHaren Myneni } 1034ca77d488SHaren Myneni machine_device_initcall(pseries, pseries_vas_init); 1035