12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 24dea2d1aSSukadev Bhattiprolu /* 34dea2d1aSSukadev Bhattiprolu * Copyright 2016-17 IBM Corp. 44dea2d1aSSukadev Bhattiprolu */ 54dea2d1aSSukadev Bhattiprolu 6180fe15aSSukadev Bhattiprolu #define pr_fmt(fmt) "vas: " fmt 7180fe15aSSukadev Bhattiprolu 84dea2d1aSSukadev Bhattiprolu #include <linux/types.h> 94dea2d1aSSukadev Bhattiprolu #include <linux/mutex.h> 10180fe15aSSukadev Bhattiprolu #include <linux/slab.h> 11180fe15aSSukadev Bhattiprolu #include <linux/io.h> 12b25b33acSSukadev Bhattiprolu #include <linux/log2.h> 1362c4eda4SSukadev Bhattiprolu #include <linux/rcupdate.h> 1462c4eda4SSukadev Bhattiprolu #include <linux/cred.h> 15db1c08a7SHaren Myneni #include <linux/sched/mm.h> 16db1c08a7SHaren Myneni #include <linux/mmu_context.h> 176c8e6bb2SSukadev Bhattiprolu #include <asm/switch_to.h> 186c8e6bb2SSukadev Bhattiprolu #include <asm/ppc-opcode.h> 19*1a0d0d5eSHaren Myneni #include <asm/vas.h> 204dea2d1aSSukadev Bhattiprolu #include "vas.h" 212392c8c8SSukadev Bhattiprolu #include "copy-paste.h" 224dea2d1aSSukadev Bhattiprolu 23007bb7d6SSukadev Bhattiprolu #define CREATE_TRACE_POINTS 24007bb7d6SSukadev Bhattiprolu #include "vas-trace.h" 25007bb7d6SSukadev Bhattiprolu 26180fe15aSSukadev Bhattiprolu /* 27180fe15aSSukadev Bhattiprolu * Compute the paste address region for the window @window using the 28180fe15aSSukadev Bhattiprolu * ->paste_base_addr and ->paste_win_id_shift we got from device tree. 29180fe15aSSukadev Bhattiprolu */ 30dda44eb2SHaren Myneni void vas_win_paste_addr(struct vas_window *window, u64 *addr, int *len) 31180fe15aSSukadev Bhattiprolu { 32180fe15aSSukadev Bhattiprolu int winid; 33180fe15aSSukadev Bhattiprolu u64 base, shift; 34180fe15aSSukadev Bhattiprolu 35180fe15aSSukadev Bhattiprolu base = window->vinst->paste_base_addr; 36180fe15aSSukadev Bhattiprolu shift = window->vinst->paste_win_id_shift; 37180fe15aSSukadev Bhattiprolu winid = window->winid; 38180fe15aSSukadev Bhattiprolu 39180fe15aSSukadev Bhattiprolu *addr = base + (winid << shift); 40180fe15aSSukadev Bhattiprolu if (len) 41180fe15aSSukadev Bhattiprolu *len = PAGE_SIZE; 42180fe15aSSukadev Bhattiprolu 43180fe15aSSukadev Bhattiprolu pr_debug("Txwin #%d: Paste addr 0x%llx\n", winid, *addr); 44180fe15aSSukadev Bhattiprolu } 45180fe15aSSukadev Bhattiprolu 46180fe15aSSukadev Bhattiprolu static inline void get_hvwc_mmio_bar(struct vas_window *window, 47180fe15aSSukadev Bhattiprolu u64 *start, int *len) 48180fe15aSSukadev Bhattiprolu { 49180fe15aSSukadev Bhattiprolu u64 pbaddr; 50180fe15aSSukadev Bhattiprolu 51180fe15aSSukadev Bhattiprolu pbaddr = window->vinst->hvwc_bar_start; 52180fe15aSSukadev Bhattiprolu *start = pbaddr + window->winid * VAS_HVWC_SIZE; 53180fe15aSSukadev Bhattiprolu *len = VAS_HVWC_SIZE; 54180fe15aSSukadev Bhattiprolu } 55180fe15aSSukadev Bhattiprolu 56180fe15aSSukadev Bhattiprolu static inline void get_uwc_mmio_bar(struct vas_window *window, 57180fe15aSSukadev Bhattiprolu u64 *start, int *len) 58180fe15aSSukadev Bhattiprolu { 59180fe15aSSukadev Bhattiprolu u64 pbaddr; 60180fe15aSSukadev Bhattiprolu 61180fe15aSSukadev Bhattiprolu pbaddr = window->vinst->uwc_bar_start; 62180fe15aSSukadev Bhattiprolu *start = pbaddr + window->winid * VAS_UWC_SIZE; 63180fe15aSSukadev Bhattiprolu *len = VAS_UWC_SIZE; 64180fe15aSSukadev Bhattiprolu } 65180fe15aSSukadev Bhattiprolu 66180fe15aSSukadev Bhattiprolu /* 67180fe15aSSukadev Bhattiprolu * Map the paste bus address of the given send window into kernel address 68180fe15aSSukadev Bhattiprolu * space. Unlike MMIO regions (map_mmio_region() below), paste region must 69180fe15aSSukadev Bhattiprolu * be mapped cache-able and is only applicable to send windows. 70180fe15aSSukadev Bhattiprolu */ 715239af67SSukadev Bhattiprolu static void *map_paste_region(struct vas_window *txwin) 72180fe15aSSukadev Bhattiprolu { 73180fe15aSSukadev Bhattiprolu int len; 74180fe15aSSukadev Bhattiprolu void *map; 75180fe15aSSukadev Bhattiprolu char *name; 76180fe15aSSukadev Bhattiprolu u64 start; 77180fe15aSSukadev Bhattiprolu 78180fe15aSSukadev Bhattiprolu name = kasprintf(GFP_KERNEL, "window-v%d-w%d", txwin->vinst->vas_id, 79180fe15aSSukadev Bhattiprolu txwin->winid); 80180fe15aSSukadev Bhattiprolu if (!name) 81180fe15aSSukadev Bhattiprolu goto free_name; 82180fe15aSSukadev Bhattiprolu 83180fe15aSSukadev Bhattiprolu txwin->paste_addr_name = name; 84dda44eb2SHaren Myneni vas_win_paste_addr(txwin, &start, &len); 85180fe15aSSukadev Bhattiprolu 86180fe15aSSukadev Bhattiprolu if (!request_mem_region(start, len, name)) { 87180fe15aSSukadev Bhattiprolu pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n", 88180fe15aSSukadev Bhattiprolu __func__, start, len); 89180fe15aSSukadev Bhattiprolu goto free_name; 90180fe15aSSukadev Bhattiprolu } 91180fe15aSSukadev Bhattiprolu 92180fe15aSSukadev Bhattiprolu map = ioremap_cache(start, len); 93180fe15aSSukadev Bhattiprolu if (!map) { 94180fe15aSSukadev Bhattiprolu pr_devel("%s(): ioremap_cache(0x%llx, %d) failed\n", __func__, 95180fe15aSSukadev Bhattiprolu start, len); 96180fe15aSSukadev Bhattiprolu goto free_name; 97180fe15aSSukadev Bhattiprolu } 98180fe15aSSukadev Bhattiprolu 99180fe15aSSukadev Bhattiprolu pr_devel("Mapped paste addr 0x%llx to kaddr 0x%p\n", start, map); 100180fe15aSSukadev Bhattiprolu return map; 101180fe15aSSukadev Bhattiprolu 102180fe15aSSukadev Bhattiprolu free_name: 103180fe15aSSukadev Bhattiprolu kfree(name); 104180fe15aSSukadev Bhattiprolu return ERR_PTR(-ENOMEM); 105180fe15aSSukadev Bhattiprolu } 106180fe15aSSukadev Bhattiprolu 107180fe15aSSukadev Bhattiprolu static void *map_mmio_region(char *name, u64 start, int len) 108180fe15aSSukadev Bhattiprolu { 109180fe15aSSukadev Bhattiprolu void *map; 110180fe15aSSukadev Bhattiprolu 111180fe15aSSukadev Bhattiprolu if (!request_mem_region(start, len, name)) { 112180fe15aSSukadev Bhattiprolu pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n", 113180fe15aSSukadev Bhattiprolu __func__, start, len); 114180fe15aSSukadev Bhattiprolu return NULL; 115180fe15aSSukadev Bhattiprolu } 116180fe15aSSukadev Bhattiprolu 117180fe15aSSukadev Bhattiprolu map = ioremap(start, len); 118180fe15aSSukadev Bhattiprolu if (!map) { 119180fe15aSSukadev Bhattiprolu pr_devel("%s(): ioremap(0x%llx, %d) failed\n", __func__, start, 120180fe15aSSukadev Bhattiprolu len); 121180fe15aSSukadev Bhattiprolu return NULL; 122180fe15aSSukadev Bhattiprolu } 123180fe15aSSukadev Bhattiprolu 124180fe15aSSukadev Bhattiprolu return map; 125180fe15aSSukadev Bhattiprolu } 126180fe15aSSukadev Bhattiprolu 127180fe15aSSukadev Bhattiprolu static void unmap_region(void *addr, u64 start, int len) 128180fe15aSSukadev Bhattiprolu { 129180fe15aSSukadev Bhattiprolu iounmap(addr); 130180fe15aSSukadev Bhattiprolu release_mem_region((phys_addr_t)start, len); 131180fe15aSSukadev Bhattiprolu } 132180fe15aSSukadev Bhattiprolu 133180fe15aSSukadev Bhattiprolu /* 134180fe15aSSukadev Bhattiprolu * Unmap the paste address region for a window. 135180fe15aSSukadev Bhattiprolu */ 13698271d41SSukadev Bhattiprolu static void unmap_paste_region(struct vas_window *window) 137180fe15aSSukadev Bhattiprolu { 138180fe15aSSukadev Bhattiprolu int len; 139180fe15aSSukadev Bhattiprolu u64 busaddr_start; 140180fe15aSSukadev Bhattiprolu 141180fe15aSSukadev Bhattiprolu if (window->paste_kaddr) { 142dda44eb2SHaren Myneni vas_win_paste_addr(window, &busaddr_start, &len); 143180fe15aSSukadev Bhattiprolu unmap_region(window->paste_kaddr, busaddr_start, len); 144180fe15aSSukadev Bhattiprolu window->paste_kaddr = NULL; 145180fe15aSSukadev Bhattiprolu kfree(window->paste_addr_name); 146180fe15aSSukadev Bhattiprolu window->paste_addr_name = NULL; 147180fe15aSSukadev Bhattiprolu } 148180fe15aSSukadev Bhattiprolu } 149180fe15aSSukadev Bhattiprolu 150180fe15aSSukadev Bhattiprolu /* 151ece4e512SSukadev Bhattiprolu * Unmap the MMIO regions for a window. Hold the vas_mutex so we don't 152ece4e512SSukadev Bhattiprolu * unmap when the window's debugfs dir is in use. This serializes close 153ece4e512SSukadev Bhattiprolu * of a window even on another VAS instance but since its not a critical 154ece4e512SSukadev Bhattiprolu * path, just minimize the time we hold the mutex for now. We can add 155ece4e512SSukadev Bhattiprolu * a per-instance mutex later if necessary. 156180fe15aSSukadev Bhattiprolu */ 157180fe15aSSukadev Bhattiprolu static void unmap_winctx_mmio_bars(struct vas_window *window) 158180fe15aSSukadev Bhattiprolu { 159180fe15aSSukadev Bhattiprolu int len; 160ece4e512SSukadev Bhattiprolu void *uwc_map; 161ece4e512SSukadev Bhattiprolu void *hvwc_map; 162180fe15aSSukadev Bhattiprolu u64 busaddr_start; 163180fe15aSSukadev Bhattiprolu 164ece4e512SSukadev Bhattiprolu mutex_lock(&vas_mutex); 165ece4e512SSukadev Bhattiprolu 166ece4e512SSukadev Bhattiprolu hvwc_map = window->hvwc_map; 167180fe15aSSukadev Bhattiprolu window->hvwc_map = NULL; 168ece4e512SSukadev Bhattiprolu 169ece4e512SSukadev Bhattiprolu uwc_map = window->uwc_map; 170ece4e512SSukadev Bhattiprolu window->uwc_map = NULL; 171ece4e512SSukadev Bhattiprolu 172ece4e512SSukadev Bhattiprolu mutex_unlock(&vas_mutex); 173ece4e512SSukadev Bhattiprolu 174ece4e512SSukadev Bhattiprolu if (hvwc_map) { 175ece4e512SSukadev Bhattiprolu get_hvwc_mmio_bar(window, &busaddr_start, &len); 176ece4e512SSukadev Bhattiprolu unmap_region(hvwc_map, busaddr_start, len); 177180fe15aSSukadev Bhattiprolu } 178180fe15aSSukadev Bhattiprolu 179ece4e512SSukadev Bhattiprolu if (uwc_map) { 180180fe15aSSukadev Bhattiprolu get_uwc_mmio_bar(window, &busaddr_start, &len); 181ece4e512SSukadev Bhattiprolu unmap_region(uwc_map, busaddr_start, len); 182180fe15aSSukadev Bhattiprolu } 183180fe15aSSukadev Bhattiprolu } 184180fe15aSSukadev Bhattiprolu 185180fe15aSSukadev Bhattiprolu /* 186180fe15aSSukadev Bhattiprolu * Find the Hypervisor Window Context (HVWC) MMIO Base Address Region and the 187180fe15aSSukadev Bhattiprolu * OS/User Window Context (UWC) MMIO Base Address Region for the given window. 188180fe15aSSukadev Bhattiprolu * Map these bus addresses and save the mapped kernel addresses in @window. 189180fe15aSSukadev Bhattiprolu */ 1903b70464aSOliver O'Halloran static int map_winctx_mmio_bars(struct vas_window *window) 191180fe15aSSukadev Bhattiprolu { 192180fe15aSSukadev Bhattiprolu int len; 193180fe15aSSukadev Bhattiprolu u64 start; 194180fe15aSSukadev Bhattiprolu 195180fe15aSSukadev Bhattiprolu get_hvwc_mmio_bar(window, &start, &len); 196180fe15aSSukadev Bhattiprolu window->hvwc_map = map_mmio_region("HVWCM_Window", start, len); 197180fe15aSSukadev Bhattiprolu 198180fe15aSSukadev Bhattiprolu get_uwc_mmio_bar(window, &start, &len); 199180fe15aSSukadev Bhattiprolu window->uwc_map = map_mmio_region("UWCM_Window", start, len); 200180fe15aSSukadev Bhattiprolu 201180fe15aSSukadev Bhattiprolu if (!window->hvwc_map || !window->uwc_map) { 202180fe15aSSukadev Bhattiprolu unmap_winctx_mmio_bars(window); 203180fe15aSSukadev Bhattiprolu return -1; 204180fe15aSSukadev Bhattiprolu } 205180fe15aSSukadev Bhattiprolu 206180fe15aSSukadev Bhattiprolu return 0; 207180fe15aSSukadev Bhattiprolu } 208180fe15aSSukadev Bhattiprolu 209b25b33acSSukadev Bhattiprolu /* 210b25b33acSSukadev Bhattiprolu * Reset all valid registers in the HV and OS/User Window Contexts for 211b25b33acSSukadev Bhattiprolu * the window identified by @window. 212b25b33acSSukadev Bhattiprolu * 213b25b33acSSukadev Bhattiprolu * NOTE: We cannot really use a for loop to reset window context. Not all 214b25b33acSSukadev Bhattiprolu * offsets in a window context are valid registers and the valid 215b25b33acSSukadev Bhattiprolu * registers are not sequential. And, we can only write to offsets 216b25b33acSSukadev Bhattiprolu * with valid registers. 217b25b33acSSukadev Bhattiprolu */ 2183b70464aSOliver O'Halloran static void reset_window_regs(struct vas_window *window) 219b25b33acSSukadev Bhattiprolu { 220b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LPID), 0ULL); 221b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(PID), 0ULL); 222b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(XLATE_MSR), 0ULL); 223b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(XLATE_LPCR), 0ULL); 224b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(XLATE_CTL), 0ULL); 225b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(AMR), 0ULL); 226b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(SEIDR), 0ULL); 227b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(FAULT_TX_WIN), 0ULL); 228b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL); 229b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), 0ULL); 230b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(PSWID), 0ULL); 231b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LFIFO_BAR), 0ULL); 232b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), 0ULL); 233b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), 0ULL); 234b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL); 235b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL); 236b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL); 237b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LRX_WCRED), 0ULL); 238b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL); 239b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(TX_WCRED), 0ULL); 240b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL); 241b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LFIFO_SIZE), 0ULL); 242b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(WINCTL), 0ULL); 243b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL); 244b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), 0ULL); 245b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(TX_RSVD_BUF_COUNT), 0ULL); 246b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), 0ULL); 247b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LNOTIFY_CTL), 0ULL); 248b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LNOTIFY_PID), 0ULL); 249b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LNOTIFY_LPID), 0ULL); 250b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LNOTIFY_TID), 0ULL); 251b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), 0ULL); 252b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL); 253b25b33acSSukadev Bhattiprolu 254b25b33acSSukadev Bhattiprolu /* Skip read-only registers: NX_UTIL and NX_UTIL_SE */ 255b25b33acSSukadev Bhattiprolu 256b25b33acSSukadev Bhattiprolu /* 257b25b33acSSukadev Bhattiprolu * The send and receive window credit adder registers are also 258b25b33acSSukadev Bhattiprolu * accessible from HVWC and have been initialized above. We don't 259b25b33acSSukadev Bhattiprolu * need to initialize from the OS/User Window Context, so skip 260b25b33acSSukadev Bhattiprolu * following calls: 261b25b33acSSukadev Bhattiprolu * 262b25b33acSSukadev Bhattiprolu * write_uwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL); 263b25b33acSSukadev Bhattiprolu * write_uwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL); 264b25b33acSSukadev Bhattiprolu */ 265b25b33acSSukadev Bhattiprolu } 266b25b33acSSukadev Bhattiprolu 267b25b33acSSukadev Bhattiprolu /* 268b25b33acSSukadev Bhattiprolu * Initialize window context registers related to Address Translation. 269b25b33acSSukadev Bhattiprolu * These registers are common to send/receive windows although they 270b25b33acSSukadev Bhattiprolu * differ for user/kernel windows. As we resolve the TODOs we may 271b25b33acSSukadev Bhattiprolu * want to add fields to vas_winctx and move the initialization to 272b25b33acSSukadev Bhattiprolu * init_vas_winctx_regs(). 273b25b33acSSukadev Bhattiprolu */ 274b25b33acSSukadev Bhattiprolu static void init_xlate_regs(struct vas_window *window, bool user_win) 275b25b33acSSukadev Bhattiprolu { 276b25b33acSSukadev Bhattiprolu u64 lpcr, val; 277b25b33acSSukadev Bhattiprolu 278b25b33acSSukadev Bhattiprolu /* 279b25b33acSSukadev Bhattiprolu * MSR_TA, MSR_US are false for both kernel and user. 280b25b33acSSukadev Bhattiprolu * MSR_DR and MSR_PR are false for kernel. 281b25b33acSSukadev Bhattiprolu */ 282b25b33acSSukadev Bhattiprolu val = 0ULL; 283b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_XLATE_MSR_HV, val, 1); 284b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_XLATE_MSR_SF, val, 1); 285b25b33acSSukadev Bhattiprolu if (user_win) { 286b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_XLATE_MSR_DR, val, 1); 287b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_XLATE_MSR_PR, val, 1); 288b25b33acSSukadev Bhattiprolu } 289b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(XLATE_MSR), val); 290b25b33acSSukadev Bhattiprolu 291b25b33acSSukadev Bhattiprolu lpcr = mfspr(SPRN_LPCR); 292b25b33acSSukadev Bhattiprolu val = 0ULL; 293b25b33acSSukadev Bhattiprolu /* 294b25b33acSSukadev Bhattiprolu * NOTE: From Section 5.7.8.1 Segment Lookaside Buffer of the 295b25b33acSSukadev Bhattiprolu * Power ISA, v3.0B, Page size encoding is 0 = 4KB, 5 = 64KB. 296b25b33acSSukadev Bhattiprolu * 297b25b33acSSukadev Bhattiprolu * NOTE: From Section 1.3.1, Address Translation Context of the 298b25b33acSSukadev Bhattiprolu * Nest MMU Workbook, LPCR_SC should be 0 for Power9. 299b25b33acSSukadev Bhattiprolu */ 300b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_XLATE_LPCR_PAGE_SIZE, val, 5); 301b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_XLATE_LPCR_ISL, val, lpcr & LPCR_ISL); 302b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_XLATE_LPCR_TC, val, lpcr & LPCR_TC); 303b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_XLATE_LPCR_SC, val, 0); 304b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(XLATE_LPCR), val); 305b25b33acSSukadev Bhattiprolu 306b25b33acSSukadev Bhattiprolu /* 307b25b33acSSukadev Bhattiprolu * Section 1.3.1 (Address translation Context) of NMMU workbook. 308b25b33acSSukadev Bhattiprolu * 0b00 Hashed Page Table mode 309b25b33acSSukadev Bhattiprolu * 0b01 Reserved 310b25b33acSSukadev Bhattiprolu * 0b10 Radix on HPT 311b25b33acSSukadev Bhattiprolu * 0b11 Radix on Radix 312b25b33acSSukadev Bhattiprolu */ 313b25b33acSSukadev Bhattiprolu val = 0ULL; 314b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_XLATE_MODE, val, radix_enabled() ? 3 : 2); 315b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(XLATE_CTL), val); 316b25b33acSSukadev Bhattiprolu 317b25b33acSSukadev Bhattiprolu /* 318b25b33acSSukadev Bhattiprolu * TODO: Can we mfspr(AMR) even for user windows? 319b25b33acSSukadev Bhattiprolu */ 320b25b33acSSukadev Bhattiprolu val = 0ULL; 321b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_AMR, val, mfspr(SPRN_AMR)); 322b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(AMR), val); 323b25b33acSSukadev Bhattiprolu 324b25b33acSSukadev Bhattiprolu val = 0ULL; 325b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_SEIDR, val, 0); 326b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(SEIDR), val); 327b25b33acSSukadev Bhattiprolu } 328b25b33acSSukadev Bhattiprolu 329b25b33acSSukadev Bhattiprolu /* 330b25b33acSSukadev Bhattiprolu * Initialize Reserved Send Buffer Count for the send window. It involves 331b25b33acSSukadev Bhattiprolu * writing to the register, reading it back to confirm that the hardware 332b25b33acSSukadev Bhattiprolu * has enough buffers to reserve. See section 1.3.1.2.1 of VAS workbook. 333b25b33acSSukadev Bhattiprolu * 334b25b33acSSukadev Bhattiprolu * Since we can only make a best-effort attempt to fulfill the request, 335b25b33acSSukadev Bhattiprolu * we don't return any errors if we cannot. 336b25b33acSSukadev Bhattiprolu * 337b25b33acSSukadev Bhattiprolu * TODO: Reserved (aka dedicated) send buffers are not supported yet. 338b25b33acSSukadev Bhattiprolu */ 339b25b33acSSukadev Bhattiprolu static void init_rsvd_tx_buf_count(struct vas_window *txwin, 340b25b33acSSukadev Bhattiprolu struct vas_winctx *winctx) 341b25b33acSSukadev Bhattiprolu { 342b25b33acSSukadev Bhattiprolu write_hvwc_reg(txwin, VREG(TX_RSVD_BUF_COUNT), 0ULL); 343b25b33acSSukadev Bhattiprolu } 344b25b33acSSukadev Bhattiprolu 345b25b33acSSukadev Bhattiprolu /* 346b25b33acSSukadev Bhattiprolu * init_winctx_regs() 347b25b33acSSukadev Bhattiprolu * Initialize window context registers for a receive window. 348b25b33acSSukadev Bhattiprolu * Except for caching control and marking window open, the registers 349b25b33acSSukadev Bhattiprolu * are initialized in the order listed in Section 3.1.4 (Window Context 350b25b33acSSukadev Bhattiprolu * Cache Register Details) of the VAS workbook although they don't need 351b25b33acSSukadev Bhattiprolu * to be. 352b25b33acSSukadev Bhattiprolu * 353b25b33acSSukadev Bhattiprolu * Design note: For NX receive windows, NX allocates the FIFO buffer in OPAL 354b25b33acSSukadev Bhattiprolu * (so that it can get a large contiguous area) and passes that buffer 355b25b33acSSukadev Bhattiprolu * to kernel via device tree. We now write that buffer address to the 356b25b33acSSukadev Bhattiprolu * FIFO BAR. Would it make sense to do this all in OPAL? i.e have OPAL 357b25b33acSSukadev Bhattiprolu * write the per-chip RX FIFO addresses to the windows during boot-up 358b25b33acSSukadev Bhattiprolu * as a one-time task? That could work for NX but what about other 359b25b33acSSukadev Bhattiprolu * receivers? Let the receivers tell us the rx-fifo buffers for now. 360b25b33acSSukadev Bhattiprolu */ 3613b70464aSOliver O'Halloran static void init_winctx_regs(struct vas_window *window, 3623b70464aSOliver O'Halloran struct vas_winctx *winctx) 363b25b33acSSukadev Bhattiprolu { 364b25b33acSSukadev Bhattiprolu u64 val; 365b25b33acSSukadev Bhattiprolu int fifo_size; 366b25b33acSSukadev Bhattiprolu 367b25b33acSSukadev Bhattiprolu reset_window_regs(window); 368b25b33acSSukadev Bhattiprolu 369b25b33acSSukadev Bhattiprolu val = 0ULL; 370b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_LPID, val, winctx->lpid); 371b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LPID), val); 372b25b33acSSukadev Bhattiprolu 373b25b33acSSukadev Bhattiprolu val = 0ULL; 374b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_PID_ID, val, winctx->pidr); 375b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(PID), val); 376b25b33acSSukadev Bhattiprolu 377b25b33acSSukadev Bhattiprolu init_xlate_regs(window, winctx->user_win); 378b25b33acSSukadev Bhattiprolu 379b25b33acSSukadev Bhattiprolu val = 0ULL; 3808b8a73dcSHaren Myneni val = SET_FIELD(VAS_FAULT_TX_WIN, val, winctx->fault_win_id); 381b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(FAULT_TX_WIN), val); 382b25b33acSSukadev Bhattiprolu 383b25b33acSSukadev Bhattiprolu /* In PowerNV, interrupts go to HV. */ 384b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL); 385b25b33acSSukadev Bhattiprolu 386b25b33acSSukadev Bhattiprolu val = 0ULL; 387b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_HV_INTR_SRC_RA, val, winctx->irq_port); 388b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), val); 389b25b33acSSukadev Bhattiprolu 390b25b33acSSukadev Bhattiprolu val = 0ULL; 391b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_PSWID_EA_HANDLE, val, winctx->pswid); 392b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(PSWID), val); 393b25b33acSSukadev Bhattiprolu 394b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(SPARE1), 0ULL); 395b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(SPARE2), 0ULL); 396b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(SPARE3), 0ULL); 397b25b33acSSukadev Bhattiprolu 398b25b33acSSukadev Bhattiprolu /* 399b25b33acSSukadev Bhattiprolu * NOTE: VAS expects the FIFO address to be copied into the LFIFO_BAR 400b25b33acSSukadev Bhattiprolu * register as is - do NOT shift the address into VAS_LFIFO_BAR 401b25b33acSSukadev Bhattiprolu * bit fields! Ok to set the page migration select fields - 402b25b33acSSukadev Bhattiprolu * VAS ignores the lower 10+ bits in the address anyway, because 403b25b33acSSukadev Bhattiprolu * the minimum FIFO size is 1K? 404b25b33acSSukadev Bhattiprolu * 405b25b33acSSukadev Bhattiprolu * See also: Design note in function header. 406b25b33acSSukadev Bhattiprolu */ 407b25b33acSSukadev Bhattiprolu val = __pa(winctx->rx_fifo); 408b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_PAGE_MIGRATION_SELECT, val, 0); 409b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LFIFO_BAR), val); 410b25b33acSSukadev Bhattiprolu 411b25b33acSSukadev Bhattiprolu val = 0ULL; 412b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_LDATA_STAMP, val, winctx->data_stamp); 413b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), val); 414b25b33acSSukadev Bhattiprolu 415b25b33acSSukadev Bhattiprolu val = 0ULL; 416b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_LDMA_TYPE, val, winctx->dma_type); 417b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_LDMA_FIFO_DISABLE, val, winctx->fifo_disable); 418b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), val); 419b25b33acSSukadev Bhattiprolu 420b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL); 421b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL); 422b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL); 423b25b33acSSukadev Bhattiprolu 424b25b33acSSukadev Bhattiprolu val = 0ULL; 425b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_LRX_WCRED, val, winctx->wcreds_max); 426b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LRX_WCRED), val); 427b25b33acSSukadev Bhattiprolu 428b25b33acSSukadev Bhattiprolu val = 0ULL; 429b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_TX_WCRED, val, winctx->wcreds_max); 430b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(TX_WCRED), val); 431b25b33acSSukadev Bhattiprolu 432b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL); 433b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL); 434b25b33acSSukadev Bhattiprolu 435b25b33acSSukadev Bhattiprolu fifo_size = winctx->rx_fifo_size / 1024; 436b25b33acSSukadev Bhattiprolu 437b25b33acSSukadev Bhattiprolu val = 0ULL; 438b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_LFIFO_SIZE, val, ilog2(fifo_size)); 439b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LFIFO_SIZE), val); 440b25b33acSSukadev Bhattiprolu 441b25b33acSSukadev Bhattiprolu /* Update window control and caching control registers last so 442b25b33acSSukadev Bhattiprolu * we mark the window open only after fully initializing it and 443b25b33acSSukadev Bhattiprolu * pushing context to cache. 444b25b33acSSukadev Bhattiprolu */ 445b25b33acSSukadev Bhattiprolu 446b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL); 447b25b33acSSukadev Bhattiprolu 448b25b33acSSukadev Bhattiprolu init_rsvd_tx_buf_count(window, winctx); 449b25b33acSSukadev Bhattiprolu 450b25b33acSSukadev Bhattiprolu /* for a send window, point to the matching receive window */ 451b25b33acSSukadev Bhattiprolu val = 0ULL; 452b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_LRX_WIN_ID, val, winctx->rx_win_id); 453b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), val); 454b25b33acSSukadev Bhattiprolu 455b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(SPARE4), 0ULL); 456b25b33acSSukadev Bhattiprolu 457b25b33acSSukadev Bhattiprolu val = 0ULL; 458b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_NOTIFY_DISABLE, val, winctx->notify_disable); 459b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_INTR_DISABLE, val, winctx->intr_disable); 460b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_NOTIFY_EARLY, val, winctx->notify_early); 461b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_NOTIFY_OSU_INTR, val, winctx->notify_os_intr_reg); 462b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LNOTIFY_CTL), val); 463b25b33acSSukadev Bhattiprolu 464b25b33acSSukadev Bhattiprolu val = 0ULL; 465b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_LNOTIFY_PID, val, winctx->lnotify_pid); 466b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LNOTIFY_PID), val); 467b25b33acSSukadev Bhattiprolu 468b25b33acSSukadev Bhattiprolu val = 0ULL; 469b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_LNOTIFY_LPID, val, winctx->lnotify_lpid); 470b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LNOTIFY_LPID), val); 471b25b33acSSukadev Bhattiprolu 472b25b33acSSukadev Bhattiprolu val = 0ULL; 473b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_LNOTIFY_TID, val, winctx->lnotify_tid); 474b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LNOTIFY_TID), val); 475b25b33acSSukadev Bhattiprolu 476b25b33acSSukadev Bhattiprolu val = 0ULL; 477b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_LNOTIFY_MIN_SCOPE, val, winctx->min_scope); 478b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_LNOTIFY_MAX_SCOPE, val, winctx->max_scope); 479b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), val); 480b25b33acSSukadev Bhattiprolu 481b25b33acSSukadev Bhattiprolu /* Skip read-only registers NX_UTIL and NX_UTIL_SE */ 482b25b33acSSukadev Bhattiprolu 483b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(SPARE5), 0ULL); 484b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL); 485b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(SPARE6), 0ULL); 486b25b33acSSukadev Bhattiprolu 487b25b33acSSukadev Bhattiprolu /* Finally, push window context to memory and... */ 488b25b33acSSukadev Bhattiprolu val = 0ULL; 489b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_PUSH_TO_MEM, val, 1); 490b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), val); 491b25b33acSSukadev Bhattiprolu 492b25b33acSSukadev Bhattiprolu /* ... mark the window open for business */ 493b25b33acSSukadev Bhattiprolu val = 0ULL; 494b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_WINCTL_REJ_NO_CREDIT, val, winctx->rej_no_credit); 495b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_WINCTL_PIN, val, winctx->pin_win); 496b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val, winctx->tx_wcred_mode); 497b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val, winctx->rx_wcred_mode); 498b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_WINCTL_TX_WORD_MODE, val, winctx->tx_word_mode); 499b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_WINCTL_RX_WORD_MODE, val, winctx->rx_word_mode); 500b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_WINCTL_FAULT_WIN, val, winctx->fault_win); 501b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_WINCTL_NX_WIN, val, winctx->nx_win); 502b25b33acSSukadev Bhattiprolu val = SET_FIELD(VAS_WINCTL_OPEN, val, 1); 503b25b33acSSukadev Bhattiprolu write_hvwc_reg(window, VREG(WINCTL), val); 504b25b33acSSukadev Bhattiprolu } 505b25b33acSSukadev Bhattiprolu 506bbfe59f8SSukadev Bhattiprolu static void vas_release_window_id(struct ida *ida, int winid) 507bbfe59f8SSukadev Bhattiprolu { 508cd38049eSMatthew Wilcox ida_free(ida, winid); 509bbfe59f8SSukadev Bhattiprolu } 510bbfe59f8SSukadev Bhattiprolu 511bbfe59f8SSukadev Bhattiprolu static int vas_assign_window_id(struct ida *ida) 512bbfe59f8SSukadev Bhattiprolu { 513cd38049eSMatthew Wilcox int winid = ida_alloc_max(ida, VAS_WINDOWS_PER_CHIP - 1, GFP_KERNEL); 514bbfe59f8SSukadev Bhattiprolu 515cd38049eSMatthew Wilcox if (winid == -ENOSPC) { 516cd38049eSMatthew Wilcox pr_err("Too many (%d) open windows\n", VAS_WINDOWS_PER_CHIP); 517bbfe59f8SSukadev Bhattiprolu return -EAGAIN; 518bbfe59f8SSukadev Bhattiprolu } 519bbfe59f8SSukadev Bhattiprolu 520bbfe59f8SSukadev Bhattiprolu return winid; 521bbfe59f8SSukadev Bhattiprolu } 522bbfe59f8SSukadev Bhattiprolu 52398271d41SSukadev Bhattiprolu static void vas_window_free(struct vas_window *window) 524bbfe59f8SSukadev Bhattiprolu { 525bbfe59f8SSukadev Bhattiprolu int winid = window->winid; 526bbfe59f8SSukadev Bhattiprolu struct vas_instance *vinst = window->vinst; 527bbfe59f8SSukadev Bhattiprolu 528bbfe59f8SSukadev Bhattiprolu unmap_winctx_mmio_bars(window); 529ece4e512SSukadev Bhattiprolu 530ece4e512SSukadev Bhattiprolu vas_window_free_dbgdir(window); 531ece4e512SSukadev Bhattiprolu 532bbfe59f8SSukadev Bhattiprolu kfree(window); 533bbfe59f8SSukadev Bhattiprolu 534bbfe59f8SSukadev Bhattiprolu vas_release_window_id(&vinst->ida, winid); 535bbfe59f8SSukadev Bhattiprolu } 536bbfe59f8SSukadev Bhattiprolu 53762c4eda4SSukadev Bhattiprolu static struct vas_window *vas_window_alloc(struct vas_instance *vinst) 538bbfe59f8SSukadev Bhattiprolu { 539bbfe59f8SSukadev Bhattiprolu int winid; 540bbfe59f8SSukadev Bhattiprolu struct vas_window *window; 541bbfe59f8SSukadev Bhattiprolu 542bbfe59f8SSukadev Bhattiprolu winid = vas_assign_window_id(&vinst->ida); 543bbfe59f8SSukadev Bhattiprolu if (winid < 0) 544bbfe59f8SSukadev Bhattiprolu return ERR_PTR(winid); 545bbfe59f8SSukadev Bhattiprolu 546bbfe59f8SSukadev Bhattiprolu window = kzalloc(sizeof(*window), GFP_KERNEL); 547bbfe59f8SSukadev Bhattiprolu if (!window) 548bbfe59f8SSukadev Bhattiprolu goto out_free; 549bbfe59f8SSukadev Bhattiprolu 550bbfe59f8SSukadev Bhattiprolu window->vinst = vinst; 551bbfe59f8SSukadev Bhattiprolu window->winid = winid; 552bbfe59f8SSukadev Bhattiprolu 553bbfe59f8SSukadev Bhattiprolu if (map_winctx_mmio_bars(window)) 554bbfe59f8SSukadev Bhattiprolu goto out_free; 555bbfe59f8SSukadev Bhattiprolu 556ece4e512SSukadev Bhattiprolu vas_window_init_dbgdir(window); 557ece4e512SSukadev Bhattiprolu 558bbfe59f8SSukadev Bhattiprolu return window; 559bbfe59f8SSukadev Bhattiprolu 560bbfe59f8SSukadev Bhattiprolu out_free: 561bbfe59f8SSukadev Bhattiprolu kfree(window); 562bbfe59f8SSukadev Bhattiprolu vas_release_window_id(&vinst->ida, winid); 563bbfe59f8SSukadev Bhattiprolu return ERR_PTR(-ENOMEM); 564bbfe59f8SSukadev Bhattiprolu } 565bbfe59f8SSukadev Bhattiprolu 56698271d41SSukadev Bhattiprolu static void put_rx_win(struct vas_window *rxwin) 56798271d41SSukadev Bhattiprolu { 56898271d41SSukadev Bhattiprolu /* Better not be a send window! */ 56998271d41SSukadev Bhattiprolu WARN_ON_ONCE(rxwin->tx_win); 57098271d41SSukadev Bhattiprolu 57198271d41SSukadev Bhattiprolu atomic_dec(&rxwin->num_txwins); 57298271d41SSukadev Bhattiprolu } 57398271d41SSukadev Bhattiprolu 57462c4eda4SSukadev Bhattiprolu /* 5756c8e6bb2SSukadev Bhattiprolu * Find the user space receive window given the @pswid. 5766c8e6bb2SSukadev Bhattiprolu * - We must have a valid vasid and it must belong to this instance. 5776c8e6bb2SSukadev Bhattiprolu * (so both send and receive windows are on the same VAS instance) 5786c8e6bb2SSukadev Bhattiprolu * - The window must refer to an OPEN, FTW, RECEIVE window. 5796c8e6bb2SSukadev Bhattiprolu * 5806c8e6bb2SSukadev Bhattiprolu * NOTE: We access ->windows[] table and assume that vinst->mutex is held. 5816c8e6bb2SSukadev Bhattiprolu */ 5826c8e6bb2SSukadev Bhattiprolu static struct vas_window *get_user_rxwin(struct vas_instance *vinst, u32 pswid) 5836c8e6bb2SSukadev Bhattiprolu { 5846c8e6bb2SSukadev Bhattiprolu int vasid, winid; 5856c8e6bb2SSukadev Bhattiprolu struct vas_window *rxwin; 5866c8e6bb2SSukadev Bhattiprolu 5876c8e6bb2SSukadev Bhattiprolu decode_pswid(pswid, &vasid, &winid); 5886c8e6bb2SSukadev Bhattiprolu 5896c8e6bb2SSukadev Bhattiprolu if (vinst->vas_id != vasid) 5906c8e6bb2SSukadev Bhattiprolu return ERR_PTR(-EINVAL); 5916c8e6bb2SSukadev Bhattiprolu 5926c8e6bb2SSukadev Bhattiprolu rxwin = vinst->windows[winid]; 5936c8e6bb2SSukadev Bhattiprolu 5946c8e6bb2SSukadev Bhattiprolu if (!rxwin || rxwin->tx_win || rxwin->cop != VAS_COP_TYPE_FTW) 5956c8e6bb2SSukadev Bhattiprolu return ERR_PTR(-EINVAL); 5966c8e6bb2SSukadev Bhattiprolu 5976c8e6bb2SSukadev Bhattiprolu return rxwin; 5986c8e6bb2SSukadev Bhattiprolu } 5996c8e6bb2SSukadev Bhattiprolu 6006c8e6bb2SSukadev Bhattiprolu /* 60162c4eda4SSukadev Bhattiprolu * Get the VAS receive window associated with NX engine identified 60262c4eda4SSukadev Bhattiprolu * by @cop and if applicable, @pswid. 60362c4eda4SSukadev Bhattiprolu * 60462c4eda4SSukadev Bhattiprolu * See also function header of set_vinst_win(). 60562c4eda4SSukadev Bhattiprolu */ 6065239af67SSukadev Bhattiprolu static struct vas_window *get_vinst_rxwin(struct vas_instance *vinst, 60762c4eda4SSukadev Bhattiprolu enum vas_cop_type cop, u32 pswid) 60862c4eda4SSukadev Bhattiprolu { 60962c4eda4SSukadev Bhattiprolu struct vas_window *rxwin; 61062c4eda4SSukadev Bhattiprolu 61162c4eda4SSukadev Bhattiprolu mutex_lock(&vinst->mutex); 61262c4eda4SSukadev Bhattiprolu 6136c8e6bb2SSukadev Bhattiprolu if (cop == VAS_COP_TYPE_FTW) 6146c8e6bb2SSukadev Bhattiprolu rxwin = get_user_rxwin(vinst, pswid); 61562c4eda4SSukadev Bhattiprolu else 6166c8e6bb2SSukadev Bhattiprolu rxwin = vinst->rxwin[cop] ?: ERR_PTR(-EINVAL); 61762c4eda4SSukadev Bhattiprolu 61862c4eda4SSukadev Bhattiprolu if (!IS_ERR(rxwin)) 61962c4eda4SSukadev Bhattiprolu atomic_inc(&rxwin->num_txwins); 62062c4eda4SSukadev Bhattiprolu 62162c4eda4SSukadev Bhattiprolu mutex_unlock(&vinst->mutex); 62262c4eda4SSukadev Bhattiprolu 62362c4eda4SSukadev Bhattiprolu return rxwin; 62462c4eda4SSukadev Bhattiprolu } 62562c4eda4SSukadev Bhattiprolu 62662c4eda4SSukadev Bhattiprolu /* 62762c4eda4SSukadev Bhattiprolu * We have two tables of windows in a VAS instance. The first one, 62862c4eda4SSukadev Bhattiprolu * ->windows[], contains all the windows in the instance and allows 62962c4eda4SSukadev Bhattiprolu * looking up a window by its id. It is used to look up send windows 63062c4eda4SSukadev Bhattiprolu * during fault handling and receive windows when pairing user space 63162c4eda4SSukadev Bhattiprolu * send/receive windows. 63262c4eda4SSukadev Bhattiprolu * 63362c4eda4SSukadev Bhattiprolu * The second table, ->rxwin[], contains receive windows that are 63462c4eda4SSukadev Bhattiprolu * associated with NX engines. This table has VAS_COP_TYPE_MAX 63562c4eda4SSukadev Bhattiprolu * entries and is used to look up a receive window by its 63662c4eda4SSukadev Bhattiprolu * coprocessor type. 63762c4eda4SSukadev Bhattiprolu * 63862c4eda4SSukadev Bhattiprolu * Here, we save @window in the ->windows[] table. If it is a receive 63962c4eda4SSukadev Bhattiprolu * window, we also save the window in the ->rxwin[] table. 64062c4eda4SSukadev Bhattiprolu */ 64162c4eda4SSukadev Bhattiprolu static void set_vinst_win(struct vas_instance *vinst, 64262c4eda4SSukadev Bhattiprolu struct vas_window *window) 64362c4eda4SSukadev Bhattiprolu { 64462c4eda4SSukadev Bhattiprolu int id = window->winid; 64562c4eda4SSukadev Bhattiprolu 64662c4eda4SSukadev Bhattiprolu mutex_lock(&vinst->mutex); 64762c4eda4SSukadev Bhattiprolu 64862c4eda4SSukadev Bhattiprolu /* 64962c4eda4SSukadev Bhattiprolu * There should only be one receive window for a coprocessor type 65062c4eda4SSukadev Bhattiprolu * unless its a user (FTW) window. 65162c4eda4SSukadev Bhattiprolu */ 65262c4eda4SSukadev Bhattiprolu if (!window->user_win && !window->tx_win) { 65362c4eda4SSukadev Bhattiprolu WARN_ON_ONCE(vinst->rxwin[window->cop]); 65462c4eda4SSukadev Bhattiprolu vinst->rxwin[window->cop] = window; 65562c4eda4SSukadev Bhattiprolu } 65662c4eda4SSukadev Bhattiprolu 65762c4eda4SSukadev Bhattiprolu WARN_ON_ONCE(vinst->windows[id] != NULL); 65862c4eda4SSukadev Bhattiprolu vinst->windows[id] = window; 65962c4eda4SSukadev Bhattiprolu 66062c4eda4SSukadev Bhattiprolu mutex_unlock(&vinst->mutex); 66162c4eda4SSukadev Bhattiprolu } 66262c4eda4SSukadev Bhattiprolu 66362c4eda4SSukadev Bhattiprolu /* 66462c4eda4SSukadev Bhattiprolu * Clear this window from the table(s) of windows for this VAS instance. 66562c4eda4SSukadev Bhattiprolu * See also function header of set_vinst_win(). 66662c4eda4SSukadev Bhattiprolu */ 66798271d41SSukadev Bhattiprolu static void clear_vinst_win(struct vas_window *window) 66862c4eda4SSukadev Bhattiprolu { 66962c4eda4SSukadev Bhattiprolu int id = window->winid; 67062c4eda4SSukadev Bhattiprolu struct vas_instance *vinst = window->vinst; 67162c4eda4SSukadev Bhattiprolu 67262c4eda4SSukadev Bhattiprolu mutex_lock(&vinst->mutex); 67362c4eda4SSukadev Bhattiprolu 67462c4eda4SSukadev Bhattiprolu if (!window->user_win && !window->tx_win) { 67562c4eda4SSukadev Bhattiprolu WARN_ON_ONCE(!vinst->rxwin[window->cop]); 67662c4eda4SSukadev Bhattiprolu vinst->rxwin[window->cop] = NULL; 67762c4eda4SSukadev Bhattiprolu } 67862c4eda4SSukadev Bhattiprolu 67962c4eda4SSukadev Bhattiprolu WARN_ON_ONCE(vinst->windows[id] != window); 68062c4eda4SSukadev Bhattiprolu vinst->windows[id] = NULL; 68162c4eda4SSukadev Bhattiprolu 68262c4eda4SSukadev Bhattiprolu mutex_unlock(&vinst->mutex); 68362c4eda4SSukadev Bhattiprolu } 68462c4eda4SSukadev Bhattiprolu 68562c4eda4SSukadev Bhattiprolu static void init_winctx_for_rxwin(struct vas_window *rxwin, 68662c4eda4SSukadev Bhattiprolu struct vas_rx_win_attr *rxattr, 68762c4eda4SSukadev Bhattiprolu struct vas_winctx *winctx) 68862c4eda4SSukadev Bhattiprolu { 68962c4eda4SSukadev Bhattiprolu /* 69062c4eda4SSukadev Bhattiprolu * We first zero (memset()) all fields and only set non-zero fields. 69162c4eda4SSukadev Bhattiprolu * Following fields are 0/false but maybe deserve a comment: 69262c4eda4SSukadev Bhattiprolu * 69362c4eda4SSukadev Bhattiprolu * ->notify_os_intr_reg In powerNV, send intrs to HV 69462c4eda4SSukadev Bhattiprolu * ->notify_disable False for NX windows 69562c4eda4SSukadev Bhattiprolu * ->intr_disable False for Fault Windows 69662c4eda4SSukadev Bhattiprolu * ->xtra_write False for NX windows 69762c4eda4SSukadev Bhattiprolu * ->notify_early NA for NX windows 69862c4eda4SSukadev Bhattiprolu * ->rsvd_txbuf_count NA for Rx windows 69962c4eda4SSukadev Bhattiprolu * ->lpid, ->pid, ->tid NA for Rx windows 70062c4eda4SSukadev Bhattiprolu */ 70162c4eda4SSukadev Bhattiprolu 70262c4eda4SSukadev Bhattiprolu memset(winctx, 0, sizeof(struct vas_winctx)); 70362c4eda4SSukadev Bhattiprolu 70462c4eda4SSukadev Bhattiprolu winctx->rx_fifo = rxattr->rx_fifo; 70562c4eda4SSukadev Bhattiprolu winctx->rx_fifo_size = rxattr->rx_fifo_size; 70662f659e0SSukadev Bhattiprolu winctx->wcreds_max = rxwin->wcreds_max; 70762c4eda4SSukadev Bhattiprolu winctx->pin_win = rxattr->pin_win; 70862c4eda4SSukadev Bhattiprolu 70962c4eda4SSukadev Bhattiprolu winctx->nx_win = rxattr->nx_win; 71062c4eda4SSukadev Bhattiprolu winctx->fault_win = rxattr->fault_win; 711e34917fbSSukadev Bhattiprolu winctx->user_win = rxattr->user_win; 712e34917fbSSukadev Bhattiprolu winctx->rej_no_credit = rxattr->rej_no_credit; 71362c4eda4SSukadev Bhattiprolu winctx->rx_word_mode = rxattr->rx_win_ord_mode; 71462c4eda4SSukadev Bhattiprolu winctx->tx_word_mode = rxattr->tx_win_ord_mode; 71562c4eda4SSukadev Bhattiprolu winctx->rx_wcred_mode = rxattr->rx_wcred_mode; 71662c4eda4SSukadev Bhattiprolu winctx->tx_wcred_mode = rxattr->tx_wcred_mode; 717e34917fbSSukadev Bhattiprolu winctx->notify_early = rxattr->notify_early; 71862c4eda4SSukadev Bhattiprolu 71962c4eda4SSukadev Bhattiprolu if (winctx->nx_win) { 72062c4eda4SSukadev Bhattiprolu winctx->data_stamp = true; 72162c4eda4SSukadev Bhattiprolu winctx->intr_disable = true; 72262c4eda4SSukadev Bhattiprolu winctx->pin_win = true; 72362c4eda4SSukadev Bhattiprolu 72462c4eda4SSukadev Bhattiprolu WARN_ON_ONCE(winctx->fault_win); 72562c4eda4SSukadev Bhattiprolu WARN_ON_ONCE(!winctx->rx_word_mode); 72662c4eda4SSukadev Bhattiprolu WARN_ON_ONCE(!winctx->tx_word_mode); 72762c4eda4SSukadev Bhattiprolu WARN_ON_ONCE(winctx->notify_after_count); 72862c4eda4SSukadev Bhattiprolu } else if (winctx->fault_win) { 72962c4eda4SSukadev Bhattiprolu winctx->notify_disable = true; 73062c4eda4SSukadev Bhattiprolu } else if (winctx->user_win) { 73162c4eda4SSukadev Bhattiprolu /* 73262c4eda4SSukadev Bhattiprolu * Section 1.8.1 Low Latency Core-Core Wake up of 73362c4eda4SSukadev Bhattiprolu * the VAS workbook: 73462c4eda4SSukadev Bhattiprolu * 73562c4eda4SSukadev Bhattiprolu * - disable credit checks ([tr]x_wcred_mode = false) 73662c4eda4SSukadev Bhattiprolu * - disable FIFO writes 73762c4eda4SSukadev Bhattiprolu * - enable ASB_Notify, disable interrupt 73862c4eda4SSukadev Bhattiprolu */ 73962c4eda4SSukadev Bhattiprolu winctx->fifo_disable = true; 74062c4eda4SSukadev Bhattiprolu winctx->intr_disable = true; 74162c4eda4SSukadev Bhattiprolu winctx->rx_fifo = NULL; 74262c4eda4SSukadev Bhattiprolu } 74362c4eda4SSukadev Bhattiprolu 74462c4eda4SSukadev Bhattiprolu winctx->lnotify_lpid = rxattr->lnotify_lpid; 74562c4eda4SSukadev Bhattiprolu winctx->lnotify_pid = rxattr->lnotify_pid; 74662c4eda4SSukadev Bhattiprolu winctx->lnotify_tid = rxattr->lnotify_tid; 74762c4eda4SSukadev Bhattiprolu winctx->pswid = rxattr->pswid; 74862c4eda4SSukadev Bhattiprolu winctx->dma_type = VAS_DMA_TYPE_INJECT; 74962c4eda4SSukadev Bhattiprolu winctx->tc_mode = rxattr->tc_mode; 75062c4eda4SSukadev Bhattiprolu 75162c4eda4SSukadev Bhattiprolu winctx->min_scope = VAS_SCOPE_LOCAL; 75262c4eda4SSukadev Bhattiprolu winctx->max_scope = VAS_SCOPE_VECTORED_GROUP; 7538b8a73dcSHaren Myneni if (rxwin->vinst->virq) 7548b8a73dcSHaren Myneni winctx->irq_port = rxwin->vinst->irq_port; 75562c4eda4SSukadev Bhattiprolu } 75662c4eda4SSukadev Bhattiprolu 75762c4eda4SSukadev Bhattiprolu static bool rx_win_args_valid(enum vas_cop_type cop, 75862c4eda4SSukadev Bhattiprolu struct vas_rx_win_attr *attr) 75962c4eda4SSukadev Bhattiprolu { 7600a2c2c24SSukadev Bhattiprolu pr_debug("Rxattr: fault %d, notify %d, intr %d, early %d, fifo %d\n", 7610a2c2c24SSukadev Bhattiprolu attr->fault_win, attr->notify_disable, 7620a2c2c24SSukadev Bhattiprolu attr->intr_disable, attr->notify_early, 7630a2c2c24SSukadev Bhattiprolu attr->rx_fifo_size); 76462c4eda4SSukadev Bhattiprolu 76562c4eda4SSukadev Bhattiprolu if (cop >= VAS_COP_TYPE_MAX) 76662c4eda4SSukadev Bhattiprolu return false; 76762c4eda4SSukadev Bhattiprolu 76862c4eda4SSukadev Bhattiprolu if (cop != VAS_COP_TYPE_FTW && 76962c4eda4SSukadev Bhattiprolu attr->rx_fifo_size < VAS_RX_FIFO_SIZE_MIN) 77062c4eda4SSukadev Bhattiprolu return false; 77162c4eda4SSukadev Bhattiprolu 77262c4eda4SSukadev Bhattiprolu if (attr->rx_fifo_size > VAS_RX_FIFO_SIZE_MAX) 77362c4eda4SSukadev Bhattiprolu return false; 77462c4eda4SSukadev Bhattiprolu 77504f6296cSHaren Myneni if (!attr->wcreds_max) 77651b53712SSukadev Bhattiprolu return false; 77751b53712SSukadev Bhattiprolu 77862c4eda4SSukadev Bhattiprolu if (attr->nx_win) { 77962c4eda4SSukadev Bhattiprolu /* cannot be fault or user window if it is nx */ 78062c4eda4SSukadev Bhattiprolu if (attr->fault_win || attr->user_win) 78162c4eda4SSukadev Bhattiprolu return false; 78262c4eda4SSukadev Bhattiprolu /* 78362c4eda4SSukadev Bhattiprolu * Section 3.1.4.32: NX Windows must not disable notification, 78462c4eda4SSukadev Bhattiprolu * and must not enable interrupts or early notification. 78562c4eda4SSukadev Bhattiprolu */ 78662c4eda4SSukadev Bhattiprolu if (attr->notify_disable || !attr->intr_disable || 78762c4eda4SSukadev Bhattiprolu attr->notify_early) 78862c4eda4SSukadev Bhattiprolu return false; 78962c4eda4SSukadev Bhattiprolu } else if (attr->fault_win) { 79062c4eda4SSukadev Bhattiprolu /* cannot be both fault and user window */ 79162c4eda4SSukadev Bhattiprolu if (attr->user_win) 79262c4eda4SSukadev Bhattiprolu return false; 79362c4eda4SSukadev Bhattiprolu 79462c4eda4SSukadev Bhattiprolu /* 79562c4eda4SSukadev Bhattiprolu * Section 3.1.4.32: Fault windows must disable notification 79662c4eda4SSukadev Bhattiprolu * but not interrupts. 79762c4eda4SSukadev Bhattiprolu */ 79862c4eda4SSukadev Bhattiprolu if (!attr->notify_disable || attr->intr_disable) 79962c4eda4SSukadev Bhattiprolu return false; 80062c4eda4SSukadev Bhattiprolu 80162c4eda4SSukadev Bhattiprolu } else if (attr->user_win) { 80262c4eda4SSukadev Bhattiprolu /* 80362c4eda4SSukadev Bhattiprolu * User receive windows are only for fast-thread-wakeup 80462c4eda4SSukadev Bhattiprolu * (FTW). They don't need a FIFO and must disable interrupts 80562c4eda4SSukadev Bhattiprolu */ 80662c4eda4SSukadev Bhattiprolu if (attr->rx_fifo || attr->rx_fifo_size || !attr->intr_disable) 80762c4eda4SSukadev Bhattiprolu return false; 80862c4eda4SSukadev Bhattiprolu } else { 80962c4eda4SSukadev Bhattiprolu /* Rx window must be one of NX or Fault or User window. */ 81062c4eda4SSukadev Bhattiprolu return false; 81162c4eda4SSukadev Bhattiprolu } 81262c4eda4SSukadev Bhattiprolu 81362c4eda4SSukadev Bhattiprolu return true; 81462c4eda4SSukadev Bhattiprolu } 81562c4eda4SSukadev Bhattiprolu 81662c4eda4SSukadev Bhattiprolu void vas_init_rx_win_attr(struct vas_rx_win_attr *rxattr, enum vas_cop_type cop) 81762c4eda4SSukadev Bhattiprolu { 81862c4eda4SSukadev Bhattiprolu memset(rxattr, 0, sizeof(*rxattr)); 81962c4eda4SSukadev Bhattiprolu 820a8c0c69bSHaren Myneni if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI || 821a8c0c69bSHaren Myneni cop == VAS_COP_TYPE_GZIP || cop == VAS_COP_TYPE_GZIP_HIPRI) { 82262c4eda4SSukadev Bhattiprolu rxattr->pin_win = true; 82362c4eda4SSukadev Bhattiprolu rxattr->nx_win = true; 82462c4eda4SSukadev Bhattiprolu rxattr->fault_win = false; 82562c4eda4SSukadev Bhattiprolu rxattr->intr_disable = true; 82662c4eda4SSukadev Bhattiprolu rxattr->rx_wcred_mode = true; 82762c4eda4SSukadev Bhattiprolu rxattr->tx_wcred_mode = true; 82862c4eda4SSukadev Bhattiprolu rxattr->rx_win_ord_mode = true; 82962c4eda4SSukadev Bhattiprolu rxattr->tx_win_ord_mode = true; 83062c4eda4SSukadev Bhattiprolu } else if (cop == VAS_COP_TYPE_FAULT) { 83162c4eda4SSukadev Bhattiprolu rxattr->pin_win = true; 83262c4eda4SSukadev Bhattiprolu rxattr->fault_win = true; 83362c4eda4SSukadev Bhattiprolu rxattr->notify_disable = true; 83462c4eda4SSukadev Bhattiprolu rxattr->rx_wcred_mode = true; 83562c4eda4SSukadev Bhattiprolu rxattr->rx_win_ord_mode = true; 8360d17de03SHaren Myneni rxattr->rej_no_credit = true; 8370d17de03SHaren Myneni rxattr->tc_mode = VAS_THRESH_DISABLED; 83862c4eda4SSukadev Bhattiprolu } else if (cop == VAS_COP_TYPE_FTW) { 83962c4eda4SSukadev Bhattiprolu rxattr->user_win = true; 84062c4eda4SSukadev Bhattiprolu rxattr->intr_disable = true; 84162c4eda4SSukadev Bhattiprolu 84262c4eda4SSukadev Bhattiprolu /* 84362c4eda4SSukadev Bhattiprolu * As noted in the VAS Workbook we disable credit checks. 84462c4eda4SSukadev Bhattiprolu * If we enable credit checks in the future, we must also 84562c4eda4SSukadev Bhattiprolu * implement a mechanism to return the user credits or new 84662c4eda4SSukadev Bhattiprolu * paste operations will fail. 84762c4eda4SSukadev Bhattiprolu */ 84862c4eda4SSukadev Bhattiprolu } 84962c4eda4SSukadev Bhattiprolu } 85062c4eda4SSukadev Bhattiprolu EXPORT_SYMBOL_GPL(vas_init_rx_win_attr); 85162c4eda4SSukadev Bhattiprolu 85262c4eda4SSukadev Bhattiprolu struct vas_window *vas_rx_win_open(int vasid, enum vas_cop_type cop, 85362c4eda4SSukadev Bhattiprolu struct vas_rx_win_attr *rxattr) 85462c4eda4SSukadev Bhattiprolu { 85562c4eda4SSukadev Bhattiprolu struct vas_window *rxwin; 85662c4eda4SSukadev Bhattiprolu struct vas_winctx winctx; 85762c4eda4SSukadev Bhattiprolu struct vas_instance *vinst; 85862c4eda4SSukadev Bhattiprolu 859007bb7d6SSukadev Bhattiprolu trace_vas_rx_win_open(current, vasid, cop, rxattr); 860007bb7d6SSukadev Bhattiprolu 86162c4eda4SSukadev Bhattiprolu if (!rx_win_args_valid(cop, rxattr)) 86262c4eda4SSukadev Bhattiprolu return ERR_PTR(-EINVAL); 86362c4eda4SSukadev Bhattiprolu 86462c4eda4SSukadev Bhattiprolu vinst = find_vas_instance(vasid); 86562c4eda4SSukadev Bhattiprolu if (!vinst) { 86662c4eda4SSukadev Bhattiprolu pr_devel("vasid %d not found!\n", vasid); 86762c4eda4SSukadev Bhattiprolu return ERR_PTR(-EINVAL); 86862c4eda4SSukadev Bhattiprolu } 86962c4eda4SSukadev Bhattiprolu pr_devel("Found instance %d\n", vasid); 87062c4eda4SSukadev Bhattiprolu 87162c4eda4SSukadev Bhattiprolu rxwin = vas_window_alloc(vinst); 87262c4eda4SSukadev Bhattiprolu if (IS_ERR(rxwin)) { 87362c4eda4SSukadev Bhattiprolu pr_devel("Unable to allocate memory for Rx window\n"); 87462c4eda4SSukadev Bhattiprolu return rxwin; 87562c4eda4SSukadev Bhattiprolu } 87662c4eda4SSukadev Bhattiprolu 87762c4eda4SSukadev Bhattiprolu rxwin->tx_win = false; 87862c4eda4SSukadev Bhattiprolu rxwin->nx_win = rxattr->nx_win; 87962c4eda4SSukadev Bhattiprolu rxwin->user_win = rxattr->user_win; 88062c4eda4SSukadev Bhattiprolu rxwin->cop = cop; 88104f6296cSHaren Myneni rxwin->wcreds_max = rxattr->wcreds_max; 88262c4eda4SSukadev Bhattiprolu 88362c4eda4SSukadev Bhattiprolu init_winctx_for_rxwin(rxwin, rxattr, &winctx); 88462c4eda4SSukadev Bhattiprolu init_winctx_regs(rxwin, &winctx); 88562c4eda4SSukadev Bhattiprolu 88662c4eda4SSukadev Bhattiprolu set_vinst_win(vinst, rxwin); 88762c4eda4SSukadev Bhattiprolu 88862c4eda4SSukadev Bhattiprolu return rxwin; 88962c4eda4SSukadev Bhattiprolu } 89062c4eda4SSukadev Bhattiprolu EXPORT_SYMBOL_GPL(vas_rx_win_open); 89162c4eda4SSukadev Bhattiprolu 8925239af67SSukadev Bhattiprolu void vas_init_tx_win_attr(struct vas_tx_win_attr *txattr, enum vas_cop_type cop) 8935239af67SSukadev Bhattiprolu { 8945239af67SSukadev Bhattiprolu memset(txattr, 0, sizeof(*txattr)); 8955239af67SSukadev Bhattiprolu 896a8c0c69bSHaren Myneni if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI || 897a8c0c69bSHaren Myneni cop == VAS_COP_TYPE_GZIP || cop == VAS_COP_TYPE_GZIP_HIPRI) { 8985239af67SSukadev Bhattiprolu txattr->rej_no_credit = false; 8995239af67SSukadev Bhattiprolu txattr->rx_wcred_mode = true; 9005239af67SSukadev Bhattiprolu txattr->tx_wcred_mode = true; 9015239af67SSukadev Bhattiprolu txattr->rx_win_ord_mode = true; 9025239af67SSukadev Bhattiprolu txattr->tx_win_ord_mode = true; 9035239af67SSukadev Bhattiprolu } else if (cop == VAS_COP_TYPE_FTW) { 9045239af67SSukadev Bhattiprolu txattr->user_win = true; 9055239af67SSukadev Bhattiprolu } 9065239af67SSukadev Bhattiprolu } 9075239af67SSukadev Bhattiprolu EXPORT_SYMBOL_GPL(vas_init_tx_win_attr); 9085239af67SSukadev Bhattiprolu 9095239af67SSukadev Bhattiprolu static void init_winctx_for_txwin(struct vas_window *txwin, 9105239af67SSukadev Bhattiprolu struct vas_tx_win_attr *txattr, 9115239af67SSukadev Bhattiprolu struct vas_winctx *winctx) 9125239af67SSukadev Bhattiprolu { 9135239af67SSukadev Bhattiprolu /* 9145239af67SSukadev Bhattiprolu * We first zero all fields and only set non-zero ones. Following 9155239af67SSukadev Bhattiprolu * are some fields set to 0/false for the stated reason: 9165239af67SSukadev Bhattiprolu * 9175239af67SSukadev Bhattiprolu * ->notify_os_intr_reg In powernv, send intrs to HV 9185239af67SSukadev Bhattiprolu * ->rsvd_txbuf_count Not supported yet. 9195239af67SSukadev Bhattiprolu * ->notify_disable False for NX windows 9205239af67SSukadev Bhattiprolu * ->xtra_write False for NX windows 9215239af67SSukadev Bhattiprolu * ->notify_early NA for NX windows 9225239af67SSukadev Bhattiprolu * ->lnotify_lpid NA for Tx windows 9235239af67SSukadev Bhattiprolu * ->lnotify_pid NA for Tx windows 9245239af67SSukadev Bhattiprolu * ->lnotify_tid NA for Tx windows 9255239af67SSukadev Bhattiprolu * ->tx_win_cred_mode Ignore for now for NX windows 9265239af67SSukadev Bhattiprolu * ->rx_win_cred_mode Ignore for now for NX windows 9275239af67SSukadev Bhattiprolu */ 9285239af67SSukadev Bhattiprolu memset(winctx, 0, sizeof(struct vas_winctx)); 9295239af67SSukadev Bhattiprolu 93062f659e0SSukadev Bhattiprolu winctx->wcreds_max = txwin->wcreds_max; 9315239af67SSukadev Bhattiprolu 9325239af67SSukadev Bhattiprolu winctx->user_win = txattr->user_win; 9335239af67SSukadev Bhattiprolu winctx->nx_win = txwin->rxwin->nx_win; 9345239af67SSukadev Bhattiprolu winctx->pin_win = txattr->pin_win; 935e34917fbSSukadev Bhattiprolu winctx->rej_no_credit = txattr->rej_no_credit; 936e34917fbSSukadev Bhattiprolu winctx->rsvd_txbuf_enable = txattr->rsvd_txbuf_enable; 9375239af67SSukadev Bhattiprolu 9385239af67SSukadev Bhattiprolu winctx->rx_wcred_mode = txattr->rx_wcred_mode; 9395239af67SSukadev Bhattiprolu winctx->tx_wcred_mode = txattr->tx_wcred_mode; 9405239af67SSukadev Bhattiprolu winctx->rx_word_mode = txattr->rx_win_ord_mode; 9415239af67SSukadev Bhattiprolu winctx->tx_word_mode = txattr->tx_win_ord_mode; 942e34917fbSSukadev Bhattiprolu winctx->rsvd_txbuf_count = txattr->rsvd_txbuf_count; 9435239af67SSukadev Bhattiprolu 9445239af67SSukadev Bhattiprolu winctx->intr_disable = true; 9456c8e6bb2SSukadev Bhattiprolu if (winctx->nx_win) 9466c8e6bb2SSukadev Bhattiprolu winctx->data_stamp = true; 9475239af67SSukadev Bhattiprolu 9485239af67SSukadev Bhattiprolu winctx->lpid = txattr->lpid; 9495239af67SSukadev Bhattiprolu winctx->pidr = txattr->pidr; 9505239af67SSukadev Bhattiprolu winctx->rx_win_id = txwin->rxwin->winid; 9518b8a73dcSHaren Myneni /* 9528b8a73dcSHaren Myneni * IRQ and fault window setup is successful. Set fault window 9538b8a73dcSHaren Myneni * for the send window so that ready to handle faults. 9548b8a73dcSHaren Myneni */ 9558b8a73dcSHaren Myneni if (txwin->vinst->virq) 9568b8a73dcSHaren Myneni winctx->fault_win_id = txwin->vinst->fault_win->winid; 9575239af67SSukadev Bhattiprolu 9585239af67SSukadev Bhattiprolu winctx->dma_type = VAS_DMA_TYPE_INJECT; 9595239af67SSukadev Bhattiprolu winctx->tc_mode = txattr->tc_mode; 9605239af67SSukadev Bhattiprolu winctx->min_scope = VAS_SCOPE_LOCAL; 9615239af67SSukadev Bhattiprolu winctx->max_scope = VAS_SCOPE_VECTORED_GROUP; 9628b8a73dcSHaren Myneni if (txwin->vinst->virq) 9638b8a73dcSHaren Myneni winctx->irq_port = txwin->vinst->irq_port; 9645239af67SSukadev Bhattiprolu 9658b8a73dcSHaren Myneni winctx->pswid = txattr->pswid ? txattr->pswid : 9668b8a73dcSHaren Myneni encode_pswid(txwin->vinst->vas_id, txwin->winid); 9675239af67SSukadev Bhattiprolu } 9685239af67SSukadev Bhattiprolu 9695239af67SSukadev Bhattiprolu static bool tx_win_args_valid(enum vas_cop_type cop, 9705239af67SSukadev Bhattiprolu struct vas_tx_win_attr *attr) 9715239af67SSukadev Bhattiprolu { 9725239af67SSukadev Bhattiprolu if (attr->tc_mode != VAS_THRESH_DISABLED) 9735239af67SSukadev Bhattiprolu return false; 9745239af67SSukadev Bhattiprolu 9755239af67SSukadev Bhattiprolu if (cop > VAS_COP_TYPE_MAX) 9765239af67SSukadev Bhattiprolu return false; 9775239af67SSukadev Bhattiprolu 97851b53712SSukadev Bhattiprolu if (attr->wcreds_max > VAS_TX_WCREDS_MAX) 97951b53712SSukadev Bhattiprolu return false; 98051b53712SSukadev Bhattiprolu 981a8c0c69bSHaren Myneni if (attr->user_win) { 982a8c0c69bSHaren Myneni if (attr->rsvd_txbuf_count) 9835239af67SSukadev Bhattiprolu return false; 9845239af67SSukadev Bhattiprolu 985a8c0c69bSHaren Myneni if (cop != VAS_COP_TYPE_FTW && cop != VAS_COP_TYPE_GZIP && 986a8c0c69bSHaren Myneni cop != VAS_COP_TYPE_GZIP_HIPRI) 987a8c0c69bSHaren Myneni return false; 988a8c0c69bSHaren Myneni } 989a8c0c69bSHaren Myneni 9905239af67SSukadev Bhattiprolu return true; 9915239af67SSukadev Bhattiprolu } 9925239af67SSukadev Bhattiprolu 9935239af67SSukadev Bhattiprolu struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop, 9945239af67SSukadev Bhattiprolu struct vas_tx_win_attr *attr) 9955239af67SSukadev Bhattiprolu { 9965239af67SSukadev Bhattiprolu int rc; 9975239af67SSukadev Bhattiprolu struct vas_window *txwin; 9985239af67SSukadev Bhattiprolu struct vas_window *rxwin; 9995239af67SSukadev Bhattiprolu struct vas_winctx winctx; 10005239af67SSukadev Bhattiprolu struct vas_instance *vinst; 10015239af67SSukadev Bhattiprolu 1002007bb7d6SSukadev Bhattiprolu trace_vas_tx_win_open(current, vasid, cop, attr); 1003007bb7d6SSukadev Bhattiprolu 10045239af67SSukadev Bhattiprolu if (!tx_win_args_valid(cop, attr)) 10055239af67SSukadev Bhattiprolu return ERR_PTR(-EINVAL); 10065239af67SSukadev Bhattiprolu 10076c8e6bb2SSukadev Bhattiprolu /* 10086c8e6bb2SSukadev Bhattiprolu * If caller did not specify a vasid but specified the PSWID of a 10096c8e6bb2SSukadev Bhattiprolu * receive window (applicable only to FTW windows), use the vasid 10106c8e6bb2SSukadev Bhattiprolu * from that receive window. 10116c8e6bb2SSukadev Bhattiprolu */ 10126c8e6bb2SSukadev Bhattiprolu if (vasid == -1 && attr->pswid) 10136c8e6bb2SSukadev Bhattiprolu decode_pswid(attr->pswid, &vasid, NULL); 10146c8e6bb2SSukadev Bhattiprolu 10155239af67SSukadev Bhattiprolu vinst = find_vas_instance(vasid); 10165239af67SSukadev Bhattiprolu if (!vinst) { 10175239af67SSukadev Bhattiprolu pr_devel("vasid %d not found!\n", vasid); 10185239af67SSukadev Bhattiprolu return ERR_PTR(-EINVAL); 10195239af67SSukadev Bhattiprolu } 10205239af67SSukadev Bhattiprolu 10215239af67SSukadev Bhattiprolu rxwin = get_vinst_rxwin(vinst, cop, attr->pswid); 10225239af67SSukadev Bhattiprolu if (IS_ERR(rxwin)) { 10235239af67SSukadev Bhattiprolu pr_devel("No RxWin for vasid %d, cop %d\n", vasid, cop); 10245239af67SSukadev Bhattiprolu return rxwin; 10255239af67SSukadev Bhattiprolu } 10265239af67SSukadev Bhattiprolu 10275239af67SSukadev Bhattiprolu txwin = vas_window_alloc(vinst); 10285239af67SSukadev Bhattiprolu if (IS_ERR(txwin)) { 10295239af67SSukadev Bhattiprolu rc = PTR_ERR(txwin); 10305239af67SSukadev Bhattiprolu goto put_rxwin; 10315239af67SSukadev Bhattiprolu } 10325239af67SSukadev Bhattiprolu 1033ece4e512SSukadev Bhattiprolu txwin->cop = cop; 10345239af67SSukadev Bhattiprolu txwin->tx_win = 1; 10355239af67SSukadev Bhattiprolu txwin->rxwin = rxwin; 10365239af67SSukadev Bhattiprolu txwin->nx_win = txwin->rxwin->nx_win; 10375239af67SSukadev Bhattiprolu txwin->user_win = attr->user_win; 103862f659e0SSukadev Bhattiprolu txwin->wcreds_max = attr->wcreds_max ?: VAS_WCREDS_DEFAULT; 10395239af67SSukadev Bhattiprolu 10405239af67SSukadev Bhattiprolu init_winctx_for_txwin(txwin, attr, &winctx); 10415239af67SSukadev Bhattiprolu 10425239af67SSukadev Bhattiprolu init_winctx_regs(txwin, &winctx); 10435239af67SSukadev Bhattiprolu 10445239af67SSukadev Bhattiprolu /* 10455239af67SSukadev Bhattiprolu * If its a kernel send window, map the window address into the 10465239af67SSukadev Bhattiprolu * kernel's address space. For user windows, user must issue an 10475239af67SSukadev Bhattiprolu * mmap() to map the window into their address space. 10485239af67SSukadev Bhattiprolu * 10495239af67SSukadev Bhattiprolu * NOTE: If kernel ever resubmits a user CRB after handling a page 10505239af67SSukadev Bhattiprolu * fault, we will need to map this into kernel as well. 10515239af67SSukadev Bhattiprolu */ 10525239af67SSukadev Bhattiprolu if (!txwin->user_win) { 10535239af67SSukadev Bhattiprolu txwin->paste_kaddr = map_paste_region(txwin); 10545239af67SSukadev Bhattiprolu if (IS_ERR(txwin->paste_kaddr)) { 10555239af67SSukadev Bhattiprolu rc = PTR_ERR(txwin->paste_kaddr); 10565239af67SSukadev Bhattiprolu goto free_window; 10575239af67SSukadev Bhattiprolu } 1058b00b6289SNicholas Piggin } else { 10596c8e6bb2SSukadev Bhattiprolu /* 10609774628aSHaren Myneni * Interrupt hanlder or fault window setup failed. Means 10619774628aSHaren Myneni * NX can not generate fault for page fault. So not 10629774628aSHaren Myneni * opening for user space tx window. 10639774628aSHaren Myneni */ 10649774628aSHaren Myneni if (!vinst->virq) { 10659774628aSHaren Myneni rc = -ENODEV; 10669774628aSHaren Myneni goto free_window; 10679774628aSHaren Myneni } 1068db1c08a7SHaren Myneni 1069db1c08a7SHaren Myneni /* 1070db1c08a7SHaren Myneni * Window opened by a child thread may not be closed when 1071db1c08a7SHaren Myneni * it exits. So take reference to its pid and release it 1072db1c08a7SHaren Myneni * when the window is free by parent thread. 1073db1c08a7SHaren Myneni * Acquire a reference to the task's pid to make sure 1074db1c08a7SHaren Myneni * pid will not be re-used - needed only for multithread 1075db1c08a7SHaren Myneni * applications. 1076db1c08a7SHaren Myneni */ 1077db1c08a7SHaren Myneni txwin->pid = get_task_pid(current, PIDTYPE_PID); 1078db1c08a7SHaren Myneni /* 1079db1c08a7SHaren Myneni * Acquire a reference to the task's mm. 1080db1c08a7SHaren Myneni */ 1081db1c08a7SHaren Myneni txwin->mm = get_task_mm(current); 1082db1c08a7SHaren Myneni 1083db1c08a7SHaren Myneni if (!txwin->mm) { 1084db1c08a7SHaren Myneni put_pid(txwin->pid); 1085db1c08a7SHaren Myneni pr_err("VAS: pid(%d): mm_struct is not found\n", 1086db1c08a7SHaren Myneni current->pid); 1087db1c08a7SHaren Myneni rc = -EPERM; 1088db1c08a7SHaren Myneni goto free_window; 1089db1c08a7SHaren Myneni } 1090db1c08a7SHaren Myneni 1091db1c08a7SHaren Myneni mmgrab(txwin->mm); 1092db1c08a7SHaren Myneni mmput(txwin->mm); 1093c420644cSHaren Myneni mm_context_add_vas_window(txwin->mm); 1094db1c08a7SHaren Myneni /* 1095db1c08a7SHaren Myneni * Process closes window during exit. In the case of 1096db1c08a7SHaren Myneni * multithread application, the child thread can open 109791cdbb95SHaren Myneni * window and can exit without closing it. so takes tgid 109891cdbb95SHaren Myneni * reference until window closed to make sure tgid is not 109991cdbb95SHaren Myneni * reused. 1100db1c08a7SHaren Myneni */ 1101db1c08a7SHaren Myneni txwin->tgid = find_get_pid(task_tgid_vnr(current)); 1102c420644cSHaren Myneni /* 1103c420644cSHaren Myneni * Even a process that has no foreign real address mapping can 1104c420644cSHaren Myneni * use an unpaired COPY instruction (to no real effect). Issue 1105c420644cSHaren Myneni * CP_ABORT to clear any pending COPY and prevent a covert 1106c420644cSHaren Myneni * channel. 1107c420644cSHaren Myneni * 1108c420644cSHaren Myneni * __switch_to() will issue CP_ABORT on future context switches 1109c420644cSHaren Myneni * if process / thread has any open VAS window (Use 1110c420644cSHaren Myneni * current->mm->context.vas_windows). 1111c420644cSHaren Myneni */ 1112c420644cSHaren Myneni asm volatile(PPC_CP_ABORT); 1113b00b6289SNicholas Piggin } 11146c8e6bb2SSukadev Bhattiprolu 11155239af67SSukadev Bhattiprolu set_vinst_win(vinst, txwin); 11165239af67SSukadev Bhattiprolu 11175239af67SSukadev Bhattiprolu return txwin; 11185239af67SSukadev Bhattiprolu 11195239af67SSukadev Bhattiprolu free_window: 11205239af67SSukadev Bhattiprolu vas_window_free(txwin); 11215239af67SSukadev Bhattiprolu 11225239af67SSukadev Bhattiprolu put_rxwin: 11235239af67SSukadev Bhattiprolu put_rx_win(rxwin); 11245239af67SSukadev Bhattiprolu return ERR_PTR(rc); 11255239af67SSukadev Bhattiprolu 11265239af67SSukadev Bhattiprolu } 11275239af67SSukadev Bhattiprolu EXPORT_SYMBOL_GPL(vas_tx_win_open); 11285239af67SSukadev Bhattiprolu 11292392c8c8SSukadev Bhattiprolu int vas_copy_crb(void *crb, int offset) 11302392c8c8SSukadev Bhattiprolu { 11312392c8c8SSukadev Bhattiprolu return vas_copy(crb, offset); 11322392c8c8SSukadev Bhattiprolu } 11332392c8c8SSukadev Bhattiprolu EXPORT_SYMBOL_GPL(vas_copy_crb); 11342392c8c8SSukadev Bhattiprolu 11352392c8c8SSukadev Bhattiprolu #define RMA_LSMP_REPORT_ENABLE PPC_BIT(53) 11362392c8c8SSukadev Bhattiprolu int vas_paste_crb(struct vas_window *txwin, int offset, bool re) 11372392c8c8SSukadev Bhattiprolu { 11382392c8c8SSukadev Bhattiprolu int rc; 11392392c8c8SSukadev Bhattiprolu void *addr; 11402392c8c8SSukadev Bhattiprolu uint64_t val; 11412392c8c8SSukadev Bhattiprolu 1142007bb7d6SSukadev Bhattiprolu trace_vas_paste_crb(current, txwin); 1143007bb7d6SSukadev Bhattiprolu 11442392c8c8SSukadev Bhattiprolu /* 11452392c8c8SSukadev Bhattiprolu * Only NX windows are supported for now and hardware assumes 11462392c8c8SSukadev Bhattiprolu * report-enable flag is set for NX windows. Ensure software 11472392c8c8SSukadev Bhattiprolu * complies too. 11482392c8c8SSukadev Bhattiprolu */ 11492392c8c8SSukadev Bhattiprolu WARN_ON_ONCE(txwin->nx_win && !re); 11502392c8c8SSukadev Bhattiprolu 11512392c8c8SSukadev Bhattiprolu addr = txwin->paste_kaddr; 11522392c8c8SSukadev Bhattiprolu if (re) { 11532392c8c8SSukadev Bhattiprolu /* 11542392c8c8SSukadev Bhattiprolu * Set the REPORT_ENABLE bit (equivalent to writing 11552392c8c8SSukadev Bhattiprolu * to 1K offset of the paste address) 11562392c8c8SSukadev Bhattiprolu */ 11572392c8c8SSukadev Bhattiprolu val = SET_FIELD(RMA_LSMP_REPORT_ENABLE, 0ULL, 1); 11582392c8c8SSukadev Bhattiprolu addr += val; 11592392c8c8SSukadev Bhattiprolu } 11602392c8c8SSukadev Bhattiprolu 11612392c8c8SSukadev Bhattiprolu /* 11622392c8c8SSukadev Bhattiprolu * Map the raw CR value from vas_paste() to an error code (there 11632392c8c8SSukadev Bhattiprolu * is just pass or fail for now though). 11642392c8c8SSukadev Bhattiprolu */ 11652392c8c8SSukadev Bhattiprolu rc = vas_paste(addr, offset); 11662392c8c8SSukadev Bhattiprolu if (rc == 2) 11672392c8c8SSukadev Bhattiprolu rc = 0; 11682392c8c8SSukadev Bhattiprolu else 11692392c8c8SSukadev Bhattiprolu rc = -EINVAL; 11702392c8c8SSukadev Bhattiprolu 11710a2c2c24SSukadev Bhattiprolu pr_debug("Txwin #%d: Msg count %llu\n", txwin->winid, 11720a2c2c24SSukadev Bhattiprolu read_hvwc_reg(txwin, VREG(LRFIFO_PUSH))); 11732392c8c8SSukadev Bhattiprolu 11742392c8c8SSukadev Bhattiprolu return rc; 11752392c8c8SSukadev Bhattiprolu } 11762392c8c8SSukadev Bhattiprolu EXPORT_SYMBOL_GPL(vas_paste_crb); 11772392c8c8SSukadev Bhattiprolu 1178dfe954e4SSukadev Bhattiprolu /* 11796fccac16SSukadev Bhattiprolu * If credit checking is enabled for this window, poll for the return 11806fccac16SSukadev Bhattiprolu * of window credits (i.e for NX engines to process any outstanding CRBs). 11816fccac16SSukadev Bhattiprolu * Since NX-842 waits for the CRBs to be processed before closing the 11826fccac16SSukadev Bhattiprolu * window, we should not have to wait for too long. 11836fccac16SSukadev Bhattiprolu * 11846fccac16SSukadev Bhattiprolu * TODO: We retry in 10ms intervals now. We could/should probably peek at 11856fccac16SSukadev Bhattiprolu * the VAS_LRFIFO_PUSH_OFFSET register to get an estimate of pending 11866fccac16SSukadev Bhattiprolu * CRBs on the FIFO and compute the delay dynamically on each retry. 11876fccac16SSukadev Bhattiprolu * But that is not really needed until we support NX-GZIP access from 11886fccac16SSukadev Bhattiprolu * user space. (NX-842 driver waits for CSB and Fast thread-wakeup 11896fccac16SSukadev Bhattiprolu * doesn't use credit checking). 11906fccac16SSukadev Bhattiprolu */ 11916fccac16SSukadev Bhattiprolu static void poll_window_credits(struct vas_window *window) 11926fccac16SSukadev Bhattiprolu { 11936fccac16SSukadev Bhattiprolu u64 val; 11946fccac16SSukadev Bhattiprolu int creds, mode; 1195bd4da68dSHaren Myneni int count = 0; 11966fccac16SSukadev Bhattiprolu 11976fccac16SSukadev Bhattiprolu val = read_hvwc_reg(window, VREG(WINCTL)); 11986fccac16SSukadev Bhattiprolu if (window->tx_win) 11996fccac16SSukadev Bhattiprolu mode = GET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val); 12006fccac16SSukadev Bhattiprolu else 12016fccac16SSukadev Bhattiprolu mode = GET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val); 12026fccac16SSukadev Bhattiprolu 12036fccac16SSukadev Bhattiprolu if (!mode) 12046fccac16SSukadev Bhattiprolu return; 12056fccac16SSukadev Bhattiprolu retry: 12066fccac16SSukadev Bhattiprolu if (window->tx_win) { 12076fccac16SSukadev Bhattiprolu val = read_hvwc_reg(window, VREG(TX_WCRED)); 12086fccac16SSukadev Bhattiprolu creds = GET_FIELD(VAS_TX_WCRED, val); 12096fccac16SSukadev Bhattiprolu } else { 12106fccac16SSukadev Bhattiprolu val = read_hvwc_reg(window, VREG(LRX_WCRED)); 12116fccac16SSukadev Bhattiprolu creds = GET_FIELD(VAS_LRX_WCRED, val); 12126fccac16SSukadev Bhattiprolu } 12136fccac16SSukadev Bhattiprolu 1214bd4da68dSHaren Myneni /* 1215bd4da68dSHaren Myneni * Takes around few milliseconds to complete all pending requests 1216bd4da68dSHaren Myneni * and return credits. 1217bd4da68dSHaren Myneni * TODO: Scan fault FIFO and invalidate CRBs points to this window 1218bd4da68dSHaren Myneni * and issue CRB Kill to stop all pending requests. Need only 1219bd4da68dSHaren Myneni * if there is a bug in NX or fault handling in kernel. 1220bd4da68dSHaren Myneni */ 12216fccac16SSukadev Bhattiprolu if (creds < window->wcreds_max) { 12226fccac16SSukadev Bhattiprolu val = 0; 12236fccac16SSukadev Bhattiprolu set_current_state(TASK_UNINTERRUPTIBLE); 12246fccac16SSukadev Bhattiprolu schedule_timeout(msecs_to_jiffies(10)); 1225bd4da68dSHaren Myneni count++; 1226bd4da68dSHaren Myneni /* 1227bd4da68dSHaren Myneni * Process can not close send window until all credits are 1228bd4da68dSHaren Myneni * returned. 1229bd4da68dSHaren Myneni */ 1230bd4da68dSHaren Myneni if (!(count % 1000)) 1231bd4da68dSHaren Myneni pr_warn_ratelimited("VAS: pid %d stuck. Waiting for credits returned for Window(%d). creds %d, Retries %d\n", 1232bd4da68dSHaren Myneni vas_window_pid(window), window->winid, 1233bd4da68dSHaren Myneni creds, count); 1234bd4da68dSHaren Myneni 12356fccac16SSukadev Bhattiprolu goto retry; 12366fccac16SSukadev Bhattiprolu } 12376fccac16SSukadev Bhattiprolu } 12386fccac16SSukadev Bhattiprolu 12396fccac16SSukadev Bhattiprolu /* 1240dfe954e4SSukadev Bhattiprolu * Wait for the window to go to "not-busy" state. It should only take a 1241dfe954e4SSukadev Bhattiprolu * short time to queue a CRB, so window should not be busy for too long. 1242dfe954e4SSukadev Bhattiprolu * Trying 5ms intervals. 1243dfe954e4SSukadev Bhattiprolu */ 124498271d41SSukadev Bhattiprolu static void poll_window_busy_state(struct vas_window *window) 124598271d41SSukadev Bhattiprolu { 124698271d41SSukadev Bhattiprolu int busy; 124798271d41SSukadev Bhattiprolu u64 val; 1248bd4da68dSHaren Myneni int count = 0; 124998271d41SSukadev Bhattiprolu 125098271d41SSukadev Bhattiprolu retry: 125198271d41SSukadev Bhattiprolu val = read_hvwc_reg(window, VREG(WIN_STATUS)); 125298271d41SSukadev Bhattiprolu busy = GET_FIELD(VAS_WIN_BUSY, val); 125398271d41SSukadev Bhattiprolu if (busy) { 125498271d41SSukadev Bhattiprolu val = 0; 125598271d41SSukadev Bhattiprolu set_current_state(TASK_UNINTERRUPTIBLE); 1256bd4da68dSHaren Myneni schedule_timeout(msecs_to_jiffies(10)); 1257bd4da68dSHaren Myneni count++; 1258bd4da68dSHaren Myneni /* 1259bd4da68dSHaren Myneni * Takes around few milliseconds to process all pending 1260bd4da68dSHaren Myneni * requests. 1261bd4da68dSHaren Myneni */ 1262bd4da68dSHaren Myneni if (!(count % 1000)) 1263bd4da68dSHaren Myneni pr_warn_ratelimited("VAS: pid %d stuck. Window (ID=%d) is in busy state. Retries %d\n", 1264bd4da68dSHaren Myneni vas_window_pid(window), window->winid, count); 1265bd4da68dSHaren Myneni 126698271d41SSukadev Bhattiprolu goto retry; 126798271d41SSukadev Bhattiprolu } 126898271d41SSukadev Bhattiprolu } 126998271d41SSukadev Bhattiprolu 12704963ac36SSukadev Bhattiprolu /* 12714963ac36SSukadev Bhattiprolu * Have the hardware cast a window out of cache and wait for it to 12724963ac36SSukadev Bhattiprolu * be completed. 12734963ac36SSukadev Bhattiprolu * 12744963ac36SSukadev Bhattiprolu * NOTE: It can take a relatively long time to cast the window context 12754963ac36SSukadev Bhattiprolu * out of the cache. It is not strictly necessary to cast out if: 12764963ac36SSukadev Bhattiprolu * 12774963ac36SSukadev Bhattiprolu * - we clear the "Pin Window" bit (so hardware is free to evict) 12784963ac36SSukadev Bhattiprolu * 12794963ac36SSukadev Bhattiprolu * - we re-initialize the window context when it is reassigned. 12804963ac36SSukadev Bhattiprolu * 12814963ac36SSukadev Bhattiprolu * We do the former in vas_win_close() and latter in vas_win_open(). 12824963ac36SSukadev Bhattiprolu * So, ignoring the cast-out for now. We can add it as needed. If 12834963ac36SSukadev Bhattiprolu * casting out becomes necessary we should consider offloading the 12844963ac36SSukadev Bhattiprolu * job to a worker thread, so the window close can proceed quickly. 12854963ac36SSukadev Bhattiprolu */ 128698271d41SSukadev Bhattiprolu static void poll_window_castout(struct vas_window *window) 128798271d41SSukadev Bhattiprolu { 12884963ac36SSukadev Bhattiprolu /* stub for now */ 128998271d41SSukadev Bhattiprolu } 129098271d41SSukadev Bhattiprolu 129198271d41SSukadev Bhattiprolu /* 129236a288feSSukadev Bhattiprolu * Unpin and close a window so no new requests are accepted and the 129336a288feSSukadev Bhattiprolu * hardware can evict this window from cache if necessary. 129436a288feSSukadev Bhattiprolu */ 129536a288feSSukadev Bhattiprolu static void unpin_close_window(struct vas_window *window) 129636a288feSSukadev Bhattiprolu { 129736a288feSSukadev Bhattiprolu u64 val; 129836a288feSSukadev Bhattiprolu 129936a288feSSukadev Bhattiprolu val = read_hvwc_reg(window, VREG(WINCTL)); 130036a288feSSukadev Bhattiprolu val = SET_FIELD(VAS_WINCTL_PIN, val, 0); 130136a288feSSukadev Bhattiprolu val = SET_FIELD(VAS_WINCTL_OPEN, val, 0); 130236a288feSSukadev Bhattiprolu write_hvwc_reg(window, VREG(WINCTL), val); 130336a288feSSukadev Bhattiprolu } 130436a288feSSukadev Bhattiprolu 130536a288feSSukadev Bhattiprolu /* 130698271d41SSukadev Bhattiprolu * Close a window. 130798271d41SSukadev Bhattiprolu * 130898271d41SSukadev Bhattiprolu * See Section 1.12.1 of VAS workbook v1.05 for details on closing window: 130998271d41SSukadev Bhattiprolu * - Disable new paste operations (unmap paste address) 131098271d41SSukadev Bhattiprolu * - Poll for the "Window Busy" bit to be cleared 131198271d41SSukadev Bhattiprolu * - Clear the Open/Enable bit for the Window. 131298271d41SSukadev Bhattiprolu * - Poll for return of window Credits (implies FIFO empty for Rx win?) 131398271d41SSukadev Bhattiprolu * - Unpin and cast window context out of cache 131498271d41SSukadev Bhattiprolu * 131598271d41SSukadev Bhattiprolu * Besides the hardware, kernel has some bookkeeping of course. 131698271d41SSukadev Bhattiprolu */ 13174dea2d1aSSukadev Bhattiprolu int vas_win_close(struct vas_window *window) 13184dea2d1aSSukadev Bhattiprolu { 131998271d41SSukadev Bhattiprolu if (!window) 132098271d41SSukadev Bhattiprolu return 0; 132198271d41SSukadev Bhattiprolu 132298271d41SSukadev Bhattiprolu if (!window->tx_win && atomic_read(&window->num_txwins) != 0) { 132398271d41SSukadev Bhattiprolu pr_devel("Attempting to close an active Rx window!\n"); 132498271d41SSukadev Bhattiprolu WARN_ON_ONCE(1); 132598271d41SSukadev Bhattiprolu return -EBUSY; 13264dea2d1aSSukadev Bhattiprolu } 132798271d41SSukadev Bhattiprolu 132898271d41SSukadev Bhattiprolu unmap_paste_region(window); 132998271d41SSukadev Bhattiprolu 133098271d41SSukadev Bhattiprolu poll_window_busy_state(window); 133198271d41SSukadev Bhattiprolu 133236a288feSSukadev Bhattiprolu unpin_close_window(window); 133398271d41SSukadev Bhattiprolu 13346fccac16SSukadev Bhattiprolu poll_window_credits(window); 13356fccac16SSukadev Bhattiprolu 13361d955f98SHaren Myneni clear_vinst_win(window); 13371d955f98SHaren Myneni 133898271d41SSukadev Bhattiprolu poll_window_castout(window); 133998271d41SSukadev Bhattiprolu 134098271d41SSukadev Bhattiprolu /* if send window, drop reference to matching receive window */ 1341db1c08a7SHaren Myneni if (window->tx_win) { 1342db1c08a7SHaren Myneni if (window->user_win) { 134391cdbb95SHaren Myneni /* Drop references to pid. tgid and mm */ 1344db1c08a7SHaren Myneni put_pid(window->pid); 134591cdbb95SHaren Myneni put_pid(window->tgid); 1346db1c08a7SHaren Myneni if (window->mm) { 1347c420644cSHaren Myneni mm_context_remove_vas_window(window->mm); 1348db1c08a7SHaren Myneni mmdrop(window->mm); 1349db1c08a7SHaren Myneni } 1350db1c08a7SHaren Myneni } 135198271d41SSukadev Bhattiprolu put_rx_win(window->rxwin); 1352db1c08a7SHaren Myneni } 135398271d41SSukadev Bhattiprolu 135498271d41SSukadev Bhattiprolu vas_window_free(window); 135598271d41SSukadev Bhattiprolu 135698271d41SSukadev Bhattiprolu return 0; 135798271d41SSukadev Bhattiprolu } 135898271d41SSukadev Bhattiprolu EXPORT_SYMBOL_GPL(vas_win_close); 13599774628aSHaren Myneni 1360461862efSHaren Myneni /* 1361461862efSHaren Myneni * Return credit for the given window. 1362461862efSHaren Myneni * Send windows and fault window uses credit mechanism as follows: 1363461862efSHaren Myneni * 1364461862efSHaren Myneni * Send windows: 1365461862efSHaren Myneni * - The default number of credits available for each send window is 1366461862efSHaren Myneni * 1024. It means 1024 requests can be issued asynchronously at the 1367461862efSHaren Myneni * same time. If the credit is not available, that request will be 1368461862efSHaren Myneni * returned with RMA_Busy. 1369461862efSHaren Myneni * - One credit is taken when NX request is issued. 1370461862efSHaren Myneni * - This credit is returned after NX processed that request. 1371461862efSHaren Myneni * - If NX encounters translation error, kernel will return the 1372461862efSHaren Myneni * credit on the specific send window after processing the fault CRB. 1373461862efSHaren Myneni * 1374461862efSHaren Myneni * Fault window: 1375461862efSHaren Myneni * - The total number credits available is FIFO_SIZE/CRB_SIZE. 1376461862efSHaren Myneni * Means 4MB/128 in the current implementation. If credit is not 1377461862efSHaren Myneni * available, RMA_Reject is returned. 1378461862efSHaren Myneni * - A credit is taken when NX pastes CRB in fault FIFO. 1379461862efSHaren Myneni * - The kernel with return credit on fault window after reading entry 1380461862efSHaren Myneni * from fault FIFO. 1381461862efSHaren Myneni */ 1382461862efSHaren Myneni void vas_return_credit(struct vas_window *window, bool tx) 1383461862efSHaren Myneni { 1384461862efSHaren Myneni uint64_t val; 1385461862efSHaren Myneni 1386461862efSHaren Myneni val = 0ULL; 1387461862efSHaren Myneni if (tx) { /* send window */ 1388461862efSHaren Myneni val = SET_FIELD(VAS_TX_WCRED, val, 1); 1389461862efSHaren Myneni write_hvwc_reg(window, VREG(TX_WCRED_ADDER), val); 1390461862efSHaren Myneni } else { 1391461862efSHaren Myneni val = SET_FIELD(VAS_LRX_WCRED, val, 1); 1392461862efSHaren Myneni write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), val); 1393461862efSHaren Myneni } 1394461862efSHaren Myneni } 1395461862efSHaren Myneni 13969774628aSHaren Myneni struct vas_window *vas_pswid_to_window(struct vas_instance *vinst, 13979774628aSHaren Myneni uint32_t pswid) 13989774628aSHaren Myneni { 13999774628aSHaren Myneni struct vas_window *window; 14009774628aSHaren Myneni int winid; 14019774628aSHaren Myneni 14029774628aSHaren Myneni if (!pswid) { 14039774628aSHaren Myneni pr_devel("%s: called for pswid 0!\n", __func__); 14049774628aSHaren Myneni return ERR_PTR(-ESRCH); 14059774628aSHaren Myneni } 14069774628aSHaren Myneni 14079774628aSHaren Myneni decode_pswid(pswid, NULL, &winid); 14089774628aSHaren Myneni 14099774628aSHaren Myneni if (winid >= VAS_WINDOWS_PER_CHIP) 14109774628aSHaren Myneni return ERR_PTR(-ESRCH); 14119774628aSHaren Myneni 14129774628aSHaren Myneni /* 14139774628aSHaren Myneni * If application closes the window before the hardware 14149774628aSHaren Myneni * returns the fault CRB, we should wait in vas_win_close() 14159774628aSHaren Myneni * for the pending requests. so the window must be active 14169774628aSHaren Myneni * and the process alive. 14179774628aSHaren Myneni * 14189774628aSHaren Myneni * If its a kernel process, we should not get any faults and 14199774628aSHaren Myneni * should not get here. 14209774628aSHaren Myneni */ 14219774628aSHaren Myneni window = vinst->windows[winid]; 14229774628aSHaren Myneni 14239774628aSHaren Myneni if (!window) { 14249774628aSHaren Myneni pr_err("PSWID decode: Could not find window for winid %d pswid %d vinst 0x%p\n", 14259774628aSHaren Myneni winid, pswid, vinst); 14269774628aSHaren Myneni return NULL; 14279774628aSHaren Myneni } 14289774628aSHaren Myneni 14299774628aSHaren Myneni /* 14309774628aSHaren Myneni * Do some sanity checks on the decoded window. Window should be 14319774628aSHaren Myneni * NX GZIP user send window. FTW windows should not incur faults 14329774628aSHaren Myneni * since their CRBs are ignored (not queued on FIFO or processed 14339774628aSHaren Myneni * by NX). 14349774628aSHaren Myneni */ 14359774628aSHaren Myneni if (!window->tx_win || !window->user_win || !window->nx_win || 14369774628aSHaren Myneni window->cop == VAS_COP_TYPE_FAULT || 14379774628aSHaren Myneni window->cop == VAS_COP_TYPE_FTW) { 14389774628aSHaren Myneni pr_err("PSWID decode: id %d, tx %d, user %d, nx %d, cop %d\n", 14399774628aSHaren Myneni winid, window->tx_win, window->user_win, 14409774628aSHaren Myneni window->nx_win, window->cop); 14419774628aSHaren Myneni WARN_ON(1); 14429774628aSHaren Myneni } 14439774628aSHaren Myneni 14449774628aSHaren Myneni return window; 14459774628aSHaren Myneni } 144606c6fad9SHaren Myneni 1447*1a0d0d5eSHaren Myneni static struct vas_window *vas_user_win_open(int vas_id, u64 flags, 1448*1a0d0d5eSHaren Myneni enum vas_cop_type cop_type) 1449*1a0d0d5eSHaren Myneni { 1450*1a0d0d5eSHaren Myneni struct vas_tx_win_attr txattr = {}; 1451*1a0d0d5eSHaren Myneni 1452*1a0d0d5eSHaren Myneni vas_init_tx_win_attr(&txattr, cop_type); 1453*1a0d0d5eSHaren Myneni 1454*1a0d0d5eSHaren Myneni txattr.lpid = mfspr(SPRN_LPID); 1455*1a0d0d5eSHaren Myneni txattr.pidr = mfspr(SPRN_PID); 1456*1a0d0d5eSHaren Myneni txattr.user_win = true; 1457*1a0d0d5eSHaren Myneni txattr.rsvd_txbuf_count = false; 1458*1a0d0d5eSHaren Myneni txattr.pswid = false; 1459*1a0d0d5eSHaren Myneni 1460*1a0d0d5eSHaren Myneni pr_devel("Pid %d: Opening txwin, PIDR %ld\n", txattr.pidr, 1461*1a0d0d5eSHaren Myneni mfspr(SPRN_PID)); 1462*1a0d0d5eSHaren Myneni 1463*1a0d0d5eSHaren Myneni return vas_tx_win_open(vas_id, cop_type, &txattr); 1464*1a0d0d5eSHaren Myneni } 1465*1a0d0d5eSHaren Myneni 1466*1a0d0d5eSHaren Myneni static u64 vas_user_win_paste_addr(struct vas_window *win) 1467*1a0d0d5eSHaren Myneni { 1468*1a0d0d5eSHaren Myneni u64 paste_addr; 1469*1a0d0d5eSHaren Myneni 1470*1a0d0d5eSHaren Myneni vas_win_paste_addr(win, &paste_addr, NULL); 1471*1a0d0d5eSHaren Myneni 1472*1a0d0d5eSHaren Myneni return paste_addr; 1473*1a0d0d5eSHaren Myneni } 1474*1a0d0d5eSHaren Myneni 1475*1a0d0d5eSHaren Myneni static int vas_user_win_close(struct vas_window *txwin) 1476*1a0d0d5eSHaren Myneni { 1477*1a0d0d5eSHaren Myneni 1478*1a0d0d5eSHaren Myneni vas_win_close(txwin); 1479*1a0d0d5eSHaren Myneni 1480*1a0d0d5eSHaren Myneni return 0; 1481*1a0d0d5eSHaren Myneni } 1482*1a0d0d5eSHaren Myneni 1483*1a0d0d5eSHaren Myneni static const struct vas_user_win_ops vops = { 1484*1a0d0d5eSHaren Myneni .open_win = vas_user_win_open, 1485*1a0d0d5eSHaren Myneni .paste_addr = vas_user_win_paste_addr, 1486*1a0d0d5eSHaren Myneni .close_win = vas_user_win_close, 1487*1a0d0d5eSHaren Myneni }; 1488*1a0d0d5eSHaren Myneni 148906c6fad9SHaren Myneni /* 149006c6fad9SHaren Myneni * Supporting only nx-gzip coprocessor type now, but this API code 149106c6fad9SHaren Myneni * extended to other coprocessor types later. 149206c6fad9SHaren Myneni */ 149306c6fad9SHaren Myneni int vas_register_api_powernv(struct module *mod, enum vas_cop_type cop_type, 149406c6fad9SHaren Myneni const char *name) 149506c6fad9SHaren Myneni { 149606c6fad9SHaren Myneni 1497*1a0d0d5eSHaren Myneni return vas_register_coproc_api(mod, cop_type, name, &vops); 149806c6fad9SHaren Myneni } 149906c6fad9SHaren Myneni EXPORT_SYMBOL_GPL(vas_register_api_powernv); 150006c6fad9SHaren Myneni 150106c6fad9SHaren Myneni void vas_unregister_api_powernv(void) 150206c6fad9SHaren Myneni { 150306c6fad9SHaren Myneni vas_unregister_coproc_api(); 150406c6fad9SHaren Myneni } 150506c6fad9SHaren Myneni EXPORT_SYMBOL_GPL(vas_unregister_api_powernv); 1506