1 /* 2 * Copyright 2016-17 IBM Corp. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; either version 7 * 2 of the License, or (at your option) any later version. 8 */ 9 10 #define pr_fmt(fmt) "vas: " fmt 11 12 #include <linux/types.h> 13 #include <linux/mutex.h> 14 #include <linux/slab.h> 15 #include <linux/io.h> 16 #include <linux/log2.h> 17 #include <linux/rcupdate.h> 18 #include <linux/cred.h> 19 20 #include "vas.h" 21 #include "copy-paste.h" 22 23 /* 24 * Compute the paste address region for the window @window using the 25 * ->paste_base_addr and ->paste_win_id_shift we got from device tree. 26 */ 27 static void compute_paste_address(struct vas_window *window, u64 *addr, int *len) 28 { 29 int winid; 30 u64 base, shift; 31 32 base = window->vinst->paste_base_addr; 33 shift = window->vinst->paste_win_id_shift; 34 winid = window->winid; 35 36 *addr = base + (winid << shift); 37 if (len) 38 *len = PAGE_SIZE; 39 40 pr_debug("Txwin #%d: Paste addr 0x%llx\n", winid, *addr); 41 } 42 43 static inline void get_hvwc_mmio_bar(struct vas_window *window, 44 u64 *start, int *len) 45 { 46 u64 pbaddr; 47 48 pbaddr = window->vinst->hvwc_bar_start; 49 *start = pbaddr + window->winid * VAS_HVWC_SIZE; 50 *len = VAS_HVWC_SIZE; 51 } 52 53 static inline void get_uwc_mmio_bar(struct vas_window *window, 54 u64 *start, int *len) 55 { 56 u64 pbaddr; 57 58 pbaddr = window->vinst->uwc_bar_start; 59 *start = pbaddr + window->winid * VAS_UWC_SIZE; 60 *len = VAS_UWC_SIZE; 61 } 62 63 /* 64 * Map the paste bus address of the given send window into kernel address 65 * space. Unlike MMIO regions (map_mmio_region() below), paste region must 66 * be mapped cache-able and is only applicable to send windows. 67 */ 68 static void *map_paste_region(struct vas_window *txwin) 69 { 70 int len; 71 void *map; 72 char *name; 73 u64 start; 74 75 name = kasprintf(GFP_KERNEL, "window-v%d-w%d", txwin->vinst->vas_id, 76 txwin->winid); 77 if (!name) 78 goto free_name; 79 80 txwin->paste_addr_name = name; 81 compute_paste_address(txwin, &start, &len); 82 83 if (!request_mem_region(start, len, name)) { 84 pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n", 85 __func__, start, len); 86 goto free_name; 87 } 88 89 map = ioremap_cache(start, len); 90 if (!map) { 91 pr_devel("%s(): ioremap_cache(0x%llx, %d) failed\n", __func__, 92 start, len); 93 goto free_name; 94 } 95 96 pr_devel("Mapped paste addr 0x%llx to kaddr 0x%p\n", start, map); 97 return map; 98 99 free_name: 100 kfree(name); 101 return ERR_PTR(-ENOMEM); 102 } 103 104 static void *map_mmio_region(char *name, u64 start, int len) 105 { 106 void *map; 107 108 if (!request_mem_region(start, len, name)) { 109 pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n", 110 __func__, start, len); 111 return NULL; 112 } 113 114 map = ioremap(start, len); 115 if (!map) { 116 pr_devel("%s(): ioremap(0x%llx, %d) failed\n", __func__, start, 117 len); 118 return NULL; 119 } 120 121 return map; 122 } 123 124 static void unmap_region(void *addr, u64 start, int len) 125 { 126 iounmap(addr); 127 release_mem_region((phys_addr_t)start, len); 128 } 129 130 /* 131 * Unmap the paste address region for a window. 132 */ 133 static void unmap_paste_region(struct vas_window *window) 134 { 135 int len; 136 u64 busaddr_start; 137 138 if (window->paste_kaddr) { 139 compute_paste_address(window, &busaddr_start, &len); 140 unmap_region(window->paste_kaddr, busaddr_start, len); 141 window->paste_kaddr = NULL; 142 kfree(window->paste_addr_name); 143 window->paste_addr_name = NULL; 144 } 145 } 146 147 /* 148 * Unmap the MMIO regions for a window. Hold the vas_mutex so we don't 149 * unmap when the window's debugfs dir is in use. This serializes close 150 * of a window even on another VAS instance but since its not a critical 151 * path, just minimize the time we hold the mutex for now. We can add 152 * a per-instance mutex later if necessary. 153 */ 154 static void unmap_winctx_mmio_bars(struct vas_window *window) 155 { 156 int len; 157 void *uwc_map; 158 void *hvwc_map; 159 u64 busaddr_start; 160 161 mutex_lock(&vas_mutex); 162 163 hvwc_map = window->hvwc_map; 164 window->hvwc_map = NULL; 165 166 uwc_map = window->uwc_map; 167 window->uwc_map = NULL; 168 169 mutex_unlock(&vas_mutex); 170 171 if (hvwc_map) { 172 get_hvwc_mmio_bar(window, &busaddr_start, &len); 173 unmap_region(hvwc_map, busaddr_start, len); 174 } 175 176 if (uwc_map) { 177 get_uwc_mmio_bar(window, &busaddr_start, &len); 178 unmap_region(uwc_map, busaddr_start, len); 179 } 180 } 181 182 /* 183 * Find the Hypervisor Window Context (HVWC) MMIO Base Address Region and the 184 * OS/User Window Context (UWC) MMIO Base Address Region for the given window. 185 * Map these bus addresses and save the mapped kernel addresses in @window. 186 */ 187 int map_winctx_mmio_bars(struct vas_window *window) 188 { 189 int len; 190 u64 start; 191 192 get_hvwc_mmio_bar(window, &start, &len); 193 window->hvwc_map = map_mmio_region("HVWCM_Window", start, len); 194 195 get_uwc_mmio_bar(window, &start, &len); 196 window->uwc_map = map_mmio_region("UWCM_Window", start, len); 197 198 if (!window->hvwc_map || !window->uwc_map) { 199 unmap_winctx_mmio_bars(window); 200 return -1; 201 } 202 203 return 0; 204 } 205 206 /* 207 * Reset all valid registers in the HV and OS/User Window Contexts for 208 * the window identified by @window. 209 * 210 * NOTE: We cannot really use a for loop to reset window context. Not all 211 * offsets in a window context are valid registers and the valid 212 * registers are not sequential. And, we can only write to offsets 213 * with valid registers. 214 */ 215 void reset_window_regs(struct vas_window *window) 216 { 217 write_hvwc_reg(window, VREG(LPID), 0ULL); 218 write_hvwc_reg(window, VREG(PID), 0ULL); 219 write_hvwc_reg(window, VREG(XLATE_MSR), 0ULL); 220 write_hvwc_reg(window, VREG(XLATE_LPCR), 0ULL); 221 write_hvwc_reg(window, VREG(XLATE_CTL), 0ULL); 222 write_hvwc_reg(window, VREG(AMR), 0ULL); 223 write_hvwc_reg(window, VREG(SEIDR), 0ULL); 224 write_hvwc_reg(window, VREG(FAULT_TX_WIN), 0ULL); 225 write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL); 226 write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), 0ULL); 227 write_hvwc_reg(window, VREG(PSWID), 0ULL); 228 write_hvwc_reg(window, VREG(LFIFO_BAR), 0ULL); 229 write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), 0ULL); 230 write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), 0ULL); 231 write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL); 232 write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL); 233 write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL); 234 write_hvwc_reg(window, VREG(LRX_WCRED), 0ULL); 235 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL); 236 write_hvwc_reg(window, VREG(TX_WCRED), 0ULL); 237 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL); 238 write_hvwc_reg(window, VREG(LFIFO_SIZE), 0ULL); 239 write_hvwc_reg(window, VREG(WINCTL), 0ULL); 240 write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL); 241 write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), 0ULL); 242 write_hvwc_reg(window, VREG(TX_RSVD_BUF_COUNT), 0ULL); 243 write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), 0ULL); 244 write_hvwc_reg(window, VREG(LNOTIFY_CTL), 0ULL); 245 write_hvwc_reg(window, VREG(LNOTIFY_PID), 0ULL); 246 write_hvwc_reg(window, VREG(LNOTIFY_LPID), 0ULL); 247 write_hvwc_reg(window, VREG(LNOTIFY_TID), 0ULL); 248 write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), 0ULL); 249 write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL); 250 251 /* Skip read-only registers: NX_UTIL and NX_UTIL_SE */ 252 253 /* 254 * The send and receive window credit adder registers are also 255 * accessible from HVWC and have been initialized above. We don't 256 * need to initialize from the OS/User Window Context, so skip 257 * following calls: 258 * 259 * write_uwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL); 260 * write_uwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL); 261 */ 262 } 263 264 /* 265 * Initialize window context registers related to Address Translation. 266 * These registers are common to send/receive windows although they 267 * differ for user/kernel windows. As we resolve the TODOs we may 268 * want to add fields to vas_winctx and move the initialization to 269 * init_vas_winctx_regs(). 270 */ 271 static void init_xlate_regs(struct vas_window *window, bool user_win) 272 { 273 u64 lpcr, val; 274 275 /* 276 * MSR_TA, MSR_US are false for both kernel and user. 277 * MSR_DR and MSR_PR are false for kernel. 278 */ 279 val = 0ULL; 280 val = SET_FIELD(VAS_XLATE_MSR_HV, val, 1); 281 val = SET_FIELD(VAS_XLATE_MSR_SF, val, 1); 282 if (user_win) { 283 val = SET_FIELD(VAS_XLATE_MSR_DR, val, 1); 284 val = SET_FIELD(VAS_XLATE_MSR_PR, val, 1); 285 } 286 write_hvwc_reg(window, VREG(XLATE_MSR), val); 287 288 lpcr = mfspr(SPRN_LPCR); 289 val = 0ULL; 290 /* 291 * NOTE: From Section 5.7.8.1 Segment Lookaside Buffer of the 292 * Power ISA, v3.0B, Page size encoding is 0 = 4KB, 5 = 64KB. 293 * 294 * NOTE: From Section 1.3.1, Address Translation Context of the 295 * Nest MMU Workbook, LPCR_SC should be 0 for Power9. 296 */ 297 val = SET_FIELD(VAS_XLATE_LPCR_PAGE_SIZE, val, 5); 298 val = SET_FIELD(VAS_XLATE_LPCR_ISL, val, lpcr & LPCR_ISL); 299 val = SET_FIELD(VAS_XLATE_LPCR_TC, val, lpcr & LPCR_TC); 300 val = SET_FIELD(VAS_XLATE_LPCR_SC, val, 0); 301 write_hvwc_reg(window, VREG(XLATE_LPCR), val); 302 303 /* 304 * Section 1.3.1 (Address translation Context) of NMMU workbook. 305 * 0b00 Hashed Page Table mode 306 * 0b01 Reserved 307 * 0b10 Radix on HPT 308 * 0b11 Radix on Radix 309 */ 310 val = 0ULL; 311 val = SET_FIELD(VAS_XLATE_MODE, val, radix_enabled() ? 3 : 2); 312 write_hvwc_reg(window, VREG(XLATE_CTL), val); 313 314 /* 315 * TODO: Can we mfspr(AMR) even for user windows? 316 */ 317 val = 0ULL; 318 val = SET_FIELD(VAS_AMR, val, mfspr(SPRN_AMR)); 319 write_hvwc_reg(window, VREG(AMR), val); 320 321 val = 0ULL; 322 val = SET_FIELD(VAS_SEIDR, val, 0); 323 write_hvwc_reg(window, VREG(SEIDR), val); 324 } 325 326 /* 327 * Initialize Reserved Send Buffer Count for the send window. It involves 328 * writing to the register, reading it back to confirm that the hardware 329 * has enough buffers to reserve. See section 1.3.1.2.1 of VAS workbook. 330 * 331 * Since we can only make a best-effort attempt to fulfill the request, 332 * we don't return any errors if we cannot. 333 * 334 * TODO: Reserved (aka dedicated) send buffers are not supported yet. 335 */ 336 static void init_rsvd_tx_buf_count(struct vas_window *txwin, 337 struct vas_winctx *winctx) 338 { 339 write_hvwc_reg(txwin, VREG(TX_RSVD_BUF_COUNT), 0ULL); 340 } 341 342 /* 343 * init_winctx_regs() 344 * Initialize window context registers for a receive window. 345 * Except for caching control and marking window open, the registers 346 * are initialized in the order listed in Section 3.1.4 (Window Context 347 * Cache Register Details) of the VAS workbook although they don't need 348 * to be. 349 * 350 * Design note: For NX receive windows, NX allocates the FIFO buffer in OPAL 351 * (so that it can get a large contiguous area) and passes that buffer 352 * to kernel via device tree. We now write that buffer address to the 353 * FIFO BAR. Would it make sense to do this all in OPAL? i.e have OPAL 354 * write the per-chip RX FIFO addresses to the windows during boot-up 355 * as a one-time task? That could work for NX but what about other 356 * receivers? Let the receivers tell us the rx-fifo buffers for now. 357 */ 358 int init_winctx_regs(struct vas_window *window, struct vas_winctx *winctx) 359 { 360 u64 val; 361 int fifo_size; 362 363 reset_window_regs(window); 364 365 val = 0ULL; 366 val = SET_FIELD(VAS_LPID, val, winctx->lpid); 367 write_hvwc_reg(window, VREG(LPID), val); 368 369 val = 0ULL; 370 val = SET_FIELD(VAS_PID_ID, val, winctx->pidr); 371 write_hvwc_reg(window, VREG(PID), val); 372 373 init_xlate_regs(window, winctx->user_win); 374 375 val = 0ULL; 376 val = SET_FIELD(VAS_FAULT_TX_WIN, val, 0); 377 write_hvwc_reg(window, VREG(FAULT_TX_WIN), val); 378 379 /* In PowerNV, interrupts go to HV. */ 380 write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL); 381 382 val = 0ULL; 383 val = SET_FIELD(VAS_HV_INTR_SRC_RA, val, winctx->irq_port); 384 write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), val); 385 386 val = 0ULL; 387 val = SET_FIELD(VAS_PSWID_EA_HANDLE, val, winctx->pswid); 388 write_hvwc_reg(window, VREG(PSWID), val); 389 390 write_hvwc_reg(window, VREG(SPARE1), 0ULL); 391 write_hvwc_reg(window, VREG(SPARE2), 0ULL); 392 write_hvwc_reg(window, VREG(SPARE3), 0ULL); 393 394 /* 395 * NOTE: VAS expects the FIFO address to be copied into the LFIFO_BAR 396 * register as is - do NOT shift the address into VAS_LFIFO_BAR 397 * bit fields! Ok to set the page migration select fields - 398 * VAS ignores the lower 10+ bits in the address anyway, because 399 * the minimum FIFO size is 1K? 400 * 401 * See also: Design note in function header. 402 */ 403 val = __pa(winctx->rx_fifo); 404 val = SET_FIELD(VAS_PAGE_MIGRATION_SELECT, val, 0); 405 write_hvwc_reg(window, VREG(LFIFO_BAR), val); 406 407 val = 0ULL; 408 val = SET_FIELD(VAS_LDATA_STAMP, val, winctx->data_stamp); 409 write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), val); 410 411 val = 0ULL; 412 val = SET_FIELD(VAS_LDMA_TYPE, val, winctx->dma_type); 413 val = SET_FIELD(VAS_LDMA_FIFO_DISABLE, val, winctx->fifo_disable); 414 write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), val); 415 416 write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL); 417 write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL); 418 write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL); 419 420 val = 0ULL; 421 val = SET_FIELD(VAS_LRX_WCRED, val, winctx->wcreds_max); 422 write_hvwc_reg(window, VREG(LRX_WCRED), val); 423 424 val = 0ULL; 425 val = SET_FIELD(VAS_TX_WCRED, val, winctx->wcreds_max); 426 write_hvwc_reg(window, VREG(TX_WCRED), val); 427 428 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL); 429 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL); 430 431 fifo_size = winctx->rx_fifo_size / 1024; 432 433 val = 0ULL; 434 val = SET_FIELD(VAS_LFIFO_SIZE, val, ilog2(fifo_size)); 435 write_hvwc_reg(window, VREG(LFIFO_SIZE), val); 436 437 /* Update window control and caching control registers last so 438 * we mark the window open only after fully initializing it and 439 * pushing context to cache. 440 */ 441 442 write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL); 443 444 init_rsvd_tx_buf_count(window, winctx); 445 446 /* for a send window, point to the matching receive window */ 447 val = 0ULL; 448 val = SET_FIELD(VAS_LRX_WIN_ID, val, winctx->rx_win_id); 449 write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), val); 450 451 write_hvwc_reg(window, VREG(SPARE4), 0ULL); 452 453 val = 0ULL; 454 val = SET_FIELD(VAS_NOTIFY_DISABLE, val, winctx->notify_disable); 455 val = SET_FIELD(VAS_INTR_DISABLE, val, winctx->intr_disable); 456 val = SET_FIELD(VAS_NOTIFY_EARLY, val, winctx->notify_early); 457 val = SET_FIELD(VAS_NOTIFY_OSU_INTR, val, winctx->notify_os_intr_reg); 458 write_hvwc_reg(window, VREG(LNOTIFY_CTL), val); 459 460 val = 0ULL; 461 val = SET_FIELD(VAS_LNOTIFY_PID, val, winctx->lnotify_pid); 462 write_hvwc_reg(window, VREG(LNOTIFY_PID), val); 463 464 val = 0ULL; 465 val = SET_FIELD(VAS_LNOTIFY_LPID, val, winctx->lnotify_lpid); 466 write_hvwc_reg(window, VREG(LNOTIFY_LPID), val); 467 468 val = 0ULL; 469 val = SET_FIELD(VAS_LNOTIFY_TID, val, winctx->lnotify_tid); 470 write_hvwc_reg(window, VREG(LNOTIFY_TID), val); 471 472 val = 0ULL; 473 val = SET_FIELD(VAS_LNOTIFY_MIN_SCOPE, val, winctx->min_scope); 474 val = SET_FIELD(VAS_LNOTIFY_MAX_SCOPE, val, winctx->max_scope); 475 write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), val); 476 477 /* Skip read-only registers NX_UTIL and NX_UTIL_SE */ 478 479 write_hvwc_reg(window, VREG(SPARE5), 0ULL); 480 write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL); 481 write_hvwc_reg(window, VREG(SPARE6), 0ULL); 482 483 /* Finally, push window context to memory and... */ 484 val = 0ULL; 485 val = SET_FIELD(VAS_PUSH_TO_MEM, val, 1); 486 write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), val); 487 488 /* ... mark the window open for business */ 489 val = 0ULL; 490 val = SET_FIELD(VAS_WINCTL_REJ_NO_CREDIT, val, winctx->rej_no_credit); 491 val = SET_FIELD(VAS_WINCTL_PIN, val, winctx->pin_win); 492 val = SET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val, winctx->tx_wcred_mode); 493 val = SET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val, winctx->rx_wcred_mode); 494 val = SET_FIELD(VAS_WINCTL_TX_WORD_MODE, val, winctx->tx_word_mode); 495 val = SET_FIELD(VAS_WINCTL_RX_WORD_MODE, val, winctx->rx_word_mode); 496 val = SET_FIELD(VAS_WINCTL_FAULT_WIN, val, winctx->fault_win); 497 val = SET_FIELD(VAS_WINCTL_NX_WIN, val, winctx->nx_win); 498 val = SET_FIELD(VAS_WINCTL_OPEN, val, 1); 499 write_hvwc_reg(window, VREG(WINCTL), val); 500 501 return 0; 502 } 503 504 static DEFINE_SPINLOCK(vas_ida_lock); 505 506 static void vas_release_window_id(struct ida *ida, int winid) 507 { 508 spin_lock(&vas_ida_lock); 509 ida_remove(ida, winid); 510 spin_unlock(&vas_ida_lock); 511 } 512 513 static int vas_assign_window_id(struct ida *ida) 514 { 515 int rc, winid; 516 517 do { 518 rc = ida_pre_get(ida, GFP_KERNEL); 519 if (!rc) 520 return -EAGAIN; 521 522 spin_lock(&vas_ida_lock); 523 rc = ida_get_new(ida, &winid); 524 spin_unlock(&vas_ida_lock); 525 } while (rc == -EAGAIN); 526 527 if (rc) 528 return rc; 529 530 if (winid > VAS_WINDOWS_PER_CHIP) { 531 pr_err("Too many (%d) open windows\n", winid); 532 vas_release_window_id(ida, winid); 533 return -EAGAIN; 534 } 535 536 return winid; 537 } 538 539 static void vas_window_free(struct vas_window *window) 540 { 541 int winid = window->winid; 542 struct vas_instance *vinst = window->vinst; 543 544 unmap_winctx_mmio_bars(window); 545 546 vas_window_free_dbgdir(window); 547 548 kfree(window); 549 550 vas_release_window_id(&vinst->ida, winid); 551 } 552 553 static struct vas_window *vas_window_alloc(struct vas_instance *vinst) 554 { 555 int winid; 556 struct vas_window *window; 557 558 winid = vas_assign_window_id(&vinst->ida); 559 if (winid < 0) 560 return ERR_PTR(winid); 561 562 window = kzalloc(sizeof(*window), GFP_KERNEL); 563 if (!window) 564 goto out_free; 565 566 window->vinst = vinst; 567 window->winid = winid; 568 569 if (map_winctx_mmio_bars(window)) 570 goto out_free; 571 572 vas_window_init_dbgdir(window); 573 574 return window; 575 576 out_free: 577 kfree(window); 578 vas_release_window_id(&vinst->ida, winid); 579 return ERR_PTR(-ENOMEM); 580 } 581 582 static void put_rx_win(struct vas_window *rxwin) 583 { 584 /* Better not be a send window! */ 585 WARN_ON_ONCE(rxwin->tx_win); 586 587 atomic_dec(&rxwin->num_txwins); 588 } 589 590 /* 591 * Get the VAS receive window associated with NX engine identified 592 * by @cop and if applicable, @pswid. 593 * 594 * See also function header of set_vinst_win(). 595 */ 596 static struct vas_window *get_vinst_rxwin(struct vas_instance *vinst, 597 enum vas_cop_type cop, u32 pswid) 598 { 599 struct vas_window *rxwin; 600 601 mutex_lock(&vinst->mutex); 602 603 if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI) 604 rxwin = vinst->rxwin[cop] ?: ERR_PTR(-EINVAL); 605 else 606 rxwin = ERR_PTR(-EINVAL); 607 608 if (!IS_ERR(rxwin)) 609 atomic_inc(&rxwin->num_txwins); 610 611 mutex_unlock(&vinst->mutex); 612 613 return rxwin; 614 } 615 616 /* 617 * We have two tables of windows in a VAS instance. The first one, 618 * ->windows[], contains all the windows in the instance and allows 619 * looking up a window by its id. It is used to look up send windows 620 * during fault handling and receive windows when pairing user space 621 * send/receive windows. 622 * 623 * The second table, ->rxwin[], contains receive windows that are 624 * associated with NX engines. This table has VAS_COP_TYPE_MAX 625 * entries and is used to look up a receive window by its 626 * coprocessor type. 627 * 628 * Here, we save @window in the ->windows[] table. If it is a receive 629 * window, we also save the window in the ->rxwin[] table. 630 */ 631 static void set_vinst_win(struct vas_instance *vinst, 632 struct vas_window *window) 633 { 634 int id = window->winid; 635 636 mutex_lock(&vinst->mutex); 637 638 /* 639 * There should only be one receive window for a coprocessor type 640 * unless its a user (FTW) window. 641 */ 642 if (!window->user_win && !window->tx_win) { 643 WARN_ON_ONCE(vinst->rxwin[window->cop]); 644 vinst->rxwin[window->cop] = window; 645 } 646 647 WARN_ON_ONCE(vinst->windows[id] != NULL); 648 vinst->windows[id] = window; 649 650 mutex_unlock(&vinst->mutex); 651 } 652 653 /* 654 * Clear this window from the table(s) of windows for this VAS instance. 655 * See also function header of set_vinst_win(). 656 */ 657 static void clear_vinst_win(struct vas_window *window) 658 { 659 int id = window->winid; 660 struct vas_instance *vinst = window->vinst; 661 662 mutex_lock(&vinst->mutex); 663 664 if (!window->user_win && !window->tx_win) { 665 WARN_ON_ONCE(!vinst->rxwin[window->cop]); 666 vinst->rxwin[window->cop] = NULL; 667 } 668 669 WARN_ON_ONCE(vinst->windows[id] != window); 670 vinst->windows[id] = NULL; 671 672 mutex_unlock(&vinst->mutex); 673 } 674 675 static void init_winctx_for_rxwin(struct vas_window *rxwin, 676 struct vas_rx_win_attr *rxattr, 677 struct vas_winctx *winctx) 678 { 679 /* 680 * We first zero (memset()) all fields and only set non-zero fields. 681 * Following fields are 0/false but maybe deserve a comment: 682 * 683 * ->notify_os_intr_reg In powerNV, send intrs to HV 684 * ->notify_disable False for NX windows 685 * ->intr_disable False for Fault Windows 686 * ->xtra_write False for NX windows 687 * ->notify_early NA for NX windows 688 * ->rsvd_txbuf_count NA for Rx windows 689 * ->lpid, ->pid, ->tid NA for Rx windows 690 */ 691 692 memset(winctx, 0, sizeof(struct vas_winctx)); 693 694 winctx->rx_fifo = rxattr->rx_fifo; 695 winctx->rx_fifo_size = rxattr->rx_fifo_size; 696 winctx->wcreds_max = rxwin->wcreds_max; 697 winctx->pin_win = rxattr->pin_win; 698 699 winctx->nx_win = rxattr->nx_win; 700 winctx->fault_win = rxattr->fault_win; 701 winctx->user_win = rxattr->user_win; 702 winctx->rej_no_credit = rxattr->rej_no_credit; 703 winctx->rx_word_mode = rxattr->rx_win_ord_mode; 704 winctx->tx_word_mode = rxattr->tx_win_ord_mode; 705 winctx->rx_wcred_mode = rxattr->rx_wcred_mode; 706 winctx->tx_wcred_mode = rxattr->tx_wcred_mode; 707 winctx->notify_early = rxattr->notify_early; 708 709 if (winctx->nx_win) { 710 winctx->data_stamp = true; 711 winctx->intr_disable = true; 712 winctx->pin_win = true; 713 714 WARN_ON_ONCE(winctx->fault_win); 715 WARN_ON_ONCE(!winctx->rx_word_mode); 716 WARN_ON_ONCE(!winctx->tx_word_mode); 717 WARN_ON_ONCE(winctx->notify_after_count); 718 } else if (winctx->fault_win) { 719 winctx->notify_disable = true; 720 } else if (winctx->user_win) { 721 /* 722 * Section 1.8.1 Low Latency Core-Core Wake up of 723 * the VAS workbook: 724 * 725 * - disable credit checks ([tr]x_wcred_mode = false) 726 * - disable FIFO writes 727 * - enable ASB_Notify, disable interrupt 728 */ 729 winctx->fifo_disable = true; 730 winctx->intr_disable = true; 731 winctx->rx_fifo = NULL; 732 } 733 734 winctx->lnotify_lpid = rxattr->lnotify_lpid; 735 winctx->lnotify_pid = rxattr->lnotify_pid; 736 winctx->lnotify_tid = rxattr->lnotify_tid; 737 winctx->pswid = rxattr->pswid; 738 winctx->dma_type = VAS_DMA_TYPE_INJECT; 739 winctx->tc_mode = rxattr->tc_mode; 740 741 winctx->min_scope = VAS_SCOPE_LOCAL; 742 winctx->max_scope = VAS_SCOPE_VECTORED_GROUP; 743 } 744 745 static bool rx_win_args_valid(enum vas_cop_type cop, 746 struct vas_rx_win_attr *attr) 747 { 748 pr_debug("Rxattr: fault %d, notify %d, intr %d, early %d, fifo %d\n", 749 attr->fault_win, attr->notify_disable, 750 attr->intr_disable, attr->notify_early, 751 attr->rx_fifo_size); 752 753 if (cop >= VAS_COP_TYPE_MAX) 754 return false; 755 756 if (cop != VAS_COP_TYPE_FTW && 757 attr->rx_fifo_size < VAS_RX_FIFO_SIZE_MIN) 758 return false; 759 760 if (attr->rx_fifo_size > VAS_RX_FIFO_SIZE_MAX) 761 return false; 762 763 if (attr->wcreds_max > VAS_RX_WCREDS_MAX) 764 return false; 765 766 if (attr->nx_win) { 767 /* cannot be fault or user window if it is nx */ 768 if (attr->fault_win || attr->user_win) 769 return false; 770 /* 771 * Section 3.1.4.32: NX Windows must not disable notification, 772 * and must not enable interrupts or early notification. 773 */ 774 if (attr->notify_disable || !attr->intr_disable || 775 attr->notify_early) 776 return false; 777 } else if (attr->fault_win) { 778 /* cannot be both fault and user window */ 779 if (attr->user_win) 780 return false; 781 782 /* 783 * Section 3.1.4.32: Fault windows must disable notification 784 * but not interrupts. 785 */ 786 if (!attr->notify_disable || attr->intr_disable) 787 return false; 788 789 } else if (attr->user_win) { 790 /* 791 * User receive windows are only for fast-thread-wakeup 792 * (FTW). They don't need a FIFO and must disable interrupts 793 */ 794 if (attr->rx_fifo || attr->rx_fifo_size || !attr->intr_disable) 795 return false; 796 } else { 797 /* Rx window must be one of NX or Fault or User window. */ 798 return false; 799 } 800 801 return true; 802 } 803 804 void vas_init_rx_win_attr(struct vas_rx_win_attr *rxattr, enum vas_cop_type cop) 805 { 806 memset(rxattr, 0, sizeof(*rxattr)); 807 808 if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI) { 809 rxattr->pin_win = true; 810 rxattr->nx_win = true; 811 rxattr->fault_win = false; 812 rxattr->intr_disable = true; 813 rxattr->rx_wcred_mode = true; 814 rxattr->tx_wcred_mode = true; 815 rxattr->rx_win_ord_mode = true; 816 rxattr->tx_win_ord_mode = true; 817 } else if (cop == VAS_COP_TYPE_FAULT) { 818 rxattr->pin_win = true; 819 rxattr->fault_win = true; 820 rxattr->notify_disable = true; 821 rxattr->rx_wcred_mode = true; 822 rxattr->tx_wcred_mode = true; 823 rxattr->rx_win_ord_mode = true; 824 rxattr->tx_win_ord_mode = true; 825 } else if (cop == VAS_COP_TYPE_FTW) { 826 rxattr->user_win = true; 827 rxattr->intr_disable = true; 828 829 /* 830 * As noted in the VAS Workbook we disable credit checks. 831 * If we enable credit checks in the future, we must also 832 * implement a mechanism to return the user credits or new 833 * paste operations will fail. 834 */ 835 } 836 } 837 EXPORT_SYMBOL_GPL(vas_init_rx_win_attr); 838 839 struct vas_window *vas_rx_win_open(int vasid, enum vas_cop_type cop, 840 struct vas_rx_win_attr *rxattr) 841 { 842 struct vas_window *rxwin; 843 struct vas_winctx winctx; 844 struct vas_instance *vinst; 845 846 if (!rx_win_args_valid(cop, rxattr)) 847 return ERR_PTR(-EINVAL); 848 849 vinst = find_vas_instance(vasid); 850 if (!vinst) { 851 pr_devel("vasid %d not found!\n", vasid); 852 return ERR_PTR(-EINVAL); 853 } 854 pr_devel("Found instance %d\n", vasid); 855 856 rxwin = vas_window_alloc(vinst); 857 if (IS_ERR(rxwin)) { 858 pr_devel("Unable to allocate memory for Rx window\n"); 859 return rxwin; 860 } 861 862 rxwin->tx_win = false; 863 rxwin->nx_win = rxattr->nx_win; 864 rxwin->user_win = rxattr->user_win; 865 rxwin->cop = cop; 866 rxwin->wcreds_max = rxattr->wcreds_max ?: VAS_WCREDS_DEFAULT; 867 if (rxattr->user_win) 868 rxwin->pid = task_pid_vnr(current); 869 870 init_winctx_for_rxwin(rxwin, rxattr, &winctx); 871 init_winctx_regs(rxwin, &winctx); 872 873 set_vinst_win(vinst, rxwin); 874 875 return rxwin; 876 } 877 EXPORT_SYMBOL_GPL(vas_rx_win_open); 878 879 void vas_init_tx_win_attr(struct vas_tx_win_attr *txattr, enum vas_cop_type cop) 880 { 881 memset(txattr, 0, sizeof(*txattr)); 882 883 if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI) { 884 txattr->rej_no_credit = false; 885 txattr->rx_wcred_mode = true; 886 txattr->tx_wcred_mode = true; 887 txattr->rx_win_ord_mode = true; 888 txattr->tx_win_ord_mode = true; 889 } else if (cop == VAS_COP_TYPE_FTW) { 890 txattr->user_win = true; 891 } 892 } 893 EXPORT_SYMBOL_GPL(vas_init_tx_win_attr); 894 895 static void init_winctx_for_txwin(struct vas_window *txwin, 896 struct vas_tx_win_attr *txattr, 897 struct vas_winctx *winctx) 898 { 899 /* 900 * We first zero all fields and only set non-zero ones. Following 901 * are some fields set to 0/false for the stated reason: 902 * 903 * ->notify_os_intr_reg In powernv, send intrs to HV 904 * ->rsvd_txbuf_count Not supported yet. 905 * ->notify_disable False for NX windows 906 * ->xtra_write False for NX windows 907 * ->notify_early NA for NX windows 908 * ->lnotify_lpid NA for Tx windows 909 * ->lnotify_pid NA for Tx windows 910 * ->lnotify_tid NA for Tx windows 911 * ->tx_win_cred_mode Ignore for now for NX windows 912 * ->rx_win_cred_mode Ignore for now for NX windows 913 */ 914 memset(winctx, 0, sizeof(struct vas_winctx)); 915 916 winctx->wcreds_max = txwin->wcreds_max; 917 918 winctx->user_win = txattr->user_win; 919 winctx->nx_win = txwin->rxwin->nx_win; 920 winctx->pin_win = txattr->pin_win; 921 winctx->rej_no_credit = txattr->rej_no_credit; 922 winctx->rsvd_txbuf_enable = txattr->rsvd_txbuf_enable; 923 924 winctx->rx_wcred_mode = txattr->rx_wcred_mode; 925 winctx->tx_wcred_mode = txattr->tx_wcred_mode; 926 winctx->rx_word_mode = txattr->rx_win_ord_mode; 927 winctx->tx_word_mode = txattr->tx_win_ord_mode; 928 winctx->rsvd_txbuf_count = txattr->rsvd_txbuf_count; 929 930 if (winctx->nx_win) { 931 winctx->data_stamp = true; 932 winctx->intr_disable = true; 933 } 934 935 winctx->lpid = txattr->lpid; 936 winctx->pidr = txattr->pidr; 937 winctx->rx_win_id = txwin->rxwin->winid; 938 939 winctx->dma_type = VAS_DMA_TYPE_INJECT; 940 winctx->tc_mode = txattr->tc_mode; 941 winctx->min_scope = VAS_SCOPE_LOCAL; 942 winctx->max_scope = VAS_SCOPE_VECTORED_GROUP; 943 944 winctx->pswid = 0; 945 } 946 947 static bool tx_win_args_valid(enum vas_cop_type cop, 948 struct vas_tx_win_attr *attr) 949 { 950 if (attr->tc_mode != VAS_THRESH_DISABLED) 951 return false; 952 953 if (cop > VAS_COP_TYPE_MAX) 954 return false; 955 956 if (attr->wcreds_max > VAS_TX_WCREDS_MAX) 957 return false; 958 959 if (attr->user_win && 960 (cop != VAS_COP_TYPE_FTW || attr->rsvd_txbuf_count)) 961 return false; 962 963 return true; 964 } 965 966 struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop, 967 struct vas_tx_win_attr *attr) 968 { 969 int rc; 970 struct vas_window *txwin; 971 struct vas_window *rxwin; 972 struct vas_winctx winctx; 973 struct vas_instance *vinst; 974 975 if (!tx_win_args_valid(cop, attr)) 976 return ERR_PTR(-EINVAL); 977 978 vinst = find_vas_instance(vasid); 979 if (!vinst) { 980 pr_devel("vasid %d not found!\n", vasid); 981 return ERR_PTR(-EINVAL); 982 } 983 984 rxwin = get_vinst_rxwin(vinst, cop, attr->pswid); 985 if (IS_ERR(rxwin)) { 986 pr_devel("No RxWin for vasid %d, cop %d\n", vasid, cop); 987 return rxwin; 988 } 989 990 txwin = vas_window_alloc(vinst); 991 if (IS_ERR(txwin)) { 992 rc = PTR_ERR(txwin); 993 goto put_rxwin; 994 } 995 996 txwin->cop = cop; 997 txwin->tx_win = 1; 998 txwin->rxwin = rxwin; 999 txwin->nx_win = txwin->rxwin->nx_win; 1000 txwin->pid = attr->pid; 1001 txwin->user_win = attr->user_win; 1002 txwin->wcreds_max = attr->wcreds_max ?: VAS_WCREDS_DEFAULT; 1003 1004 init_winctx_for_txwin(txwin, attr, &winctx); 1005 1006 init_winctx_regs(txwin, &winctx); 1007 1008 /* 1009 * If its a kernel send window, map the window address into the 1010 * kernel's address space. For user windows, user must issue an 1011 * mmap() to map the window into their address space. 1012 * 1013 * NOTE: If kernel ever resubmits a user CRB after handling a page 1014 * fault, we will need to map this into kernel as well. 1015 */ 1016 if (!txwin->user_win) { 1017 txwin->paste_kaddr = map_paste_region(txwin); 1018 if (IS_ERR(txwin->paste_kaddr)) { 1019 rc = PTR_ERR(txwin->paste_kaddr); 1020 goto free_window; 1021 } 1022 } 1023 1024 set_vinst_win(vinst, txwin); 1025 1026 return txwin; 1027 1028 free_window: 1029 vas_window_free(txwin); 1030 1031 put_rxwin: 1032 put_rx_win(rxwin); 1033 return ERR_PTR(rc); 1034 1035 } 1036 EXPORT_SYMBOL_GPL(vas_tx_win_open); 1037 1038 int vas_copy_crb(void *crb, int offset) 1039 { 1040 return vas_copy(crb, offset); 1041 } 1042 EXPORT_SYMBOL_GPL(vas_copy_crb); 1043 1044 #define RMA_LSMP_REPORT_ENABLE PPC_BIT(53) 1045 int vas_paste_crb(struct vas_window *txwin, int offset, bool re) 1046 { 1047 int rc; 1048 void *addr; 1049 uint64_t val; 1050 1051 /* 1052 * Only NX windows are supported for now and hardware assumes 1053 * report-enable flag is set for NX windows. Ensure software 1054 * complies too. 1055 */ 1056 WARN_ON_ONCE(txwin->nx_win && !re); 1057 1058 addr = txwin->paste_kaddr; 1059 if (re) { 1060 /* 1061 * Set the REPORT_ENABLE bit (equivalent to writing 1062 * to 1K offset of the paste address) 1063 */ 1064 val = SET_FIELD(RMA_LSMP_REPORT_ENABLE, 0ULL, 1); 1065 addr += val; 1066 } 1067 1068 /* 1069 * Map the raw CR value from vas_paste() to an error code (there 1070 * is just pass or fail for now though). 1071 */ 1072 rc = vas_paste(addr, offset); 1073 if (rc == 2) 1074 rc = 0; 1075 else 1076 rc = -EINVAL; 1077 1078 pr_debug("Txwin #%d: Msg count %llu\n", txwin->winid, 1079 read_hvwc_reg(txwin, VREG(LRFIFO_PUSH))); 1080 1081 return rc; 1082 } 1083 EXPORT_SYMBOL_GPL(vas_paste_crb); 1084 1085 /* 1086 * If credit checking is enabled for this window, poll for the return 1087 * of window credits (i.e for NX engines to process any outstanding CRBs). 1088 * Since NX-842 waits for the CRBs to be processed before closing the 1089 * window, we should not have to wait for too long. 1090 * 1091 * TODO: We retry in 10ms intervals now. We could/should probably peek at 1092 * the VAS_LRFIFO_PUSH_OFFSET register to get an estimate of pending 1093 * CRBs on the FIFO and compute the delay dynamically on each retry. 1094 * But that is not really needed until we support NX-GZIP access from 1095 * user space. (NX-842 driver waits for CSB and Fast thread-wakeup 1096 * doesn't use credit checking). 1097 */ 1098 static void poll_window_credits(struct vas_window *window) 1099 { 1100 u64 val; 1101 int creds, mode; 1102 1103 val = read_hvwc_reg(window, VREG(WINCTL)); 1104 if (window->tx_win) 1105 mode = GET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val); 1106 else 1107 mode = GET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val); 1108 1109 if (!mode) 1110 return; 1111 retry: 1112 if (window->tx_win) { 1113 val = read_hvwc_reg(window, VREG(TX_WCRED)); 1114 creds = GET_FIELD(VAS_TX_WCRED, val); 1115 } else { 1116 val = read_hvwc_reg(window, VREG(LRX_WCRED)); 1117 creds = GET_FIELD(VAS_LRX_WCRED, val); 1118 } 1119 1120 if (creds < window->wcreds_max) { 1121 val = 0; 1122 set_current_state(TASK_UNINTERRUPTIBLE); 1123 schedule_timeout(msecs_to_jiffies(10)); 1124 goto retry; 1125 } 1126 } 1127 1128 /* 1129 * Wait for the window to go to "not-busy" state. It should only take a 1130 * short time to queue a CRB, so window should not be busy for too long. 1131 * Trying 5ms intervals. 1132 */ 1133 static void poll_window_busy_state(struct vas_window *window) 1134 { 1135 int busy; 1136 u64 val; 1137 1138 retry: 1139 val = read_hvwc_reg(window, VREG(WIN_STATUS)); 1140 busy = GET_FIELD(VAS_WIN_BUSY, val); 1141 if (busy) { 1142 val = 0; 1143 set_current_state(TASK_UNINTERRUPTIBLE); 1144 schedule_timeout(msecs_to_jiffies(5)); 1145 goto retry; 1146 } 1147 } 1148 1149 /* 1150 * Have the hardware cast a window out of cache and wait for it to 1151 * be completed. 1152 * 1153 * NOTE: It can take a relatively long time to cast the window context 1154 * out of the cache. It is not strictly necessary to cast out if: 1155 * 1156 * - we clear the "Pin Window" bit (so hardware is free to evict) 1157 * 1158 * - we re-initialize the window context when it is reassigned. 1159 * 1160 * We do the former in vas_win_close() and latter in vas_win_open(). 1161 * So, ignoring the cast-out for now. We can add it as needed. If 1162 * casting out becomes necessary we should consider offloading the 1163 * job to a worker thread, so the window close can proceed quickly. 1164 */ 1165 static void poll_window_castout(struct vas_window *window) 1166 { 1167 /* stub for now */ 1168 } 1169 1170 /* 1171 * Unpin and close a window so no new requests are accepted and the 1172 * hardware can evict this window from cache if necessary. 1173 */ 1174 static void unpin_close_window(struct vas_window *window) 1175 { 1176 u64 val; 1177 1178 val = read_hvwc_reg(window, VREG(WINCTL)); 1179 val = SET_FIELD(VAS_WINCTL_PIN, val, 0); 1180 val = SET_FIELD(VAS_WINCTL_OPEN, val, 0); 1181 write_hvwc_reg(window, VREG(WINCTL), val); 1182 } 1183 1184 /* 1185 * Close a window. 1186 * 1187 * See Section 1.12.1 of VAS workbook v1.05 for details on closing window: 1188 * - Disable new paste operations (unmap paste address) 1189 * - Poll for the "Window Busy" bit to be cleared 1190 * - Clear the Open/Enable bit for the Window. 1191 * - Poll for return of window Credits (implies FIFO empty for Rx win?) 1192 * - Unpin and cast window context out of cache 1193 * 1194 * Besides the hardware, kernel has some bookkeeping of course. 1195 */ 1196 int vas_win_close(struct vas_window *window) 1197 { 1198 if (!window) 1199 return 0; 1200 1201 if (!window->tx_win && atomic_read(&window->num_txwins) != 0) { 1202 pr_devel("Attempting to close an active Rx window!\n"); 1203 WARN_ON_ONCE(1); 1204 return -EBUSY; 1205 } 1206 1207 unmap_paste_region(window); 1208 1209 clear_vinst_win(window); 1210 1211 poll_window_busy_state(window); 1212 1213 unpin_close_window(window); 1214 1215 poll_window_credits(window); 1216 1217 poll_window_castout(window); 1218 1219 /* if send window, drop reference to matching receive window */ 1220 if (window->tx_win) 1221 put_rx_win(window->rxwin); 1222 1223 vas_window_free(window); 1224 1225 return 0; 1226 } 1227 EXPORT_SYMBOL_GPL(vas_win_close); 1228