1 /* 2 * Copyright 2016-17 IBM Corp. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; either version 7 * 2 of the License, or (at your option) any later version. 8 */ 9 10 #define pr_fmt(fmt) "vas: " fmt 11 12 #include <linux/types.h> 13 #include <linux/mutex.h> 14 #include <linux/slab.h> 15 #include <linux/io.h> 16 #include <linux/log2.h> 17 #include <linux/rcupdate.h> 18 #include <linux/cred.h> 19 20 #include "vas.h" 21 #include "copy-paste.h" 22 23 /* 24 * Compute the paste address region for the window @window using the 25 * ->paste_base_addr and ->paste_win_id_shift we got from device tree. 26 */ 27 static void compute_paste_address(struct vas_window *window, u64 *addr, int *len) 28 { 29 int winid; 30 u64 base, shift; 31 32 base = window->vinst->paste_base_addr; 33 shift = window->vinst->paste_win_id_shift; 34 winid = window->winid; 35 36 *addr = base + (winid << shift); 37 if (len) 38 *len = PAGE_SIZE; 39 40 pr_debug("Txwin #%d: Paste addr 0x%llx\n", winid, *addr); 41 } 42 43 static inline void get_hvwc_mmio_bar(struct vas_window *window, 44 u64 *start, int *len) 45 { 46 u64 pbaddr; 47 48 pbaddr = window->vinst->hvwc_bar_start; 49 *start = pbaddr + window->winid * VAS_HVWC_SIZE; 50 *len = VAS_HVWC_SIZE; 51 } 52 53 static inline void get_uwc_mmio_bar(struct vas_window *window, 54 u64 *start, int *len) 55 { 56 u64 pbaddr; 57 58 pbaddr = window->vinst->uwc_bar_start; 59 *start = pbaddr + window->winid * VAS_UWC_SIZE; 60 *len = VAS_UWC_SIZE; 61 } 62 63 /* 64 * Map the paste bus address of the given send window into kernel address 65 * space. Unlike MMIO regions (map_mmio_region() below), paste region must 66 * be mapped cache-able and is only applicable to send windows. 67 */ 68 static void *map_paste_region(struct vas_window *txwin) 69 { 70 int len; 71 void *map; 72 char *name; 73 u64 start; 74 75 name = kasprintf(GFP_KERNEL, "window-v%d-w%d", txwin->vinst->vas_id, 76 txwin->winid); 77 if (!name) 78 goto free_name; 79 80 txwin->paste_addr_name = name; 81 compute_paste_address(txwin, &start, &len); 82 83 if (!request_mem_region(start, len, name)) { 84 pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n", 85 __func__, start, len); 86 goto free_name; 87 } 88 89 map = ioremap_cache(start, len); 90 if (!map) { 91 pr_devel("%s(): ioremap_cache(0x%llx, %d) failed\n", __func__, 92 start, len); 93 goto free_name; 94 } 95 96 pr_devel("Mapped paste addr 0x%llx to kaddr 0x%p\n", start, map); 97 return map; 98 99 free_name: 100 kfree(name); 101 return ERR_PTR(-ENOMEM); 102 } 103 104 static void *map_mmio_region(char *name, u64 start, int len) 105 { 106 void *map; 107 108 if (!request_mem_region(start, len, name)) { 109 pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n", 110 __func__, start, len); 111 return NULL; 112 } 113 114 map = ioremap(start, len); 115 if (!map) { 116 pr_devel("%s(): ioremap(0x%llx, %d) failed\n", __func__, start, 117 len); 118 return NULL; 119 } 120 121 return map; 122 } 123 124 static void unmap_region(void *addr, u64 start, int len) 125 { 126 iounmap(addr); 127 release_mem_region((phys_addr_t)start, len); 128 } 129 130 /* 131 * Unmap the paste address region for a window. 132 */ 133 static void unmap_paste_region(struct vas_window *window) 134 { 135 int len; 136 u64 busaddr_start; 137 138 if (window->paste_kaddr) { 139 compute_paste_address(window, &busaddr_start, &len); 140 unmap_region(window->paste_kaddr, busaddr_start, len); 141 window->paste_kaddr = NULL; 142 kfree(window->paste_addr_name); 143 window->paste_addr_name = NULL; 144 } 145 } 146 147 /* 148 * Unmap the MMIO regions for a window. 149 */ 150 static void unmap_winctx_mmio_bars(struct vas_window *window) 151 { 152 int len; 153 u64 busaddr_start; 154 155 if (window->hvwc_map) { 156 get_hvwc_mmio_bar(window, &busaddr_start, &len); 157 unmap_region(window->hvwc_map, busaddr_start, len); 158 window->hvwc_map = NULL; 159 } 160 161 if (window->uwc_map) { 162 get_uwc_mmio_bar(window, &busaddr_start, &len); 163 unmap_region(window->uwc_map, busaddr_start, len); 164 window->uwc_map = NULL; 165 } 166 } 167 168 /* 169 * Find the Hypervisor Window Context (HVWC) MMIO Base Address Region and the 170 * OS/User Window Context (UWC) MMIO Base Address Region for the given window. 171 * Map these bus addresses and save the mapped kernel addresses in @window. 172 */ 173 int map_winctx_mmio_bars(struct vas_window *window) 174 { 175 int len; 176 u64 start; 177 178 get_hvwc_mmio_bar(window, &start, &len); 179 window->hvwc_map = map_mmio_region("HVWCM_Window", start, len); 180 181 get_uwc_mmio_bar(window, &start, &len); 182 window->uwc_map = map_mmio_region("UWCM_Window", start, len); 183 184 if (!window->hvwc_map || !window->uwc_map) { 185 unmap_winctx_mmio_bars(window); 186 return -1; 187 } 188 189 return 0; 190 } 191 192 /* 193 * Reset all valid registers in the HV and OS/User Window Contexts for 194 * the window identified by @window. 195 * 196 * NOTE: We cannot really use a for loop to reset window context. Not all 197 * offsets in a window context are valid registers and the valid 198 * registers are not sequential. And, we can only write to offsets 199 * with valid registers. 200 */ 201 void reset_window_regs(struct vas_window *window) 202 { 203 write_hvwc_reg(window, VREG(LPID), 0ULL); 204 write_hvwc_reg(window, VREG(PID), 0ULL); 205 write_hvwc_reg(window, VREG(XLATE_MSR), 0ULL); 206 write_hvwc_reg(window, VREG(XLATE_LPCR), 0ULL); 207 write_hvwc_reg(window, VREG(XLATE_CTL), 0ULL); 208 write_hvwc_reg(window, VREG(AMR), 0ULL); 209 write_hvwc_reg(window, VREG(SEIDR), 0ULL); 210 write_hvwc_reg(window, VREG(FAULT_TX_WIN), 0ULL); 211 write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL); 212 write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), 0ULL); 213 write_hvwc_reg(window, VREG(PSWID), 0ULL); 214 write_hvwc_reg(window, VREG(LFIFO_BAR), 0ULL); 215 write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), 0ULL); 216 write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), 0ULL); 217 write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL); 218 write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL); 219 write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL); 220 write_hvwc_reg(window, VREG(LRX_WCRED), 0ULL); 221 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL); 222 write_hvwc_reg(window, VREG(TX_WCRED), 0ULL); 223 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL); 224 write_hvwc_reg(window, VREG(LFIFO_SIZE), 0ULL); 225 write_hvwc_reg(window, VREG(WINCTL), 0ULL); 226 write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL); 227 write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), 0ULL); 228 write_hvwc_reg(window, VREG(TX_RSVD_BUF_COUNT), 0ULL); 229 write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), 0ULL); 230 write_hvwc_reg(window, VREG(LNOTIFY_CTL), 0ULL); 231 write_hvwc_reg(window, VREG(LNOTIFY_PID), 0ULL); 232 write_hvwc_reg(window, VREG(LNOTIFY_LPID), 0ULL); 233 write_hvwc_reg(window, VREG(LNOTIFY_TID), 0ULL); 234 write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), 0ULL); 235 write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL); 236 237 /* Skip read-only registers: NX_UTIL and NX_UTIL_SE */ 238 239 /* 240 * The send and receive window credit adder registers are also 241 * accessible from HVWC and have been initialized above. We don't 242 * need to initialize from the OS/User Window Context, so skip 243 * following calls: 244 * 245 * write_uwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL); 246 * write_uwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL); 247 */ 248 } 249 250 /* 251 * Initialize window context registers related to Address Translation. 252 * These registers are common to send/receive windows although they 253 * differ for user/kernel windows. As we resolve the TODOs we may 254 * want to add fields to vas_winctx and move the initialization to 255 * init_vas_winctx_regs(). 256 */ 257 static void init_xlate_regs(struct vas_window *window, bool user_win) 258 { 259 u64 lpcr, val; 260 261 /* 262 * MSR_TA, MSR_US are false for both kernel and user. 263 * MSR_DR and MSR_PR are false for kernel. 264 */ 265 val = 0ULL; 266 val = SET_FIELD(VAS_XLATE_MSR_HV, val, 1); 267 val = SET_FIELD(VAS_XLATE_MSR_SF, val, 1); 268 if (user_win) { 269 val = SET_FIELD(VAS_XLATE_MSR_DR, val, 1); 270 val = SET_FIELD(VAS_XLATE_MSR_PR, val, 1); 271 } 272 write_hvwc_reg(window, VREG(XLATE_MSR), val); 273 274 lpcr = mfspr(SPRN_LPCR); 275 val = 0ULL; 276 /* 277 * NOTE: From Section 5.7.8.1 Segment Lookaside Buffer of the 278 * Power ISA, v3.0B, Page size encoding is 0 = 4KB, 5 = 64KB. 279 * 280 * NOTE: From Section 1.3.1, Address Translation Context of the 281 * Nest MMU Workbook, LPCR_SC should be 0 for Power9. 282 */ 283 val = SET_FIELD(VAS_XLATE_LPCR_PAGE_SIZE, val, 5); 284 val = SET_FIELD(VAS_XLATE_LPCR_ISL, val, lpcr & LPCR_ISL); 285 val = SET_FIELD(VAS_XLATE_LPCR_TC, val, lpcr & LPCR_TC); 286 val = SET_FIELD(VAS_XLATE_LPCR_SC, val, 0); 287 write_hvwc_reg(window, VREG(XLATE_LPCR), val); 288 289 /* 290 * Section 1.3.1 (Address translation Context) of NMMU workbook. 291 * 0b00 Hashed Page Table mode 292 * 0b01 Reserved 293 * 0b10 Radix on HPT 294 * 0b11 Radix on Radix 295 */ 296 val = 0ULL; 297 val = SET_FIELD(VAS_XLATE_MODE, val, radix_enabled() ? 3 : 2); 298 write_hvwc_reg(window, VREG(XLATE_CTL), val); 299 300 /* 301 * TODO: Can we mfspr(AMR) even for user windows? 302 */ 303 val = 0ULL; 304 val = SET_FIELD(VAS_AMR, val, mfspr(SPRN_AMR)); 305 write_hvwc_reg(window, VREG(AMR), val); 306 307 val = 0ULL; 308 val = SET_FIELD(VAS_SEIDR, val, 0); 309 write_hvwc_reg(window, VREG(SEIDR), val); 310 } 311 312 /* 313 * Initialize Reserved Send Buffer Count for the send window. It involves 314 * writing to the register, reading it back to confirm that the hardware 315 * has enough buffers to reserve. See section 1.3.1.2.1 of VAS workbook. 316 * 317 * Since we can only make a best-effort attempt to fulfill the request, 318 * we don't return any errors if we cannot. 319 * 320 * TODO: Reserved (aka dedicated) send buffers are not supported yet. 321 */ 322 static void init_rsvd_tx_buf_count(struct vas_window *txwin, 323 struct vas_winctx *winctx) 324 { 325 write_hvwc_reg(txwin, VREG(TX_RSVD_BUF_COUNT), 0ULL); 326 } 327 328 /* 329 * init_winctx_regs() 330 * Initialize window context registers for a receive window. 331 * Except for caching control and marking window open, the registers 332 * are initialized in the order listed in Section 3.1.4 (Window Context 333 * Cache Register Details) of the VAS workbook although they don't need 334 * to be. 335 * 336 * Design note: For NX receive windows, NX allocates the FIFO buffer in OPAL 337 * (so that it can get a large contiguous area) and passes that buffer 338 * to kernel via device tree. We now write that buffer address to the 339 * FIFO BAR. Would it make sense to do this all in OPAL? i.e have OPAL 340 * write the per-chip RX FIFO addresses to the windows during boot-up 341 * as a one-time task? That could work for NX but what about other 342 * receivers? Let the receivers tell us the rx-fifo buffers for now. 343 */ 344 int init_winctx_regs(struct vas_window *window, struct vas_winctx *winctx) 345 { 346 u64 val; 347 int fifo_size; 348 349 reset_window_regs(window); 350 351 val = 0ULL; 352 val = SET_FIELD(VAS_LPID, val, winctx->lpid); 353 write_hvwc_reg(window, VREG(LPID), val); 354 355 val = 0ULL; 356 val = SET_FIELD(VAS_PID_ID, val, winctx->pidr); 357 write_hvwc_reg(window, VREG(PID), val); 358 359 init_xlate_regs(window, winctx->user_win); 360 361 val = 0ULL; 362 val = SET_FIELD(VAS_FAULT_TX_WIN, val, 0); 363 write_hvwc_reg(window, VREG(FAULT_TX_WIN), val); 364 365 /* In PowerNV, interrupts go to HV. */ 366 write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL); 367 368 val = 0ULL; 369 val = SET_FIELD(VAS_HV_INTR_SRC_RA, val, winctx->irq_port); 370 write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), val); 371 372 val = 0ULL; 373 val = SET_FIELD(VAS_PSWID_EA_HANDLE, val, winctx->pswid); 374 write_hvwc_reg(window, VREG(PSWID), val); 375 376 write_hvwc_reg(window, VREG(SPARE1), 0ULL); 377 write_hvwc_reg(window, VREG(SPARE2), 0ULL); 378 write_hvwc_reg(window, VREG(SPARE3), 0ULL); 379 380 /* 381 * NOTE: VAS expects the FIFO address to be copied into the LFIFO_BAR 382 * register as is - do NOT shift the address into VAS_LFIFO_BAR 383 * bit fields! Ok to set the page migration select fields - 384 * VAS ignores the lower 10+ bits in the address anyway, because 385 * the minimum FIFO size is 1K? 386 * 387 * See also: Design note in function header. 388 */ 389 val = __pa(winctx->rx_fifo); 390 val = SET_FIELD(VAS_PAGE_MIGRATION_SELECT, val, 0); 391 write_hvwc_reg(window, VREG(LFIFO_BAR), val); 392 393 val = 0ULL; 394 val = SET_FIELD(VAS_LDATA_STAMP, val, winctx->data_stamp); 395 write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), val); 396 397 val = 0ULL; 398 val = SET_FIELD(VAS_LDMA_TYPE, val, winctx->dma_type); 399 val = SET_FIELD(VAS_LDMA_FIFO_DISABLE, val, winctx->fifo_disable); 400 write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), val); 401 402 write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL); 403 write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL); 404 write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL); 405 406 val = 0ULL; 407 val = SET_FIELD(VAS_LRX_WCRED, val, winctx->wcreds_max); 408 write_hvwc_reg(window, VREG(LRX_WCRED), val); 409 410 val = 0ULL; 411 val = SET_FIELD(VAS_TX_WCRED, val, winctx->wcreds_max); 412 write_hvwc_reg(window, VREG(TX_WCRED), val); 413 414 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL); 415 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL); 416 417 fifo_size = winctx->rx_fifo_size / 1024; 418 419 val = 0ULL; 420 val = SET_FIELD(VAS_LFIFO_SIZE, val, ilog2(fifo_size)); 421 write_hvwc_reg(window, VREG(LFIFO_SIZE), val); 422 423 /* Update window control and caching control registers last so 424 * we mark the window open only after fully initializing it and 425 * pushing context to cache. 426 */ 427 428 write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL); 429 430 init_rsvd_tx_buf_count(window, winctx); 431 432 /* for a send window, point to the matching receive window */ 433 val = 0ULL; 434 val = SET_FIELD(VAS_LRX_WIN_ID, val, winctx->rx_win_id); 435 write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), val); 436 437 write_hvwc_reg(window, VREG(SPARE4), 0ULL); 438 439 val = 0ULL; 440 val = SET_FIELD(VAS_NOTIFY_DISABLE, val, winctx->notify_disable); 441 val = SET_FIELD(VAS_INTR_DISABLE, val, winctx->intr_disable); 442 val = SET_FIELD(VAS_NOTIFY_EARLY, val, winctx->notify_early); 443 val = SET_FIELD(VAS_NOTIFY_OSU_INTR, val, winctx->notify_os_intr_reg); 444 write_hvwc_reg(window, VREG(LNOTIFY_CTL), val); 445 446 val = 0ULL; 447 val = SET_FIELD(VAS_LNOTIFY_PID, val, winctx->lnotify_pid); 448 write_hvwc_reg(window, VREG(LNOTIFY_PID), val); 449 450 val = 0ULL; 451 val = SET_FIELD(VAS_LNOTIFY_LPID, val, winctx->lnotify_lpid); 452 write_hvwc_reg(window, VREG(LNOTIFY_LPID), val); 453 454 val = 0ULL; 455 val = SET_FIELD(VAS_LNOTIFY_TID, val, winctx->lnotify_tid); 456 write_hvwc_reg(window, VREG(LNOTIFY_TID), val); 457 458 val = 0ULL; 459 val = SET_FIELD(VAS_LNOTIFY_MIN_SCOPE, val, winctx->min_scope); 460 val = SET_FIELD(VAS_LNOTIFY_MAX_SCOPE, val, winctx->max_scope); 461 write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), val); 462 463 /* Skip read-only registers NX_UTIL and NX_UTIL_SE */ 464 465 write_hvwc_reg(window, VREG(SPARE5), 0ULL); 466 write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL); 467 write_hvwc_reg(window, VREG(SPARE6), 0ULL); 468 469 /* Finally, push window context to memory and... */ 470 val = 0ULL; 471 val = SET_FIELD(VAS_PUSH_TO_MEM, val, 1); 472 write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), val); 473 474 /* ... mark the window open for business */ 475 val = 0ULL; 476 val = SET_FIELD(VAS_WINCTL_REJ_NO_CREDIT, val, winctx->rej_no_credit); 477 val = SET_FIELD(VAS_WINCTL_PIN, val, winctx->pin_win); 478 val = SET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val, winctx->tx_wcred_mode); 479 val = SET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val, winctx->rx_wcred_mode); 480 val = SET_FIELD(VAS_WINCTL_TX_WORD_MODE, val, winctx->tx_word_mode); 481 val = SET_FIELD(VAS_WINCTL_RX_WORD_MODE, val, winctx->rx_word_mode); 482 val = SET_FIELD(VAS_WINCTL_FAULT_WIN, val, winctx->fault_win); 483 val = SET_FIELD(VAS_WINCTL_NX_WIN, val, winctx->nx_win); 484 val = SET_FIELD(VAS_WINCTL_OPEN, val, 1); 485 write_hvwc_reg(window, VREG(WINCTL), val); 486 487 return 0; 488 } 489 490 static DEFINE_SPINLOCK(vas_ida_lock); 491 492 static void vas_release_window_id(struct ida *ida, int winid) 493 { 494 spin_lock(&vas_ida_lock); 495 ida_remove(ida, winid); 496 spin_unlock(&vas_ida_lock); 497 } 498 499 static int vas_assign_window_id(struct ida *ida) 500 { 501 int rc, winid; 502 503 do { 504 rc = ida_pre_get(ida, GFP_KERNEL); 505 if (!rc) 506 return -EAGAIN; 507 508 spin_lock(&vas_ida_lock); 509 rc = ida_get_new(ida, &winid); 510 spin_unlock(&vas_ida_lock); 511 } while (rc == -EAGAIN); 512 513 if (rc) 514 return rc; 515 516 if (winid > VAS_WINDOWS_PER_CHIP) { 517 pr_err("Too many (%d) open windows\n", winid); 518 vas_release_window_id(ida, winid); 519 return -EAGAIN; 520 } 521 522 return winid; 523 } 524 525 static void vas_window_free(struct vas_window *window) 526 { 527 int winid = window->winid; 528 struct vas_instance *vinst = window->vinst; 529 530 unmap_winctx_mmio_bars(window); 531 kfree(window); 532 533 vas_release_window_id(&vinst->ida, winid); 534 } 535 536 static struct vas_window *vas_window_alloc(struct vas_instance *vinst) 537 { 538 int winid; 539 struct vas_window *window; 540 541 winid = vas_assign_window_id(&vinst->ida); 542 if (winid < 0) 543 return ERR_PTR(winid); 544 545 window = kzalloc(sizeof(*window), GFP_KERNEL); 546 if (!window) 547 goto out_free; 548 549 window->vinst = vinst; 550 window->winid = winid; 551 552 if (map_winctx_mmio_bars(window)) 553 goto out_free; 554 555 return window; 556 557 out_free: 558 kfree(window); 559 vas_release_window_id(&vinst->ida, winid); 560 return ERR_PTR(-ENOMEM); 561 } 562 563 static void put_rx_win(struct vas_window *rxwin) 564 { 565 /* Better not be a send window! */ 566 WARN_ON_ONCE(rxwin->tx_win); 567 568 atomic_dec(&rxwin->num_txwins); 569 } 570 571 /* 572 * Get the VAS receive window associated with NX engine identified 573 * by @cop and if applicable, @pswid. 574 * 575 * See also function header of set_vinst_win(). 576 */ 577 static struct vas_window *get_vinst_rxwin(struct vas_instance *vinst, 578 enum vas_cop_type cop, u32 pswid) 579 { 580 struct vas_window *rxwin; 581 582 mutex_lock(&vinst->mutex); 583 584 if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI) 585 rxwin = vinst->rxwin[cop] ?: ERR_PTR(-EINVAL); 586 else 587 rxwin = ERR_PTR(-EINVAL); 588 589 if (!IS_ERR(rxwin)) 590 atomic_inc(&rxwin->num_txwins); 591 592 mutex_unlock(&vinst->mutex); 593 594 return rxwin; 595 } 596 597 /* 598 * We have two tables of windows in a VAS instance. The first one, 599 * ->windows[], contains all the windows in the instance and allows 600 * looking up a window by its id. It is used to look up send windows 601 * during fault handling and receive windows when pairing user space 602 * send/receive windows. 603 * 604 * The second table, ->rxwin[], contains receive windows that are 605 * associated with NX engines. This table has VAS_COP_TYPE_MAX 606 * entries and is used to look up a receive window by its 607 * coprocessor type. 608 * 609 * Here, we save @window in the ->windows[] table. If it is a receive 610 * window, we also save the window in the ->rxwin[] table. 611 */ 612 static void set_vinst_win(struct vas_instance *vinst, 613 struct vas_window *window) 614 { 615 int id = window->winid; 616 617 mutex_lock(&vinst->mutex); 618 619 /* 620 * There should only be one receive window for a coprocessor type 621 * unless its a user (FTW) window. 622 */ 623 if (!window->user_win && !window->tx_win) { 624 WARN_ON_ONCE(vinst->rxwin[window->cop]); 625 vinst->rxwin[window->cop] = window; 626 } 627 628 WARN_ON_ONCE(vinst->windows[id] != NULL); 629 vinst->windows[id] = window; 630 631 mutex_unlock(&vinst->mutex); 632 } 633 634 /* 635 * Clear this window from the table(s) of windows for this VAS instance. 636 * See also function header of set_vinst_win(). 637 */ 638 static void clear_vinst_win(struct vas_window *window) 639 { 640 int id = window->winid; 641 struct vas_instance *vinst = window->vinst; 642 643 mutex_lock(&vinst->mutex); 644 645 if (!window->user_win && !window->tx_win) { 646 WARN_ON_ONCE(!vinst->rxwin[window->cop]); 647 vinst->rxwin[window->cop] = NULL; 648 } 649 650 WARN_ON_ONCE(vinst->windows[id] != window); 651 vinst->windows[id] = NULL; 652 653 mutex_unlock(&vinst->mutex); 654 } 655 656 static void init_winctx_for_rxwin(struct vas_window *rxwin, 657 struct vas_rx_win_attr *rxattr, 658 struct vas_winctx *winctx) 659 { 660 /* 661 * We first zero (memset()) all fields and only set non-zero fields. 662 * Following fields are 0/false but maybe deserve a comment: 663 * 664 * ->notify_os_intr_reg In powerNV, send intrs to HV 665 * ->notify_disable False for NX windows 666 * ->intr_disable False for Fault Windows 667 * ->xtra_write False for NX windows 668 * ->notify_early NA for NX windows 669 * ->rsvd_txbuf_count NA for Rx windows 670 * ->lpid, ->pid, ->tid NA for Rx windows 671 */ 672 673 memset(winctx, 0, sizeof(struct vas_winctx)); 674 675 winctx->rx_fifo = rxattr->rx_fifo; 676 winctx->rx_fifo_size = rxattr->rx_fifo_size; 677 winctx->wcreds_max = rxattr->wcreds_max ?: VAS_WCREDS_DEFAULT; 678 winctx->pin_win = rxattr->pin_win; 679 680 winctx->nx_win = rxattr->nx_win; 681 winctx->fault_win = rxattr->fault_win; 682 winctx->user_win = rxattr->user_win; 683 winctx->rej_no_credit = rxattr->rej_no_credit; 684 winctx->rx_word_mode = rxattr->rx_win_ord_mode; 685 winctx->tx_word_mode = rxattr->tx_win_ord_mode; 686 winctx->rx_wcred_mode = rxattr->rx_wcred_mode; 687 winctx->tx_wcred_mode = rxattr->tx_wcred_mode; 688 winctx->notify_early = rxattr->notify_early; 689 690 if (winctx->nx_win) { 691 winctx->data_stamp = true; 692 winctx->intr_disable = true; 693 winctx->pin_win = true; 694 695 WARN_ON_ONCE(winctx->fault_win); 696 WARN_ON_ONCE(!winctx->rx_word_mode); 697 WARN_ON_ONCE(!winctx->tx_word_mode); 698 WARN_ON_ONCE(winctx->notify_after_count); 699 } else if (winctx->fault_win) { 700 winctx->notify_disable = true; 701 } else if (winctx->user_win) { 702 /* 703 * Section 1.8.1 Low Latency Core-Core Wake up of 704 * the VAS workbook: 705 * 706 * - disable credit checks ([tr]x_wcred_mode = false) 707 * - disable FIFO writes 708 * - enable ASB_Notify, disable interrupt 709 */ 710 winctx->fifo_disable = true; 711 winctx->intr_disable = true; 712 winctx->rx_fifo = NULL; 713 } 714 715 winctx->lnotify_lpid = rxattr->lnotify_lpid; 716 winctx->lnotify_pid = rxattr->lnotify_pid; 717 winctx->lnotify_tid = rxattr->lnotify_tid; 718 winctx->pswid = rxattr->pswid; 719 winctx->dma_type = VAS_DMA_TYPE_INJECT; 720 winctx->tc_mode = rxattr->tc_mode; 721 722 winctx->min_scope = VAS_SCOPE_LOCAL; 723 winctx->max_scope = VAS_SCOPE_VECTORED_GROUP; 724 } 725 726 static bool rx_win_args_valid(enum vas_cop_type cop, 727 struct vas_rx_win_attr *attr) 728 { 729 pr_debug("Rxattr: fault %d, notify %d, intr %d, early %d, fifo %d\n", 730 attr->fault_win, attr->notify_disable, 731 attr->intr_disable, attr->notify_early, 732 attr->rx_fifo_size); 733 734 if (cop >= VAS_COP_TYPE_MAX) 735 return false; 736 737 if (cop != VAS_COP_TYPE_FTW && 738 attr->rx_fifo_size < VAS_RX_FIFO_SIZE_MIN) 739 return false; 740 741 if (attr->rx_fifo_size > VAS_RX_FIFO_SIZE_MAX) 742 return false; 743 744 if (attr->wcreds_max > VAS_RX_WCREDS_MAX) 745 return false; 746 747 if (attr->nx_win) { 748 /* cannot be fault or user window if it is nx */ 749 if (attr->fault_win || attr->user_win) 750 return false; 751 /* 752 * Section 3.1.4.32: NX Windows must not disable notification, 753 * and must not enable interrupts or early notification. 754 */ 755 if (attr->notify_disable || !attr->intr_disable || 756 attr->notify_early) 757 return false; 758 } else if (attr->fault_win) { 759 /* cannot be both fault and user window */ 760 if (attr->user_win) 761 return false; 762 763 /* 764 * Section 3.1.4.32: Fault windows must disable notification 765 * but not interrupts. 766 */ 767 if (!attr->notify_disable || attr->intr_disable) 768 return false; 769 770 } else if (attr->user_win) { 771 /* 772 * User receive windows are only for fast-thread-wakeup 773 * (FTW). They don't need a FIFO and must disable interrupts 774 */ 775 if (attr->rx_fifo || attr->rx_fifo_size || !attr->intr_disable) 776 return false; 777 } else { 778 /* Rx window must be one of NX or Fault or User window. */ 779 return false; 780 } 781 782 return true; 783 } 784 785 void vas_init_rx_win_attr(struct vas_rx_win_attr *rxattr, enum vas_cop_type cop) 786 { 787 memset(rxattr, 0, sizeof(*rxattr)); 788 789 if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI) { 790 rxattr->pin_win = true; 791 rxattr->nx_win = true; 792 rxattr->fault_win = false; 793 rxattr->intr_disable = true; 794 rxattr->rx_wcred_mode = true; 795 rxattr->tx_wcred_mode = true; 796 rxattr->rx_win_ord_mode = true; 797 rxattr->tx_win_ord_mode = true; 798 } else if (cop == VAS_COP_TYPE_FAULT) { 799 rxattr->pin_win = true; 800 rxattr->fault_win = true; 801 rxattr->notify_disable = true; 802 rxattr->rx_wcred_mode = true; 803 rxattr->tx_wcred_mode = true; 804 rxattr->rx_win_ord_mode = true; 805 rxattr->tx_win_ord_mode = true; 806 } else if (cop == VAS_COP_TYPE_FTW) { 807 rxattr->user_win = true; 808 rxattr->intr_disable = true; 809 810 /* 811 * As noted in the VAS Workbook we disable credit checks. 812 * If we enable credit checks in the future, we must also 813 * implement a mechanism to return the user credits or new 814 * paste operations will fail. 815 */ 816 } 817 } 818 EXPORT_SYMBOL_GPL(vas_init_rx_win_attr); 819 820 struct vas_window *vas_rx_win_open(int vasid, enum vas_cop_type cop, 821 struct vas_rx_win_attr *rxattr) 822 { 823 struct vas_window *rxwin; 824 struct vas_winctx winctx; 825 struct vas_instance *vinst; 826 827 if (!rx_win_args_valid(cop, rxattr)) 828 return ERR_PTR(-EINVAL); 829 830 vinst = find_vas_instance(vasid); 831 if (!vinst) { 832 pr_devel("vasid %d not found!\n", vasid); 833 return ERR_PTR(-EINVAL); 834 } 835 pr_devel("Found instance %d\n", vasid); 836 837 rxwin = vas_window_alloc(vinst); 838 if (IS_ERR(rxwin)) { 839 pr_devel("Unable to allocate memory for Rx window\n"); 840 return rxwin; 841 } 842 843 rxwin->tx_win = false; 844 rxwin->nx_win = rxattr->nx_win; 845 rxwin->user_win = rxattr->user_win; 846 rxwin->cop = cop; 847 if (rxattr->user_win) 848 rxwin->pid = task_pid_vnr(current); 849 850 init_winctx_for_rxwin(rxwin, rxattr, &winctx); 851 init_winctx_regs(rxwin, &winctx); 852 853 set_vinst_win(vinst, rxwin); 854 855 return rxwin; 856 } 857 EXPORT_SYMBOL_GPL(vas_rx_win_open); 858 859 void vas_init_tx_win_attr(struct vas_tx_win_attr *txattr, enum vas_cop_type cop) 860 { 861 memset(txattr, 0, sizeof(*txattr)); 862 863 if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI) { 864 txattr->rej_no_credit = false; 865 txattr->rx_wcred_mode = true; 866 txattr->tx_wcred_mode = true; 867 txattr->rx_win_ord_mode = true; 868 txattr->tx_win_ord_mode = true; 869 } else if (cop == VAS_COP_TYPE_FTW) { 870 txattr->user_win = true; 871 } 872 } 873 EXPORT_SYMBOL_GPL(vas_init_tx_win_attr); 874 875 static void init_winctx_for_txwin(struct vas_window *txwin, 876 struct vas_tx_win_attr *txattr, 877 struct vas_winctx *winctx) 878 { 879 /* 880 * We first zero all fields and only set non-zero ones. Following 881 * are some fields set to 0/false for the stated reason: 882 * 883 * ->notify_os_intr_reg In powernv, send intrs to HV 884 * ->rsvd_txbuf_count Not supported yet. 885 * ->notify_disable False for NX windows 886 * ->xtra_write False for NX windows 887 * ->notify_early NA for NX windows 888 * ->lnotify_lpid NA for Tx windows 889 * ->lnotify_pid NA for Tx windows 890 * ->lnotify_tid NA for Tx windows 891 * ->tx_win_cred_mode Ignore for now for NX windows 892 * ->rx_win_cred_mode Ignore for now for NX windows 893 */ 894 memset(winctx, 0, sizeof(struct vas_winctx)); 895 896 winctx->wcreds_max = txattr->wcreds_max ?: VAS_WCREDS_DEFAULT; 897 898 winctx->user_win = txattr->user_win; 899 winctx->nx_win = txwin->rxwin->nx_win; 900 winctx->pin_win = txattr->pin_win; 901 winctx->rej_no_credit = txattr->rej_no_credit; 902 winctx->rsvd_txbuf_enable = txattr->rsvd_txbuf_enable; 903 904 winctx->rx_wcred_mode = txattr->rx_wcred_mode; 905 winctx->tx_wcred_mode = txattr->tx_wcred_mode; 906 winctx->rx_word_mode = txattr->rx_win_ord_mode; 907 winctx->tx_word_mode = txattr->tx_win_ord_mode; 908 winctx->rsvd_txbuf_count = txattr->rsvd_txbuf_count; 909 910 if (winctx->nx_win) { 911 winctx->data_stamp = true; 912 winctx->intr_disable = true; 913 } 914 915 winctx->lpid = txattr->lpid; 916 winctx->pidr = txattr->pidr; 917 winctx->rx_win_id = txwin->rxwin->winid; 918 919 winctx->dma_type = VAS_DMA_TYPE_INJECT; 920 winctx->tc_mode = txattr->tc_mode; 921 winctx->min_scope = VAS_SCOPE_LOCAL; 922 winctx->max_scope = VAS_SCOPE_VECTORED_GROUP; 923 924 winctx->pswid = 0; 925 } 926 927 static bool tx_win_args_valid(enum vas_cop_type cop, 928 struct vas_tx_win_attr *attr) 929 { 930 if (attr->tc_mode != VAS_THRESH_DISABLED) 931 return false; 932 933 if (cop > VAS_COP_TYPE_MAX) 934 return false; 935 936 if (attr->wcreds_max > VAS_TX_WCREDS_MAX) 937 return false; 938 939 if (attr->user_win && 940 (cop != VAS_COP_TYPE_FTW || attr->rsvd_txbuf_count)) 941 return false; 942 943 return true; 944 } 945 946 struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop, 947 struct vas_tx_win_attr *attr) 948 { 949 int rc; 950 struct vas_window *txwin; 951 struct vas_window *rxwin; 952 struct vas_winctx winctx; 953 struct vas_instance *vinst; 954 955 if (!tx_win_args_valid(cop, attr)) 956 return ERR_PTR(-EINVAL); 957 958 vinst = find_vas_instance(vasid); 959 if (!vinst) { 960 pr_devel("vasid %d not found!\n", vasid); 961 return ERR_PTR(-EINVAL); 962 } 963 964 rxwin = get_vinst_rxwin(vinst, cop, attr->pswid); 965 if (IS_ERR(rxwin)) { 966 pr_devel("No RxWin for vasid %d, cop %d\n", vasid, cop); 967 return rxwin; 968 } 969 970 txwin = vas_window_alloc(vinst); 971 if (IS_ERR(txwin)) { 972 rc = PTR_ERR(txwin); 973 goto put_rxwin; 974 } 975 976 txwin->tx_win = 1; 977 txwin->rxwin = rxwin; 978 txwin->nx_win = txwin->rxwin->nx_win; 979 txwin->pid = attr->pid; 980 txwin->user_win = attr->user_win; 981 982 init_winctx_for_txwin(txwin, attr, &winctx); 983 984 init_winctx_regs(txwin, &winctx); 985 986 /* 987 * If its a kernel send window, map the window address into the 988 * kernel's address space. For user windows, user must issue an 989 * mmap() to map the window into their address space. 990 * 991 * NOTE: If kernel ever resubmits a user CRB after handling a page 992 * fault, we will need to map this into kernel as well. 993 */ 994 if (!txwin->user_win) { 995 txwin->paste_kaddr = map_paste_region(txwin); 996 if (IS_ERR(txwin->paste_kaddr)) { 997 rc = PTR_ERR(txwin->paste_kaddr); 998 goto free_window; 999 } 1000 } 1001 1002 set_vinst_win(vinst, txwin); 1003 1004 return txwin; 1005 1006 free_window: 1007 vas_window_free(txwin); 1008 1009 put_rxwin: 1010 put_rx_win(rxwin); 1011 return ERR_PTR(rc); 1012 1013 } 1014 EXPORT_SYMBOL_GPL(vas_tx_win_open); 1015 1016 int vas_copy_crb(void *crb, int offset) 1017 { 1018 return vas_copy(crb, offset); 1019 } 1020 EXPORT_SYMBOL_GPL(vas_copy_crb); 1021 1022 #define RMA_LSMP_REPORT_ENABLE PPC_BIT(53) 1023 int vas_paste_crb(struct vas_window *txwin, int offset, bool re) 1024 { 1025 int rc; 1026 void *addr; 1027 uint64_t val; 1028 1029 /* 1030 * Only NX windows are supported for now and hardware assumes 1031 * report-enable flag is set for NX windows. Ensure software 1032 * complies too. 1033 */ 1034 WARN_ON_ONCE(txwin->nx_win && !re); 1035 1036 addr = txwin->paste_kaddr; 1037 if (re) { 1038 /* 1039 * Set the REPORT_ENABLE bit (equivalent to writing 1040 * to 1K offset of the paste address) 1041 */ 1042 val = SET_FIELD(RMA_LSMP_REPORT_ENABLE, 0ULL, 1); 1043 addr += val; 1044 } 1045 1046 /* 1047 * Map the raw CR value from vas_paste() to an error code (there 1048 * is just pass or fail for now though). 1049 */ 1050 rc = vas_paste(addr, offset); 1051 if (rc == 2) 1052 rc = 0; 1053 else 1054 rc = -EINVAL; 1055 1056 pr_debug("Txwin #%d: Msg count %llu\n", txwin->winid, 1057 read_hvwc_reg(txwin, VREG(LRFIFO_PUSH))); 1058 1059 return rc; 1060 } 1061 EXPORT_SYMBOL_GPL(vas_paste_crb); 1062 1063 static void poll_window_busy_state(struct vas_window *window) 1064 { 1065 int busy; 1066 u64 val; 1067 1068 retry: 1069 /* 1070 * Poll Window Busy flag 1071 */ 1072 val = read_hvwc_reg(window, VREG(WIN_STATUS)); 1073 busy = GET_FIELD(VAS_WIN_BUSY, val); 1074 if (busy) { 1075 val = 0; 1076 set_current_state(TASK_UNINTERRUPTIBLE); 1077 schedule_timeout(HZ); 1078 goto retry; 1079 } 1080 } 1081 1082 /* 1083 * Have the hardware cast a window out of cache and wait for it to 1084 * be completed. 1085 * 1086 * NOTE: It can take a relatively long time to cast the window context 1087 * out of the cache. It is not strictly necessary to cast out if: 1088 * 1089 * - we clear the "Pin Window" bit (so hardware is free to evict) 1090 * 1091 * - we re-initialize the window context when it is reassigned. 1092 * 1093 * We do the former in vas_win_close() and latter in vas_win_open(). 1094 * So, ignoring the cast-out for now. We can add it as needed. If 1095 * casting out becomes necessary we should consider offloading the 1096 * job to a worker thread, so the window close can proceed quickly. 1097 */ 1098 static void poll_window_castout(struct vas_window *window) 1099 { 1100 /* stub for now */ 1101 } 1102 1103 /* 1104 * Close a window. 1105 * 1106 * See Section 1.12.1 of VAS workbook v1.05 for details on closing window: 1107 * - Disable new paste operations (unmap paste address) 1108 * - Poll for the "Window Busy" bit to be cleared 1109 * - Clear the Open/Enable bit for the Window. 1110 * - Poll for return of window Credits (implies FIFO empty for Rx win?) 1111 * - Unpin and cast window context out of cache 1112 * 1113 * Besides the hardware, kernel has some bookkeeping of course. 1114 */ 1115 int vas_win_close(struct vas_window *window) 1116 { 1117 u64 val; 1118 1119 if (!window) 1120 return 0; 1121 1122 if (!window->tx_win && atomic_read(&window->num_txwins) != 0) { 1123 pr_devel("Attempting to close an active Rx window!\n"); 1124 WARN_ON_ONCE(1); 1125 return -EBUSY; 1126 } 1127 1128 unmap_paste_region(window); 1129 1130 clear_vinst_win(window); 1131 1132 poll_window_busy_state(window); 1133 1134 /* Unpin window from cache and close it */ 1135 val = read_hvwc_reg(window, VREG(WINCTL)); 1136 val = SET_FIELD(VAS_WINCTL_PIN, val, 0); 1137 val = SET_FIELD(VAS_WINCTL_OPEN, val, 0); 1138 write_hvwc_reg(window, VREG(WINCTL), val); 1139 1140 poll_window_castout(window); 1141 1142 /* if send window, drop reference to matching receive window */ 1143 if (window->tx_win) 1144 put_rx_win(window->rxwin); 1145 1146 vas_window_free(window); 1147 1148 return 0; 1149 } 1150 EXPORT_SYMBOL_GPL(vas_win_close); 1151