1 /* 2 * Copyright 2016-17 IBM Corp. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; either version 7 * 2 of the License, or (at your option) any later version. 8 */ 9 10 #define pr_fmt(fmt) "vas: " fmt 11 12 #include <linux/types.h> 13 #include <linux/mutex.h> 14 #include <linux/slab.h> 15 #include <linux/io.h> 16 #include <linux/log2.h> 17 18 #include "vas.h" 19 20 /* 21 * Compute the paste address region for the window @window using the 22 * ->paste_base_addr and ->paste_win_id_shift we got from device tree. 23 */ 24 static void compute_paste_address(struct vas_window *window, u64 *addr, int *len) 25 { 26 int winid; 27 u64 base, shift; 28 29 base = window->vinst->paste_base_addr; 30 shift = window->vinst->paste_win_id_shift; 31 winid = window->winid; 32 33 *addr = base + (winid << shift); 34 if (len) 35 *len = PAGE_SIZE; 36 37 pr_debug("Txwin #%d: Paste addr 0x%llx\n", winid, *addr); 38 } 39 40 static inline void get_hvwc_mmio_bar(struct vas_window *window, 41 u64 *start, int *len) 42 { 43 u64 pbaddr; 44 45 pbaddr = window->vinst->hvwc_bar_start; 46 *start = pbaddr + window->winid * VAS_HVWC_SIZE; 47 *len = VAS_HVWC_SIZE; 48 } 49 50 static inline void get_uwc_mmio_bar(struct vas_window *window, 51 u64 *start, int *len) 52 { 53 u64 pbaddr; 54 55 pbaddr = window->vinst->uwc_bar_start; 56 *start = pbaddr + window->winid * VAS_UWC_SIZE; 57 *len = VAS_UWC_SIZE; 58 } 59 60 /* 61 * Map the paste bus address of the given send window into kernel address 62 * space. Unlike MMIO regions (map_mmio_region() below), paste region must 63 * be mapped cache-able and is only applicable to send windows. 64 */ 65 void *map_paste_region(struct vas_window *txwin) 66 { 67 int len; 68 void *map; 69 char *name; 70 u64 start; 71 72 name = kasprintf(GFP_KERNEL, "window-v%d-w%d", txwin->vinst->vas_id, 73 txwin->winid); 74 if (!name) 75 goto free_name; 76 77 txwin->paste_addr_name = name; 78 compute_paste_address(txwin, &start, &len); 79 80 if (!request_mem_region(start, len, name)) { 81 pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n", 82 __func__, start, len); 83 goto free_name; 84 } 85 86 map = ioremap_cache(start, len); 87 if (!map) { 88 pr_devel("%s(): ioremap_cache(0x%llx, %d) failed\n", __func__, 89 start, len); 90 goto free_name; 91 } 92 93 pr_devel("Mapped paste addr 0x%llx to kaddr 0x%p\n", start, map); 94 return map; 95 96 free_name: 97 kfree(name); 98 return ERR_PTR(-ENOMEM); 99 } 100 101 102 static void *map_mmio_region(char *name, u64 start, int len) 103 { 104 void *map; 105 106 if (!request_mem_region(start, len, name)) { 107 pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n", 108 __func__, start, len); 109 return NULL; 110 } 111 112 map = ioremap(start, len); 113 if (!map) { 114 pr_devel("%s(): ioremap(0x%llx, %d) failed\n", __func__, start, 115 len); 116 return NULL; 117 } 118 119 return map; 120 } 121 122 static void unmap_region(void *addr, u64 start, int len) 123 { 124 iounmap(addr); 125 release_mem_region((phys_addr_t)start, len); 126 } 127 128 /* 129 * Unmap the paste address region for a window. 130 */ 131 void unmap_paste_region(struct vas_window *window) 132 { 133 int len; 134 u64 busaddr_start; 135 136 if (window->paste_kaddr) { 137 compute_paste_address(window, &busaddr_start, &len); 138 unmap_region(window->paste_kaddr, busaddr_start, len); 139 window->paste_kaddr = NULL; 140 kfree(window->paste_addr_name); 141 window->paste_addr_name = NULL; 142 } 143 } 144 145 /* 146 * Unmap the MMIO regions for a window. 147 */ 148 static void unmap_winctx_mmio_bars(struct vas_window *window) 149 { 150 int len; 151 u64 busaddr_start; 152 153 if (window->hvwc_map) { 154 get_hvwc_mmio_bar(window, &busaddr_start, &len); 155 unmap_region(window->hvwc_map, busaddr_start, len); 156 window->hvwc_map = NULL; 157 } 158 159 if (window->uwc_map) { 160 get_uwc_mmio_bar(window, &busaddr_start, &len); 161 unmap_region(window->uwc_map, busaddr_start, len); 162 window->uwc_map = NULL; 163 } 164 } 165 166 /* 167 * Find the Hypervisor Window Context (HVWC) MMIO Base Address Region and the 168 * OS/User Window Context (UWC) MMIO Base Address Region for the given window. 169 * Map these bus addresses and save the mapped kernel addresses in @window. 170 */ 171 int map_winctx_mmio_bars(struct vas_window *window) 172 { 173 int len; 174 u64 start; 175 176 get_hvwc_mmio_bar(window, &start, &len); 177 window->hvwc_map = map_mmio_region("HVWCM_Window", start, len); 178 179 get_uwc_mmio_bar(window, &start, &len); 180 window->uwc_map = map_mmio_region("UWCM_Window", start, len); 181 182 if (!window->hvwc_map || !window->uwc_map) { 183 unmap_winctx_mmio_bars(window); 184 return -1; 185 } 186 187 return 0; 188 } 189 190 /* 191 * Reset all valid registers in the HV and OS/User Window Contexts for 192 * the window identified by @window. 193 * 194 * NOTE: We cannot really use a for loop to reset window context. Not all 195 * offsets in a window context are valid registers and the valid 196 * registers are not sequential. And, we can only write to offsets 197 * with valid registers. 198 */ 199 void reset_window_regs(struct vas_window *window) 200 { 201 write_hvwc_reg(window, VREG(LPID), 0ULL); 202 write_hvwc_reg(window, VREG(PID), 0ULL); 203 write_hvwc_reg(window, VREG(XLATE_MSR), 0ULL); 204 write_hvwc_reg(window, VREG(XLATE_LPCR), 0ULL); 205 write_hvwc_reg(window, VREG(XLATE_CTL), 0ULL); 206 write_hvwc_reg(window, VREG(AMR), 0ULL); 207 write_hvwc_reg(window, VREG(SEIDR), 0ULL); 208 write_hvwc_reg(window, VREG(FAULT_TX_WIN), 0ULL); 209 write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL); 210 write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), 0ULL); 211 write_hvwc_reg(window, VREG(PSWID), 0ULL); 212 write_hvwc_reg(window, VREG(LFIFO_BAR), 0ULL); 213 write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), 0ULL); 214 write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), 0ULL); 215 write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL); 216 write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL); 217 write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL); 218 write_hvwc_reg(window, VREG(LRX_WCRED), 0ULL); 219 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL); 220 write_hvwc_reg(window, VREG(TX_WCRED), 0ULL); 221 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL); 222 write_hvwc_reg(window, VREG(LFIFO_SIZE), 0ULL); 223 write_hvwc_reg(window, VREG(WINCTL), 0ULL); 224 write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL); 225 write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), 0ULL); 226 write_hvwc_reg(window, VREG(TX_RSVD_BUF_COUNT), 0ULL); 227 write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), 0ULL); 228 write_hvwc_reg(window, VREG(LNOTIFY_CTL), 0ULL); 229 write_hvwc_reg(window, VREG(LNOTIFY_PID), 0ULL); 230 write_hvwc_reg(window, VREG(LNOTIFY_LPID), 0ULL); 231 write_hvwc_reg(window, VREG(LNOTIFY_TID), 0ULL); 232 write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), 0ULL); 233 write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL); 234 235 /* Skip read-only registers: NX_UTIL and NX_UTIL_SE */ 236 237 /* 238 * The send and receive window credit adder registers are also 239 * accessible from HVWC and have been initialized above. We don't 240 * need to initialize from the OS/User Window Context, so skip 241 * following calls: 242 * 243 * write_uwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL); 244 * write_uwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL); 245 */ 246 } 247 248 /* 249 * Initialize window context registers related to Address Translation. 250 * These registers are common to send/receive windows although they 251 * differ for user/kernel windows. As we resolve the TODOs we may 252 * want to add fields to vas_winctx and move the initialization to 253 * init_vas_winctx_regs(). 254 */ 255 static void init_xlate_regs(struct vas_window *window, bool user_win) 256 { 257 u64 lpcr, val; 258 259 /* 260 * MSR_TA, MSR_US are false for both kernel and user. 261 * MSR_DR and MSR_PR are false for kernel. 262 */ 263 val = 0ULL; 264 val = SET_FIELD(VAS_XLATE_MSR_HV, val, 1); 265 val = SET_FIELD(VAS_XLATE_MSR_SF, val, 1); 266 if (user_win) { 267 val = SET_FIELD(VAS_XLATE_MSR_DR, val, 1); 268 val = SET_FIELD(VAS_XLATE_MSR_PR, val, 1); 269 } 270 write_hvwc_reg(window, VREG(XLATE_MSR), val); 271 272 lpcr = mfspr(SPRN_LPCR); 273 val = 0ULL; 274 /* 275 * NOTE: From Section 5.7.8.1 Segment Lookaside Buffer of the 276 * Power ISA, v3.0B, Page size encoding is 0 = 4KB, 5 = 64KB. 277 * 278 * NOTE: From Section 1.3.1, Address Translation Context of the 279 * Nest MMU Workbook, LPCR_SC should be 0 for Power9. 280 */ 281 val = SET_FIELD(VAS_XLATE_LPCR_PAGE_SIZE, val, 5); 282 val = SET_FIELD(VAS_XLATE_LPCR_ISL, val, lpcr & LPCR_ISL); 283 val = SET_FIELD(VAS_XLATE_LPCR_TC, val, lpcr & LPCR_TC); 284 val = SET_FIELD(VAS_XLATE_LPCR_SC, val, 0); 285 write_hvwc_reg(window, VREG(XLATE_LPCR), val); 286 287 /* 288 * Section 1.3.1 (Address translation Context) of NMMU workbook. 289 * 0b00 Hashed Page Table mode 290 * 0b01 Reserved 291 * 0b10 Radix on HPT 292 * 0b11 Radix on Radix 293 */ 294 val = 0ULL; 295 val = SET_FIELD(VAS_XLATE_MODE, val, radix_enabled() ? 3 : 2); 296 write_hvwc_reg(window, VREG(XLATE_CTL), val); 297 298 /* 299 * TODO: Can we mfspr(AMR) even for user windows? 300 */ 301 val = 0ULL; 302 val = SET_FIELD(VAS_AMR, val, mfspr(SPRN_AMR)); 303 write_hvwc_reg(window, VREG(AMR), val); 304 305 val = 0ULL; 306 val = SET_FIELD(VAS_SEIDR, val, 0); 307 write_hvwc_reg(window, VREG(SEIDR), val); 308 } 309 310 /* 311 * Initialize Reserved Send Buffer Count for the send window. It involves 312 * writing to the register, reading it back to confirm that the hardware 313 * has enough buffers to reserve. See section 1.3.1.2.1 of VAS workbook. 314 * 315 * Since we can only make a best-effort attempt to fulfill the request, 316 * we don't return any errors if we cannot. 317 * 318 * TODO: Reserved (aka dedicated) send buffers are not supported yet. 319 */ 320 static void init_rsvd_tx_buf_count(struct vas_window *txwin, 321 struct vas_winctx *winctx) 322 { 323 write_hvwc_reg(txwin, VREG(TX_RSVD_BUF_COUNT), 0ULL); 324 } 325 326 /* 327 * init_winctx_regs() 328 * Initialize window context registers for a receive window. 329 * Except for caching control and marking window open, the registers 330 * are initialized in the order listed in Section 3.1.4 (Window Context 331 * Cache Register Details) of the VAS workbook although they don't need 332 * to be. 333 * 334 * Design note: For NX receive windows, NX allocates the FIFO buffer in OPAL 335 * (so that it can get a large contiguous area) and passes that buffer 336 * to kernel via device tree. We now write that buffer address to the 337 * FIFO BAR. Would it make sense to do this all in OPAL? i.e have OPAL 338 * write the per-chip RX FIFO addresses to the windows during boot-up 339 * as a one-time task? That could work for NX but what about other 340 * receivers? Let the receivers tell us the rx-fifo buffers for now. 341 */ 342 int init_winctx_regs(struct vas_window *window, struct vas_winctx *winctx) 343 { 344 u64 val; 345 int fifo_size; 346 347 reset_window_regs(window); 348 349 val = 0ULL; 350 val = SET_FIELD(VAS_LPID, val, winctx->lpid); 351 write_hvwc_reg(window, VREG(LPID), val); 352 353 val = 0ULL; 354 val = SET_FIELD(VAS_PID_ID, val, winctx->pidr); 355 write_hvwc_reg(window, VREG(PID), val); 356 357 init_xlate_regs(window, winctx->user_win); 358 359 val = 0ULL; 360 val = SET_FIELD(VAS_FAULT_TX_WIN, val, 0); 361 write_hvwc_reg(window, VREG(FAULT_TX_WIN), val); 362 363 /* In PowerNV, interrupts go to HV. */ 364 write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL); 365 366 val = 0ULL; 367 val = SET_FIELD(VAS_HV_INTR_SRC_RA, val, winctx->irq_port); 368 write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), val); 369 370 val = 0ULL; 371 val = SET_FIELD(VAS_PSWID_EA_HANDLE, val, winctx->pswid); 372 write_hvwc_reg(window, VREG(PSWID), val); 373 374 write_hvwc_reg(window, VREG(SPARE1), 0ULL); 375 write_hvwc_reg(window, VREG(SPARE2), 0ULL); 376 write_hvwc_reg(window, VREG(SPARE3), 0ULL); 377 378 /* 379 * NOTE: VAS expects the FIFO address to be copied into the LFIFO_BAR 380 * register as is - do NOT shift the address into VAS_LFIFO_BAR 381 * bit fields! Ok to set the page migration select fields - 382 * VAS ignores the lower 10+ bits in the address anyway, because 383 * the minimum FIFO size is 1K? 384 * 385 * See also: Design note in function header. 386 */ 387 val = __pa(winctx->rx_fifo); 388 val = SET_FIELD(VAS_PAGE_MIGRATION_SELECT, val, 0); 389 write_hvwc_reg(window, VREG(LFIFO_BAR), val); 390 391 val = 0ULL; 392 val = SET_FIELD(VAS_LDATA_STAMP, val, winctx->data_stamp); 393 write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), val); 394 395 val = 0ULL; 396 val = SET_FIELD(VAS_LDMA_TYPE, val, winctx->dma_type); 397 val = SET_FIELD(VAS_LDMA_FIFO_DISABLE, val, winctx->fifo_disable); 398 write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), val); 399 400 write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL); 401 write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL); 402 write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL); 403 404 val = 0ULL; 405 val = SET_FIELD(VAS_LRX_WCRED, val, winctx->wcreds_max); 406 write_hvwc_reg(window, VREG(LRX_WCRED), val); 407 408 val = 0ULL; 409 val = SET_FIELD(VAS_TX_WCRED, val, winctx->wcreds_max); 410 write_hvwc_reg(window, VREG(TX_WCRED), val); 411 412 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL); 413 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL); 414 415 fifo_size = winctx->rx_fifo_size / 1024; 416 417 val = 0ULL; 418 val = SET_FIELD(VAS_LFIFO_SIZE, val, ilog2(fifo_size)); 419 write_hvwc_reg(window, VREG(LFIFO_SIZE), val); 420 421 /* Update window control and caching control registers last so 422 * we mark the window open only after fully initializing it and 423 * pushing context to cache. 424 */ 425 426 write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL); 427 428 init_rsvd_tx_buf_count(window, winctx); 429 430 /* for a send window, point to the matching receive window */ 431 val = 0ULL; 432 val = SET_FIELD(VAS_LRX_WIN_ID, val, winctx->rx_win_id); 433 write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), val); 434 435 write_hvwc_reg(window, VREG(SPARE4), 0ULL); 436 437 val = 0ULL; 438 val = SET_FIELD(VAS_NOTIFY_DISABLE, val, winctx->notify_disable); 439 val = SET_FIELD(VAS_INTR_DISABLE, val, winctx->intr_disable); 440 val = SET_FIELD(VAS_NOTIFY_EARLY, val, winctx->notify_early); 441 val = SET_FIELD(VAS_NOTIFY_OSU_INTR, val, winctx->notify_os_intr_reg); 442 write_hvwc_reg(window, VREG(LNOTIFY_CTL), val); 443 444 val = 0ULL; 445 val = SET_FIELD(VAS_LNOTIFY_PID, val, winctx->lnotify_pid); 446 write_hvwc_reg(window, VREG(LNOTIFY_PID), val); 447 448 val = 0ULL; 449 val = SET_FIELD(VAS_LNOTIFY_LPID, val, winctx->lnotify_lpid); 450 write_hvwc_reg(window, VREG(LNOTIFY_LPID), val); 451 452 val = 0ULL; 453 val = SET_FIELD(VAS_LNOTIFY_TID, val, winctx->lnotify_tid); 454 write_hvwc_reg(window, VREG(LNOTIFY_TID), val); 455 456 val = 0ULL; 457 val = SET_FIELD(VAS_LNOTIFY_MIN_SCOPE, val, winctx->min_scope); 458 val = SET_FIELD(VAS_LNOTIFY_MAX_SCOPE, val, winctx->max_scope); 459 write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), val); 460 461 /* Skip read-only registers NX_UTIL and NX_UTIL_SE */ 462 463 write_hvwc_reg(window, VREG(SPARE5), 0ULL); 464 write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL); 465 write_hvwc_reg(window, VREG(SPARE6), 0ULL); 466 467 /* Finally, push window context to memory and... */ 468 val = 0ULL; 469 val = SET_FIELD(VAS_PUSH_TO_MEM, val, 1); 470 write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), val); 471 472 /* ... mark the window open for business */ 473 val = 0ULL; 474 val = SET_FIELD(VAS_WINCTL_REJ_NO_CREDIT, val, winctx->rej_no_credit); 475 val = SET_FIELD(VAS_WINCTL_PIN, val, winctx->pin_win); 476 val = SET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val, winctx->tx_wcred_mode); 477 val = SET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val, winctx->rx_wcred_mode); 478 val = SET_FIELD(VAS_WINCTL_TX_WORD_MODE, val, winctx->tx_word_mode); 479 val = SET_FIELD(VAS_WINCTL_RX_WORD_MODE, val, winctx->rx_word_mode); 480 val = SET_FIELD(VAS_WINCTL_FAULT_WIN, val, winctx->fault_win); 481 val = SET_FIELD(VAS_WINCTL_NX_WIN, val, winctx->nx_win); 482 val = SET_FIELD(VAS_WINCTL_OPEN, val, 1); 483 write_hvwc_reg(window, VREG(WINCTL), val); 484 485 return 0; 486 } 487 488 /* stub for now */ 489 int vas_win_close(struct vas_window *window) 490 { 491 return -1; 492 } 493