1 /* 2 * Copyright 2016-17 IBM Corp. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; either version 7 * 2 of the License, or (at your option) any later version. 8 */ 9 10 #ifndef _VAS_H 11 #define _VAS_H 12 #include <linux/atomic.h> 13 #include <linux/idr.h> 14 #include <asm/vas.h> 15 #include <linux/io.h> 16 #include <linux/dcache.h> 17 #include <linux/mutex.h> 18 19 /* 20 * Overview of Virtual Accelerator Switchboard (VAS). 21 * 22 * VAS is a hardware "switchboard" that allows senders and receivers to 23 * exchange messages with _minimal_ kernel involvment. The receivers are 24 * typically NX coprocessor engines that perform compression or encryption 25 * in hardware, but receivers can also be other software threads. 26 * 27 * Senders are user/kernel threads that submit compression/encryption or 28 * other requests to the receivers. Senders must format their messages as 29 * Coprocessor Request Blocks (CRB)s and submit them using the "copy" and 30 * "paste" instructions which were introduced in Power9. 31 * 32 * A Power node can have (upto?) 8 Power chips. There is one instance of 33 * VAS in each Power9 chip. Each instance of VAS has 64K windows or ports, 34 * Senders and receivers must each connect to a separate window before they 35 * can exchange messages through the switchboard. 36 * 37 * Each window is described by two types of window contexts: 38 * 39 * Hypervisor Window Context (HVWC) of size VAS_HVWC_SIZE bytes 40 * 41 * OS/User Window Context (UWC) of size VAS_UWC_SIZE bytes. 42 * 43 * A window context can be viewed as a set of 64-bit registers. The settings 44 * in these registers configure/control/determine the behavior of the VAS 45 * hardware when messages are sent/received through the window. The registers 46 * in the HVWC are configured by the kernel while the registers in the UWC can 47 * be configured by the kernel or by the user space application that is using 48 * the window. 49 * 50 * The HVWCs for all windows on a specific instance of VAS are in a contiguous 51 * range of hardware addresses or Base address region (BAR) referred to as the 52 * HVWC BAR for the instance. Similarly the UWCs for all windows on an instance 53 * are referred to as the UWC BAR for the instance. 54 * 55 * The two BARs for each instance are defined Power9 MMIO Ranges spreadsheet 56 * and available to the kernel in the VAS node's "reg" property in the device 57 * tree: 58 * 59 * /proc/device-tree/vasm@.../reg 60 * 61 * (see vas_probe() for details on the reg property). 62 * 63 * The kernel maps the HVWC and UWC BAR regions into the kernel address 64 * space (hvwc_map and uwc_map). The kernel can then access the window 65 * contexts of a specific window using: 66 * 67 * hvwc = hvwc_map + winid * VAS_HVWC_SIZE. 68 * uwc = uwc_map + winid * VAS_UWC_SIZE. 69 * 70 * where winid is the window index (0..64K). 71 * 72 * As mentioned, a window context is used to "configure" a window. Besides 73 * this configuration address, each _send_ window also has a unique hardware 74 * "paste" address that is used to submit requests/CRBs (see vas_paste_crb()). 75 * 76 * The hardware paste address for a window is computed using the "paste 77 * base address" and "paste win id shift" reg properties in the VAS device 78 * tree node using: 79 * 80 * paste_addr = paste_base + ((winid << paste_win_id_shift)) 81 * 82 * (again, see vas_probe() for ->paste_base_addr and ->paste_win_id_shift). 83 * 84 * The kernel maps this hardware address into the sender's address space 85 * after which they can use the 'paste' instruction (new in Power9) to 86 * send a message (submit a request aka CRB) to the coprocessor. 87 * 88 * NOTE: In the initial version, senders can only in-kernel drivers/threads. 89 * Support for user space threads will be added in follow-on patches. 90 * 91 * TODO: Do we need to map the UWC into user address space so they can return 92 * credits? Its NA for NX but may be needed for other receive windows. 93 * 94 */ 95 96 #define VAS_WINDOWS_PER_CHIP (64 << 10) 97 98 /* 99 * Hypervisor and OS/USer Window Context sizes 100 */ 101 #define VAS_HVWC_SIZE 512 102 #define VAS_UWC_SIZE PAGE_SIZE 103 104 /* 105 * Initial per-process credits. 106 * Max send window credits: 4K-1 (12-bits in VAS_TX_WCRED) 107 * Max receive window credits: 64K-1 (16 bits in VAS_LRX_WCRED) 108 * 109 * TODO: Needs tuning for per-process credits 110 */ 111 #define VAS_RX_WCREDS_MAX ((64 << 10) - 1) 112 #define VAS_TX_WCREDS_MAX ((4 << 10) - 1) 113 #define VAS_WCREDS_DEFAULT (1 << 10) 114 115 /* 116 * VAS Window Context Register Offsets and bitmasks. 117 * See Section 3.1.4 of VAS Work book 118 */ 119 #define VAS_LPID_OFFSET 0x010 120 #define VAS_LPID PPC_BITMASK(0, 11) 121 122 #define VAS_PID_OFFSET 0x018 123 #define VAS_PID_ID PPC_BITMASK(0, 19) 124 125 #define VAS_XLATE_MSR_OFFSET 0x020 126 #define VAS_XLATE_MSR_DR PPC_BIT(0) 127 #define VAS_XLATE_MSR_TA PPC_BIT(1) 128 #define VAS_XLATE_MSR_PR PPC_BIT(2) 129 #define VAS_XLATE_MSR_US PPC_BIT(3) 130 #define VAS_XLATE_MSR_HV PPC_BIT(4) 131 #define VAS_XLATE_MSR_SF PPC_BIT(5) 132 133 #define VAS_XLATE_LPCR_OFFSET 0x028 134 #define VAS_XLATE_LPCR_PAGE_SIZE PPC_BITMASK(0, 2) 135 #define VAS_XLATE_LPCR_ISL PPC_BIT(3) 136 #define VAS_XLATE_LPCR_TC PPC_BIT(4) 137 #define VAS_XLATE_LPCR_SC PPC_BIT(5) 138 139 #define VAS_XLATE_CTL_OFFSET 0x030 140 #define VAS_XLATE_MODE PPC_BITMASK(0, 1) 141 142 #define VAS_AMR_OFFSET 0x040 143 #define VAS_AMR PPC_BITMASK(0, 63) 144 145 #define VAS_SEIDR_OFFSET 0x048 146 #define VAS_SEIDR PPC_BITMASK(0, 63) 147 148 #define VAS_FAULT_TX_WIN_OFFSET 0x050 149 #define VAS_FAULT_TX_WIN PPC_BITMASK(48, 63) 150 151 #define VAS_OSU_INTR_SRC_RA_OFFSET 0x060 152 #define VAS_OSU_INTR_SRC_RA PPC_BITMASK(8, 63) 153 154 #define VAS_HV_INTR_SRC_RA_OFFSET 0x070 155 #define VAS_HV_INTR_SRC_RA PPC_BITMASK(8, 63) 156 157 #define VAS_PSWID_OFFSET 0x078 158 #define VAS_PSWID_EA_HANDLE PPC_BITMASK(0, 31) 159 160 #define VAS_SPARE1_OFFSET 0x080 161 #define VAS_SPARE2_OFFSET 0x088 162 #define VAS_SPARE3_OFFSET 0x090 163 #define VAS_SPARE4_OFFSET 0x130 164 #define VAS_SPARE5_OFFSET 0x160 165 #define VAS_SPARE6_OFFSET 0x188 166 167 #define VAS_LFIFO_BAR_OFFSET 0x0A0 168 #define VAS_LFIFO_BAR PPC_BITMASK(8, 53) 169 #define VAS_PAGE_MIGRATION_SELECT PPC_BITMASK(54, 56) 170 171 #define VAS_LDATA_STAMP_CTL_OFFSET 0x0A8 172 #define VAS_LDATA_STAMP PPC_BITMASK(0, 1) 173 #define VAS_XTRA_WRITE PPC_BIT(2) 174 175 #define VAS_LDMA_CACHE_CTL_OFFSET 0x0B0 176 #define VAS_LDMA_TYPE PPC_BITMASK(0, 1) 177 #define VAS_LDMA_FIFO_DISABLE PPC_BIT(2) 178 179 #define VAS_LRFIFO_PUSH_OFFSET 0x0B8 180 #define VAS_LRFIFO_PUSH PPC_BITMASK(0, 15) 181 182 #define VAS_CURR_MSG_COUNT_OFFSET 0x0C0 183 #define VAS_CURR_MSG_COUNT PPC_BITMASK(0, 7) 184 185 #define VAS_LNOTIFY_AFTER_COUNT_OFFSET 0x0C8 186 #define VAS_LNOTIFY_AFTER_COUNT PPC_BITMASK(0, 7) 187 188 #define VAS_LRX_WCRED_OFFSET 0x0E0 189 #define VAS_LRX_WCRED PPC_BITMASK(0, 15) 190 191 #define VAS_LRX_WCRED_ADDER_OFFSET 0x190 192 #define VAS_LRX_WCRED_ADDER PPC_BITMASK(0, 15) 193 194 #define VAS_TX_WCRED_OFFSET 0x0F0 195 #define VAS_TX_WCRED PPC_BITMASK(4, 15) 196 197 #define VAS_TX_WCRED_ADDER_OFFSET 0x1A0 198 #define VAS_TX_WCRED_ADDER PPC_BITMASK(4, 15) 199 200 #define VAS_LFIFO_SIZE_OFFSET 0x100 201 #define VAS_LFIFO_SIZE PPC_BITMASK(0, 3) 202 203 #define VAS_WINCTL_OFFSET 0x108 204 #define VAS_WINCTL_OPEN PPC_BIT(0) 205 #define VAS_WINCTL_REJ_NO_CREDIT PPC_BIT(1) 206 #define VAS_WINCTL_PIN PPC_BIT(2) 207 #define VAS_WINCTL_TX_WCRED_MODE PPC_BIT(3) 208 #define VAS_WINCTL_RX_WCRED_MODE PPC_BIT(4) 209 #define VAS_WINCTL_TX_WORD_MODE PPC_BIT(5) 210 #define VAS_WINCTL_RX_WORD_MODE PPC_BIT(6) 211 #define VAS_WINCTL_RSVD_TXBUF PPC_BIT(7) 212 #define VAS_WINCTL_THRESH_CTL PPC_BITMASK(8, 9) 213 #define VAS_WINCTL_FAULT_WIN PPC_BIT(10) 214 #define VAS_WINCTL_NX_WIN PPC_BIT(11) 215 216 #define VAS_WIN_STATUS_OFFSET 0x110 217 #define VAS_WIN_BUSY PPC_BIT(1) 218 219 #define VAS_WIN_CTX_CACHING_CTL_OFFSET 0x118 220 #define VAS_CASTOUT_REQ PPC_BIT(0) 221 #define VAS_PUSH_TO_MEM PPC_BIT(1) 222 #define VAS_WIN_CACHE_STATUS PPC_BIT(4) 223 224 #define VAS_TX_RSVD_BUF_COUNT_OFFSET 0x120 225 #define VAS_RXVD_BUF_COUNT PPC_BITMASK(58, 63) 226 227 #define VAS_LRFIFO_WIN_PTR_OFFSET 0x128 228 #define VAS_LRX_WIN_ID PPC_BITMASK(0, 15) 229 230 /* 231 * Local Notification Control Register controls what happens in _response_ 232 * to a paste command and hence applies only to receive windows. 233 */ 234 #define VAS_LNOTIFY_CTL_OFFSET 0x138 235 #define VAS_NOTIFY_DISABLE PPC_BIT(0) 236 #define VAS_INTR_DISABLE PPC_BIT(1) 237 #define VAS_NOTIFY_EARLY PPC_BIT(2) 238 #define VAS_NOTIFY_OSU_INTR PPC_BIT(3) 239 240 #define VAS_LNOTIFY_PID_OFFSET 0x140 241 #define VAS_LNOTIFY_PID PPC_BITMASK(0, 19) 242 243 #define VAS_LNOTIFY_LPID_OFFSET 0x148 244 #define VAS_LNOTIFY_LPID PPC_BITMASK(0, 11) 245 246 #define VAS_LNOTIFY_TID_OFFSET 0x150 247 #define VAS_LNOTIFY_TID PPC_BITMASK(0, 15) 248 249 #define VAS_LNOTIFY_SCOPE_OFFSET 0x158 250 #define VAS_LNOTIFY_MIN_SCOPE PPC_BITMASK(0, 1) 251 #define VAS_LNOTIFY_MAX_SCOPE PPC_BITMASK(2, 3) 252 253 #define VAS_NX_UTIL_OFFSET 0x1B0 254 #define VAS_NX_UTIL PPC_BITMASK(0, 63) 255 256 /* SE: Side effects */ 257 #define VAS_NX_UTIL_SE_OFFSET 0x1B8 258 #define VAS_NX_UTIL_SE PPC_BITMASK(0, 63) 259 260 #define VAS_NX_UTIL_ADDER_OFFSET 0x180 261 #define VAS_NX_UTIL_ADDER PPC_BITMASK(32, 63) 262 263 /* 264 * VREG(x): 265 * Expand a register's short name (eg: LPID) into two parameters: 266 * - the register's short name in string form ("LPID"), and 267 * - the name of the macro (eg: VAS_LPID_OFFSET), defining the 268 * register's offset in the window context 269 */ 270 #define VREG_SFX(n, s) __stringify(n), VAS_##n##s 271 #define VREG(r) VREG_SFX(r, _OFFSET) 272 273 /* 274 * Local Notify Scope Control Register. (Receive windows only). 275 */ 276 enum vas_notify_scope { 277 VAS_SCOPE_LOCAL, 278 VAS_SCOPE_GROUP, 279 VAS_SCOPE_VECTORED_GROUP, 280 VAS_SCOPE_UNUSED, 281 }; 282 283 /* 284 * Local DMA Cache Control Register (Receive windows only). 285 */ 286 enum vas_dma_type { 287 VAS_DMA_TYPE_INJECT, 288 VAS_DMA_TYPE_WRITE, 289 }; 290 291 /* 292 * Local Notify Scope Control Register. (Receive windows only). 293 * Not applicable to NX receive windows. 294 */ 295 enum vas_notify_after_count { 296 VAS_NOTIFY_AFTER_256 = 0, 297 VAS_NOTIFY_NONE, 298 VAS_NOTIFY_AFTER_2 299 }; 300 301 /* 302 * One per instance of VAS. Each instance will have a separate set of 303 * receive windows, one per coprocessor type. 304 * 305 * See also function header of set_vinst_win() for details on ->windows[] 306 * and ->rxwin[] tables. 307 */ 308 struct vas_instance { 309 int vas_id; 310 struct ida ida; 311 struct list_head node; 312 struct platform_device *pdev; 313 314 u64 hvwc_bar_start; 315 u64 uwc_bar_start; 316 u64 paste_base_addr; 317 u64 paste_win_id_shift; 318 319 struct mutex mutex; 320 struct vas_window *rxwin[VAS_COP_TYPE_MAX]; 321 struct vas_window *windows[VAS_WINDOWS_PER_CHIP]; 322 323 char *dbgname; 324 struct dentry *dbgdir; 325 }; 326 327 /* 328 * In-kernel state a VAS window. One per window. 329 */ 330 struct vas_window { 331 /* Fields common to send and receive windows */ 332 struct vas_instance *vinst; 333 int winid; 334 bool tx_win; /* True if send window */ 335 bool nx_win; /* True if NX window */ 336 bool user_win; /* True if user space window */ 337 void *hvwc_map; /* HV window context */ 338 void *uwc_map; /* OS/User window context */ 339 pid_t pid; /* Linux process id of owner */ 340 int wcreds_max; /* Window credits */ 341 342 char *dbgname; 343 struct dentry *dbgdir; 344 345 /* Fields applicable only to send windows */ 346 void *paste_kaddr; 347 char *paste_addr_name; 348 struct vas_window *rxwin; 349 350 /* Feilds applicable only to receive windows */ 351 enum vas_cop_type cop; 352 atomic_t num_txwins; 353 }; 354 355 /* 356 * Container for the hardware state of a window. One per-window. 357 * 358 * A VAS Window context is a 512-byte area in the hardware that contains 359 * a set of 64-bit registers. Individual bit-fields in these registers 360 * determine the configuration/operation of the hardware. struct vas_winctx 361 * is a container for the register fields in the window context. 362 */ 363 struct vas_winctx { 364 void *rx_fifo; 365 int rx_fifo_size; 366 int wcreds_max; 367 int rsvd_txbuf_count; 368 369 bool user_win; 370 bool nx_win; 371 bool fault_win; 372 bool rsvd_txbuf_enable; 373 bool pin_win; 374 bool rej_no_credit; 375 bool tx_wcred_mode; 376 bool rx_wcred_mode; 377 bool tx_word_mode; 378 bool rx_word_mode; 379 bool data_stamp; 380 bool xtra_write; 381 bool notify_disable; 382 bool intr_disable; 383 bool fifo_disable; 384 bool notify_early; 385 bool notify_os_intr_reg; 386 387 int lpid; 388 int pidr; /* value from SPRN_PID, not linux pid */ 389 int lnotify_lpid; 390 int lnotify_pid; 391 int lnotify_tid; 392 u32 pswid; 393 int rx_win_id; 394 int fault_win_id; 395 int tc_mode; 396 397 u64 irq_port; 398 399 enum vas_dma_type dma_type; 400 enum vas_notify_scope min_scope; 401 enum vas_notify_scope max_scope; 402 enum vas_notify_after_count notify_after_count; 403 }; 404 405 extern struct mutex vas_mutex; 406 407 extern struct vas_instance *find_vas_instance(int vasid); 408 extern void vas_init_dbgdir(void); 409 extern void vas_instance_init_dbgdir(struct vas_instance *vinst); 410 extern void vas_window_init_dbgdir(struct vas_window *win); 411 extern void vas_window_free_dbgdir(struct vas_window *win); 412 413 static inline void vas_log_write(struct vas_window *win, char *name, 414 void *regptr, u64 val) 415 { 416 if (val) 417 pr_debug("%swin #%d: %s reg %p, val 0x%016llx\n", 418 win->tx_win ? "Tx" : "Rx", win->winid, name, 419 regptr, val); 420 } 421 422 static inline void write_uwc_reg(struct vas_window *win, char *name, 423 s32 reg, u64 val) 424 { 425 void *regptr; 426 427 regptr = win->uwc_map + reg; 428 vas_log_write(win, name, regptr, val); 429 430 out_be64(regptr, val); 431 } 432 433 static inline void write_hvwc_reg(struct vas_window *win, char *name, 434 s32 reg, u64 val) 435 { 436 void *regptr; 437 438 regptr = win->hvwc_map + reg; 439 vas_log_write(win, name, regptr, val); 440 441 out_be64(regptr, val); 442 } 443 444 static inline u64 read_hvwc_reg(struct vas_window *win, 445 char *name __maybe_unused, s32 reg) 446 { 447 return in_be64(win->hvwc_map+reg); 448 } 449 450 /* 451 * Encode/decode the Partition Send Window ID (PSWID) for a window in 452 * a way that we can uniquely identify any window in the system. i.e. 453 * we should be able to locate the 'struct vas_window' given the PSWID. 454 * 455 * Bits Usage 456 * 0:7 VAS id (8 bits) 457 * 8:15 Unused, 0 (3 bits) 458 * 16:31 Window id (16 bits) 459 */ 460 static inline u32 encode_pswid(int vasid, int winid) 461 { 462 u32 pswid = 0; 463 464 pswid |= vasid << (31 - 7); 465 pswid |= winid; 466 467 return pswid; 468 } 469 470 static inline void decode_pswid(u32 pswid, int *vasid, int *winid) 471 { 472 if (vasid) 473 *vasid = pswid >> (31 - 7) & 0xFF; 474 475 if (winid) 476 *winid = pswid & 0xFFFF; 477 } 478 #endif /* _VAS_H */ 479