1 /* 2 * Copyright 2016-17 IBM Corp. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; either version 7 * 2 of the License, or (at your option) any later version. 8 */ 9 10 #ifndef _VAS_H 11 #define _VAS_H 12 #include <linux/atomic.h> 13 #include <linux/idr.h> 14 #include <asm/vas.h> 15 #include <linux/io.h> 16 17 /* 18 * Overview of Virtual Accelerator Switchboard (VAS). 19 * 20 * VAS is a hardware "switchboard" that allows senders and receivers to 21 * exchange messages with _minimal_ kernel involvment. The receivers are 22 * typically NX coprocessor engines that perform compression or encryption 23 * in hardware, but receivers can also be other software threads. 24 * 25 * Senders are user/kernel threads that submit compression/encryption or 26 * other requests to the receivers. Senders must format their messages as 27 * Coprocessor Request Blocks (CRB)s and submit them using the "copy" and 28 * "paste" instructions which were introduced in Power9. 29 * 30 * A Power node can have (upto?) 8 Power chips. There is one instance of 31 * VAS in each Power9 chip. Each instance of VAS has 64K windows or ports, 32 * Senders and receivers must each connect to a separate window before they 33 * can exchange messages through the switchboard. 34 * 35 * Each window is described by two types of window contexts: 36 * 37 * Hypervisor Window Context (HVWC) of size VAS_HVWC_SIZE bytes 38 * 39 * OS/User Window Context (UWC) of size VAS_UWC_SIZE bytes. 40 * 41 * A window context can be viewed as a set of 64-bit registers. The settings 42 * in these registers configure/control/determine the behavior of the VAS 43 * hardware when messages are sent/received through the window. The registers 44 * in the HVWC are configured by the kernel while the registers in the UWC can 45 * be configured by the kernel or by the user space application that is using 46 * the window. 47 * 48 * The HVWCs for all windows on a specific instance of VAS are in a contiguous 49 * range of hardware addresses or Base address region (BAR) referred to as the 50 * HVWC BAR for the instance. Similarly the UWCs for all windows on an instance 51 * are referred to as the UWC BAR for the instance. 52 * 53 * The two BARs for each instance are defined Power9 MMIO Ranges spreadsheet 54 * and available to the kernel in the VAS node's "reg" property in the device 55 * tree: 56 * 57 * /proc/device-tree/vasm@.../reg 58 * 59 * (see vas_probe() for details on the reg property). 60 * 61 * The kernel maps the HVWC and UWC BAR regions into the kernel address 62 * space (hvwc_map and uwc_map). The kernel can then access the window 63 * contexts of a specific window using: 64 * 65 * hvwc = hvwc_map + winid * VAS_HVWC_SIZE. 66 * uwc = uwc_map + winid * VAS_UWC_SIZE. 67 * 68 * where winid is the window index (0..64K). 69 * 70 * As mentioned, a window context is used to "configure" a window. Besides 71 * this configuration address, each _send_ window also has a unique hardware 72 * "paste" address that is used to submit requests/CRBs (see vas_paste_crb()). 73 * 74 * The hardware paste address for a window is computed using the "paste 75 * base address" and "paste win id shift" reg properties in the VAS device 76 * tree node using: 77 * 78 * paste_addr = paste_base + ((winid << paste_win_id_shift)) 79 * 80 * (again, see vas_probe() for ->paste_base_addr and ->paste_win_id_shift). 81 * 82 * The kernel maps this hardware address into the sender's address space 83 * after which they can use the 'paste' instruction (new in Power9) to 84 * send a message (submit a request aka CRB) to the coprocessor. 85 * 86 * NOTE: In the initial version, senders can only in-kernel drivers/threads. 87 * Support for user space threads will be added in follow-on patches. 88 * 89 * TODO: Do we need to map the UWC into user address space so they can return 90 * credits? Its NA for NX but may be needed for other receive windows. 91 * 92 */ 93 94 #define VAS_WINDOWS_PER_CHIP (64 << 10) 95 96 /* 97 * Hypervisor and OS/USer Window Context sizes 98 */ 99 #define VAS_HVWC_SIZE 512 100 #define VAS_UWC_SIZE PAGE_SIZE 101 102 /* 103 * Initial per-process credits. 104 * Max send window credits: 4K-1 (12-bits in VAS_TX_WCRED) 105 * Max receive window credits: 64K-1 (16 bits in VAS_LRX_WCRED) 106 * 107 * TODO: Needs tuning for per-process credits 108 */ 109 #define VAS_WCREDS_MIN 16 110 #define VAS_WCREDS_MAX ((64 << 10) - 1) 111 #define VAS_WCREDS_DEFAULT (1 << 10) 112 113 /* 114 * VAS Window Context Register Offsets and bitmasks. 115 * See Section 3.1.4 of VAS Work book 116 */ 117 #define VAS_LPID_OFFSET 0x010 118 #define VAS_LPID PPC_BITMASK(0, 11) 119 120 #define VAS_PID_OFFSET 0x018 121 #define VAS_PID_ID PPC_BITMASK(0, 19) 122 123 #define VAS_XLATE_MSR_OFFSET 0x020 124 #define VAS_XLATE_MSR_DR PPC_BIT(0) 125 #define VAS_XLATE_MSR_TA PPC_BIT(1) 126 #define VAS_XLATE_MSR_PR PPC_BIT(2) 127 #define VAS_XLATE_MSR_US PPC_BIT(3) 128 #define VAS_XLATE_MSR_HV PPC_BIT(4) 129 #define VAS_XLATE_MSR_SF PPC_BIT(5) 130 131 #define VAS_XLATE_LPCR_OFFSET 0x028 132 #define VAS_XLATE_LPCR_PAGE_SIZE PPC_BITMASK(0, 2) 133 #define VAS_XLATE_LPCR_ISL PPC_BIT(3) 134 #define VAS_XLATE_LPCR_TC PPC_BIT(4) 135 #define VAS_XLATE_LPCR_SC PPC_BIT(5) 136 137 #define VAS_XLATE_CTL_OFFSET 0x030 138 #define VAS_XLATE_MODE PPC_BITMASK(0, 1) 139 140 #define VAS_AMR_OFFSET 0x040 141 #define VAS_AMR PPC_BITMASK(0, 63) 142 143 #define VAS_SEIDR_OFFSET 0x048 144 #define VAS_SEIDR PPC_BITMASK(0, 63) 145 146 #define VAS_FAULT_TX_WIN_OFFSET 0x050 147 #define VAS_FAULT_TX_WIN PPC_BITMASK(48, 63) 148 149 #define VAS_OSU_INTR_SRC_RA_OFFSET 0x060 150 #define VAS_OSU_INTR_SRC_RA PPC_BITMASK(8, 63) 151 152 #define VAS_HV_INTR_SRC_RA_OFFSET 0x070 153 #define VAS_HV_INTR_SRC_RA PPC_BITMASK(8, 63) 154 155 #define VAS_PSWID_OFFSET 0x078 156 #define VAS_PSWID_EA_HANDLE PPC_BITMASK(0, 31) 157 158 #define VAS_SPARE1_OFFSET 0x080 159 #define VAS_SPARE2_OFFSET 0x088 160 #define VAS_SPARE3_OFFSET 0x090 161 #define VAS_SPARE4_OFFSET 0x130 162 #define VAS_SPARE5_OFFSET 0x160 163 #define VAS_SPARE6_OFFSET 0x188 164 165 #define VAS_LFIFO_BAR_OFFSET 0x0A0 166 #define VAS_LFIFO_BAR PPC_BITMASK(8, 53) 167 #define VAS_PAGE_MIGRATION_SELECT PPC_BITMASK(54, 56) 168 169 #define VAS_LDATA_STAMP_CTL_OFFSET 0x0A8 170 #define VAS_LDATA_STAMP PPC_BITMASK(0, 1) 171 #define VAS_XTRA_WRITE PPC_BIT(2) 172 173 #define VAS_LDMA_CACHE_CTL_OFFSET 0x0B0 174 #define VAS_LDMA_TYPE PPC_BITMASK(0, 1) 175 #define VAS_LDMA_FIFO_DISABLE PPC_BIT(2) 176 177 #define VAS_LRFIFO_PUSH_OFFSET 0x0B8 178 #define VAS_LRFIFO_PUSH PPC_BITMASK(0, 15) 179 180 #define VAS_CURR_MSG_COUNT_OFFSET 0x0C0 181 #define VAS_CURR_MSG_COUNT PPC_BITMASK(0, 7) 182 183 #define VAS_LNOTIFY_AFTER_COUNT_OFFSET 0x0C8 184 #define VAS_LNOTIFY_AFTER_COUNT PPC_BITMASK(0, 7) 185 186 #define VAS_LRX_WCRED_OFFSET 0x0E0 187 #define VAS_LRX_WCRED PPC_BITMASK(0, 15) 188 189 #define VAS_LRX_WCRED_ADDER_OFFSET 0x190 190 #define VAS_LRX_WCRED_ADDER PPC_BITMASK(0, 15) 191 192 #define VAS_TX_WCRED_OFFSET 0x0F0 193 #define VAS_TX_WCRED PPC_BITMASK(4, 15) 194 195 #define VAS_TX_WCRED_ADDER_OFFSET 0x1A0 196 #define VAS_TX_WCRED_ADDER PPC_BITMASK(4, 15) 197 198 #define VAS_LFIFO_SIZE_OFFSET 0x100 199 #define VAS_LFIFO_SIZE PPC_BITMASK(0, 3) 200 201 #define VAS_WINCTL_OFFSET 0x108 202 #define VAS_WINCTL_OPEN PPC_BIT(0) 203 #define VAS_WINCTL_REJ_NO_CREDIT PPC_BIT(1) 204 #define VAS_WINCTL_PIN PPC_BIT(2) 205 #define VAS_WINCTL_TX_WCRED_MODE PPC_BIT(3) 206 #define VAS_WINCTL_RX_WCRED_MODE PPC_BIT(4) 207 #define VAS_WINCTL_TX_WORD_MODE PPC_BIT(5) 208 #define VAS_WINCTL_RX_WORD_MODE PPC_BIT(6) 209 #define VAS_WINCTL_RSVD_TXBUF PPC_BIT(7) 210 #define VAS_WINCTL_THRESH_CTL PPC_BITMASK(8, 9) 211 #define VAS_WINCTL_FAULT_WIN PPC_BIT(10) 212 #define VAS_WINCTL_NX_WIN PPC_BIT(11) 213 214 #define VAS_WIN_STATUS_OFFSET 0x110 215 #define VAS_WIN_BUSY PPC_BIT(1) 216 217 #define VAS_WIN_CTX_CACHING_CTL_OFFSET 0x118 218 #define VAS_CASTOUT_REQ PPC_BIT(0) 219 #define VAS_PUSH_TO_MEM PPC_BIT(1) 220 #define VAS_WIN_CACHE_STATUS PPC_BIT(4) 221 222 #define VAS_TX_RSVD_BUF_COUNT_OFFSET 0x120 223 #define VAS_RXVD_BUF_COUNT PPC_BITMASK(58, 63) 224 225 #define VAS_LRFIFO_WIN_PTR_OFFSET 0x128 226 #define VAS_LRX_WIN_ID PPC_BITMASK(0, 15) 227 228 /* 229 * Local Notification Control Register controls what happens in _response_ 230 * to a paste command and hence applies only to receive windows. 231 */ 232 #define VAS_LNOTIFY_CTL_OFFSET 0x138 233 #define VAS_NOTIFY_DISABLE PPC_BIT(0) 234 #define VAS_INTR_DISABLE PPC_BIT(1) 235 #define VAS_NOTIFY_EARLY PPC_BIT(2) 236 #define VAS_NOTIFY_OSU_INTR PPC_BIT(3) 237 238 #define VAS_LNOTIFY_PID_OFFSET 0x140 239 #define VAS_LNOTIFY_PID PPC_BITMASK(0, 19) 240 241 #define VAS_LNOTIFY_LPID_OFFSET 0x148 242 #define VAS_LNOTIFY_LPID PPC_BITMASK(0, 11) 243 244 #define VAS_LNOTIFY_TID_OFFSET 0x150 245 #define VAS_LNOTIFY_TID PPC_BITMASK(0, 15) 246 247 #define VAS_LNOTIFY_SCOPE_OFFSET 0x158 248 #define VAS_LNOTIFY_MIN_SCOPE PPC_BITMASK(0, 1) 249 #define VAS_LNOTIFY_MAX_SCOPE PPC_BITMASK(2, 3) 250 251 #define VAS_NX_UTIL_OFFSET 0x1B0 252 #define VAS_NX_UTIL PPC_BITMASK(0, 63) 253 254 /* SE: Side effects */ 255 #define VAS_NX_UTIL_SE_OFFSET 0x1B8 256 #define VAS_NX_UTIL_SE PPC_BITMASK(0, 63) 257 258 #define VAS_NX_UTIL_ADDER_OFFSET 0x180 259 #define VAS_NX_UTIL_ADDER PPC_BITMASK(32, 63) 260 261 /* 262 * Local Notify Scope Control Register. (Receive windows only). 263 */ 264 enum vas_notify_scope { 265 VAS_SCOPE_LOCAL, 266 VAS_SCOPE_GROUP, 267 VAS_SCOPE_VECTORED_GROUP, 268 VAS_SCOPE_UNUSED, 269 }; 270 271 /* 272 * Local DMA Cache Control Register (Receive windows only). 273 */ 274 enum vas_dma_type { 275 VAS_DMA_TYPE_INJECT, 276 VAS_DMA_TYPE_WRITE, 277 }; 278 279 /* 280 * Local Notify Scope Control Register. (Receive windows only). 281 * Not applicable to NX receive windows. 282 */ 283 enum vas_notify_after_count { 284 VAS_NOTIFY_AFTER_256 = 0, 285 VAS_NOTIFY_NONE, 286 VAS_NOTIFY_AFTER_2 287 }; 288 289 /* 290 * One per instance of VAS. Each instance will have a separate set of 291 * receive windows, one per coprocessor type. 292 * 293 * See also function header of set_vinst_win() for details on ->windows[] 294 * and ->rxwin[] tables. 295 */ 296 struct vas_instance { 297 int vas_id; 298 struct ida ida; 299 struct list_head node; 300 struct platform_device *pdev; 301 302 u64 hvwc_bar_start; 303 u64 uwc_bar_start; 304 u64 paste_base_addr; 305 u64 paste_win_id_shift; 306 307 struct mutex mutex; 308 struct vas_window *rxwin[VAS_COP_TYPE_MAX]; 309 struct vas_window *windows[VAS_WINDOWS_PER_CHIP]; 310 }; 311 312 /* 313 * In-kernel state a VAS window. One per window. 314 */ 315 struct vas_window { 316 /* Fields common to send and receive windows */ 317 struct vas_instance *vinst; 318 int winid; 319 bool tx_win; /* True if send window */ 320 bool nx_win; /* True if NX window */ 321 bool user_win; /* True if user space window */ 322 void *hvwc_map; /* HV window context */ 323 void *uwc_map; /* OS/User window context */ 324 pid_t pid; /* Linux process id of owner */ 325 326 /* Fields applicable only to send windows */ 327 void *paste_kaddr; 328 char *paste_addr_name; 329 struct vas_window *rxwin; 330 331 /* Feilds applicable only to receive windows */ 332 enum vas_cop_type cop; 333 atomic_t num_txwins; 334 }; 335 336 /* 337 * Container for the hardware state of a window. One per-window. 338 * 339 * A VAS Window context is a 512-byte area in the hardware that contains 340 * a set of 64-bit registers. Individual bit-fields in these registers 341 * determine the configuration/operation of the hardware. struct vas_winctx 342 * is a container for the register fields in the window context. 343 */ 344 struct vas_winctx { 345 void *rx_fifo; 346 int rx_fifo_size; 347 int wcreds_max; 348 int rsvd_txbuf_count; 349 350 bool user_win; 351 bool nx_win; 352 bool fault_win; 353 bool rsvd_txbuf_enable; 354 bool pin_win; 355 bool rej_no_credit; 356 bool tx_wcred_mode; 357 bool rx_wcred_mode; 358 bool tx_word_mode; 359 bool rx_word_mode; 360 bool data_stamp; 361 bool xtra_write; 362 bool notify_disable; 363 bool intr_disable; 364 bool fifo_disable; 365 bool notify_early; 366 bool notify_os_intr_reg; 367 368 int lpid; 369 int pidr; /* value from SPRN_PID, not linux pid */ 370 int lnotify_lpid; 371 int lnotify_pid; 372 int lnotify_tid; 373 u32 pswid; 374 int rx_win_id; 375 int fault_win_id; 376 int tc_mode; 377 378 u64 irq_port; 379 380 enum vas_dma_type dma_type; 381 enum vas_notify_scope min_scope; 382 enum vas_notify_scope max_scope; 383 enum vas_notify_after_count notify_after_count; 384 }; 385 386 extern struct vas_instance *find_vas_instance(int vasid); 387 388 /* 389 * VREG(x): 390 * Expand a register's short name (eg: LPID) into two parameters: 391 * - the register's short name in string form ("LPID"), and 392 * - the name of the macro (eg: VAS_LPID_OFFSET), defining the 393 * register's offset in the window context 394 */ 395 #define VREG_SFX(n, s) __stringify(n), VAS_##n##s 396 #define VREG(r) VREG_SFX(r, _OFFSET) 397 398 #ifdef vas_debug 399 static inline void dump_rx_win_attr(struct vas_rx_win_attr *attr) 400 { 401 pr_err("fault %d, notify %d, intr %d early %d\n", 402 attr->fault_win, attr->notify_disable, 403 attr->intr_disable, attr->notify_early); 404 405 pr_err("rx_fifo_size %d, max value %d\n", 406 attr->rx_fifo_size, VAS_RX_FIFO_SIZE_MAX); 407 } 408 409 static inline void vas_log_write(struct vas_window *win, char *name, 410 void *regptr, u64 val) 411 { 412 if (val) 413 pr_err("%swin #%d: %s reg %p, val 0x%016llx\n", 414 win->tx_win ? "Tx" : "Rx", win->winid, name, 415 regptr, val); 416 } 417 418 #else /* vas_debug */ 419 420 #define vas_log_write(win, name, reg, val) 421 #define dump_rx_win_attr(attr) 422 423 #endif /* vas_debug */ 424 425 static inline void write_uwc_reg(struct vas_window *win, char *name, 426 s32 reg, u64 val) 427 { 428 void *regptr; 429 430 regptr = win->uwc_map + reg; 431 vas_log_write(win, name, regptr, val); 432 433 out_be64(regptr, val); 434 } 435 436 static inline void write_hvwc_reg(struct vas_window *win, char *name, 437 s32 reg, u64 val) 438 { 439 void *regptr; 440 441 regptr = win->hvwc_map + reg; 442 vas_log_write(win, name, regptr, val); 443 444 out_be64(regptr, val); 445 } 446 447 static inline u64 read_hvwc_reg(struct vas_window *win, 448 char *name __maybe_unused, s32 reg) 449 { 450 return in_be64(win->hvwc_map+reg); 451 } 452 453 #ifdef vas_debug 454 455 static void print_fifo_msg_count(struct vas_window *txwin) 456 { 457 uint64_t read_hvwc_reg(struct vas_window *w, char *n, uint64_t o); 458 pr_devel("Winid %d, Msg count %llu\n", txwin->winid, 459 (uint64_t)read_hvwc_reg(txwin, VREG(LRFIFO_PUSH))); 460 } 461 #else /* vas_debug */ 462 463 #define print_fifo_msg_count(window) 464 465 #endif /* vas_debug */ 466 467 #endif /* _VAS_H */ 468