1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3  * Copyright 2016-17 IBM Corp.
4  */
5 
6 #ifndef _VAS_H
7 #define _VAS_H
8 #include <linux/atomic.h>
9 #include <linux/idr.h>
10 #include <asm/vas.h>
11 #include <linux/io.h>
12 #include <linux/dcache.h>
13 #include <linux/mutex.h>
14 #include <linux/stringify.h>
15 
16 /*
17  * Overview of Virtual Accelerator Switchboard (VAS).
18  *
19  * VAS is a hardware "switchboard" that allows senders and receivers to
20  * exchange messages with _minimal_ kernel involvment. The receivers are
21  * typically NX coprocessor engines that perform compression or encryption
22  * in hardware, but receivers can also be other software threads.
23  *
24  * Senders are user/kernel threads that submit compression/encryption or
25  * other requests to the receivers. Senders must format their messages as
26  * Coprocessor Request Blocks (CRB)s and submit them using the "copy" and
27  * "paste" instructions which were introduced in Power9.
28  *
29  * A Power node can have (upto?) 8 Power chips. There is one instance of
30  * VAS in each Power9 chip. Each instance of VAS has 64K windows or ports,
31  * Senders and receivers must each connect to a separate window before they
32  * can exchange messages through the switchboard.
33  *
34  * Each window is described by two types of window contexts:
35  *
36  *	Hypervisor Window Context (HVWC) of size VAS_HVWC_SIZE bytes
37  *
38  *	OS/User Window Context (UWC) of size VAS_UWC_SIZE bytes.
39  *
40  * A window context can be viewed as a set of 64-bit registers. The settings
41  * in these registers configure/control/determine the behavior of the VAS
42  * hardware when messages are sent/received through the window. The registers
43  * in the HVWC are configured by the kernel while the registers in the UWC can
44  * be configured by the kernel or by the user space application that is using
45  * the window.
46  *
47  * The HVWCs for all windows on a specific instance of VAS are in a contiguous
48  * range of hardware addresses or Base address region (BAR) referred to as the
49  * HVWC BAR for the instance. Similarly the UWCs for all windows on an instance
50  * are referred to as the UWC BAR for the instance.
51  *
52  * The two BARs for each instance are defined Power9 MMIO Ranges spreadsheet
53  * and available to the kernel in the VAS node's "reg" property in the device
54  * tree:
55  *
56  *	/proc/device-tree/vasm@.../reg
57  *
58  * (see vas_probe() for details on the reg property).
59  *
60  * The kernel maps the HVWC and UWC BAR regions into the kernel address
61  * space (hvwc_map and uwc_map). The kernel can then access the window
62  * contexts of a specific window using:
63  *
64  *	 hvwc = hvwc_map + winid * VAS_HVWC_SIZE.
65  *	 uwc = uwc_map + winid * VAS_UWC_SIZE.
66  *
67  * where winid is the window index (0..64K).
68  *
69  * As mentioned, a window context is used to "configure" a window. Besides
70  * this configuration address, each _send_ window also has a unique hardware
71  * "paste" address that is used to submit requests/CRBs (see vas_paste_crb()).
72  *
73  * The hardware paste address for a window is computed using the "paste
74  * base address" and "paste win id shift" reg properties in the VAS device
75  * tree node using:
76  *
77  *	paste_addr = paste_base + ((winid << paste_win_id_shift))
78  *
79  * (again, see vas_probe() for ->paste_base_addr and ->paste_win_id_shift).
80  *
81  * The kernel maps this hardware address into the sender's address space
82  * after which they can use the 'paste' instruction (new in Power9) to
83  * send a message (submit a request aka CRB) to the coprocessor.
84  *
85  * NOTE: In the initial version, senders can only in-kernel drivers/threads.
86  *	 Support for user space threads will be added in follow-on patches.
87  *
88  * TODO: Do we need to map the UWC into user address space so they can return
89  *	 credits? Its NA for NX but may be needed for other receive windows.
90  *
91  */
92 
93 #define VAS_WINDOWS_PER_CHIP		(64 << 10)
94 
95 /*
96  * Hypervisor and OS/USer Window Context sizes
97  */
98 #define VAS_HVWC_SIZE			512
99 #define VAS_UWC_SIZE			PAGE_SIZE
100 
101 /*
102  * Initial per-process credits.
103  * Max send window credits:    4K-1 (12-bits in VAS_TX_WCRED)
104  *
105  * TODO: Needs tuning for per-process credits
106  */
107 #define VAS_TX_WCREDS_MAX		((4 << 10) - 1)
108 #define VAS_WCREDS_DEFAULT		(1 << 10)
109 
110 /*
111  * VAS Window Context Register Offsets and bitmasks.
112  * See Section 3.1.4 of VAS Work book
113  */
114 #define VAS_LPID_OFFSET			0x010
115 #define VAS_LPID			PPC_BITMASK(0, 11)
116 
117 #define VAS_PID_OFFSET			0x018
118 #define VAS_PID_ID			PPC_BITMASK(0, 19)
119 
120 #define VAS_XLATE_MSR_OFFSET		0x020
121 #define VAS_XLATE_MSR_DR		PPC_BIT(0)
122 #define VAS_XLATE_MSR_TA		PPC_BIT(1)
123 #define VAS_XLATE_MSR_PR		PPC_BIT(2)
124 #define VAS_XLATE_MSR_US		PPC_BIT(3)
125 #define VAS_XLATE_MSR_HV		PPC_BIT(4)
126 #define VAS_XLATE_MSR_SF		PPC_BIT(5)
127 
128 #define VAS_XLATE_LPCR_OFFSET		0x028
129 #define VAS_XLATE_LPCR_PAGE_SIZE	PPC_BITMASK(0, 2)
130 #define VAS_XLATE_LPCR_ISL		PPC_BIT(3)
131 #define VAS_XLATE_LPCR_TC		PPC_BIT(4)
132 #define VAS_XLATE_LPCR_SC		PPC_BIT(5)
133 
134 #define VAS_XLATE_CTL_OFFSET		0x030
135 #define VAS_XLATE_MODE			PPC_BITMASK(0, 1)
136 
137 #define VAS_AMR_OFFSET			0x040
138 #define VAS_AMR				PPC_BITMASK(0, 63)
139 
140 #define VAS_SEIDR_OFFSET		0x048
141 #define VAS_SEIDR			PPC_BITMASK(0, 63)
142 
143 #define VAS_FAULT_TX_WIN_OFFSET		0x050
144 #define VAS_FAULT_TX_WIN		PPC_BITMASK(48, 63)
145 
146 #define VAS_OSU_INTR_SRC_RA_OFFSET	0x060
147 #define VAS_OSU_INTR_SRC_RA		PPC_BITMASK(8, 63)
148 
149 #define VAS_HV_INTR_SRC_RA_OFFSET	0x070
150 #define VAS_HV_INTR_SRC_RA		PPC_BITMASK(8, 63)
151 
152 #define VAS_PSWID_OFFSET		0x078
153 #define VAS_PSWID_EA_HANDLE		PPC_BITMASK(0, 31)
154 
155 #define VAS_SPARE1_OFFSET		0x080
156 #define VAS_SPARE2_OFFSET		0x088
157 #define VAS_SPARE3_OFFSET		0x090
158 #define VAS_SPARE4_OFFSET		0x130
159 #define VAS_SPARE5_OFFSET		0x160
160 #define VAS_SPARE6_OFFSET		0x188
161 
162 #define VAS_LFIFO_BAR_OFFSET		0x0A0
163 #define VAS_LFIFO_BAR			PPC_BITMASK(8, 53)
164 #define VAS_PAGE_MIGRATION_SELECT	PPC_BITMASK(54, 56)
165 
166 #define VAS_LDATA_STAMP_CTL_OFFSET	0x0A8
167 #define VAS_LDATA_STAMP			PPC_BITMASK(0, 1)
168 #define VAS_XTRA_WRITE			PPC_BIT(2)
169 
170 #define VAS_LDMA_CACHE_CTL_OFFSET	0x0B0
171 #define VAS_LDMA_TYPE			PPC_BITMASK(0, 1)
172 #define VAS_LDMA_FIFO_DISABLE		PPC_BIT(2)
173 
174 #define VAS_LRFIFO_PUSH_OFFSET		0x0B8
175 #define VAS_LRFIFO_PUSH			PPC_BITMASK(0, 15)
176 
177 #define VAS_CURR_MSG_COUNT_OFFSET	0x0C0
178 #define VAS_CURR_MSG_COUNT		PPC_BITMASK(0, 7)
179 
180 #define VAS_LNOTIFY_AFTER_COUNT_OFFSET	0x0C8
181 #define VAS_LNOTIFY_AFTER_COUNT		PPC_BITMASK(0, 7)
182 
183 #define VAS_LRX_WCRED_OFFSET		0x0E0
184 #define VAS_LRX_WCRED			PPC_BITMASK(0, 15)
185 
186 #define VAS_LRX_WCRED_ADDER_OFFSET	0x190
187 #define VAS_LRX_WCRED_ADDER		PPC_BITMASK(0, 15)
188 
189 #define VAS_TX_WCRED_OFFSET		0x0F0
190 #define VAS_TX_WCRED			PPC_BITMASK(4, 15)
191 
192 #define VAS_TX_WCRED_ADDER_OFFSET	0x1A0
193 #define VAS_TX_WCRED_ADDER		PPC_BITMASK(4, 15)
194 
195 #define VAS_LFIFO_SIZE_OFFSET		0x100
196 #define VAS_LFIFO_SIZE			PPC_BITMASK(0, 3)
197 
198 #define VAS_WINCTL_OFFSET		0x108
199 #define VAS_WINCTL_OPEN			PPC_BIT(0)
200 #define VAS_WINCTL_REJ_NO_CREDIT	PPC_BIT(1)
201 #define VAS_WINCTL_PIN			PPC_BIT(2)
202 #define VAS_WINCTL_TX_WCRED_MODE	PPC_BIT(3)
203 #define VAS_WINCTL_RX_WCRED_MODE	PPC_BIT(4)
204 #define VAS_WINCTL_TX_WORD_MODE		PPC_BIT(5)
205 #define VAS_WINCTL_RX_WORD_MODE		PPC_BIT(6)
206 #define VAS_WINCTL_RSVD_TXBUF		PPC_BIT(7)
207 #define VAS_WINCTL_THRESH_CTL		PPC_BITMASK(8, 9)
208 #define VAS_WINCTL_FAULT_WIN		PPC_BIT(10)
209 #define VAS_WINCTL_NX_WIN		PPC_BIT(11)
210 
211 #define VAS_WIN_STATUS_OFFSET		0x110
212 #define VAS_WIN_BUSY			PPC_BIT(1)
213 
214 #define VAS_WIN_CTX_CACHING_CTL_OFFSET	0x118
215 #define VAS_CASTOUT_REQ			PPC_BIT(0)
216 #define VAS_PUSH_TO_MEM			PPC_BIT(1)
217 #define VAS_WIN_CACHE_STATUS		PPC_BIT(4)
218 
219 #define VAS_TX_RSVD_BUF_COUNT_OFFSET	0x120
220 #define VAS_RXVD_BUF_COUNT		PPC_BITMASK(58, 63)
221 
222 #define VAS_LRFIFO_WIN_PTR_OFFSET	0x128
223 #define VAS_LRX_WIN_ID			PPC_BITMASK(0, 15)
224 
225 /*
226  * Local Notification Control Register controls what happens in _response_
227  * to a paste command and hence applies only to receive windows.
228  */
229 #define VAS_LNOTIFY_CTL_OFFSET		0x138
230 #define VAS_NOTIFY_DISABLE		PPC_BIT(0)
231 #define VAS_INTR_DISABLE		PPC_BIT(1)
232 #define VAS_NOTIFY_EARLY		PPC_BIT(2)
233 #define VAS_NOTIFY_OSU_INTR		PPC_BIT(3)
234 
235 #define VAS_LNOTIFY_PID_OFFSET		0x140
236 #define VAS_LNOTIFY_PID			PPC_BITMASK(0, 19)
237 
238 #define VAS_LNOTIFY_LPID_OFFSET		0x148
239 #define VAS_LNOTIFY_LPID		PPC_BITMASK(0, 11)
240 
241 #define VAS_LNOTIFY_TID_OFFSET		0x150
242 #define VAS_LNOTIFY_TID			PPC_BITMASK(0, 15)
243 
244 #define VAS_LNOTIFY_SCOPE_OFFSET	0x158
245 #define VAS_LNOTIFY_MIN_SCOPE		PPC_BITMASK(0, 1)
246 #define VAS_LNOTIFY_MAX_SCOPE		PPC_BITMASK(2, 3)
247 
248 #define VAS_NX_UTIL_OFFSET		0x1B0
249 #define VAS_NX_UTIL			PPC_BITMASK(0, 63)
250 
251 /* SE: Side effects */
252 #define VAS_NX_UTIL_SE_OFFSET		0x1B8
253 #define VAS_NX_UTIL_SE			PPC_BITMASK(0, 63)
254 
255 #define VAS_NX_UTIL_ADDER_OFFSET	0x180
256 #define VAS_NX_UTIL_ADDER		PPC_BITMASK(32, 63)
257 
258 /*
259  * VREG(x):
260  * Expand a register's short name (eg: LPID) into two parameters:
261  *	- the register's short name in string form ("LPID"), and
262  *	- the name of the macro (eg: VAS_LPID_OFFSET), defining the
263  *	  register's offset in the window context
264  */
265 #define VREG_SFX(n, s)	__stringify(n), VAS_##n##s
266 #define VREG(r)		VREG_SFX(r, _OFFSET)
267 
268 /*
269  * Local Notify Scope Control Register. (Receive windows only).
270  */
271 enum vas_notify_scope {
272 	VAS_SCOPE_LOCAL,
273 	VAS_SCOPE_GROUP,
274 	VAS_SCOPE_VECTORED_GROUP,
275 	VAS_SCOPE_UNUSED,
276 };
277 
278 /*
279  * Local DMA Cache Control Register (Receive windows only).
280  */
281 enum vas_dma_type {
282 	VAS_DMA_TYPE_INJECT,
283 	VAS_DMA_TYPE_WRITE,
284 };
285 
286 /*
287  * Local Notify Scope Control Register. (Receive windows only).
288  * Not applicable to NX receive windows.
289  */
290 enum vas_notify_after_count {
291 	VAS_NOTIFY_AFTER_256 = 0,
292 	VAS_NOTIFY_NONE,
293 	VAS_NOTIFY_AFTER_2
294 };
295 
296 /*
297  * NX can generate an interrupt for multiple faults and expects kernel
298  * to process all of them. So read all valid CRB entries until find the
299  * invalid one. So use pswid which is pasted by NX and ccw[0] (reserved
300  * bit in BE) to check valid CRB. CCW[0] will not be touched by user
301  * space. Application gets CRB formt error if it updates this bit.
302  *
303  * Invalidate FIFO during allocation and process all entries from last
304  * successful read until finds invalid pswid and ccw[0] values.
305  * After reading each CRB entry from fault FIFO, the kernel invalidate
306  * it by updating pswid with FIFO_INVALID_ENTRY and CCW[0] with
307  * CCW0_INVALID.
308  */
309 #define FIFO_INVALID_ENTRY	0xffffffff
310 #define CCW0_INVALID		1
311 
312 /*
313  * One per instance of VAS. Each instance will have a separate set of
314  * receive windows, one per coprocessor type.
315  *
316  * See also function header of set_vinst_win() for details on ->windows[]
317  * and ->rxwin[] tables.
318  */
319 struct vas_instance {
320 	int vas_id;
321 	struct ida ida;
322 	struct list_head node;
323 	struct platform_device *pdev;
324 
325 	u64 hvwc_bar_start;
326 	u64 uwc_bar_start;
327 	u64 paste_base_addr;
328 	u64 paste_win_id_shift;
329 
330 	u64 irq_port;
331 	int virq;
332 	int fault_crbs;
333 	int fault_fifo_size;
334 	int fifo_in_progress;	/* To wake up thread or return IRQ_HANDLED */
335 	spinlock_t fault_lock;	/* Protects fifo_in_progress update */
336 	void *fault_fifo;
337 	struct vas_window *fault_win; /* Fault window */
338 
339 	struct mutex mutex;
340 	struct vas_window *rxwin[VAS_COP_TYPE_MAX];
341 	struct vas_window *windows[VAS_WINDOWS_PER_CHIP];
342 
343 	char *name;
344 	char *dbgname;
345 	struct dentry *dbgdir;
346 };
347 
348 /*
349  * In-kernel state a VAS window. One per window.
350  */
351 struct vas_window {
352 	/* Fields common to send and receive windows */
353 	struct vas_instance *vinst;
354 	int winid;
355 	bool tx_win;		/* True if send window */
356 	bool nx_win;		/* True if NX window */
357 	bool user_win;		/* True if user space window */
358 	void *hvwc_map;		/* HV window context */
359 	void *uwc_map;		/* OS/User window context */
360 	struct pid *pid;	/* Linux process id of owner */
361 	struct pid *tgid;	/* Thread group ID of owner */
362 	struct mm_struct *mm;	/* Linux process mm_struct */
363 	int wcreds_max;		/* Window credits */
364 
365 	char *dbgname;
366 	struct dentry *dbgdir;
367 
368 	/* Fields applicable only to send windows */
369 	void *paste_kaddr;
370 	char *paste_addr_name;
371 	struct vas_window *rxwin;
372 
373 	/* Feilds applicable only to receive windows */
374 	enum vas_cop_type cop;
375 	atomic_t num_txwins;
376 };
377 
378 /*
379  * Container for the hardware state of a window. One per-window.
380  *
381  * A VAS Window context is a 512-byte area in the hardware that contains
382  * a set of 64-bit registers. Individual bit-fields in these registers
383  * determine the configuration/operation of the hardware. struct vas_winctx
384  * is a container for the register fields in the window context.
385  */
386 struct vas_winctx {
387 	void *rx_fifo;
388 	int rx_fifo_size;
389 	int wcreds_max;
390 	int rsvd_txbuf_count;
391 
392 	bool user_win;
393 	bool nx_win;
394 	bool fault_win;
395 	bool rsvd_txbuf_enable;
396 	bool pin_win;
397 	bool rej_no_credit;
398 	bool tx_wcred_mode;
399 	bool rx_wcred_mode;
400 	bool tx_word_mode;
401 	bool rx_word_mode;
402 	bool data_stamp;
403 	bool xtra_write;
404 	bool notify_disable;
405 	bool intr_disable;
406 	bool fifo_disable;
407 	bool notify_early;
408 	bool notify_os_intr_reg;
409 
410 	int lpid;
411 	int pidr;		/* value from SPRN_PID, not linux pid */
412 	int lnotify_lpid;
413 	int lnotify_pid;
414 	int lnotify_tid;
415 	u32 pswid;
416 	int rx_win_id;
417 	int fault_win_id;
418 	int tc_mode;
419 
420 	u64 irq_port;
421 
422 	enum vas_dma_type dma_type;
423 	enum vas_notify_scope min_scope;
424 	enum vas_notify_scope max_scope;
425 	enum vas_notify_after_count notify_after_count;
426 };
427 
428 extern struct mutex vas_mutex;
429 
430 extern struct vas_instance *find_vas_instance(int vasid);
431 extern void vas_init_dbgdir(void);
432 extern void vas_instance_init_dbgdir(struct vas_instance *vinst);
433 extern void vas_window_init_dbgdir(struct vas_window *win);
434 extern void vas_window_free_dbgdir(struct vas_window *win);
435 extern int vas_setup_fault_window(struct vas_instance *vinst);
436 extern irqreturn_t vas_fault_thread_fn(int irq, void *data);
437 extern irqreturn_t vas_fault_handler(int irq, void *dev_id);
438 extern void vas_return_credit(struct vas_window *window, bool tx);
439 extern struct vas_window *vas_pswid_to_window(struct vas_instance *vinst,
440 						uint32_t pswid);
441 extern void vas_win_paste_addr(struct vas_window *window, u64 *addr,
442 					int *len);
443 
444 static inline int vas_window_pid(struct vas_window *window)
445 {
446 	return pid_vnr(window->pid);
447 }
448 
449 static inline void vas_log_write(struct vas_window *win, char *name,
450 			void *regptr, u64 val)
451 {
452 	if (val)
453 		pr_debug("%swin #%d: %s reg %p, val 0x%016llx\n",
454 				win->tx_win ? "Tx" : "Rx", win->winid, name,
455 				regptr, val);
456 }
457 
458 static inline void write_uwc_reg(struct vas_window *win, char *name,
459 			s32 reg, u64 val)
460 {
461 	void *regptr;
462 
463 	regptr = win->uwc_map + reg;
464 	vas_log_write(win, name, regptr, val);
465 
466 	out_be64(regptr, val);
467 }
468 
469 static inline void write_hvwc_reg(struct vas_window *win, char *name,
470 			s32 reg, u64 val)
471 {
472 	void *regptr;
473 
474 	regptr = win->hvwc_map + reg;
475 	vas_log_write(win, name, regptr, val);
476 
477 	out_be64(regptr, val);
478 }
479 
480 static inline u64 read_hvwc_reg(struct vas_window *win,
481 			char *name __maybe_unused, s32 reg)
482 {
483 	return in_be64(win->hvwc_map+reg);
484 }
485 
486 /*
487  * Encode/decode the Partition Send Window ID (PSWID) for a window in
488  * a way that we can uniquely identify any window in the system. i.e.
489  * we should be able to locate the 'struct vas_window' given the PSWID.
490  *
491  *	Bits	Usage
492  *	0:7	VAS id (8 bits)
493  *	8:15	Unused, 0 (3 bits)
494  *	16:31	Window id (16 bits)
495  */
496 static inline u32 encode_pswid(int vasid, int winid)
497 {
498 	return ((u32)winid | (vasid << (31 - 7)));
499 }
500 
501 static inline void decode_pswid(u32 pswid, int *vasid, int *winid)
502 {
503 	if (vasid)
504 		*vasid = pswid >> (31 - 7) & 0xFF;
505 
506 	if (winid)
507 		*winid = pswid & 0xFFFF;
508 }
509 #endif /* _VAS_H */
510