1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright 2016-17 IBM Corp.
4  */
5 
6 #define pr_fmt(fmt) "vas: " fmt
7 
8 #include <linux/types.h>
9 #include <linux/mutex.h>
10 #include <linux/slab.h>
11 #include <linux/io.h>
12 #include <linux/log2.h>
13 #include <linux/rcupdate.h>
14 #include <linux/cred.h>
15 #include <linux/sched/mm.h>
16 #include <linux/mmu_context.h>
17 #include <asm/switch_to.h>
18 #include <asm/ppc-opcode.h>
19 #include "vas.h"
20 #include "copy-paste.h"
21 
22 #define CREATE_TRACE_POINTS
23 #include "vas-trace.h"
24 
25 /*
26  * Compute the paste address region for the window @window using the
27  * ->paste_base_addr and ->paste_win_id_shift we got from device tree.
28  */
29 void vas_win_paste_addr(struct vas_window *window, u64 *addr, int *len)
30 {
31 	int winid;
32 	u64 base, shift;
33 
34 	base = window->vinst->paste_base_addr;
35 	shift = window->vinst->paste_win_id_shift;
36 	winid = window->winid;
37 
38 	*addr  = base + (winid << shift);
39 	if (len)
40 		*len = PAGE_SIZE;
41 
42 	pr_debug("Txwin #%d: Paste addr 0x%llx\n", winid, *addr);
43 }
44 
45 static inline void get_hvwc_mmio_bar(struct vas_window *window,
46 			u64 *start, int *len)
47 {
48 	u64 pbaddr;
49 
50 	pbaddr = window->vinst->hvwc_bar_start;
51 	*start = pbaddr + window->winid * VAS_HVWC_SIZE;
52 	*len = VAS_HVWC_SIZE;
53 }
54 
55 static inline void get_uwc_mmio_bar(struct vas_window *window,
56 			u64 *start, int *len)
57 {
58 	u64 pbaddr;
59 
60 	pbaddr = window->vinst->uwc_bar_start;
61 	*start = pbaddr + window->winid * VAS_UWC_SIZE;
62 	*len = VAS_UWC_SIZE;
63 }
64 
65 /*
66  * Map the paste bus address of the given send window into kernel address
67  * space. Unlike MMIO regions (map_mmio_region() below), paste region must
68  * be mapped cache-able and is only applicable to send windows.
69  */
70 static void *map_paste_region(struct vas_window *txwin)
71 {
72 	int len;
73 	void *map;
74 	char *name;
75 	u64 start;
76 
77 	name = kasprintf(GFP_KERNEL, "window-v%d-w%d", txwin->vinst->vas_id,
78 				txwin->winid);
79 	if (!name)
80 		goto free_name;
81 
82 	txwin->paste_addr_name = name;
83 	vas_win_paste_addr(txwin, &start, &len);
84 
85 	if (!request_mem_region(start, len, name)) {
86 		pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
87 				__func__, start, len);
88 		goto free_name;
89 	}
90 
91 	map = ioremap_cache(start, len);
92 	if (!map) {
93 		pr_devel("%s(): ioremap_cache(0x%llx, %d) failed\n", __func__,
94 				start, len);
95 		goto free_name;
96 	}
97 
98 	pr_devel("Mapped paste addr 0x%llx to kaddr 0x%p\n", start, map);
99 	return map;
100 
101 free_name:
102 	kfree(name);
103 	return ERR_PTR(-ENOMEM);
104 }
105 
106 static void *map_mmio_region(char *name, u64 start, int len)
107 {
108 	void *map;
109 
110 	if (!request_mem_region(start, len, name)) {
111 		pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
112 				__func__, start, len);
113 		return NULL;
114 	}
115 
116 	map = ioremap(start, len);
117 	if (!map) {
118 		pr_devel("%s(): ioremap(0x%llx, %d) failed\n", __func__, start,
119 				len);
120 		return NULL;
121 	}
122 
123 	return map;
124 }
125 
126 static void unmap_region(void *addr, u64 start, int len)
127 {
128 	iounmap(addr);
129 	release_mem_region((phys_addr_t)start, len);
130 }
131 
132 /*
133  * Unmap the paste address region for a window.
134  */
135 static void unmap_paste_region(struct vas_window *window)
136 {
137 	int len;
138 	u64 busaddr_start;
139 
140 	if (window->paste_kaddr) {
141 		vas_win_paste_addr(window, &busaddr_start, &len);
142 		unmap_region(window->paste_kaddr, busaddr_start, len);
143 		window->paste_kaddr = NULL;
144 		kfree(window->paste_addr_name);
145 		window->paste_addr_name = NULL;
146 	}
147 }
148 
149 /*
150  * Unmap the MMIO regions for a window. Hold the vas_mutex so we don't
151  * unmap when the window's debugfs dir is in use. This serializes close
152  * of a window even on another VAS instance but since its not a critical
153  * path, just minimize the time we hold the mutex for now. We can add
154  * a per-instance mutex later if necessary.
155  */
156 static void unmap_winctx_mmio_bars(struct vas_window *window)
157 {
158 	int len;
159 	void *uwc_map;
160 	void *hvwc_map;
161 	u64 busaddr_start;
162 
163 	mutex_lock(&vas_mutex);
164 
165 	hvwc_map = window->hvwc_map;
166 	window->hvwc_map = NULL;
167 
168 	uwc_map = window->uwc_map;
169 	window->uwc_map = NULL;
170 
171 	mutex_unlock(&vas_mutex);
172 
173 	if (hvwc_map) {
174 		get_hvwc_mmio_bar(window, &busaddr_start, &len);
175 		unmap_region(hvwc_map, busaddr_start, len);
176 	}
177 
178 	if (uwc_map) {
179 		get_uwc_mmio_bar(window, &busaddr_start, &len);
180 		unmap_region(uwc_map, busaddr_start, len);
181 	}
182 }
183 
184 /*
185  * Find the Hypervisor Window Context (HVWC) MMIO Base Address Region and the
186  * OS/User Window Context (UWC) MMIO Base Address Region for the given window.
187  * Map these bus addresses and save the mapped kernel addresses in @window.
188  */
189 static int map_winctx_mmio_bars(struct vas_window *window)
190 {
191 	int len;
192 	u64 start;
193 
194 	get_hvwc_mmio_bar(window, &start, &len);
195 	window->hvwc_map = map_mmio_region("HVWCM_Window", start, len);
196 
197 	get_uwc_mmio_bar(window, &start, &len);
198 	window->uwc_map = map_mmio_region("UWCM_Window", start, len);
199 
200 	if (!window->hvwc_map || !window->uwc_map) {
201 		unmap_winctx_mmio_bars(window);
202 		return -1;
203 	}
204 
205 	return 0;
206 }
207 
208 /*
209  * Reset all valid registers in the HV and OS/User Window Contexts for
210  * the window identified by @window.
211  *
212  * NOTE: We cannot really use a for loop to reset window context. Not all
213  *	 offsets in a window context are valid registers and the valid
214  *	 registers are not sequential. And, we can only write to offsets
215  *	 with valid registers.
216  */
217 static void reset_window_regs(struct vas_window *window)
218 {
219 	write_hvwc_reg(window, VREG(LPID), 0ULL);
220 	write_hvwc_reg(window, VREG(PID), 0ULL);
221 	write_hvwc_reg(window, VREG(XLATE_MSR), 0ULL);
222 	write_hvwc_reg(window, VREG(XLATE_LPCR), 0ULL);
223 	write_hvwc_reg(window, VREG(XLATE_CTL), 0ULL);
224 	write_hvwc_reg(window, VREG(AMR), 0ULL);
225 	write_hvwc_reg(window, VREG(SEIDR), 0ULL);
226 	write_hvwc_reg(window, VREG(FAULT_TX_WIN), 0ULL);
227 	write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL);
228 	write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), 0ULL);
229 	write_hvwc_reg(window, VREG(PSWID), 0ULL);
230 	write_hvwc_reg(window, VREG(LFIFO_BAR), 0ULL);
231 	write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), 0ULL);
232 	write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), 0ULL);
233 	write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL);
234 	write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL);
235 	write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL);
236 	write_hvwc_reg(window, VREG(LRX_WCRED), 0ULL);
237 	write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
238 	write_hvwc_reg(window, VREG(TX_WCRED), 0ULL);
239 	write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
240 	write_hvwc_reg(window, VREG(LFIFO_SIZE), 0ULL);
241 	write_hvwc_reg(window, VREG(WINCTL), 0ULL);
242 	write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL);
243 	write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), 0ULL);
244 	write_hvwc_reg(window, VREG(TX_RSVD_BUF_COUNT), 0ULL);
245 	write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), 0ULL);
246 	write_hvwc_reg(window, VREG(LNOTIFY_CTL), 0ULL);
247 	write_hvwc_reg(window, VREG(LNOTIFY_PID), 0ULL);
248 	write_hvwc_reg(window, VREG(LNOTIFY_LPID), 0ULL);
249 	write_hvwc_reg(window, VREG(LNOTIFY_TID), 0ULL);
250 	write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), 0ULL);
251 	write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL);
252 
253 	/* Skip read-only registers: NX_UTIL and NX_UTIL_SE */
254 
255 	/*
256 	 * The send and receive window credit adder registers are also
257 	 * accessible from HVWC and have been initialized above. We don't
258 	 * need to initialize from the OS/User Window Context, so skip
259 	 * following calls:
260 	 *
261 	 *	write_uwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
262 	 *	write_uwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
263 	 */
264 }
265 
266 /*
267  * Initialize window context registers related to Address Translation.
268  * These registers are common to send/receive windows although they
269  * differ for user/kernel windows. As we resolve the TODOs we may
270  * want to add fields to vas_winctx and move the initialization to
271  * init_vas_winctx_regs().
272  */
273 static void init_xlate_regs(struct vas_window *window, bool user_win)
274 {
275 	u64 lpcr, val;
276 
277 	/*
278 	 * MSR_TA, MSR_US are false for both kernel and user.
279 	 * MSR_DR and MSR_PR are false for kernel.
280 	 */
281 	val = 0ULL;
282 	val = SET_FIELD(VAS_XLATE_MSR_HV, val, 1);
283 	val = SET_FIELD(VAS_XLATE_MSR_SF, val, 1);
284 	if (user_win) {
285 		val = SET_FIELD(VAS_XLATE_MSR_DR, val, 1);
286 		val = SET_FIELD(VAS_XLATE_MSR_PR, val, 1);
287 	}
288 	write_hvwc_reg(window, VREG(XLATE_MSR), val);
289 
290 	lpcr = mfspr(SPRN_LPCR);
291 	val = 0ULL;
292 	/*
293 	 * NOTE: From Section 5.7.8.1 Segment Lookaside Buffer of the
294 	 *	 Power ISA, v3.0B, Page size encoding is 0 = 4KB, 5 = 64KB.
295 	 *
296 	 * NOTE: From Section 1.3.1, Address Translation Context of the
297 	 *	 Nest MMU Workbook, LPCR_SC should be 0 for Power9.
298 	 */
299 	val = SET_FIELD(VAS_XLATE_LPCR_PAGE_SIZE, val, 5);
300 	val = SET_FIELD(VAS_XLATE_LPCR_ISL, val, lpcr & LPCR_ISL);
301 	val = SET_FIELD(VAS_XLATE_LPCR_TC, val, lpcr & LPCR_TC);
302 	val = SET_FIELD(VAS_XLATE_LPCR_SC, val, 0);
303 	write_hvwc_reg(window, VREG(XLATE_LPCR), val);
304 
305 	/*
306 	 * Section 1.3.1 (Address translation Context) of NMMU workbook.
307 	 *	0b00	Hashed Page Table mode
308 	 *	0b01	Reserved
309 	 *	0b10	Radix on HPT
310 	 *	0b11	Radix on Radix
311 	 */
312 	val = 0ULL;
313 	val = SET_FIELD(VAS_XLATE_MODE, val, radix_enabled() ? 3 : 2);
314 	write_hvwc_reg(window, VREG(XLATE_CTL), val);
315 
316 	/*
317 	 * TODO: Can we mfspr(AMR) even for user windows?
318 	 */
319 	val = 0ULL;
320 	val = SET_FIELD(VAS_AMR, val, mfspr(SPRN_AMR));
321 	write_hvwc_reg(window, VREG(AMR), val);
322 
323 	val = 0ULL;
324 	val = SET_FIELD(VAS_SEIDR, val, 0);
325 	write_hvwc_reg(window, VREG(SEIDR), val);
326 }
327 
328 /*
329  * Initialize Reserved Send Buffer Count for the send window. It involves
330  * writing to the register, reading it back to confirm that the hardware
331  * has enough buffers to reserve. See section 1.3.1.2.1 of VAS workbook.
332  *
333  * Since we can only make a best-effort attempt to fulfill the request,
334  * we don't return any errors if we cannot.
335  *
336  * TODO: Reserved (aka dedicated) send buffers are not supported yet.
337  */
338 static void init_rsvd_tx_buf_count(struct vas_window *txwin,
339 				struct vas_winctx *winctx)
340 {
341 	write_hvwc_reg(txwin, VREG(TX_RSVD_BUF_COUNT), 0ULL);
342 }
343 
344 /*
345  * init_winctx_regs()
346  *	Initialize window context registers for a receive window.
347  *	Except for caching control and marking window open, the registers
348  *	are initialized in the order listed in Section 3.1.4 (Window Context
349  *	Cache Register Details) of the VAS workbook although they don't need
350  *	to be.
351  *
352  * Design note: For NX receive windows, NX allocates the FIFO buffer in OPAL
353  *	(so that it can get a large contiguous area) and passes that buffer
354  *	to kernel via device tree. We now write that buffer address to the
355  *	FIFO BAR. Would it make sense to do this all in OPAL? i.e have OPAL
356  *	write the per-chip RX FIFO addresses to the windows during boot-up
357  *	as a one-time task? That could work for NX but what about other
358  *	receivers?  Let the receivers tell us the rx-fifo buffers for now.
359  */
360 static void init_winctx_regs(struct vas_window *window,
361 			     struct vas_winctx *winctx)
362 {
363 	u64 val;
364 	int fifo_size;
365 
366 	reset_window_regs(window);
367 
368 	val = 0ULL;
369 	val = SET_FIELD(VAS_LPID, val, winctx->lpid);
370 	write_hvwc_reg(window, VREG(LPID), val);
371 
372 	val = 0ULL;
373 	val = SET_FIELD(VAS_PID_ID, val, winctx->pidr);
374 	write_hvwc_reg(window, VREG(PID), val);
375 
376 	init_xlate_regs(window, winctx->user_win);
377 
378 	val = 0ULL;
379 	val = SET_FIELD(VAS_FAULT_TX_WIN, val, winctx->fault_win_id);
380 	write_hvwc_reg(window, VREG(FAULT_TX_WIN), val);
381 
382 	/* In PowerNV, interrupts go to HV. */
383 	write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL);
384 
385 	val = 0ULL;
386 	val = SET_FIELD(VAS_HV_INTR_SRC_RA, val, winctx->irq_port);
387 	write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), val);
388 
389 	val = 0ULL;
390 	val = SET_FIELD(VAS_PSWID_EA_HANDLE, val, winctx->pswid);
391 	write_hvwc_reg(window, VREG(PSWID), val);
392 
393 	write_hvwc_reg(window, VREG(SPARE1), 0ULL);
394 	write_hvwc_reg(window, VREG(SPARE2), 0ULL);
395 	write_hvwc_reg(window, VREG(SPARE3), 0ULL);
396 
397 	/*
398 	 * NOTE: VAS expects the FIFO address to be copied into the LFIFO_BAR
399 	 *	 register as is - do NOT shift the address into VAS_LFIFO_BAR
400 	 *	 bit fields! Ok to set the page migration select fields -
401 	 *	 VAS ignores the lower 10+ bits in the address anyway, because
402 	 *	 the minimum FIFO size is 1K?
403 	 *
404 	 * See also: Design note in function header.
405 	 */
406 	val = __pa(winctx->rx_fifo);
407 	val = SET_FIELD(VAS_PAGE_MIGRATION_SELECT, val, 0);
408 	write_hvwc_reg(window, VREG(LFIFO_BAR), val);
409 
410 	val = 0ULL;
411 	val = SET_FIELD(VAS_LDATA_STAMP, val, winctx->data_stamp);
412 	write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), val);
413 
414 	val = 0ULL;
415 	val = SET_FIELD(VAS_LDMA_TYPE, val, winctx->dma_type);
416 	val = SET_FIELD(VAS_LDMA_FIFO_DISABLE, val, winctx->fifo_disable);
417 	write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), val);
418 
419 	write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL);
420 	write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL);
421 	write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL);
422 
423 	val = 0ULL;
424 	val = SET_FIELD(VAS_LRX_WCRED, val, winctx->wcreds_max);
425 	write_hvwc_reg(window, VREG(LRX_WCRED), val);
426 
427 	val = 0ULL;
428 	val = SET_FIELD(VAS_TX_WCRED, val, winctx->wcreds_max);
429 	write_hvwc_reg(window, VREG(TX_WCRED), val);
430 
431 	write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
432 	write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
433 
434 	fifo_size = winctx->rx_fifo_size / 1024;
435 
436 	val = 0ULL;
437 	val = SET_FIELD(VAS_LFIFO_SIZE, val, ilog2(fifo_size));
438 	write_hvwc_reg(window, VREG(LFIFO_SIZE), val);
439 
440 	/* Update window control and caching control registers last so
441 	 * we mark the window open only after fully initializing it and
442 	 * pushing context to cache.
443 	 */
444 
445 	write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL);
446 
447 	init_rsvd_tx_buf_count(window, winctx);
448 
449 	/* for a send window, point to the matching receive window */
450 	val = 0ULL;
451 	val = SET_FIELD(VAS_LRX_WIN_ID, val, winctx->rx_win_id);
452 	write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), val);
453 
454 	write_hvwc_reg(window, VREG(SPARE4), 0ULL);
455 
456 	val = 0ULL;
457 	val = SET_FIELD(VAS_NOTIFY_DISABLE, val, winctx->notify_disable);
458 	val = SET_FIELD(VAS_INTR_DISABLE, val, winctx->intr_disable);
459 	val = SET_FIELD(VAS_NOTIFY_EARLY, val, winctx->notify_early);
460 	val = SET_FIELD(VAS_NOTIFY_OSU_INTR, val, winctx->notify_os_intr_reg);
461 	write_hvwc_reg(window, VREG(LNOTIFY_CTL), val);
462 
463 	val = 0ULL;
464 	val = SET_FIELD(VAS_LNOTIFY_PID, val, winctx->lnotify_pid);
465 	write_hvwc_reg(window, VREG(LNOTIFY_PID), val);
466 
467 	val = 0ULL;
468 	val = SET_FIELD(VAS_LNOTIFY_LPID, val, winctx->lnotify_lpid);
469 	write_hvwc_reg(window, VREG(LNOTIFY_LPID), val);
470 
471 	val = 0ULL;
472 	val = SET_FIELD(VAS_LNOTIFY_TID, val, winctx->lnotify_tid);
473 	write_hvwc_reg(window, VREG(LNOTIFY_TID), val);
474 
475 	val = 0ULL;
476 	val = SET_FIELD(VAS_LNOTIFY_MIN_SCOPE, val, winctx->min_scope);
477 	val = SET_FIELD(VAS_LNOTIFY_MAX_SCOPE, val, winctx->max_scope);
478 	write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), val);
479 
480 	/* Skip read-only registers NX_UTIL and NX_UTIL_SE */
481 
482 	write_hvwc_reg(window, VREG(SPARE5), 0ULL);
483 	write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL);
484 	write_hvwc_reg(window, VREG(SPARE6), 0ULL);
485 
486 	/* Finally, push window context to memory and... */
487 	val = 0ULL;
488 	val = SET_FIELD(VAS_PUSH_TO_MEM, val, 1);
489 	write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), val);
490 
491 	/* ... mark the window open for business */
492 	val = 0ULL;
493 	val = SET_FIELD(VAS_WINCTL_REJ_NO_CREDIT, val, winctx->rej_no_credit);
494 	val = SET_FIELD(VAS_WINCTL_PIN, val, winctx->pin_win);
495 	val = SET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val, winctx->tx_wcred_mode);
496 	val = SET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val, winctx->rx_wcred_mode);
497 	val = SET_FIELD(VAS_WINCTL_TX_WORD_MODE, val, winctx->tx_word_mode);
498 	val = SET_FIELD(VAS_WINCTL_RX_WORD_MODE, val, winctx->rx_word_mode);
499 	val = SET_FIELD(VAS_WINCTL_FAULT_WIN, val, winctx->fault_win);
500 	val = SET_FIELD(VAS_WINCTL_NX_WIN, val, winctx->nx_win);
501 	val = SET_FIELD(VAS_WINCTL_OPEN, val, 1);
502 	write_hvwc_reg(window, VREG(WINCTL), val);
503 }
504 
505 static void vas_release_window_id(struct ida *ida, int winid)
506 {
507 	ida_free(ida, winid);
508 }
509 
510 static int vas_assign_window_id(struct ida *ida)
511 {
512 	int winid = ida_alloc_max(ida, VAS_WINDOWS_PER_CHIP - 1, GFP_KERNEL);
513 
514 	if (winid == -ENOSPC) {
515 		pr_err("Too many (%d) open windows\n", VAS_WINDOWS_PER_CHIP);
516 		return -EAGAIN;
517 	}
518 
519 	return winid;
520 }
521 
522 static void vas_window_free(struct vas_window *window)
523 {
524 	int winid = window->winid;
525 	struct vas_instance *vinst = window->vinst;
526 
527 	unmap_winctx_mmio_bars(window);
528 
529 	vas_window_free_dbgdir(window);
530 
531 	kfree(window);
532 
533 	vas_release_window_id(&vinst->ida, winid);
534 }
535 
536 static struct vas_window *vas_window_alloc(struct vas_instance *vinst)
537 {
538 	int winid;
539 	struct vas_window *window;
540 
541 	winid = vas_assign_window_id(&vinst->ida);
542 	if (winid < 0)
543 		return ERR_PTR(winid);
544 
545 	window = kzalloc(sizeof(*window), GFP_KERNEL);
546 	if (!window)
547 		goto out_free;
548 
549 	window->vinst = vinst;
550 	window->winid = winid;
551 
552 	if (map_winctx_mmio_bars(window))
553 		goto out_free;
554 
555 	vas_window_init_dbgdir(window);
556 
557 	return window;
558 
559 out_free:
560 	kfree(window);
561 	vas_release_window_id(&vinst->ida, winid);
562 	return ERR_PTR(-ENOMEM);
563 }
564 
565 static void put_rx_win(struct vas_window *rxwin)
566 {
567 	/* Better not be a send window! */
568 	WARN_ON_ONCE(rxwin->tx_win);
569 
570 	atomic_dec(&rxwin->num_txwins);
571 }
572 
573 /*
574  * Find the user space receive window given the @pswid.
575  *      - We must have a valid vasid and it must belong to this instance.
576  *        (so both send and receive windows are on the same VAS instance)
577  *      - The window must refer to an OPEN, FTW, RECEIVE window.
578  *
579  * NOTE: We access ->windows[] table and assume that vinst->mutex is held.
580  */
581 static struct vas_window *get_user_rxwin(struct vas_instance *vinst, u32 pswid)
582 {
583 	int vasid, winid;
584 	struct vas_window *rxwin;
585 
586 	decode_pswid(pswid, &vasid, &winid);
587 
588 	if (vinst->vas_id != vasid)
589 		return ERR_PTR(-EINVAL);
590 
591 	rxwin = vinst->windows[winid];
592 
593 	if (!rxwin || rxwin->tx_win || rxwin->cop != VAS_COP_TYPE_FTW)
594 		return ERR_PTR(-EINVAL);
595 
596 	return rxwin;
597 }
598 
599 /*
600  * Get the VAS receive window associated with NX engine identified
601  * by @cop and if applicable, @pswid.
602  *
603  * See also function header of set_vinst_win().
604  */
605 static struct vas_window *get_vinst_rxwin(struct vas_instance *vinst,
606 			enum vas_cop_type cop, u32 pswid)
607 {
608 	struct vas_window *rxwin;
609 
610 	mutex_lock(&vinst->mutex);
611 
612 	if (cop == VAS_COP_TYPE_FTW)
613 		rxwin = get_user_rxwin(vinst, pswid);
614 	else
615 		rxwin = vinst->rxwin[cop] ?: ERR_PTR(-EINVAL);
616 
617 	if (!IS_ERR(rxwin))
618 		atomic_inc(&rxwin->num_txwins);
619 
620 	mutex_unlock(&vinst->mutex);
621 
622 	return rxwin;
623 }
624 
625 /*
626  * We have two tables of windows in a VAS instance. The first one,
627  * ->windows[], contains all the windows in the instance and allows
628  * looking up a window by its id. It is used to look up send windows
629  * during fault handling and receive windows when pairing user space
630  * send/receive windows.
631  *
632  * The second table, ->rxwin[], contains receive windows that are
633  * associated with NX engines. This table has VAS_COP_TYPE_MAX
634  * entries and is used to look up a receive window by its
635  * coprocessor type.
636  *
637  * Here, we save @window in the ->windows[] table. If it is a receive
638  * window, we also save the window in the ->rxwin[] table.
639  */
640 static void set_vinst_win(struct vas_instance *vinst,
641 			struct vas_window *window)
642 {
643 	int id = window->winid;
644 
645 	mutex_lock(&vinst->mutex);
646 
647 	/*
648 	 * There should only be one receive window for a coprocessor type
649 	 * unless its a user (FTW) window.
650 	 */
651 	if (!window->user_win && !window->tx_win) {
652 		WARN_ON_ONCE(vinst->rxwin[window->cop]);
653 		vinst->rxwin[window->cop] = window;
654 	}
655 
656 	WARN_ON_ONCE(vinst->windows[id] != NULL);
657 	vinst->windows[id] = window;
658 
659 	mutex_unlock(&vinst->mutex);
660 }
661 
662 /*
663  * Clear this window from the table(s) of windows for this VAS instance.
664  * See also function header of set_vinst_win().
665  */
666 static void clear_vinst_win(struct vas_window *window)
667 {
668 	int id = window->winid;
669 	struct vas_instance *vinst = window->vinst;
670 
671 	mutex_lock(&vinst->mutex);
672 
673 	if (!window->user_win && !window->tx_win) {
674 		WARN_ON_ONCE(!vinst->rxwin[window->cop]);
675 		vinst->rxwin[window->cop] = NULL;
676 	}
677 
678 	WARN_ON_ONCE(vinst->windows[id] != window);
679 	vinst->windows[id] = NULL;
680 
681 	mutex_unlock(&vinst->mutex);
682 }
683 
684 static void init_winctx_for_rxwin(struct vas_window *rxwin,
685 			struct vas_rx_win_attr *rxattr,
686 			struct vas_winctx *winctx)
687 {
688 	/*
689 	 * We first zero (memset()) all fields and only set non-zero fields.
690 	 * Following fields are 0/false but maybe deserve a comment:
691 	 *
692 	 *	->notify_os_intr_reg	In powerNV, send intrs to HV
693 	 *	->notify_disable	False for NX windows
694 	 *	->intr_disable		False for Fault Windows
695 	 *	->xtra_write		False for NX windows
696 	 *	->notify_early		NA for NX windows
697 	 *	->rsvd_txbuf_count	NA for Rx windows
698 	 *	->lpid, ->pid, ->tid	NA for Rx windows
699 	 */
700 
701 	memset(winctx, 0, sizeof(struct vas_winctx));
702 
703 	winctx->rx_fifo = rxattr->rx_fifo;
704 	winctx->rx_fifo_size = rxattr->rx_fifo_size;
705 	winctx->wcreds_max = rxwin->wcreds_max;
706 	winctx->pin_win = rxattr->pin_win;
707 
708 	winctx->nx_win = rxattr->nx_win;
709 	winctx->fault_win = rxattr->fault_win;
710 	winctx->user_win = rxattr->user_win;
711 	winctx->rej_no_credit = rxattr->rej_no_credit;
712 	winctx->rx_word_mode = rxattr->rx_win_ord_mode;
713 	winctx->tx_word_mode = rxattr->tx_win_ord_mode;
714 	winctx->rx_wcred_mode = rxattr->rx_wcred_mode;
715 	winctx->tx_wcred_mode = rxattr->tx_wcred_mode;
716 	winctx->notify_early = rxattr->notify_early;
717 
718 	if (winctx->nx_win) {
719 		winctx->data_stamp = true;
720 		winctx->intr_disable = true;
721 		winctx->pin_win = true;
722 
723 		WARN_ON_ONCE(winctx->fault_win);
724 		WARN_ON_ONCE(!winctx->rx_word_mode);
725 		WARN_ON_ONCE(!winctx->tx_word_mode);
726 		WARN_ON_ONCE(winctx->notify_after_count);
727 	} else if (winctx->fault_win) {
728 		winctx->notify_disable = true;
729 	} else if (winctx->user_win) {
730 		/*
731 		 * Section 1.8.1 Low Latency Core-Core Wake up of
732 		 * the VAS workbook:
733 		 *
734 		 *      - disable credit checks ([tr]x_wcred_mode = false)
735 		 *      - disable FIFO writes
736 		 *      - enable ASB_Notify, disable interrupt
737 		 */
738 		winctx->fifo_disable = true;
739 		winctx->intr_disable = true;
740 		winctx->rx_fifo = NULL;
741 	}
742 
743 	winctx->lnotify_lpid = rxattr->lnotify_lpid;
744 	winctx->lnotify_pid = rxattr->lnotify_pid;
745 	winctx->lnotify_tid = rxattr->lnotify_tid;
746 	winctx->pswid = rxattr->pswid;
747 	winctx->dma_type = VAS_DMA_TYPE_INJECT;
748 	winctx->tc_mode = rxattr->tc_mode;
749 
750 	winctx->min_scope = VAS_SCOPE_LOCAL;
751 	winctx->max_scope = VAS_SCOPE_VECTORED_GROUP;
752 	if (rxwin->vinst->virq)
753 		winctx->irq_port = rxwin->vinst->irq_port;
754 }
755 
756 static bool rx_win_args_valid(enum vas_cop_type cop,
757 			struct vas_rx_win_attr *attr)
758 {
759 	pr_debug("Rxattr: fault %d, notify %d, intr %d, early %d, fifo %d\n",
760 			attr->fault_win, attr->notify_disable,
761 			attr->intr_disable, attr->notify_early,
762 			attr->rx_fifo_size);
763 
764 	if (cop >= VAS_COP_TYPE_MAX)
765 		return false;
766 
767 	if (cop != VAS_COP_TYPE_FTW &&
768 				attr->rx_fifo_size < VAS_RX_FIFO_SIZE_MIN)
769 		return false;
770 
771 	if (attr->rx_fifo_size > VAS_RX_FIFO_SIZE_MAX)
772 		return false;
773 
774 	if (!attr->wcreds_max)
775 		return false;
776 
777 	if (attr->nx_win) {
778 		/* cannot be fault or user window if it is nx */
779 		if (attr->fault_win || attr->user_win)
780 			return false;
781 		/*
782 		 * Section 3.1.4.32: NX Windows must not disable notification,
783 		 *	and must not enable interrupts or early notification.
784 		 */
785 		if (attr->notify_disable || !attr->intr_disable ||
786 				attr->notify_early)
787 			return false;
788 	} else if (attr->fault_win) {
789 		/* cannot be both fault and user window */
790 		if (attr->user_win)
791 			return false;
792 
793 		/*
794 		 * Section 3.1.4.32: Fault windows must disable notification
795 		 *	but not interrupts.
796 		 */
797 		if (!attr->notify_disable || attr->intr_disable)
798 			return false;
799 
800 	} else if (attr->user_win) {
801 		/*
802 		 * User receive windows are only for fast-thread-wakeup
803 		 * (FTW). They don't need a FIFO and must disable interrupts
804 		 */
805 		if (attr->rx_fifo || attr->rx_fifo_size || !attr->intr_disable)
806 			return false;
807 	} else {
808 		/* Rx window must be one of NX or Fault or User window. */
809 		return false;
810 	}
811 
812 	return true;
813 }
814 
815 void vas_init_rx_win_attr(struct vas_rx_win_attr *rxattr, enum vas_cop_type cop)
816 {
817 	memset(rxattr, 0, sizeof(*rxattr));
818 
819 	if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI ||
820 		cop == VAS_COP_TYPE_GZIP || cop == VAS_COP_TYPE_GZIP_HIPRI) {
821 		rxattr->pin_win = true;
822 		rxattr->nx_win = true;
823 		rxattr->fault_win = false;
824 		rxattr->intr_disable = true;
825 		rxattr->rx_wcred_mode = true;
826 		rxattr->tx_wcred_mode = true;
827 		rxattr->rx_win_ord_mode = true;
828 		rxattr->tx_win_ord_mode = true;
829 	} else if (cop == VAS_COP_TYPE_FAULT) {
830 		rxattr->pin_win = true;
831 		rxattr->fault_win = true;
832 		rxattr->notify_disable = true;
833 		rxattr->rx_wcred_mode = true;
834 		rxattr->rx_win_ord_mode = true;
835 		rxattr->rej_no_credit = true;
836 		rxattr->tc_mode = VAS_THRESH_DISABLED;
837 	} else if (cop == VAS_COP_TYPE_FTW) {
838 		rxattr->user_win = true;
839 		rxattr->intr_disable = true;
840 
841 		/*
842 		 * As noted in the VAS Workbook we disable credit checks.
843 		 * If we enable credit checks in the future, we must also
844 		 * implement a mechanism to return the user credits or new
845 		 * paste operations will fail.
846 		 */
847 	}
848 }
849 EXPORT_SYMBOL_GPL(vas_init_rx_win_attr);
850 
851 struct vas_window *vas_rx_win_open(int vasid, enum vas_cop_type cop,
852 			struct vas_rx_win_attr *rxattr)
853 {
854 	struct vas_window *rxwin;
855 	struct vas_winctx winctx;
856 	struct vas_instance *vinst;
857 
858 	trace_vas_rx_win_open(current, vasid, cop, rxattr);
859 
860 	if (!rx_win_args_valid(cop, rxattr))
861 		return ERR_PTR(-EINVAL);
862 
863 	vinst = find_vas_instance(vasid);
864 	if (!vinst) {
865 		pr_devel("vasid %d not found!\n", vasid);
866 		return ERR_PTR(-EINVAL);
867 	}
868 	pr_devel("Found instance %d\n", vasid);
869 
870 	rxwin = vas_window_alloc(vinst);
871 	if (IS_ERR(rxwin)) {
872 		pr_devel("Unable to allocate memory for Rx window\n");
873 		return rxwin;
874 	}
875 
876 	rxwin->tx_win = false;
877 	rxwin->nx_win = rxattr->nx_win;
878 	rxwin->user_win = rxattr->user_win;
879 	rxwin->cop = cop;
880 	rxwin->wcreds_max = rxattr->wcreds_max;
881 
882 	init_winctx_for_rxwin(rxwin, rxattr, &winctx);
883 	init_winctx_regs(rxwin, &winctx);
884 
885 	set_vinst_win(vinst, rxwin);
886 
887 	return rxwin;
888 }
889 EXPORT_SYMBOL_GPL(vas_rx_win_open);
890 
891 void vas_init_tx_win_attr(struct vas_tx_win_attr *txattr, enum vas_cop_type cop)
892 {
893 	memset(txattr, 0, sizeof(*txattr));
894 
895 	if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI ||
896 		cop == VAS_COP_TYPE_GZIP || cop == VAS_COP_TYPE_GZIP_HIPRI) {
897 		txattr->rej_no_credit = false;
898 		txattr->rx_wcred_mode = true;
899 		txattr->tx_wcred_mode = true;
900 		txattr->rx_win_ord_mode = true;
901 		txattr->tx_win_ord_mode = true;
902 	} else if (cop == VAS_COP_TYPE_FTW) {
903 		txattr->user_win = true;
904 	}
905 }
906 EXPORT_SYMBOL_GPL(vas_init_tx_win_attr);
907 
908 static void init_winctx_for_txwin(struct vas_window *txwin,
909 			struct vas_tx_win_attr *txattr,
910 			struct vas_winctx *winctx)
911 {
912 	/*
913 	 * We first zero all fields and only set non-zero ones. Following
914 	 * are some fields set to 0/false for the stated reason:
915 	 *
916 	 *	->notify_os_intr_reg	In powernv, send intrs to HV
917 	 *	->rsvd_txbuf_count	Not supported yet.
918 	 *	->notify_disable	False for NX windows
919 	 *	->xtra_write		False for NX windows
920 	 *	->notify_early		NA for NX windows
921 	 *	->lnotify_lpid		NA for Tx windows
922 	 *	->lnotify_pid		NA for Tx windows
923 	 *	->lnotify_tid		NA for Tx windows
924 	 *	->tx_win_cred_mode	Ignore for now for NX windows
925 	 *	->rx_win_cred_mode	Ignore for now for NX windows
926 	 */
927 	memset(winctx, 0, sizeof(struct vas_winctx));
928 
929 	winctx->wcreds_max = txwin->wcreds_max;
930 
931 	winctx->user_win = txattr->user_win;
932 	winctx->nx_win = txwin->rxwin->nx_win;
933 	winctx->pin_win = txattr->pin_win;
934 	winctx->rej_no_credit = txattr->rej_no_credit;
935 	winctx->rsvd_txbuf_enable = txattr->rsvd_txbuf_enable;
936 
937 	winctx->rx_wcred_mode = txattr->rx_wcred_mode;
938 	winctx->tx_wcred_mode = txattr->tx_wcred_mode;
939 	winctx->rx_word_mode = txattr->rx_win_ord_mode;
940 	winctx->tx_word_mode = txattr->tx_win_ord_mode;
941 	winctx->rsvd_txbuf_count = txattr->rsvd_txbuf_count;
942 
943 	winctx->intr_disable = true;
944 	if (winctx->nx_win)
945 		winctx->data_stamp = true;
946 
947 	winctx->lpid = txattr->lpid;
948 	winctx->pidr = txattr->pidr;
949 	winctx->rx_win_id = txwin->rxwin->winid;
950 	/*
951 	 * IRQ and fault window setup is successful. Set fault window
952 	 * for the send window so that ready to handle faults.
953 	 */
954 	if (txwin->vinst->virq)
955 		winctx->fault_win_id = txwin->vinst->fault_win->winid;
956 
957 	winctx->dma_type = VAS_DMA_TYPE_INJECT;
958 	winctx->tc_mode = txattr->tc_mode;
959 	winctx->min_scope = VAS_SCOPE_LOCAL;
960 	winctx->max_scope = VAS_SCOPE_VECTORED_GROUP;
961 	if (txwin->vinst->virq)
962 		winctx->irq_port = txwin->vinst->irq_port;
963 
964 	winctx->pswid = txattr->pswid ? txattr->pswid :
965 			encode_pswid(txwin->vinst->vas_id, txwin->winid);
966 }
967 
968 static bool tx_win_args_valid(enum vas_cop_type cop,
969 			struct vas_tx_win_attr *attr)
970 {
971 	if (attr->tc_mode != VAS_THRESH_DISABLED)
972 		return false;
973 
974 	if (cop > VAS_COP_TYPE_MAX)
975 		return false;
976 
977 	if (attr->wcreds_max > VAS_TX_WCREDS_MAX)
978 		return false;
979 
980 	if (attr->user_win) {
981 		if (attr->rsvd_txbuf_count)
982 			return false;
983 
984 		if (cop != VAS_COP_TYPE_FTW && cop != VAS_COP_TYPE_GZIP &&
985 			cop != VAS_COP_TYPE_GZIP_HIPRI)
986 			return false;
987 	}
988 
989 	return true;
990 }
991 
992 struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop,
993 			struct vas_tx_win_attr *attr)
994 {
995 	int rc;
996 	struct vas_window *txwin;
997 	struct vas_window *rxwin;
998 	struct vas_winctx winctx;
999 	struct vas_instance *vinst;
1000 
1001 	trace_vas_tx_win_open(current, vasid, cop, attr);
1002 
1003 	if (!tx_win_args_valid(cop, attr))
1004 		return ERR_PTR(-EINVAL);
1005 
1006 	/*
1007 	 * If caller did not specify a vasid but specified the PSWID of a
1008 	 * receive window (applicable only to FTW windows), use the vasid
1009 	 * from that receive window.
1010 	 */
1011 	if (vasid == -1 && attr->pswid)
1012 		decode_pswid(attr->pswid, &vasid, NULL);
1013 
1014 	vinst = find_vas_instance(vasid);
1015 	if (!vinst) {
1016 		pr_devel("vasid %d not found!\n", vasid);
1017 		return ERR_PTR(-EINVAL);
1018 	}
1019 
1020 	rxwin = get_vinst_rxwin(vinst, cop, attr->pswid);
1021 	if (IS_ERR(rxwin)) {
1022 		pr_devel("No RxWin for vasid %d, cop %d\n", vasid, cop);
1023 		return rxwin;
1024 	}
1025 
1026 	txwin = vas_window_alloc(vinst);
1027 	if (IS_ERR(txwin)) {
1028 		rc = PTR_ERR(txwin);
1029 		goto put_rxwin;
1030 	}
1031 
1032 	txwin->cop = cop;
1033 	txwin->tx_win = 1;
1034 	txwin->rxwin = rxwin;
1035 	txwin->nx_win = txwin->rxwin->nx_win;
1036 	txwin->user_win = attr->user_win;
1037 	txwin->wcreds_max = attr->wcreds_max ?: VAS_WCREDS_DEFAULT;
1038 
1039 	init_winctx_for_txwin(txwin, attr, &winctx);
1040 
1041 	init_winctx_regs(txwin, &winctx);
1042 
1043 	/*
1044 	 * If its a kernel send window, map the window address into the
1045 	 * kernel's address space. For user windows, user must issue an
1046 	 * mmap() to map the window into their address space.
1047 	 *
1048 	 * NOTE: If kernel ever resubmits a user CRB after handling a page
1049 	 *	 fault, we will need to map this into kernel as well.
1050 	 */
1051 	if (!txwin->user_win) {
1052 		txwin->paste_kaddr = map_paste_region(txwin);
1053 		if (IS_ERR(txwin->paste_kaddr)) {
1054 			rc = PTR_ERR(txwin->paste_kaddr);
1055 			goto free_window;
1056 		}
1057 	} else {
1058 		/*
1059 		 * Interrupt hanlder or fault window setup failed. Means
1060 		 * NX can not generate fault for page fault. So not
1061 		 * opening for user space tx window.
1062 		 */
1063 		if (!vinst->virq) {
1064 			rc = -ENODEV;
1065 			goto free_window;
1066 		}
1067 
1068 		/*
1069 		 * Window opened by a child thread may not be closed when
1070 		 * it exits. So take reference to its pid and release it
1071 		 * when the window is free by parent thread.
1072 		 * Acquire a reference to the task's pid to make sure
1073 		 * pid will not be re-used - needed only for multithread
1074 		 * applications.
1075 		 */
1076 		txwin->pid = get_task_pid(current, PIDTYPE_PID);
1077 		/*
1078 		 * Acquire a reference to the task's mm.
1079 		 */
1080 		txwin->mm = get_task_mm(current);
1081 
1082 		if (!txwin->mm) {
1083 			put_pid(txwin->pid);
1084 			pr_err("VAS: pid(%d): mm_struct is not found\n",
1085 					current->pid);
1086 			rc = -EPERM;
1087 			goto free_window;
1088 		}
1089 
1090 		mmgrab(txwin->mm);
1091 		mmput(txwin->mm);
1092 		mm_context_add_vas_window(txwin->mm);
1093 		/*
1094 		 * Process closes window during exit. In the case of
1095 		 * multithread application, the child thread can open
1096 		 * window and can exit without closing it. Expects parent
1097 		 * thread to use and close the window. So do not need
1098 		 * to take pid reference for parent thread.
1099 		 */
1100 		txwin->tgid = find_get_pid(task_tgid_vnr(current));
1101 		/*
1102 		 * Even a process that has no foreign real address mapping can
1103 		 * use an unpaired COPY instruction (to no real effect). Issue
1104 		 * CP_ABORT to clear any pending COPY and prevent a covert
1105 		 * channel.
1106 		 *
1107 		 * __switch_to() will issue CP_ABORT on future context switches
1108 		 * if process / thread has any open VAS window (Use
1109 		 * current->mm->context.vas_windows).
1110 		 */
1111 		asm volatile(PPC_CP_ABORT);
1112 	}
1113 
1114 	set_vinst_win(vinst, txwin);
1115 
1116 	return txwin;
1117 
1118 free_window:
1119 	vas_window_free(txwin);
1120 
1121 put_rxwin:
1122 	put_rx_win(rxwin);
1123 	return ERR_PTR(rc);
1124 
1125 }
1126 EXPORT_SYMBOL_GPL(vas_tx_win_open);
1127 
1128 int vas_copy_crb(void *crb, int offset)
1129 {
1130 	return vas_copy(crb, offset);
1131 }
1132 EXPORT_SYMBOL_GPL(vas_copy_crb);
1133 
1134 #define RMA_LSMP_REPORT_ENABLE PPC_BIT(53)
1135 int vas_paste_crb(struct vas_window *txwin, int offset, bool re)
1136 {
1137 	int rc;
1138 	void *addr;
1139 	uint64_t val;
1140 
1141 	trace_vas_paste_crb(current, txwin);
1142 
1143 	/*
1144 	 * Only NX windows are supported for now and hardware assumes
1145 	 * report-enable flag is set for NX windows. Ensure software
1146 	 * complies too.
1147 	 */
1148 	WARN_ON_ONCE(txwin->nx_win && !re);
1149 
1150 	addr = txwin->paste_kaddr;
1151 	if (re) {
1152 		/*
1153 		 * Set the REPORT_ENABLE bit (equivalent to writing
1154 		 * to 1K offset of the paste address)
1155 		 */
1156 		val = SET_FIELD(RMA_LSMP_REPORT_ENABLE, 0ULL, 1);
1157 		addr += val;
1158 	}
1159 
1160 	/*
1161 	 * Map the raw CR value from vas_paste() to an error code (there
1162 	 * is just pass or fail for now though).
1163 	 */
1164 	rc = vas_paste(addr, offset);
1165 	if (rc == 2)
1166 		rc = 0;
1167 	else
1168 		rc = -EINVAL;
1169 
1170 	pr_debug("Txwin #%d: Msg count %llu\n", txwin->winid,
1171 			read_hvwc_reg(txwin, VREG(LRFIFO_PUSH)));
1172 
1173 	return rc;
1174 }
1175 EXPORT_SYMBOL_GPL(vas_paste_crb);
1176 
1177 /*
1178  * If credit checking is enabled for this window, poll for the return
1179  * of window credits (i.e for NX engines to process any outstanding CRBs).
1180  * Since NX-842 waits for the CRBs to be processed before closing the
1181  * window, we should not have to wait for too long.
1182  *
1183  * TODO: We retry in 10ms intervals now. We could/should probably peek at
1184  *	the VAS_LRFIFO_PUSH_OFFSET register to get an estimate of pending
1185  *	CRBs on the FIFO and compute the delay dynamically on each retry.
1186  *	But that is not really needed until we support NX-GZIP access from
1187  *	user space. (NX-842 driver waits for CSB and Fast thread-wakeup
1188  *	doesn't use credit checking).
1189  */
1190 static void poll_window_credits(struct vas_window *window)
1191 {
1192 	u64 val;
1193 	int creds, mode;
1194 	int count = 0;
1195 
1196 	val = read_hvwc_reg(window, VREG(WINCTL));
1197 	if (window->tx_win)
1198 		mode = GET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val);
1199 	else
1200 		mode = GET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val);
1201 
1202 	if (!mode)
1203 		return;
1204 retry:
1205 	if (window->tx_win) {
1206 		val = read_hvwc_reg(window, VREG(TX_WCRED));
1207 		creds = GET_FIELD(VAS_TX_WCRED, val);
1208 	} else {
1209 		val = read_hvwc_reg(window, VREG(LRX_WCRED));
1210 		creds = GET_FIELD(VAS_LRX_WCRED, val);
1211 	}
1212 
1213 	/*
1214 	 * Takes around few milliseconds to complete all pending requests
1215 	 * and return credits.
1216 	 * TODO: Scan fault FIFO and invalidate CRBs points to this window
1217 	 *       and issue CRB Kill to stop all pending requests. Need only
1218 	 *       if there is a bug in NX or fault handling in kernel.
1219 	 */
1220 	if (creds < window->wcreds_max) {
1221 		val = 0;
1222 		set_current_state(TASK_UNINTERRUPTIBLE);
1223 		schedule_timeout(msecs_to_jiffies(10));
1224 		count++;
1225 		/*
1226 		 * Process can not close send window until all credits are
1227 		 * returned.
1228 		 */
1229 		if (!(count % 1000))
1230 			pr_warn_ratelimited("VAS: pid %d stuck. Waiting for credits returned for Window(%d). creds %d, Retries %d\n",
1231 				vas_window_pid(window), window->winid,
1232 				creds, count);
1233 
1234 		goto retry;
1235 	}
1236 }
1237 
1238 /*
1239  * Wait for the window to go to "not-busy" state. It should only take a
1240  * short time to queue a CRB, so window should not be busy for too long.
1241  * Trying 5ms intervals.
1242  */
1243 static void poll_window_busy_state(struct vas_window *window)
1244 {
1245 	int busy;
1246 	u64 val;
1247 	int count = 0;
1248 
1249 retry:
1250 	val = read_hvwc_reg(window, VREG(WIN_STATUS));
1251 	busy = GET_FIELD(VAS_WIN_BUSY, val);
1252 	if (busy) {
1253 		val = 0;
1254 		set_current_state(TASK_UNINTERRUPTIBLE);
1255 		schedule_timeout(msecs_to_jiffies(10));
1256 		count++;
1257 		/*
1258 		 * Takes around few milliseconds to process all pending
1259 		 * requests.
1260 		 */
1261 		if (!(count % 1000))
1262 			pr_warn_ratelimited("VAS: pid %d stuck. Window (ID=%d) is in busy state. Retries %d\n",
1263 				vas_window_pid(window), window->winid, count);
1264 
1265 		goto retry;
1266 	}
1267 }
1268 
1269 /*
1270  * Have the hardware cast a window out of cache and wait for it to
1271  * be completed.
1272  *
1273  * NOTE: It can take a relatively long time to cast the window context
1274  *	out of the cache. It is not strictly necessary to cast out if:
1275  *
1276  *	- we clear the "Pin Window" bit (so hardware is free to evict)
1277  *
1278  *	- we re-initialize the window context when it is reassigned.
1279  *
1280  *	We do the former in vas_win_close() and latter in vas_win_open().
1281  *	So, ignoring the cast-out for now. We can add it as needed. If
1282  *	casting out becomes necessary we should consider offloading the
1283  *	job to a worker thread, so the window close can proceed quickly.
1284  */
1285 static void poll_window_castout(struct vas_window *window)
1286 {
1287 	/* stub for now */
1288 }
1289 
1290 /*
1291  * Unpin and close a window so no new requests are accepted and the
1292  * hardware can evict this window from cache if necessary.
1293  */
1294 static void unpin_close_window(struct vas_window *window)
1295 {
1296 	u64 val;
1297 
1298 	val = read_hvwc_reg(window, VREG(WINCTL));
1299 	val = SET_FIELD(VAS_WINCTL_PIN, val, 0);
1300 	val = SET_FIELD(VAS_WINCTL_OPEN, val, 0);
1301 	write_hvwc_reg(window, VREG(WINCTL), val);
1302 }
1303 
1304 /*
1305  * Close a window.
1306  *
1307  * See Section 1.12.1 of VAS workbook v1.05 for details on closing window:
1308  *	- Disable new paste operations (unmap paste address)
1309  *	- Poll for the "Window Busy" bit to be cleared
1310  *	- Clear the Open/Enable bit for the Window.
1311  *	- Poll for return of window Credits (implies FIFO empty for Rx win?)
1312  *	- Unpin and cast window context out of cache
1313  *
1314  * Besides the hardware, kernel has some bookkeeping of course.
1315  */
1316 int vas_win_close(struct vas_window *window)
1317 {
1318 	if (!window)
1319 		return 0;
1320 
1321 	if (!window->tx_win && atomic_read(&window->num_txwins) != 0) {
1322 		pr_devel("Attempting to close an active Rx window!\n");
1323 		WARN_ON_ONCE(1);
1324 		return -EBUSY;
1325 	}
1326 
1327 	unmap_paste_region(window);
1328 
1329 	poll_window_busy_state(window);
1330 
1331 	unpin_close_window(window);
1332 
1333 	poll_window_credits(window);
1334 
1335 	clear_vinst_win(window);
1336 
1337 	poll_window_castout(window);
1338 
1339 	/* if send window, drop reference to matching receive window */
1340 	if (window->tx_win) {
1341 		if (window->user_win) {
1342 			/* Drop references to pid and mm */
1343 			put_pid(window->pid);
1344 			if (window->mm) {
1345 				mm_context_remove_vas_window(window->mm);
1346 				mmdrop(window->mm);
1347 			}
1348 		}
1349 		put_rx_win(window->rxwin);
1350 	}
1351 
1352 	vas_window_free(window);
1353 
1354 	return 0;
1355 }
1356 EXPORT_SYMBOL_GPL(vas_win_close);
1357 
1358 /*
1359  * Return credit for the given window.
1360  * Send windows and fault window uses credit mechanism as follows:
1361  *
1362  * Send windows:
1363  * - The default number of credits available for each send window is
1364  *   1024. It means 1024 requests can be issued asynchronously at the
1365  *   same time. If the credit is not available, that request will be
1366  *   returned with RMA_Busy.
1367  * - One credit is taken when NX request is issued.
1368  * - This credit is returned after NX processed that request.
1369  * - If NX encounters translation error, kernel will return the
1370  *   credit on the specific send window after processing the fault CRB.
1371  *
1372  * Fault window:
1373  * - The total number credits available is FIFO_SIZE/CRB_SIZE.
1374  *   Means 4MB/128 in the current implementation. If credit is not
1375  *   available, RMA_Reject is returned.
1376  * - A credit is taken when NX pastes CRB in fault FIFO.
1377  * - The kernel with return credit on fault window after reading entry
1378  *   from fault FIFO.
1379  */
1380 void vas_return_credit(struct vas_window *window, bool tx)
1381 {
1382 	uint64_t val;
1383 
1384 	val = 0ULL;
1385 	if (tx) { /* send window */
1386 		val = SET_FIELD(VAS_TX_WCRED, val, 1);
1387 		write_hvwc_reg(window, VREG(TX_WCRED_ADDER), val);
1388 	} else {
1389 		val = SET_FIELD(VAS_LRX_WCRED, val, 1);
1390 		write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), val);
1391 	}
1392 }
1393 
1394 struct vas_window *vas_pswid_to_window(struct vas_instance *vinst,
1395 		uint32_t pswid)
1396 {
1397 	struct vas_window *window;
1398 	int winid;
1399 
1400 	if (!pswid) {
1401 		pr_devel("%s: called for pswid 0!\n", __func__);
1402 		return ERR_PTR(-ESRCH);
1403 	}
1404 
1405 	decode_pswid(pswid, NULL, &winid);
1406 
1407 	if (winid >= VAS_WINDOWS_PER_CHIP)
1408 		return ERR_PTR(-ESRCH);
1409 
1410 	/*
1411 	 * If application closes the window before the hardware
1412 	 * returns the fault CRB, we should wait in vas_win_close()
1413 	 * for the pending requests. so the window must be active
1414 	 * and the process alive.
1415 	 *
1416 	 * If its a kernel process, we should not get any faults and
1417 	 * should not get here.
1418 	 */
1419 	window = vinst->windows[winid];
1420 
1421 	if (!window) {
1422 		pr_err("PSWID decode: Could not find window for winid %d pswid %d vinst 0x%p\n",
1423 			winid, pswid, vinst);
1424 		return NULL;
1425 	}
1426 
1427 	/*
1428 	 * Do some sanity checks on the decoded window.  Window should be
1429 	 * NX GZIP user send window. FTW windows should not incur faults
1430 	 * since their CRBs are ignored (not queued on FIFO or processed
1431 	 * by NX).
1432 	 */
1433 	if (!window->tx_win || !window->user_win || !window->nx_win ||
1434 			window->cop == VAS_COP_TYPE_FAULT ||
1435 			window->cop == VAS_COP_TYPE_FTW) {
1436 		pr_err("PSWID decode: id %d, tx %d, user %d, nx %d, cop %d\n",
1437 			winid, window->tx_win, window->user_win,
1438 			window->nx_win, window->cop);
1439 		WARN_ON(1);
1440 	}
1441 
1442 	return window;
1443 }
1444