1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright 2016-17 IBM Corp.
4  */
5 
6 #define pr_fmt(fmt) "vas: " fmt
7 
8 #include <linux/types.h>
9 #include <linux/mutex.h>
10 #include <linux/slab.h>
11 #include <linux/io.h>
12 #include <linux/log2.h>
13 #include <linux/rcupdate.h>
14 #include <linux/cred.h>
15 #include <linux/sched/mm.h>
16 #include <linux/mmu_context.h>
17 #include <asm/switch_to.h>
18 #include <asm/ppc-opcode.h>
19 #include <asm/vas.h>
20 #include "vas.h"
21 #include "copy-paste.h"
22 
23 #define CREATE_TRACE_POINTS
24 #include "vas-trace.h"
25 
26 /*
27  * Compute the paste address region for the window @window using the
28  * ->paste_base_addr and ->paste_win_id_shift we got from device tree.
29  */
30 void vas_win_paste_addr(struct pnv_vas_window *window, u64 *addr, int *len)
31 {
32 	int winid;
33 	u64 base, shift;
34 
35 	base = window->vinst->paste_base_addr;
36 	shift = window->vinst->paste_win_id_shift;
37 	winid = window->vas_win.winid;
38 
39 	*addr  = base + (winid << shift);
40 	if (len)
41 		*len = PAGE_SIZE;
42 
43 	pr_debug("Txwin #%d: Paste addr 0x%llx\n", winid, *addr);
44 }
45 
46 static inline void get_hvwc_mmio_bar(struct pnv_vas_window *window,
47 			u64 *start, int *len)
48 {
49 	u64 pbaddr;
50 
51 	pbaddr = window->vinst->hvwc_bar_start;
52 	*start = pbaddr + window->vas_win.winid * VAS_HVWC_SIZE;
53 	*len = VAS_HVWC_SIZE;
54 }
55 
56 static inline void get_uwc_mmio_bar(struct pnv_vas_window *window,
57 			u64 *start, int *len)
58 {
59 	u64 pbaddr;
60 
61 	pbaddr = window->vinst->uwc_bar_start;
62 	*start = pbaddr + window->vas_win.winid * VAS_UWC_SIZE;
63 	*len = VAS_UWC_SIZE;
64 }
65 
66 /*
67  * Map the paste bus address of the given send window into kernel address
68  * space. Unlike MMIO regions (map_mmio_region() below), paste region must
69  * be mapped cache-able and is only applicable to send windows.
70  */
71 static void *map_paste_region(struct pnv_vas_window *txwin)
72 {
73 	int len;
74 	void *map;
75 	char *name;
76 	u64 start;
77 
78 	name = kasprintf(GFP_KERNEL, "window-v%d-w%d", txwin->vinst->vas_id,
79 				txwin->vas_win.winid);
80 	if (!name)
81 		goto free_name;
82 
83 	txwin->paste_addr_name = name;
84 	vas_win_paste_addr(txwin, &start, &len);
85 
86 	if (!request_mem_region(start, len, name)) {
87 		pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
88 				__func__, start, len);
89 		goto free_name;
90 	}
91 
92 	map = ioremap_cache(start, len);
93 	if (!map) {
94 		pr_devel("%s(): ioremap_cache(0x%llx, %d) failed\n", __func__,
95 				start, len);
96 		goto free_name;
97 	}
98 
99 	pr_devel("Mapped paste addr 0x%llx to kaddr 0x%p\n", start, map);
100 	return map;
101 
102 free_name:
103 	kfree(name);
104 	return ERR_PTR(-ENOMEM);
105 }
106 
107 static void *map_mmio_region(char *name, u64 start, int len)
108 {
109 	void *map;
110 
111 	if (!request_mem_region(start, len, name)) {
112 		pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
113 				__func__, start, len);
114 		return NULL;
115 	}
116 
117 	map = ioremap(start, len);
118 	if (!map) {
119 		pr_devel("%s(): ioremap(0x%llx, %d) failed\n", __func__, start,
120 				len);
121 		return NULL;
122 	}
123 
124 	return map;
125 }
126 
127 static void unmap_region(void *addr, u64 start, int len)
128 {
129 	iounmap(addr);
130 	release_mem_region((phys_addr_t)start, len);
131 }
132 
133 /*
134  * Unmap the paste address region for a window.
135  */
136 static void unmap_paste_region(struct pnv_vas_window *window)
137 {
138 	int len;
139 	u64 busaddr_start;
140 
141 	if (window->paste_kaddr) {
142 		vas_win_paste_addr(window, &busaddr_start, &len);
143 		unmap_region(window->paste_kaddr, busaddr_start, len);
144 		window->paste_kaddr = NULL;
145 		kfree(window->paste_addr_name);
146 		window->paste_addr_name = NULL;
147 	}
148 }
149 
150 /*
151  * Unmap the MMIO regions for a window. Hold the vas_mutex so we don't
152  * unmap when the window's debugfs dir is in use. This serializes close
153  * of a window even on another VAS instance but since its not a critical
154  * path, just minimize the time we hold the mutex for now. We can add
155  * a per-instance mutex later if necessary.
156  */
157 static void unmap_winctx_mmio_bars(struct pnv_vas_window *window)
158 {
159 	int len;
160 	void *uwc_map;
161 	void *hvwc_map;
162 	u64 busaddr_start;
163 
164 	mutex_lock(&vas_mutex);
165 
166 	hvwc_map = window->hvwc_map;
167 	window->hvwc_map = NULL;
168 
169 	uwc_map = window->uwc_map;
170 	window->uwc_map = NULL;
171 
172 	mutex_unlock(&vas_mutex);
173 
174 	if (hvwc_map) {
175 		get_hvwc_mmio_bar(window, &busaddr_start, &len);
176 		unmap_region(hvwc_map, busaddr_start, len);
177 	}
178 
179 	if (uwc_map) {
180 		get_uwc_mmio_bar(window, &busaddr_start, &len);
181 		unmap_region(uwc_map, busaddr_start, len);
182 	}
183 }
184 
185 /*
186  * Find the Hypervisor Window Context (HVWC) MMIO Base Address Region and the
187  * OS/User Window Context (UWC) MMIO Base Address Region for the given window.
188  * Map these bus addresses and save the mapped kernel addresses in @window.
189  */
190 static int map_winctx_mmio_bars(struct pnv_vas_window *window)
191 {
192 	int len;
193 	u64 start;
194 
195 	get_hvwc_mmio_bar(window, &start, &len);
196 	window->hvwc_map = map_mmio_region("HVWCM_Window", start, len);
197 
198 	get_uwc_mmio_bar(window, &start, &len);
199 	window->uwc_map = map_mmio_region("UWCM_Window", start, len);
200 
201 	if (!window->hvwc_map || !window->uwc_map) {
202 		unmap_winctx_mmio_bars(window);
203 		return -1;
204 	}
205 
206 	return 0;
207 }
208 
209 /*
210  * Reset all valid registers in the HV and OS/User Window Contexts for
211  * the window identified by @window.
212  *
213  * NOTE: We cannot really use a for loop to reset window context. Not all
214  *	 offsets in a window context are valid registers and the valid
215  *	 registers are not sequential. And, we can only write to offsets
216  *	 with valid registers.
217  */
218 static void reset_window_regs(struct pnv_vas_window *window)
219 {
220 	write_hvwc_reg(window, VREG(LPID), 0ULL);
221 	write_hvwc_reg(window, VREG(PID), 0ULL);
222 	write_hvwc_reg(window, VREG(XLATE_MSR), 0ULL);
223 	write_hvwc_reg(window, VREG(XLATE_LPCR), 0ULL);
224 	write_hvwc_reg(window, VREG(XLATE_CTL), 0ULL);
225 	write_hvwc_reg(window, VREG(AMR), 0ULL);
226 	write_hvwc_reg(window, VREG(SEIDR), 0ULL);
227 	write_hvwc_reg(window, VREG(FAULT_TX_WIN), 0ULL);
228 	write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL);
229 	write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), 0ULL);
230 	write_hvwc_reg(window, VREG(PSWID), 0ULL);
231 	write_hvwc_reg(window, VREG(LFIFO_BAR), 0ULL);
232 	write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), 0ULL);
233 	write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), 0ULL);
234 	write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL);
235 	write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL);
236 	write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL);
237 	write_hvwc_reg(window, VREG(LRX_WCRED), 0ULL);
238 	write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
239 	write_hvwc_reg(window, VREG(TX_WCRED), 0ULL);
240 	write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
241 	write_hvwc_reg(window, VREG(LFIFO_SIZE), 0ULL);
242 	write_hvwc_reg(window, VREG(WINCTL), 0ULL);
243 	write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL);
244 	write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), 0ULL);
245 	write_hvwc_reg(window, VREG(TX_RSVD_BUF_COUNT), 0ULL);
246 	write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), 0ULL);
247 	write_hvwc_reg(window, VREG(LNOTIFY_CTL), 0ULL);
248 	write_hvwc_reg(window, VREG(LNOTIFY_PID), 0ULL);
249 	write_hvwc_reg(window, VREG(LNOTIFY_LPID), 0ULL);
250 	write_hvwc_reg(window, VREG(LNOTIFY_TID), 0ULL);
251 	write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), 0ULL);
252 	write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL);
253 
254 	/* Skip read-only registers: NX_UTIL and NX_UTIL_SE */
255 
256 	/*
257 	 * The send and receive window credit adder registers are also
258 	 * accessible from HVWC and have been initialized above. We don't
259 	 * need to initialize from the OS/User Window Context, so skip
260 	 * following calls:
261 	 *
262 	 *	write_uwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
263 	 *	write_uwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
264 	 */
265 }
266 
267 /*
268  * Initialize window context registers related to Address Translation.
269  * These registers are common to send/receive windows although they
270  * differ for user/kernel windows. As we resolve the TODOs we may
271  * want to add fields to vas_winctx and move the initialization to
272  * init_vas_winctx_regs().
273  */
274 static void init_xlate_regs(struct pnv_vas_window *window, bool user_win)
275 {
276 	u64 lpcr, val;
277 
278 	/*
279 	 * MSR_TA, MSR_US are false for both kernel and user.
280 	 * MSR_DR and MSR_PR are false for kernel.
281 	 */
282 	val = 0ULL;
283 	val = SET_FIELD(VAS_XLATE_MSR_HV, val, 1);
284 	val = SET_FIELD(VAS_XLATE_MSR_SF, val, 1);
285 	if (user_win) {
286 		val = SET_FIELD(VAS_XLATE_MSR_DR, val, 1);
287 		val = SET_FIELD(VAS_XLATE_MSR_PR, val, 1);
288 	}
289 	write_hvwc_reg(window, VREG(XLATE_MSR), val);
290 
291 	lpcr = mfspr(SPRN_LPCR);
292 	val = 0ULL;
293 	/*
294 	 * NOTE: From Section 5.7.8.1 Segment Lookaside Buffer of the
295 	 *	 Power ISA, v3.0B, Page size encoding is 0 = 4KB, 5 = 64KB.
296 	 *
297 	 * NOTE: From Section 1.3.1, Address Translation Context of the
298 	 *	 Nest MMU Workbook, LPCR_SC should be 0 for Power9.
299 	 */
300 	val = SET_FIELD(VAS_XLATE_LPCR_PAGE_SIZE, val, 5);
301 	val = SET_FIELD(VAS_XLATE_LPCR_ISL, val, lpcr & LPCR_ISL);
302 	val = SET_FIELD(VAS_XLATE_LPCR_TC, val, lpcr & LPCR_TC);
303 	val = SET_FIELD(VAS_XLATE_LPCR_SC, val, 0);
304 	write_hvwc_reg(window, VREG(XLATE_LPCR), val);
305 
306 	/*
307 	 * Section 1.3.1 (Address translation Context) of NMMU workbook.
308 	 *	0b00	Hashed Page Table mode
309 	 *	0b01	Reserved
310 	 *	0b10	Radix on HPT
311 	 *	0b11	Radix on Radix
312 	 */
313 	val = 0ULL;
314 	val = SET_FIELD(VAS_XLATE_MODE, val, radix_enabled() ? 3 : 2);
315 	write_hvwc_reg(window, VREG(XLATE_CTL), val);
316 
317 	/*
318 	 * TODO: Can we mfspr(AMR) even for user windows?
319 	 */
320 	val = 0ULL;
321 	val = SET_FIELD(VAS_AMR, val, mfspr(SPRN_AMR));
322 	write_hvwc_reg(window, VREG(AMR), val);
323 
324 	val = 0ULL;
325 	val = SET_FIELD(VAS_SEIDR, val, 0);
326 	write_hvwc_reg(window, VREG(SEIDR), val);
327 }
328 
329 /*
330  * Initialize Reserved Send Buffer Count for the send window. It involves
331  * writing to the register, reading it back to confirm that the hardware
332  * has enough buffers to reserve. See section 1.3.1.2.1 of VAS workbook.
333  *
334  * Since we can only make a best-effort attempt to fulfill the request,
335  * we don't return any errors if we cannot.
336  *
337  * TODO: Reserved (aka dedicated) send buffers are not supported yet.
338  */
339 static void init_rsvd_tx_buf_count(struct pnv_vas_window *txwin,
340 				struct vas_winctx *winctx)
341 {
342 	write_hvwc_reg(txwin, VREG(TX_RSVD_BUF_COUNT), 0ULL);
343 }
344 
345 /*
346  * init_winctx_regs()
347  *	Initialize window context registers for a receive window.
348  *	Except for caching control and marking window open, the registers
349  *	are initialized in the order listed in Section 3.1.4 (Window Context
350  *	Cache Register Details) of the VAS workbook although they don't need
351  *	to be.
352  *
353  * Design note: For NX receive windows, NX allocates the FIFO buffer in OPAL
354  *	(so that it can get a large contiguous area) and passes that buffer
355  *	to kernel via device tree. We now write that buffer address to the
356  *	FIFO BAR. Would it make sense to do this all in OPAL? i.e have OPAL
357  *	write the per-chip RX FIFO addresses to the windows during boot-up
358  *	as a one-time task? That could work for NX but what about other
359  *	receivers?  Let the receivers tell us the rx-fifo buffers for now.
360  */
361 static void init_winctx_regs(struct pnv_vas_window *window,
362 			     struct vas_winctx *winctx)
363 {
364 	u64 val;
365 	int fifo_size;
366 
367 	reset_window_regs(window);
368 
369 	val = 0ULL;
370 	val = SET_FIELD(VAS_LPID, val, winctx->lpid);
371 	write_hvwc_reg(window, VREG(LPID), val);
372 
373 	val = 0ULL;
374 	val = SET_FIELD(VAS_PID_ID, val, winctx->pidr);
375 	write_hvwc_reg(window, VREG(PID), val);
376 
377 	init_xlate_regs(window, winctx->user_win);
378 
379 	val = 0ULL;
380 	val = SET_FIELD(VAS_FAULT_TX_WIN, val, winctx->fault_win_id);
381 	write_hvwc_reg(window, VREG(FAULT_TX_WIN), val);
382 
383 	/* In PowerNV, interrupts go to HV. */
384 	write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL);
385 
386 	val = 0ULL;
387 	val = SET_FIELD(VAS_HV_INTR_SRC_RA, val, winctx->irq_port);
388 	write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), val);
389 
390 	val = 0ULL;
391 	val = SET_FIELD(VAS_PSWID_EA_HANDLE, val, winctx->pswid);
392 	write_hvwc_reg(window, VREG(PSWID), val);
393 
394 	write_hvwc_reg(window, VREG(SPARE1), 0ULL);
395 	write_hvwc_reg(window, VREG(SPARE2), 0ULL);
396 	write_hvwc_reg(window, VREG(SPARE3), 0ULL);
397 
398 	/*
399 	 * NOTE: VAS expects the FIFO address to be copied into the LFIFO_BAR
400 	 *	 register as is - do NOT shift the address into VAS_LFIFO_BAR
401 	 *	 bit fields! Ok to set the page migration select fields -
402 	 *	 VAS ignores the lower 10+ bits in the address anyway, because
403 	 *	 the minimum FIFO size is 1K?
404 	 *
405 	 * See also: Design note in function header.
406 	 */
407 	val = winctx->rx_fifo;
408 	val = SET_FIELD(VAS_PAGE_MIGRATION_SELECT, val, 0);
409 	write_hvwc_reg(window, VREG(LFIFO_BAR), val);
410 
411 	val = 0ULL;
412 	val = SET_FIELD(VAS_LDATA_STAMP, val, winctx->data_stamp);
413 	write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), val);
414 
415 	val = 0ULL;
416 	val = SET_FIELD(VAS_LDMA_TYPE, val, winctx->dma_type);
417 	val = SET_FIELD(VAS_LDMA_FIFO_DISABLE, val, winctx->fifo_disable);
418 	write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), val);
419 
420 	write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL);
421 	write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL);
422 	write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL);
423 
424 	val = 0ULL;
425 	val = SET_FIELD(VAS_LRX_WCRED, val, winctx->wcreds_max);
426 	write_hvwc_reg(window, VREG(LRX_WCRED), val);
427 
428 	val = 0ULL;
429 	val = SET_FIELD(VAS_TX_WCRED, val, winctx->wcreds_max);
430 	write_hvwc_reg(window, VREG(TX_WCRED), val);
431 
432 	write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
433 	write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
434 
435 	fifo_size = winctx->rx_fifo_size / 1024;
436 
437 	val = 0ULL;
438 	val = SET_FIELD(VAS_LFIFO_SIZE, val, ilog2(fifo_size));
439 	write_hvwc_reg(window, VREG(LFIFO_SIZE), val);
440 
441 	/* Update window control and caching control registers last so
442 	 * we mark the window open only after fully initializing it and
443 	 * pushing context to cache.
444 	 */
445 
446 	write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL);
447 
448 	init_rsvd_tx_buf_count(window, winctx);
449 
450 	/* for a send window, point to the matching receive window */
451 	val = 0ULL;
452 	val = SET_FIELD(VAS_LRX_WIN_ID, val, winctx->rx_win_id);
453 	write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), val);
454 
455 	write_hvwc_reg(window, VREG(SPARE4), 0ULL);
456 
457 	val = 0ULL;
458 	val = SET_FIELD(VAS_NOTIFY_DISABLE, val, winctx->notify_disable);
459 	val = SET_FIELD(VAS_INTR_DISABLE, val, winctx->intr_disable);
460 	val = SET_FIELD(VAS_NOTIFY_EARLY, val, winctx->notify_early);
461 	val = SET_FIELD(VAS_NOTIFY_OSU_INTR, val, winctx->notify_os_intr_reg);
462 	write_hvwc_reg(window, VREG(LNOTIFY_CTL), val);
463 
464 	val = 0ULL;
465 	val = SET_FIELD(VAS_LNOTIFY_PID, val, winctx->lnotify_pid);
466 	write_hvwc_reg(window, VREG(LNOTIFY_PID), val);
467 
468 	val = 0ULL;
469 	val = SET_FIELD(VAS_LNOTIFY_LPID, val, winctx->lnotify_lpid);
470 	write_hvwc_reg(window, VREG(LNOTIFY_LPID), val);
471 
472 	val = 0ULL;
473 	val = SET_FIELD(VAS_LNOTIFY_TID, val, winctx->lnotify_tid);
474 	write_hvwc_reg(window, VREG(LNOTIFY_TID), val);
475 
476 	val = 0ULL;
477 	val = SET_FIELD(VAS_LNOTIFY_MIN_SCOPE, val, winctx->min_scope);
478 	val = SET_FIELD(VAS_LNOTIFY_MAX_SCOPE, val, winctx->max_scope);
479 	write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), val);
480 
481 	/* Skip read-only registers NX_UTIL and NX_UTIL_SE */
482 
483 	write_hvwc_reg(window, VREG(SPARE5), 0ULL);
484 	write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL);
485 	write_hvwc_reg(window, VREG(SPARE6), 0ULL);
486 
487 	/* Finally, push window context to memory and... */
488 	val = 0ULL;
489 	val = SET_FIELD(VAS_PUSH_TO_MEM, val, 1);
490 	write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), val);
491 
492 	/* ... mark the window open for business */
493 	val = 0ULL;
494 	val = SET_FIELD(VAS_WINCTL_REJ_NO_CREDIT, val, winctx->rej_no_credit);
495 	val = SET_FIELD(VAS_WINCTL_PIN, val, winctx->pin_win);
496 	val = SET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val, winctx->tx_wcred_mode);
497 	val = SET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val, winctx->rx_wcred_mode);
498 	val = SET_FIELD(VAS_WINCTL_TX_WORD_MODE, val, winctx->tx_word_mode);
499 	val = SET_FIELD(VAS_WINCTL_RX_WORD_MODE, val, winctx->rx_word_mode);
500 	val = SET_FIELD(VAS_WINCTL_FAULT_WIN, val, winctx->fault_win);
501 	val = SET_FIELD(VAS_WINCTL_NX_WIN, val, winctx->nx_win);
502 	val = SET_FIELD(VAS_WINCTL_OPEN, val, 1);
503 	write_hvwc_reg(window, VREG(WINCTL), val);
504 }
505 
506 static void vas_release_window_id(struct ida *ida, int winid)
507 {
508 	ida_free(ida, winid);
509 }
510 
511 static int vas_assign_window_id(struct ida *ida)
512 {
513 	int winid = ida_alloc_max(ida, VAS_WINDOWS_PER_CHIP - 1, GFP_KERNEL);
514 
515 	if (winid == -ENOSPC) {
516 		pr_err("Too many (%d) open windows\n", VAS_WINDOWS_PER_CHIP);
517 		return -EAGAIN;
518 	}
519 
520 	return winid;
521 }
522 
523 static void vas_window_free(struct pnv_vas_window *window)
524 {
525 	struct vas_instance *vinst = window->vinst;
526 	int winid = window->vas_win.winid;
527 
528 	unmap_winctx_mmio_bars(window);
529 
530 	vas_window_free_dbgdir(window);
531 
532 	kfree(window);
533 
534 	vas_release_window_id(&vinst->ida, winid);
535 }
536 
537 static struct pnv_vas_window *vas_window_alloc(struct vas_instance *vinst)
538 {
539 	int winid;
540 	struct pnv_vas_window *window;
541 
542 	winid = vas_assign_window_id(&vinst->ida);
543 	if (winid < 0)
544 		return ERR_PTR(winid);
545 
546 	window = kzalloc(sizeof(*window), GFP_KERNEL);
547 	if (!window)
548 		goto out_free;
549 
550 	window->vinst = vinst;
551 	window->vas_win.winid = winid;
552 
553 	if (map_winctx_mmio_bars(window))
554 		goto out_free;
555 
556 	vas_window_init_dbgdir(window);
557 
558 	return window;
559 
560 out_free:
561 	kfree(window);
562 	vas_release_window_id(&vinst->ida, winid);
563 	return ERR_PTR(-ENOMEM);
564 }
565 
566 static void put_rx_win(struct pnv_vas_window *rxwin)
567 {
568 	/* Better not be a send window! */
569 	WARN_ON_ONCE(rxwin->tx_win);
570 
571 	atomic_dec(&rxwin->num_txwins);
572 }
573 
574 /*
575  * Find the user space receive window given the @pswid.
576  *      - We must have a valid vasid and it must belong to this instance.
577  *        (so both send and receive windows are on the same VAS instance)
578  *      - The window must refer to an OPEN, FTW, RECEIVE window.
579  *
580  * NOTE: We access ->windows[] table and assume that vinst->mutex is held.
581  */
582 static struct pnv_vas_window *get_user_rxwin(struct vas_instance *vinst,
583 					     u32 pswid)
584 {
585 	int vasid, winid;
586 	struct pnv_vas_window *rxwin;
587 
588 	decode_pswid(pswid, &vasid, &winid);
589 
590 	if (vinst->vas_id != vasid)
591 		return ERR_PTR(-EINVAL);
592 
593 	rxwin = vinst->windows[winid];
594 
595 	if (!rxwin || rxwin->tx_win || rxwin->vas_win.cop != VAS_COP_TYPE_FTW)
596 		return ERR_PTR(-EINVAL);
597 
598 	return rxwin;
599 }
600 
601 /*
602  * Get the VAS receive window associated with NX engine identified
603  * by @cop and if applicable, @pswid.
604  *
605  * See also function header of set_vinst_win().
606  */
607 static struct pnv_vas_window *get_vinst_rxwin(struct vas_instance *vinst,
608 			enum vas_cop_type cop, u32 pswid)
609 {
610 	struct pnv_vas_window *rxwin;
611 
612 	mutex_lock(&vinst->mutex);
613 
614 	if (cop == VAS_COP_TYPE_FTW)
615 		rxwin = get_user_rxwin(vinst, pswid);
616 	else
617 		rxwin = vinst->rxwin[cop] ?: ERR_PTR(-EINVAL);
618 
619 	if (!IS_ERR(rxwin))
620 		atomic_inc(&rxwin->num_txwins);
621 
622 	mutex_unlock(&vinst->mutex);
623 
624 	return rxwin;
625 }
626 
627 /*
628  * We have two tables of windows in a VAS instance. The first one,
629  * ->windows[], contains all the windows in the instance and allows
630  * looking up a window by its id. It is used to look up send windows
631  * during fault handling and receive windows when pairing user space
632  * send/receive windows.
633  *
634  * The second table, ->rxwin[], contains receive windows that are
635  * associated with NX engines. This table has VAS_COP_TYPE_MAX
636  * entries and is used to look up a receive window by its
637  * coprocessor type.
638  *
639  * Here, we save @window in the ->windows[] table. If it is a receive
640  * window, we also save the window in the ->rxwin[] table.
641  */
642 static void set_vinst_win(struct vas_instance *vinst,
643 			struct pnv_vas_window *window)
644 {
645 	int id = window->vas_win.winid;
646 
647 	mutex_lock(&vinst->mutex);
648 
649 	/*
650 	 * There should only be one receive window for a coprocessor type
651 	 * unless its a user (FTW) window.
652 	 */
653 	if (!window->user_win && !window->tx_win) {
654 		WARN_ON_ONCE(vinst->rxwin[window->vas_win.cop]);
655 		vinst->rxwin[window->vas_win.cop] = window;
656 	}
657 
658 	WARN_ON_ONCE(vinst->windows[id] != NULL);
659 	vinst->windows[id] = window;
660 
661 	mutex_unlock(&vinst->mutex);
662 }
663 
664 /*
665  * Clear this window from the table(s) of windows for this VAS instance.
666  * See also function header of set_vinst_win().
667  */
668 static void clear_vinst_win(struct pnv_vas_window *window)
669 {
670 	int id = window->vas_win.winid;
671 	struct vas_instance *vinst = window->vinst;
672 
673 	mutex_lock(&vinst->mutex);
674 
675 	if (!window->user_win && !window->tx_win) {
676 		WARN_ON_ONCE(!vinst->rxwin[window->vas_win.cop]);
677 		vinst->rxwin[window->vas_win.cop] = NULL;
678 	}
679 
680 	WARN_ON_ONCE(vinst->windows[id] != window);
681 	vinst->windows[id] = NULL;
682 
683 	mutex_unlock(&vinst->mutex);
684 }
685 
686 static void init_winctx_for_rxwin(struct pnv_vas_window *rxwin,
687 			struct vas_rx_win_attr *rxattr,
688 			struct vas_winctx *winctx)
689 {
690 	/*
691 	 * We first zero (memset()) all fields and only set non-zero fields.
692 	 * Following fields are 0/false but maybe deserve a comment:
693 	 *
694 	 *	->notify_os_intr_reg	In powerNV, send intrs to HV
695 	 *	->notify_disable	False for NX windows
696 	 *	->intr_disable		False for Fault Windows
697 	 *	->xtra_write		False for NX windows
698 	 *	->notify_early		NA for NX windows
699 	 *	->rsvd_txbuf_count	NA for Rx windows
700 	 *	->lpid, ->pid, ->tid	NA for Rx windows
701 	 */
702 
703 	memset(winctx, 0, sizeof(struct vas_winctx));
704 
705 	winctx->rx_fifo = rxattr->rx_fifo;
706 	winctx->rx_fifo_size = rxattr->rx_fifo_size;
707 	winctx->wcreds_max = rxwin->vas_win.wcreds_max;
708 	winctx->pin_win = rxattr->pin_win;
709 
710 	winctx->nx_win = rxattr->nx_win;
711 	winctx->fault_win = rxattr->fault_win;
712 	winctx->user_win = rxattr->user_win;
713 	winctx->rej_no_credit = rxattr->rej_no_credit;
714 	winctx->rx_word_mode = rxattr->rx_win_ord_mode;
715 	winctx->tx_word_mode = rxattr->tx_win_ord_mode;
716 	winctx->rx_wcred_mode = rxattr->rx_wcred_mode;
717 	winctx->tx_wcred_mode = rxattr->tx_wcred_mode;
718 	winctx->notify_early = rxattr->notify_early;
719 
720 	if (winctx->nx_win) {
721 		winctx->data_stamp = true;
722 		winctx->intr_disable = true;
723 		winctx->pin_win = true;
724 
725 		WARN_ON_ONCE(winctx->fault_win);
726 		WARN_ON_ONCE(!winctx->rx_word_mode);
727 		WARN_ON_ONCE(!winctx->tx_word_mode);
728 		WARN_ON_ONCE(winctx->notify_after_count);
729 	} else if (winctx->fault_win) {
730 		winctx->notify_disable = true;
731 	} else if (winctx->user_win) {
732 		/*
733 		 * Section 1.8.1 Low Latency Core-Core Wake up of
734 		 * the VAS workbook:
735 		 *
736 		 *      - disable credit checks ([tr]x_wcred_mode = false)
737 		 *      - disable FIFO writes
738 		 *      - enable ASB_Notify, disable interrupt
739 		 */
740 		winctx->fifo_disable = true;
741 		winctx->intr_disable = true;
742 		winctx->rx_fifo = 0;
743 	}
744 
745 	winctx->lnotify_lpid = rxattr->lnotify_lpid;
746 	winctx->lnotify_pid = rxattr->lnotify_pid;
747 	winctx->lnotify_tid = rxattr->lnotify_tid;
748 	winctx->pswid = rxattr->pswid;
749 	winctx->dma_type = VAS_DMA_TYPE_INJECT;
750 	winctx->tc_mode = rxattr->tc_mode;
751 
752 	winctx->min_scope = VAS_SCOPE_LOCAL;
753 	winctx->max_scope = VAS_SCOPE_VECTORED_GROUP;
754 	if (rxwin->vinst->virq)
755 		winctx->irq_port = rxwin->vinst->irq_port;
756 }
757 
758 static bool rx_win_args_valid(enum vas_cop_type cop,
759 			struct vas_rx_win_attr *attr)
760 {
761 	pr_debug("Rxattr: fault %d, notify %d, intr %d, early %d, fifo %d\n",
762 			attr->fault_win, attr->notify_disable,
763 			attr->intr_disable, attr->notify_early,
764 			attr->rx_fifo_size);
765 
766 	if (cop >= VAS_COP_TYPE_MAX)
767 		return false;
768 
769 	if (cop != VAS_COP_TYPE_FTW &&
770 				attr->rx_fifo_size < VAS_RX_FIFO_SIZE_MIN)
771 		return false;
772 
773 	if (attr->rx_fifo_size > VAS_RX_FIFO_SIZE_MAX)
774 		return false;
775 
776 	if (!attr->wcreds_max)
777 		return false;
778 
779 	if (attr->nx_win) {
780 		/* cannot be fault or user window if it is nx */
781 		if (attr->fault_win || attr->user_win)
782 			return false;
783 		/*
784 		 * Section 3.1.4.32: NX Windows must not disable notification,
785 		 *	and must not enable interrupts or early notification.
786 		 */
787 		if (attr->notify_disable || !attr->intr_disable ||
788 				attr->notify_early)
789 			return false;
790 	} else if (attr->fault_win) {
791 		/* cannot be both fault and user window */
792 		if (attr->user_win)
793 			return false;
794 
795 		/*
796 		 * Section 3.1.4.32: Fault windows must disable notification
797 		 *	but not interrupts.
798 		 */
799 		if (!attr->notify_disable || attr->intr_disable)
800 			return false;
801 
802 	} else if (attr->user_win) {
803 		/*
804 		 * User receive windows are only for fast-thread-wakeup
805 		 * (FTW). They don't need a FIFO and must disable interrupts
806 		 */
807 		if (attr->rx_fifo || attr->rx_fifo_size || !attr->intr_disable)
808 			return false;
809 	} else {
810 		/* Rx window must be one of NX or Fault or User window. */
811 		return false;
812 	}
813 
814 	return true;
815 }
816 
817 void vas_init_rx_win_attr(struct vas_rx_win_attr *rxattr, enum vas_cop_type cop)
818 {
819 	memset(rxattr, 0, sizeof(*rxattr));
820 
821 	if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI ||
822 		cop == VAS_COP_TYPE_GZIP || cop == VAS_COP_TYPE_GZIP_HIPRI) {
823 		rxattr->pin_win = true;
824 		rxattr->nx_win = true;
825 		rxattr->fault_win = false;
826 		rxattr->intr_disable = true;
827 		rxattr->rx_wcred_mode = true;
828 		rxattr->tx_wcred_mode = true;
829 		rxattr->rx_win_ord_mode = true;
830 		rxattr->tx_win_ord_mode = true;
831 	} else if (cop == VAS_COP_TYPE_FAULT) {
832 		rxattr->pin_win = true;
833 		rxattr->fault_win = true;
834 		rxattr->notify_disable = true;
835 		rxattr->rx_wcred_mode = true;
836 		rxattr->rx_win_ord_mode = true;
837 		rxattr->rej_no_credit = true;
838 		rxattr->tc_mode = VAS_THRESH_DISABLED;
839 	} else if (cop == VAS_COP_TYPE_FTW) {
840 		rxattr->user_win = true;
841 		rxattr->intr_disable = true;
842 
843 		/*
844 		 * As noted in the VAS Workbook we disable credit checks.
845 		 * If we enable credit checks in the future, we must also
846 		 * implement a mechanism to return the user credits or new
847 		 * paste operations will fail.
848 		 */
849 	}
850 }
851 EXPORT_SYMBOL_GPL(vas_init_rx_win_attr);
852 
853 struct vas_window *vas_rx_win_open(int vasid, enum vas_cop_type cop,
854 			struct vas_rx_win_attr *rxattr)
855 {
856 	struct pnv_vas_window *rxwin;
857 	struct vas_winctx winctx;
858 	struct vas_instance *vinst;
859 
860 	trace_vas_rx_win_open(current, vasid, cop, rxattr);
861 
862 	if (!rx_win_args_valid(cop, rxattr))
863 		return ERR_PTR(-EINVAL);
864 
865 	vinst = find_vas_instance(vasid);
866 	if (!vinst) {
867 		pr_devel("vasid %d not found!\n", vasid);
868 		return ERR_PTR(-EINVAL);
869 	}
870 	pr_devel("Found instance %d\n", vasid);
871 
872 	rxwin = vas_window_alloc(vinst);
873 	if (IS_ERR(rxwin)) {
874 		pr_devel("Unable to allocate memory for Rx window\n");
875 		return (struct vas_window *)rxwin;
876 	}
877 
878 	rxwin->tx_win = false;
879 	rxwin->nx_win = rxattr->nx_win;
880 	rxwin->user_win = rxattr->user_win;
881 	rxwin->vas_win.cop = cop;
882 	rxwin->vas_win.wcreds_max = rxattr->wcreds_max;
883 
884 	init_winctx_for_rxwin(rxwin, rxattr, &winctx);
885 	init_winctx_regs(rxwin, &winctx);
886 
887 	set_vinst_win(vinst, rxwin);
888 
889 	return &rxwin->vas_win;
890 }
891 EXPORT_SYMBOL_GPL(vas_rx_win_open);
892 
893 void vas_init_tx_win_attr(struct vas_tx_win_attr *txattr, enum vas_cop_type cop)
894 {
895 	memset(txattr, 0, sizeof(*txattr));
896 
897 	if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI ||
898 		cop == VAS_COP_TYPE_GZIP || cop == VAS_COP_TYPE_GZIP_HIPRI) {
899 		txattr->rej_no_credit = false;
900 		txattr->rx_wcred_mode = true;
901 		txattr->tx_wcred_mode = true;
902 		txattr->rx_win_ord_mode = true;
903 		txattr->tx_win_ord_mode = true;
904 	} else if (cop == VAS_COP_TYPE_FTW) {
905 		txattr->user_win = true;
906 	}
907 }
908 EXPORT_SYMBOL_GPL(vas_init_tx_win_attr);
909 
910 static void init_winctx_for_txwin(struct pnv_vas_window *txwin,
911 			struct vas_tx_win_attr *txattr,
912 			struct vas_winctx *winctx)
913 {
914 	/*
915 	 * We first zero all fields and only set non-zero ones. Following
916 	 * are some fields set to 0/false for the stated reason:
917 	 *
918 	 *	->notify_os_intr_reg	In powernv, send intrs to HV
919 	 *	->rsvd_txbuf_count	Not supported yet.
920 	 *	->notify_disable	False for NX windows
921 	 *	->xtra_write		False for NX windows
922 	 *	->notify_early		NA for NX windows
923 	 *	->lnotify_lpid		NA for Tx windows
924 	 *	->lnotify_pid		NA for Tx windows
925 	 *	->lnotify_tid		NA for Tx windows
926 	 *	->tx_win_cred_mode	Ignore for now for NX windows
927 	 *	->rx_win_cred_mode	Ignore for now for NX windows
928 	 */
929 	memset(winctx, 0, sizeof(struct vas_winctx));
930 
931 	winctx->wcreds_max = txwin->vas_win.wcreds_max;
932 
933 	winctx->user_win = txattr->user_win;
934 	winctx->nx_win = txwin->rxwin->nx_win;
935 	winctx->pin_win = txattr->pin_win;
936 	winctx->rej_no_credit = txattr->rej_no_credit;
937 	winctx->rsvd_txbuf_enable = txattr->rsvd_txbuf_enable;
938 
939 	winctx->rx_wcred_mode = txattr->rx_wcred_mode;
940 	winctx->tx_wcred_mode = txattr->tx_wcred_mode;
941 	winctx->rx_word_mode = txattr->rx_win_ord_mode;
942 	winctx->tx_word_mode = txattr->tx_win_ord_mode;
943 	winctx->rsvd_txbuf_count = txattr->rsvd_txbuf_count;
944 
945 	winctx->intr_disable = true;
946 	if (winctx->nx_win)
947 		winctx->data_stamp = true;
948 
949 	winctx->lpid = txattr->lpid;
950 	winctx->pidr = txattr->pidr;
951 	winctx->rx_win_id = txwin->rxwin->vas_win.winid;
952 	/*
953 	 * IRQ and fault window setup is successful. Set fault window
954 	 * for the send window so that ready to handle faults.
955 	 */
956 	if (txwin->vinst->virq)
957 		winctx->fault_win_id = txwin->vinst->fault_win->vas_win.winid;
958 
959 	winctx->dma_type = VAS_DMA_TYPE_INJECT;
960 	winctx->tc_mode = txattr->tc_mode;
961 	winctx->min_scope = VAS_SCOPE_LOCAL;
962 	winctx->max_scope = VAS_SCOPE_VECTORED_GROUP;
963 	if (txwin->vinst->virq)
964 		winctx->irq_port = txwin->vinst->irq_port;
965 
966 	winctx->pswid = txattr->pswid ? txattr->pswid :
967 			encode_pswid(txwin->vinst->vas_id,
968 			txwin->vas_win.winid);
969 }
970 
971 static bool tx_win_args_valid(enum vas_cop_type cop,
972 			struct vas_tx_win_attr *attr)
973 {
974 	if (attr->tc_mode != VAS_THRESH_DISABLED)
975 		return false;
976 
977 	if (cop > VAS_COP_TYPE_MAX)
978 		return false;
979 
980 	if (attr->wcreds_max > VAS_TX_WCREDS_MAX)
981 		return false;
982 
983 	if (attr->user_win) {
984 		if (attr->rsvd_txbuf_count)
985 			return false;
986 
987 		if (cop != VAS_COP_TYPE_FTW && cop != VAS_COP_TYPE_GZIP &&
988 			cop != VAS_COP_TYPE_GZIP_HIPRI)
989 			return false;
990 	}
991 
992 	return true;
993 }
994 
995 struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop,
996 			struct vas_tx_win_attr *attr)
997 {
998 	int rc;
999 	struct pnv_vas_window *txwin;
1000 	struct pnv_vas_window *rxwin;
1001 	struct vas_winctx winctx;
1002 	struct vas_instance *vinst;
1003 
1004 	trace_vas_tx_win_open(current, vasid, cop, attr);
1005 
1006 	if (!tx_win_args_valid(cop, attr))
1007 		return ERR_PTR(-EINVAL);
1008 
1009 	/*
1010 	 * If caller did not specify a vasid but specified the PSWID of a
1011 	 * receive window (applicable only to FTW windows), use the vasid
1012 	 * from that receive window.
1013 	 */
1014 	if (vasid == -1 && attr->pswid)
1015 		decode_pswid(attr->pswid, &vasid, NULL);
1016 
1017 	vinst = find_vas_instance(vasid);
1018 	if (!vinst) {
1019 		pr_devel("vasid %d not found!\n", vasid);
1020 		return ERR_PTR(-EINVAL);
1021 	}
1022 
1023 	rxwin = get_vinst_rxwin(vinst, cop, attr->pswid);
1024 	if (IS_ERR(rxwin)) {
1025 		pr_devel("No RxWin for vasid %d, cop %d\n", vasid, cop);
1026 		return (struct vas_window *)rxwin;
1027 	}
1028 
1029 	txwin = vas_window_alloc(vinst);
1030 	if (IS_ERR(txwin)) {
1031 		rc = PTR_ERR(txwin);
1032 		goto put_rxwin;
1033 	}
1034 
1035 	txwin->vas_win.cop = cop;
1036 	txwin->tx_win = 1;
1037 	txwin->rxwin = rxwin;
1038 	txwin->nx_win = txwin->rxwin->nx_win;
1039 	txwin->user_win = attr->user_win;
1040 	txwin->vas_win.wcreds_max = attr->wcreds_max ?: VAS_WCREDS_DEFAULT;
1041 
1042 	init_winctx_for_txwin(txwin, attr, &winctx);
1043 
1044 	init_winctx_regs(txwin, &winctx);
1045 
1046 	/*
1047 	 * If its a kernel send window, map the window address into the
1048 	 * kernel's address space. For user windows, user must issue an
1049 	 * mmap() to map the window into their address space.
1050 	 *
1051 	 * NOTE: If kernel ever resubmits a user CRB after handling a page
1052 	 *	 fault, we will need to map this into kernel as well.
1053 	 */
1054 	if (!txwin->user_win) {
1055 		txwin->paste_kaddr = map_paste_region(txwin);
1056 		if (IS_ERR(txwin->paste_kaddr)) {
1057 			rc = PTR_ERR(txwin->paste_kaddr);
1058 			goto free_window;
1059 		}
1060 	} else {
1061 		/*
1062 		 * Interrupt hanlder or fault window setup failed. Means
1063 		 * NX can not generate fault for page fault. So not
1064 		 * opening for user space tx window.
1065 		 */
1066 		if (!vinst->virq) {
1067 			rc = -ENODEV;
1068 			goto free_window;
1069 		}
1070 		rc = get_vas_user_win_ref(&txwin->vas_win.task_ref);
1071 		if (rc)
1072 			goto free_window;
1073 
1074 		vas_user_win_add_mm_context(&txwin->vas_win.task_ref);
1075 	}
1076 
1077 	set_vinst_win(vinst, txwin);
1078 
1079 	return &txwin->vas_win;
1080 
1081 free_window:
1082 	vas_window_free(txwin);
1083 
1084 put_rxwin:
1085 	put_rx_win(rxwin);
1086 	return ERR_PTR(rc);
1087 
1088 }
1089 EXPORT_SYMBOL_GPL(vas_tx_win_open);
1090 
1091 int vas_copy_crb(void *crb, int offset)
1092 {
1093 	return vas_copy(crb, offset);
1094 }
1095 EXPORT_SYMBOL_GPL(vas_copy_crb);
1096 
1097 #define RMA_LSMP_REPORT_ENABLE PPC_BIT(53)
1098 int vas_paste_crb(struct vas_window *vwin, int offset, bool re)
1099 {
1100 	struct pnv_vas_window *txwin;
1101 	int rc;
1102 	void *addr;
1103 	uint64_t val;
1104 
1105 	txwin = container_of(vwin, struct pnv_vas_window, vas_win);
1106 	trace_vas_paste_crb(current, txwin);
1107 
1108 	/*
1109 	 * Only NX windows are supported for now and hardware assumes
1110 	 * report-enable flag is set for NX windows. Ensure software
1111 	 * complies too.
1112 	 */
1113 	WARN_ON_ONCE(txwin->nx_win && !re);
1114 
1115 	addr = txwin->paste_kaddr;
1116 	if (re) {
1117 		/*
1118 		 * Set the REPORT_ENABLE bit (equivalent to writing
1119 		 * to 1K offset of the paste address)
1120 		 */
1121 		val = SET_FIELD(RMA_LSMP_REPORT_ENABLE, 0ULL, 1);
1122 		addr += val;
1123 	}
1124 
1125 	/*
1126 	 * Map the raw CR value from vas_paste() to an error code (there
1127 	 * is just pass or fail for now though).
1128 	 */
1129 	rc = vas_paste(addr, offset);
1130 	if (rc == 2)
1131 		rc = 0;
1132 	else
1133 		rc = -EINVAL;
1134 
1135 	pr_debug("Txwin #%d: Msg count %llu\n", txwin->vas_win.winid,
1136 			read_hvwc_reg(txwin, VREG(LRFIFO_PUSH)));
1137 
1138 	return rc;
1139 }
1140 EXPORT_SYMBOL_GPL(vas_paste_crb);
1141 
1142 /*
1143  * If credit checking is enabled for this window, poll for the return
1144  * of window credits (i.e for NX engines to process any outstanding CRBs).
1145  * Since NX-842 waits for the CRBs to be processed before closing the
1146  * window, we should not have to wait for too long.
1147  *
1148  * TODO: We retry in 10ms intervals now. We could/should probably peek at
1149  *	the VAS_LRFIFO_PUSH_OFFSET register to get an estimate of pending
1150  *	CRBs on the FIFO and compute the delay dynamically on each retry.
1151  *	But that is not really needed until we support NX-GZIP access from
1152  *	user space. (NX-842 driver waits for CSB and Fast thread-wakeup
1153  *	doesn't use credit checking).
1154  */
1155 static void poll_window_credits(struct pnv_vas_window *window)
1156 {
1157 	u64 val;
1158 	int creds, mode;
1159 	int count = 0;
1160 
1161 	val = read_hvwc_reg(window, VREG(WINCTL));
1162 	if (window->tx_win)
1163 		mode = GET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val);
1164 	else
1165 		mode = GET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val);
1166 
1167 	if (!mode)
1168 		return;
1169 retry:
1170 	if (window->tx_win) {
1171 		val = read_hvwc_reg(window, VREG(TX_WCRED));
1172 		creds = GET_FIELD(VAS_TX_WCRED, val);
1173 	} else {
1174 		val = read_hvwc_reg(window, VREG(LRX_WCRED));
1175 		creds = GET_FIELD(VAS_LRX_WCRED, val);
1176 	}
1177 
1178 	/*
1179 	 * Takes around few milliseconds to complete all pending requests
1180 	 * and return credits.
1181 	 * TODO: Scan fault FIFO and invalidate CRBs points to this window
1182 	 *       and issue CRB Kill to stop all pending requests. Need only
1183 	 *       if there is a bug in NX or fault handling in kernel.
1184 	 */
1185 	if (creds < window->vas_win.wcreds_max) {
1186 		val = 0;
1187 		set_current_state(TASK_UNINTERRUPTIBLE);
1188 		schedule_timeout(msecs_to_jiffies(10));
1189 		count++;
1190 		/*
1191 		 * Process can not close send window until all credits are
1192 		 * returned.
1193 		 */
1194 		if (!(count % 1000))
1195 			pr_warn_ratelimited("VAS: pid %d stuck. Waiting for credits returned for Window(%d). creds %d, Retries %d\n",
1196 				vas_window_pid(&window->vas_win),
1197 				window->vas_win.winid,
1198 				creds, count);
1199 
1200 		goto retry;
1201 	}
1202 }
1203 
1204 /*
1205  * Wait for the window to go to "not-busy" state. It should only take a
1206  * short time to queue a CRB, so window should not be busy for too long.
1207  * Trying 5ms intervals.
1208  */
1209 static void poll_window_busy_state(struct pnv_vas_window *window)
1210 {
1211 	int busy;
1212 	u64 val;
1213 	int count = 0;
1214 
1215 retry:
1216 	val = read_hvwc_reg(window, VREG(WIN_STATUS));
1217 	busy = GET_FIELD(VAS_WIN_BUSY, val);
1218 	if (busy) {
1219 		val = 0;
1220 		set_current_state(TASK_UNINTERRUPTIBLE);
1221 		schedule_timeout(msecs_to_jiffies(10));
1222 		count++;
1223 		/*
1224 		 * Takes around few milliseconds to process all pending
1225 		 * requests.
1226 		 */
1227 		if (!(count % 1000))
1228 			pr_warn_ratelimited("VAS: pid %d stuck. Window (ID=%d) is in busy state. Retries %d\n",
1229 				vas_window_pid(&window->vas_win),
1230 				window->vas_win.winid, count);
1231 
1232 		goto retry;
1233 	}
1234 }
1235 
1236 /*
1237  * Have the hardware cast a window out of cache and wait for it to
1238  * be completed.
1239  *
1240  * NOTE: It can take a relatively long time to cast the window context
1241  *	out of the cache. It is not strictly necessary to cast out if:
1242  *
1243  *	- we clear the "Pin Window" bit (so hardware is free to evict)
1244  *
1245  *	- we re-initialize the window context when it is reassigned.
1246  *
1247  *	We do the former in vas_win_close() and latter in vas_win_open().
1248  *	So, ignoring the cast-out for now. We can add it as needed. If
1249  *	casting out becomes necessary we should consider offloading the
1250  *	job to a worker thread, so the window close can proceed quickly.
1251  */
1252 static void poll_window_castout(struct pnv_vas_window *window)
1253 {
1254 	/* stub for now */
1255 }
1256 
1257 /*
1258  * Unpin and close a window so no new requests are accepted and the
1259  * hardware can evict this window from cache if necessary.
1260  */
1261 static void unpin_close_window(struct pnv_vas_window *window)
1262 {
1263 	u64 val;
1264 
1265 	val = read_hvwc_reg(window, VREG(WINCTL));
1266 	val = SET_FIELD(VAS_WINCTL_PIN, val, 0);
1267 	val = SET_FIELD(VAS_WINCTL_OPEN, val, 0);
1268 	write_hvwc_reg(window, VREG(WINCTL), val);
1269 }
1270 
1271 /*
1272  * Close a window.
1273  *
1274  * See Section 1.12.1 of VAS workbook v1.05 for details on closing window:
1275  *	- Disable new paste operations (unmap paste address)
1276  *	- Poll for the "Window Busy" bit to be cleared
1277  *	- Clear the Open/Enable bit for the Window.
1278  *	- Poll for return of window Credits (implies FIFO empty for Rx win?)
1279  *	- Unpin and cast window context out of cache
1280  *
1281  * Besides the hardware, kernel has some bookkeeping of course.
1282  */
1283 int vas_win_close(struct vas_window *vwin)
1284 {
1285 	struct pnv_vas_window *window;
1286 
1287 	if (!vwin)
1288 		return 0;
1289 
1290 	window = container_of(vwin, struct pnv_vas_window, vas_win);
1291 
1292 	if (!window->tx_win && atomic_read(&window->num_txwins) != 0) {
1293 		pr_devel("Attempting to close an active Rx window!\n");
1294 		WARN_ON_ONCE(1);
1295 		return -EBUSY;
1296 	}
1297 
1298 	unmap_paste_region(window);
1299 
1300 	poll_window_busy_state(window);
1301 
1302 	unpin_close_window(window);
1303 
1304 	poll_window_credits(window);
1305 
1306 	clear_vinst_win(window);
1307 
1308 	poll_window_castout(window);
1309 
1310 	/* if send window, drop reference to matching receive window */
1311 	if (window->tx_win) {
1312 		if (window->user_win) {
1313 			put_vas_user_win_ref(&vwin->task_ref);
1314 			mm_context_remove_vas_window(vwin->task_ref.mm);
1315 		}
1316 		put_rx_win(window->rxwin);
1317 	}
1318 
1319 	vas_window_free(window);
1320 
1321 	return 0;
1322 }
1323 EXPORT_SYMBOL_GPL(vas_win_close);
1324 
1325 /*
1326  * Return credit for the given window.
1327  * Send windows and fault window uses credit mechanism as follows:
1328  *
1329  * Send windows:
1330  * - The default number of credits available for each send window is
1331  *   1024. It means 1024 requests can be issued asynchronously at the
1332  *   same time. If the credit is not available, that request will be
1333  *   returned with RMA_Busy.
1334  * - One credit is taken when NX request is issued.
1335  * - This credit is returned after NX processed that request.
1336  * - If NX encounters translation error, kernel will return the
1337  *   credit on the specific send window after processing the fault CRB.
1338  *
1339  * Fault window:
1340  * - The total number credits available is FIFO_SIZE/CRB_SIZE.
1341  *   Means 4MB/128 in the current implementation. If credit is not
1342  *   available, RMA_Reject is returned.
1343  * - A credit is taken when NX pastes CRB in fault FIFO.
1344  * - The kernel with return credit on fault window after reading entry
1345  *   from fault FIFO.
1346  */
1347 void vas_return_credit(struct pnv_vas_window *window, bool tx)
1348 {
1349 	uint64_t val;
1350 
1351 	val = 0ULL;
1352 	if (tx) { /* send window */
1353 		val = SET_FIELD(VAS_TX_WCRED, val, 1);
1354 		write_hvwc_reg(window, VREG(TX_WCRED_ADDER), val);
1355 	} else {
1356 		val = SET_FIELD(VAS_LRX_WCRED, val, 1);
1357 		write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), val);
1358 	}
1359 }
1360 
1361 struct pnv_vas_window *vas_pswid_to_window(struct vas_instance *vinst,
1362 		uint32_t pswid)
1363 {
1364 	struct pnv_vas_window *window;
1365 	int winid;
1366 
1367 	if (!pswid) {
1368 		pr_devel("%s: called for pswid 0!\n", __func__);
1369 		return ERR_PTR(-ESRCH);
1370 	}
1371 
1372 	decode_pswid(pswid, NULL, &winid);
1373 
1374 	if (winid >= VAS_WINDOWS_PER_CHIP)
1375 		return ERR_PTR(-ESRCH);
1376 
1377 	/*
1378 	 * If application closes the window before the hardware
1379 	 * returns the fault CRB, we should wait in vas_win_close()
1380 	 * for the pending requests. so the window must be active
1381 	 * and the process alive.
1382 	 *
1383 	 * If its a kernel process, we should not get any faults and
1384 	 * should not get here.
1385 	 */
1386 	window = vinst->windows[winid];
1387 
1388 	if (!window) {
1389 		pr_err("PSWID decode: Could not find window for winid %d pswid %d vinst 0x%p\n",
1390 			winid, pswid, vinst);
1391 		return NULL;
1392 	}
1393 
1394 	/*
1395 	 * Do some sanity checks on the decoded window.  Window should be
1396 	 * NX GZIP user send window. FTW windows should not incur faults
1397 	 * since their CRBs are ignored (not queued on FIFO or processed
1398 	 * by NX).
1399 	 */
1400 	if (!window->tx_win || !window->user_win || !window->nx_win ||
1401 			window->vas_win.cop == VAS_COP_TYPE_FAULT ||
1402 			window->vas_win.cop == VAS_COP_TYPE_FTW) {
1403 		pr_err("PSWID decode: id %d, tx %d, user %d, nx %d, cop %d\n",
1404 			winid, window->tx_win, window->user_win,
1405 			window->nx_win, window->vas_win.cop);
1406 		WARN_ON(1);
1407 	}
1408 
1409 	return window;
1410 }
1411 
1412 static struct vas_window *vas_user_win_open(int vas_id, u64 flags,
1413 				enum vas_cop_type cop_type)
1414 {
1415 	struct vas_tx_win_attr txattr = {};
1416 
1417 	vas_init_tx_win_attr(&txattr, cop_type);
1418 
1419 	txattr.lpid = mfspr(SPRN_LPID);
1420 	txattr.pidr = mfspr(SPRN_PID);
1421 	txattr.user_win = true;
1422 	txattr.rsvd_txbuf_count = false;
1423 	txattr.pswid = false;
1424 
1425 	pr_devel("Pid %d: Opening txwin, PIDR %ld\n", txattr.pidr,
1426 				mfspr(SPRN_PID));
1427 
1428 	return vas_tx_win_open(vas_id, cop_type, &txattr);
1429 }
1430 
1431 static u64 vas_user_win_paste_addr(struct vas_window *txwin)
1432 {
1433 	struct pnv_vas_window *win;
1434 	u64 paste_addr;
1435 
1436 	win = container_of(txwin, struct pnv_vas_window, vas_win);
1437 	vas_win_paste_addr(win, &paste_addr, NULL);
1438 
1439 	return paste_addr;
1440 }
1441 
1442 static int vas_user_win_close(struct vas_window *txwin)
1443 {
1444 	vas_win_close(txwin);
1445 
1446 	return 0;
1447 }
1448 
1449 static const struct vas_user_win_ops vops =  {
1450 	.open_win	=	vas_user_win_open,
1451 	.paste_addr	=	vas_user_win_paste_addr,
1452 	.close_win	=	vas_user_win_close,
1453 };
1454 
1455 /*
1456  * Supporting only nx-gzip coprocessor type now, but this API code
1457  * extended to other coprocessor types later.
1458  */
1459 int vas_register_api_powernv(struct module *mod, enum vas_cop_type cop_type,
1460 			     const char *name)
1461 {
1462 
1463 	return vas_register_coproc_api(mod, cop_type, name, &vops);
1464 }
1465 EXPORT_SYMBOL_GPL(vas_register_api_powernv);
1466 
1467 void vas_unregister_api_powernv(void)
1468 {
1469 	vas_unregister_coproc_api();
1470 }
1471 EXPORT_SYMBOL_GPL(vas_unregister_api_powernv);
1472