1 // SPDX-License-Identifier: GPL-2.0-or-later
2 
3 /*
4  * Copyright 2020 IBM Corp.
5  *
6  * Author: Bulent Abali <abali@us.ibm.com>
7  *
8  */
9 #include <stdio.h>
10 #include <stdlib.h>
11 #include <string.h>
12 #include <unistd.h>
13 #include <stdint.h>
14 #include <sys/types.h>
15 #include <sys/stat.h>
16 #include <sys/time.h>
17 #include <sys/fcntl.h>
18 #include <sys/mman.h>
19 #include <endian.h>
20 #include <bits/endian.h>
21 #include <sys/ioctl.h>
22 #include <assert.h>
23 #include <errno.h>
24 #include <signal.h>
25 #include "vas-api.h"
26 #include "nx.h"
27 #include "copy-paste.h"
28 #include "nxu.h"
29 #include "nx_dbg.h"
30 #include <sys/platform/ppc.h>
31 
32 #define barrier()
33 #define hwsync()    ({ asm volatile("sync" ::: "memory"); })
34 
35 #ifndef NX_NO_CPU_PRI
36 #define cpu_pri_default()  ({ asm volatile ("or 2, 2, 2"); })
37 #define cpu_pri_low()      ({ asm volatile ("or 31, 31, 31"); })
38 #else
39 #define cpu_pri_default()
40 #define cpu_pri_low()
41 #endif
42 
43 void *nx_fault_storage_address;
44 
45 struct nx_handle {
46 	int fd;
47 	int function;
48 	void *paste_addr;
49 };
50 
51 static int open_device_nodes(char *devname, int pri, struct nx_handle *handle)
52 {
53 	int rc, fd;
54 	void *addr;
55 	struct vas_tx_win_open_attr txattr;
56 
57 	fd = open(devname, O_RDWR);
58 	if (fd < 0) {
59 		fprintf(stderr, " open device name %s\n", devname);
60 		return -errno;
61 	}
62 
63 	memset(&txattr, 0, sizeof(txattr));
64 	txattr.version = 1;
65 	txattr.vas_id = pri;
66 	rc = ioctl(fd, VAS_TX_WIN_OPEN, (unsigned long)&txattr);
67 	if (rc < 0) {
68 		fprintf(stderr, "ioctl() n %d, error %d\n", rc, errno);
69 		rc = -errno;
70 		goto out;
71 	}
72 
73 	addr = mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0ULL);
74 	if (addr == MAP_FAILED) {
75 		fprintf(stderr, "mmap() failed, errno %d\n", errno);
76 		rc = -errno;
77 		goto out;
78 	}
79 	handle->fd = fd;
80 	handle->paste_addr = (void *)((char *)addr + 0x400);
81 
82 	rc = 0;
83 out:
84 	close(fd);
85 	return rc;
86 }
87 
88 void *nx_function_begin(int function, int pri)
89 {
90 	int rc;
91 	char *devname = "/dev/crypto/nx-gzip";
92 	struct nx_handle *nxhandle;
93 
94 	if (function != NX_FUNC_COMP_GZIP) {
95 		errno = EINVAL;
96 		fprintf(stderr, " NX_FUNC_COMP_GZIP not found\n");
97 		return NULL;
98 	}
99 
100 
101 	nxhandle = malloc(sizeof(*nxhandle));
102 	if (!nxhandle) {
103 		errno = ENOMEM;
104 		fprintf(stderr, " No memory\n");
105 		return NULL;
106 	}
107 
108 	nxhandle->function = function;
109 	rc = open_device_nodes(devname, pri, nxhandle);
110 	if (rc < 0) {
111 		errno = -rc;
112 		fprintf(stderr, " open_device_nodes failed\n");
113 		return NULL;
114 	}
115 
116 	return nxhandle;
117 }
118 
119 int nx_function_end(void *handle)
120 {
121 	int rc = 0;
122 	struct nx_handle *nxhandle = handle;
123 
124 	rc = munmap(nxhandle->paste_addr - 0x400, 4096);
125 	if (rc < 0) {
126 		fprintf(stderr, "munmap() failed, errno %d\n", errno);
127 		return rc;
128 	}
129 	close(nxhandle->fd);
130 	free(nxhandle);
131 
132 	return rc;
133 }
134 
135 static int nx_wait_for_csb(struct nx_gzip_crb_cpb_t *cmdp)
136 {
137 	long poll = 0;
138 	uint64_t t;
139 
140 	/* Save power and let other threads use the h/w. top may show
141 	 * 100% but only because OS doesn't know we slowed the this
142 	 * h/w thread while polling. We're letting other threads have
143 	 * higher throughput on the core.
144 	 */
145 	cpu_pri_low();
146 
147 #define CSB_MAX_POLL 200000000UL
148 #define USLEEP_TH     300000UL
149 
150 	t = __ppc_get_timebase();
151 
152 	while (getnn(cmdp->crb.csb, csb_v) == 0) {
153 		++poll;
154 		hwsync();
155 
156 		cpu_pri_low();
157 
158 		/* usleep(0) takes around 29000 ticks ~60 us.
159 		 * 300000 is spinning for about 600 us then
160 		 * start sleeping.
161 		 */
162 		if ((__ppc_get_timebase() - t) > USLEEP_TH) {
163 			cpu_pri_default();
164 			usleep(1);
165 		}
166 
167 		if (poll > CSB_MAX_POLL)
168 			break;
169 
170 		/* Fault address from signal handler */
171 		if (nx_fault_storage_address) {
172 			cpu_pri_default();
173 			return -EAGAIN;
174 		}
175 
176 	}
177 
178 	cpu_pri_default();
179 
180 	/* hw has updated csb and output buffer */
181 	hwsync();
182 
183 	/* Check CSB flags. */
184 	if (getnn(cmdp->crb.csb, csb_v) == 0) {
185 		fprintf(stderr, "CSB still not valid after %d polls.\n",
186 			(int) poll);
187 		prt_err("CSB still not valid after %d polls, giving up.\n",
188 			(int) poll);
189 		return -ETIMEDOUT;
190 	}
191 
192 	return 0;
193 }
194 
195 static int nxu_run_job(struct nx_gzip_crb_cpb_t *cmdp, void *handle)
196 {
197 	int i, ret, retries;
198 	struct nx_handle *nxhandle = handle;
199 
200 	assert(handle != NULL);
201 	i = 0;
202 	retries = 5000;
203 	while (i++ < retries) {
204 		hwsync();
205 		vas_copy(&cmdp->crb, 0);
206 		ret = vas_paste(nxhandle->paste_addr, 0);
207 		hwsync();
208 
209 		NXPRT(fprintf(stderr, "Paste attempt %d/%d returns 0x%x\n",
210 				i, retries, ret));
211 
212 		if ((ret == 2) || (ret == 3)) {
213 
214 			ret = nx_wait_for_csb(cmdp);
215 			if (!ret) {
216 				goto out;
217 			} else if (ret == -EAGAIN) {
218 				long x;
219 
220 				prt_err("Touching address %p, 0x%lx\n",
221 					 nx_fault_storage_address,
222 					 *(long *) nx_fault_storage_address);
223 				x = *(long *) nx_fault_storage_address;
224 				*(long *) nx_fault_storage_address = x;
225 				nx_fault_storage_address = 0;
226 				continue;
227 			} else {
228 				prt_err("wait_for_csb() returns %d\n", ret);
229 				break;
230 			}
231 		} else {
232 			if (i < 10) {
233 				/* spin for few ticks */
234 #define SPIN_TH 500UL
235 				uint64_t fail_spin;
236 
237 				fail_spin = __ppc_get_timebase();
238 				while ((__ppc_get_timebase() - fail_spin) <
239 					 SPIN_TH)
240 					;
241 			} else {
242 				/* sleep */
243 				unsigned int pr = 0;
244 
245 				if (pr++ % 100 == 0) {
246 					prt_err("Paste attempt %d/", i);
247 					prt_err("%d, failed pid= %d\n", retries,
248 						getpid());
249 				}
250 				usleep(1);
251 			}
252 			continue;
253 		}
254 	}
255 
256 out:
257 	cpu_pri_default();
258 
259 	return ret;
260 }
261 
262 int nxu_submit_job(struct nx_gzip_crb_cpb_t *cmdp, void *handle)
263 {
264 	int cc;
265 
266 	cc = nxu_run_job(cmdp, handle);
267 
268 	if (!cc)
269 		cc = getnn(cmdp->crb.csb, csb_cc);      /* CC Table 6-8 */
270 
271 	return cc;
272 }
273 
274 
275 void nxu_sigsegv_handler(int sig, siginfo_t *info, void *ctx)
276 {
277 	fprintf(stderr, "%d: Got signal %d si_code %d, si_addr %p\n", getpid(),
278 		sig, info->si_code, info->si_addr);
279 
280 	nx_fault_storage_address = info->si_addr;
281 }
282 
283 /*
284  * Fault in pages prior to NX job submission.  wr=1 may be required to
285  * touch writeable pages.  System zero pages do not fault-in the page as
286  * intended.  Typically set wr=1 for NX target pages and set wr=0 for NX
287  * source pages.
288  */
289 int nxu_touch_pages(void *buf, long buf_len, long page_len, int wr)
290 {
291 	char *begin = buf;
292 	char *end = (char *) buf + buf_len - 1;
293 	volatile char t;
294 
295 	assert(buf_len >= 0 && !!buf);
296 
297 	NXPRT(fprintf(stderr, "touch %p %p len 0x%lx wr=%d\n", buf,
298 			(buf + buf_len), buf_len, wr));
299 
300 	if (buf_len <= 0 || buf == NULL)
301 		return -1;
302 
303 	do {
304 		t = *begin;
305 		if (wr)
306 			*begin = t;
307 		begin = begin + page_len;
308 	} while (begin < end);
309 
310 	/* When buf_sz is small or buf tail is in another page */
311 	t = *end;
312 	if (wr)
313 		*end = t;
314 
315 	return 0;
316 }
317