1647c734fSRaphael Moreira Zinsly // SPDX-License-Identifier: GPL-2.0-or-later
2647c734fSRaphael Moreira Zinsly
3647c734fSRaphael Moreira Zinsly /*
4647c734fSRaphael Moreira Zinsly * Copyright 2020 IBM Corp.
5647c734fSRaphael Moreira Zinsly *
6647c734fSRaphael Moreira Zinsly * Author: Bulent Abali <abali@us.ibm.com>
7647c734fSRaphael Moreira Zinsly *
8647c734fSRaphael Moreira Zinsly */
9647c734fSRaphael Moreira Zinsly #include <stdio.h>
10647c734fSRaphael Moreira Zinsly #include <stdlib.h>
11647c734fSRaphael Moreira Zinsly #include <string.h>
12647c734fSRaphael Moreira Zinsly #include <unistd.h>
13647c734fSRaphael Moreira Zinsly #include <stdint.h>
14647c734fSRaphael Moreira Zinsly #include <sys/types.h>
15647c734fSRaphael Moreira Zinsly #include <sys/stat.h>
16647c734fSRaphael Moreira Zinsly #include <sys/time.h>
17647c734fSRaphael Moreira Zinsly #include <sys/fcntl.h>
18647c734fSRaphael Moreira Zinsly #include <sys/mman.h>
19647c734fSRaphael Moreira Zinsly #include <endian.h>
20647c734fSRaphael Moreira Zinsly #include <bits/endian.h>
21647c734fSRaphael Moreira Zinsly #include <sys/ioctl.h>
22647c734fSRaphael Moreira Zinsly #include <assert.h>
23647c734fSRaphael Moreira Zinsly #include <errno.h>
24647c734fSRaphael Moreira Zinsly #include <signal.h>
25647c734fSRaphael Moreira Zinsly #include "vas-api.h"
26647c734fSRaphael Moreira Zinsly #include "nx.h"
27647c734fSRaphael Moreira Zinsly #include "copy-paste.h"
28647c734fSRaphael Moreira Zinsly #include "nxu.h"
29647c734fSRaphael Moreira Zinsly #include "nx_dbg.h"
30647c734fSRaphael Moreira Zinsly #include <sys/platform/ppc.h>
31647c734fSRaphael Moreira Zinsly
32647c734fSRaphael Moreira Zinsly #define barrier()
33647c734fSRaphael Moreira Zinsly #define hwsync() ({ asm volatile("sync" ::: "memory"); })
34647c734fSRaphael Moreira Zinsly
35647c734fSRaphael Moreira Zinsly #ifndef NX_NO_CPU_PRI
36647c734fSRaphael Moreira Zinsly #define cpu_pri_default() ({ asm volatile ("or 2, 2, 2"); })
37647c734fSRaphael Moreira Zinsly #define cpu_pri_low() ({ asm volatile ("or 31, 31, 31"); })
38647c734fSRaphael Moreira Zinsly #else
39647c734fSRaphael Moreira Zinsly #define cpu_pri_default()
40647c734fSRaphael Moreira Zinsly #define cpu_pri_low()
41647c734fSRaphael Moreira Zinsly #endif
42647c734fSRaphael Moreira Zinsly
43647c734fSRaphael Moreira Zinsly void *nx_fault_storage_address;
44647c734fSRaphael Moreira Zinsly
45647c734fSRaphael Moreira Zinsly struct nx_handle {
46647c734fSRaphael Moreira Zinsly int fd;
47647c734fSRaphael Moreira Zinsly int function;
48647c734fSRaphael Moreira Zinsly void *paste_addr;
49647c734fSRaphael Moreira Zinsly };
50647c734fSRaphael Moreira Zinsly
open_device_nodes(char * devname,int pri,struct nx_handle * handle)51647c734fSRaphael Moreira Zinsly static int open_device_nodes(char *devname, int pri, struct nx_handle *handle)
52647c734fSRaphael Moreira Zinsly {
53647c734fSRaphael Moreira Zinsly int rc, fd;
54647c734fSRaphael Moreira Zinsly void *addr;
55647c734fSRaphael Moreira Zinsly struct vas_tx_win_open_attr txattr;
56647c734fSRaphael Moreira Zinsly
57647c734fSRaphael Moreira Zinsly fd = open(devname, O_RDWR);
58647c734fSRaphael Moreira Zinsly if (fd < 0) {
59647c734fSRaphael Moreira Zinsly fprintf(stderr, " open device name %s\n", devname);
60647c734fSRaphael Moreira Zinsly return -errno;
61647c734fSRaphael Moreira Zinsly }
62647c734fSRaphael Moreira Zinsly
63647c734fSRaphael Moreira Zinsly memset(&txattr, 0, sizeof(txattr));
64647c734fSRaphael Moreira Zinsly txattr.version = 1;
65647c734fSRaphael Moreira Zinsly txattr.vas_id = pri;
66647c734fSRaphael Moreira Zinsly rc = ioctl(fd, VAS_TX_WIN_OPEN, (unsigned long)&txattr);
67647c734fSRaphael Moreira Zinsly if (rc < 0) {
68647c734fSRaphael Moreira Zinsly fprintf(stderr, "ioctl() n %d, error %d\n", rc, errno);
69647c734fSRaphael Moreira Zinsly rc = -errno;
70647c734fSRaphael Moreira Zinsly goto out;
71647c734fSRaphael Moreira Zinsly }
72647c734fSRaphael Moreira Zinsly
73647c734fSRaphael Moreira Zinsly addr = mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0ULL);
74647c734fSRaphael Moreira Zinsly if (addr == MAP_FAILED) {
75647c734fSRaphael Moreira Zinsly fprintf(stderr, "mmap() failed, errno %d\n", errno);
76647c734fSRaphael Moreira Zinsly rc = -errno;
77647c734fSRaphael Moreira Zinsly goto out;
78647c734fSRaphael Moreira Zinsly }
79647c734fSRaphael Moreira Zinsly handle->fd = fd;
80647c734fSRaphael Moreira Zinsly handle->paste_addr = (void *)((char *)addr + 0x400);
81647c734fSRaphael Moreira Zinsly
82647c734fSRaphael Moreira Zinsly rc = 0;
83647c734fSRaphael Moreira Zinsly out:
84647c734fSRaphael Moreira Zinsly close(fd);
85647c734fSRaphael Moreira Zinsly return rc;
86647c734fSRaphael Moreira Zinsly }
87647c734fSRaphael Moreira Zinsly
nx_function_begin(int function,int pri)88647c734fSRaphael Moreira Zinsly void *nx_function_begin(int function, int pri)
89647c734fSRaphael Moreira Zinsly {
90647c734fSRaphael Moreira Zinsly int rc;
91647c734fSRaphael Moreira Zinsly char *devname = "/dev/crypto/nx-gzip";
92647c734fSRaphael Moreira Zinsly struct nx_handle *nxhandle;
93647c734fSRaphael Moreira Zinsly
94647c734fSRaphael Moreira Zinsly if (function != NX_FUNC_COMP_GZIP) {
95647c734fSRaphael Moreira Zinsly errno = EINVAL;
96647c734fSRaphael Moreira Zinsly fprintf(stderr, " NX_FUNC_COMP_GZIP not found\n");
97647c734fSRaphael Moreira Zinsly return NULL;
98647c734fSRaphael Moreira Zinsly }
99647c734fSRaphael Moreira Zinsly
100647c734fSRaphael Moreira Zinsly
101647c734fSRaphael Moreira Zinsly nxhandle = malloc(sizeof(*nxhandle));
102647c734fSRaphael Moreira Zinsly if (!nxhandle) {
103647c734fSRaphael Moreira Zinsly errno = ENOMEM;
104647c734fSRaphael Moreira Zinsly fprintf(stderr, " No memory\n");
105647c734fSRaphael Moreira Zinsly return NULL;
106647c734fSRaphael Moreira Zinsly }
107647c734fSRaphael Moreira Zinsly
108647c734fSRaphael Moreira Zinsly nxhandle->function = function;
109647c734fSRaphael Moreira Zinsly rc = open_device_nodes(devname, pri, nxhandle);
110647c734fSRaphael Moreira Zinsly if (rc < 0) {
111647c734fSRaphael Moreira Zinsly errno = -rc;
112647c734fSRaphael Moreira Zinsly fprintf(stderr, " open_device_nodes failed\n");
113647c734fSRaphael Moreira Zinsly return NULL;
114647c734fSRaphael Moreira Zinsly }
115647c734fSRaphael Moreira Zinsly
116647c734fSRaphael Moreira Zinsly return nxhandle;
117647c734fSRaphael Moreira Zinsly }
118647c734fSRaphael Moreira Zinsly
nx_function_end(void * handle)119647c734fSRaphael Moreira Zinsly int nx_function_end(void *handle)
120647c734fSRaphael Moreira Zinsly {
121647c734fSRaphael Moreira Zinsly int rc = 0;
122647c734fSRaphael Moreira Zinsly struct nx_handle *nxhandle = handle;
123647c734fSRaphael Moreira Zinsly
124647c734fSRaphael Moreira Zinsly rc = munmap(nxhandle->paste_addr - 0x400, 4096);
125647c734fSRaphael Moreira Zinsly if (rc < 0) {
126647c734fSRaphael Moreira Zinsly fprintf(stderr, "munmap() failed, errno %d\n", errno);
127647c734fSRaphael Moreira Zinsly return rc;
128647c734fSRaphael Moreira Zinsly }
129647c734fSRaphael Moreira Zinsly close(nxhandle->fd);
130647c734fSRaphael Moreira Zinsly free(nxhandle);
131647c734fSRaphael Moreira Zinsly
132647c734fSRaphael Moreira Zinsly return rc;
133647c734fSRaphael Moreira Zinsly }
134647c734fSRaphael Moreira Zinsly
nx_wait_for_csb(struct nx_gzip_crb_cpb_t * cmdp)135647c734fSRaphael Moreira Zinsly static int nx_wait_for_csb(struct nx_gzip_crb_cpb_t *cmdp)
136647c734fSRaphael Moreira Zinsly {
137647c734fSRaphael Moreira Zinsly long poll = 0;
138647c734fSRaphael Moreira Zinsly uint64_t t;
139647c734fSRaphael Moreira Zinsly
140647c734fSRaphael Moreira Zinsly /* Save power and let other threads use the h/w. top may show
141647c734fSRaphael Moreira Zinsly * 100% but only because OS doesn't know we slowed the this
142647c734fSRaphael Moreira Zinsly * h/w thread while polling. We're letting other threads have
143647c734fSRaphael Moreira Zinsly * higher throughput on the core.
144647c734fSRaphael Moreira Zinsly */
145647c734fSRaphael Moreira Zinsly cpu_pri_low();
146647c734fSRaphael Moreira Zinsly
147647c734fSRaphael Moreira Zinsly #define CSB_MAX_POLL 200000000UL
148647c734fSRaphael Moreira Zinsly #define USLEEP_TH 300000UL
149647c734fSRaphael Moreira Zinsly
150647c734fSRaphael Moreira Zinsly t = __ppc_get_timebase();
151647c734fSRaphael Moreira Zinsly
152647c734fSRaphael Moreira Zinsly while (getnn(cmdp->crb.csb, csb_v) == 0) {
153647c734fSRaphael Moreira Zinsly ++poll;
154647c734fSRaphael Moreira Zinsly hwsync();
155647c734fSRaphael Moreira Zinsly
156647c734fSRaphael Moreira Zinsly cpu_pri_low();
157647c734fSRaphael Moreira Zinsly
158647c734fSRaphael Moreira Zinsly /* usleep(0) takes around 29000 ticks ~60 us.
159647c734fSRaphael Moreira Zinsly * 300000 is spinning for about 600 us then
160647c734fSRaphael Moreira Zinsly * start sleeping.
161647c734fSRaphael Moreira Zinsly */
162647c734fSRaphael Moreira Zinsly if ((__ppc_get_timebase() - t) > USLEEP_TH) {
163647c734fSRaphael Moreira Zinsly cpu_pri_default();
164647c734fSRaphael Moreira Zinsly usleep(1);
165647c734fSRaphael Moreira Zinsly }
166647c734fSRaphael Moreira Zinsly
167647c734fSRaphael Moreira Zinsly if (poll > CSB_MAX_POLL)
168647c734fSRaphael Moreira Zinsly break;
169647c734fSRaphael Moreira Zinsly
170647c734fSRaphael Moreira Zinsly /* Fault address from signal handler */
171647c734fSRaphael Moreira Zinsly if (nx_fault_storage_address) {
172647c734fSRaphael Moreira Zinsly cpu_pri_default();
173647c734fSRaphael Moreira Zinsly return -EAGAIN;
174647c734fSRaphael Moreira Zinsly }
175647c734fSRaphael Moreira Zinsly
176647c734fSRaphael Moreira Zinsly }
177647c734fSRaphael Moreira Zinsly
178647c734fSRaphael Moreira Zinsly cpu_pri_default();
179647c734fSRaphael Moreira Zinsly
180647c734fSRaphael Moreira Zinsly /* hw has updated csb and output buffer */
181647c734fSRaphael Moreira Zinsly hwsync();
182647c734fSRaphael Moreira Zinsly
183647c734fSRaphael Moreira Zinsly /* Check CSB flags. */
184647c734fSRaphael Moreira Zinsly if (getnn(cmdp->crb.csb, csb_v) == 0) {
185647c734fSRaphael Moreira Zinsly fprintf(stderr, "CSB still not valid after %d polls.\n",
186647c734fSRaphael Moreira Zinsly (int) poll);
187647c734fSRaphael Moreira Zinsly prt_err("CSB still not valid after %d polls, giving up.\n",
188647c734fSRaphael Moreira Zinsly (int) poll);
189647c734fSRaphael Moreira Zinsly return -ETIMEDOUT;
190647c734fSRaphael Moreira Zinsly }
191647c734fSRaphael Moreira Zinsly
192647c734fSRaphael Moreira Zinsly return 0;
193647c734fSRaphael Moreira Zinsly }
194647c734fSRaphael Moreira Zinsly
nxu_run_job(struct nx_gzip_crb_cpb_t * cmdp,void * handle)195647c734fSRaphael Moreira Zinsly static int nxu_run_job(struct nx_gzip_crb_cpb_t *cmdp, void *handle)
196647c734fSRaphael Moreira Zinsly {
197647c734fSRaphael Moreira Zinsly int i, ret, retries;
198647c734fSRaphael Moreira Zinsly struct nx_handle *nxhandle = handle;
199647c734fSRaphael Moreira Zinsly
200647c734fSRaphael Moreira Zinsly assert(handle != NULL);
201647c734fSRaphael Moreira Zinsly i = 0;
202647c734fSRaphael Moreira Zinsly retries = 5000;
203647c734fSRaphael Moreira Zinsly while (i++ < retries) {
204647c734fSRaphael Moreira Zinsly hwsync();
205647c734fSRaphael Moreira Zinsly vas_copy(&cmdp->crb, 0);
206647c734fSRaphael Moreira Zinsly ret = vas_paste(nxhandle->paste_addr, 0);
207647c734fSRaphael Moreira Zinsly hwsync();
208647c734fSRaphael Moreira Zinsly
209647c734fSRaphael Moreira Zinsly NXPRT(fprintf(stderr, "Paste attempt %d/%d returns 0x%x\n",
210647c734fSRaphael Moreira Zinsly i, retries, ret));
211647c734fSRaphael Moreira Zinsly
212647c734fSRaphael Moreira Zinsly if ((ret == 2) || (ret == 3)) {
213647c734fSRaphael Moreira Zinsly
214647c734fSRaphael Moreira Zinsly ret = nx_wait_for_csb(cmdp);
215647c734fSRaphael Moreira Zinsly if (!ret) {
216647c734fSRaphael Moreira Zinsly goto out;
217647c734fSRaphael Moreira Zinsly } else if (ret == -EAGAIN) {
218647c734fSRaphael Moreira Zinsly long x;
219647c734fSRaphael Moreira Zinsly
220647c734fSRaphael Moreira Zinsly prt_err("Touching address %p, 0x%lx\n",
221647c734fSRaphael Moreira Zinsly nx_fault_storage_address,
222647c734fSRaphael Moreira Zinsly *(long *) nx_fault_storage_address);
223647c734fSRaphael Moreira Zinsly x = *(long *) nx_fault_storage_address;
224647c734fSRaphael Moreira Zinsly *(long *) nx_fault_storage_address = x;
225647c734fSRaphael Moreira Zinsly nx_fault_storage_address = 0;
226647c734fSRaphael Moreira Zinsly continue;
227647c734fSRaphael Moreira Zinsly } else {
228647c734fSRaphael Moreira Zinsly prt_err("wait_for_csb() returns %d\n", ret);
229647c734fSRaphael Moreira Zinsly break;
230647c734fSRaphael Moreira Zinsly }
231647c734fSRaphael Moreira Zinsly } else {
232647c734fSRaphael Moreira Zinsly if (i < 10) {
233647c734fSRaphael Moreira Zinsly /* spin for few ticks */
234647c734fSRaphael Moreira Zinsly #define SPIN_TH 500UL
235647c734fSRaphael Moreira Zinsly uint64_t fail_spin;
236647c734fSRaphael Moreira Zinsly
237647c734fSRaphael Moreira Zinsly fail_spin = __ppc_get_timebase();
238647c734fSRaphael Moreira Zinsly while ((__ppc_get_timebase() - fail_spin) <
239647c734fSRaphael Moreira Zinsly SPIN_TH)
240647c734fSRaphael Moreira Zinsly ;
241647c734fSRaphael Moreira Zinsly } else {
242647c734fSRaphael Moreira Zinsly /* sleep */
243647c734fSRaphael Moreira Zinsly unsigned int pr = 0;
244647c734fSRaphael Moreira Zinsly
245647c734fSRaphael Moreira Zinsly if (pr++ % 100 == 0) {
246647c734fSRaphael Moreira Zinsly prt_err("Paste attempt %d/", i);
247647c734fSRaphael Moreira Zinsly prt_err("%d, failed pid= %d\n", retries,
248647c734fSRaphael Moreira Zinsly getpid());
249647c734fSRaphael Moreira Zinsly }
250647c734fSRaphael Moreira Zinsly usleep(1);
251647c734fSRaphael Moreira Zinsly }
252647c734fSRaphael Moreira Zinsly continue;
253647c734fSRaphael Moreira Zinsly }
254647c734fSRaphael Moreira Zinsly }
255647c734fSRaphael Moreira Zinsly
256647c734fSRaphael Moreira Zinsly out:
257647c734fSRaphael Moreira Zinsly cpu_pri_default();
258647c734fSRaphael Moreira Zinsly
259647c734fSRaphael Moreira Zinsly return ret;
260647c734fSRaphael Moreira Zinsly }
261647c734fSRaphael Moreira Zinsly
nxu_submit_job(struct nx_gzip_crb_cpb_t * cmdp,void * handle)262647c734fSRaphael Moreira Zinsly int nxu_submit_job(struct nx_gzip_crb_cpb_t *cmdp, void *handle)
263647c734fSRaphael Moreira Zinsly {
264647c734fSRaphael Moreira Zinsly int cc;
265647c734fSRaphael Moreira Zinsly
266647c734fSRaphael Moreira Zinsly cc = nxu_run_job(cmdp, handle);
267647c734fSRaphael Moreira Zinsly
268647c734fSRaphael Moreira Zinsly if (!cc)
269647c734fSRaphael Moreira Zinsly cc = getnn(cmdp->crb.csb, csb_cc); /* CC Table 6-8 */
270647c734fSRaphael Moreira Zinsly
271647c734fSRaphael Moreira Zinsly return cc;
272647c734fSRaphael Moreira Zinsly }
273647c734fSRaphael Moreira Zinsly
274647c734fSRaphael Moreira Zinsly
nxu_sigsegv_handler(int sig,siginfo_t * info,void * ctx)275647c734fSRaphael Moreira Zinsly void nxu_sigsegv_handler(int sig, siginfo_t *info, void *ctx)
276647c734fSRaphael Moreira Zinsly {
277647c734fSRaphael Moreira Zinsly fprintf(stderr, "%d: Got signal %d si_code %d, si_addr %p\n", getpid(),
278647c734fSRaphael Moreira Zinsly sig, info->si_code, info->si_addr);
279647c734fSRaphael Moreira Zinsly
280647c734fSRaphael Moreira Zinsly nx_fault_storage_address = info->si_addr;
281647c734fSRaphael Moreira Zinsly }
282647c734fSRaphael Moreira Zinsly
283647c734fSRaphael Moreira Zinsly /*
284647c734fSRaphael Moreira Zinsly * Fault in pages prior to NX job submission. wr=1 may be required to
285647c734fSRaphael Moreira Zinsly * touch writeable pages. System zero pages do not fault-in the page as
286647c734fSRaphael Moreira Zinsly * intended. Typically set wr=1 for NX target pages and set wr=0 for NX
287647c734fSRaphael Moreira Zinsly * source pages.
288647c734fSRaphael Moreira Zinsly */
nxu_touch_pages(void * buf,long buf_len,long page_len,int wr)289647c734fSRaphael Moreira Zinsly int nxu_touch_pages(void *buf, long buf_len, long page_len, int wr)
290647c734fSRaphael Moreira Zinsly {
291647c734fSRaphael Moreira Zinsly char *begin = buf;
292647c734fSRaphael Moreira Zinsly char *end = (char *) buf + buf_len - 1;
293647c734fSRaphael Moreira Zinsly volatile char t;
294647c734fSRaphael Moreira Zinsly
295647c734fSRaphael Moreira Zinsly assert(buf_len >= 0 && !!buf);
296647c734fSRaphael Moreira Zinsly
297647c734fSRaphael Moreira Zinsly NXPRT(fprintf(stderr, "touch %p %p len 0x%lx wr=%d\n", buf,
298647c734fSRaphael Moreira Zinsly (buf + buf_len), buf_len, wr));
299647c734fSRaphael Moreira Zinsly
300647c734fSRaphael Moreira Zinsly if (buf_len <= 0 || buf == NULL)
301647c734fSRaphael Moreira Zinsly return -1;
302647c734fSRaphael Moreira Zinsly
303647c734fSRaphael Moreira Zinsly do {
304647c734fSRaphael Moreira Zinsly t = *begin;
305647c734fSRaphael Moreira Zinsly if (wr)
306647c734fSRaphael Moreira Zinsly *begin = t;
307647c734fSRaphael Moreira Zinsly begin = begin + page_len;
308647c734fSRaphael Moreira Zinsly } while (begin < end);
309647c734fSRaphael Moreira Zinsly
310647c734fSRaphael Moreira Zinsly /* When buf_sz is small or buf tail is in another page */
311647c734fSRaphael Moreira Zinsly t = *end;
312647c734fSRaphael Moreira Zinsly if (wr)
313647c734fSRaphael Moreira Zinsly *end = t;
314647c734fSRaphael Moreira Zinsly
315647c734fSRaphael Moreira Zinsly return 0;
316647c734fSRaphael Moreira Zinsly }
317