1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Driver for IBM Power 842 compression accelerator
4 *
5 * Copyright (C) IBM Corporation, 2012
6 *
7 * Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
8 * Seth Jennings <sjenning@linux.vnet.ibm.com>
9 */
10
11 #include <asm/vio.h>
12 #include <asm/hvcall.h>
13 #include <asm/vas.h>
14
15 #include "nx-842.h"
16 #include "nx_csbcpb.h" /* struct nx_csbcpb */
17
18 MODULE_LICENSE("GPL");
19 MODULE_AUTHOR("Robert Jennings <rcj@linux.vnet.ibm.com>");
20 MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors");
21 MODULE_ALIAS_CRYPTO("842");
22 MODULE_ALIAS_CRYPTO("842-nx");
23
24 /*
25 * Coprocessor type specific capabilities from the hypervisor.
26 */
27 struct hv_nx_cop_caps {
28 __be64 descriptor;
29 __be64 req_max_processed_len; /* Max bytes in one GZIP request */
30 __be64 min_compress_len; /* Min compression size in bytes */
31 __be64 min_decompress_len; /* Min decompression size in bytes */
32 } __packed __aligned(0x1000);
33
34 /*
35 * Coprocessor type specific capabilities.
36 */
37 struct nx_cop_caps {
38 u64 descriptor;
39 u64 req_max_processed_len; /* Max bytes in one GZIP request */
40 u64 min_compress_len; /* Min compression in bytes */
41 u64 min_decompress_len; /* Min decompression in bytes */
42 };
43
44 static u64 caps_feat;
45 static struct nx_cop_caps nx_cop_caps;
46
47 static struct nx842_constraints nx842_pseries_constraints = {
48 .alignment = DDE_BUFFER_ALIGN,
49 .multiple = DDE_BUFFER_LAST_MULT,
50 .minimum = DDE_BUFFER_LAST_MULT,
51 .maximum = PAGE_SIZE, /* dynamic, max_sync_size */
52 };
53
check_constraints(unsigned long buf,unsigned int * len,bool in)54 static int check_constraints(unsigned long buf, unsigned int *len, bool in)
55 {
56 if (!IS_ALIGNED(buf, nx842_pseries_constraints.alignment)) {
57 pr_debug("%s buffer 0x%lx not aligned to 0x%x\n",
58 in ? "input" : "output", buf,
59 nx842_pseries_constraints.alignment);
60 return -EINVAL;
61 }
62 if (*len % nx842_pseries_constraints.multiple) {
63 pr_debug("%s buffer len 0x%x not multiple of 0x%x\n",
64 in ? "input" : "output", *len,
65 nx842_pseries_constraints.multiple);
66 if (in)
67 return -EINVAL;
68 *len = round_down(*len, nx842_pseries_constraints.multiple);
69 }
70 if (*len < nx842_pseries_constraints.minimum) {
71 pr_debug("%s buffer len 0x%x under minimum 0x%x\n",
72 in ? "input" : "output", *len,
73 nx842_pseries_constraints.minimum);
74 return -EINVAL;
75 }
76 if (*len > nx842_pseries_constraints.maximum) {
77 pr_debug("%s buffer len 0x%x over maximum 0x%x\n",
78 in ? "input" : "output", *len,
79 nx842_pseries_constraints.maximum);
80 if (in)
81 return -EINVAL;
82 *len = nx842_pseries_constraints.maximum;
83 }
84 return 0;
85 }
86
87 /* I assume we need to align the CSB? */
88 #define WORKMEM_ALIGN (256)
89
90 struct nx842_workmem {
91 /* scatterlist */
92 char slin[4096];
93 char slout[4096];
94 /* coprocessor status/parameter block */
95 struct nx_csbcpb csbcpb;
96
97 char padding[WORKMEM_ALIGN];
98 } __aligned(WORKMEM_ALIGN);
99
100 /* Macros for fields within nx_csbcpb */
101 /* Check the valid bit within the csbcpb valid field */
102 #define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7))
103
104 /* CE macros operate on the completion_extension field bits in the csbcpb.
105 * CE0 0=full completion, 1=partial completion
106 * CE1 0=CE0 indicates completion, 1=termination (output may be modified)
107 * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */
108 #define NX842_CSBCPB_CE0(x) (x & BIT_MASK(7))
109 #define NX842_CSBCPB_CE1(x) (x & BIT_MASK(6))
110 #define NX842_CSBCPB_CE2(x) (x & BIT_MASK(5))
111
112 /* The NX unit accepts data only on 4K page boundaries */
113 #define NX842_HW_PAGE_SIZE (4096)
114 #define NX842_HW_PAGE_MASK (~(NX842_HW_PAGE_SIZE-1))
115
116 struct ibm_nx842_counters {
117 atomic64_t comp_complete;
118 atomic64_t comp_failed;
119 atomic64_t decomp_complete;
120 atomic64_t decomp_failed;
121 atomic64_t swdecomp;
122 atomic64_t comp_times[32];
123 atomic64_t decomp_times[32];
124 };
125
126 struct nx842_devdata {
127 struct vio_dev *vdev;
128 struct device *dev;
129 struct ibm_nx842_counters *counters;
130 unsigned int max_sg_len;
131 unsigned int max_sync_size;
132 unsigned int max_sync_sg;
133 };
134
135 static struct nx842_devdata __rcu *devdata;
136 static DEFINE_SPINLOCK(devdata_mutex);
137
138 #define NX842_COUNTER_INC(_x) \
139 static inline void nx842_inc_##_x( \
140 const struct nx842_devdata *dev) { \
141 if (dev) \
142 atomic64_inc(&dev->counters->_x); \
143 }
144 NX842_COUNTER_INC(comp_complete);
145 NX842_COUNTER_INC(comp_failed);
146 NX842_COUNTER_INC(decomp_complete);
147 NX842_COUNTER_INC(decomp_failed);
148 NX842_COUNTER_INC(swdecomp);
149
150 #define NX842_HIST_SLOTS 16
151
ibm_nx842_incr_hist(atomic64_t * times,unsigned int time)152 static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time)
153 {
154 int bucket = fls(time);
155
156 if (bucket)
157 bucket = min((NX842_HIST_SLOTS - 1), bucket - 1);
158
159 atomic64_inc(×[bucket]);
160 }
161
162 /* NX unit operation flags */
163 #define NX842_OP_COMPRESS 0x0
164 #define NX842_OP_CRC 0x1
165 #define NX842_OP_DECOMPRESS 0x2
166 #define NX842_OP_COMPRESS_CRC (NX842_OP_COMPRESS | NX842_OP_CRC)
167 #define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC)
168 #define NX842_OP_ASYNC (1<<23)
169 #define NX842_OP_NOTIFY (1<<22)
170 #define NX842_OP_NOTIFY_INT(x) ((x & 0xff)<<8)
171
nx842_get_desired_dma(struct vio_dev * viodev)172 static unsigned long nx842_get_desired_dma(struct vio_dev *viodev)
173 {
174 /* No use of DMA mappings within the driver. */
175 return 0;
176 }
177
178 struct nx842_slentry {
179 __be64 ptr; /* Real address (use __pa()) */
180 __be64 len;
181 };
182
183 /* pHyp scatterlist entry */
184 struct nx842_scatterlist {
185 int entry_nr; /* number of slentries */
186 struct nx842_slentry *entries; /* ptr to array of slentries */
187 };
188
189 /* Does not include sizeof(entry_nr) in the size */
nx842_get_scatterlist_size(struct nx842_scatterlist * sl)190 static inline unsigned long nx842_get_scatterlist_size(
191 struct nx842_scatterlist *sl)
192 {
193 return sl->entry_nr * sizeof(struct nx842_slentry);
194 }
195
nx842_build_scatterlist(unsigned long buf,int len,struct nx842_scatterlist * sl)196 static int nx842_build_scatterlist(unsigned long buf, int len,
197 struct nx842_scatterlist *sl)
198 {
199 unsigned long entrylen;
200 struct nx842_slentry *entry;
201
202 sl->entry_nr = 0;
203
204 entry = sl->entries;
205 while (len) {
206 entry->ptr = cpu_to_be64(nx842_get_pa((void *)buf));
207 entrylen = min_t(int, len,
208 LEN_ON_SIZE(buf, NX842_HW_PAGE_SIZE));
209 entry->len = cpu_to_be64(entrylen);
210
211 len -= entrylen;
212 buf += entrylen;
213
214 sl->entry_nr++;
215 entry++;
216 }
217
218 return 0;
219 }
220
nx842_validate_result(struct device * dev,struct cop_status_block * csb)221 static int nx842_validate_result(struct device *dev,
222 struct cop_status_block *csb)
223 {
224 /* The csb must be valid after returning from vio_h_cop_sync */
225 if (!NX842_CSBCBP_VALID_CHK(csb->valid)) {
226 dev_err(dev, "%s: cspcbp not valid upon completion.\n",
227 __func__);
228 dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n",
229 csb->valid,
230 csb->crb_seq_number,
231 csb->completion_code,
232 csb->completion_extension);
233 dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n",
234 be32_to_cpu(csb->processed_byte_count),
235 (unsigned long)be64_to_cpu(csb->address));
236 return -EIO;
237 }
238
239 /* Check return values from the hardware in the CSB */
240 switch (csb->completion_code) {
241 case 0: /* Completed without error */
242 break;
243 case 64: /* Compression ok, but output larger than input */
244 dev_dbg(dev, "%s: output size larger than input size\n",
245 __func__);
246 break;
247 case 13: /* Output buffer too small */
248 dev_dbg(dev, "%s: Out of space in output buffer\n",
249 __func__);
250 return -ENOSPC;
251 case 65: /* Calculated CRC doesn't match the passed value */
252 dev_dbg(dev, "%s: CRC mismatch for decompression\n",
253 __func__);
254 return -EINVAL;
255 case 66: /* Input data contains an illegal template field */
256 case 67: /* Template indicates data past the end of the input stream */
257 dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n",
258 __func__, csb->completion_code);
259 return -EINVAL;
260 default:
261 dev_dbg(dev, "%s: Unspecified error (code:%d)\n",
262 __func__, csb->completion_code);
263 return -EIO;
264 }
265
266 /* Hardware sanity check */
267 if (!NX842_CSBCPB_CE2(csb->completion_extension)) {
268 dev_err(dev, "%s: No error returned by hardware, but "
269 "data returned is unusable, contact support.\n"
270 "(Additional info: csbcbp->processed bytes "
271 "does not specify processed bytes for the "
272 "target buffer.)\n", __func__);
273 return -EIO;
274 }
275
276 return 0;
277 }
278
279 /**
280 * nx842_pseries_compress - Compress data using the 842 algorithm
281 *
282 * Compression provide by the NX842 coprocessor on IBM Power systems.
283 * The input buffer is compressed and the result is stored in the
284 * provided output buffer.
285 *
286 * Upon return from this function @outlen contains the length of the
287 * compressed data. If there is an error then @outlen will be 0 and an
288 * error will be specified by the return code from this function.
289 *
290 * @in: Pointer to input buffer
291 * @inlen: Length of input buffer
292 * @out: Pointer to output buffer
293 * @outlen: Length of output buffer
294 * @wmem: ptr to buffer for working memory, size determined by
295 * nx842_pseries_driver.workmem_size
296 *
297 * Returns:
298 * 0 Success, output of length @outlen stored in the buffer at @out
299 * -ENOMEM Unable to allocate internal buffers
300 * -ENOSPC Output buffer is to small
301 * -EIO Internal error
302 * -ENODEV Hardware unavailable
303 */
nx842_pseries_compress(const unsigned char * in,unsigned int inlen,unsigned char * out,unsigned int * outlen,void * wmem)304 static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen,
305 unsigned char *out, unsigned int *outlen,
306 void *wmem)
307 {
308 struct nx842_devdata *local_devdata;
309 struct device *dev = NULL;
310 struct nx842_workmem *workmem;
311 struct nx842_scatterlist slin, slout;
312 struct nx_csbcpb *csbcpb;
313 int ret = 0;
314 unsigned long inbuf, outbuf;
315 struct vio_pfo_op op = {
316 .done = NULL,
317 .handle = 0,
318 .timeout = 0,
319 };
320 unsigned long start = get_tb();
321
322 inbuf = (unsigned long)in;
323 if (check_constraints(inbuf, &inlen, true))
324 return -EINVAL;
325
326 outbuf = (unsigned long)out;
327 if (check_constraints(outbuf, outlen, false))
328 return -EINVAL;
329
330 rcu_read_lock();
331 local_devdata = rcu_dereference(devdata);
332 if (!local_devdata || !local_devdata->dev) {
333 rcu_read_unlock();
334 return -ENODEV;
335 }
336 dev = local_devdata->dev;
337
338 /* Init scatterlist */
339 workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
340 slin.entries = (struct nx842_slentry *)workmem->slin;
341 slout.entries = (struct nx842_slentry *)workmem->slout;
342
343 /* Init operation */
344 op.flags = NX842_OP_COMPRESS_CRC;
345 csbcpb = &workmem->csbcpb;
346 memset(csbcpb, 0, sizeof(*csbcpb));
347 op.csbcpb = nx842_get_pa(csbcpb);
348
349 if ((inbuf & NX842_HW_PAGE_MASK) ==
350 ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
351 /* Create direct DDE */
352 op.in = nx842_get_pa((void *)inbuf);
353 op.inlen = inlen;
354 } else {
355 /* Create indirect DDE (scatterlist) */
356 nx842_build_scatterlist(inbuf, inlen, &slin);
357 op.in = nx842_get_pa(slin.entries);
358 op.inlen = -nx842_get_scatterlist_size(&slin);
359 }
360
361 if ((outbuf & NX842_HW_PAGE_MASK) ==
362 ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
363 /* Create direct DDE */
364 op.out = nx842_get_pa((void *)outbuf);
365 op.outlen = *outlen;
366 } else {
367 /* Create indirect DDE (scatterlist) */
368 nx842_build_scatterlist(outbuf, *outlen, &slout);
369 op.out = nx842_get_pa(slout.entries);
370 op.outlen = -nx842_get_scatterlist_size(&slout);
371 }
372
373 dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
374 __func__, (unsigned long)op.in, (long)op.inlen,
375 (unsigned long)op.out, (long)op.outlen);
376
377 /* Send request to pHyp */
378 ret = vio_h_cop_sync(local_devdata->vdev, &op);
379
380 /* Check for pHyp error */
381 if (ret) {
382 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
383 __func__, ret, op.hcall_err);
384 ret = -EIO;
385 goto unlock;
386 }
387
388 /* Check for hardware error */
389 ret = nx842_validate_result(dev, &csbcpb->csb);
390 if (ret)
391 goto unlock;
392
393 *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
394 dev_dbg(dev, "%s: processed_bytes=%d\n", __func__, *outlen);
395
396 unlock:
397 if (ret)
398 nx842_inc_comp_failed(local_devdata);
399 else {
400 nx842_inc_comp_complete(local_devdata);
401 ibm_nx842_incr_hist(local_devdata->counters->comp_times,
402 (get_tb() - start) / tb_ticks_per_usec);
403 }
404 rcu_read_unlock();
405 return ret;
406 }
407
408 /**
409 * nx842_pseries_decompress - Decompress data using the 842 algorithm
410 *
411 * Decompression provide by the NX842 coprocessor on IBM Power systems.
412 * The input buffer is decompressed and the result is stored in the
413 * provided output buffer. The size allocated to the output buffer is
414 * provided by the caller of this function in @outlen. Upon return from
415 * this function @outlen contains the length of the decompressed data.
416 * If there is an error then @outlen will be 0 and an error will be
417 * specified by the return code from this function.
418 *
419 * @in: Pointer to input buffer
420 * @inlen: Length of input buffer
421 * @out: Pointer to output buffer
422 * @outlen: Length of output buffer
423 * @wmem: ptr to buffer for working memory, size determined by
424 * nx842_pseries_driver.workmem_size
425 *
426 * Returns:
427 * 0 Success, output of length @outlen stored in the buffer at @out
428 * -ENODEV Hardware decompression device is unavailable
429 * -ENOMEM Unable to allocate internal buffers
430 * -ENOSPC Output buffer is to small
431 * -EINVAL Bad input data encountered when attempting decompress
432 * -EIO Internal error
433 */
nx842_pseries_decompress(const unsigned char * in,unsigned int inlen,unsigned char * out,unsigned int * outlen,void * wmem)434 static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen,
435 unsigned char *out, unsigned int *outlen,
436 void *wmem)
437 {
438 struct nx842_devdata *local_devdata;
439 struct device *dev = NULL;
440 struct nx842_workmem *workmem;
441 struct nx842_scatterlist slin, slout;
442 struct nx_csbcpb *csbcpb;
443 int ret = 0;
444 unsigned long inbuf, outbuf;
445 struct vio_pfo_op op = {
446 .done = NULL,
447 .handle = 0,
448 .timeout = 0,
449 };
450 unsigned long start = get_tb();
451
452 /* Ensure page alignment and size */
453 inbuf = (unsigned long)in;
454 if (check_constraints(inbuf, &inlen, true))
455 return -EINVAL;
456
457 outbuf = (unsigned long)out;
458 if (check_constraints(outbuf, outlen, false))
459 return -EINVAL;
460
461 rcu_read_lock();
462 local_devdata = rcu_dereference(devdata);
463 if (!local_devdata || !local_devdata->dev) {
464 rcu_read_unlock();
465 return -ENODEV;
466 }
467 dev = local_devdata->dev;
468
469 workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
470
471 /* Init scatterlist */
472 slin.entries = (struct nx842_slentry *)workmem->slin;
473 slout.entries = (struct nx842_slentry *)workmem->slout;
474
475 /* Init operation */
476 op.flags = NX842_OP_DECOMPRESS_CRC;
477 csbcpb = &workmem->csbcpb;
478 memset(csbcpb, 0, sizeof(*csbcpb));
479 op.csbcpb = nx842_get_pa(csbcpb);
480
481 if ((inbuf & NX842_HW_PAGE_MASK) ==
482 ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
483 /* Create direct DDE */
484 op.in = nx842_get_pa((void *)inbuf);
485 op.inlen = inlen;
486 } else {
487 /* Create indirect DDE (scatterlist) */
488 nx842_build_scatterlist(inbuf, inlen, &slin);
489 op.in = nx842_get_pa(slin.entries);
490 op.inlen = -nx842_get_scatterlist_size(&slin);
491 }
492
493 if ((outbuf & NX842_HW_PAGE_MASK) ==
494 ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
495 /* Create direct DDE */
496 op.out = nx842_get_pa((void *)outbuf);
497 op.outlen = *outlen;
498 } else {
499 /* Create indirect DDE (scatterlist) */
500 nx842_build_scatterlist(outbuf, *outlen, &slout);
501 op.out = nx842_get_pa(slout.entries);
502 op.outlen = -nx842_get_scatterlist_size(&slout);
503 }
504
505 dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
506 __func__, (unsigned long)op.in, (long)op.inlen,
507 (unsigned long)op.out, (long)op.outlen);
508
509 /* Send request to pHyp */
510 ret = vio_h_cop_sync(local_devdata->vdev, &op);
511
512 /* Check for pHyp error */
513 if (ret) {
514 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
515 __func__, ret, op.hcall_err);
516 goto unlock;
517 }
518
519 /* Check for hardware error */
520 ret = nx842_validate_result(dev, &csbcpb->csb);
521 if (ret)
522 goto unlock;
523
524 *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
525
526 unlock:
527 if (ret)
528 /* decompress fail */
529 nx842_inc_decomp_failed(local_devdata);
530 else {
531 nx842_inc_decomp_complete(local_devdata);
532 ibm_nx842_incr_hist(local_devdata->counters->decomp_times,
533 (get_tb() - start) / tb_ticks_per_usec);
534 }
535
536 rcu_read_unlock();
537 return ret;
538 }
539
540 /**
541 * nx842_OF_set_defaults -- Set default (disabled) values for devdata
542 *
543 * @devdata: struct nx842_devdata to update
544 *
545 * Returns:
546 * 0 on success
547 * -ENOENT if @devdata ptr is NULL
548 */
nx842_OF_set_defaults(struct nx842_devdata * devdata)549 static int nx842_OF_set_defaults(struct nx842_devdata *devdata)
550 {
551 if (devdata) {
552 devdata->max_sync_size = 0;
553 devdata->max_sync_sg = 0;
554 devdata->max_sg_len = 0;
555 return 0;
556 } else
557 return -ENOENT;
558 }
559
560 /**
561 * nx842_OF_upd_status -- Check the device info from OF status prop
562 *
563 * The status property indicates if the accelerator is enabled. If the
564 * device is in the OF tree it indicates that the hardware is present.
565 * The status field indicates if the device is enabled when the status
566 * is 'okay'. Otherwise the device driver will be disabled.
567 *
568 * @devdata: struct nx842_devdata to use for dev_info
569 * @prop: struct property point containing the maxsyncop for the update
570 *
571 * Returns:
572 * 0 - Device is available
573 * -ENODEV - Device is not available
574 */
nx842_OF_upd_status(struct nx842_devdata * devdata,struct property * prop)575 static int nx842_OF_upd_status(struct nx842_devdata *devdata,
576 struct property *prop)
577 {
578 const char *status = (const char *)prop->value;
579
580 if (!strncmp(status, "okay", (size_t)prop->length))
581 return 0;
582 if (!strncmp(status, "disabled", (size_t)prop->length))
583 return -ENODEV;
584 dev_info(devdata->dev, "%s: unknown status '%s'\n", __func__, status);
585
586 return -EINVAL;
587 }
588
589 /**
590 * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop
591 *
592 * Definition of the 'ibm,max-sg-len' OF property:
593 * This field indicates the maximum byte length of a scatter list
594 * for the platform facility. It is a single cell encoded as with encode-int.
595 *
596 * Example:
597 * # od -x ibm,max-sg-len
598 * 0000000 0000 0ff0
599 *
600 * In this example, the maximum byte length of a scatter list is
601 * 0x0ff0 (4,080).
602 *
603 * @devdata: struct nx842_devdata to update
604 * @prop: struct property point containing the maxsyncop for the update
605 *
606 * Returns:
607 * 0 on success
608 * -EINVAL on failure
609 */
nx842_OF_upd_maxsglen(struct nx842_devdata * devdata,struct property * prop)610 static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata,
611 struct property *prop) {
612 int ret = 0;
613 const unsigned int maxsglen = of_read_number(prop->value, 1);
614
615 if (prop->length != sizeof(maxsglen)) {
616 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__);
617 dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__,
618 prop->length, sizeof(maxsglen));
619 ret = -EINVAL;
620 } else {
621 devdata->max_sg_len = min_t(unsigned int,
622 maxsglen, NX842_HW_PAGE_SIZE);
623 }
624
625 return ret;
626 }
627
628 /**
629 * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop
630 *
631 * Definition of the 'ibm,max-sync-cop' OF property:
632 * Two series of cells. The first series of cells represents the maximums
633 * that can be synchronously compressed. The second series of cells
634 * represents the maximums that can be synchronously decompressed.
635 * 1. The first cell in each series contains the count of the number of
636 * data length, scatter list elements pairs that follow – each being
637 * of the form
638 * a. One cell data byte length
639 * b. One cell total number of scatter list elements
640 *
641 * Example:
642 * # od -x ibm,max-sync-cop
643 * 0000000 0000 0001 0000 1000 0000 01fe 0000 0001
644 * 0000020 0000 1000 0000 01fe
645 *
646 * In this example, compression supports 0x1000 (4,096) data byte length
647 * and 0x1fe (510) total scatter list elements. Decompression supports
648 * 0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list
649 * elements.
650 *
651 * @devdata: struct nx842_devdata to update
652 * @prop: struct property point containing the maxsyncop for the update
653 *
654 * Returns:
655 * 0 on success
656 * -EINVAL on failure
657 */
nx842_OF_upd_maxsyncop(struct nx842_devdata * devdata,struct property * prop)658 static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata,
659 struct property *prop) {
660 int ret = 0;
661 unsigned int comp_data_limit, decomp_data_limit;
662 unsigned int comp_sg_limit, decomp_sg_limit;
663 const struct maxsynccop_t {
664 __be32 comp_elements;
665 __be32 comp_data_limit;
666 __be32 comp_sg_limit;
667 __be32 decomp_elements;
668 __be32 decomp_data_limit;
669 __be32 decomp_sg_limit;
670 } *maxsynccop;
671
672 if (prop->length != sizeof(*maxsynccop)) {
673 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__);
674 dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length,
675 sizeof(*maxsynccop));
676 ret = -EINVAL;
677 goto out;
678 }
679
680 maxsynccop = (const struct maxsynccop_t *)prop->value;
681 comp_data_limit = be32_to_cpu(maxsynccop->comp_data_limit);
682 comp_sg_limit = be32_to_cpu(maxsynccop->comp_sg_limit);
683 decomp_data_limit = be32_to_cpu(maxsynccop->decomp_data_limit);
684 decomp_sg_limit = be32_to_cpu(maxsynccop->decomp_sg_limit);
685
686 /* Use one limit rather than separate limits for compression and
687 * decompression. Set a maximum for this so as not to exceed the
688 * size that the header can support and round the value down to
689 * the hardware page size (4K) */
690 devdata->max_sync_size = min(comp_data_limit, decomp_data_limit);
691
692 devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size,
693 65536);
694
695 if (devdata->max_sync_size < 4096) {
696 dev_err(devdata->dev, "%s: hardware max data size (%u) is "
697 "less than the driver minimum, unable to use "
698 "the hardware device\n",
699 __func__, devdata->max_sync_size);
700 ret = -EINVAL;
701 goto out;
702 }
703
704 nx842_pseries_constraints.maximum = devdata->max_sync_size;
705
706 devdata->max_sync_sg = min(comp_sg_limit, decomp_sg_limit);
707 if (devdata->max_sync_sg < 1) {
708 dev_err(devdata->dev, "%s: hardware max sg size (%u) is "
709 "less than the driver minimum, unable to use "
710 "the hardware device\n",
711 __func__, devdata->max_sync_sg);
712 ret = -EINVAL;
713 goto out;
714 }
715
716 out:
717 return ret;
718 }
719
720 /**
721 * nx842_OF_upd -- Handle OF properties updates for the device.
722 *
723 * Set all properties from the OF tree. Optionally, a new property
724 * can be provided by the @new_prop pointer to overwrite an existing value.
725 * The device will remain disabled until all values are valid, this function
726 * will return an error for updates unless all values are valid.
727 *
728 * @new_prop: If not NULL, this property is being updated. If NULL, update
729 * all properties from the current values in the OF tree.
730 *
731 * Returns:
732 * 0 - Success
733 * -ENOMEM - Could not allocate memory for new devdata structure
734 * -EINVAL - property value not found, new_prop is not a recognized
735 * property for the device or property value is not valid.
736 * -ENODEV - Device is not available
737 */
nx842_OF_upd(struct property * new_prop)738 static int nx842_OF_upd(struct property *new_prop)
739 {
740 struct nx842_devdata *old_devdata = NULL;
741 struct nx842_devdata *new_devdata = NULL;
742 struct device_node *of_node = NULL;
743 struct property *status = NULL;
744 struct property *maxsglen = NULL;
745 struct property *maxsyncop = NULL;
746 int ret = 0;
747 unsigned long flags;
748
749 new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
750 if (!new_devdata)
751 return -ENOMEM;
752
753 spin_lock_irqsave(&devdata_mutex, flags);
754 old_devdata = rcu_dereference_check(devdata,
755 lockdep_is_held(&devdata_mutex));
756 if (old_devdata)
757 of_node = old_devdata->dev->of_node;
758
759 if (!old_devdata || !of_node) {
760 pr_err("%s: device is not available\n", __func__);
761 spin_unlock_irqrestore(&devdata_mutex, flags);
762 kfree(new_devdata);
763 return -ENODEV;
764 }
765
766 memcpy(new_devdata, old_devdata, sizeof(*old_devdata));
767 new_devdata->counters = old_devdata->counters;
768
769 /* Set ptrs for existing properties */
770 status = of_find_property(of_node, "status", NULL);
771 maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL);
772 maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL);
773 if (!status || !maxsglen || !maxsyncop) {
774 dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__);
775 ret = -EINVAL;
776 goto error_out;
777 }
778
779 /*
780 * If this is a property update, there are only certain properties that
781 * we care about. Bail if it isn't in the below list
782 */
783 if (new_prop && (strncmp(new_prop->name, "status", new_prop->length) ||
784 strncmp(new_prop->name, "ibm,max-sg-len", new_prop->length) ||
785 strncmp(new_prop->name, "ibm,max-sync-cop", new_prop->length)))
786 goto out;
787
788 /* Perform property updates */
789 ret = nx842_OF_upd_status(new_devdata, status);
790 if (ret)
791 goto error_out;
792
793 ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen);
794 if (ret)
795 goto error_out;
796
797 ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop);
798 if (ret)
799 goto error_out;
800
801 out:
802 dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n",
803 __func__, new_devdata->max_sync_size,
804 old_devdata->max_sync_size);
805 dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n",
806 __func__, new_devdata->max_sync_sg,
807 old_devdata->max_sync_sg);
808 dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n",
809 __func__, new_devdata->max_sg_len,
810 old_devdata->max_sg_len);
811
812 rcu_assign_pointer(devdata, new_devdata);
813 spin_unlock_irqrestore(&devdata_mutex, flags);
814 synchronize_rcu();
815 dev_set_drvdata(new_devdata->dev, new_devdata);
816 kfree(old_devdata);
817 return 0;
818
819 error_out:
820 if (new_devdata) {
821 dev_info(old_devdata->dev, "%s: device disabled\n", __func__);
822 nx842_OF_set_defaults(new_devdata);
823 rcu_assign_pointer(devdata, new_devdata);
824 spin_unlock_irqrestore(&devdata_mutex, flags);
825 synchronize_rcu();
826 dev_set_drvdata(new_devdata->dev, new_devdata);
827 kfree(old_devdata);
828 } else {
829 dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__);
830 spin_unlock_irqrestore(&devdata_mutex, flags);
831 }
832
833 if (!ret)
834 ret = -EINVAL;
835 return ret;
836 }
837
838 /**
839 * nx842_OF_notifier - Process updates to OF properties for the device
840 *
841 * @np: notifier block
842 * @action: notifier action
843 * @data: struct of_reconfig_data pointer
844 *
845 * Returns:
846 * NOTIFY_OK on success
847 * NOTIFY_BAD encoded with error number on failure, use
848 * notifier_to_errno() to decode this value
849 */
nx842_OF_notifier(struct notifier_block * np,unsigned long action,void * data)850 static int nx842_OF_notifier(struct notifier_block *np, unsigned long action,
851 void *data)
852 {
853 struct of_reconfig_data *upd = data;
854 struct nx842_devdata *local_devdata;
855 struct device_node *node = NULL;
856
857 rcu_read_lock();
858 local_devdata = rcu_dereference(devdata);
859 if (local_devdata)
860 node = local_devdata->dev->of_node;
861
862 if (local_devdata &&
863 action == OF_RECONFIG_UPDATE_PROPERTY &&
864 !strcmp(upd->dn->name, node->name)) {
865 rcu_read_unlock();
866 nx842_OF_upd(upd->prop);
867 } else
868 rcu_read_unlock();
869
870 return NOTIFY_OK;
871 }
872
873 static struct notifier_block nx842_of_nb = {
874 .notifier_call = nx842_OF_notifier,
875 };
876
877 #define nx842_counter_read(_name) \
878 static ssize_t nx842_##_name##_show(struct device *dev, \
879 struct device_attribute *attr, \
880 char *buf) { \
881 struct nx842_devdata *local_devdata; \
882 int p = 0; \
883 rcu_read_lock(); \
884 local_devdata = rcu_dereference(devdata); \
885 if (local_devdata) \
886 p = snprintf(buf, PAGE_SIZE, "%lld\n", \
887 atomic64_read(&local_devdata->counters->_name)); \
888 rcu_read_unlock(); \
889 return p; \
890 }
891
892 #define NX842DEV_COUNTER_ATTR_RO(_name) \
893 nx842_counter_read(_name); \
894 static struct device_attribute dev_attr_##_name = __ATTR(_name, \
895 0444, \
896 nx842_##_name##_show,\
897 NULL);
898
899 NX842DEV_COUNTER_ATTR_RO(comp_complete);
900 NX842DEV_COUNTER_ATTR_RO(comp_failed);
901 NX842DEV_COUNTER_ATTR_RO(decomp_complete);
902 NX842DEV_COUNTER_ATTR_RO(decomp_failed);
903 NX842DEV_COUNTER_ATTR_RO(swdecomp);
904
905 static ssize_t nx842_timehist_show(struct device *,
906 struct device_attribute *, char *);
907
908 static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444,
909 nx842_timehist_show, NULL);
910 static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times,
911 0444, nx842_timehist_show, NULL);
912
nx842_timehist_show(struct device * dev,struct device_attribute * attr,char * buf)913 static ssize_t nx842_timehist_show(struct device *dev,
914 struct device_attribute *attr, char *buf) {
915 char *p = buf;
916 struct nx842_devdata *local_devdata;
917 atomic64_t *times;
918 int bytes_remain = PAGE_SIZE;
919 int bytes;
920 int i;
921
922 rcu_read_lock();
923 local_devdata = rcu_dereference(devdata);
924 if (!local_devdata) {
925 rcu_read_unlock();
926 return 0;
927 }
928
929 if (attr == &dev_attr_comp_times)
930 times = local_devdata->counters->comp_times;
931 else if (attr == &dev_attr_decomp_times)
932 times = local_devdata->counters->decomp_times;
933 else {
934 rcu_read_unlock();
935 return 0;
936 }
937
938 for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) {
939 bytes = snprintf(p, bytes_remain, "%u-%uus:\t%lld\n",
940 i ? (2<<(i-1)) : 0, (2<<i)-1,
941 atomic64_read(×[i]));
942 bytes_remain -= bytes;
943 p += bytes;
944 }
945 /* The last bucket holds everything over
946 * 2<<(NX842_HIST_SLOTS - 2) us */
947 bytes = snprintf(p, bytes_remain, "%uus - :\t%lld\n",
948 2<<(NX842_HIST_SLOTS - 2),
949 atomic64_read(×[(NX842_HIST_SLOTS - 1)]));
950 p += bytes;
951
952 rcu_read_unlock();
953 return p - buf;
954 }
955
956 static struct attribute *nx842_sysfs_entries[] = {
957 &dev_attr_comp_complete.attr,
958 &dev_attr_comp_failed.attr,
959 &dev_attr_decomp_complete.attr,
960 &dev_attr_decomp_failed.attr,
961 &dev_attr_swdecomp.attr,
962 &dev_attr_comp_times.attr,
963 &dev_attr_decomp_times.attr,
964 NULL,
965 };
966
967 static const struct attribute_group nx842_attribute_group = {
968 .name = NULL, /* put in device directory */
969 .attrs = nx842_sysfs_entries,
970 };
971
972 #define nxcop_caps_read(_name) \
973 static ssize_t nxcop_##_name##_show(struct device *dev, \
974 struct device_attribute *attr, char *buf) \
975 { \
976 return sprintf(buf, "%lld\n", nx_cop_caps._name); \
977 }
978
979 #define NXCT_ATTR_RO(_name) \
980 nxcop_caps_read(_name); \
981 static struct device_attribute dev_attr_##_name = __ATTR(_name, \
982 0444, \
983 nxcop_##_name##_show, \
984 NULL);
985
986 NXCT_ATTR_RO(req_max_processed_len);
987 NXCT_ATTR_RO(min_compress_len);
988 NXCT_ATTR_RO(min_decompress_len);
989
990 static struct attribute *nxcop_caps_sysfs_entries[] = {
991 &dev_attr_req_max_processed_len.attr,
992 &dev_attr_min_compress_len.attr,
993 &dev_attr_min_decompress_len.attr,
994 NULL,
995 };
996
997 static const struct attribute_group nxcop_caps_attr_group = {
998 .name = "nx_gzip_caps",
999 .attrs = nxcop_caps_sysfs_entries,
1000 };
1001
1002 static struct nx842_driver nx842_pseries_driver = {
1003 .name = KBUILD_MODNAME,
1004 .owner = THIS_MODULE,
1005 .workmem_size = sizeof(struct nx842_workmem),
1006 .constraints = &nx842_pseries_constraints,
1007 .compress = nx842_pseries_compress,
1008 .decompress = nx842_pseries_decompress,
1009 };
1010
nx842_pseries_crypto_init(struct crypto_tfm * tfm)1011 static int nx842_pseries_crypto_init(struct crypto_tfm *tfm)
1012 {
1013 return nx842_crypto_init(tfm, &nx842_pseries_driver);
1014 }
1015
1016 static struct crypto_alg nx842_pseries_alg = {
1017 .cra_name = "842",
1018 .cra_driver_name = "842-nx",
1019 .cra_priority = 300,
1020 .cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
1021 .cra_ctxsize = sizeof(struct nx842_crypto_ctx),
1022 .cra_module = THIS_MODULE,
1023 .cra_init = nx842_pseries_crypto_init,
1024 .cra_exit = nx842_crypto_exit,
1025 .cra_u = { .compress = {
1026 .coa_compress = nx842_crypto_compress,
1027 .coa_decompress = nx842_crypto_decompress } }
1028 };
1029
nx842_probe(struct vio_dev * viodev,const struct vio_device_id * id)1030 static int nx842_probe(struct vio_dev *viodev,
1031 const struct vio_device_id *id)
1032 {
1033 struct nx842_devdata *old_devdata, *new_devdata = NULL;
1034 unsigned long flags;
1035 int ret = 0;
1036
1037 new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
1038 if (!new_devdata)
1039 return -ENOMEM;
1040
1041 new_devdata->counters = kzalloc(sizeof(*new_devdata->counters),
1042 GFP_NOFS);
1043 if (!new_devdata->counters) {
1044 kfree(new_devdata);
1045 return -ENOMEM;
1046 }
1047
1048 spin_lock_irqsave(&devdata_mutex, flags);
1049 old_devdata = rcu_dereference_check(devdata,
1050 lockdep_is_held(&devdata_mutex));
1051
1052 if (old_devdata && old_devdata->vdev != NULL) {
1053 dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__);
1054 ret = -1;
1055 goto error_unlock;
1056 }
1057
1058 dev_set_drvdata(&viodev->dev, NULL);
1059
1060 new_devdata->vdev = viodev;
1061 new_devdata->dev = &viodev->dev;
1062 nx842_OF_set_defaults(new_devdata);
1063
1064 rcu_assign_pointer(devdata, new_devdata);
1065 spin_unlock_irqrestore(&devdata_mutex, flags);
1066 synchronize_rcu();
1067 kfree(old_devdata);
1068
1069 of_reconfig_notifier_register(&nx842_of_nb);
1070
1071 ret = nx842_OF_upd(NULL);
1072 if (ret)
1073 goto error;
1074
1075 ret = crypto_register_alg(&nx842_pseries_alg);
1076 if (ret) {
1077 dev_err(&viodev->dev, "could not register comp alg: %d\n", ret);
1078 goto error;
1079 }
1080
1081 rcu_read_lock();
1082 dev_set_drvdata(&viodev->dev, rcu_dereference(devdata));
1083 rcu_read_unlock();
1084
1085 if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) {
1086 dev_err(&viodev->dev, "could not create sysfs device attributes\n");
1087 ret = -1;
1088 goto error;
1089 }
1090
1091 if (caps_feat) {
1092 if (sysfs_create_group(&viodev->dev.kobj,
1093 &nxcop_caps_attr_group)) {
1094 dev_err(&viodev->dev,
1095 "Could not create sysfs NX capability entries\n");
1096 ret = -1;
1097 goto error;
1098 }
1099 }
1100
1101 return 0;
1102
1103 error_unlock:
1104 spin_unlock_irqrestore(&devdata_mutex, flags);
1105 if (new_devdata)
1106 kfree(new_devdata->counters);
1107 kfree(new_devdata);
1108 error:
1109 return ret;
1110 }
1111
nx842_remove(struct vio_dev * viodev)1112 static void nx842_remove(struct vio_dev *viodev)
1113 {
1114 struct nx842_devdata *old_devdata;
1115 unsigned long flags;
1116
1117 pr_info("Removing IBM Power 842 compression device\n");
1118 sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group);
1119
1120 if (caps_feat)
1121 sysfs_remove_group(&viodev->dev.kobj, &nxcop_caps_attr_group);
1122
1123 crypto_unregister_alg(&nx842_pseries_alg);
1124
1125 spin_lock_irqsave(&devdata_mutex, flags);
1126 old_devdata = rcu_dereference_check(devdata,
1127 lockdep_is_held(&devdata_mutex));
1128 of_reconfig_notifier_unregister(&nx842_of_nb);
1129 RCU_INIT_POINTER(devdata, NULL);
1130 spin_unlock_irqrestore(&devdata_mutex, flags);
1131 synchronize_rcu();
1132 dev_set_drvdata(&viodev->dev, NULL);
1133 if (old_devdata)
1134 kfree(old_devdata->counters);
1135 kfree(old_devdata);
1136 }
1137
1138 /*
1139 * Get NX capabilities from the hypervisor.
1140 * Only NXGZIP capabilities are provided by the hypersvisor right
1141 * now and these values are available to user space with sysfs.
1142 */
nxcop_get_capabilities(void)1143 static void __init nxcop_get_capabilities(void)
1144 {
1145 struct hv_vas_all_caps *hv_caps;
1146 struct hv_nx_cop_caps *hv_nxc;
1147 int rc;
1148
1149 hv_caps = kmalloc(sizeof(*hv_caps), GFP_KERNEL);
1150 if (!hv_caps)
1151 return;
1152 /*
1153 * Get NX overall capabilities with feature type=0
1154 */
1155 rc = h_query_vas_capabilities(H_QUERY_NX_CAPABILITIES, 0,
1156 (u64)virt_to_phys(hv_caps));
1157 if (rc)
1158 goto out;
1159
1160 caps_feat = be64_to_cpu(hv_caps->feat_type);
1161 /*
1162 * NX-GZIP feature available
1163 */
1164 if (caps_feat & VAS_NX_GZIP_FEAT_BIT) {
1165 hv_nxc = kmalloc(sizeof(*hv_nxc), GFP_KERNEL);
1166 if (!hv_nxc)
1167 goto out;
1168 /*
1169 * Get capabilities for NX-GZIP feature
1170 */
1171 rc = h_query_vas_capabilities(H_QUERY_NX_CAPABILITIES,
1172 VAS_NX_GZIP_FEAT,
1173 (u64)virt_to_phys(hv_nxc));
1174 } else {
1175 pr_err("NX-GZIP feature is not available\n");
1176 rc = -EINVAL;
1177 }
1178
1179 if (!rc) {
1180 nx_cop_caps.descriptor = be64_to_cpu(hv_nxc->descriptor);
1181 nx_cop_caps.req_max_processed_len =
1182 be64_to_cpu(hv_nxc->req_max_processed_len);
1183 nx_cop_caps.min_compress_len =
1184 be64_to_cpu(hv_nxc->min_compress_len);
1185 nx_cop_caps.min_decompress_len =
1186 be64_to_cpu(hv_nxc->min_decompress_len);
1187 } else {
1188 caps_feat = 0;
1189 }
1190
1191 kfree(hv_nxc);
1192 out:
1193 kfree(hv_caps);
1194 }
1195
1196 static const struct vio_device_id nx842_vio_driver_ids[] = {
1197 {"ibm,compression-v1", "ibm,compression"},
1198 {"", ""},
1199 };
1200 MODULE_DEVICE_TABLE(vio, nx842_vio_driver_ids);
1201
1202 static struct vio_driver nx842_vio_driver = {
1203 .name = KBUILD_MODNAME,
1204 .probe = nx842_probe,
1205 .remove = nx842_remove,
1206 .get_desired_dma = nx842_get_desired_dma,
1207 .id_table = nx842_vio_driver_ids,
1208 };
1209
nx842_pseries_init(void)1210 static int __init nx842_pseries_init(void)
1211 {
1212 struct nx842_devdata *new_devdata;
1213 struct device_node *np;
1214 int ret;
1215
1216 np = of_find_compatible_node(NULL, NULL, "ibm,compression");
1217 if (!np)
1218 return -ENODEV;
1219 of_node_put(np);
1220
1221 RCU_INIT_POINTER(devdata, NULL);
1222 new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL);
1223 if (!new_devdata)
1224 return -ENOMEM;
1225
1226 RCU_INIT_POINTER(devdata, new_devdata);
1227 /*
1228 * Get NX capabilities from the hypervisor.
1229 */
1230 nxcop_get_capabilities();
1231
1232 ret = vio_register_driver(&nx842_vio_driver);
1233 if (ret) {
1234 pr_err("Could not register VIO driver %d\n", ret);
1235
1236 kfree(new_devdata);
1237 return ret;
1238 }
1239
1240 ret = vas_register_api_pseries(THIS_MODULE, VAS_COP_TYPE_GZIP,
1241 "nx-gzip");
1242
1243 if (ret)
1244 pr_err("NX-GZIP is not supported. Returned=%d\n", ret);
1245
1246 return 0;
1247 }
1248
1249 module_init(nx842_pseries_init);
1250
nx842_pseries_exit(void)1251 static void __exit nx842_pseries_exit(void)
1252 {
1253 struct nx842_devdata *old_devdata;
1254 unsigned long flags;
1255
1256 vas_unregister_api_pseries();
1257
1258 crypto_unregister_alg(&nx842_pseries_alg);
1259
1260 spin_lock_irqsave(&devdata_mutex, flags);
1261 old_devdata = rcu_dereference_check(devdata,
1262 lockdep_is_held(&devdata_mutex));
1263 RCU_INIT_POINTER(devdata, NULL);
1264 spin_unlock_irqrestore(&devdata_mutex, flags);
1265 synchronize_rcu();
1266 if (old_devdata && old_devdata->dev)
1267 dev_set_drvdata(old_devdata->dev, NULL);
1268 kfree(old_devdata);
1269 vio_unregister_driver(&nx842_vio_driver);
1270 }
1271
1272 module_exit(nx842_pseries_exit);
1273
1274