1*a4b16dadSTom Zanussi // SPDX-License-Identifier: GPL-2.0-only
2*a4b16dadSTom Zanussi /* Copyright(c) 2022 Intel Corporation */
3*a4b16dadSTom Zanussi #include <linux/crypto.h>
4*a4b16dadSTom Zanussi #include <crypto/acompress.h>
5*a4b16dadSTom Zanussi #include <crypto/internal/acompress.h>
6*a4b16dadSTom Zanussi #include <crypto/scatterwalk.h>
7*a4b16dadSTom Zanussi #include <linux/dma-mapping.h>
8*a4b16dadSTom Zanussi #include <linux/workqueue.h>
9*a4b16dadSTom Zanussi #include "adf_accel_devices.h"
10*a4b16dadSTom Zanussi #include "adf_common_drv.h"
11*a4b16dadSTom Zanussi #include "qat_bl.h"
12*a4b16dadSTom Zanussi #include "qat_comp_req.h"
13*a4b16dadSTom Zanussi #include "qat_compression.h"
14*a4b16dadSTom Zanussi #include "qat_algs_send.h"
15*a4b16dadSTom Zanussi
16*a4b16dadSTom Zanussi #define QAT_RFC_1950_HDR_SIZE 2
17*a4b16dadSTom Zanussi #define QAT_RFC_1950_FOOTER_SIZE 4
18*a4b16dadSTom Zanussi #define QAT_RFC_1950_CM_DEFLATE 8
19*a4b16dadSTom Zanussi #define QAT_RFC_1950_CM_DEFLATE_CINFO_32K 7
20*a4b16dadSTom Zanussi #define QAT_RFC_1950_CM_MASK 0x0f
21*a4b16dadSTom Zanussi #define QAT_RFC_1950_CM_OFFSET 4
22*a4b16dadSTom Zanussi #define QAT_RFC_1950_DICT_MASK 0x20
23*a4b16dadSTom Zanussi #define QAT_RFC_1950_COMP_HDR 0x785e
24*a4b16dadSTom Zanussi
25*a4b16dadSTom Zanussi static DEFINE_MUTEX(algs_lock);
26*a4b16dadSTom Zanussi static unsigned int active_devs;
27*a4b16dadSTom Zanussi
28*a4b16dadSTom Zanussi enum direction {
29*a4b16dadSTom Zanussi DECOMPRESSION = 0,
30*a4b16dadSTom Zanussi COMPRESSION = 1,
31*a4b16dadSTom Zanussi };
32*a4b16dadSTom Zanussi
33*a4b16dadSTom Zanussi struct qat_compression_req;
34*a4b16dadSTom Zanussi
35*a4b16dadSTom Zanussi struct qat_compression_ctx {
36*a4b16dadSTom Zanussi u8 comp_ctx[QAT_COMP_CTX_SIZE];
37*a4b16dadSTom Zanussi struct qat_compression_instance *inst;
38*a4b16dadSTom Zanussi int (*qat_comp_callback)(struct qat_compression_req *qat_req, void *resp);
39*a4b16dadSTom Zanussi };
40*a4b16dadSTom Zanussi
41*a4b16dadSTom Zanussi struct qat_dst {
42*a4b16dadSTom Zanussi bool is_null;
43*a4b16dadSTom Zanussi int resubmitted;
44*a4b16dadSTom Zanussi };
45*a4b16dadSTom Zanussi
46*a4b16dadSTom Zanussi struct qat_compression_req {
47*a4b16dadSTom Zanussi u8 req[QAT_COMP_REQ_SIZE];
48*a4b16dadSTom Zanussi struct qat_compression_ctx *qat_compression_ctx;
49*a4b16dadSTom Zanussi struct acomp_req *acompress_req;
50*a4b16dadSTom Zanussi struct qat_request_buffs buf;
51*a4b16dadSTom Zanussi enum direction dir;
52*a4b16dadSTom Zanussi int actual_dlen;
53*a4b16dadSTom Zanussi struct qat_alg_req alg_req;
54*a4b16dadSTom Zanussi struct work_struct resubmit;
55*a4b16dadSTom Zanussi struct qat_dst dst;
56*a4b16dadSTom Zanussi };
57*a4b16dadSTom Zanussi
qat_alg_send_dc_message(struct qat_compression_req * qat_req,struct qat_compression_instance * inst,struct crypto_async_request * base)58*a4b16dadSTom Zanussi static int qat_alg_send_dc_message(struct qat_compression_req *qat_req,
59*a4b16dadSTom Zanussi struct qat_compression_instance *inst,
60*a4b16dadSTom Zanussi struct crypto_async_request *base)
61*a4b16dadSTom Zanussi {
62*a4b16dadSTom Zanussi struct qat_alg_req *alg_req = &qat_req->alg_req;
63*a4b16dadSTom Zanussi
64*a4b16dadSTom Zanussi alg_req->fw_req = (u32 *)&qat_req->req;
65*a4b16dadSTom Zanussi alg_req->tx_ring = inst->dc_tx;
66*a4b16dadSTom Zanussi alg_req->base = base;
67*a4b16dadSTom Zanussi alg_req->backlog = &inst->backlog;
68*a4b16dadSTom Zanussi
69*a4b16dadSTom Zanussi return qat_alg_send_message(alg_req);
70*a4b16dadSTom Zanussi }
71*a4b16dadSTom Zanussi
qat_comp_resubmit(struct work_struct * work)72*a4b16dadSTom Zanussi static void qat_comp_resubmit(struct work_struct *work)
73*a4b16dadSTom Zanussi {
74*a4b16dadSTom Zanussi struct qat_compression_req *qat_req =
75*a4b16dadSTom Zanussi container_of(work, struct qat_compression_req, resubmit);
76*a4b16dadSTom Zanussi struct qat_compression_ctx *ctx = qat_req->qat_compression_ctx;
77*a4b16dadSTom Zanussi struct adf_accel_dev *accel_dev = ctx->inst->accel_dev;
78*a4b16dadSTom Zanussi struct qat_request_buffs *qat_bufs = &qat_req->buf;
79*a4b16dadSTom Zanussi struct qat_compression_instance *inst = ctx->inst;
80*a4b16dadSTom Zanussi struct acomp_req *areq = qat_req->acompress_req;
81*a4b16dadSTom Zanussi struct crypto_acomp *tfm = crypto_acomp_reqtfm(areq);
82*a4b16dadSTom Zanussi unsigned int dlen = CRYPTO_ACOMP_DST_MAX;
83*a4b16dadSTom Zanussi u8 *req = qat_req->req;
84*a4b16dadSTom Zanussi dma_addr_t dfbuf;
85*a4b16dadSTom Zanussi int ret;
86*a4b16dadSTom Zanussi
87*a4b16dadSTom Zanussi areq->dlen = dlen;
88*a4b16dadSTom Zanussi
89*a4b16dadSTom Zanussi dev_dbg(&GET_DEV(accel_dev), "[%s][%s] retry NULL dst request - dlen = %d\n",
90*a4b16dadSTom Zanussi crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm)),
91*a4b16dadSTom Zanussi qat_req->dir == COMPRESSION ? "comp" : "decomp", dlen);
92*a4b16dadSTom Zanussi
93*a4b16dadSTom Zanussi ret = qat_bl_realloc_map_new_dst(accel_dev, &areq->dst, dlen, qat_bufs,
94*a4b16dadSTom Zanussi qat_algs_alloc_flags(&areq->base));
95*a4b16dadSTom Zanussi if (ret)
96*a4b16dadSTom Zanussi goto err;
97*a4b16dadSTom Zanussi
98*a4b16dadSTom Zanussi qat_req->dst.resubmitted = true;
99*a4b16dadSTom Zanussi
100*a4b16dadSTom Zanussi dfbuf = qat_req->buf.bloutp;
101*a4b16dadSTom Zanussi qat_comp_override_dst(req, dfbuf, dlen);
102*a4b16dadSTom Zanussi
103*a4b16dadSTom Zanussi ret = qat_alg_send_dc_message(qat_req, inst, &areq->base);
104*a4b16dadSTom Zanussi if (ret != -ENOSPC)
105*a4b16dadSTom Zanussi return;
106*a4b16dadSTom Zanussi
107*a4b16dadSTom Zanussi err:
108*a4b16dadSTom Zanussi qat_bl_free_bufl(accel_dev, qat_bufs);
109*a4b16dadSTom Zanussi acomp_request_complete(areq, ret);
110*a4b16dadSTom Zanussi }
111*a4b16dadSTom Zanussi
parse_zlib_header(u16 zlib_h)112*a4b16dadSTom Zanussi static int parse_zlib_header(u16 zlib_h)
113*a4b16dadSTom Zanussi {
114*a4b16dadSTom Zanussi int ret = -EINVAL;
115*a4b16dadSTom Zanussi __be16 header;
116*a4b16dadSTom Zanussi u8 *header_p;
117*a4b16dadSTom Zanussi u8 cmf, flg;
118*a4b16dadSTom Zanussi
119*a4b16dadSTom Zanussi header = cpu_to_be16(zlib_h);
120*a4b16dadSTom Zanussi header_p = (u8 *)&header;
121*a4b16dadSTom Zanussi
122*a4b16dadSTom Zanussi flg = header_p[0];
123*a4b16dadSTom Zanussi cmf = header_p[1];
124*a4b16dadSTom Zanussi
125*a4b16dadSTom Zanussi if (cmf >> QAT_RFC_1950_CM_OFFSET > QAT_RFC_1950_CM_DEFLATE_CINFO_32K)
126*a4b16dadSTom Zanussi return ret;
127*a4b16dadSTom Zanussi
128*a4b16dadSTom Zanussi if ((cmf & QAT_RFC_1950_CM_MASK) != QAT_RFC_1950_CM_DEFLATE)
129*a4b16dadSTom Zanussi return ret;
130*a4b16dadSTom Zanussi
131*a4b16dadSTom Zanussi if (flg & QAT_RFC_1950_DICT_MASK)
132*a4b16dadSTom Zanussi return ret;
133*a4b16dadSTom Zanussi
134*a4b16dadSTom Zanussi return 0;
135*a4b16dadSTom Zanussi }
136*a4b16dadSTom Zanussi
qat_comp_rfc1950_callback(struct qat_compression_req * qat_req,void * resp)137*a4b16dadSTom Zanussi static int qat_comp_rfc1950_callback(struct qat_compression_req *qat_req,
138*a4b16dadSTom Zanussi void *resp)
139*a4b16dadSTom Zanussi {
140*a4b16dadSTom Zanussi struct acomp_req *areq = qat_req->acompress_req;
141*a4b16dadSTom Zanussi enum direction dir = qat_req->dir;
142*a4b16dadSTom Zanussi __be32 qat_produced_adler;
143*a4b16dadSTom Zanussi
144*a4b16dadSTom Zanussi qat_produced_adler = cpu_to_be32(qat_comp_get_produced_adler32(resp));
145*a4b16dadSTom Zanussi
146*a4b16dadSTom Zanussi if (dir == COMPRESSION) {
147*a4b16dadSTom Zanussi __be16 zlib_header;
148*a4b16dadSTom Zanussi
149*a4b16dadSTom Zanussi zlib_header = cpu_to_be16(QAT_RFC_1950_COMP_HDR);
150*a4b16dadSTom Zanussi scatterwalk_map_and_copy(&zlib_header, areq->dst, 0, QAT_RFC_1950_HDR_SIZE, 1);
151*a4b16dadSTom Zanussi areq->dlen += QAT_RFC_1950_HDR_SIZE;
152*a4b16dadSTom Zanussi
153*a4b16dadSTom Zanussi scatterwalk_map_and_copy(&qat_produced_adler, areq->dst, areq->dlen,
154*a4b16dadSTom Zanussi QAT_RFC_1950_FOOTER_SIZE, 1);
155*a4b16dadSTom Zanussi areq->dlen += QAT_RFC_1950_FOOTER_SIZE;
156*a4b16dadSTom Zanussi } else {
157*a4b16dadSTom Zanussi __be32 decomp_adler;
158*a4b16dadSTom Zanussi int footer_offset;
159*a4b16dadSTom Zanussi int consumed;
160*a4b16dadSTom Zanussi
161*a4b16dadSTom Zanussi consumed = qat_comp_get_consumed_ctr(resp);
162*a4b16dadSTom Zanussi footer_offset = consumed + QAT_RFC_1950_HDR_SIZE;
163*a4b16dadSTom Zanussi if (footer_offset + QAT_RFC_1950_FOOTER_SIZE > areq->slen)
164*a4b16dadSTom Zanussi return -EBADMSG;
165*a4b16dadSTom Zanussi
166*a4b16dadSTom Zanussi scatterwalk_map_and_copy(&decomp_adler, areq->src, footer_offset,
167*a4b16dadSTom Zanussi QAT_RFC_1950_FOOTER_SIZE, 0);
168*a4b16dadSTom Zanussi
169*a4b16dadSTom Zanussi if (qat_produced_adler != decomp_adler)
170*a4b16dadSTom Zanussi return -EBADMSG;
171*a4b16dadSTom Zanussi }
172*a4b16dadSTom Zanussi return 0;
173*a4b16dadSTom Zanussi }
174*a4b16dadSTom Zanussi
qat_comp_generic_callback(struct qat_compression_req * qat_req,void * resp)175*a4b16dadSTom Zanussi static void qat_comp_generic_callback(struct qat_compression_req *qat_req,
176*a4b16dadSTom Zanussi void *resp)
177*a4b16dadSTom Zanussi {
178*a4b16dadSTom Zanussi struct acomp_req *areq = qat_req->acompress_req;
179*a4b16dadSTom Zanussi struct qat_compression_ctx *ctx = qat_req->qat_compression_ctx;
180*a4b16dadSTom Zanussi struct adf_accel_dev *accel_dev = ctx->inst->accel_dev;
181*a4b16dadSTom Zanussi struct crypto_acomp *tfm = crypto_acomp_reqtfm(areq);
182*a4b16dadSTom Zanussi struct qat_compression_instance *inst = ctx->inst;
183*a4b16dadSTom Zanussi int consumed, produced;
184*a4b16dadSTom Zanussi s8 cmp_err, xlt_err;
185*a4b16dadSTom Zanussi int res = -EBADMSG;
186*a4b16dadSTom Zanussi int status;
187*a4b16dadSTom Zanussi u8 cnv;
188*a4b16dadSTom Zanussi
189*a4b16dadSTom Zanussi status = qat_comp_get_cmp_status(resp);
190*a4b16dadSTom Zanussi status |= qat_comp_get_xlt_status(resp);
191*a4b16dadSTom Zanussi cmp_err = qat_comp_get_cmp_err(resp);
192*a4b16dadSTom Zanussi xlt_err = qat_comp_get_xlt_err(resp);
193*a4b16dadSTom Zanussi
194*a4b16dadSTom Zanussi consumed = qat_comp_get_consumed_ctr(resp);
195*a4b16dadSTom Zanussi produced = qat_comp_get_produced_ctr(resp);
196*a4b16dadSTom Zanussi
197*a4b16dadSTom Zanussi dev_dbg(&GET_DEV(accel_dev),
198*a4b16dadSTom Zanussi "[%s][%s][%s] slen = %8d dlen = %8d consumed = %8d produced = %8d cmp_err = %3d xlt_err = %3d",
199*a4b16dadSTom Zanussi crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm)),
200*a4b16dadSTom Zanussi qat_req->dir == COMPRESSION ? "comp " : "decomp",
201*a4b16dadSTom Zanussi status ? "ERR" : "OK ",
202*a4b16dadSTom Zanussi areq->slen, areq->dlen, consumed, produced, cmp_err, xlt_err);
203*a4b16dadSTom Zanussi
204*a4b16dadSTom Zanussi areq->dlen = 0;
205*a4b16dadSTom Zanussi
206*a4b16dadSTom Zanussi if (qat_req->dir == DECOMPRESSION && qat_req->dst.is_null) {
207*a4b16dadSTom Zanussi if (cmp_err == ERR_CODE_OVERFLOW_ERROR) {
208*a4b16dadSTom Zanussi if (qat_req->dst.resubmitted) {
209*a4b16dadSTom Zanussi dev_dbg(&GET_DEV(accel_dev),
210*a4b16dadSTom Zanussi "Output does not fit destination buffer\n");
211*a4b16dadSTom Zanussi res = -EOVERFLOW;
212*a4b16dadSTom Zanussi goto end;
213*a4b16dadSTom Zanussi }
214*a4b16dadSTom Zanussi
215*a4b16dadSTom Zanussi INIT_WORK(&qat_req->resubmit, qat_comp_resubmit);
216*a4b16dadSTom Zanussi adf_misc_wq_queue_work(&qat_req->resubmit);
217*a4b16dadSTom Zanussi return;
218*a4b16dadSTom Zanussi }
219*a4b16dadSTom Zanussi }
220*a4b16dadSTom Zanussi
221*a4b16dadSTom Zanussi if (unlikely(status != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
222*a4b16dadSTom Zanussi goto end;
223*a4b16dadSTom Zanussi
224*a4b16dadSTom Zanussi if (qat_req->dir == COMPRESSION) {
225*a4b16dadSTom Zanussi cnv = qat_comp_get_cmp_cnv_flag(resp);
226*a4b16dadSTom Zanussi if (unlikely(!cnv)) {
227*a4b16dadSTom Zanussi dev_err(&GET_DEV(accel_dev),
228*a4b16dadSTom Zanussi "Verified compression not supported\n");
229*a4b16dadSTom Zanussi goto end;
230*a4b16dadSTom Zanussi }
231*a4b16dadSTom Zanussi
232*a4b16dadSTom Zanussi if (unlikely(produced > qat_req->actual_dlen)) {
233*a4b16dadSTom Zanussi memset(inst->dc_data->ovf_buff, 0,
234*a4b16dadSTom Zanussi inst->dc_data->ovf_buff_sz);
235*a4b16dadSTom Zanussi dev_dbg(&GET_DEV(accel_dev),
236*a4b16dadSTom Zanussi "Actual buffer overflow: produced=%d, dlen=%d\n",
237*a4b16dadSTom Zanussi produced, qat_req->actual_dlen);
238*a4b16dadSTom Zanussi goto end;
239*a4b16dadSTom Zanussi }
240*a4b16dadSTom Zanussi }
241*a4b16dadSTom Zanussi
242*a4b16dadSTom Zanussi res = 0;
243*a4b16dadSTom Zanussi areq->dlen = produced;
244*a4b16dadSTom Zanussi
245*a4b16dadSTom Zanussi if (ctx->qat_comp_callback)
246*a4b16dadSTom Zanussi res = ctx->qat_comp_callback(qat_req, resp);
247*a4b16dadSTom Zanussi
248*a4b16dadSTom Zanussi end:
249*a4b16dadSTom Zanussi qat_bl_free_bufl(accel_dev, &qat_req->buf);
250*a4b16dadSTom Zanussi acomp_request_complete(areq, res);
251*a4b16dadSTom Zanussi }
252*a4b16dadSTom Zanussi
qat_comp_alg_callback(void * resp)253*a4b16dadSTom Zanussi void qat_comp_alg_callback(void *resp)
254*a4b16dadSTom Zanussi {
255*a4b16dadSTom Zanussi struct qat_compression_req *qat_req =
256*a4b16dadSTom Zanussi (void *)(__force long)qat_comp_get_opaque(resp);
257*a4b16dadSTom Zanussi struct qat_instance_backlog *backlog = qat_req->alg_req.backlog;
258*a4b16dadSTom Zanussi
259*a4b16dadSTom Zanussi qat_comp_generic_callback(qat_req, resp);
260*a4b16dadSTom Zanussi
261*a4b16dadSTom Zanussi qat_alg_send_backlog(backlog);
262*a4b16dadSTom Zanussi }
263*a4b16dadSTom Zanussi
qat_comp_alg_init_tfm(struct crypto_acomp * acomp_tfm)264*a4b16dadSTom Zanussi static int qat_comp_alg_init_tfm(struct crypto_acomp *acomp_tfm)
265*a4b16dadSTom Zanussi {
266*a4b16dadSTom Zanussi struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm);
267*a4b16dadSTom Zanussi struct qat_compression_ctx *ctx = crypto_tfm_ctx(tfm);
268*a4b16dadSTom Zanussi struct qat_compression_instance *inst;
269*a4b16dadSTom Zanussi int node;
270*a4b16dadSTom Zanussi
271*a4b16dadSTom Zanussi if (tfm->node == NUMA_NO_NODE)
272*a4b16dadSTom Zanussi node = numa_node_id();
273*a4b16dadSTom Zanussi else
274*a4b16dadSTom Zanussi node = tfm->node;
275*a4b16dadSTom Zanussi
276*a4b16dadSTom Zanussi memset(ctx, 0, sizeof(*ctx));
277*a4b16dadSTom Zanussi inst = qat_compression_get_instance_node(node);
278*a4b16dadSTom Zanussi if (!inst)
279*a4b16dadSTom Zanussi return -EINVAL;
280*a4b16dadSTom Zanussi ctx->inst = inst;
281*a4b16dadSTom Zanussi
282*a4b16dadSTom Zanussi ctx->inst->build_deflate_ctx(ctx->comp_ctx);
283*a4b16dadSTom Zanussi
284*a4b16dadSTom Zanussi return 0;
285*a4b16dadSTom Zanussi }
286*a4b16dadSTom Zanussi
qat_comp_alg_exit_tfm(struct crypto_acomp * acomp_tfm)287*a4b16dadSTom Zanussi static void qat_comp_alg_exit_tfm(struct crypto_acomp *acomp_tfm)
288*a4b16dadSTom Zanussi {
289*a4b16dadSTom Zanussi struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm);
290*a4b16dadSTom Zanussi struct qat_compression_ctx *ctx = crypto_tfm_ctx(tfm);
291*a4b16dadSTom Zanussi
292*a4b16dadSTom Zanussi qat_compression_put_instance(ctx->inst);
293*a4b16dadSTom Zanussi memset(ctx, 0, sizeof(*ctx));
294*a4b16dadSTom Zanussi }
295*a4b16dadSTom Zanussi
qat_comp_alg_rfc1950_init_tfm(struct crypto_acomp * acomp_tfm)296*a4b16dadSTom Zanussi static int qat_comp_alg_rfc1950_init_tfm(struct crypto_acomp *acomp_tfm)
297*a4b16dadSTom Zanussi {
298*a4b16dadSTom Zanussi struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm);
299*a4b16dadSTom Zanussi struct qat_compression_ctx *ctx = crypto_tfm_ctx(tfm);
300*a4b16dadSTom Zanussi int ret;
301*a4b16dadSTom Zanussi
302*a4b16dadSTom Zanussi ret = qat_comp_alg_init_tfm(acomp_tfm);
303*a4b16dadSTom Zanussi ctx->qat_comp_callback = &qat_comp_rfc1950_callback;
304*a4b16dadSTom Zanussi
305*a4b16dadSTom Zanussi return ret;
306*a4b16dadSTom Zanussi }
307*a4b16dadSTom Zanussi
qat_comp_alg_compress_decompress(struct acomp_req * areq,enum direction dir,unsigned int shdr,unsigned int sftr,unsigned int dhdr,unsigned int dftr)308*a4b16dadSTom Zanussi static int qat_comp_alg_compress_decompress(struct acomp_req *areq, enum direction dir,
309*a4b16dadSTom Zanussi unsigned int shdr, unsigned int sftr,
310*a4b16dadSTom Zanussi unsigned int dhdr, unsigned int dftr)
311*a4b16dadSTom Zanussi {
312*a4b16dadSTom Zanussi struct qat_compression_req *qat_req = acomp_request_ctx(areq);
313*a4b16dadSTom Zanussi struct crypto_acomp *acomp_tfm = crypto_acomp_reqtfm(areq);
314*a4b16dadSTom Zanussi struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm);
315*a4b16dadSTom Zanussi struct qat_compression_ctx *ctx = crypto_tfm_ctx(tfm);
316*a4b16dadSTom Zanussi struct qat_compression_instance *inst = ctx->inst;
317*a4b16dadSTom Zanussi gfp_t f = qat_algs_alloc_flags(&areq->base);
318*a4b16dadSTom Zanussi struct qat_sgl_to_bufl_params params = {0};
319*a4b16dadSTom Zanussi int slen = areq->slen - shdr - sftr;
320*a4b16dadSTom Zanussi int dlen = areq->dlen - dhdr - dftr;
321*a4b16dadSTom Zanussi dma_addr_t sfbuf, dfbuf;
322*a4b16dadSTom Zanussi u8 *req = qat_req->req;
323*a4b16dadSTom Zanussi size_t ovf_buff_sz;
324*a4b16dadSTom Zanussi int ret;
325*a4b16dadSTom Zanussi
326*a4b16dadSTom Zanussi params.sskip = shdr;
327*a4b16dadSTom Zanussi params.dskip = dhdr;
328*a4b16dadSTom Zanussi
329*a4b16dadSTom Zanussi if (!areq->src || !slen)
330*a4b16dadSTom Zanussi return -EINVAL;
331*a4b16dadSTom Zanussi
332*a4b16dadSTom Zanussi if (areq->dst && !dlen)
333*a4b16dadSTom Zanussi return -EINVAL;
334*a4b16dadSTom Zanussi
335*a4b16dadSTom Zanussi qat_req->dst.is_null = false;
336*a4b16dadSTom Zanussi
337*a4b16dadSTom Zanussi /* Handle acomp requests that require the allocation of a destination
338*a4b16dadSTom Zanussi * buffer. The size of the destination buffer is double the source
339*a4b16dadSTom Zanussi * buffer (rounded up to the size of a page) to fit the decompressed
340*a4b16dadSTom Zanussi * output or an expansion on the data for compression.
341*a4b16dadSTom Zanussi */
342*a4b16dadSTom Zanussi if (!areq->dst) {
343*a4b16dadSTom Zanussi qat_req->dst.is_null = true;
344*a4b16dadSTom Zanussi
345*a4b16dadSTom Zanussi dlen = round_up(2 * slen, PAGE_SIZE);
346*a4b16dadSTom Zanussi areq->dst = sgl_alloc(dlen, f, NULL);
347*a4b16dadSTom Zanussi if (!areq->dst)
348*a4b16dadSTom Zanussi return -ENOMEM;
349*a4b16dadSTom Zanussi
350*a4b16dadSTom Zanussi dlen -= dhdr + dftr;
351*a4b16dadSTom Zanussi areq->dlen = dlen;
352*a4b16dadSTom Zanussi qat_req->dst.resubmitted = false;
353*a4b16dadSTom Zanussi }
354*a4b16dadSTom Zanussi
355*a4b16dadSTom Zanussi if (dir == COMPRESSION) {
356*a4b16dadSTom Zanussi params.extra_dst_buff = inst->dc_data->ovf_buff_p;
357*a4b16dadSTom Zanussi ovf_buff_sz = inst->dc_data->ovf_buff_sz;
358*a4b16dadSTom Zanussi params.sz_extra_dst_buff = ovf_buff_sz;
359*a4b16dadSTom Zanussi }
360*a4b16dadSTom Zanussi
361*a4b16dadSTom Zanussi ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, areq->src, areq->dst,
362*a4b16dadSTom Zanussi &qat_req->buf, ¶ms, f);
363*a4b16dadSTom Zanussi if (unlikely(ret))
364*a4b16dadSTom Zanussi return ret;
365*a4b16dadSTom Zanussi
366*a4b16dadSTom Zanussi sfbuf = qat_req->buf.blp;
367*a4b16dadSTom Zanussi dfbuf = qat_req->buf.bloutp;
368*a4b16dadSTom Zanussi qat_req->qat_compression_ctx = ctx;
369*a4b16dadSTom Zanussi qat_req->acompress_req = areq;
370*a4b16dadSTom Zanussi qat_req->dir = dir;
371*a4b16dadSTom Zanussi
372*a4b16dadSTom Zanussi if (dir == COMPRESSION) {
373*a4b16dadSTom Zanussi qat_req->actual_dlen = dlen;
374*a4b16dadSTom Zanussi dlen += ovf_buff_sz;
375*a4b16dadSTom Zanussi qat_comp_create_compression_req(ctx->comp_ctx, req,
376*a4b16dadSTom Zanussi (u64)(__force long)sfbuf, slen,
377*a4b16dadSTom Zanussi (u64)(__force long)dfbuf, dlen,
378*a4b16dadSTom Zanussi (u64)(__force long)qat_req);
379*a4b16dadSTom Zanussi } else {
380*a4b16dadSTom Zanussi qat_comp_create_decompression_req(ctx->comp_ctx, req,
381*a4b16dadSTom Zanussi (u64)(__force long)sfbuf, slen,
382*a4b16dadSTom Zanussi (u64)(__force long)dfbuf, dlen,
383*a4b16dadSTom Zanussi (u64)(__force long)qat_req);
384*a4b16dadSTom Zanussi }
385*a4b16dadSTom Zanussi
386*a4b16dadSTom Zanussi ret = qat_alg_send_dc_message(qat_req, inst, &areq->base);
387*a4b16dadSTom Zanussi if (ret == -ENOSPC)
388*a4b16dadSTom Zanussi qat_bl_free_bufl(inst->accel_dev, &qat_req->buf);
389*a4b16dadSTom Zanussi
390*a4b16dadSTom Zanussi return ret;
391*a4b16dadSTom Zanussi }
392*a4b16dadSTom Zanussi
qat_comp_alg_compress(struct acomp_req * req)393*a4b16dadSTom Zanussi static int qat_comp_alg_compress(struct acomp_req *req)
394*a4b16dadSTom Zanussi {
395*a4b16dadSTom Zanussi return qat_comp_alg_compress_decompress(req, COMPRESSION, 0, 0, 0, 0);
396*a4b16dadSTom Zanussi }
397*a4b16dadSTom Zanussi
qat_comp_alg_decompress(struct acomp_req * req)398*a4b16dadSTom Zanussi static int qat_comp_alg_decompress(struct acomp_req *req)
399*a4b16dadSTom Zanussi {
400*a4b16dadSTom Zanussi return qat_comp_alg_compress_decompress(req, DECOMPRESSION, 0, 0, 0, 0);
401*a4b16dadSTom Zanussi }
402*a4b16dadSTom Zanussi
qat_comp_alg_rfc1950_compress(struct acomp_req * req)403*a4b16dadSTom Zanussi static int qat_comp_alg_rfc1950_compress(struct acomp_req *req)
404*a4b16dadSTom Zanussi {
405*a4b16dadSTom Zanussi if (!req->dst && req->dlen != 0)
406*a4b16dadSTom Zanussi return -EINVAL;
407*a4b16dadSTom Zanussi
408*a4b16dadSTom Zanussi if (req->dst && req->dlen <= QAT_RFC_1950_HDR_SIZE + QAT_RFC_1950_FOOTER_SIZE)
409*a4b16dadSTom Zanussi return -EINVAL;
410*a4b16dadSTom Zanussi
411*a4b16dadSTom Zanussi return qat_comp_alg_compress_decompress(req, COMPRESSION, 0, 0,
412*a4b16dadSTom Zanussi QAT_RFC_1950_HDR_SIZE,
413*a4b16dadSTom Zanussi QAT_RFC_1950_FOOTER_SIZE);
414*a4b16dadSTom Zanussi }
415*a4b16dadSTom Zanussi
qat_comp_alg_rfc1950_decompress(struct acomp_req * req)416*a4b16dadSTom Zanussi static int qat_comp_alg_rfc1950_decompress(struct acomp_req *req)
417*a4b16dadSTom Zanussi {
418*a4b16dadSTom Zanussi struct crypto_acomp *acomp_tfm = crypto_acomp_reqtfm(req);
419*a4b16dadSTom Zanussi struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm);
420*a4b16dadSTom Zanussi struct qat_compression_ctx *ctx = crypto_tfm_ctx(tfm);
421*a4b16dadSTom Zanussi struct adf_accel_dev *accel_dev = ctx->inst->accel_dev;
422*a4b16dadSTom Zanussi u16 zlib_header;
423*a4b16dadSTom Zanussi int ret;
424*a4b16dadSTom Zanussi
425*a4b16dadSTom Zanussi if (req->slen <= QAT_RFC_1950_HDR_SIZE + QAT_RFC_1950_FOOTER_SIZE)
426*a4b16dadSTom Zanussi return -EBADMSG;
427*a4b16dadSTom Zanussi
428*a4b16dadSTom Zanussi scatterwalk_map_and_copy(&zlib_header, req->src, 0, QAT_RFC_1950_HDR_SIZE, 0);
429*a4b16dadSTom Zanussi
430*a4b16dadSTom Zanussi ret = parse_zlib_header(zlib_header);
431*a4b16dadSTom Zanussi if (ret) {
432*a4b16dadSTom Zanussi dev_dbg(&GET_DEV(accel_dev), "Error parsing zlib header\n");
433*a4b16dadSTom Zanussi return ret;
434*a4b16dadSTom Zanussi }
435*a4b16dadSTom Zanussi
436*a4b16dadSTom Zanussi return qat_comp_alg_compress_decompress(req, DECOMPRESSION, QAT_RFC_1950_HDR_SIZE,
437*a4b16dadSTom Zanussi QAT_RFC_1950_FOOTER_SIZE, 0, 0);
438*a4b16dadSTom Zanussi }
439*a4b16dadSTom Zanussi
440*a4b16dadSTom Zanussi static struct acomp_alg qat_acomp[] = { {
441*a4b16dadSTom Zanussi .base = {
442*a4b16dadSTom Zanussi .cra_name = "deflate",
443*a4b16dadSTom Zanussi .cra_driver_name = "qat_deflate",
444*a4b16dadSTom Zanussi .cra_priority = 4001,
445*a4b16dadSTom Zanussi .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
446*a4b16dadSTom Zanussi .cra_ctxsize = sizeof(struct qat_compression_ctx),
447*a4b16dadSTom Zanussi .cra_module = THIS_MODULE,
448*a4b16dadSTom Zanussi },
449*a4b16dadSTom Zanussi .init = qat_comp_alg_init_tfm,
450*a4b16dadSTom Zanussi .exit = qat_comp_alg_exit_tfm,
451*a4b16dadSTom Zanussi .compress = qat_comp_alg_compress,
452*a4b16dadSTom Zanussi .decompress = qat_comp_alg_decompress,
453*a4b16dadSTom Zanussi .dst_free = sgl_free,
454*a4b16dadSTom Zanussi .reqsize = sizeof(struct qat_compression_req),
455*a4b16dadSTom Zanussi }, {
456*a4b16dadSTom Zanussi .base = {
457*a4b16dadSTom Zanussi .cra_name = "zlib-deflate",
458*a4b16dadSTom Zanussi .cra_driver_name = "qat_zlib_deflate",
459*a4b16dadSTom Zanussi .cra_priority = 4001,
460*a4b16dadSTom Zanussi .cra_flags = CRYPTO_ALG_ASYNC,
461*a4b16dadSTom Zanussi .cra_ctxsize = sizeof(struct qat_compression_ctx),
462*a4b16dadSTom Zanussi .cra_module = THIS_MODULE,
463*a4b16dadSTom Zanussi },
464*a4b16dadSTom Zanussi .init = qat_comp_alg_rfc1950_init_tfm,
465*a4b16dadSTom Zanussi .exit = qat_comp_alg_exit_tfm,
466*a4b16dadSTom Zanussi .compress = qat_comp_alg_rfc1950_compress,
467*a4b16dadSTom Zanussi .decompress = qat_comp_alg_rfc1950_decompress,
468*a4b16dadSTom Zanussi .dst_free = sgl_free,
469*a4b16dadSTom Zanussi .reqsize = sizeof(struct qat_compression_req),
470*a4b16dadSTom Zanussi } };
471*a4b16dadSTom Zanussi
qat_comp_algs_register(void)472*a4b16dadSTom Zanussi int qat_comp_algs_register(void)
473*a4b16dadSTom Zanussi {
474*a4b16dadSTom Zanussi int ret = 0;
475*a4b16dadSTom Zanussi
476*a4b16dadSTom Zanussi mutex_lock(&algs_lock);
477*a4b16dadSTom Zanussi if (++active_devs == 1)
478*a4b16dadSTom Zanussi ret = crypto_register_acomps(qat_acomp, ARRAY_SIZE(qat_acomp));
479*a4b16dadSTom Zanussi mutex_unlock(&algs_lock);
480*a4b16dadSTom Zanussi return ret;
481*a4b16dadSTom Zanussi }
482*a4b16dadSTom Zanussi
qat_comp_algs_unregister(void)483*a4b16dadSTom Zanussi void qat_comp_algs_unregister(void)
484*a4b16dadSTom Zanussi {
485*a4b16dadSTom Zanussi mutex_lock(&algs_lock);
486*a4b16dadSTom Zanussi if (--active_devs == 0)
487*a4b16dadSTom Zanussi crypto_unregister_acomps(qat_acomp, ARRAY_SIZE(qat_acomp));
488*a4b16dadSTom Zanussi mutex_unlock(&algs_lock);
489*a4b16dadSTom Zanussi }
490