1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2020 Chelsio Communications.  All rights reserved. */
3 
4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
5 
6 #include <linux/skbuff.h>
7 #include <linux/module.h>
8 #include <linux/highmem.h>
9 #include <linux/ip.h>
10 #include <net/ipv6.h>
11 #include <linux/netdevice.h>
12 #include <crypto/aes.h>
13 #include "chcr_ktls.h"
14 
15 static LIST_HEAD(uld_ctx_list);
16 static DEFINE_MUTEX(dev_mutex);
17 
18 /* chcr_get_nfrags_to_send: get the remaining nfrags after start offset
19  * @skb: skb
20  * @start: start offset.
21  * @len: how much data to send after @start
22  */
chcr_get_nfrags_to_send(struct sk_buff * skb,u32 start,u32 len)23 static int chcr_get_nfrags_to_send(struct sk_buff *skb, u32 start, u32 len)
24 {
25 	struct skb_shared_info *si = skb_shinfo(skb);
26 	u32 frag_size, skb_linear_data_len = skb_headlen(skb);
27 	u8 nfrags = 0, frag_idx = 0;
28 	skb_frag_t *frag;
29 
30 	/* if its a linear skb then return 1 */
31 	if (!skb_is_nonlinear(skb))
32 		return 1;
33 
34 	if (unlikely(start < skb_linear_data_len)) {
35 		frag_size = min(len, skb_linear_data_len - start);
36 	} else {
37 		start -= skb_linear_data_len;
38 
39 		frag = &si->frags[frag_idx];
40 		frag_size = skb_frag_size(frag);
41 		while (start >= frag_size) {
42 			start -= frag_size;
43 			frag_idx++;
44 			frag = &si->frags[frag_idx];
45 			frag_size = skb_frag_size(frag);
46 		}
47 		frag_size = min(len, skb_frag_size(frag) - start);
48 	}
49 	len -= frag_size;
50 	nfrags++;
51 
52 	while (len) {
53 		frag_size = min(len, skb_frag_size(&si->frags[frag_idx]));
54 		len -= frag_size;
55 		nfrags++;
56 		frag_idx++;
57 	}
58 	return nfrags;
59 }
60 
61 static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info);
62 static void clear_conn_resources(struct chcr_ktls_info *tx_info);
63 /*
64  * chcr_ktls_save_keys: calculate and save crypto keys.
65  * @tx_info - driver specific tls info.
66  * @crypto_info - tls crypto information.
67  * @direction - TX/RX direction.
68  * return - SUCCESS/FAILURE.
69  */
chcr_ktls_save_keys(struct chcr_ktls_info * tx_info,struct tls_crypto_info * crypto_info,enum tls_offload_ctx_dir direction)70 static int chcr_ktls_save_keys(struct chcr_ktls_info *tx_info,
71 			       struct tls_crypto_info *crypto_info,
72 			       enum tls_offload_ctx_dir direction)
73 {
74 	int ck_size, key_ctx_size, mac_key_size, keylen, ghash_size, ret;
75 	unsigned char ghash_h[TLS_CIPHER_AES_GCM_256_TAG_SIZE];
76 	struct tls12_crypto_info_aes_gcm_128 *info_128_gcm;
77 	struct ktls_key_ctx *kctx = &tx_info->key_ctx;
78 	struct crypto_aes_ctx aes_ctx;
79 	unsigned char *key, *salt;
80 
81 	switch (crypto_info->cipher_type) {
82 	case TLS_CIPHER_AES_GCM_128:
83 		info_128_gcm =
84 			(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
85 		keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE;
86 		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
87 		tx_info->salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE;
88 		mac_key_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
89 		tx_info->iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
90 		tx_info->iv = be64_to_cpu(*(__be64 *)info_128_gcm->iv);
91 
92 		ghash_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
93 		key = info_128_gcm->key;
94 		salt = info_128_gcm->salt;
95 		tx_info->record_no = *(u64 *)info_128_gcm->rec_seq;
96 
97 		/* The SCMD fields used when encrypting a full TLS
98 		 * record. Its a one time calculation till the
99 		 * connection exists.
100 		 */
101 		tx_info->scmd0_seqno_numivs =
102 			SCMD_SEQ_NO_CTRL_V(CHCR_SCMD_SEQ_NO_CTRL_64BIT) |
103 			SCMD_CIPH_AUTH_SEQ_CTRL_F |
104 			SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_TLS) |
105 			SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_GCM) |
106 			SCMD_AUTH_MODE_V(CHCR_SCMD_AUTH_MODE_GHASH) |
107 			SCMD_IV_SIZE_V(TLS_CIPHER_AES_GCM_128_IV_SIZE >> 1) |
108 			SCMD_NUM_IVS_V(1);
109 
110 		/* keys will be sent inline. */
111 		tx_info->scmd0_ivgen_hdrlen = SCMD_KEY_CTX_INLINE_F;
112 
113 		/* The SCMD fields used when encrypting a partial TLS
114 		 * record (no trailer and possibly a truncated payload).
115 		 */
116 		tx_info->scmd0_short_seqno_numivs =
117 			SCMD_CIPH_AUTH_SEQ_CTRL_F |
118 			SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) |
119 			SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_CTR) |
120 			SCMD_IV_SIZE_V(AES_BLOCK_LEN >> 1);
121 
122 		tx_info->scmd0_short_ivgen_hdrlen =
123 			tx_info->scmd0_ivgen_hdrlen | SCMD_AADIVDROP_F;
124 
125 		break;
126 
127 	default:
128 		pr_err("GCM: cipher type 0x%x not supported\n",
129 		       crypto_info->cipher_type);
130 		ret = -EINVAL;
131 		goto out;
132 	}
133 
134 	key_ctx_size = CHCR_KTLS_KEY_CTX_LEN +
135 		       roundup(keylen, 16) + ghash_size;
136 	/* Calculate the H = CIPH(K, 0 repeated 16 times).
137 	 * It will go in key context
138 	 */
139 
140 	ret = aes_expandkey(&aes_ctx, key, keylen);
141 	if (ret)
142 		goto out;
143 
144 	memset(ghash_h, 0, ghash_size);
145 	aes_encrypt(&aes_ctx, ghash_h, ghash_h);
146 	memzero_explicit(&aes_ctx, sizeof(aes_ctx));
147 
148 	/* fill the Key context */
149 	if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
150 		kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
151 						 mac_key_size,
152 						 key_ctx_size >> 4);
153 	} else {
154 		ret = -EINVAL;
155 		goto out;
156 	}
157 
158 	memcpy(kctx->salt, salt, tx_info->salt_size);
159 	memcpy(kctx->key, key, keylen);
160 	memcpy(kctx->key + keylen, ghash_h, ghash_size);
161 	tx_info->key_ctx_len = key_ctx_size;
162 
163 out:
164 	return ret;
165 }
166 
167 /*
168  * chcr_ktls_act_open_req: creates TCB entry for ipv4 connection.
169  * @sk - tcp socket.
170  * @tx_info - driver specific tls info.
171  * @atid - connection active tid.
172  * return - send success/failure.
173  */
chcr_ktls_act_open_req(struct sock * sk,struct chcr_ktls_info * tx_info,int atid)174 static int chcr_ktls_act_open_req(struct sock *sk,
175 				  struct chcr_ktls_info *tx_info,
176 				  int atid)
177 {
178 	struct inet_sock *inet = inet_sk(sk);
179 	struct cpl_t6_act_open_req *cpl6;
180 	struct cpl_act_open_req *cpl;
181 	struct sk_buff *skb;
182 	unsigned int len;
183 	int qid_atid;
184 	u64 options;
185 
186 	len = sizeof(*cpl6);
187 	skb = alloc_skb(len, GFP_KERNEL);
188 	if (unlikely(!skb))
189 		return -ENOMEM;
190 	/* mark it a control pkt */
191 	set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
192 
193 	cpl6 = __skb_put_zero(skb, len);
194 	cpl = (struct cpl_act_open_req *)cpl6;
195 	INIT_TP_WR(cpl6, 0);
196 	qid_atid = TID_QID_V(tx_info->rx_qid) |
197 		   TID_TID_V(atid);
198 	OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid));
199 	cpl->local_port = inet->inet_sport;
200 	cpl->peer_port = inet->inet_dport;
201 	cpl->local_ip = inet->inet_rcv_saddr;
202 	cpl->peer_ip = inet->inet_daddr;
203 
204 	/* fill first 64 bit option field. */
205 	options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
206 		  SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
207 	cpl->opt0 = cpu_to_be64(options);
208 
209 	/* next 64 bit option field. */
210 	options =
211 		TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
212 	cpl->opt2 = htonl(options);
213 
214 	return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
215 }
216 
217 #if IS_ENABLED(CONFIG_IPV6)
218 /*
219  * chcr_ktls_act_open_req6: creates TCB entry for ipv6 connection.
220  * @sk - tcp socket.
221  * @tx_info - driver specific tls info.
222  * @atid - connection active tid.
223  * return - send success/failure.
224  */
chcr_ktls_act_open_req6(struct sock * sk,struct chcr_ktls_info * tx_info,int atid)225 static int chcr_ktls_act_open_req6(struct sock *sk,
226 				   struct chcr_ktls_info *tx_info,
227 				   int atid)
228 {
229 	struct inet_sock *inet = inet_sk(sk);
230 	struct cpl_t6_act_open_req6 *cpl6;
231 	struct cpl_act_open_req6 *cpl;
232 	struct sk_buff *skb;
233 	unsigned int len;
234 	int qid_atid;
235 	u64 options;
236 
237 	len = sizeof(*cpl6);
238 	skb = alloc_skb(len, GFP_KERNEL);
239 	if (unlikely(!skb))
240 		return -ENOMEM;
241 	/* mark it a control pkt */
242 	set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
243 
244 	cpl6 = __skb_put_zero(skb, len);
245 	cpl = (struct cpl_act_open_req6 *)cpl6;
246 	INIT_TP_WR(cpl6, 0);
247 	qid_atid = TID_QID_V(tx_info->rx_qid) | TID_TID_V(atid);
248 	OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, qid_atid));
249 	cpl->local_port = inet->inet_sport;
250 	cpl->peer_port = inet->inet_dport;
251 	cpl->local_ip_hi = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[0];
252 	cpl->local_ip_lo = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[8];
253 	cpl->peer_ip_hi = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[0];
254 	cpl->peer_ip_lo = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[8];
255 
256 	/* first 64 bit option field. */
257 	options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
258 		  SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
259 	cpl->opt0 = cpu_to_be64(options);
260 	/* next 64 bit option field. */
261 	options =
262 		TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
263 	cpl->opt2 = htonl(options);
264 
265 	return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
266 }
267 #endif /* #if IS_ENABLED(CONFIG_IPV6) */
268 
269 /*
270  * chcr_setup_connection:  create a TCB entry so that TP will form tcp packets.
271  * @sk - tcp socket.
272  * @tx_info - driver specific tls info.
273  * return: NET_TX_OK/NET_XMIT_DROP
274  */
chcr_setup_connection(struct sock * sk,struct chcr_ktls_info * tx_info)275 static int chcr_setup_connection(struct sock *sk,
276 				 struct chcr_ktls_info *tx_info)
277 {
278 	struct tid_info *t = &tx_info->adap->tids;
279 	int atid, ret = 0;
280 
281 	atid = cxgb4_alloc_atid(t, tx_info);
282 	if (atid == -1)
283 		return -EINVAL;
284 
285 	tx_info->atid = atid;
286 
287 	if (tx_info->ip_family == AF_INET) {
288 		ret = chcr_ktls_act_open_req(sk, tx_info, atid);
289 #if IS_ENABLED(CONFIG_IPV6)
290 	} else {
291 		ret = cxgb4_clip_get(tx_info->netdev, (const u32 *)
292 				     &sk->sk_v6_rcv_saddr,
293 				     1);
294 		if (ret)
295 			return ret;
296 		ret = chcr_ktls_act_open_req6(sk, tx_info, atid);
297 #endif
298 	}
299 
300 	/* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret
301 	 * success, if any other return type clear atid and return that failure.
302 	 */
303 	if (ret) {
304 		if (ret == NET_XMIT_CN) {
305 			ret = 0;
306 		} else {
307 #if IS_ENABLED(CONFIG_IPV6)
308 			/* clear clip entry */
309 			if (tx_info->ip_family == AF_INET6)
310 				cxgb4_clip_release(tx_info->netdev,
311 						   (const u32 *)
312 						   &sk->sk_v6_rcv_saddr,
313 						   1);
314 #endif
315 			cxgb4_free_atid(t, atid);
316 		}
317 	}
318 
319 	return ret;
320 }
321 
322 /*
323  * chcr_set_tcb_field: update tcb fields.
324  * @tx_info - driver specific tls info.
325  * @word - TCB word.
326  * @mask - TCB word related mask.
327  * @val - TCB word related value.
328  * @no_reply - set 1 if not looking for TP response.
329  */
chcr_set_tcb_field(struct chcr_ktls_info * tx_info,u16 word,u64 mask,u64 val,int no_reply)330 static int chcr_set_tcb_field(struct chcr_ktls_info *tx_info, u16 word,
331 			      u64 mask, u64 val, int no_reply)
332 {
333 	struct cpl_set_tcb_field *req;
334 	struct sk_buff *skb;
335 
336 	skb = alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_ATOMIC);
337 	if (!skb)
338 		return -ENOMEM;
339 
340 	req = (struct cpl_set_tcb_field *)__skb_put_zero(skb, sizeof(*req));
341 	INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, tx_info->tid);
342 	req->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
343 				NO_REPLY_V(no_reply));
344 	req->word_cookie = htons(TCB_WORD_V(word));
345 	req->mask = cpu_to_be64(mask);
346 	req->val = cpu_to_be64(val);
347 
348 	set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
349 	return cxgb4_ofld_send(tx_info->netdev, skb);
350 }
351 
352 /*
353  * chcr_ktls_dev_del:  call back for tls_dev_del.
354  * Remove the tid and l2t entry and close the connection.
355  * it per connection basis.
356  * @netdev - net device.
357  * @tls_cts - tls context.
358  * @direction - TX/RX crypto direction
359  */
chcr_ktls_dev_del(struct net_device * netdev,struct tls_context * tls_ctx,enum tls_offload_ctx_dir direction)360 static void chcr_ktls_dev_del(struct net_device *netdev,
361 			      struct tls_context *tls_ctx,
362 			      enum tls_offload_ctx_dir direction)
363 {
364 	struct chcr_ktls_ofld_ctx_tx *tx_ctx =
365 				chcr_get_ktls_tx_context(tls_ctx);
366 	struct chcr_ktls_info *tx_info = tx_ctx->chcr_info;
367 	struct ch_ktls_port_stats_debug *port_stats;
368 	struct chcr_ktls_uld_ctx *u_ctx;
369 
370 	if (!tx_info)
371 		return;
372 
373 	u_ctx = tx_info->adap->uld[CXGB4_ULD_KTLS].handle;
374 	if (u_ctx && u_ctx->detach)
375 		return;
376 	/* clear l2t entry */
377 	if (tx_info->l2te)
378 		cxgb4_l2t_release(tx_info->l2te);
379 
380 #if IS_ENABLED(CONFIG_IPV6)
381 	/* clear clip entry */
382 	if (tx_info->ip_family == AF_INET6)
383 		cxgb4_clip_release(netdev, (const u32 *)
384 				   &tx_info->sk->sk_v6_rcv_saddr,
385 				   1);
386 #endif
387 
388 	/* clear tid */
389 	if (tx_info->tid != -1) {
390 		cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
391 				 tx_info->tid, tx_info->ip_family);
392 
393 		xa_erase(&u_ctx->tid_list, tx_info->tid);
394 	}
395 
396 	port_stats = &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id];
397 	atomic64_inc(&port_stats->ktls_tx_connection_close);
398 	kvfree(tx_info);
399 	tx_ctx->chcr_info = NULL;
400 	/* release module refcount */
401 	module_put(THIS_MODULE);
402 }
403 
404 /*
405  * chcr_ktls_dev_add:  call back for tls_dev_add.
406  * Create a tcb entry for TP. Also add l2t entry for the connection. And
407  * generate keys & save those keys locally.
408  * @netdev - net device.
409  * @tls_cts - tls context.
410  * @direction - TX/RX crypto direction
411  * return: SUCCESS/FAILURE.
412  */
chcr_ktls_dev_add(struct net_device * netdev,struct sock * sk,enum tls_offload_ctx_dir direction,struct tls_crypto_info * crypto_info,u32 start_offload_tcp_sn)413 static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk,
414 			     enum tls_offload_ctx_dir direction,
415 			     struct tls_crypto_info *crypto_info,
416 			     u32 start_offload_tcp_sn)
417 {
418 	struct tls_context *tls_ctx = tls_get_ctx(sk);
419 	struct ch_ktls_port_stats_debug *port_stats;
420 	struct chcr_ktls_ofld_ctx_tx *tx_ctx;
421 	struct chcr_ktls_uld_ctx *u_ctx;
422 	struct chcr_ktls_info *tx_info;
423 	struct dst_entry *dst;
424 	struct adapter *adap;
425 	struct port_info *pi;
426 	struct neighbour *n;
427 	u8 daaddr[16];
428 	int ret = -1;
429 
430 	tx_ctx = chcr_get_ktls_tx_context(tls_ctx);
431 
432 	pi = netdev_priv(netdev);
433 	adap = pi->adapter;
434 	port_stats = &adap->ch_ktls_stats.ktls_port[pi->port_id];
435 	atomic64_inc(&port_stats->ktls_tx_connection_open);
436 	u_ctx = adap->uld[CXGB4_ULD_KTLS].handle;
437 
438 	if (direction == TLS_OFFLOAD_CTX_DIR_RX) {
439 		pr_err("not expecting for RX direction\n");
440 		goto out;
441 	}
442 
443 	if (tx_ctx->chcr_info)
444 		goto out;
445 
446 	if (u_ctx && u_ctx->detach)
447 		goto out;
448 
449 	tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL);
450 	if (!tx_info)
451 		goto out;
452 
453 	tx_info->sk = sk;
454 	spin_lock_init(&tx_info->lock);
455 	/* initialize tid and atid to -1, 0 is a also a valid id. */
456 	tx_info->tid = -1;
457 	tx_info->atid = -1;
458 
459 	tx_info->adap = adap;
460 	tx_info->netdev = netdev;
461 	tx_info->first_qset = pi->first_qset;
462 	tx_info->tx_chan = pi->tx_chan;
463 	tx_info->smt_idx = pi->smt_idx;
464 	tx_info->port_id = pi->port_id;
465 	tx_info->prev_ack = 0;
466 	tx_info->prev_win = 0;
467 
468 	tx_info->rx_qid = chcr_get_first_rx_qid(adap);
469 	if (unlikely(tx_info->rx_qid < 0))
470 		goto free_tx_info;
471 
472 	tx_info->prev_seq = start_offload_tcp_sn;
473 	tx_info->tcp_start_seq_number = start_offload_tcp_sn;
474 
475 	/* save crypto keys */
476 	ret = chcr_ktls_save_keys(tx_info, crypto_info, direction);
477 	if (ret < 0)
478 		goto free_tx_info;
479 
480 	/* get peer ip */
481 	if (sk->sk_family == AF_INET) {
482 		memcpy(daaddr, &sk->sk_daddr, 4);
483 		tx_info->ip_family = AF_INET;
484 #if IS_ENABLED(CONFIG_IPV6)
485 	} else {
486 		if (!ipv6_only_sock(sk) &&
487 		    ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) {
488 			memcpy(daaddr, &sk->sk_daddr, 4);
489 			tx_info->ip_family = AF_INET;
490 		} else {
491 			memcpy(daaddr, sk->sk_v6_daddr.in6_u.u6_addr8, 16);
492 			tx_info->ip_family = AF_INET6;
493 		}
494 #endif
495 	}
496 
497 	/* get the l2t index */
498 	dst = sk_dst_get(sk);
499 	if (!dst) {
500 		pr_err("DST entry not found\n");
501 		goto free_tx_info;
502 	}
503 	n = dst_neigh_lookup(dst, daaddr);
504 	if (!n || !n->dev) {
505 		pr_err("neighbour not found\n");
506 		dst_release(dst);
507 		goto free_tx_info;
508 	}
509 	tx_info->l2te  = cxgb4_l2t_get(adap->l2t, n, n->dev, 0);
510 
511 	neigh_release(n);
512 	dst_release(dst);
513 
514 	if (!tx_info->l2te) {
515 		pr_err("l2t entry not found\n");
516 		goto free_tx_info;
517 	}
518 
519 	/* Driver shouldn't be removed until any single connection exists */
520 	if (!try_module_get(THIS_MODULE))
521 		goto free_l2t;
522 
523 	init_completion(&tx_info->completion);
524 	/* create a filter and call cxgb4_l2t_send to send the packet out, which
525 	 * will take care of updating l2t entry in hw if not already done.
526 	 */
527 	tx_info->open_state = CH_KTLS_OPEN_PENDING;
528 
529 	if (chcr_setup_connection(sk, tx_info))
530 		goto put_module;
531 
532 	/* Wait for reply */
533 	wait_for_completion_timeout(&tx_info->completion, 30 * HZ);
534 	spin_lock_bh(&tx_info->lock);
535 	if (tx_info->open_state) {
536 		/* need to wait for hw response, can't free tx_info yet. */
537 		if (tx_info->open_state == CH_KTLS_OPEN_PENDING)
538 			tx_info->pending_close = true;
539 		else
540 			spin_unlock_bh(&tx_info->lock);
541 		/* if in pending close, free the lock after the cleanup */
542 		goto put_module;
543 	}
544 	spin_unlock_bh(&tx_info->lock);
545 
546 	/* initialize tcb */
547 	reinit_completion(&tx_info->completion);
548 	/* mark it pending for hw response */
549 	tx_info->open_state = CH_KTLS_OPEN_PENDING;
550 
551 	if (chcr_init_tcb_fields(tx_info))
552 		goto free_tid;
553 
554 	/* Wait for reply */
555 	wait_for_completion_timeout(&tx_info->completion, 30 * HZ);
556 	spin_lock_bh(&tx_info->lock);
557 	if (tx_info->open_state) {
558 		/* need to wait for hw response, can't free tx_info yet. */
559 		tx_info->pending_close = true;
560 		/* free the lock after cleanup */
561 		goto free_tid;
562 	}
563 	spin_unlock_bh(&tx_info->lock);
564 
565 	if (!cxgb4_check_l2t_valid(tx_info->l2te))
566 		goto free_tid;
567 
568 	atomic64_inc(&port_stats->ktls_tx_ctx);
569 	tx_ctx->chcr_info = tx_info;
570 
571 	return 0;
572 
573 free_tid:
574 #if IS_ENABLED(CONFIG_IPV6)
575 	/* clear clip entry */
576 	if (tx_info->ip_family == AF_INET6)
577 		cxgb4_clip_release(netdev, (const u32 *)
578 				   &sk->sk_v6_rcv_saddr,
579 				   1);
580 #endif
581 	cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
582 			 tx_info->tid, tx_info->ip_family);
583 
584 	xa_erase(&u_ctx->tid_list, tx_info->tid);
585 
586 put_module:
587 	/* release module refcount */
588 	module_put(THIS_MODULE);
589 free_l2t:
590 	cxgb4_l2t_release(tx_info->l2te);
591 free_tx_info:
592 	if (tx_info->pending_close)
593 		spin_unlock_bh(&tx_info->lock);
594 	else
595 		kvfree(tx_info);
596 out:
597 	atomic64_inc(&port_stats->ktls_tx_connection_fail);
598 	return -1;
599 }
600 
601 /*
602  * chcr_init_tcb_fields:  Initialize tcb fields to handle TCP seq number
603  *			  handling.
604  * @tx_info - driver specific tls info.
605  * return: NET_TX_OK/NET_XMIT_DROP
606  */
chcr_init_tcb_fields(struct chcr_ktls_info * tx_info)607 static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info)
608 {
609 	int  ret = 0;
610 
611 	/* set tcb in offload and bypass */
612 	ret =
613 	chcr_set_tcb_field(tx_info, TCB_T_FLAGS_W,
614 			   TCB_T_FLAGS_V(TF_CORE_BYPASS_F | TF_NON_OFFLOAD_F),
615 			   TCB_T_FLAGS_V(TF_CORE_BYPASS_F), 1);
616 	if (ret)
617 		return ret;
618 	/* reset snd_una and snd_next fields in tcb */
619 	ret = chcr_set_tcb_field(tx_info, TCB_SND_UNA_RAW_W,
620 				 TCB_SND_NXT_RAW_V(TCB_SND_NXT_RAW_M) |
621 				 TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M),
622 				 0, 1);
623 	if (ret)
624 		return ret;
625 
626 	/* reset send max */
627 	ret = chcr_set_tcb_field(tx_info, TCB_SND_MAX_RAW_W,
628 				 TCB_SND_MAX_RAW_V(TCB_SND_MAX_RAW_M),
629 				 0, 1);
630 	if (ret)
631 		return ret;
632 
633 	/* update l2t index and request for tp reply to confirm tcb is
634 	 * initialised to handle tx traffic.
635 	 */
636 	ret = chcr_set_tcb_field(tx_info, TCB_L2T_IX_W,
637 				 TCB_L2T_IX_V(TCB_L2T_IX_M),
638 				 TCB_L2T_IX_V(tx_info->l2te->idx), 0);
639 	return ret;
640 }
641 
642 /*
643  * chcr_ktls_cpl_act_open_rpl: connection reply received from TP.
644  */
chcr_ktls_cpl_act_open_rpl(struct adapter * adap,unsigned char * input)645 static int chcr_ktls_cpl_act_open_rpl(struct adapter *adap,
646 				      unsigned char *input)
647 {
648 	const struct cpl_act_open_rpl *p = (void *)input;
649 	struct chcr_ktls_info *tx_info = NULL;
650 	struct chcr_ktls_ofld_ctx_tx *tx_ctx;
651 	struct chcr_ktls_uld_ctx *u_ctx;
652 	unsigned int atid, tid, status;
653 	struct tls_context *tls_ctx;
654 	struct tid_info *t;
655 	int ret = 0;
656 
657 	tid = GET_TID(p);
658 	status = AOPEN_STATUS_G(ntohl(p->atid_status));
659 	atid = TID_TID_G(AOPEN_ATID_G(ntohl(p->atid_status)));
660 
661 	t = &adap->tids;
662 	tx_info = lookup_atid(t, atid);
663 
664 	if (!tx_info || tx_info->atid != atid) {
665 		pr_err("%s: incorrect tx_info or atid\n", __func__);
666 		return -1;
667 	}
668 
669 	cxgb4_free_atid(t, atid);
670 	tx_info->atid = -1;
671 
672 	spin_lock(&tx_info->lock);
673 	/* HW response is very close, finish pending cleanup */
674 	if (tx_info->pending_close) {
675 		spin_unlock(&tx_info->lock);
676 		if (!status) {
677 			cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
678 					 tid, tx_info->ip_family);
679 		}
680 		kvfree(tx_info);
681 		return 0;
682 	}
683 
684 	if (!status) {
685 		tx_info->tid = tid;
686 		cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family);
687 		/* Adding tid */
688 		tls_ctx = tls_get_ctx(tx_info->sk);
689 		tx_ctx = chcr_get_ktls_tx_context(tls_ctx);
690 		u_ctx = adap->uld[CXGB4_ULD_KTLS].handle;
691 		if (u_ctx) {
692 			ret = xa_insert_bh(&u_ctx->tid_list, tid, tx_ctx,
693 					   GFP_NOWAIT);
694 			if (ret < 0) {
695 				pr_err("%s: Failed to allocate tid XA entry = %d\n",
696 				       __func__, tx_info->tid);
697 				tx_info->open_state = CH_KTLS_OPEN_FAILURE;
698 				goto out;
699 			}
700 		}
701 		tx_info->open_state = CH_KTLS_OPEN_SUCCESS;
702 	} else {
703 		tx_info->open_state = CH_KTLS_OPEN_FAILURE;
704 	}
705 out:
706 	spin_unlock(&tx_info->lock);
707 
708 	complete(&tx_info->completion);
709 	return ret;
710 }
711 
712 /*
713  * chcr_ktls_cpl_set_tcb_rpl: TCB reply received from TP.
714  */
chcr_ktls_cpl_set_tcb_rpl(struct adapter * adap,unsigned char * input)715 static int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input)
716 {
717 	const struct cpl_set_tcb_rpl *p = (void *)input;
718 	struct chcr_ktls_info *tx_info = NULL;
719 	struct tid_info *t;
720 	u32 tid;
721 
722 	tid = GET_TID(p);
723 
724 	t = &adap->tids;
725 	tx_info = lookup_tid(t, tid);
726 
727 	if (!tx_info || tx_info->tid != tid) {
728 		pr_err("%s: incorrect tx_info or tid\n", __func__);
729 		return -1;
730 	}
731 
732 	spin_lock(&tx_info->lock);
733 	if (tx_info->pending_close) {
734 		spin_unlock(&tx_info->lock);
735 		kvfree(tx_info);
736 		return 0;
737 	}
738 	tx_info->open_state = CH_KTLS_OPEN_SUCCESS;
739 	spin_unlock(&tx_info->lock);
740 
741 	complete(&tx_info->completion);
742 	return 0;
743 }
744 
__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info * tx_info,u32 tid,void * pos,u16 word,struct sge_eth_txq * q,u64 mask,u64 val,u32 reply)745 static void *__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,
746 					u32 tid, void *pos, u16 word,
747 					struct sge_eth_txq *q, u64 mask,
748 					u64 val, u32 reply)
749 {
750 	struct cpl_set_tcb_field_core *cpl;
751 	struct ulptx_idata *idata;
752 	struct ulp_txpkt *txpkt;
753 
754 	/* ULP_TXPKT */
755 	txpkt = pos;
756 	txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
757 				ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
758 				ULP_TXPKT_FID_V(q->q.cntxt_id) |
759 				ULP_TXPKT_RO_F);
760 	txpkt->len = htonl(DIV_ROUND_UP(CHCR_SET_TCB_FIELD_LEN, 16));
761 
762 	/* ULPTX_IDATA sub-command */
763 	idata = (struct ulptx_idata *)(txpkt + 1);
764 	idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM));
765 	idata->len = htonl(sizeof(*cpl));
766 	pos = idata + 1;
767 
768 	cpl = pos;
769 	/* CPL_SET_TCB_FIELD */
770 	OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
771 	cpl->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
772 			NO_REPLY_V(!reply));
773 	cpl->word_cookie = htons(TCB_WORD_V(word));
774 	cpl->mask = cpu_to_be64(mask);
775 	cpl->val = cpu_to_be64(val);
776 
777 	/* ULPTX_NOOP */
778 	idata = (struct ulptx_idata *)(cpl + 1);
779 	idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
780 	idata->len = htonl(0);
781 	pos = idata + 1;
782 
783 	return pos;
784 }
785 
786 
787 /*
788  * chcr_write_cpl_set_tcb_ulp: update tcb values.
789  * TCB is responsible to create tcp headers, so all the related values
790  * should be correctly updated.
791  * @tx_info - driver specific tls info.
792  * @q - tx queue on which packet is going out.
793  * @tid - TCB identifier.
794  * @pos - current index where should we start writing.
795  * @word - TCB word.
796  * @mask - TCB word related mask.
797  * @val - TCB word related value.
798  * @reply - set 1 if looking for TP response.
799  * return - next position to write.
800  */
chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info * tx_info,struct sge_eth_txq * q,u32 tid,void * pos,u16 word,u64 mask,u64 val,u32 reply)801 static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,
802 					struct sge_eth_txq *q, u32 tid,
803 					void *pos, u16 word, u64 mask,
804 					u64 val, u32 reply)
805 {
806 	int left = (void *)q->q.stat - pos;
807 
808 	if (unlikely(left < CHCR_SET_TCB_FIELD_LEN)) {
809 		if (!left) {
810 			pos = q->q.desc;
811 		} else {
812 			u8 buf[48] = {0};
813 
814 			__chcr_write_cpl_set_tcb_ulp(tx_info, tid, buf, word, q,
815 						     mask, val, reply);
816 
817 			return chcr_copy_to_txd(buf, &q->q, pos,
818 						CHCR_SET_TCB_FIELD_LEN);
819 		}
820 	}
821 
822 	pos = __chcr_write_cpl_set_tcb_ulp(tx_info, tid, pos, word, q,
823 					   mask, val, reply);
824 
825 	/* check again if we are at the end of the queue */
826 	if (left == CHCR_SET_TCB_FIELD_LEN)
827 		pos = q->q.desc;
828 
829 	return pos;
830 }
831 
832 /*
833  * chcr_ktls_xmit_tcb_cpls: update tcb entry so that TP will create the header
834  * with updated values like tcp seq, ack, window etc.
835  * @tx_info - driver specific tls info.
836  * @q - TX queue.
837  * @tcp_seq
838  * @tcp_ack
839  * @tcp_win
840  * return: NETDEV_TX_BUSY/NET_TX_OK.
841  */
chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info * tx_info,struct sge_eth_txq * q,u64 tcp_seq,u64 tcp_ack,u64 tcp_win,bool offset)842 static int chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info *tx_info,
843 				   struct sge_eth_txq *q, u64 tcp_seq,
844 				   u64 tcp_ack, u64 tcp_win, bool offset)
845 {
846 	bool first_wr = ((tx_info->prev_ack == 0) && (tx_info->prev_win == 0));
847 	struct ch_ktls_port_stats_debug *port_stats;
848 	u32 len, cpl = 0, ndesc, wr_len, wr_mid = 0;
849 	struct fw_ulptx_wr *wr;
850 	int credits;
851 	void *pos;
852 
853 	wr_len = sizeof(*wr);
854 	/* there can be max 4 cpls, check if we have enough credits */
855 	len = wr_len + 4 * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
856 	ndesc = DIV_ROUND_UP(len, 64);
857 
858 	credits = chcr_txq_avail(&q->q) - ndesc;
859 	if (unlikely(credits < 0)) {
860 		chcr_eth_txq_stop(q);
861 		return NETDEV_TX_BUSY;
862 	}
863 
864 	if (unlikely(credits < ETHTXQ_STOP_THRES)) {
865 		chcr_eth_txq_stop(q);
866 		wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
867 	}
868 
869 	pos = &q->q.desc[q->q.pidx];
870 	/* make space for WR, we'll fill it later when we know all the cpls
871 	 * being sent out and have complete length.
872 	 */
873 	wr = pos;
874 	pos += wr_len;
875 	/* update tx_max if its a re-transmit or the first wr */
876 	if (first_wr || tcp_seq != tx_info->prev_seq) {
877 		pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
878 						 TCB_TX_MAX_W,
879 						 TCB_TX_MAX_V(TCB_TX_MAX_M),
880 						 TCB_TX_MAX_V(tcp_seq), 0);
881 		cpl++;
882 	}
883 	/* reset snd una if it's a re-transmit pkt */
884 	if (tcp_seq != tx_info->prev_seq || offset) {
885 		/* reset snd_una */
886 		port_stats =
887 			&tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id];
888 		pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
889 						 TCB_SND_UNA_RAW_W,
890 						 TCB_SND_UNA_RAW_V
891 						 (TCB_SND_UNA_RAW_M),
892 						 TCB_SND_UNA_RAW_V(0), 0);
893 		if (tcp_seq != tx_info->prev_seq)
894 			atomic64_inc(&port_stats->ktls_tx_ooo);
895 		cpl++;
896 	}
897 	/* update ack */
898 	if (first_wr || tx_info->prev_ack != tcp_ack) {
899 		pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
900 						 TCB_RCV_NXT_W,
901 						 TCB_RCV_NXT_V(TCB_RCV_NXT_M),
902 						 TCB_RCV_NXT_V(tcp_ack), 0);
903 		tx_info->prev_ack = tcp_ack;
904 		cpl++;
905 	}
906 	/* update receive window */
907 	if (first_wr || tx_info->prev_win != tcp_win) {
908 		chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
909 					   TCB_RCV_WND_W,
910 					   TCB_RCV_WND_V(TCB_RCV_WND_M),
911 					   TCB_RCV_WND_V(tcp_win), 0);
912 		tx_info->prev_win = tcp_win;
913 		cpl++;
914 	}
915 
916 	if (cpl) {
917 		/* get the actual length */
918 		len = wr_len + cpl * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
919 		/* ULPTX wr */
920 		wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
921 		wr->cookie = 0;
922 		/* fill len in wr field */
923 		wr->flowid_len16 = htonl(wr_mid |
924 					 FW_WR_LEN16_V(DIV_ROUND_UP(len, 16)));
925 
926 		ndesc = DIV_ROUND_UP(len, 64);
927 		chcr_txq_advance(&q->q, ndesc);
928 		cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
929 	}
930 	return 0;
931 }
932 
933 /*
934  * chcr_ktls_get_tx_flits
935  * returns number of flits to be sent out, it includes key context length, WR
936  * size and skb fragments.
937  */
938 static unsigned int
chcr_ktls_get_tx_flits(u32 nr_frags,unsigned int key_ctx_len)939 chcr_ktls_get_tx_flits(u32 nr_frags, unsigned int key_ctx_len)
940 {
941 	return chcr_sgl_len(nr_frags) +
942 	       DIV_ROUND_UP(key_ctx_len + CHCR_KTLS_WR_SIZE, 8);
943 }
944 
945 /*
946  * chcr_ktls_check_tcp_options: To check if there is any TCP option available
947  * other than timestamp.
948  * @skb - skb contains partial record..
949  * return: 1 / 0
950  */
951 static int
chcr_ktls_check_tcp_options(struct tcphdr * tcp)952 chcr_ktls_check_tcp_options(struct tcphdr *tcp)
953 {
954 	int cnt, opt, optlen;
955 	u_char *cp;
956 
957 	cp = (u_char *)(tcp + 1);
958 	cnt = (tcp->doff << 2) - sizeof(struct tcphdr);
959 	for (; cnt > 0; cnt -= optlen, cp += optlen) {
960 		opt = cp[0];
961 		if (opt == TCPOPT_EOL)
962 			break;
963 		if (opt == TCPOPT_NOP) {
964 			optlen = 1;
965 		} else {
966 			if (cnt < 2)
967 				break;
968 			optlen = cp[1];
969 			if (optlen < 2 || optlen > cnt)
970 				break;
971 		}
972 		switch (opt) {
973 		case TCPOPT_NOP:
974 			break;
975 		default:
976 			return 1;
977 		}
978 	}
979 	return 0;
980 }
981 
982 /*
983  * chcr_ktls_write_tcp_options : TP can't send out all the options, we need to
984  * send out separately.
985  * @tx_info - driver specific tls info.
986  * @skb - skb contains partial record..
987  * @q - TX queue.
988  * @tx_chan - channel number.
989  * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
990  */
991 static int
chcr_ktls_write_tcp_options(struct chcr_ktls_info * tx_info,struct sk_buff * skb,struct sge_eth_txq * q,uint32_t tx_chan)992 chcr_ktls_write_tcp_options(struct chcr_ktls_info *tx_info, struct sk_buff *skb,
993 			    struct sge_eth_txq *q, uint32_t tx_chan)
994 {
995 	struct fw_eth_tx_pkt_wr *wr;
996 	struct cpl_tx_pkt_core *cpl;
997 	u32 ctrl, iplen, maclen;
998 	struct ipv6hdr *ip6;
999 	unsigned int ndesc;
1000 	struct tcphdr *tcp;
1001 	int len16, pktlen;
1002 	struct iphdr *ip;
1003 	u32 wr_mid = 0;
1004 	int credits;
1005 	u8 buf[150];
1006 	u64 cntrl1;
1007 	void *pos;
1008 
1009 	iplen = skb_network_header_len(skb);
1010 	maclen = skb_mac_header_len(skb);
1011 
1012 	/* packet length = eth hdr len + ip hdr len + tcp hdr len
1013 	 * (including options).
1014 	 */
1015 	pktlen = skb_tcp_all_headers(skb);
1016 
1017 	ctrl = sizeof(*cpl) + pktlen;
1018 	len16 = DIV_ROUND_UP(sizeof(*wr) + ctrl, 16);
1019 	/* check how many descriptors needed */
1020 	ndesc = DIV_ROUND_UP(len16, 4);
1021 
1022 	credits = chcr_txq_avail(&q->q) - ndesc;
1023 	if (unlikely(credits < 0)) {
1024 		chcr_eth_txq_stop(q);
1025 		return NETDEV_TX_BUSY;
1026 	}
1027 
1028 	if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1029 		chcr_eth_txq_stop(q);
1030 		wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1031 	}
1032 
1033 	pos = &q->q.desc[q->q.pidx];
1034 	wr = pos;
1035 
1036 	/* Firmware work request header */
1037 	wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
1038 			       FW_WR_IMMDLEN_V(ctrl));
1039 
1040 	wr->equiq_to_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1041 	wr->r3 = 0;
1042 
1043 	cpl = (void *)(wr + 1);
1044 
1045 	/* CPL header */
1046 	cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) | TXPKT_INTF_V(tx_chan) |
1047 			   TXPKT_PF_V(tx_info->adap->pf));
1048 	cpl->pack = 0;
1049 	cpl->len = htons(pktlen);
1050 
1051 	memcpy(buf, skb->data, pktlen);
1052 	if (!IS_ENABLED(CONFIG_IPV6) || tx_info->ip_family == AF_INET) {
1053 		/* we need to correct ip header len */
1054 		ip = (struct iphdr *)(buf + maclen);
1055 		ip->tot_len = htons(pktlen - maclen);
1056 		cntrl1 = TXPKT_CSUM_TYPE_V(TX_CSUM_TCPIP);
1057 	} else {
1058 		ip6 = (struct ipv6hdr *)(buf + maclen);
1059 		ip6->payload_len = htons(pktlen - maclen - iplen);
1060 		cntrl1 = TXPKT_CSUM_TYPE_V(TX_CSUM_TCPIP6);
1061 	}
1062 
1063 	cntrl1 |= T6_TXPKT_ETHHDR_LEN_V(maclen - ETH_HLEN) |
1064 		  TXPKT_IPHDR_LEN_V(iplen);
1065 	/* checksum offload */
1066 	cpl->ctrl1 = cpu_to_be64(cntrl1);
1067 
1068 	pos = cpl + 1;
1069 
1070 	/* now take care of the tcp header, if fin is not set then clear push
1071 	 * bit as well, and if fin is set, it will be sent at the last so we
1072 	 * need to update the tcp sequence number as per the last packet.
1073 	 */
1074 	tcp = (struct tcphdr *)(buf + maclen + iplen);
1075 
1076 	if (!tcp->fin)
1077 		tcp->psh = 0;
1078 	else
1079 		tcp->seq = htonl(tx_info->prev_seq);
1080 
1081 	chcr_copy_to_txd(buf, &q->q, pos, pktlen);
1082 
1083 	chcr_txq_advance(&q->q, ndesc);
1084 	cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1085 	return 0;
1086 }
1087 
1088 /*
1089  * chcr_ktls_xmit_wr_complete: This sends out the complete record. If an skb
1090  * received has partial end part of the record, send out the complete record, so
1091  * that crypto block will be able to generate TAG/HASH.
1092  * @skb - segment which has complete or partial end part.
1093  * @tx_info - driver specific tls info.
1094  * @q - TX queue.
1095  * @tcp_seq
1096  * @tcp_push - tcp push bit.
1097  * @mss - segment size.
1098  * return: NETDEV_TX_BUSY/NET_TX_OK.
1099  */
chcr_ktls_xmit_wr_complete(struct sk_buff * skb,struct chcr_ktls_info * tx_info,struct sge_eth_txq * q,u32 tcp_seq,bool is_last_wr,u32 data_len,u32 skb_offset,u32 nfrags,bool tcp_push,u32 mss)1100 static int chcr_ktls_xmit_wr_complete(struct sk_buff *skb,
1101 				      struct chcr_ktls_info *tx_info,
1102 				      struct sge_eth_txq *q, u32 tcp_seq,
1103 				      bool is_last_wr, u32 data_len,
1104 				      u32 skb_offset, u32 nfrags,
1105 				      bool tcp_push, u32 mss)
1106 {
1107 	u32 len16, wr_mid = 0, flits = 0, ndesc, cipher_start;
1108 	struct adapter *adap = tx_info->adap;
1109 	int credits, left, last_desc;
1110 	struct tx_sw_desc *sgl_sdesc;
1111 	struct cpl_tx_data *tx_data;
1112 	struct cpl_tx_sec_pdu *cpl;
1113 	struct ulptx_idata *idata;
1114 	struct ulp_txpkt *ulptx;
1115 	struct fw_ulptx_wr *wr;
1116 	void *pos;
1117 	u64 *end;
1118 
1119 	/* get the number of flits required */
1120 	flits = chcr_ktls_get_tx_flits(nfrags, tx_info->key_ctx_len);
1121 	/* number of descriptors */
1122 	ndesc = chcr_flits_to_desc(flits);
1123 	/* check if enough credits available */
1124 	credits = chcr_txq_avail(&q->q) - ndesc;
1125 	if (unlikely(credits < 0)) {
1126 		chcr_eth_txq_stop(q);
1127 		return NETDEV_TX_BUSY;
1128 	}
1129 
1130 	if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1131 		/* Credits are below the threshold values, stop the queue after
1132 		 * injecting the Work Request for this packet.
1133 		 */
1134 		chcr_eth_txq_stop(q);
1135 		wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1136 	}
1137 
1138 	last_desc = q->q.pidx + ndesc - 1;
1139 	if (last_desc >= q->q.size)
1140 		last_desc -= q->q.size;
1141 	sgl_sdesc = &q->q.sdesc[last_desc];
1142 
1143 	if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
1144 		memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1145 		q->mapping_err++;
1146 		return NETDEV_TX_BUSY;
1147 	}
1148 
1149 	if (!is_last_wr)
1150 		skb_get(skb);
1151 
1152 	pos = &q->q.desc[q->q.pidx];
1153 	end = (u64 *)pos + flits;
1154 	/* FW_ULPTX_WR */
1155 	wr = pos;
1156 	/* WR will need len16 */
1157 	len16 = DIV_ROUND_UP(flits, 2);
1158 	wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1159 	wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1160 	wr->cookie = 0;
1161 	pos += sizeof(*wr);
1162 	/* ULP_TXPKT */
1163 	ulptx = pos;
1164 	ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1165 				ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1166 				ULP_TXPKT_FID_V(q->q.cntxt_id) |
1167 				ULP_TXPKT_RO_F);
1168 	ulptx->len = htonl(len16 - 1);
1169 	/* ULPTX_IDATA sub-command */
1170 	idata = (struct ulptx_idata *)(ulptx + 1);
1171 	idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1172 	/* idata length will include cpl_tx_sec_pdu + key context size +
1173 	 * cpl_tx_data header.
1174 	 */
1175 	idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +
1176 			   sizeof(*tx_data));
1177 	/* SEC CPL */
1178 	cpl = (struct cpl_tx_sec_pdu *)(idata + 1);
1179 	cpl->op_ivinsrtofst =
1180 		htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
1181 		      CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |
1182 		      CPL_TX_SEC_PDU_PLACEHOLDER_V(1) |
1183 		      CPL_TX_SEC_PDU_IVINSRTOFST_V(TLS_HEADER_SIZE + 1));
1184 	cpl->pldlen = htonl(data_len);
1185 
1186 	/* encryption should start after tls header size + iv size */
1187 	cipher_start = TLS_HEADER_SIZE + tx_info->iv_size + 1;
1188 
1189 	cpl->aadstart_cipherstop_hi =
1190 		htonl(CPL_TX_SEC_PDU_AADSTART_V(1) |
1191 		      CPL_TX_SEC_PDU_AADSTOP_V(TLS_HEADER_SIZE) |
1192 		      CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));
1193 
1194 	/* authentication will also start after tls header + iv size */
1195 	cpl->cipherstop_lo_authinsert =
1196 	htonl(CPL_TX_SEC_PDU_AUTHSTART_V(cipher_start) |
1197 	      CPL_TX_SEC_PDU_AUTHSTOP_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE) |
1198 	      CPL_TX_SEC_PDU_AUTHINSERT_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE));
1199 
1200 	/* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1201 	cpl->seqno_numivs = htonl(tx_info->scmd0_seqno_numivs);
1202 	cpl->ivgen_hdrlen = htonl(tx_info->scmd0_ivgen_hdrlen);
1203 	cpl->scmd1 = cpu_to_be64(tx_info->record_no);
1204 
1205 	pos = cpl + 1;
1206 	/* check if space left to fill the keys */
1207 	left = (void *)q->q.stat - pos;
1208 	if (!left) {
1209 		left = (void *)end - (void *)q->q.stat;
1210 		pos = q->q.desc;
1211 		end = pos + left;
1212 	}
1213 
1214 	pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,
1215 			       tx_info->key_ctx_len);
1216 	left = (void *)q->q.stat - pos;
1217 
1218 	if (!left) {
1219 		left = (void *)end - (void *)q->q.stat;
1220 		pos = q->q.desc;
1221 		end = pos + left;
1222 	}
1223 	/* CPL_TX_DATA */
1224 	tx_data = (void *)pos;
1225 	OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1226 	tx_data->len = htonl(TX_DATA_MSS_V(mss) | TX_LENGTH_V(data_len));
1227 
1228 	tx_data->rsvd = htonl(tcp_seq);
1229 
1230 	tx_data->flags = htonl(TX_BYPASS_F);
1231 	if (tcp_push)
1232 		tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1233 
1234 	/* check left again, it might go beyond queue limit */
1235 	pos = tx_data + 1;
1236 	left = (void *)q->q.stat - pos;
1237 
1238 	/* check the position again */
1239 	if (!left) {
1240 		left = (void *)end - (void *)q->q.stat;
1241 		pos = q->q.desc;
1242 		end = pos + left;
1243 	}
1244 
1245 	/* send the complete packet except the header */
1246 	cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,
1247 				skb_offset, data_len);
1248 	sgl_sdesc->skb = skb;
1249 
1250 	chcr_txq_advance(&q->q, ndesc);
1251 	cxgb4_ring_tx_db(adap, &q->q, ndesc);
1252 	atomic64_inc(&adap->ch_ktls_stats.ktls_tx_send_records);
1253 
1254 	return 0;
1255 }
1256 
1257 /*
1258  * chcr_ktls_xmit_wr_short: This is to send out partial records. If its
1259  * a middle part of a record, fetch the prior data to make it 16 byte aligned
1260  * and then only send it out.
1261  *
1262  * @skb - skb contains partial record..
1263  * @tx_info - driver specific tls info.
1264  * @q - TX queue.
1265  * @tcp_seq
1266  * @tcp_push - tcp push bit.
1267  * @mss - segment size.
1268  * @tls_rec_offset - offset from start of the tls record.
1269  * @perior_data - data before the current segment, required to make this record
1270  *		  16 byte aligned.
1271  * @prior_data_len - prior_data length (less than 16)
1272  * return: NETDEV_TX_BUSY/NET_TX_OK.
1273  */
chcr_ktls_xmit_wr_short(struct sk_buff * skb,struct chcr_ktls_info * tx_info,struct sge_eth_txq * q,u32 tcp_seq,bool tcp_push,u32 mss,u32 tls_rec_offset,u8 * prior_data,u32 prior_data_len,u32 data_len,u32 skb_offset)1274 static int chcr_ktls_xmit_wr_short(struct sk_buff *skb,
1275 				   struct chcr_ktls_info *tx_info,
1276 				   struct sge_eth_txq *q,
1277 				   u32 tcp_seq, bool tcp_push, u32 mss,
1278 				   u32 tls_rec_offset, u8 *prior_data,
1279 				   u32 prior_data_len, u32 data_len,
1280 				   u32 skb_offset)
1281 {
1282 	u32 len16, wr_mid = 0, cipher_start, nfrags;
1283 	struct adapter *adap = tx_info->adap;
1284 	unsigned int flits = 0, ndesc;
1285 	int credits, left, last_desc;
1286 	struct tx_sw_desc *sgl_sdesc;
1287 	struct cpl_tx_data *tx_data;
1288 	struct cpl_tx_sec_pdu *cpl;
1289 	struct ulptx_idata *idata;
1290 	struct ulp_txpkt *ulptx;
1291 	struct fw_ulptx_wr *wr;
1292 	__be64 iv_record;
1293 	void *pos;
1294 	u64 *end;
1295 
1296 	nfrags = chcr_get_nfrags_to_send(skb, skb_offset, data_len);
1297 	/* get the number of flits required, it's a partial record so 2 flits
1298 	 * (AES_BLOCK_SIZE) will be added.
1299 	 */
1300 	flits = chcr_ktls_get_tx_flits(nfrags, tx_info->key_ctx_len) + 2;
1301 	/* get the correct 8 byte IV of this record */
1302 	iv_record = cpu_to_be64(tx_info->iv + tx_info->record_no);
1303 	/* If it's a middle record and not 16 byte aligned to run AES CTR, need
1304 	 * to make it 16 byte aligned. So atleadt 2 extra flits of immediate
1305 	 * data will be added.
1306 	 */
1307 	if (prior_data_len)
1308 		flits += 2;
1309 	/* number of descriptors */
1310 	ndesc = chcr_flits_to_desc(flits);
1311 	/* check if enough credits available */
1312 	credits = chcr_txq_avail(&q->q) - ndesc;
1313 	if (unlikely(credits < 0)) {
1314 		chcr_eth_txq_stop(q);
1315 		return NETDEV_TX_BUSY;
1316 	}
1317 
1318 	if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1319 		chcr_eth_txq_stop(q);
1320 		wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1321 	}
1322 
1323 	last_desc = q->q.pidx + ndesc - 1;
1324 	if (last_desc >= q->q.size)
1325 		last_desc -= q->q.size;
1326 	sgl_sdesc = &q->q.sdesc[last_desc];
1327 
1328 	if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
1329 		memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1330 		q->mapping_err++;
1331 		return NETDEV_TX_BUSY;
1332 	}
1333 
1334 	pos = &q->q.desc[q->q.pidx];
1335 	end = (u64 *)pos + flits;
1336 	/* FW_ULPTX_WR */
1337 	wr = pos;
1338 	/* WR will need len16 */
1339 	len16 = DIV_ROUND_UP(flits, 2);
1340 	wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1341 	wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1342 	wr->cookie = 0;
1343 	pos += sizeof(*wr);
1344 	/* ULP_TXPKT */
1345 	ulptx = pos;
1346 	ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1347 				ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1348 				ULP_TXPKT_FID_V(q->q.cntxt_id) |
1349 				ULP_TXPKT_RO_F);
1350 	ulptx->len = htonl(len16 - 1);
1351 	/* ULPTX_IDATA sub-command */
1352 	idata = (struct ulptx_idata *)(ulptx + 1);
1353 	idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1354 	/* idata length will include cpl_tx_sec_pdu + key context size +
1355 	 * cpl_tx_data header.
1356 	 */
1357 	idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +
1358 			   sizeof(*tx_data) + AES_BLOCK_LEN + prior_data_len);
1359 	/* SEC CPL */
1360 	cpl = (struct cpl_tx_sec_pdu *)(idata + 1);
1361 	/* cipher start will have tls header + iv size extra if its a header
1362 	 * part of tls record. else only 16 byte IV will be added.
1363 	 */
1364 	cipher_start =
1365 		AES_BLOCK_LEN + 1 +
1366 		(!tls_rec_offset ? TLS_HEADER_SIZE + tx_info->iv_size : 0);
1367 
1368 	cpl->op_ivinsrtofst =
1369 		htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
1370 		      CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |
1371 		      CPL_TX_SEC_PDU_IVINSRTOFST_V(1));
1372 	cpl->pldlen = htonl(data_len + AES_BLOCK_LEN + prior_data_len);
1373 	cpl->aadstart_cipherstop_hi =
1374 		htonl(CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));
1375 	cpl->cipherstop_lo_authinsert = 0;
1376 	/* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1377 	cpl->seqno_numivs = htonl(tx_info->scmd0_short_seqno_numivs);
1378 	cpl->ivgen_hdrlen = htonl(tx_info->scmd0_short_ivgen_hdrlen);
1379 	cpl->scmd1 = 0;
1380 
1381 	pos = cpl + 1;
1382 	/* check if space left to fill the keys */
1383 	left = (void *)q->q.stat - pos;
1384 	if (!left) {
1385 		left = (void *)end - (void *)q->q.stat;
1386 		pos = q->q.desc;
1387 		end = pos + left;
1388 	}
1389 
1390 	pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,
1391 			       tx_info->key_ctx_len);
1392 	left = (void *)q->q.stat - pos;
1393 
1394 	if (!left) {
1395 		left = (void *)end - (void *)q->q.stat;
1396 		pos = q->q.desc;
1397 		end = pos + left;
1398 	}
1399 	/* CPL_TX_DATA */
1400 	tx_data = (void *)pos;
1401 	OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1402 	tx_data->len = htonl(TX_DATA_MSS_V(mss) |
1403 			     TX_LENGTH_V(data_len + prior_data_len));
1404 	tx_data->rsvd = htonl(tcp_seq);
1405 	tx_data->flags = htonl(TX_BYPASS_F);
1406 	if (tcp_push)
1407 		tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1408 
1409 	/* check left again, it might go beyond queue limit */
1410 	pos = tx_data + 1;
1411 	left = (void *)q->q.stat - pos;
1412 
1413 	/* check the position again */
1414 	if (!left) {
1415 		left = (void *)end - (void *)q->q.stat;
1416 		pos = q->q.desc;
1417 		end = pos + left;
1418 	}
1419 	/* copy the 16 byte IV for AES-CTR, which includes 4 bytes of salt, 8
1420 	 * bytes of actual IV and 4 bytes of 16 byte-sequence.
1421 	 */
1422 	memcpy(pos, tx_info->key_ctx.salt, tx_info->salt_size);
1423 	memcpy(pos + tx_info->salt_size, &iv_record, tx_info->iv_size);
1424 	*(__be32 *)(pos + tx_info->salt_size + tx_info->iv_size) =
1425 		htonl(2 + (tls_rec_offset ? ((tls_rec_offset -
1426 		(TLS_HEADER_SIZE + tx_info->iv_size)) / AES_BLOCK_LEN) : 0));
1427 
1428 	pos += 16;
1429 	/* Prior_data_len will always be less than 16 bytes, fill the
1430 	 * prio_data_len after AES_CTRL_BLOCK and clear the remaining length
1431 	 * to 0.
1432 	 */
1433 	if (prior_data_len)
1434 		pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);
1435 	/* send the complete packet except the header */
1436 	cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,
1437 				skb_offset, data_len);
1438 	sgl_sdesc->skb = skb;
1439 
1440 	chcr_txq_advance(&q->q, ndesc);
1441 	cxgb4_ring_tx_db(adap, &q->q, ndesc);
1442 
1443 	return 0;
1444 }
1445 
1446 /*
1447  * chcr_ktls_tx_plaintxt: This handler will take care of the records which has
1448  * only plain text (only tls header and iv)
1449  * @tx_info - driver specific tls info.
1450  * @skb - skb contains partial record..
1451  * @tcp_seq
1452  * @mss - segment size.
1453  * @tcp_push - tcp push bit.
1454  * @q - TX queue.
1455  * @port_id : port number
1456  * @perior_data - data before the current segment, required to make this record
1457  *		 16 byte aligned.
1458  * @prior_data_len - prior_data length (less than 16)
1459  * return: NETDEV_TX_BUSY/NET_TX_OK.
1460  */
chcr_ktls_tx_plaintxt(struct chcr_ktls_info * tx_info,struct sk_buff * skb,u32 tcp_seq,u32 mss,bool tcp_push,struct sge_eth_txq * q,u32 port_id,u8 * prior_data,u32 data_len,u32 skb_offset,u32 prior_data_len)1461 static int chcr_ktls_tx_plaintxt(struct chcr_ktls_info *tx_info,
1462 				 struct sk_buff *skb, u32 tcp_seq, u32 mss,
1463 				 bool tcp_push, struct sge_eth_txq *q,
1464 				 u32 port_id, u8 *prior_data,
1465 				 u32 data_len, u32 skb_offset,
1466 				 u32 prior_data_len)
1467 {
1468 	int credits, left, len16, last_desc;
1469 	unsigned int flits = 0, ndesc;
1470 	struct tx_sw_desc *sgl_sdesc;
1471 	struct cpl_tx_data *tx_data;
1472 	struct ulptx_idata *idata;
1473 	struct ulp_txpkt *ulptx;
1474 	struct fw_ulptx_wr *wr;
1475 	u32 wr_mid = 0, nfrags;
1476 	void *pos;
1477 	u64 *end;
1478 
1479 	flits = DIV_ROUND_UP(CHCR_PLAIN_TX_DATA_LEN, 8);
1480 	nfrags = chcr_get_nfrags_to_send(skb, skb_offset, data_len);
1481 	flits += chcr_sgl_len(nfrags);
1482 	if (prior_data_len)
1483 		flits += 2;
1484 
1485 	/* WR will need len16 */
1486 	len16 = DIV_ROUND_UP(flits, 2);
1487 	/* check how many descriptors needed */
1488 	ndesc = DIV_ROUND_UP(flits, 8);
1489 
1490 	credits = chcr_txq_avail(&q->q) - ndesc;
1491 	if (unlikely(credits < 0)) {
1492 		chcr_eth_txq_stop(q);
1493 		return NETDEV_TX_BUSY;
1494 	}
1495 
1496 	if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1497 		chcr_eth_txq_stop(q);
1498 		wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1499 	}
1500 
1501 	last_desc = q->q.pidx + ndesc - 1;
1502 	if (last_desc >= q->q.size)
1503 		last_desc -= q->q.size;
1504 	sgl_sdesc = &q->q.sdesc[last_desc];
1505 
1506 	if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb,
1507 				   sgl_sdesc->addr) < 0)) {
1508 		memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1509 		q->mapping_err++;
1510 		return NETDEV_TX_BUSY;
1511 	}
1512 
1513 	pos = &q->q.desc[q->q.pidx];
1514 	end = (u64 *)pos + flits;
1515 	/* FW_ULPTX_WR */
1516 	wr = pos;
1517 	wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1518 	wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1519 	wr->cookie = 0;
1520 	/* ULP_TXPKT */
1521 	ulptx = (struct ulp_txpkt *)(wr + 1);
1522 	ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1523 			ULP_TXPKT_DATAMODIFY_V(0) |
1524 			ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1525 			ULP_TXPKT_DEST_V(0) |
1526 			ULP_TXPKT_FID_V(q->q.cntxt_id) | ULP_TXPKT_RO_V(1));
1527 	ulptx->len = htonl(len16 - 1);
1528 	/* ULPTX_IDATA sub-command */
1529 	idata = (struct ulptx_idata *)(ulptx + 1);
1530 	idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1531 	idata->len = htonl(sizeof(*tx_data) + prior_data_len);
1532 	/* CPL_TX_DATA */
1533 	tx_data = (struct cpl_tx_data *)(idata + 1);
1534 	OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1535 	tx_data->len = htonl(TX_DATA_MSS_V(mss) |
1536 			     TX_LENGTH_V(data_len + prior_data_len));
1537 	/* set tcp seq number */
1538 	tx_data->rsvd = htonl(tcp_seq);
1539 	tx_data->flags = htonl(TX_BYPASS_F);
1540 	if (tcp_push)
1541 		tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1542 
1543 	pos = tx_data + 1;
1544 	/* apart from prior_data_len, we should set remaining part of 16 bytes
1545 	 * to be zero.
1546 	 */
1547 	if (prior_data_len)
1548 		pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);
1549 
1550 	/* check left again, it might go beyond queue limit */
1551 	left = (void *)q->q.stat - pos;
1552 
1553 	/* check the position again */
1554 	if (!left) {
1555 		left = (void *)end - (void *)q->q.stat;
1556 		pos = q->q.desc;
1557 		end = pos + left;
1558 	}
1559 	/* send the complete packet including the header */
1560 	cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,
1561 				skb_offset, data_len);
1562 	sgl_sdesc->skb = skb;
1563 
1564 	chcr_txq_advance(&q->q, ndesc);
1565 	cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1566 	return 0;
1567 }
1568 
chcr_ktls_tunnel_pkt(struct chcr_ktls_info * tx_info,struct sk_buff * skb,struct sge_eth_txq * q)1569 static int chcr_ktls_tunnel_pkt(struct chcr_ktls_info *tx_info,
1570 				struct sk_buff *skb,
1571 				struct sge_eth_txq *q)
1572 {
1573 	u32 ctrl, iplen, maclen, wr_mid = 0, len16;
1574 	struct tx_sw_desc *sgl_sdesc;
1575 	struct fw_eth_tx_pkt_wr *wr;
1576 	struct cpl_tx_pkt_core *cpl;
1577 	unsigned int flits, ndesc;
1578 	int credits, last_desc;
1579 	u64 cntrl1, *end;
1580 	void *pos;
1581 
1582 	ctrl = sizeof(*cpl);
1583 	flits = DIV_ROUND_UP(sizeof(*wr) + ctrl, 8);
1584 
1585 	flits += chcr_sgl_len(skb_shinfo(skb)->nr_frags + 1);
1586 	len16 = DIV_ROUND_UP(flits, 2);
1587 	/* check how many descriptors needed */
1588 	ndesc = DIV_ROUND_UP(flits, 8);
1589 
1590 	credits = chcr_txq_avail(&q->q) - ndesc;
1591 	if (unlikely(credits < 0)) {
1592 		chcr_eth_txq_stop(q);
1593 		return -ENOMEM;
1594 	}
1595 
1596 	if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1597 		chcr_eth_txq_stop(q);
1598 		wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1599 	}
1600 
1601 	last_desc = q->q.pidx + ndesc - 1;
1602 	if (last_desc >= q->q.size)
1603 		last_desc -= q->q.size;
1604 	sgl_sdesc = &q->q.sdesc[last_desc];
1605 
1606 	if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb,
1607 				   sgl_sdesc->addr) < 0)) {
1608 		memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1609 		q->mapping_err++;
1610 		return -ENOMEM;
1611 	}
1612 
1613 	iplen = skb_network_header_len(skb);
1614 	maclen = skb_mac_header_len(skb);
1615 
1616 	pos = &q->q.desc[q->q.pidx];
1617 	end = (u64 *)pos + flits;
1618 	wr = pos;
1619 
1620 	/* Firmware work request header */
1621 	wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
1622 			       FW_WR_IMMDLEN_V(ctrl));
1623 
1624 	wr->equiq_to_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1625 	wr->r3 = 0;
1626 
1627 	cpl = (void *)(wr + 1);
1628 
1629 	/* CPL header */
1630 	cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) |
1631 			   TXPKT_INTF_V(tx_info->tx_chan) |
1632 			   TXPKT_PF_V(tx_info->adap->pf));
1633 	cpl->pack = 0;
1634 	cntrl1 = TXPKT_CSUM_TYPE_V(tx_info->ip_family == AF_INET ?
1635 				   TX_CSUM_TCPIP : TX_CSUM_TCPIP6);
1636 	cntrl1 |= T6_TXPKT_ETHHDR_LEN_V(maclen - ETH_HLEN) |
1637 		  TXPKT_IPHDR_LEN_V(iplen);
1638 	/* checksum offload */
1639 	cpl->ctrl1 = cpu_to_be64(cntrl1);
1640 	cpl->len = htons(skb->len);
1641 
1642 	pos = cpl + 1;
1643 
1644 	cxgb4_write_sgl(skb, &q->q, pos, end, 0, sgl_sdesc->addr);
1645 	sgl_sdesc->skb = skb;
1646 	chcr_txq_advance(&q->q, ndesc);
1647 	cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1648 	return 0;
1649 }
1650 
1651 /*
1652  * chcr_ktls_copy_record_in_skb
1653  * @nskb - new skb where the frags to be added.
1654  * @skb - old skb, to copy socket and destructor details.
1655  * @record - specific record which has complete 16k record in frags.
1656  */
chcr_ktls_copy_record_in_skb(struct sk_buff * nskb,struct sk_buff * skb,struct tls_record_info * record)1657 static void chcr_ktls_copy_record_in_skb(struct sk_buff *nskb,
1658 					 struct sk_buff *skb,
1659 					 struct tls_record_info *record)
1660 {
1661 	int i = 0;
1662 
1663 	for (i = 0; i < record->num_frags; i++) {
1664 		skb_shinfo(nskb)->frags[i] = record->frags[i];
1665 		/* increase the frag ref count */
1666 		__skb_frag_ref(&skb_shinfo(nskb)->frags[i]);
1667 	}
1668 
1669 	skb_shinfo(nskb)->nr_frags = record->num_frags;
1670 	nskb->data_len = record->len;
1671 	nskb->len += record->len;
1672 	nskb->truesize += record->len;
1673 	nskb->sk = skb->sk;
1674 	nskb->destructor = skb->destructor;
1675 	refcount_add(nskb->truesize, &nskb->sk->sk_wmem_alloc);
1676 }
1677 
1678 /*
1679  * chcr_end_part_handler: This handler will handle the record which
1680  * is complete or if record's end part is received. T6 adapter has a issue that
1681  * it can't send out TAG with partial record so if its an end part then we have
1682  * to send TAG as well and for which we need to fetch the complete record and
1683  * send it to crypto module.
1684  * @tx_info - driver specific tls info.
1685  * @skb - skb contains partial record.
1686  * @record - complete record of 16K size.
1687  * @tcp_seq
1688  * @mss - segment size in which TP needs to chop a packet.
1689  * @tcp_push_no_fin - tcp push if fin is not set.
1690  * @q - TX queue.
1691  * @tls_end_offset - offset from end of the record.
1692  * @last wr : check if this is the last part of the skb going out.
1693  * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
1694  */
chcr_end_part_handler(struct chcr_ktls_info * tx_info,struct sk_buff * skb,struct tls_record_info * record,u32 tcp_seq,int mss,bool tcp_push_no_fin,struct sge_eth_txq * q,u32 skb_offset,u32 tls_end_offset,bool last_wr)1695 static int chcr_end_part_handler(struct chcr_ktls_info *tx_info,
1696 				 struct sk_buff *skb,
1697 				 struct tls_record_info *record,
1698 				 u32 tcp_seq, int mss, bool tcp_push_no_fin,
1699 				 struct sge_eth_txq *q, u32 skb_offset,
1700 				 u32 tls_end_offset, bool last_wr)
1701 {
1702 	bool free_skb_if_tx_fails = false;
1703 	struct sk_buff *nskb = NULL;
1704 
1705 	/* check if it is a complete record */
1706 	if (tls_end_offset == record->len) {
1707 		nskb = skb;
1708 		atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_complete_pkts);
1709 	} else {
1710 		nskb = alloc_skb(0, GFP_ATOMIC);
1711 		if (!nskb) {
1712 			dev_kfree_skb_any(skb);
1713 			return NETDEV_TX_BUSY;
1714 		}
1715 
1716 		/* copy complete record in skb */
1717 		chcr_ktls_copy_record_in_skb(nskb, skb, record);
1718 		/* packet is being sent from the beginning, update the tcp_seq
1719 		 * accordingly.
1720 		 */
1721 		tcp_seq = tls_record_start_seq(record);
1722 		/* reset skb offset */
1723 		skb_offset = 0;
1724 
1725 		if (last_wr)
1726 			dev_kfree_skb_any(skb);
1727 		else
1728 			free_skb_if_tx_fails = true;
1729 
1730 		last_wr = true;
1731 
1732 		atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_end_pkts);
1733 	}
1734 
1735 	if (chcr_ktls_xmit_wr_complete(nskb, tx_info, q, tcp_seq,
1736 				       last_wr, record->len, skb_offset,
1737 				       record->num_frags,
1738 				       (last_wr && tcp_push_no_fin),
1739 				       mss)) {
1740 		if (free_skb_if_tx_fails)
1741 			dev_kfree_skb_any(skb);
1742 		goto out;
1743 	}
1744 	tx_info->prev_seq = record->end_seq;
1745 	return 0;
1746 out:
1747 	dev_kfree_skb_any(nskb);
1748 	return NETDEV_TX_BUSY;
1749 }
1750 
1751 /*
1752  * chcr_short_record_handler: This handler will take care of the records which
1753  * doesn't have end part (1st part or the middle part(/s) of a record). In such
1754  * cases, AES CTR will be used in place of AES GCM to send out partial packet.
1755  * This partial record might be the first part of the record, or the middle
1756  * part. In case of middle record we should fetch the prior data to make it 16
1757  * byte aligned. If it has a partial tls header or iv then get to the start of
1758  * tls header. And if it has partial TAG, then remove the complete TAG and send
1759  * only the payload.
1760  * There is one more possibility that it gets a partial header, send that
1761  * portion as a plaintext.
1762  * @tx_info - driver specific tls info.
1763  * @skb - skb contains partial record..
1764  * @record - complete record of 16K size.
1765  * @tcp_seq
1766  * @mss - segment size in which TP needs to chop a packet.
1767  * @tcp_push_no_fin - tcp push if fin is not set.
1768  * @q - TX queue.
1769  * @tls_end_offset - offset from end of the record.
1770  * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
1771  */
chcr_short_record_handler(struct chcr_ktls_info * tx_info,struct sk_buff * skb,struct tls_record_info * record,u32 tcp_seq,int mss,bool tcp_push_no_fin,u32 data_len,u32 skb_offset,struct sge_eth_txq * q,u32 tls_end_offset)1772 static int chcr_short_record_handler(struct chcr_ktls_info *tx_info,
1773 				     struct sk_buff *skb,
1774 				     struct tls_record_info *record,
1775 				     u32 tcp_seq, int mss, bool tcp_push_no_fin,
1776 				     u32 data_len, u32 skb_offset,
1777 				     struct sge_eth_txq *q, u32 tls_end_offset)
1778 {
1779 	u32 tls_rec_offset = tcp_seq - tls_record_start_seq(record);
1780 	u8 prior_data[16] = {0};
1781 	u32 prior_data_len = 0;
1782 
1783 	/* check if the skb is ending in middle of tag/HASH, its a big
1784 	 * trouble, send the packet before the HASH.
1785 	 */
1786 	int remaining_record = tls_end_offset - data_len;
1787 
1788 	if (remaining_record > 0 &&
1789 	    remaining_record < TLS_CIPHER_AES_GCM_128_TAG_SIZE) {
1790 		int trimmed_len = 0;
1791 
1792 		if (tls_end_offset > TLS_CIPHER_AES_GCM_128_TAG_SIZE)
1793 			trimmed_len = data_len -
1794 				      (TLS_CIPHER_AES_GCM_128_TAG_SIZE -
1795 				       remaining_record);
1796 		if (!trimmed_len)
1797 			return FALLBACK;
1798 
1799 		WARN_ON(trimmed_len > data_len);
1800 
1801 		data_len = trimmed_len;
1802 		atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_trimmed_pkts);
1803 	}
1804 
1805 	/* check if it is only the header part. */
1806 	if (tls_rec_offset + data_len <= (TLS_HEADER_SIZE + tx_info->iv_size)) {
1807 		if (chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss,
1808 					  tcp_push_no_fin, q,
1809 					  tx_info->port_id, prior_data,
1810 					  data_len, skb_offset, prior_data_len))
1811 			goto out;
1812 
1813 		tx_info->prev_seq = tcp_seq + data_len;
1814 		return 0;
1815 	}
1816 
1817 	/* check if the middle record's start point is 16 byte aligned. CTR
1818 	 * needs 16 byte aligned start point to start encryption.
1819 	 */
1820 	if (tls_rec_offset) {
1821 		/* there is an offset from start, means its a middle record */
1822 		int remaining = 0;
1823 
1824 		if (tls_rec_offset < (TLS_HEADER_SIZE + tx_info->iv_size)) {
1825 			prior_data_len = tls_rec_offset;
1826 			tls_rec_offset = 0;
1827 			remaining = 0;
1828 		} else {
1829 			prior_data_len =
1830 				(tls_rec_offset -
1831 				(TLS_HEADER_SIZE + tx_info->iv_size))
1832 				% AES_BLOCK_LEN;
1833 			remaining = tls_rec_offset - prior_data_len;
1834 		}
1835 
1836 		/* if prior_data_len is not zero, means we need to fetch prior
1837 		 * data to make this record 16 byte aligned, or we need to reach
1838 		 * to start offset.
1839 		 */
1840 		if (prior_data_len) {
1841 			int i = 0;
1842 			skb_frag_t *f;
1843 			int frag_size = 0, frag_delta = 0;
1844 
1845 			while (remaining > 0) {
1846 				frag_size = skb_frag_size(&record->frags[i]);
1847 				if (remaining < frag_size)
1848 					break;
1849 
1850 				remaining -= frag_size;
1851 				i++;
1852 			}
1853 			f = &record->frags[i];
1854 			frag_delta = skb_frag_size(f) - remaining;
1855 
1856 			if (frag_delta >= prior_data_len) {
1857 				memcpy_from_page(prior_data, skb_frag_page(f),
1858 						 skb_frag_off(f) + remaining,
1859 						 prior_data_len);
1860 			} else {
1861 				memcpy_from_page(prior_data, skb_frag_page(f),
1862 						 skb_frag_off(f) + remaining,
1863 						 frag_delta);
1864 
1865 				/* get the next page */
1866 				f = &record->frags[i + 1];
1867 
1868 				memcpy_from_page(prior_data + frag_delta,
1869 						 skb_frag_page(f),
1870 						 skb_frag_off(f),
1871 						 prior_data_len - frag_delta);
1872 			}
1873 			/* reset tcp_seq as per the prior_data_required len */
1874 			tcp_seq -= prior_data_len;
1875 		}
1876 		atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_middle_pkts);
1877 	} else {
1878 		atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_start_pkts);
1879 	}
1880 
1881 	if (chcr_ktls_xmit_wr_short(skb, tx_info, q, tcp_seq, tcp_push_no_fin,
1882 				    mss, tls_rec_offset, prior_data,
1883 				    prior_data_len, data_len, skb_offset)) {
1884 		goto out;
1885 	}
1886 
1887 	tx_info->prev_seq = tcp_seq + data_len + prior_data_len;
1888 	return 0;
1889 out:
1890 	dev_kfree_skb_any(skb);
1891 	return NETDEV_TX_BUSY;
1892 }
1893 
chcr_ktls_sw_fallback(struct sk_buff * skb,struct chcr_ktls_info * tx_info,struct sge_eth_txq * q)1894 static int chcr_ktls_sw_fallback(struct sk_buff *skb,
1895 				 struct chcr_ktls_info *tx_info,
1896 				 struct sge_eth_txq *q)
1897 {
1898 	u32 data_len, skb_offset;
1899 	struct sk_buff *nskb;
1900 	struct tcphdr *th;
1901 
1902 	nskb = tls_encrypt_skb(skb);
1903 
1904 	if (!nskb)
1905 		return 0;
1906 
1907 	th = tcp_hdr(nskb);
1908 	skb_offset = skb_tcp_all_headers(nskb);
1909 	data_len = nskb->len - skb_offset;
1910 	skb_tx_timestamp(nskb);
1911 
1912 	if (chcr_ktls_tunnel_pkt(tx_info, nskb, q))
1913 		goto out;
1914 
1915 	tx_info->prev_seq = ntohl(th->seq) + data_len;
1916 	atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_fallback);
1917 	return 0;
1918 out:
1919 	dev_kfree_skb_any(nskb);
1920 	return 0;
1921 }
1922 /* nic tls TX handler */
chcr_ktls_xmit(struct sk_buff * skb,struct net_device * dev)1923 static int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev)
1924 {
1925 	u32 tls_end_offset, tcp_seq, skb_data_len, skb_offset;
1926 	struct ch_ktls_port_stats_debug *port_stats;
1927 	struct chcr_ktls_ofld_ctx_tx *tx_ctx;
1928 	struct ch_ktls_stats_debug *stats;
1929 	struct tcphdr *th = tcp_hdr(skb);
1930 	int data_len, qidx, ret = 0, mss;
1931 	struct tls_record_info *record;
1932 	struct chcr_ktls_info *tx_info;
1933 	struct net_device *tls_netdev;
1934 	struct tls_context *tls_ctx;
1935 	struct sge_eth_txq *q;
1936 	struct adapter *adap;
1937 	unsigned long flags;
1938 
1939 	tcp_seq = ntohl(th->seq);
1940 	skb_offset = skb_tcp_all_headers(skb);
1941 	skb_data_len = skb->len - skb_offset;
1942 	data_len = skb_data_len;
1943 
1944 	mss = skb_is_gso(skb) ? skb_shinfo(skb)->gso_size : data_len;
1945 
1946 	tls_ctx = tls_get_ctx(skb->sk);
1947 	tls_netdev = rcu_dereference_bh(tls_ctx->netdev);
1948 	/* Don't quit on NULL: if tls_device_down is running in parallel,
1949 	 * netdev might become NULL, even if tls_is_skb_tx_device_offloaded was
1950 	 * true. Rather continue processing this packet.
1951 	 */
1952 	if (unlikely(tls_netdev && tls_netdev != dev))
1953 		goto out;
1954 
1955 	tx_ctx = chcr_get_ktls_tx_context(tls_ctx);
1956 	tx_info = tx_ctx->chcr_info;
1957 
1958 	if (unlikely(!tx_info))
1959 		goto out;
1960 
1961 	adap = tx_info->adap;
1962 	stats = &adap->ch_ktls_stats;
1963 	port_stats = &stats->ktls_port[tx_info->port_id];
1964 
1965 	qidx = skb->queue_mapping;
1966 	q = &adap->sge.ethtxq[qidx + tx_info->first_qset];
1967 	cxgb4_reclaim_completed_tx(adap, &q->q, true);
1968 	/* if tcp options are set but finish is not send the options first */
1969 	if (!th->fin && chcr_ktls_check_tcp_options(th)) {
1970 		ret = chcr_ktls_write_tcp_options(tx_info, skb, q,
1971 						  tx_info->tx_chan);
1972 		if (ret)
1973 			return NETDEV_TX_BUSY;
1974 	}
1975 
1976 	/* TCP segments can be in received either complete or partial.
1977 	 * chcr_end_part_handler will handle cases if complete record or end
1978 	 * part of the record is received. In case of partial end part of record,
1979 	 * we will send the complete record again.
1980 	 */
1981 
1982 	spin_lock_irqsave(&tx_ctx->base.lock, flags);
1983 
1984 	do {
1985 
1986 		cxgb4_reclaim_completed_tx(adap, &q->q, true);
1987 		/* fetch the tls record */
1988 		record = tls_get_record(&tx_ctx->base, tcp_seq,
1989 					&tx_info->record_no);
1990 		/* By the time packet reached to us, ACK is received, and record
1991 		 * won't be found in that case, handle it gracefully.
1992 		 */
1993 		if (unlikely(!record)) {
1994 			spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
1995 			atomic64_inc(&port_stats->ktls_tx_drop_no_sync_data);
1996 			goto out;
1997 		}
1998 
1999 		tls_end_offset = record->end_seq - tcp_seq;
2000 
2001 		pr_debug("seq 0x%x, end_seq 0x%x prev_seq 0x%x, datalen 0x%x\n",
2002 			 tcp_seq, record->end_seq, tx_info->prev_seq, data_len);
2003 		/* update tcb for the skb */
2004 		if (skb_data_len == data_len) {
2005 			u32 tx_max = tcp_seq;
2006 
2007 			if (!tls_record_is_start_marker(record) &&
2008 			    tls_end_offset < TLS_CIPHER_AES_GCM_128_TAG_SIZE)
2009 				tx_max = record->end_seq -
2010 					TLS_CIPHER_AES_GCM_128_TAG_SIZE;
2011 
2012 			ret = chcr_ktls_xmit_tcb_cpls(tx_info, q, tx_max,
2013 						      ntohl(th->ack_seq),
2014 						      ntohs(th->window),
2015 						      tls_end_offset !=
2016 						      record->len);
2017 			if (ret) {
2018 				spin_unlock_irqrestore(&tx_ctx->base.lock,
2019 						       flags);
2020 				goto out;
2021 			}
2022 
2023 			if (th->fin)
2024 				skb_get(skb);
2025 		}
2026 
2027 		if (unlikely(tls_record_is_start_marker(record))) {
2028 			atomic64_inc(&port_stats->ktls_tx_skip_no_sync_data);
2029 			/* If tls_end_offset < data_len, means there is some
2030 			 * data after start marker, which needs encryption, send
2031 			 * plaintext first and take skb refcount. else send out
2032 			 * complete pkt as plaintext.
2033 			 */
2034 			if (tls_end_offset < data_len)
2035 				skb_get(skb);
2036 			else
2037 				tls_end_offset = data_len;
2038 
2039 			ret = chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss,
2040 						    (!th->fin && th->psh), q,
2041 						    tx_info->port_id, NULL,
2042 						    tls_end_offset, skb_offset,
2043 						    0);
2044 
2045 			if (ret) {
2046 				/* free the refcount taken earlier */
2047 				if (tls_end_offset < data_len)
2048 					dev_kfree_skb_any(skb);
2049 				spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
2050 				goto out;
2051 			}
2052 
2053 			data_len -= tls_end_offset;
2054 			tcp_seq = record->end_seq;
2055 			skb_offset += tls_end_offset;
2056 			continue;
2057 		}
2058 
2059 		/* if a tls record is finishing in this SKB */
2060 		if (tls_end_offset <= data_len) {
2061 			ret = chcr_end_part_handler(tx_info, skb, record,
2062 						    tcp_seq, mss,
2063 						    (!th->fin && th->psh), q,
2064 						    skb_offset,
2065 						    tls_end_offset,
2066 						    skb_offset +
2067 						    tls_end_offset == skb->len);
2068 
2069 			data_len -= tls_end_offset;
2070 			/* tcp_seq increment is required to handle next record.
2071 			 */
2072 			tcp_seq += tls_end_offset;
2073 			skb_offset += tls_end_offset;
2074 		} else {
2075 			ret = chcr_short_record_handler(tx_info, skb,
2076 							record, tcp_seq, mss,
2077 							(!th->fin && th->psh),
2078 							data_len, skb_offset,
2079 							q, tls_end_offset);
2080 			data_len = 0;
2081 		}
2082 
2083 		/* if any failure, come out from the loop. */
2084 		if (ret) {
2085 			spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
2086 			if (th->fin)
2087 				dev_kfree_skb_any(skb);
2088 
2089 			if (ret == FALLBACK)
2090 				return chcr_ktls_sw_fallback(skb, tx_info, q);
2091 
2092 			return NETDEV_TX_OK;
2093 		}
2094 
2095 		/* length should never be less than 0 */
2096 		WARN_ON(data_len < 0);
2097 
2098 	} while (data_len > 0);
2099 
2100 	spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
2101 	atomic64_inc(&port_stats->ktls_tx_encrypted_packets);
2102 	atomic64_add(skb_data_len, &port_stats->ktls_tx_encrypted_bytes);
2103 
2104 	/* tcp finish is set, send a separate tcp msg including all the options
2105 	 * as well.
2106 	 */
2107 	if (th->fin) {
2108 		chcr_ktls_write_tcp_options(tx_info, skb, q, tx_info->tx_chan);
2109 		dev_kfree_skb_any(skb);
2110 	}
2111 
2112 	return NETDEV_TX_OK;
2113 out:
2114 	dev_kfree_skb_any(skb);
2115 	return NETDEV_TX_OK;
2116 }
2117 
chcr_ktls_uld_add(const struct cxgb4_lld_info * lldi)2118 static void *chcr_ktls_uld_add(const struct cxgb4_lld_info *lldi)
2119 {
2120 	struct chcr_ktls_uld_ctx *u_ctx;
2121 
2122 	pr_info_once("%s - version %s\n", CHCR_KTLS_DRV_DESC,
2123 		     CHCR_KTLS_DRV_VERSION);
2124 	u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL);
2125 	if (!u_ctx) {
2126 		u_ctx = ERR_PTR(-ENOMEM);
2127 		goto out;
2128 	}
2129 	u_ctx->lldi = *lldi;
2130 	u_ctx->detach = false;
2131 	xa_init_flags(&u_ctx->tid_list, XA_FLAGS_LOCK_BH);
2132 out:
2133 	return u_ctx;
2134 }
2135 
2136 static const struct tlsdev_ops chcr_ktls_ops = {
2137 	.tls_dev_add = chcr_ktls_dev_add,
2138 	.tls_dev_del = chcr_ktls_dev_del,
2139 };
2140 
2141 static chcr_handler_func work_handlers[NUM_CPL_CMDS] = {
2142 	[CPL_ACT_OPEN_RPL] = chcr_ktls_cpl_act_open_rpl,
2143 	[CPL_SET_TCB_RPL] = chcr_ktls_cpl_set_tcb_rpl,
2144 };
2145 
chcr_ktls_uld_rx_handler(void * handle,const __be64 * rsp,const struct pkt_gl * pgl)2146 static int chcr_ktls_uld_rx_handler(void *handle, const __be64 *rsp,
2147 				    const struct pkt_gl *pgl)
2148 {
2149 	const struct cpl_act_open_rpl *rpl = (struct cpl_act_open_rpl *)rsp;
2150 	struct chcr_ktls_uld_ctx *u_ctx = handle;
2151 	u8 opcode = rpl->ot.opcode;
2152 	struct adapter *adap;
2153 
2154 	adap = pci_get_drvdata(u_ctx->lldi.pdev);
2155 
2156 	if (!work_handlers[opcode]) {
2157 		pr_err("Unsupported opcode %d received\n", opcode);
2158 		return 0;
2159 	}
2160 
2161 	work_handlers[opcode](adap, (unsigned char *)&rsp[1]);
2162 	return 0;
2163 }
2164 
clear_conn_resources(struct chcr_ktls_info * tx_info)2165 static void clear_conn_resources(struct chcr_ktls_info *tx_info)
2166 {
2167 	/* clear l2t entry */
2168 	if (tx_info->l2te)
2169 		cxgb4_l2t_release(tx_info->l2te);
2170 
2171 #if IS_ENABLED(CONFIG_IPV6)
2172 	/* clear clip entry */
2173 	if (tx_info->ip_family == AF_INET6)
2174 		cxgb4_clip_release(tx_info->netdev, (const u32 *)
2175 				   &tx_info->sk->sk_v6_rcv_saddr,
2176 				   1);
2177 #endif
2178 
2179 	/* clear tid */
2180 	if (tx_info->tid != -1)
2181 		cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
2182 				 tx_info->tid, tx_info->ip_family);
2183 }
2184 
ch_ktls_reset_all_conn(struct chcr_ktls_uld_ctx * u_ctx)2185 static void ch_ktls_reset_all_conn(struct chcr_ktls_uld_ctx *u_ctx)
2186 {
2187 	struct ch_ktls_port_stats_debug *port_stats;
2188 	struct chcr_ktls_ofld_ctx_tx *tx_ctx;
2189 	struct chcr_ktls_info *tx_info;
2190 	unsigned long index;
2191 
2192 	xa_for_each(&u_ctx->tid_list, index, tx_ctx) {
2193 		tx_info = tx_ctx->chcr_info;
2194 		clear_conn_resources(tx_info);
2195 		port_stats = &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id];
2196 		atomic64_inc(&port_stats->ktls_tx_connection_close);
2197 		kvfree(tx_info);
2198 		tx_ctx->chcr_info = NULL;
2199 		/* release module refcount */
2200 		module_put(THIS_MODULE);
2201 	}
2202 }
2203 
chcr_ktls_uld_state_change(void * handle,enum cxgb4_state new_state)2204 static int chcr_ktls_uld_state_change(void *handle, enum cxgb4_state new_state)
2205 {
2206 	struct chcr_ktls_uld_ctx *u_ctx = handle;
2207 
2208 	switch (new_state) {
2209 	case CXGB4_STATE_UP:
2210 		pr_info("%s: Up\n", pci_name(u_ctx->lldi.pdev));
2211 		mutex_lock(&dev_mutex);
2212 		list_add_tail(&u_ctx->entry, &uld_ctx_list);
2213 		mutex_unlock(&dev_mutex);
2214 		break;
2215 	case CXGB4_STATE_START_RECOVERY:
2216 	case CXGB4_STATE_DOWN:
2217 	case CXGB4_STATE_DETACH:
2218 		pr_info("%s: Down\n", pci_name(u_ctx->lldi.pdev));
2219 		mutex_lock(&dev_mutex);
2220 		u_ctx->detach = true;
2221 		list_del(&u_ctx->entry);
2222 		ch_ktls_reset_all_conn(u_ctx);
2223 		xa_destroy(&u_ctx->tid_list);
2224 		mutex_unlock(&dev_mutex);
2225 		break;
2226 	default:
2227 		break;
2228 	}
2229 
2230 	return 0;
2231 }
2232 
2233 static struct cxgb4_uld_info chcr_ktls_uld_info = {
2234 	.name = CHCR_KTLS_DRV_MODULE_NAME,
2235 	.nrxq = 1,
2236 	.rxq_size = 1024,
2237 	.add = chcr_ktls_uld_add,
2238 	.tx_handler = chcr_ktls_xmit,
2239 	.rx_handler = chcr_ktls_uld_rx_handler,
2240 	.state_change = chcr_ktls_uld_state_change,
2241 	.tlsdev_ops = &chcr_ktls_ops,
2242 };
2243 
chcr_ktls_init(void)2244 static int __init chcr_ktls_init(void)
2245 {
2246 	cxgb4_register_uld(CXGB4_ULD_KTLS, &chcr_ktls_uld_info);
2247 	return 0;
2248 }
2249 
chcr_ktls_exit(void)2250 static void __exit chcr_ktls_exit(void)
2251 {
2252 	struct chcr_ktls_uld_ctx *u_ctx, *tmp;
2253 	struct adapter *adap;
2254 
2255 	pr_info("driver unloaded\n");
2256 
2257 	mutex_lock(&dev_mutex);
2258 	list_for_each_entry_safe(u_ctx, tmp, &uld_ctx_list, entry) {
2259 		adap = pci_get_drvdata(u_ctx->lldi.pdev);
2260 		memset(&adap->ch_ktls_stats, 0, sizeof(adap->ch_ktls_stats));
2261 		list_del(&u_ctx->entry);
2262 		xa_destroy(&u_ctx->tid_list);
2263 		kfree(u_ctx);
2264 	}
2265 	mutex_unlock(&dev_mutex);
2266 	cxgb4_unregister_uld(CXGB4_ULD_KTLS);
2267 }
2268 
2269 module_init(chcr_ktls_init);
2270 module_exit(chcr_ktls_exit);
2271 
2272 MODULE_DESCRIPTION("Chelsio NIC TLS ULD driver");
2273 MODULE_LICENSE("GPL");
2274 MODULE_AUTHOR("Chelsio Communications");
2275 MODULE_VERSION(CHCR_KTLS_DRV_VERSION);
2276