144fd1c1fSVinay Kumar Yadav // SPDX-License-Identifier: GPL-2.0-only 244fd1c1fSVinay Kumar Yadav /* 344fd1c1fSVinay Kumar Yadav * Copyright (c) 2018 Chelsio Communications, Inc. 444fd1c1fSVinay Kumar Yadav * 544fd1c1fSVinay Kumar Yadav * Written by: Atul Gupta (atul.gupta@chelsio.com) 644fd1c1fSVinay Kumar Yadav */ 744fd1c1fSVinay Kumar Yadav 844fd1c1fSVinay Kumar Yadav #include <linux/module.h> 944fd1c1fSVinay Kumar Yadav #include <linux/list.h> 1044fd1c1fSVinay Kumar Yadav #include <linux/workqueue.h> 1144fd1c1fSVinay Kumar Yadav #include <linux/skbuff.h> 1244fd1c1fSVinay Kumar Yadav #include <linux/timer.h> 1344fd1c1fSVinay Kumar Yadav #include <linux/notifier.h> 1444fd1c1fSVinay Kumar Yadav #include <linux/inetdevice.h> 1544fd1c1fSVinay Kumar Yadav #include <linux/ip.h> 1644fd1c1fSVinay Kumar Yadav #include <linux/tcp.h> 1744fd1c1fSVinay Kumar Yadav #include <linux/tls.h> 1844fd1c1fSVinay Kumar Yadav #include <net/tls.h> 1944fd1c1fSVinay Kumar Yadav 2044fd1c1fSVinay Kumar Yadav #include "chtls.h" 2144fd1c1fSVinay Kumar Yadav #include "chtls_cm.h" 2244fd1c1fSVinay Kumar Yadav 2344fd1c1fSVinay Kumar Yadav static void __set_tcb_field_direct(struct chtls_sock *csk, 2444fd1c1fSVinay Kumar Yadav struct cpl_set_tcb_field *req, u16 word, 2544fd1c1fSVinay Kumar Yadav u64 mask, u64 val, u8 cookie, int no_reply) 2644fd1c1fSVinay Kumar Yadav { 2744fd1c1fSVinay Kumar Yadav struct ulptx_idata *sc; 2844fd1c1fSVinay Kumar Yadav 2944fd1c1fSVinay Kumar Yadav INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, csk->tid); 3044fd1c1fSVinay Kumar Yadav req->wr.wr_mid |= htonl(FW_WR_FLOWID_V(csk->tid)); 3144fd1c1fSVinay Kumar Yadav req->reply_ctrl = htons(NO_REPLY_V(no_reply) | 3244fd1c1fSVinay Kumar Yadav QUEUENO_V(csk->rss_qid)); 3344fd1c1fSVinay Kumar Yadav req->word_cookie = htons(TCB_WORD_V(word) | TCB_COOKIE_V(cookie)); 3444fd1c1fSVinay Kumar Yadav req->mask = cpu_to_be64(mask); 3544fd1c1fSVinay Kumar Yadav req->val = cpu_to_be64(val); 3644fd1c1fSVinay Kumar Yadav sc = (struct ulptx_idata *)(req + 1); 3744fd1c1fSVinay Kumar Yadav sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP)); 3844fd1c1fSVinay Kumar Yadav sc->len = htonl(0); 3944fd1c1fSVinay Kumar Yadav } 4044fd1c1fSVinay Kumar Yadav 4144fd1c1fSVinay Kumar Yadav static void __set_tcb_field(struct sock *sk, struct sk_buff *skb, u16 word, 4244fd1c1fSVinay Kumar Yadav u64 mask, u64 val, u8 cookie, int no_reply) 4344fd1c1fSVinay Kumar Yadav { 4444fd1c1fSVinay Kumar Yadav struct cpl_set_tcb_field *req; 4544fd1c1fSVinay Kumar Yadav struct chtls_sock *csk; 4644fd1c1fSVinay Kumar Yadav struct ulptx_idata *sc; 4744fd1c1fSVinay Kumar Yadav unsigned int wrlen; 4844fd1c1fSVinay Kumar Yadav 4944fd1c1fSVinay Kumar Yadav wrlen = roundup(sizeof(*req) + sizeof(*sc), 16); 5044fd1c1fSVinay Kumar Yadav csk = rcu_dereference_sk_user_data(sk); 5144fd1c1fSVinay Kumar Yadav 5244fd1c1fSVinay Kumar Yadav req = (struct cpl_set_tcb_field *)__skb_put(skb, wrlen); 5344fd1c1fSVinay Kumar Yadav __set_tcb_field_direct(csk, req, word, mask, val, cookie, no_reply); 5444fd1c1fSVinay Kumar Yadav set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id); 5544fd1c1fSVinay Kumar Yadav } 5644fd1c1fSVinay Kumar Yadav 5744fd1c1fSVinay Kumar Yadav /* 5844fd1c1fSVinay Kumar Yadav * Send control message to HW, message go as immediate data and packet 5944fd1c1fSVinay Kumar Yadav * is freed immediately. 6044fd1c1fSVinay Kumar Yadav */ 6144fd1c1fSVinay Kumar Yadav static int chtls_set_tcb_field(struct sock *sk, u16 word, u64 mask, u64 val) 6244fd1c1fSVinay Kumar Yadav { 6344fd1c1fSVinay Kumar Yadav struct cpl_set_tcb_field *req; 6444fd1c1fSVinay Kumar Yadav unsigned int credits_needed; 6544fd1c1fSVinay Kumar Yadav struct chtls_sock *csk; 6644fd1c1fSVinay Kumar Yadav struct ulptx_idata *sc; 6744fd1c1fSVinay Kumar Yadav struct sk_buff *skb; 6844fd1c1fSVinay Kumar Yadav unsigned int wrlen; 6944fd1c1fSVinay Kumar Yadav int ret; 7044fd1c1fSVinay Kumar Yadav 7144fd1c1fSVinay Kumar Yadav wrlen = roundup(sizeof(*req) + sizeof(*sc), 16); 7244fd1c1fSVinay Kumar Yadav 7344fd1c1fSVinay Kumar Yadav skb = alloc_skb(wrlen, GFP_ATOMIC); 7444fd1c1fSVinay Kumar Yadav if (!skb) 7544fd1c1fSVinay Kumar Yadav return -ENOMEM; 7644fd1c1fSVinay Kumar Yadav 7744fd1c1fSVinay Kumar Yadav credits_needed = DIV_ROUND_UP(wrlen, 16); 7844fd1c1fSVinay Kumar Yadav csk = rcu_dereference_sk_user_data(sk); 7944fd1c1fSVinay Kumar Yadav 8044fd1c1fSVinay Kumar Yadav __set_tcb_field(sk, skb, word, mask, val, 0, 1); 8144fd1c1fSVinay Kumar Yadav skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA); 8244fd1c1fSVinay Kumar Yadav csk->wr_credits -= credits_needed; 8344fd1c1fSVinay Kumar Yadav csk->wr_unacked += credits_needed; 8444fd1c1fSVinay Kumar Yadav enqueue_wr(csk, skb); 8544fd1c1fSVinay Kumar Yadav ret = cxgb4_ofld_send(csk->egress_dev, skb); 8644fd1c1fSVinay Kumar Yadav if (ret < 0) 8744fd1c1fSVinay Kumar Yadav kfree_skb(skb); 8844fd1c1fSVinay Kumar Yadav return ret < 0 ? ret : 0; 8944fd1c1fSVinay Kumar Yadav } 9044fd1c1fSVinay Kumar Yadav 9144fd1c1fSVinay Kumar Yadav /* 9244fd1c1fSVinay Kumar Yadav * Set one of the t_flags bits in the TCB. 9344fd1c1fSVinay Kumar Yadav */ 9444fd1c1fSVinay Kumar Yadav int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val) 9544fd1c1fSVinay Kumar Yadav { 9644fd1c1fSVinay Kumar Yadav return chtls_set_tcb_field(sk, 1, 1ULL << bit_pos, 9744fd1c1fSVinay Kumar Yadav (u64)val << bit_pos); 9844fd1c1fSVinay Kumar Yadav } 9944fd1c1fSVinay Kumar Yadav 10044fd1c1fSVinay Kumar Yadav static int chtls_set_tcb_keyid(struct sock *sk, int keyid) 10144fd1c1fSVinay Kumar Yadav { 10244fd1c1fSVinay Kumar Yadav return chtls_set_tcb_field(sk, 31, 0xFFFFFFFFULL, keyid); 10344fd1c1fSVinay Kumar Yadav } 10444fd1c1fSVinay Kumar Yadav 10544fd1c1fSVinay Kumar Yadav static int chtls_set_tcb_seqno(struct sock *sk) 10644fd1c1fSVinay Kumar Yadav { 10744fd1c1fSVinay Kumar Yadav return chtls_set_tcb_field(sk, 28, ~0ULL, 0); 10844fd1c1fSVinay Kumar Yadav } 10944fd1c1fSVinay Kumar Yadav 11044fd1c1fSVinay Kumar Yadav static int chtls_set_tcb_quiesce(struct sock *sk, int val) 11144fd1c1fSVinay Kumar Yadav { 11244fd1c1fSVinay Kumar Yadav return chtls_set_tcb_field(sk, 1, (1ULL << TF_RX_QUIESCE_S), 11344fd1c1fSVinay Kumar Yadav TF_RX_QUIESCE_V(val)); 11444fd1c1fSVinay Kumar Yadav } 11544fd1c1fSVinay Kumar Yadav 11644fd1c1fSVinay Kumar Yadav /* TLS Key bitmap processing */ 11744fd1c1fSVinay Kumar Yadav int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi) 11844fd1c1fSVinay Kumar Yadav { 11944fd1c1fSVinay Kumar Yadav unsigned int num_key_ctx, bsize; 12044fd1c1fSVinay Kumar Yadav int ksize; 12144fd1c1fSVinay Kumar Yadav 12244fd1c1fSVinay Kumar Yadav num_key_ctx = (lldi->vr->key.size / TLS_KEY_CONTEXT_SZ); 12344fd1c1fSVinay Kumar Yadav bsize = BITS_TO_LONGS(num_key_ctx); 12444fd1c1fSVinay Kumar Yadav 12544fd1c1fSVinay Kumar Yadav cdev->kmap.size = num_key_ctx; 12644fd1c1fSVinay Kumar Yadav cdev->kmap.available = bsize; 12744fd1c1fSVinay Kumar Yadav ksize = sizeof(*cdev->kmap.addr) * bsize; 12844fd1c1fSVinay Kumar Yadav cdev->kmap.addr = kvzalloc(ksize, GFP_KERNEL); 12944fd1c1fSVinay Kumar Yadav if (!cdev->kmap.addr) 13044fd1c1fSVinay Kumar Yadav return -ENOMEM; 13144fd1c1fSVinay Kumar Yadav 13244fd1c1fSVinay Kumar Yadav cdev->kmap.start = lldi->vr->key.start; 13344fd1c1fSVinay Kumar Yadav spin_lock_init(&cdev->kmap.lock); 13444fd1c1fSVinay Kumar Yadav return 0; 13544fd1c1fSVinay Kumar Yadav } 13644fd1c1fSVinay Kumar Yadav 13744fd1c1fSVinay Kumar Yadav static int get_new_keyid(struct chtls_sock *csk, u32 optname) 13844fd1c1fSVinay Kumar Yadav { 13944fd1c1fSVinay Kumar Yadav struct net_device *dev = csk->egress_dev; 14044fd1c1fSVinay Kumar Yadav struct chtls_dev *cdev = csk->cdev; 14144fd1c1fSVinay Kumar Yadav struct chtls_hws *hws; 14244fd1c1fSVinay Kumar Yadav struct adapter *adap; 14344fd1c1fSVinay Kumar Yadav int keyid; 14444fd1c1fSVinay Kumar Yadav 14544fd1c1fSVinay Kumar Yadav adap = netdev2adap(dev); 14644fd1c1fSVinay Kumar Yadav hws = &csk->tlshws; 14744fd1c1fSVinay Kumar Yadav 14844fd1c1fSVinay Kumar Yadav spin_lock_bh(&cdev->kmap.lock); 14944fd1c1fSVinay Kumar Yadav keyid = find_first_zero_bit(cdev->kmap.addr, cdev->kmap.size); 15044fd1c1fSVinay Kumar Yadav if (keyid < cdev->kmap.size) { 15144fd1c1fSVinay Kumar Yadav __set_bit(keyid, cdev->kmap.addr); 15244fd1c1fSVinay Kumar Yadav if (optname == TLS_RX) 15344fd1c1fSVinay Kumar Yadav hws->rxkey = keyid; 15444fd1c1fSVinay Kumar Yadav else 15544fd1c1fSVinay Kumar Yadav hws->txkey = keyid; 15644fd1c1fSVinay Kumar Yadav atomic_inc(&adap->chcr_stats.tls_key); 15744fd1c1fSVinay Kumar Yadav } else { 15844fd1c1fSVinay Kumar Yadav keyid = -1; 15944fd1c1fSVinay Kumar Yadav } 16044fd1c1fSVinay Kumar Yadav spin_unlock_bh(&cdev->kmap.lock); 16144fd1c1fSVinay Kumar Yadav return keyid; 16244fd1c1fSVinay Kumar Yadav } 16344fd1c1fSVinay Kumar Yadav 16444fd1c1fSVinay Kumar Yadav void free_tls_keyid(struct sock *sk) 16544fd1c1fSVinay Kumar Yadav { 16644fd1c1fSVinay Kumar Yadav struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); 16744fd1c1fSVinay Kumar Yadav struct net_device *dev = csk->egress_dev; 16844fd1c1fSVinay Kumar Yadav struct chtls_dev *cdev = csk->cdev; 16944fd1c1fSVinay Kumar Yadav struct chtls_hws *hws; 17044fd1c1fSVinay Kumar Yadav struct adapter *adap; 17144fd1c1fSVinay Kumar Yadav 17244fd1c1fSVinay Kumar Yadav if (!cdev->kmap.addr) 17344fd1c1fSVinay Kumar Yadav return; 17444fd1c1fSVinay Kumar Yadav 17544fd1c1fSVinay Kumar Yadav adap = netdev2adap(dev); 17644fd1c1fSVinay Kumar Yadav hws = &csk->tlshws; 17744fd1c1fSVinay Kumar Yadav 17844fd1c1fSVinay Kumar Yadav spin_lock_bh(&cdev->kmap.lock); 17944fd1c1fSVinay Kumar Yadav if (hws->rxkey >= 0) { 18044fd1c1fSVinay Kumar Yadav __clear_bit(hws->rxkey, cdev->kmap.addr); 18144fd1c1fSVinay Kumar Yadav atomic_dec(&adap->chcr_stats.tls_key); 18244fd1c1fSVinay Kumar Yadav hws->rxkey = -1; 18344fd1c1fSVinay Kumar Yadav } 18444fd1c1fSVinay Kumar Yadav if (hws->txkey >= 0) { 18544fd1c1fSVinay Kumar Yadav __clear_bit(hws->txkey, cdev->kmap.addr); 18644fd1c1fSVinay Kumar Yadav atomic_dec(&adap->chcr_stats.tls_key); 18744fd1c1fSVinay Kumar Yadav hws->txkey = -1; 18844fd1c1fSVinay Kumar Yadav } 18944fd1c1fSVinay Kumar Yadav spin_unlock_bh(&cdev->kmap.lock); 19044fd1c1fSVinay Kumar Yadav } 19144fd1c1fSVinay Kumar Yadav 19244fd1c1fSVinay Kumar Yadav unsigned int keyid_to_addr(int start_addr, int keyid) 19344fd1c1fSVinay Kumar Yadav { 19444fd1c1fSVinay Kumar Yadav return (start_addr + (keyid * TLS_KEY_CONTEXT_SZ)) >> 5; 19544fd1c1fSVinay Kumar Yadav } 19644fd1c1fSVinay Kumar Yadav 19744fd1c1fSVinay Kumar Yadav static void chtls_rxkey_ivauth(struct _key_ctx *kctx) 19844fd1c1fSVinay Kumar Yadav { 19944fd1c1fSVinay Kumar Yadav kctx->iv_to_auth = cpu_to_be64(KEYCTX_TX_WR_IV_V(6ULL) | 20044fd1c1fSVinay Kumar Yadav KEYCTX_TX_WR_AAD_V(1ULL) | 20144fd1c1fSVinay Kumar Yadav KEYCTX_TX_WR_AADST_V(5ULL) | 20244fd1c1fSVinay Kumar Yadav KEYCTX_TX_WR_CIPHER_V(14ULL) | 20344fd1c1fSVinay Kumar Yadav KEYCTX_TX_WR_CIPHERST_V(0ULL) | 20444fd1c1fSVinay Kumar Yadav KEYCTX_TX_WR_AUTH_V(14ULL) | 20544fd1c1fSVinay Kumar Yadav KEYCTX_TX_WR_AUTHST_V(16ULL) | 20644fd1c1fSVinay Kumar Yadav KEYCTX_TX_WR_AUTHIN_V(16ULL)); 20744fd1c1fSVinay Kumar Yadav } 20844fd1c1fSVinay Kumar Yadav 20944fd1c1fSVinay Kumar Yadav static int chtls_key_info(struct chtls_sock *csk, 21044fd1c1fSVinay Kumar Yadav struct _key_ctx *kctx, 21144fd1c1fSVinay Kumar Yadav u32 keylen, u32 optname, 21244fd1c1fSVinay Kumar Yadav int cipher_type) 21344fd1c1fSVinay Kumar Yadav { 21444fd1c1fSVinay Kumar Yadav unsigned char key[AES_MAX_KEY_SIZE]; 21544fd1c1fSVinay Kumar Yadav unsigned char *key_p, *salt; 21644fd1c1fSVinay Kumar Yadav unsigned char ghash_h[AEAD_H_SIZE]; 21744fd1c1fSVinay Kumar Yadav int ck_size, key_ctx_size, kctx_mackey_size, salt_size; 21844fd1c1fSVinay Kumar Yadav struct crypto_aes_ctx aes; 21944fd1c1fSVinay Kumar Yadav int ret; 22044fd1c1fSVinay Kumar Yadav 22144fd1c1fSVinay Kumar Yadav key_ctx_size = sizeof(struct _key_ctx) + 22244fd1c1fSVinay Kumar Yadav roundup(keylen, 16) + AEAD_H_SIZE; 22344fd1c1fSVinay Kumar Yadav 22444fd1c1fSVinay Kumar Yadav /* GCM mode of AES supports 128 and 256 bit encryption, so 22544fd1c1fSVinay Kumar Yadav * prepare key context base on GCM cipher type 22644fd1c1fSVinay Kumar Yadav */ 22744fd1c1fSVinay Kumar Yadav switch (cipher_type) { 22844fd1c1fSVinay Kumar Yadav case TLS_CIPHER_AES_GCM_128: { 22944fd1c1fSVinay Kumar Yadav struct tls12_crypto_info_aes_gcm_128 *gcm_ctx_128 = 23044fd1c1fSVinay Kumar Yadav (struct tls12_crypto_info_aes_gcm_128 *) 23144fd1c1fSVinay Kumar Yadav &csk->tlshws.crypto_info; 23244fd1c1fSVinay Kumar Yadav memcpy(key, gcm_ctx_128->key, keylen); 23344fd1c1fSVinay Kumar Yadav 23444fd1c1fSVinay Kumar Yadav key_p = gcm_ctx_128->key; 23544fd1c1fSVinay Kumar Yadav salt = gcm_ctx_128->salt; 23644fd1c1fSVinay Kumar Yadav ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 23744fd1c1fSVinay Kumar Yadav salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE; 23844fd1c1fSVinay Kumar Yadav kctx_mackey_size = CHCR_KEYCTX_MAC_KEY_SIZE_128; 23944fd1c1fSVinay Kumar Yadav break; 24044fd1c1fSVinay Kumar Yadav } 24144fd1c1fSVinay Kumar Yadav case TLS_CIPHER_AES_GCM_256: { 24244fd1c1fSVinay Kumar Yadav struct tls12_crypto_info_aes_gcm_256 *gcm_ctx_256 = 24344fd1c1fSVinay Kumar Yadav (struct tls12_crypto_info_aes_gcm_256 *) 24444fd1c1fSVinay Kumar Yadav &csk->tlshws.crypto_info; 24544fd1c1fSVinay Kumar Yadav memcpy(key, gcm_ctx_256->key, keylen); 24644fd1c1fSVinay Kumar Yadav 24744fd1c1fSVinay Kumar Yadav key_p = gcm_ctx_256->key; 24844fd1c1fSVinay Kumar Yadav salt = gcm_ctx_256->salt; 24944fd1c1fSVinay Kumar Yadav ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 25044fd1c1fSVinay Kumar Yadav salt_size = TLS_CIPHER_AES_GCM_256_SALT_SIZE; 25144fd1c1fSVinay Kumar Yadav kctx_mackey_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 25244fd1c1fSVinay Kumar Yadav break; 25344fd1c1fSVinay Kumar Yadav } 25444fd1c1fSVinay Kumar Yadav default: 25544fd1c1fSVinay Kumar Yadav pr_err("GCM: Invalid key length %d\n", keylen); 25644fd1c1fSVinay Kumar Yadav return -EINVAL; 25744fd1c1fSVinay Kumar Yadav } 25844fd1c1fSVinay Kumar Yadav 25944fd1c1fSVinay Kumar Yadav /* Calculate the H = CIPH(K, 0 repeated 16 times). 26044fd1c1fSVinay Kumar Yadav * It will go in key context 26144fd1c1fSVinay Kumar Yadav */ 26244fd1c1fSVinay Kumar Yadav ret = aes_expandkey(&aes, key, keylen); 26344fd1c1fSVinay Kumar Yadav if (ret) 26444fd1c1fSVinay Kumar Yadav return ret; 26544fd1c1fSVinay Kumar Yadav 26644fd1c1fSVinay Kumar Yadav memset(ghash_h, 0, AEAD_H_SIZE); 26744fd1c1fSVinay Kumar Yadav aes_encrypt(&aes, ghash_h, ghash_h); 26844fd1c1fSVinay Kumar Yadav memzero_explicit(&aes, sizeof(aes)); 26944fd1c1fSVinay Kumar Yadav csk->tlshws.keylen = key_ctx_size; 27044fd1c1fSVinay Kumar Yadav 27144fd1c1fSVinay Kumar Yadav /* Copy the Key context */ 27244fd1c1fSVinay Kumar Yadav if (optname == TLS_RX) { 27344fd1c1fSVinay Kumar Yadav int key_ctx; 27444fd1c1fSVinay Kumar Yadav 27544fd1c1fSVinay Kumar Yadav key_ctx = ((key_ctx_size >> 4) << 3); 27644fd1c1fSVinay Kumar Yadav kctx->ctx_hdr = FILL_KEY_CRX_HDR(ck_size, 27744fd1c1fSVinay Kumar Yadav kctx_mackey_size, 27844fd1c1fSVinay Kumar Yadav 0, 0, key_ctx); 27944fd1c1fSVinay Kumar Yadav chtls_rxkey_ivauth(kctx); 28044fd1c1fSVinay Kumar Yadav } else { 28144fd1c1fSVinay Kumar Yadav kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size, 28244fd1c1fSVinay Kumar Yadav kctx_mackey_size, 28344fd1c1fSVinay Kumar Yadav 0, 0, key_ctx_size >> 4); 28444fd1c1fSVinay Kumar Yadav } 28544fd1c1fSVinay Kumar Yadav 28644fd1c1fSVinay Kumar Yadav memcpy(kctx->salt, salt, salt_size); 28744fd1c1fSVinay Kumar Yadav memcpy(kctx->key, key_p, keylen); 28844fd1c1fSVinay Kumar Yadav memcpy(kctx->key + keylen, ghash_h, AEAD_H_SIZE); 28944fd1c1fSVinay Kumar Yadav /* erase key info from driver */ 29044fd1c1fSVinay Kumar Yadav memset(key_p, 0, keylen); 29144fd1c1fSVinay Kumar Yadav 29244fd1c1fSVinay Kumar Yadav return 0; 29344fd1c1fSVinay Kumar Yadav } 29444fd1c1fSVinay Kumar Yadav 29544fd1c1fSVinay Kumar Yadav static void chtls_set_scmd(struct chtls_sock *csk) 29644fd1c1fSVinay Kumar Yadav { 29744fd1c1fSVinay Kumar Yadav struct chtls_hws *hws = &csk->tlshws; 29844fd1c1fSVinay Kumar Yadav 29944fd1c1fSVinay Kumar Yadav hws->scmd.seqno_numivs = 30044fd1c1fSVinay Kumar Yadav SCMD_SEQ_NO_CTRL_V(3) | 30144fd1c1fSVinay Kumar Yadav SCMD_PROTO_VERSION_V(0) | 30244fd1c1fSVinay Kumar Yadav SCMD_ENC_DEC_CTRL_V(0) | 30344fd1c1fSVinay Kumar Yadav SCMD_CIPH_AUTH_SEQ_CTRL_V(1) | 30444fd1c1fSVinay Kumar Yadav SCMD_CIPH_MODE_V(2) | 30544fd1c1fSVinay Kumar Yadav SCMD_AUTH_MODE_V(4) | 30644fd1c1fSVinay Kumar Yadav SCMD_HMAC_CTRL_V(0) | 30744fd1c1fSVinay Kumar Yadav SCMD_IV_SIZE_V(4) | 30844fd1c1fSVinay Kumar Yadav SCMD_NUM_IVS_V(1); 30944fd1c1fSVinay Kumar Yadav 31044fd1c1fSVinay Kumar Yadav hws->scmd.ivgen_hdrlen = 31144fd1c1fSVinay Kumar Yadav SCMD_IV_GEN_CTRL_V(1) | 31244fd1c1fSVinay Kumar Yadav SCMD_KEY_CTX_INLINE_V(0) | 31344fd1c1fSVinay Kumar Yadav SCMD_TLS_FRAG_ENABLE_V(1); 31444fd1c1fSVinay Kumar Yadav } 31544fd1c1fSVinay Kumar Yadav 31644fd1c1fSVinay Kumar Yadav int chtls_setkey(struct chtls_sock *csk, u32 keylen, 31744fd1c1fSVinay Kumar Yadav u32 optname, int cipher_type) 31844fd1c1fSVinay Kumar Yadav { 31944fd1c1fSVinay Kumar Yadav struct tls_key_req *kwr; 32044fd1c1fSVinay Kumar Yadav struct chtls_dev *cdev; 32144fd1c1fSVinay Kumar Yadav struct _key_ctx *kctx; 32244fd1c1fSVinay Kumar Yadav int wrlen, klen, len; 32344fd1c1fSVinay Kumar Yadav struct sk_buff *skb; 32444fd1c1fSVinay Kumar Yadav struct sock *sk; 32544fd1c1fSVinay Kumar Yadav int keyid; 32644fd1c1fSVinay Kumar Yadav int kaddr; 32744fd1c1fSVinay Kumar Yadav int ret; 32844fd1c1fSVinay Kumar Yadav 32944fd1c1fSVinay Kumar Yadav cdev = csk->cdev; 33044fd1c1fSVinay Kumar Yadav sk = csk->sk; 33144fd1c1fSVinay Kumar Yadav 33244fd1c1fSVinay Kumar Yadav klen = roundup((keylen + AEAD_H_SIZE) + sizeof(*kctx), 32); 33344fd1c1fSVinay Kumar Yadav wrlen = roundup(sizeof(*kwr), 16); 33444fd1c1fSVinay Kumar Yadav len = klen + wrlen; 33544fd1c1fSVinay Kumar Yadav 33644fd1c1fSVinay Kumar Yadav /* Flush out-standing data before new key takes effect */ 33744fd1c1fSVinay Kumar Yadav if (optname == TLS_TX) { 33844fd1c1fSVinay Kumar Yadav lock_sock(sk); 33944fd1c1fSVinay Kumar Yadav if (skb_queue_len(&csk->txq)) 34044fd1c1fSVinay Kumar Yadav chtls_push_frames(csk, 0); 34144fd1c1fSVinay Kumar Yadav release_sock(sk); 34244fd1c1fSVinay Kumar Yadav } 34344fd1c1fSVinay Kumar Yadav 34444fd1c1fSVinay Kumar Yadav skb = alloc_skb(len, GFP_KERNEL); 34544fd1c1fSVinay Kumar Yadav if (!skb) 34644fd1c1fSVinay Kumar Yadav return -ENOMEM; 34744fd1c1fSVinay Kumar Yadav 34844fd1c1fSVinay Kumar Yadav keyid = get_new_keyid(csk, optname); 34944fd1c1fSVinay Kumar Yadav if (keyid < 0) { 35044fd1c1fSVinay Kumar Yadav ret = -ENOSPC; 35144fd1c1fSVinay Kumar Yadav goto out_nokey; 35244fd1c1fSVinay Kumar Yadav } 35344fd1c1fSVinay Kumar Yadav 35444fd1c1fSVinay Kumar Yadav kaddr = keyid_to_addr(cdev->kmap.start, keyid); 35544fd1c1fSVinay Kumar Yadav kwr = (struct tls_key_req *)__skb_put_zero(skb, len); 35644fd1c1fSVinay Kumar Yadav kwr->wr.op_to_compl = 35744fd1c1fSVinay Kumar Yadav cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR) | FW_WR_COMPL_F | 35844fd1c1fSVinay Kumar Yadav FW_WR_ATOMIC_V(1U)); 35944fd1c1fSVinay Kumar Yadav kwr->wr.flowid_len16 = 36044fd1c1fSVinay Kumar Yadav cpu_to_be32(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16) | 36144fd1c1fSVinay Kumar Yadav FW_WR_FLOWID_V(csk->tid))); 36244fd1c1fSVinay Kumar Yadav kwr->wr.protocol = 0; 36344fd1c1fSVinay Kumar Yadav kwr->wr.mfs = htons(TLS_MFS); 36444fd1c1fSVinay Kumar Yadav kwr->wr.reneg_to_write_rx = optname; 36544fd1c1fSVinay Kumar Yadav 36644fd1c1fSVinay Kumar Yadav /* ulptx command */ 36744fd1c1fSVinay Kumar Yadav kwr->req.cmd = cpu_to_be32(ULPTX_CMD_V(ULP_TX_MEM_WRITE) | 36844fd1c1fSVinay Kumar Yadav T5_ULP_MEMIO_ORDER_V(1) | 36944fd1c1fSVinay Kumar Yadav T5_ULP_MEMIO_IMM_V(1)); 37044fd1c1fSVinay Kumar Yadav kwr->req.len16 = cpu_to_be32((csk->tid << 8) | 37144fd1c1fSVinay Kumar Yadav DIV_ROUND_UP(len - sizeof(kwr->wr), 16)); 37244fd1c1fSVinay Kumar Yadav kwr->req.dlen = cpu_to_be32(ULP_MEMIO_DATA_LEN_V(klen >> 5)); 37344fd1c1fSVinay Kumar Yadav kwr->req.lock_addr = cpu_to_be32(ULP_MEMIO_ADDR_V(kaddr)); 37444fd1c1fSVinay Kumar Yadav 37544fd1c1fSVinay Kumar Yadav /* sub command */ 37644fd1c1fSVinay Kumar Yadav kwr->sc_imm.cmd_more = cpu_to_be32(ULPTX_CMD_V(ULP_TX_SC_IMM)); 37744fd1c1fSVinay Kumar Yadav kwr->sc_imm.len = cpu_to_be32(klen); 37844fd1c1fSVinay Kumar Yadav 37944fd1c1fSVinay Kumar Yadav lock_sock(sk); 38044fd1c1fSVinay Kumar Yadav /* key info */ 38144fd1c1fSVinay Kumar Yadav kctx = (struct _key_ctx *)(kwr + 1); 38244fd1c1fSVinay Kumar Yadav ret = chtls_key_info(csk, kctx, keylen, optname, cipher_type); 38344fd1c1fSVinay Kumar Yadav if (ret) 38444fd1c1fSVinay Kumar Yadav goto out_notcb; 38544fd1c1fSVinay Kumar Yadav 3868080b462SVinay Kumar Yadav if (unlikely(csk_flag(sk, CSK_ABORT_SHUTDOWN))) 3878080b462SVinay Kumar Yadav goto out_notcb; 3888080b462SVinay Kumar Yadav 38944fd1c1fSVinay Kumar Yadav set_wr_txq(skb, CPL_PRIORITY_DATA, csk->tlshws.txqid); 39044fd1c1fSVinay Kumar Yadav csk->wr_credits -= DIV_ROUND_UP(len, 16); 39144fd1c1fSVinay Kumar Yadav csk->wr_unacked += DIV_ROUND_UP(len, 16); 39244fd1c1fSVinay Kumar Yadav enqueue_wr(csk, skb); 39344fd1c1fSVinay Kumar Yadav cxgb4_ofld_send(csk->egress_dev, skb); 394391119fbSDan Carpenter skb = NULL; 39544fd1c1fSVinay Kumar Yadav 39644fd1c1fSVinay Kumar Yadav chtls_set_scmd(csk); 39744fd1c1fSVinay Kumar Yadav /* Clear quiesce for Rx key */ 39844fd1c1fSVinay Kumar Yadav if (optname == TLS_RX) { 39944fd1c1fSVinay Kumar Yadav ret = chtls_set_tcb_keyid(sk, keyid); 40044fd1c1fSVinay Kumar Yadav if (ret) 40144fd1c1fSVinay Kumar Yadav goto out_notcb; 40244fd1c1fSVinay Kumar Yadav ret = chtls_set_tcb_field(sk, 0, 40344fd1c1fSVinay Kumar Yadav TCB_ULP_RAW_V(TCB_ULP_RAW_M), 40444fd1c1fSVinay Kumar Yadav TCB_ULP_RAW_V((TF_TLS_KEY_SIZE_V(1) | 40544fd1c1fSVinay Kumar Yadav TF_TLS_CONTROL_V(1) | 40644fd1c1fSVinay Kumar Yadav TF_TLS_ACTIVE_V(1) | 40744fd1c1fSVinay Kumar Yadav TF_TLS_ENABLE_V(1)))); 40844fd1c1fSVinay Kumar Yadav if (ret) 40944fd1c1fSVinay Kumar Yadav goto out_notcb; 41044fd1c1fSVinay Kumar Yadav ret = chtls_set_tcb_seqno(sk); 41144fd1c1fSVinay Kumar Yadav if (ret) 41244fd1c1fSVinay Kumar Yadav goto out_notcb; 41344fd1c1fSVinay Kumar Yadav ret = chtls_set_tcb_quiesce(sk, 0); 41444fd1c1fSVinay Kumar Yadav if (ret) 41544fd1c1fSVinay Kumar Yadav goto out_notcb; 41644fd1c1fSVinay Kumar Yadav csk->tlshws.rxkey = keyid; 41744fd1c1fSVinay Kumar Yadav } else { 41844fd1c1fSVinay Kumar Yadav csk->tlshws.tx_seq_no = 0; 41944fd1c1fSVinay Kumar Yadav csk->tlshws.txkey = keyid; 42044fd1c1fSVinay Kumar Yadav } 42144fd1c1fSVinay Kumar Yadav 42244fd1c1fSVinay Kumar Yadav release_sock(sk); 42344fd1c1fSVinay Kumar Yadav return ret; 42444fd1c1fSVinay Kumar Yadav out_notcb: 42544fd1c1fSVinay Kumar Yadav release_sock(sk); 42644fd1c1fSVinay Kumar Yadav free_tls_keyid(sk); 42744fd1c1fSVinay Kumar Yadav out_nokey: 42844fd1c1fSVinay Kumar Yadav kfree_skb(skb); 42944fd1c1fSVinay Kumar Yadav return ret; 43044fd1c1fSVinay Kumar Yadav } 431