1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * CQHCI crypto engine (inline encryption) support 4 * 5 * Copyright 2020 Google LLC 6 */ 7 8 #include <linux/blk-crypto.h> 9 #include <linux/keyslot-manager.h> 10 #include <linux/mmc/host.h> 11 12 #include "cqhci-crypto.h" 13 14 /* Map from blk-crypto modes to CQHCI crypto algorithm IDs and key sizes */ 15 static const struct cqhci_crypto_alg_entry { 16 enum cqhci_crypto_alg alg; 17 enum cqhci_crypto_key_size key_size; 18 } cqhci_crypto_algs[BLK_ENCRYPTION_MODE_MAX] = { 19 [BLK_ENCRYPTION_MODE_AES_256_XTS] = { 20 .alg = CQHCI_CRYPTO_ALG_AES_XTS, 21 .key_size = CQHCI_CRYPTO_KEY_SIZE_256, 22 }, 23 }; 24 25 static inline struct cqhci_host * 26 cqhci_host_from_ksm(struct blk_keyslot_manager *ksm) 27 { 28 struct mmc_host *mmc = container_of(ksm, struct mmc_host, ksm); 29 30 return mmc->cqe_private; 31 } 32 33 static int cqhci_crypto_program_key(struct cqhci_host *cq_host, 34 const union cqhci_crypto_cfg_entry *cfg, 35 int slot) 36 { 37 u32 slot_offset = cq_host->crypto_cfg_register + slot * sizeof(*cfg); 38 int i; 39 40 if (cq_host->ops->program_key) 41 return cq_host->ops->program_key(cq_host, cfg, slot); 42 43 /* Clear CFGE */ 44 cqhci_writel(cq_host, 0, slot_offset + 16 * sizeof(cfg->reg_val[0])); 45 46 /* Write the key */ 47 for (i = 0; i < 16; i++) { 48 cqhci_writel(cq_host, le32_to_cpu(cfg->reg_val[i]), 49 slot_offset + i * sizeof(cfg->reg_val[0])); 50 } 51 /* Write dword 17 */ 52 cqhci_writel(cq_host, le32_to_cpu(cfg->reg_val[17]), 53 slot_offset + 17 * sizeof(cfg->reg_val[0])); 54 /* Write dword 16, which includes the new value of CFGE */ 55 cqhci_writel(cq_host, le32_to_cpu(cfg->reg_val[16]), 56 slot_offset + 16 * sizeof(cfg->reg_val[0])); 57 return 0; 58 } 59 60 static int cqhci_crypto_keyslot_program(struct blk_keyslot_manager *ksm, 61 const struct blk_crypto_key *key, 62 unsigned int slot) 63 64 { 65 struct cqhci_host *cq_host = cqhci_host_from_ksm(ksm); 66 const union cqhci_crypto_cap_entry *ccap_array = 67 cq_host->crypto_cap_array; 68 const struct cqhci_crypto_alg_entry *alg = 69 &cqhci_crypto_algs[key->crypto_cfg.crypto_mode]; 70 u8 data_unit_mask = key->crypto_cfg.data_unit_size / 512; 71 int i; 72 int cap_idx = -1; 73 union cqhci_crypto_cfg_entry cfg = {}; 74 int err; 75 76 BUILD_BUG_ON(CQHCI_CRYPTO_KEY_SIZE_INVALID != 0); 77 for (i = 0; i < cq_host->crypto_capabilities.num_crypto_cap; i++) { 78 if (ccap_array[i].algorithm_id == alg->alg && 79 ccap_array[i].key_size == alg->key_size && 80 (ccap_array[i].sdus_mask & data_unit_mask)) { 81 cap_idx = i; 82 break; 83 } 84 } 85 if (WARN_ON(cap_idx < 0)) 86 return -EOPNOTSUPP; 87 88 cfg.data_unit_size = data_unit_mask; 89 cfg.crypto_cap_idx = cap_idx; 90 cfg.config_enable = CQHCI_CRYPTO_CONFIGURATION_ENABLE; 91 92 if (ccap_array[cap_idx].algorithm_id == CQHCI_CRYPTO_ALG_AES_XTS) { 93 /* In XTS mode, the blk_crypto_key's size is already doubled */ 94 memcpy(cfg.crypto_key, key->raw, key->size/2); 95 memcpy(cfg.crypto_key + CQHCI_CRYPTO_KEY_MAX_SIZE/2, 96 key->raw + key->size/2, key->size/2); 97 } else { 98 memcpy(cfg.crypto_key, key->raw, key->size); 99 } 100 101 err = cqhci_crypto_program_key(cq_host, &cfg, slot); 102 103 memzero_explicit(&cfg, sizeof(cfg)); 104 return err; 105 } 106 107 static int cqhci_crypto_clear_keyslot(struct cqhci_host *cq_host, int slot) 108 { 109 /* 110 * Clear the crypto cfg on the device. Clearing CFGE 111 * might not be sufficient, so just clear the entire cfg. 112 */ 113 union cqhci_crypto_cfg_entry cfg = {}; 114 115 return cqhci_crypto_program_key(cq_host, &cfg, slot); 116 } 117 118 static int cqhci_crypto_keyslot_evict(struct blk_keyslot_manager *ksm, 119 const struct blk_crypto_key *key, 120 unsigned int slot) 121 { 122 struct cqhci_host *cq_host = cqhci_host_from_ksm(ksm); 123 124 return cqhci_crypto_clear_keyslot(cq_host, slot); 125 } 126 127 /* 128 * The keyslot management operations for CQHCI crypto. 129 * 130 * Note that the block layer ensures that these are never called while the host 131 * controller is runtime-suspended. However, the CQE won't necessarily be 132 * "enabled" when these are called, i.e. CQHCI_ENABLE might not be set in the 133 * CQHCI_CFG register. But the hardware allows that. 134 */ 135 static const struct blk_ksm_ll_ops cqhci_ksm_ops = { 136 .keyslot_program = cqhci_crypto_keyslot_program, 137 .keyslot_evict = cqhci_crypto_keyslot_evict, 138 }; 139 140 static enum blk_crypto_mode_num 141 cqhci_find_blk_crypto_mode(union cqhci_crypto_cap_entry cap) 142 { 143 int i; 144 145 for (i = 0; i < ARRAY_SIZE(cqhci_crypto_algs); i++) { 146 BUILD_BUG_ON(CQHCI_CRYPTO_KEY_SIZE_INVALID != 0); 147 if (cqhci_crypto_algs[i].alg == cap.algorithm_id && 148 cqhci_crypto_algs[i].key_size == cap.key_size) 149 return i; 150 } 151 return BLK_ENCRYPTION_MODE_INVALID; 152 } 153 154 /** 155 * cqhci_crypto_init - initialize CQHCI crypto support 156 * @cq_host: a cqhci host 157 * 158 * If the driver previously set MMC_CAP2_CRYPTO and the CQE declares 159 * CQHCI_CAP_CS, initialize the crypto support. This involves reading the 160 * crypto capability registers, initializing the keyslot manager, clearing all 161 * keyslots, and enabling 128-bit task descriptors. 162 * 163 * Return: 0 if crypto was initialized or isn't supported; whether 164 * MMC_CAP2_CRYPTO remains set indicates which one of those cases it is. 165 * Also can return a negative errno value on unexpected error. 166 */ 167 int cqhci_crypto_init(struct cqhci_host *cq_host) 168 { 169 struct mmc_host *mmc = cq_host->mmc; 170 struct device *dev = mmc_dev(mmc); 171 struct blk_keyslot_manager *ksm = &mmc->ksm; 172 unsigned int num_keyslots; 173 unsigned int cap_idx; 174 enum blk_crypto_mode_num blk_mode_num; 175 unsigned int slot; 176 int err = 0; 177 178 if (!(mmc->caps2 & MMC_CAP2_CRYPTO) || 179 !(cqhci_readl(cq_host, CQHCI_CAP) & CQHCI_CAP_CS)) 180 goto out; 181 182 cq_host->crypto_capabilities.reg_val = 183 cpu_to_le32(cqhci_readl(cq_host, CQHCI_CCAP)); 184 185 cq_host->crypto_cfg_register = 186 (u32)cq_host->crypto_capabilities.config_array_ptr * 0x100; 187 188 cq_host->crypto_cap_array = 189 devm_kcalloc(dev, cq_host->crypto_capabilities.num_crypto_cap, 190 sizeof(cq_host->crypto_cap_array[0]), GFP_KERNEL); 191 if (!cq_host->crypto_cap_array) { 192 err = -ENOMEM; 193 goto out; 194 } 195 196 /* 197 * CCAP.CFGC is off by one, so the actual number of crypto 198 * configurations (a.k.a. keyslots) is CCAP.CFGC + 1. 199 */ 200 num_keyslots = cq_host->crypto_capabilities.config_count + 1; 201 202 err = devm_blk_ksm_init(dev, ksm, num_keyslots); 203 if (err) 204 goto out; 205 206 ksm->ksm_ll_ops = cqhci_ksm_ops; 207 ksm->dev = dev; 208 209 /* Unfortunately, CQHCI crypto only supports 32 DUN bits. */ 210 ksm->max_dun_bytes_supported = 4; 211 212 /* 213 * Cache all the crypto capabilities and advertise the supported crypto 214 * modes and data unit sizes to the block layer. 215 */ 216 for (cap_idx = 0; cap_idx < cq_host->crypto_capabilities.num_crypto_cap; 217 cap_idx++) { 218 cq_host->crypto_cap_array[cap_idx].reg_val = 219 cpu_to_le32(cqhci_readl(cq_host, 220 CQHCI_CRYPTOCAP + 221 cap_idx * sizeof(__le32))); 222 blk_mode_num = cqhci_find_blk_crypto_mode( 223 cq_host->crypto_cap_array[cap_idx]); 224 if (blk_mode_num == BLK_ENCRYPTION_MODE_INVALID) 225 continue; 226 ksm->crypto_modes_supported[blk_mode_num] |= 227 cq_host->crypto_cap_array[cap_idx].sdus_mask * 512; 228 } 229 230 /* Clear all the keyslots so that we start in a known state. */ 231 for (slot = 0; slot < num_keyslots; slot++) 232 cqhci_crypto_clear_keyslot(cq_host, slot); 233 234 /* CQHCI crypto requires the use of 128-bit task descriptors. */ 235 cq_host->caps |= CQHCI_TASK_DESC_SZ_128; 236 237 return 0; 238 239 out: 240 mmc->caps2 &= ~MMC_CAP2_CRYPTO; 241 return err; 242 } 243