11e80709bSEric Biggers // SPDX-License-Identifier: GPL-2.0-only
21e80709bSEric Biggers /*
31e80709bSEric Biggers * CQHCI crypto engine (inline encryption) support
41e80709bSEric Biggers *
51e80709bSEric Biggers * Copyright 2020 Google LLC
61e80709bSEric Biggers */
71e80709bSEric Biggers
81e80709bSEric Biggers #include <linux/blk-crypto.h>
91e8d44bdSEric Biggers #include <linux/blk-crypto-profile.h>
101e80709bSEric Biggers #include <linux/mmc/host.h>
111e80709bSEric Biggers
121e80709bSEric Biggers #include "cqhci-crypto.h"
131e80709bSEric Biggers
141e80709bSEric Biggers /* Map from blk-crypto modes to CQHCI crypto algorithm IDs and key sizes */
151e80709bSEric Biggers static const struct cqhci_crypto_alg_entry {
161e80709bSEric Biggers enum cqhci_crypto_alg alg;
171e80709bSEric Biggers enum cqhci_crypto_key_size key_size;
181e80709bSEric Biggers } cqhci_crypto_algs[BLK_ENCRYPTION_MODE_MAX] = {
191e80709bSEric Biggers [BLK_ENCRYPTION_MODE_AES_256_XTS] = {
201e80709bSEric Biggers .alg = CQHCI_CRYPTO_ALG_AES_XTS,
211e80709bSEric Biggers .key_size = CQHCI_CRYPTO_KEY_SIZE_256,
221e80709bSEric Biggers },
231e80709bSEric Biggers };
241e80709bSEric Biggers
251e80709bSEric Biggers static inline struct cqhci_host *
cqhci_host_from_crypto_profile(struct blk_crypto_profile * profile)26*cb77cb5aSEric Biggers cqhci_host_from_crypto_profile(struct blk_crypto_profile *profile)
271e80709bSEric Biggers {
28*cb77cb5aSEric Biggers struct mmc_host *mmc =
29*cb77cb5aSEric Biggers container_of(profile, struct mmc_host, crypto_profile);
301e80709bSEric Biggers
311e80709bSEric Biggers return mmc->cqe_private;
321e80709bSEric Biggers }
331e80709bSEric Biggers
cqhci_crypto_program_key(struct cqhci_host * cq_host,const union cqhci_crypto_cfg_entry * cfg,int slot)340a0c866fSEric Biggers static int cqhci_crypto_program_key(struct cqhci_host *cq_host,
351e80709bSEric Biggers const union cqhci_crypto_cfg_entry *cfg,
361e80709bSEric Biggers int slot)
371e80709bSEric Biggers {
381e80709bSEric Biggers u32 slot_offset = cq_host->crypto_cfg_register + slot * sizeof(*cfg);
391e80709bSEric Biggers int i;
401e80709bSEric Biggers
410a0c866fSEric Biggers if (cq_host->ops->program_key)
420a0c866fSEric Biggers return cq_host->ops->program_key(cq_host, cfg, slot);
430a0c866fSEric Biggers
441e80709bSEric Biggers /* Clear CFGE */
451e80709bSEric Biggers cqhci_writel(cq_host, 0, slot_offset + 16 * sizeof(cfg->reg_val[0]));
461e80709bSEric Biggers
471e80709bSEric Biggers /* Write the key */
481e80709bSEric Biggers for (i = 0; i < 16; i++) {
491e80709bSEric Biggers cqhci_writel(cq_host, le32_to_cpu(cfg->reg_val[i]),
501e80709bSEric Biggers slot_offset + i * sizeof(cfg->reg_val[0]));
511e80709bSEric Biggers }
521e80709bSEric Biggers /* Write dword 17 */
531e80709bSEric Biggers cqhci_writel(cq_host, le32_to_cpu(cfg->reg_val[17]),
541e80709bSEric Biggers slot_offset + 17 * sizeof(cfg->reg_val[0]));
551e80709bSEric Biggers /* Write dword 16, which includes the new value of CFGE */
561e80709bSEric Biggers cqhci_writel(cq_host, le32_to_cpu(cfg->reg_val[16]),
571e80709bSEric Biggers slot_offset + 16 * sizeof(cfg->reg_val[0]));
580a0c866fSEric Biggers return 0;
591e80709bSEric Biggers }
601e80709bSEric Biggers
cqhci_crypto_keyslot_program(struct blk_crypto_profile * profile,const struct blk_crypto_key * key,unsigned int slot)61*cb77cb5aSEric Biggers static int cqhci_crypto_keyslot_program(struct blk_crypto_profile *profile,
621e80709bSEric Biggers const struct blk_crypto_key *key,
631e80709bSEric Biggers unsigned int slot)
641e80709bSEric Biggers
651e80709bSEric Biggers {
66*cb77cb5aSEric Biggers struct cqhci_host *cq_host = cqhci_host_from_crypto_profile(profile);
671e80709bSEric Biggers const union cqhci_crypto_cap_entry *ccap_array =
681e80709bSEric Biggers cq_host->crypto_cap_array;
691e80709bSEric Biggers const struct cqhci_crypto_alg_entry *alg =
701e80709bSEric Biggers &cqhci_crypto_algs[key->crypto_cfg.crypto_mode];
711e80709bSEric Biggers u8 data_unit_mask = key->crypto_cfg.data_unit_size / 512;
721e80709bSEric Biggers int i;
731e80709bSEric Biggers int cap_idx = -1;
741e80709bSEric Biggers union cqhci_crypto_cfg_entry cfg = {};
750a0c866fSEric Biggers int err;
761e80709bSEric Biggers
771e80709bSEric Biggers BUILD_BUG_ON(CQHCI_CRYPTO_KEY_SIZE_INVALID != 0);
781e80709bSEric Biggers for (i = 0; i < cq_host->crypto_capabilities.num_crypto_cap; i++) {
791e80709bSEric Biggers if (ccap_array[i].algorithm_id == alg->alg &&
801e80709bSEric Biggers ccap_array[i].key_size == alg->key_size &&
811e80709bSEric Biggers (ccap_array[i].sdus_mask & data_unit_mask)) {
821e80709bSEric Biggers cap_idx = i;
831e80709bSEric Biggers break;
841e80709bSEric Biggers }
851e80709bSEric Biggers }
861e80709bSEric Biggers if (WARN_ON(cap_idx < 0))
871e80709bSEric Biggers return -EOPNOTSUPP;
881e80709bSEric Biggers
891e80709bSEric Biggers cfg.data_unit_size = data_unit_mask;
901e80709bSEric Biggers cfg.crypto_cap_idx = cap_idx;
911e80709bSEric Biggers cfg.config_enable = CQHCI_CRYPTO_CONFIGURATION_ENABLE;
921e80709bSEric Biggers
931e80709bSEric Biggers if (ccap_array[cap_idx].algorithm_id == CQHCI_CRYPTO_ALG_AES_XTS) {
941e80709bSEric Biggers /* In XTS mode, the blk_crypto_key's size is already doubled */
951e80709bSEric Biggers memcpy(cfg.crypto_key, key->raw, key->size/2);
961e80709bSEric Biggers memcpy(cfg.crypto_key + CQHCI_CRYPTO_KEY_MAX_SIZE/2,
971e80709bSEric Biggers key->raw + key->size/2, key->size/2);
981e80709bSEric Biggers } else {
991e80709bSEric Biggers memcpy(cfg.crypto_key, key->raw, key->size);
1001e80709bSEric Biggers }
1011e80709bSEric Biggers
1020a0c866fSEric Biggers err = cqhci_crypto_program_key(cq_host, &cfg, slot);
1031e80709bSEric Biggers
1041e80709bSEric Biggers memzero_explicit(&cfg, sizeof(cfg));
1050a0c866fSEric Biggers return err;
1061e80709bSEric Biggers }
1071e80709bSEric Biggers
cqhci_crypto_clear_keyslot(struct cqhci_host * cq_host,int slot)1080a0c866fSEric Biggers static int cqhci_crypto_clear_keyslot(struct cqhci_host *cq_host, int slot)
1091e80709bSEric Biggers {
1101e80709bSEric Biggers /*
1111e80709bSEric Biggers * Clear the crypto cfg on the device. Clearing CFGE
1121e80709bSEric Biggers * might not be sufficient, so just clear the entire cfg.
1131e80709bSEric Biggers */
1141e80709bSEric Biggers union cqhci_crypto_cfg_entry cfg = {};
1151e80709bSEric Biggers
1160a0c866fSEric Biggers return cqhci_crypto_program_key(cq_host, &cfg, slot);
1171e80709bSEric Biggers }
1181e80709bSEric Biggers
cqhci_crypto_keyslot_evict(struct blk_crypto_profile * profile,const struct blk_crypto_key * key,unsigned int slot)119*cb77cb5aSEric Biggers static int cqhci_crypto_keyslot_evict(struct blk_crypto_profile *profile,
1201e80709bSEric Biggers const struct blk_crypto_key *key,
1211e80709bSEric Biggers unsigned int slot)
1221e80709bSEric Biggers {
123*cb77cb5aSEric Biggers struct cqhci_host *cq_host = cqhci_host_from_crypto_profile(profile);
1241e80709bSEric Biggers
1250a0c866fSEric Biggers return cqhci_crypto_clear_keyslot(cq_host, slot);
1261e80709bSEric Biggers }
1271e80709bSEric Biggers
1281e80709bSEric Biggers /*
1291e80709bSEric Biggers * The keyslot management operations for CQHCI crypto.
1301e80709bSEric Biggers *
1311e80709bSEric Biggers * Note that the block layer ensures that these are never called while the host
1321e80709bSEric Biggers * controller is runtime-suspended. However, the CQE won't necessarily be
1331e80709bSEric Biggers * "enabled" when these are called, i.e. CQHCI_ENABLE might not be set in the
1341e80709bSEric Biggers * CQHCI_CFG register. But the hardware allows that.
1351e80709bSEric Biggers */
136*cb77cb5aSEric Biggers static const struct blk_crypto_ll_ops cqhci_crypto_ops = {
1371e80709bSEric Biggers .keyslot_program = cqhci_crypto_keyslot_program,
1381e80709bSEric Biggers .keyslot_evict = cqhci_crypto_keyslot_evict,
1391e80709bSEric Biggers };
1401e80709bSEric Biggers
1411e80709bSEric Biggers static enum blk_crypto_mode_num
cqhci_find_blk_crypto_mode(union cqhci_crypto_cap_entry cap)1421e80709bSEric Biggers cqhci_find_blk_crypto_mode(union cqhci_crypto_cap_entry cap)
1431e80709bSEric Biggers {
1441e80709bSEric Biggers int i;
1451e80709bSEric Biggers
1461e80709bSEric Biggers for (i = 0; i < ARRAY_SIZE(cqhci_crypto_algs); i++) {
1471e80709bSEric Biggers BUILD_BUG_ON(CQHCI_CRYPTO_KEY_SIZE_INVALID != 0);
1481e80709bSEric Biggers if (cqhci_crypto_algs[i].alg == cap.algorithm_id &&
1491e80709bSEric Biggers cqhci_crypto_algs[i].key_size == cap.key_size)
1501e80709bSEric Biggers return i;
1511e80709bSEric Biggers }
1521e80709bSEric Biggers return BLK_ENCRYPTION_MODE_INVALID;
1531e80709bSEric Biggers }
1541e80709bSEric Biggers
1551e80709bSEric Biggers /**
1561e80709bSEric Biggers * cqhci_crypto_init - initialize CQHCI crypto support
1571e80709bSEric Biggers * @cq_host: a cqhci host
1581e80709bSEric Biggers *
1591e80709bSEric Biggers * If the driver previously set MMC_CAP2_CRYPTO and the CQE declares
1601e80709bSEric Biggers * CQHCI_CAP_CS, initialize the crypto support. This involves reading the
161*cb77cb5aSEric Biggers * crypto capability registers, initializing the blk_crypto_profile, clearing
162*cb77cb5aSEric Biggers * all keyslots, and enabling 128-bit task descriptors.
1631e80709bSEric Biggers *
1641e80709bSEric Biggers * Return: 0 if crypto was initialized or isn't supported; whether
1651e80709bSEric Biggers * MMC_CAP2_CRYPTO remains set indicates which one of those cases it is.
1661e80709bSEric Biggers * Also can return a negative errno value on unexpected error.
1671e80709bSEric Biggers */
cqhci_crypto_init(struct cqhci_host * cq_host)1681e80709bSEric Biggers int cqhci_crypto_init(struct cqhci_host *cq_host)
1691e80709bSEric Biggers {
1701e80709bSEric Biggers struct mmc_host *mmc = cq_host->mmc;
1711e80709bSEric Biggers struct device *dev = mmc_dev(mmc);
172*cb77cb5aSEric Biggers struct blk_crypto_profile *profile = &mmc->crypto_profile;
1731e80709bSEric Biggers unsigned int num_keyslots;
1741e80709bSEric Biggers unsigned int cap_idx;
1751e80709bSEric Biggers enum blk_crypto_mode_num blk_mode_num;
1761e80709bSEric Biggers unsigned int slot;
1771e80709bSEric Biggers int err = 0;
1781e80709bSEric Biggers
1791e80709bSEric Biggers if (!(mmc->caps2 & MMC_CAP2_CRYPTO) ||
1801e80709bSEric Biggers !(cqhci_readl(cq_host, CQHCI_CAP) & CQHCI_CAP_CS))
1811e80709bSEric Biggers goto out;
1821e80709bSEric Biggers
1831e80709bSEric Biggers cq_host->crypto_capabilities.reg_val =
1841e80709bSEric Biggers cpu_to_le32(cqhci_readl(cq_host, CQHCI_CCAP));
1851e80709bSEric Biggers
1861e80709bSEric Biggers cq_host->crypto_cfg_register =
1871e80709bSEric Biggers (u32)cq_host->crypto_capabilities.config_array_ptr * 0x100;
1881e80709bSEric Biggers
1891e80709bSEric Biggers cq_host->crypto_cap_array =
1901e80709bSEric Biggers devm_kcalloc(dev, cq_host->crypto_capabilities.num_crypto_cap,
1911e80709bSEric Biggers sizeof(cq_host->crypto_cap_array[0]), GFP_KERNEL);
1921e80709bSEric Biggers if (!cq_host->crypto_cap_array) {
1931e80709bSEric Biggers err = -ENOMEM;
1941e80709bSEric Biggers goto out;
1951e80709bSEric Biggers }
1961e80709bSEric Biggers
1971e80709bSEric Biggers /*
1981e80709bSEric Biggers * CCAP.CFGC is off by one, so the actual number of crypto
1991e80709bSEric Biggers * configurations (a.k.a. keyslots) is CCAP.CFGC + 1.
2001e80709bSEric Biggers */
2011e80709bSEric Biggers num_keyslots = cq_host->crypto_capabilities.config_count + 1;
2021e80709bSEric Biggers
203*cb77cb5aSEric Biggers err = devm_blk_crypto_profile_init(dev, profile, num_keyslots);
2041e80709bSEric Biggers if (err)
2051e80709bSEric Biggers goto out;
2061e80709bSEric Biggers
207*cb77cb5aSEric Biggers profile->ll_ops = cqhci_crypto_ops;
208*cb77cb5aSEric Biggers profile->dev = dev;
2091e80709bSEric Biggers
2101e80709bSEric Biggers /* Unfortunately, CQHCI crypto only supports 32 DUN bits. */
211*cb77cb5aSEric Biggers profile->max_dun_bytes_supported = 4;
2121e80709bSEric Biggers
2131e80709bSEric Biggers /*
2141e80709bSEric Biggers * Cache all the crypto capabilities and advertise the supported crypto
2151e80709bSEric Biggers * modes and data unit sizes to the block layer.
2161e80709bSEric Biggers */
2171e80709bSEric Biggers for (cap_idx = 0; cap_idx < cq_host->crypto_capabilities.num_crypto_cap;
2181e80709bSEric Biggers cap_idx++) {
2191e80709bSEric Biggers cq_host->crypto_cap_array[cap_idx].reg_val =
2201e80709bSEric Biggers cpu_to_le32(cqhci_readl(cq_host,
2211e80709bSEric Biggers CQHCI_CRYPTOCAP +
2221e80709bSEric Biggers cap_idx * sizeof(__le32)));
2231e80709bSEric Biggers blk_mode_num = cqhci_find_blk_crypto_mode(
2241e80709bSEric Biggers cq_host->crypto_cap_array[cap_idx]);
2251e80709bSEric Biggers if (blk_mode_num == BLK_ENCRYPTION_MODE_INVALID)
2261e80709bSEric Biggers continue;
227*cb77cb5aSEric Biggers profile->modes_supported[blk_mode_num] |=
2281e80709bSEric Biggers cq_host->crypto_cap_array[cap_idx].sdus_mask * 512;
2291e80709bSEric Biggers }
2301e80709bSEric Biggers
2311e80709bSEric Biggers /* Clear all the keyslots so that we start in a known state. */
2321e80709bSEric Biggers for (slot = 0; slot < num_keyslots; slot++)
2331e80709bSEric Biggers cqhci_crypto_clear_keyslot(cq_host, slot);
2341e80709bSEric Biggers
2351e80709bSEric Biggers /* CQHCI crypto requires the use of 128-bit task descriptors. */
2361e80709bSEric Biggers cq_host->caps |= CQHCI_TASK_DESC_SZ_128;
2371e80709bSEric Biggers
2381e80709bSEric Biggers return 0;
2391e80709bSEric Biggers
2401e80709bSEric Biggers out:
2411e80709bSEric Biggers mmc->caps2 &= ~MMC_CAP2_CRYPTO;
2421e80709bSEric Biggers return err;
2431e80709bSEric Biggers }
244