1 /* 2 * Copyright (C) 2017 Marvell 3 * 4 * Antoine Tenart <antoine.tenart@free-electrons.com> 5 * 6 * This file is licensed under the terms of the GNU General Public 7 * License version 2. This program is licensed "as is" without any 8 * warranty of any kind, whether express or implied. 9 */ 10 11 #include <linux/clk.h> 12 #include <linux/device.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/dmapool.h> 15 #include <linux/firmware.h> 16 #include <linux/interrupt.h> 17 #include <linux/module.h> 18 #include <linux/of_platform.h> 19 #include <linux/of_irq.h> 20 #include <linux/platform_device.h> 21 #include <linux/workqueue.h> 22 23 #include <crypto/internal/hash.h> 24 #include <crypto/internal/skcipher.h> 25 26 #include "safexcel.h" 27 28 static u32 max_rings = EIP197_MAX_RINGS; 29 module_param(max_rings, uint, 0644); 30 MODULE_PARM_DESC(max_rings, "Maximum number of rings to use."); 31 32 static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv) 33 { 34 u32 val, htable_offset; 35 int i; 36 37 /* Enable the record cache memory access */ 38 val = readl(priv->base + EIP197_CS_RAM_CTRL); 39 val &= ~EIP197_TRC_ENABLE_MASK; 40 val |= EIP197_TRC_ENABLE_0; 41 writel(val, priv->base + EIP197_CS_RAM_CTRL); 42 43 /* Clear all ECC errors */ 44 writel(0, priv->base + EIP197_TRC_ECCCTRL); 45 46 /* 47 * Make sure the cache memory is accessible by taking record cache into 48 * reset. 49 */ 50 val = readl(priv->base + EIP197_TRC_PARAMS); 51 val |= EIP197_TRC_PARAMS_SW_RESET; 52 val &= ~EIP197_TRC_PARAMS_DATA_ACCESS; 53 writel(val, priv->base + EIP197_TRC_PARAMS); 54 55 /* Clear all records */ 56 for (i = 0; i < EIP197_CS_RC_MAX; i++) { 57 u32 val, offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE; 58 59 writel(EIP197_CS_RC_NEXT(EIP197_RC_NULL) | 60 EIP197_CS_RC_PREV(EIP197_RC_NULL), 61 priv->base + offset); 62 63 val = EIP197_CS_RC_NEXT(i+1) | EIP197_CS_RC_PREV(i-1); 64 if (i == 0) 65 val |= EIP197_CS_RC_PREV(EIP197_RC_NULL); 66 else if (i == EIP197_CS_RC_MAX - 1) 67 val |= EIP197_CS_RC_NEXT(EIP197_RC_NULL); 68 writel(val, priv->base + offset + sizeof(u32)); 69 } 70 71 /* Clear the hash table entries */ 72 htable_offset = EIP197_CS_RC_MAX * EIP197_CS_RC_SIZE; 73 for (i = 0; i < 64; i++) 74 writel(GENMASK(29, 0), 75 priv->base + EIP197_CLASSIFICATION_RAMS + htable_offset + i * sizeof(u32)); 76 77 /* Disable the record cache memory access */ 78 val = readl(priv->base + EIP197_CS_RAM_CTRL); 79 val &= ~EIP197_TRC_ENABLE_MASK; 80 writel(val, priv->base + EIP197_CS_RAM_CTRL); 81 82 /* Write head and tail pointers of the record free chain */ 83 val = EIP197_TRC_FREECHAIN_HEAD_PTR(0) | 84 EIP197_TRC_FREECHAIN_TAIL_PTR(EIP197_CS_RC_MAX - 1); 85 writel(val, priv->base + EIP197_TRC_FREECHAIN); 86 87 /* Configure the record cache #1 */ 88 val = EIP197_TRC_PARAMS2_RC_SZ_SMALL(EIP197_CS_TRC_REC_WC) | 89 EIP197_TRC_PARAMS2_HTABLE_PTR(EIP197_CS_RC_MAX); 90 writel(val, priv->base + EIP197_TRC_PARAMS2); 91 92 /* Configure the record cache #2 */ 93 val = EIP197_TRC_PARAMS_RC_SZ_LARGE(EIP197_CS_TRC_LG_REC_WC) | 94 EIP197_TRC_PARAMS_BLK_TIMER_SPEED(1) | 95 EIP197_TRC_PARAMS_HTABLE_SZ(2); 96 writel(val, priv->base + EIP197_TRC_PARAMS); 97 } 98 99 static void eip197_write_firmware(struct safexcel_crypto_priv *priv, 100 const struct firmware *fw, u32 ctrl, 101 u32 prog_en) 102 { 103 const u32 *data = (const u32 *)fw->data; 104 u32 val; 105 int i; 106 107 /* Reset the engine to make its program memory accessible */ 108 writel(EIP197_PE_ICE_x_CTRL_SW_RESET | 109 EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR | 110 EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR, 111 EIP197_PE(priv) + ctrl); 112 113 /* Enable access to the program memory */ 114 writel(prog_en, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL); 115 116 /* Write the firmware */ 117 for (i = 0; i < fw->size / sizeof(u32); i++) 118 writel(be32_to_cpu(data[i]), 119 priv->base + EIP197_CLASSIFICATION_RAMS + i * sizeof(u32)); 120 121 /* Disable access to the program memory */ 122 writel(0, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL); 123 124 /* Release engine from reset */ 125 val = readl(EIP197_PE(priv) + ctrl); 126 val &= ~EIP197_PE_ICE_x_CTRL_SW_RESET; 127 writel(val, EIP197_PE(priv) + ctrl); 128 } 129 130 static int eip197_load_firmwares(struct safexcel_crypto_priv *priv) 131 { 132 const char *fw_name[] = {"ifpp.bin", "ipue.bin"}; 133 const struct firmware *fw[FW_NB]; 134 int i, j, ret = 0; 135 u32 val; 136 137 for (i = 0; i < FW_NB; i++) { 138 ret = request_firmware(&fw[i], fw_name[i], priv->dev); 139 if (ret) { 140 dev_err(priv->dev, 141 "Failed to request firmware %s (%d)\n", 142 fw_name[i], ret); 143 goto release_fw; 144 } 145 } 146 147 /* Clear the scratchpad memory */ 148 val = readl(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL); 149 val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER | 150 EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN | 151 EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS | 152 EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS; 153 writel(val, EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL); 154 155 memset(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_RAM, 0, 156 EIP197_NUM_OF_SCRATCH_BLOCKS * sizeof(u32)); 157 158 eip197_write_firmware(priv, fw[FW_IFPP], EIP197_PE_ICE_FPP_CTRL, 159 EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN); 160 161 eip197_write_firmware(priv, fw[FW_IPUE], EIP197_PE_ICE_PUE_CTRL, 162 EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN); 163 164 release_fw: 165 for (j = 0; j < i; j++) 166 release_firmware(fw[j]); 167 168 return ret; 169 } 170 171 static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv) 172 { 173 u32 hdw, cd_size_rnd, val; 174 int i; 175 176 hdw = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS); 177 hdw &= GENMASK(27, 25); 178 hdw >>= 25; 179 180 cd_size_rnd = (priv->config.cd_size + (BIT(hdw) - 1)) >> hdw; 181 182 for (i = 0; i < priv->config.rings; i++) { 183 /* ring base address */ 184 writel(lower_32_bits(priv->ring[i].cdr.base_dma), 185 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); 186 writel(upper_32_bits(priv->ring[i].cdr.base_dma), 187 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); 188 189 writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.cd_offset << 16) | 190 priv->config.cd_size, 191 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE); 192 writel(((EIP197_FETCH_COUNT * (cd_size_rnd << hdw)) << 16) | 193 (EIP197_FETCH_COUNT * priv->config.cd_offset), 194 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG); 195 196 /* Configure DMA tx control */ 197 val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS); 198 val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS); 199 writel(val, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DMA_CFG); 200 201 /* clear any pending interrupt */ 202 writel(GENMASK(5, 0), 203 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT); 204 } 205 206 return 0; 207 } 208 209 static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv) 210 { 211 u32 hdw, rd_size_rnd, val; 212 int i; 213 214 hdw = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS); 215 hdw &= GENMASK(27, 25); 216 hdw >>= 25; 217 218 rd_size_rnd = (priv->config.rd_size + (BIT(hdw) - 1)) >> hdw; 219 220 for (i = 0; i < priv->config.rings; i++) { 221 /* ring base address */ 222 writel(lower_32_bits(priv->ring[i].rdr.base_dma), 223 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); 224 writel(upper_32_bits(priv->ring[i].rdr.base_dma), 225 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); 226 227 writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 16) | 228 priv->config.rd_size, 229 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE); 230 231 writel(((EIP197_FETCH_COUNT * (rd_size_rnd << hdw)) << 16) | 232 (EIP197_FETCH_COUNT * priv->config.rd_offset), 233 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG); 234 235 /* Configure DMA tx control */ 236 val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS); 237 val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS); 238 val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUF; 239 writel(val, 240 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DMA_CFG); 241 242 /* clear any pending interrupt */ 243 writel(GENMASK(7, 0), 244 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT); 245 246 /* enable ring interrupt */ 247 val = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i)); 248 val |= EIP197_RDR_IRQ(i); 249 writel(val, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i)); 250 } 251 252 return 0; 253 } 254 255 static int safexcel_hw_init(struct safexcel_crypto_priv *priv) 256 { 257 u32 version, val; 258 int i, ret; 259 260 /* Determine endianess and configure byte swap */ 261 version = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_VERSION); 262 val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL); 263 264 if ((version & 0xffff) == EIP197_HIA_VERSION_BE) 265 val |= EIP197_MST_CTRL_BYTE_SWAP; 266 else if (((version >> 16) & 0xffff) == EIP197_HIA_VERSION_LE) 267 val |= (EIP197_MST_CTRL_NO_BYTE_SWAP >> 24); 268 269 writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL); 270 271 /* Configure wr/rd cache values */ 272 writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) | 273 EIP197_MST_CTRL_WD_CACHE(WR_CACHE_4BITS), 274 EIP197_HIA_GEN_CFG(priv) + EIP197_MST_CTRL); 275 276 /* Interrupts reset */ 277 278 /* Disable all global interrupts */ 279 writel(0, EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ENABLE_CTRL); 280 281 /* Clear any pending interrupt */ 282 writel(GENMASK(31, 0), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK); 283 284 /* Data Fetch Engine configuration */ 285 286 /* Reset all DFE threads */ 287 writel(EIP197_DxE_THR_CTRL_RESET_PE, 288 EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL); 289 290 if (priv->version == EIP197) { 291 /* Reset HIA input interface arbiter */ 292 writel(EIP197_HIA_RA_PE_CTRL_RESET, 293 EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL); 294 } 295 296 /* DMA transfer size to use */ 297 val = EIP197_HIA_DFE_CFG_DIS_DEBUG; 298 val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(5) | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(9); 299 val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(5) | EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7); 300 val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS); 301 val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS); 302 writel(val, EIP197_HIA_DFE(priv) + EIP197_HIA_DFE_CFG); 303 304 /* Leave the DFE threads reset state */ 305 writel(0, EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL); 306 307 /* Configure the procesing engine thresholds */ 308 writel(EIP197_PE_IN_xBUF_THRES_MIN(5) | EIP197_PE_IN_xBUF_THRES_MAX(9), 309 EIP197_PE(priv) + EIP197_PE_IN_DBUF_THRES); 310 writel(EIP197_PE_IN_xBUF_THRES_MIN(5) | EIP197_PE_IN_xBUF_THRES_MAX(7), 311 EIP197_PE(priv) + EIP197_PE_IN_TBUF_THRES); 312 313 if (priv->version == EIP197) { 314 /* enable HIA input interface arbiter and rings */ 315 writel(EIP197_HIA_RA_PE_CTRL_EN | 316 GENMASK(priv->config.rings - 1, 0), 317 EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL); 318 } 319 320 /* Data Store Engine configuration */ 321 322 /* Reset all DSE threads */ 323 writel(EIP197_DxE_THR_CTRL_RESET_PE, 324 EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL); 325 326 /* Wait for all DSE threads to complete */ 327 while ((readl(EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_STAT) & 328 GENMASK(15, 12)) != GENMASK(15, 12)) 329 ; 330 331 /* DMA transfer size to use */ 332 val = EIP197_HIA_DSE_CFG_DIS_DEBUG; 333 val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(7) | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8); 334 val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS); 335 val |= EIP197_HIA_DSE_CFG_ALWAYS_BUFFERABLE; 336 /* FIXME: instability issues can occur for EIP97 but disabling it impact 337 * performances. 338 */ 339 if (priv->version == EIP197) 340 val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR; 341 writel(val, EIP197_HIA_DSE(priv) + EIP197_HIA_DSE_CFG); 342 343 /* Leave the DSE threads reset state */ 344 writel(0, EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL); 345 346 /* Configure the procesing engine thresholds */ 347 writel(EIP197_PE_OUT_DBUF_THRES_MIN(7) | EIP197_PE_OUT_DBUF_THRES_MAX(8), 348 EIP197_PE(priv) + EIP197_PE_OUT_DBUF_THRES); 349 350 /* Processing Engine configuration */ 351 352 /* H/W capabilities selection */ 353 val = EIP197_FUNCTION_RSVD; 354 val |= EIP197_PROTOCOL_ENCRYPT_ONLY | EIP197_PROTOCOL_HASH_ONLY; 355 val |= EIP197_ALG_AES_ECB | EIP197_ALG_AES_CBC; 356 val |= EIP197_ALG_SHA1 | EIP197_ALG_HMAC_SHA1; 357 val |= EIP197_ALG_SHA2 | EIP197_ALG_HMAC_SHA2; 358 writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION_EN); 359 360 /* Command Descriptor Rings prepare */ 361 for (i = 0; i < priv->config.rings; i++) { 362 /* Clear interrupts for this ring */ 363 writel(GENMASK(31, 0), 364 EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CLR(i)); 365 366 /* Disable external triggering */ 367 writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG); 368 369 /* Clear the pending prepared counter */ 370 writel(EIP197_xDR_PREP_CLR_COUNT, 371 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT); 372 373 /* Clear the pending processed counter */ 374 writel(EIP197_xDR_PROC_CLR_COUNT, 375 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT); 376 377 writel(0, 378 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR); 379 writel(0, 380 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR); 381 382 writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset) << 2, 383 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_SIZE); 384 } 385 386 /* Result Descriptor Ring prepare */ 387 for (i = 0; i < priv->config.rings; i++) { 388 /* Disable external triggering*/ 389 writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG); 390 391 /* Clear the pending prepared counter */ 392 writel(EIP197_xDR_PREP_CLR_COUNT, 393 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT); 394 395 /* Clear the pending processed counter */ 396 writel(EIP197_xDR_PROC_CLR_COUNT, 397 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT); 398 399 writel(0, 400 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR); 401 writel(0, 402 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR); 403 404 /* Ring size */ 405 writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset) << 2, 406 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_SIZE); 407 } 408 409 /* Enable command descriptor rings */ 410 writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0), 411 EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL); 412 413 /* Enable result descriptor rings */ 414 writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0), 415 EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL); 416 417 /* Clear any HIA interrupt */ 418 writel(GENMASK(30, 20), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK); 419 420 if (priv->version == EIP197) { 421 eip197_trc_cache_init(priv); 422 423 ret = eip197_load_firmwares(priv); 424 if (ret) 425 return ret; 426 } 427 428 safexcel_hw_setup_cdesc_rings(priv); 429 safexcel_hw_setup_rdesc_rings(priv); 430 431 return 0; 432 } 433 434 /* Called with ring's lock taken */ 435 static void safexcel_try_push_requests(struct safexcel_crypto_priv *priv, 436 int ring) 437 { 438 int coal = min_t(int, priv->ring[ring].requests, EIP197_MAX_BATCH_SZ); 439 440 if (!coal) 441 return; 442 443 /* Configure when we want an interrupt */ 444 writel(EIP197_HIA_RDR_THRESH_PKT_MODE | 445 EIP197_HIA_RDR_THRESH_PROC_PKT(coal), 446 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_THRESH); 447 } 448 449 void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring) 450 { 451 struct crypto_async_request *req, *backlog; 452 struct safexcel_context *ctx; 453 struct safexcel_request *request; 454 int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results; 455 456 /* If a request wasn't properly dequeued because of a lack of resources, 457 * proceeded it first, 458 */ 459 req = priv->ring[ring].req; 460 backlog = priv->ring[ring].backlog; 461 if (req) 462 goto handle_req; 463 464 while (true) { 465 spin_lock_bh(&priv->ring[ring].queue_lock); 466 backlog = crypto_get_backlog(&priv->ring[ring].queue); 467 req = crypto_dequeue_request(&priv->ring[ring].queue); 468 spin_unlock_bh(&priv->ring[ring].queue_lock); 469 470 if (!req) { 471 priv->ring[ring].req = NULL; 472 priv->ring[ring].backlog = NULL; 473 goto finalize; 474 } 475 476 handle_req: 477 request = kzalloc(sizeof(*request), EIP197_GFP_FLAGS(*req)); 478 if (!request) 479 goto request_failed; 480 481 ctx = crypto_tfm_ctx(req->tfm); 482 ret = ctx->send(req, ring, request, &commands, &results); 483 if (ret) { 484 kfree(request); 485 goto request_failed; 486 } 487 488 if (backlog) 489 backlog->complete(backlog, -EINPROGRESS); 490 491 /* In case the send() helper did not issue any command to push 492 * to the engine because the input data was cached, continue to 493 * dequeue other requests as this is valid and not an error. 494 */ 495 if (!commands && !results) { 496 kfree(request); 497 continue; 498 } 499 500 spin_lock_bh(&priv->ring[ring].egress_lock); 501 list_add_tail(&request->list, &priv->ring[ring].list); 502 spin_unlock_bh(&priv->ring[ring].egress_lock); 503 504 cdesc += commands; 505 rdesc += results; 506 nreq++; 507 } 508 509 request_failed: 510 /* Not enough resources to handle all the requests. Bail out and save 511 * the request and the backlog for the next dequeue call (per-ring). 512 */ 513 priv->ring[ring].req = req; 514 priv->ring[ring].backlog = backlog; 515 516 finalize: 517 if (!nreq) 518 return; 519 520 spin_lock_bh(&priv->ring[ring].egress_lock); 521 522 priv->ring[ring].requests += nreq; 523 524 if (!priv->ring[ring].busy) { 525 safexcel_try_push_requests(priv, ring); 526 priv->ring[ring].busy = true; 527 } 528 529 spin_unlock_bh(&priv->ring[ring].egress_lock); 530 531 /* let the RDR know we have pending descriptors */ 532 writel((rdesc * priv->config.rd_offset) << 2, 533 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT); 534 535 /* let the CDR know we have pending descriptors */ 536 writel((cdesc * priv->config.cd_offset) << 2, 537 EIP197_HIA_CDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT); 538 } 539 540 void safexcel_complete(struct safexcel_crypto_priv *priv, int ring) 541 { 542 struct safexcel_command_desc *cdesc; 543 544 /* Acknowledge the command descriptors */ 545 do { 546 cdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].cdr); 547 if (IS_ERR(cdesc)) { 548 dev_err(priv->dev, 549 "Could not retrieve the command descriptor\n"); 550 return; 551 } 552 } while (!cdesc->last_seg); 553 } 554 555 void safexcel_inv_complete(struct crypto_async_request *req, int error) 556 { 557 struct safexcel_inv_result *result = req->data; 558 559 if (error == -EINPROGRESS) 560 return; 561 562 result->error = error; 563 complete(&result->completion); 564 } 565 566 int safexcel_invalidate_cache(struct crypto_async_request *async, 567 struct safexcel_crypto_priv *priv, 568 dma_addr_t ctxr_dma, int ring, 569 struct safexcel_request *request) 570 { 571 struct safexcel_command_desc *cdesc; 572 struct safexcel_result_desc *rdesc; 573 int ret = 0; 574 575 spin_lock_bh(&priv->ring[ring].egress_lock); 576 577 /* Prepare command descriptor */ 578 cdesc = safexcel_add_cdesc(priv, ring, true, true, 0, 0, 0, ctxr_dma); 579 if (IS_ERR(cdesc)) { 580 ret = PTR_ERR(cdesc); 581 goto unlock; 582 } 583 584 cdesc->control_data.type = EIP197_TYPE_EXTENDED; 585 cdesc->control_data.options = 0; 586 cdesc->control_data.refresh = 0; 587 cdesc->control_data.control0 = CONTEXT_CONTROL_INV_TR; 588 589 /* Prepare result descriptor */ 590 rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0); 591 592 if (IS_ERR(rdesc)) { 593 ret = PTR_ERR(rdesc); 594 goto cdesc_rollback; 595 } 596 597 request->req = async; 598 goto unlock; 599 600 cdesc_rollback: 601 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); 602 603 unlock: 604 spin_unlock_bh(&priv->ring[ring].egress_lock); 605 return ret; 606 } 607 608 static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv *priv, 609 int ring) 610 { 611 struct safexcel_request *sreq; 612 struct safexcel_context *ctx; 613 int ret, i, nreq, ndesc, tot_descs, handled = 0; 614 bool should_complete; 615 616 handle_results: 617 tot_descs = 0; 618 619 nreq = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT); 620 nreq >>= EIP197_xDR_PROC_xD_PKT_OFFSET; 621 nreq &= EIP197_xDR_PROC_xD_PKT_MASK; 622 if (!nreq) 623 goto requests_left; 624 625 for (i = 0; i < nreq; i++) { 626 spin_lock_bh(&priv->ring[ring].egress_lock); 627 sreq = list_first_entry(&priv->ring[ring].list, 628 struct safexcel_request, list); 629 list_del(&sreq->list); 630 spin_unlock_bh(&priv->ring[ring].egress_lock); 631 632 ctx = crypto_tfm_ctx(sreq->req->tfm); 633 ndesc = ctx->handle_result(priv, ring, sreq->req, 634 &should_complete, &ret); 635 if (ndesc < 0) { 636 kfree(sreq); 637 dev_err(priv->dev, "failed to handle result (%d)", ndesc); 638 goto acknowledge; 639 } 640 641 if (should_complete) { 642 local_bh_disable(); 643 sreq->req->complete(sreq->req, ret); 644 local_bh_enable(); 645 } 646 647 kfree(sreq); 648 tot_descs += ndesc; 649 handled++; 650 } 651 652 acknowledge: 653 if (i) { 654 writel(EIP197_xDR_PROC_xD_PKT(i) | 655 EIP197_xDR_PROC_xD_COUNT(tot_descs * priv->config.rd_offset), 656 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT); 657 } 658 659 /* If the number of requests overflowed the counter, try to proceed more 660 * requests. 661 */ 662 if (nreq == EIP197_xDR_PROC_xD_PKT_MASK) 663 goto handle_results; 664 665 requests_left: 666 spin_lock_bh(&priv->ring[ring].egress_lock); 667 668 priv->ring[ring].requests -= handled; 669 safexcel_try_push_requests(priv, ring); 670 671 if (!priv->ring[ring].requests) 672 priv->ring[ring].busy = false; 673 674 spin_unlock_bh(&priv->ring[ring].egress_lock); 675 } 676 677 static void safexcel_dequeue_work(struct work_struct *work) 678 { 679 struct safexcel_work_data *data = 680 container_of(work, struct safexcel_work_data, work); 681 682 safexcel_dequeue(data->priv, data->ring); 683 } 684 685 struct safexcel_ring_irq_data { 686 struct safexcel_crypto_priv *priv; 687 int ring; 688 }; 689 690 static irqreturn_t safexcel_irq_ring(int irq, void *data) 691 { 692 struct safexcel_ring_irq_data *irq_data = data; 693 struct safexcel_crypto_priv *priv = irq_data->priv; 694 int ring = irq_data->ring, rc = IRQ_NONE; 695 u32 status, stat; 696 697 status = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLED_STAT(ring)); 698 if (!status) 699 return rc; 700 701 /* RDR interrupts */ 702 if (status & EIP197_RDR_IRQ(ring)) { 703 stat = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT); 704 705 if (unlikely(stat & EIP197_xDR_ERR)) { 706 /* 707 * Fatal error, the RDR is unusable and must be 708 * reinitialized. This should not happen under 709 * normal circumstances. 710 */ 711 dev_err(priv->dev, "RDR: fatal error."); 712 } else if (likely(stat & EIP197_xDR_THRESH)) { 713 rc = IRQ_WAKE_THREAD; 714 } 715 716 /* ACK the interrupts */ 717 writel(stat & 0xff, 718 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT); 719 } 720 721 /* ACK the interrupts */ 722 writel(status, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ACK(ring)); 723 724 return rc; 725 } 726 727 static irqreturn_t safexcel_irq_ring_thread(int irq, void *data) 728 { 729 struct safexcel_ring_irq_data *irq_data = data; 730 struct safexcel_crypto_priv *priv = irq_data->priv; 731 int ring = irq_data->ring; 732 733 safexcel_handle_result_descriptor(priv, ring); 734 735 queue_work(priv->ring[ring].workqueue, 736 &priv->ring[ring].work_data.work); 737 738 return IRQ_HANDLED; 739 } 740 741 static int safexcel_request_ring_irq(struct platform_device *pdev, const char *name, 742 irq_handler_t handler, 743 irq_handler_t threaded_handler, 744 struct safexcel_ring_irq_data *ring_irq_priv) 745 { 746 int ret, irq = platform_get_irq_byname(pdev, name); 747 748 if (irq < 0) { 749 dev_err(&pdev->dev, "unable to get IRQ '%s'\n", name); 750 return irq; 751 } 752 753 ret = devm_request_threaded_irq(&pdev->dev, irq, handler, 754 threaded_handler, IRQF_ONESHOT, 755 dev_name(&pdev->dev), ring_irq_priv); 756 if (ret) { 757 dev_err(&pdev->dev, "unable to request IRQ %d\n", irq); 758 return ret; 759 } 760 761 return irq; 762 } 763 764 static struct safexcel_alg_template *safexcel_algs[] = { 765 &safexcel_alg_ecb_aes, 766 &safexcel_alg_cbc_aes, 767 &safexcel_alg_sha1, 768 &safexcel_alg_sha224, 769 &safexcel_alg_sha256, 770 &safexcel_alg_hmac_sha1, 771 &safexcel_alg_hmac_sha224, 772 &safexcel_alg_hmac_sha256, 773 }; 774 775 static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) 776 { 777 int i, j, ret = 0; 778 779 for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) { 780 safexcel_algs[i]->priv = priv; 781 782 if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER) 783 ret = crypto_register_skcipher(&safexcel_algs[i]->alg.skcipher); 784 else 785 ret = crypto_register_ahash(&safexcel_algs[i]->alg.ahash); 786 787 if (ret) 788 goto fail; 789 } 790 791 return 0; 792 793 fail: 794 for (j = 0; j < i; j++) { 795 if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER) 796 crypto_unregister_skcipher(&safexcel_algs[j]->alg.skcipher); 797 else 798 crypto_unregister_ahash(&safexcel_algs[j]->alg.ahash); 799 } 800 801 return ret; 802 } 803 804 static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv) 805 { 806 int i; 807 808 for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) { 809 if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER) 810 crypto_unregister_skcipher(&safexcel_algs[i]->alg.skcipher); 811 else 812 crypto_unregister_ahash(&safexcel_algs[i]->alg.ahash); 813 } 814 } 815 816 static void safexcel_configure(struct safexcel_crypto_priv *priv) 817 { 818 u32 val, mask; 819 820 val = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS); 821 val = (val & GENMASK(27, 25)) >> 25; 822 mask = BIT(val) - 1; 823 824 val = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS); 825 priv->config.rings = min_t(u32, val & GENMASK(3, 0), max_rings); 826 827 priv->config.cd_size = (sizeof(struct safexcel_command_desc) / sizeof(u32)); 828 priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask; 829 830 priv->config.rd_size = (sizeof(struct safexcel_result_desc) / sizeof(u32)); 831 priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask; 832 } 833 834 static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv) 835 { 836 struct safexcel_register_offsets *offsets = &priv->offsets; 837 838 if (priv->version == EIP197) { 839 offsets->hia_aic = EIP197_HIA_AIC_BASE; 840 offsets->hia_aic_g = EIP197_HIA_AIC_G_BASE; 841 offsets->hia_aic_r = EIP197_HIA_AIC_R_BASE; 842 offsets->hia_aic_xdr = EIP197_HIA_AIC_xDR_BASE; 843 offsets->hia_dfe = EIP197_HIA_DFE_BASE; 844 offsets->hia_dfe_thr = EIP197_HIA_DFE_THR_BASE; 845 offsets->hia_dse = EIP197_HIA_DSE_BASE; 846 offsets->hia_dse_thr = EIP197_HIA_DSE_THR_BASE; 847 offsets->hia_gen_cfg = EIP197_HIA_GEN_CFG_BASE; 848 offsets->pe = EIP197_PE_BASE; 849 } else { 850 offsets->hia_aic = EIP97_HIA_AIC_BASE; 851 offsets->hia_aic_g = EIP97_HIA_AIC_G_BASE; 852 offsets->hia_aic_r = EIP97_HIA_AIC_R_BASE; 853 offsets->hia_aic_xdr = EIP97_HIA_AIC_xDR_BASE; 854 offsets->hia_dfe = EIP97_HIA_DFE_BASE; 855 offsets->hia_dfe_thr = EIP97_HIA_DFE_THR_BASE; 856 offsets->hia_dse = EIP97_HIA_DSE_BASE; 857 offsets->hia_dse_thr = EIP97_HIA_DSE_THR_BASE; 858 offsets->hia_gen_cfg = EIP97_HIA_GEN_CFG_BASE; 859 offsets->pe = EIP97_PE_BASE; 860 } 861 } 862 863 static int safexcel_probe(struct platform_device *pdev) 864 { 865 struct device *dev = &pdev->dev; 866 struct resource *res; 867 struct safexcel_crypto_priv *priv; 868 int i, ret; 869 870 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); 871 if (!priv) 872 return -ENOMEM; 873 874 priv->dev = dev; 875 priv->version = (enum safexcel_eip_version)of_device_get_match_data(dev); 876 877 safexcel_init_register_offsets(priv); 878 879 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 880 priv->base = devm_ioremap_resource(dev, res); 881 if (IS_ERR(priv->base)) { 882 dev_err(dev, "failed to get resource\n"); 883 return PTR_ERR(priv->base); 884 } 885 886 priv->clk = devm_clk_get(&pdev->dev, NULL); 887 ret = PTR_ERR_OR_ZERO(priv->clk); 888 /* The clock isn't mandatory */ 889 if (ret != -ENOENT) { 890 if (ret) 891 return ret; 892 893 ret = clk_prepare_enable(priv->clk); 894 if (ret) { 895 dev_err(dev, "unable to enable clk (%d)\n", ret); 896 return ret; 897 } 898 } 899 900 priv->reg_clk = devm_clk_get(&pdev->dev, "reg"); 901 ret = PTR_ERR_OR_ZERO(priv->reg_clk); 902 /* The clock isn't mandatory */ 903 if (ret != -ENOENT) { 904 if (ret) 905 goto err_core_clk; 906 907 ret = clk_prepare_enable(priv->reg_clk); 908 if (ret) { 909 dev_err(dev, "unable to enable reg clk (%d)\n", ret); 910 goto err_core_clk; 911 } 912 } 913 914 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); 915 if (ret) 916 goto err_reg_clk; 917 918 priv->context_pool = dmam_pool_create("safexcel-context", dev, 919 sizeof(struct safexcel_context_record), 920 1, 0); 921 if (!priv->context_pool) { 922 ret = -ENOMEM; 923 goto err_reg_clk; 924 } 925 926 safexcel_configure(priv); 927 928 for (i = 0; i < priv->config.rings; i++) { 929 char irq_name[6] = {0}; /* "ringX\0" */ 930 char wq_name[9] = {0}; /* "wq_ringX\0" */ 931 int irq; 932 struct safexcel_ring_irq_data *ring_irq; 933 934 ret = safexcel_init_ring_descriptors(priv, 935 &priv->ring[i].cdr, 936 &priv->ring[i].rdr); 937 if (ret) 938 goto err_reg_clk; 939 940 ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL); 941 if (!ring_irq) { 942 ret = -ENOMEM; 943 goto err_reg_clk; 944 } 945 946 ring_irq->priv = priv; 947 ring_irq->ring = i; 948 949 snprintf(irq_name, 6, "ring%d", i); 950 irq = safexcel_request_ring_irq(pdev, irq_name, safexcel_irq_ring, 951 safexcel_irq_ring_thread, 952 ring_irq); 953 if (irq < 0) { 954 ret = irq; 955 goto err_reg_clk; 956 } 957 958 priv->ring[i].work_data.priv = priv; 959 priv->ring[i].work_data.ring = i; 960 INIT_WORK(&priv->ring[i].work_data.work, safexcel_dequeue_work); 961 962 snprintf(wq_name, 9, "wq_ring%d", i); 963 priv->ring[i].workqueue = create_singlethread_workqueue(wq_name); 964 if (!priv->ring[i].workqueue) { 965 ret = -ENOMEM; 966 goto err_reg_clk; 967 } 968 969 priv->ring[i].requests = 0; 970 priv->ring[i].busy = false; 971 972 crypto_init_queue(&priv->ring[i].queue, 973 EIP197_DEFAULT_RING_SIZE); 974 975 INIT_LIST_HEAD(&priv->ring[i].list); 976 spin_lock_init(&priv->ring[i].lock); 977 spin_lock_init(&priv->ring[i].egress_lock); 978 spin_lock_init(&priv->ring[i].queue_lock); 979 } 980 981 platform_set_drvdata(pdev, priv); 982 atomic_set(&priv->ring_used, 0); 983 984 ret = safexcel_hw_init(priv); 985 if (ret) { 986 dev_err(dev, "EIP h/w init failed (%d)\n", ret); 987 goto err_reg_clk; 988 } 989 990 ret = safexcel_register_algorithms(priv); 991 if (ret) { 992 dev_err(dev, "Failed to register algorithms (%d)\n", ret); 993 goto err_reg_clk; 994 } 995 996 return 0; 997 998 err_reg_clk: 999 clk_disable_unprepare(priv->reg_clk); 1000 err_core_clk: 1001 clk_disable_unprepare(priv->clk); 1002 return ret; 1003 } 1004 1005 1006 static int safexcel_remove(struct platform_device *pdev) 1007 { 1008 struct safexcel_crypto_priv *priv = platform_get_drvdata(pdev); 1009 int i; 1010 1011 safexcel_unregister_algorithms(priv); 1012 clk_disable_unprepare(priv->clk); 1013 1014 for (i = 0; i < priv->config.rings; i++) 1015 destroy_workqueue(priv->ring[i].workqueue); 1016 1017 return 0; 1018 } 1019 1020 static const struct of_device_id safexcel_of_match_table[] = { 1021 { 1022 .compatible = "inside-secure,safexcel-eip97", 1023 .data = (void *)EIP97, 1024 }, 1025 { 1026 .compatible = "inside-secure,safexcel-eip197", 1027 .data = (void *)EIP197, 1028 }, 1029 {}, 1030 }; 1031 1032 1033 static struct platform_driver crypto_safexcel = { 1034 .probe = safexcel_probe, 1035 .remove = safexcel_remove, 1036 .driver = { 1037 .name = "crypto-safexcel", 1038 .of_match_table = safexcel_of_match_table, 1039 }, 1040 }; 1041 module_platform_driver(crypto_safexcel); 1042 1043 MODULE_AUTHOR("Antoine Tenart <antoine.tenart@free-electrons.com>"); 1044 MODULE_AUTHOR("Ofer Heifetz <oferh@marvell.com>"); 1045 MODULE_AUTHOR("Igal Liberman <igall@marvell.com>"); 1046 MODULE_DESCRIPTION("Support for SafeXcel cryptographic engine EIP197"); 1047 MODULE_LICENSE("GPL v2"); 1048