1 /* 2 * AMD Cryptographic Coprocessor (CCP) driver 3 * 4 * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. 5 * 6 * Author: Tom Lendacky <thomas.lendacky@amd.com> 7 * Author: Gary R Hook <gary.hook@amd.com> 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 as 11 * published by the Free Software Foundation. 12 */ 13 14 #ifndef __CCP_DEV_H__ 15 #define __CCP_DEV_H__ 16 17 #include <linux/device.h> 18 #include <linux/pci.h> 19 #include <linux/spinlock.h> 20 #include <linux/mutex.h> 21 #include <linux/list.h> 22 #include <linux/wait.h> 23 #include <linux/dmapool.h> 24 #include <linux/hw_random.h> 25 #include <linux/bitops.h> 26 #include <linux/interrupt.h> 27 #include <linux/irqreturn.h> 28 #include <linux/dmaengine.h> 29 30 #define MAX_CCP_NAME_LEN 16 31 #define MAX_DMAPOOL_NAME_LEN 32 32 33 #define MAX_HW_QUEUES 5 34 #define MAX_CMD_QLEN 100 35 36 #define TRNG_RETRIES 10 37 38 #define CACHE_NONE 0x00 39 #define CACHE_WB_NO_ALLOC 0xb7 40 41 /****** Register Mappings ******/ 42 #define Q_MASK_REG 0x000 43 #define TRNG_OUT_REG 0x00c 44 #define IRQ_MASK_REG 0x040 45 #define IRQ_STATUS_REG 0x200 46 47 #define DEL_CMD_Q_JOB 0x124 48 #define DEL_Q_ACTIVE 0x00000200 49 #define DEL_Q_ID_SHIFT 6 50 51 #define CMD_REQ0 0x180 52 #define CMD_REQ_INCR 0x04 53 54 #define CMD_Q_STATUS_BASE 0x210 55 #define CMD_Q_INT_STATUS_BASE 0x214 56 #define CMD_Q_STATUS_INCR 0x20 57 58 #define CMD_Q_CACHE_BASE 0x228 59 #define CMD_Q_CACHE_INC 0x20 60 61 #define CMD_Q_ERROR(__qs) ((__qs) & 0x0000003f) 62 #define CMD_Q_DEPTH(__qs) (((__qs) >> 12) & 0x0000000f) 63 64 /* ------------------------ CCP Version 5 Specifics ------------------------ */ 65 #define CMD5_QUEUE_MASK_OFFSET 0x00 66 #define CMD5_REQID_CONFIG_OFFSET 0x08 67 #define LSB_PUBLIC_MASK_LO_OFFSET 0x18 68 #define LSB_PUBLIC_MASK_HI_OFFSET 0x1C 69 #define LSB_PRIVATE_MASK_LO_OFFSET 0x20 70 #define LSB_PRIVATE_MASK_HI_OFFSET 0x24 71 72 #define CMD5_Q_CONTROL_BASE 0x0000 73 #define CMD5_Q_TAIL_LO_BASE 0x0004 74 #define CMD5_Q_HEAD_LO_BASE 0x0008 75 #define CMD5_Q_INT_ENABLE_BASE 0x000C 76 #define CMD5_Q_INTERRUPT_STATUS_BASE 0x0010 77 78 #define CMD5_Q_STATUS_BASE 0x0100 79 #define CMD5_Q_INT_STATUS_BASE 0x0104 80 #define CMD5_Q_DMA_STATUS_BASE 0x0108 81 #define CMD5_Q_DMA_READ_STATUS_BASE 0x010C 82 #define CMD5_Q_DMA_WRITE_STATUS_BASE 0x0110 83 #define CMD5_Q_ABORT_BASE 0x0114 84 #define CMD5_Q_AX_CACHE_BASE 0x0118 85 86 /* Address offset between two virtual queue registers */ 87 #define CMD5_Q_STATUS_INCR 0x1000 88 89 /* Bit masks */ 90 #define CMD5_Q_RUN 0x1 91 #define CMD5_Q_HALT 0x2 92 #define CMD5_Q_MEM_LOCATION 0x4 93 #define CMD5_Q_SIZE 0x1F 94 #define CMD5_Q_SHIFT 3 95 #define COMMANDS_PER_QUEUE 16 96 #define QUEUE_SIZE_VAL ((ffs(COMMANDS_PER_QUEUE) - 2) & \ 97 CMD5_Q_SIZE) 98 #define Q_PTR_MASK (2 << (QUEUE_SIZE_VAL + 5) - 1) 99 #define Q_DESC_SIZE sizeof(struct ccp5_desc) 100 #define Q_SIZE(n) (COMMANDS_PER_QUEUE*(n)) 101 102 #define INT_COMPLETION 0x1 103 #define INT_ERROR 0x2 104 #define INT_QUEUE_STOPPED 0x4 105 #define ALL_INTERRUPTS (INT_COMPLETION| \ 106 INT_ERROR| \ 107 INT_QUEUE_STOPPED) 108 109 #define LSB_REGION_WIDTH 5 110 #define MAX_LSB_CNT 8 111 112 #define LSB_SIZE 16 113 #define LSB_ITEM_SIZE 32 114 #define PLSB_MAP_SIZE (LSB_SIZE) 115 #define SLSB_MAP_SIZE (MAX_LSB_CNT * LSB_SIZE) 116 117 #define LSB_ENTRY_NUMBER(LSB_ADDR) (LSB_ADDR / LSB_ITEM_SIZE) 118 119 /* ------------------------ CCP Version 3 Specifics ------------------------ */ 120 #define REQ0_WAIT_FOR_WRITE 0x00000004 121 #define REQ0_INT_ON_COMPLETE 0x00000002 122 #define REQ0_STOP_ON_COMPLETE 0x00000001 123 124 #define REQ0_CMD_Q_SHIFT 9 125 #define REQ0_JOBID_SHIFT 3 126 127 /****** REQ1 Related Values ******/ 128 #define REQ1_PROTECT_SHIFT 27 129 #define REQ1_ENGINE_SHIFT 23 130 #define REQ1_KEY_KSB_SHIFT 2 131 132 #define REQ1_EOM 0x00000002 133 #define REQ1_INIT 0x00000001 134 135 /* AES Related Values */ 136 #define REQ1_AES_TYPE_SHIFT 21 137 #define REQ1_AES_MODE_SHIFT 18 138 #define REQ1_AES_ACTION_SHIFT 17 139 #define REQ1_AES_CFB_SIZE_SHIFT 10 140 141 /* XTS-AES Related Values */ 142 #define REQ1_XTS_AES_SIZE_SHIFT 10 143 144 /* SHA Related Values */ 145 #define REQ1_SHA_TYPE_SHIFT 21 146 147 /* RSA Related Values */ 148 #define REQ1_RSA_MOD_SIZE_SHIFT 10 149 150 /* Pass-Through Related Values */ 151 #define REQ1_PT_BW_SHIFT 12 152 #define REQ1_PT_BS_SHIFT 10 153 154 /* ECC Related Values */ 155 #define REQ1_ECC_AFFINE_CONVERT 0x00200000 156 #define REQ1_ECC_FUNCTION_SHIFT 18 157 158 /****** REQ4 Related Values ******/ 159 #define REQ4_KSB_SHIFT 18 160 #define REQ4_MEMTYPE_SHIFT 16 161 162 /****** REQ6 Related Values ******/ 163 #define REQ6_MEMTYPE_SHIFT 16 164 165 /****** Key Storage Block ******/ 166 #define KSB_START 77 167 #define KSB_END 127 168 #define KSB_COUNT (KSB_END - KSB_START + 1) 169 #define CCP_SB_BITS 256 170 171 #define CCP_JOBID_MASK 0x0000003f 172 173 /* ------------------------ General CCP Defines ------------------------ */ 174 175 #define CCP_DMAPOOL_MAX_SIZE 64 176 #define CCP_DMAPOOL_ALIGN BIT(5) 177 178 #define CCP_REVERSE_BUF_SIZE 64 179 180 #define CCP_AES_KEY_SB_COUNT 1 181 #define CCP_AES_CTX_SB_COUNT 1 182 183 #define CCP_XTS_AES_KEY_SB_COUNT 1 184 #define CCP_XTS_AES_CTX_SB_COUNT 1 185 186 #define CCP_SHA_SB_COUNT 1 187 188 #define CCP_RSA_MAX_WIDTH 4096 189 190 #define CCP_PASSTHRU_BLOCKSIZE 256 191 #define CCP_PASSTHRU_MASKSIZE 32 192 #define CCP_PASSTHRU_SB_COUNT 1 193 194 #define CCP_ECC_MODULUS_BYTES 48 /* 384-bits */ 195 #define CCP_ECC_MAX_OPERANDS 6 196 #define CCP_ECC_MAX_OUTPUTS 3 197 #define CCP_ECC_SRC_BUF_SIZE 448 198 #define CCP_ECC_DST_BUF_SIZE 192 199 #define CCP_ECC_OPERAND_SIZE 64 200 #define CCP_ECC_OUTPUT_SIZE 64 201 #define CCP_ECC_RESULT_OFFSET 60 202 #define CCP_ECC_RESULT_SUCCESS 0x0001 203 204 #define CCP_SB_BYTES 32 205 206 struct ccp_op; 207 struct ccp_device; 208 struct ccp_cmd; 209 struct ccp_fns; 210 211 struct ccp_dma_cmd { 212 struct list_head entry; 213 214 struct ccp_cmd ccp_cmd; 215 }; 216 217 struct ccp_dma_desc { 218 struct list_head entry; 219 220 struct ccp_device *ccp; 221 222 struct list_head pending; 223 struct list_head active; 224 225 enum dma_status status; 226 struct dma_async_tx_descriptor tx_desc; 227 size_t len; 228 }; 229 230 struct ccp_dma_chan { 231 struct ccp_device *ccp; 232 233 spinlock_t lock; 234 struct list_head pending; 235 struct list_head active; 236 struct list_head complete; 237 238 struct tasklet_struct cleanup_tasklet; 239 240 enum dma_status status; 241 struct dma_chan dma_chan; 242 }; 243 244 struct ccp_cmd_queue { 245 struct ccp_device *ccp; 246 247 /* Queue identifier */ 248 u32 id; 249 250 /* Queue dma pool */ 251 struct dma_pool *dma_pool; 252 253 /* Queue base address (not neccessarily aligned)*/ 254 struct ccp5_desc *qbase; 255 256 /* Aligned queue start address (per requirement) */ 257 struct mutex q_mutex ____cacheline_aligned; 258 unsigned int qidx; 259 260 /* Version 5 has different requirements for queue memory */ 261 unsigned int qsize; 262 dma_addr_t qbase_dma; 263 dma_addr_t qdma_tail; 264 265 /* Per-queue reserved storage block(s) */ 266 u32 sb_key; 267 u32 sb_ctx; 268 269 /* Bitmap of LSBs that can be accessed by this queue */ 270 DECLARE_BITMAP(lsbmask, MAX_LSB_CNT); 271 /* Private LSB that is assigned to this queue, or -1 if none. 272 * Bitmap for my private LSB, unused otherwise 273 */ 274 unsigned int lsb; 275 DECLARE_BITMAP(lsbmap, PLSB_MAP_SIZE); 276 277 /* Queue processing thread */ 278 struct task_struct *kthread; 279 unsigned int active; 280 unsigned int suspended; 281 282 /* Number of free command slots available */ 283 unsigned int free_slots; 284 285 /* Interrupt masks */ 286 u32 int_ok; 287 u32 int_err; 288 289 /* Register addresses for queue */ 290 void __iomem *reg_control; 291 void __iomem *reg_tail_lo; 292 void __iomem *reg_head_lo; 293 void __iomem *reg_int_enable; 294 void __iomem *reg_interrupt_status; 295 void __iomem *reg_status; 296 void __iomem *reg_int_status; 297 void __iomem *reg_dma_status; 298 void __iomem *reg_dma_read_status; 299 void __iomem *reg_dma_write_status; 300 u32 qcontrol; /* Cached control register */ 301 302 /* Status values from job */ 303 u32 int_status; 304 u32 q_status; 305 u32 q_int_status; 306 u32 cmd_error; 307 308 /* Interrupt wait queue */ 309 wait_queue_head_t int_queue; 310 unsigned int int_rcvd; 311 } ____cacheline_aligned; 312 313 struct ccp_device { 314 struct list_head entry; 315 316 struct ccp_vdata *vdata; 317 unsigned int ord; 318 char name[MAX_CCP_NAME_LEN]; 319 char rngname[MAX_CCP_NAME_LEN]; 320 321 struct device *dev; 322 323 /* Bus specific device information 324 */ 325 void *dev_specific; 326 int (*get_irq)(struct ccp_device *ccp); 327 void (*free_irq)(struct ccp_device *ccp); 328 unsigned int irq; 329 330 /* I/O area used for device communication. The register mapping 331 * starts at an offset into the mapped bar. 332 * The CMD_REQx registers and the Delete_Cmd_Queue_Job register 333 * need to be protected while a command queue thread is accessing 334 * them. 335 */ 336 struct mutex req_mutex ____cacheline_aligned; 337 void __iomem *io_map; 338 void __iomem *io_regs; 339 340 /* Master lists that all cmds are queued on. Because there can be 341 * more than one CCP command queue that can process a cmd a separate 342 * backlog list is neeeded so that the backlog completion call 343 * completes before the cmd is available for execution. 344 */ 345 spinlock_t cmd_lock ____cacheline_aligned; 346 unsigned int cmd_count; 347 struct list_head cmd; 348 struct list_head backlog; 349 350 /* The command queues. These represent the queues available on the 351 * CCP that are available for processing cmds 352 */ 353 struct ccp_cmd_queue cmd_q[MAX_HW_QUEUES]; 354 unsigned int cmd_q_count; 355 356 /* Support for the CCP True RNG 357 */ 358 struct hwrng hwrng; 359 unsigned int hwrng_retries; 360 361 /* Support for the CCP DMA capabilities 362 */ 363 struct dma_device dma_dev; 364 struct ccp_dma_chan *ccp_dma_chan; 365 struct kmem_cache *dma_cmd_cache; 366 struct kmem_cache *dma_desc_cache; 367 368 /* A counter used to generate job-ids for cmds submitted to the CCP 369 */ 370 atomic_t current_id ____cacheline_aligned; 371 372 /* The v3 CCP uses key storage blocks (SB) to maintain context for 373 * certain operations. To prevent multiple cmds from using the same 374 * SB range a command queue reserves an SB range for the duration of 375 * the cmd. Each queue, will however, reserve 2 SB blocks for 376 * operations that only require single SB entries (eg. AES context/iv 377 * and key) in order to avoid allocation contention. This will reserve 378 * at most 10 SB entries, leaving 40 SB entries available for dynamic 379 * allocation. 380 * 381 * The v5 CCP Local Storage Block (LSB) is broken up into 8 382 * memrory ranges, each of which can be enabled for access by one 383 * or more queues. Device initialization takes this into account, 384 * and attempts to assign one region for exclusive use by each 385 * available queue; the rest are then aggregated as "public" use. 386 * If there are fewer regions than queues, all regions are shared 387 * amongst all queues. 388 */ 389 struct mutex sb_mutex ____cacheline_aligned; 390 DECLARE_BITMAP(sb, KSB_COUNT); 391 wait_queue_head_t sb_queue; 392 unsigned int sb_avail; 393 unsigned int sb_count; 394 u32 sb_start; 395 396 /* Bitmap of shared LSBs, if any */ 397 DECLARE_BITMAP(lsbmap, SLSB_MAP_SIZE); 398 399 /* Suspend support */ 400 unsigned int suspending; 401 wait_queue_head_t suspend_queue; 402 403 /* DMA caching attribute support */ 404 unsigned int axcache; 405 }; 406 407 enum ccp_memtype { 408 CCP_MEMTYPE_SYSTEM = 0, 409 CCP_MEMTYPE_SB, 410 CCP_MEMTYPE_LOCAL, 411 CCP_MEMTYPE__LAST, 412 }; 413 #define CCP_MEMTYPE_LSB CCP_MEMTYPE_KSB 414 415 struct ccp_dma_info { 416 dma_addr_t address; 417 unsigned int offset; 418 unsigned int length; 419 enum dma_data_direction dir; 420 }; 421 422 struct ccp_dm_workarea { 423 struct device *dev; 424 struct dma_pool *dma_pool; 425 unsigned int length; 426 427 u8 *address; 428 struct ccp_dma_info dma; 429 }; 430 431 struct ccp_sg_workarea { 432 struct scatterlist *sg; 433 int nents; 434 435 struct scatterlist *dma_sg; 436 struct device *dma_dev; 437 unsigned int dma_count; 438 enum dma_data_direction dma_dir; 439 440 unsigned int sg_used; 441 442 u64 bytes_left; 443 }; 444 445 struct ccp_data { 446 struct ccp_sg_workarea sg_wa; 447 struct ccp_dm_workarea dm_wa; 448 }; 449 450 struct ccp_mem { 451 enum ccp_memtype type; 452 union { 453 struct ccp_dma_info dma; 454 u32 sb; 455 } u; 456 }; 457 458 struct ccp_aes_op { 459 enum ccp_aes_type type; 460 enum ccp_aes_mode mode; 461 enum ccp_aes_action action; 462 }; 463 464 struct ccp_xts_aes_op { 465 enum ccp_aes_action action; 466 enum ccp_xts_aes_unit_size unit_size; 467 }; 468 469 struct ccp_sha_op { 470 enum ccp_sha_type type; 471 u64 msg_bits; 472 }; 473 474 struct ccp_rsa_op { 475 u32 mod_size; 476 u32 input_len; 477 }; 478 479 struct ccp_passthru_op { 480 enum ccp_passthru_bitwise bit_mod; 481 enum ccp_passthru_byteswap byte_swap; 482 }; 483 484 struct ccp_ecc_op { 485 enum ccp_ecc_function function; 486 }; 487 488 struct ccp_op { 489 struct ccp_cmd_queue *cmd_q; 490 491 u32 jobid; 492 u32 ioc; 493 u32 soc; 494 u32 sb_key; 495 u32 sb_ctx; 496 u32 init; 497 u32 eom; 498 499 struct ccp_mem src; 500 struct ccp_mem dst; 501 struct ccp_mem exp; 502 503 union { 504 struct ccp_aes_op aes; 505 struct ccp_xts_aes_op xts; 506 struct ccp_sha_op sha; 507 struct ccp_rsa_op rsa; 508 struct ccp_passthru_op passthru; 509 struct ccp_ecc_op ecc; 510 } u; 511 struct ccp_mem key; 512 }; 513 514 static inline u32 ccp_addr_lo(struct ccp_dma_info *info) 515 { 516 return lower_32_bits(info->address + info->offset); 517 } 518 519 static inline u32 ccp_addr_hi(struct ccp_dma_info *info) 520 { 521 return upper_32_bits(info->address + info->offset) & 0x0000ffff; 522 } 523 524 /** 525 * descriptor for version 5 CPP commands 526 * 8 32-bit words: 527 * word 0: function; engine; control bits 528 * word 1: length of source data 529 * word 2: low 32 bits of source pointer 530 * word 3: upper 16 bits of source pointer; source memory type 531 * word 4: low 32 bits of destination pointer 532 * word 5: upper 16 bits of destination pointer; destination memory type 533 * word 6: low 32 bits of key pointer 534 * word 7: upper 16 bits of key pointer; key memory type 535 */ 536 struct dword0 { 537 __le32 soc:1; 538 __le32 ioc:1; 539 __le32 rsvd1:1; 540 __le32 init:1; 541 __le32 eom:1; /* AES/SHA only */ 542 __le32 function:15; 543 __le32 engine:4; 544 __le32 prot:1; 545 __le32 rsvd2:7; 546 }; 547 548 struct dword3 { 549 __le32 src_hi:16; 550 __le32 src_mem:2; 551 __le32 lsb_cxt_id:8; 552 __le32 rsvd1:5; 553 __le32 fixed:1; 554 }; 555 556 union dword4 { 557 __le32 dst_lo; /* NON-SHA */ 558 __le32 sha_len_lo; /* SHA */ 559 }; 560 561 union dword5 { 562 struct { 563 __le32 dst_hi:16; 564 __le32 dst_mem:2; 565 __le32 rsvd1:13; 566 __le32 fixed:1; 567 } fields; 568 __le32 sha_len_hi; 569 }; 570 571 struct dword7 { 572 __le32 key_hi:16; 573 __le32 key_mem:2; 574 __le32 rsvd1:14; 575 }; 576 577 struct ccp5_desc { 578 struct dword0 dw0; 579 __le32 length; 580 __le32 src_lo; 581 struct dword3 dw3; 582 union dword4 dw4; 583 union dword5 dw5; 584 __le32 key_lo; 585 struct dword7 dw7; 586 }; 587 588 int ccp_pci_init(void); 589 void ccp_pci_exit(void); 590 591 int ccp_platform_init(void); 592 void ccp_platform_exit(void); 593 594 void ccp_add_device(struct ccp_device *ccp); 595 void ccp_del_device(struct ccp_device *ccp); 596 597 struct ccp_device *ccp_alloc_struct(struct device *dev); 598 bool ccp_queues_suspended(struct ccp_device *ccp); 599 int ccp_cmd_queue_thread(void *data); 600 int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait); 601 602 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd); 603 604 int ccp_register_rng(struct ccp_device *ccp); 605 void ccp_unregister_rng(struct ccp_device *ccp); 606 int ccp_dmaengine_register(struct ccp_device *ccp); 607 void ccp_dmaengine_unregister(struct ccp_device *ccp); 608 609 /* Structure for computation functions that are device-specific */ 610 struct ccp_actions { 611 int (*aes)(struct ccp_op *); 612 int (*xts_aes)(struct ccp_op *); 613 int (*sha)(struct ccp_op *); 614 int (*rsa)(struct ccp_op *); 615 int (*passthru)(struct ccp_op *); 616 int (*ecc)(struct ccp_op *); 617 u32 (*sballoc)(struct ccp_cmd_queue *, unsigned int); 618 void (*sbfree)(struct ccp_cmd_queue *, unsigned int, 619 unsigned int); 620 unsigned int (*get_free_slots)(struct ccp_cmd_queue *); 621 int (*init)(struct ccp_device *); 622 void (*destroy)(struct ccp_device *); 623 irqreturn_t (*irqhandler)(int, void *); 624 }; 625 626 /* Structure to hold CCP version-specific values */ 627 struct ccp_vdata { 628 const unsigned int version; 629 void (*setup)(struct ccp_device *); 630 const struct ccp_actions *perform; 631 const unsigned int bar; 632 const unsigned int offset; 633 }; 634 635 extern struct ccp_vdata ccpv3; 636 extern struct ccp_vdata ccpv5; 637 638 #endif 639