1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) 2 /* Copyright(c) 2020 Intel Corporation */ 3 #include <linux/device.h> 4 #include <linux/module.h> 5 #include <linux/pci.h> 6 7 #include <adf_accel_devices.h> 8 #include <adf_cfg.h> 9 #include <adf_common_drv.h> 10 11 #include "adf_4xxx_hw_data.h" 12 #include "qat_compression.h" 13 #include "qat_crypto.h" 14 #include "adf_transport_access_macros.h" 15 16 static const struct pci_device_id adf_pci_tbl[] = { 17 { PCI_VDEVICE(INTEL, ADF_4XXX_PCI_DEVICE_ID), }, 18 { PCI_VDEVICE(INTEL, ADF_401XX_PCI_DEVICE_ID), }, 19 { PCI_VDEVICE(INTEL, ADF_402XX_PCI_DEVICE_ID), }, 20 { } 21 }; 22 MODULE_DEVICE_TABLE(pci, adf_pci_tbl); 23 24 enum configs { 25 DEV_CFG_CY = 0, 26 DEV_CFG_DC, 27 }; 28 29 static const char * const services_operations[] = { 30 ADF_CFG_CY, 31 ADF_CFG_DC, 32 }; 33 34 static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) 35 { 36 if (accel_dev->hw_device) { 37 adf_clean_hw_data_4xxx(accel_dev->hw_device); 38 accel_dev->hw_device = NULL; 39 } 40 adf_cfg_dev_remove(accel_dev); 41 debugfs_remove(accel_dev->debugfs_dir); 42 adf_devmgr_rm_dev(accel_dev, NULL); 43 } 44 45 static int adf_cfg_dev_init(struct adf_accel_dev *accel_dev) 46 { 47 const char *config; 48 int ret; 49 50 config = accel_dev->accel_id % 2 ? ADF_CFG_DC : ADF_CFG_CY; 51 52 ret = adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC); 53 if (ret) 54 return ret; 55 56 /* Default configuration is crypto only for even devices 57 * and compression for odd devices 58 */ 59 ret = adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC, 60 ADF_SERVICES_ENABLED, config, 61 ADF_STR); 62 if (ret) 63 return ret; 64 65 return 0; 66 } 67 68 static int adf_crypto_dev_config(struct adf_accel_dev *accel_dev) 69 { 70 char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; 71 int banks = GET_MAX_BANKS(accel_dev); 72 int cpus = num_online_cpus(); 73 unsigned long bank, val; 74 int instances; 75 int ret; 76 int i; 77 78 if (adf_hw_dev_has_crypto(accel_dev)) 79 instances = min(cpus, banks / 2); 80 else 81 instances = 0; 82 83 for (i = 0; i < instances; i++) { 84 val = i; 85 bank = i * 2; 86 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i); 87 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 88 key, &bank, ADF_DEC); 89 if (ret) 90 goto err; 91 92 bank += 1; 93 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i); 94 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 95 key, &bank, ADF_DEC); 96 if (ret) 97 goto err; 98 99 snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, 100 i); 101 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 102 key, &val, ADF_DEC); 103 if (ret) 104 goto err; 105 106 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); 107 val = 128; 108 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 109 key, &val, ADF_DEC); 110 if (ret) 111 goto err; 112 113 val = 512; 114 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); 115 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 116 key, &val, ADF_DEC); 117 if (ret) 118 goto err; 119 120 val = 0; 121 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); 122 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 123 key, &val, ADF_DEC); 124 if (ret) 125 goto err; 126 127 val = 0; 128 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); 129 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 130 key, &val, ADF_DEC); 131 if (ret) 132 goto err; 133 134 val = 1; 135 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); 136 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 137 key, &val, ADF_DEC); 138 if (ret) 139 goto err; 140 141 val = 1; 142 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); 143 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 144 key, &val, ADF_DEC); 145 if (ret) 146 goto err; 147 148 val = ADF_COALESCING_DEF_TIME; 149 snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); 150 ret = adf_cfg_add_key_value_param(accel_dev, "Accelerator0", 151 key, &val, ADF_DEC); 152 if (ret) 153 goto err; 154 } 155 156 val = i; 157 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY, 158 &val, ADF_DEC); 159 if (ret) 160 goto err; 161 162 val = 0; 163 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_DC, 164 &val, ADF_DEC); 165 if (ret) 166 goto err; 167 168 return 0; 169 err: 170 dev_err(&GET_DEV(accel_dev), "Failed to add configuration for crypto\n"); 171 return ret; 172 } 173 174 static int adf_comp_dev_config(struct adf_accel_dev *accel_dev) 175 { 176 char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; 177 int banks = GET_MAX_BANKS(accel_dev); 178 int cpus = num_online_cpus(); 179 unsigned long val; 180 int instances; 181 int ret; 182 int i; 183 184 if (adf_hw_dev_has_compression(accel_dev)) 185 instances = min(cpus, banks); 186 else 187 instances = 0; 188 189 for (i = 0; i < instances; i++) { 190 val = i; 191 snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_BANK_NUM, i); 192 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 193 key, &val, ADF_DEC); 194 if (ret) 195 goto err; 196 197 val = 512; 198 snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_SIZE, i); 199 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 200 key, &val, ADF_DEC); 201 if (ret) 202 goto err; 203 204 val = 0; 205 snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_TX, i); 206 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 207 key, &val, ADF_DEC); 208 if (ret) 209 goto err; 210 211 val = 1; 212 snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_RX, i); 213 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 214 key, &val, ADF_DEC); 215 if (ret) 216 goto err; 217 218 val = ADF_COALESCING_DEF_TIME; 219 snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); 220 ret = adf_cfg_add_key_value_param(accel_dev, "Accelerator0", 221 key, &val, ADF_DEC); 222 if (ret) 223 goto err; 224 } 225 226 val = i; 227 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_DC, 228 &val, ADF_DEC); 229 if (ret) 230 goto err; 231 232 val = 0; 233 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY, 234 &val, ADF_DEC); 235 if (ret) 236 goto err; 237 238 return 0; 239 err: 240 dev_err(&GET_DEV(accel_dev), "Failed to add configuration for compression\n"); 241 return ret; 242 } 243 244 int adf_gen4_dev_config(struct adf_accel_dev *accel_dev) 245 { 246 char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0}; 247 int ret; 248 249 ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC); 250 if (ret) 251 goto err; 252 253 ret = adf_cfg_section_add(accel_dev, "Accelerator0"); 254 if (ret) 255 goto err; 256 257 ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, 258 ADF_SERVICES_ENABLED, services); 259 if (ret) 260 goto err; 261 262 ret = sysfs_match_string(services_operations, services); 263 if (ret < 0) 264 goto err; 265 266 switch (ret) { 267 case DEV_CFG_CY: 268 ret = adf_crypto_dev_config(accel_dev); 269 break; 270 case DEV_CFG_DC: 271 ret = adf_comp_dev_config(accel_dev); 272 break; 273 } 274 275 if (ret) 276 goto err; 277 278 set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); 279 280 return ret; 281 282 err: 283 dev_err(&GET_DEV(accel_dev), "Failed to configure QAT driver\n"); 284 return ret; 285 } 286 287 static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 288 { 289 struct adf_accel_dev *accel_dev; 290 struct adf_accel_pci *accel_pci_dev; 291 struct adf_hw_device_data *hw_data; 292 char name[ADF_DEVICE_NAME_LENGTH]; 293 unsigned int i, bar_nr; 294 unsigned long bar_mask; 295 struct adf_bar *bar; 296 int ret; 297 298 if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) { 299 /* 300 * If the accelerator is connected to a node with no memory 301 * there is no point in using the accelerator since the remote 302 * memory transaction will be very slow. 303 */ 304 dev_err(&pdev->dev, "Invalid NUMA configuration.\n"); 305 return -EINVAL; 306 } 307 308 accel_dev = devm_kzalloc(&pdev->dev, sizeof(*accel_dev), GFP_KERNEL); 309 if (!accel_dev) 310 return -ENOMEM; 311 312 INIT_LIST_HEAD(&accel_dev->crypto_list); 313 accel_pci_dev = &accel_dev->accel_pci_dev; 314 accel_pci_dev->pci_dev = pdev; 315 316 /* 317 * Add accel device to accel table 318 * This should be called before adf_cleanup_accel is called 319 */ 320 if (adf_devmgr_add_dev(accel_dev, NULL)) { 321 dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); 322 return -EFAULT; 323 } 324 325 accel_dev->owner = THIS_MODULE; 326 /* Allocate and initialise device hardware meta-data structure */ 327 hw_data = devm_kzalloc(&pdev->dev, sizeof(*hw_data), GFP_KERNEL); 328 if (!hw_data) { 329 ret = -ENOMEM; 330 goto out_err; 331 } 332 333 accel_dev->hw_device = hw_data; 334 adf_init_hw_data_4xxx(accel_dev->hw_device, ent->device); 335 336 pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); 337 pci_read_config_dword(pdev, ADF_4XXX_FUSECTL4_OFFSET, &hw_data->fuses); 338 339 /* Get Accelerators and Accelerators Engines masks */ 340 hw_data->accel_mask = hw_data->get_accel_mask(hw_data); 341 hw_data->ae_mask = hw_data->get_ae_mask(hw_data); 342 accel_pci_dev->sku = hw_data->get_sku(hw_data); 343 /* If the device has no acceleration engines then ignore it */ 344 if (!hw_data->accel_mask || !hw_data->ae_mask || 345 (~hw_data->ae_mask & 0x01)) { 346 dev_err(&pdev->dev, "No acceleration units found.\n"); 347 ret = -EFAULT; 348 goto out_err; 349 } 350 351 /* Create dev top level debugfs entry */ 352 snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, 353 hw_data->dev_class->name, pci_name(pdev)); 354 355 accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); 356 357 /* Create device configuration table */ 358 ret = adf_cfg_dev_add(accel_dev); 359 if (ret) 360 goto out_err; 361 362 /* Enable PCI device */ 363 ret = pcim_enable_device(pdev); 364 if (ret) { 365 dev_err(&pdev->dev, "Can't enable PCI device.\n"); 366 goto out_err; 367 } 368 369 /* Set DMA identifier */ 370 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 371 if (ret) { 372 dev_err(&pdev->dev, "No usable DMA configuration.\n"); 373 goto out_err; 374 } 375 376 ret = adf_cfg_dev_init(accel_dev); 377 if (ret) { 378 dev_err(&pdev->dev, "Failed to initialize configuration.\n"); 379 goto out_err; 380 } 381 382 /* Get accelerator capabilities mask */ 383 hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); 384 if (!hw_data->accel_capabilities_mask) { 385 dev_err(&pdev->dev, "Failed to get capabilities mask.\n"); 386 ret = -EINVAL; 387 goto out_err; 388 } 389 390 /* Find and map all the device's BARS */ 391 bar_mask = pci_select_bars(pdev, IORESOURCE_MEM) & ADF_4XXX_BAR_MASK; 392 393 ret = pcim_iomap_regions_request_all(pdev, bar_mask, pci_name(pdev)); 394 if (ret) { 395 dev_err(&pdev->dev, "Failed to map pci regions.\n"); 396 goto out_err; 397 } 398 399 i = 0; 400 for_each_set_bit(bar_nr, &bar_mask, PCI_STD_NUM_BARS) { 401 bar = &accel_pci_dev->pci_bars[i++]; 402 bar->virt_addr = pcim_iomap_table(pdev)[bar_nr]; 403 } 404 405 pci_set_master(pdev); 406 407 if (pci_save_state(pdev)) { 408 dev_err(&pdev->dev, "Failed to save pci state.\n"); 409 ret = -ENOMEM; 410 goto out_err; 411 } 412 413 ret = adf_dev_up(accel_dev, true); 414 if (ret) 415 goto out_err_dev_stop; 416 417 ret = adf_sysfs_init(accel_dev); 418 if (ret) 419 goto out_err_dev_stop; 420 421 return ret; 422 423 out_err_dev_stop: 424 adf_dev_down(accel_dev, false); 425 out_err: 426 adf_cleanup_accel(accel_dev); 427 return ret; 428 } 429 430 static void adf_remove(struct pci_dev *pdev) 431 { 432 struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); 433 434 if (!accel_dev) { 435 pr_err("QAT: Driver removal failed\n"); 436 return; 437 } 438 adf_dev_down(accel_dev, false); 439 adf_cleanup_accel(accel_dev); 440 } 441 442 static struct pci_driver adf_driver = { 443 .id_table = adf_pci_tbl, 444 .name = ADF_4XXX_DEVICE_NAME, 445 .probe = adf_probe, 446 .remove = adf_remove, 447 .sriov_configure = adf_sriov_configure, 448 .err_handler = &adf_err_handler, 449 }; 450 451 module_pci_driver(adf_driver); 452 453 MODULE_LICENSE("Dual BSD/GPL"); 454 MODULE_AUTHOR("Intel"); 455 MODULE_FIRMWARE(ADF_4XXX_FW); 456 MODULE_FIRMWARE(ADF_4XXX_MMP); 457 MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); 458 MODULE_VERSION(ADF_DRV_VERSION); 459 MODULE_SOFTDEP("pre: crypto-intel_qat"); 460