1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3 #include <linux/init.h>
4 #include <linux/kernel.h>
5 #include <linux/module.h>
6 #include <linux/slab.h>
7 #include <linux/pci.h>
8 #include <linux/interrupt.h>
9 #include <linux/delay.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/workqueue.h>
12 #include <linux/fs.h>
13 #include <linux/io-64-nonatomic-lo-hi.h>
14 #include <linux/device.h>
15 #include <linux/idr.h>
16 #include <linux/iommu.h>
17 #include <uapi/linux/idxd.h>
18 #include <linux/dmaengine.h>
19 #include "../dmaengine.h"
20 #include "registers.h"
21 #include "idxd.h"
22 #include "perfmon.h"
23
24 MODULE_VERSION(IDXD_DRIVER_VERSION);
25 MODULE_LICENSE("GPL v2");
26 MODULE_AUTHOR("Intel Corporation");
27 MODULE_IMPORT_NS(IDXD);
28
29 static bool sva = true;
30 module_param(sva, bool, 0644);
31 MODULE_PARM_DESC(sva, "Toggle SVA support on/off");
32
33 bool tc_override;
34 module_param(tc_override, bool, 0644);
35 MODULE_PARM_DESC(tc_override, "Override traffic class defaults");
36
37 #define DRV_NAME "idxd"
38
39 bool support_enqcmd;
40 DEFINE_IDA(idxd_ida);
41
42 static struct idxd_driver_data idxd_driver_data[] = {
43 [IDXD_TYPE_DSA] = {
44 .name_prefix = "dsa",
45 .type = IDXD_TYPE_DSA,
46 .compl_size = sizeof(struct dsa_completion_record),
47 .align = 32,
48 .dev_type = &dsa_device_type,
49 .evl_cr_off = offsetof(struct dsa_evl_entry, cr),
50 .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */
51 .cr_status_off = offsetof(struct dsa_completion_record, status),
52 .cr_result_off = offsetof(struct dsa_completion_record, result),
53 },
54 [IDXD_TYPE_IAX] = {
55 .name_prefix = "iax",
56 .type = IDXD_TYPE_IAX,
57 .compl_size = sizeof(struct iax_completion_record),
58 .align = 64,
59 .dev_type = &iax_device_type,
60 .evl_cr_off = offsetof(struct iax_evl_entry, cr),
61 .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */
62 .cr_status_off = offsetof(struct iax_completion_record, status),
63 .cr_result_off = offsetof(struct iax_completion_record, error_code),
64 },
65 };
66
67 static struct pci_device_id idxd_pci_tbl[] = {
68 /* DSA ver 1.0 platforms */
69 { PCI_DEVICE_DATA(INTEL, DSA_SPR0, &idxd_driver_data[IDXD_TYPE_DSA]) },
70
71 /* IAX ver 1.0 platforms */
72 { PCI_DEVICE_DATA(INTEL, IAX_SPR0, &idxd_driver_data[IDXD_TYPE_IAX]) },
73 { 0, }
74 };
75 MODULE_DEVICE_TABLE(pci, idxd_pci_tbl);
76
idxd_setup_interrupts(struct idxd_device * idxd)77 static int idxd_setup_interrupts(struct idxd_device *idxd)
78 {
79 struct pci_dev *pdev = idxd->pdev;
80 struct device *dev = &pdev->dev;
81 struct idxd_irq_entry *ie;
82 int i, msixcnt;
83 int rc = 0;
84
85 msixcnt = pci_msix_vec_count(pdev);
86 if (msixcnt < 0) {
87 dev_err(dev, "Not MSI-X interrupt capable.\n");
88 return -ENOSPC;
89 }
90 idxd->irq_cnt = msixcnt;
91
92 rc = pci_alloc_irq_vectors(pdev, msixcnt, msixcnt, PCI_IRQ_MSIX);
93 if (rc != msixcnt) {
94 dev_err(dev, "Failed enabling %d MSIX entries: %d\n", msixcnt, rc);
95 return -ENOSPC;
96 }
97 dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt);
98
99
100 ie = idxd_get_ie(idxd, 0);
101 ie->vector = pci_irq_vector(pdev, 0);
102 rc = request_threaded_irq(ie->vector, NULL, idxd_misc_thread, 0, "idxd-misc", ie);
103 if (rc < 0) {
104 dev_err(dev, "Failed to allocate misc interrupt.\n");
105 goto err_misc_irq;
106 }
107 dev_dbg(dev, "Requested idxd-misc handler on msix vector %d\n", ie->vector);
108
109 for (i = 0; i < idxd->max_wqs; i++) {
110 int msix_idx = i + 1;
111
112 ie = idxd_get_ie(idxd, msix_idx);
113 ie->id = msix_idx;
114 ie->int_handle = INVALID_INT_HANDLE;
115 ie->pasid = IOMMU_PASID_INVALID;
116
117 spin_lock_init(&ie->list_lock);
118 init_llist_head(&ie->pending_llist);
119 INIT_LIST_HEAD(&ie->work_list);
120 }
121
122 idxd_unmask_error_interrupts(idxd);
123 return 0;
124
125 err_misc_irq:
126 idxd_mask_error_interrupts(idxd);
127 pci_free_irq_vectors(pdev);
128 dev_err(dev, "No usable interrupts\n");
129 return rc;
130 }
131
idxd_cleanup_interrupts(struct idxd_device * idxd)132 static void idxd_cleanup_interrupts(struct idxd_device *idxd)
133 {
134 struct pci_dev *pdev = idxd->pdev;
135 struct idxd_irq_entry *ie;
136 int msixcnt;
137
138 msixcnt = pci_msix_vec_count(pdev);
139 if (msixcnt <= 0)
140 return;
141
142 ie = idxd_get_ie(idxd, 0);
143 idxd_mask_error_interrupts(idxd);
144 free_irq(ie->vector, ie);
145 pci_free_irq_vectors(pdev);
146 }
147
idxd_setup_wqs(struct idxd_device * idxd)148 static int idxd_setup_wqs(struct idxd_device *idxd)
149 {
150 struct device *dev = &idxd->pdev->dev;
151 struct idxd_wq *wq;
152 struct device *conf_dev;
153 int i, rc;
154
155 idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *),
156 GFP_KERNEL, dev_to_node(dev));
157 if (!idxd->wqs)
158 return -ENOMEM;
159
160 idxd->wq_enable_map = bitmap_zalloc_node(idxd->max_wqs, GFP_KERNEL, dev_to_node(dev));
161 if (!idxd->wq_enable_map) {
162 kfree(idxd->wqs);
163 return -ENOMEM;
164 }
165
166 for (i = 0; i < idxd->max_wqs; i++) {
167 wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev));
168 if (!wq) {
169 rc = -ENOMEM;
170 goto err;
171 }
172
173 idxd_dev_set_type(&wq->idxd_dev, IDXD_DEV_WQ);
174 conf_dev = wq_confdev(wq);
175 wq->id = i;
176 wq->idxd = idxd;
177 device_initialize(wq_confdev(wq));
178 conf_dev->parent = idxd_confdev(idxd);
179 conf_dev->bus = &dsa_bus_type;
180 conf_dev->type = &idxd_wq_device_type;
181 rc = dev_set_name(conf_dev, "wq%d.%d", idxd->id, wq->id);
182 if (rc < 0) {
183 put_device(conf_dev);
184 goto err;
185 }
186
187 mutex_init(&wq->wq_lock);
188 init_waitqueue_head(&wq->err_queue);
189 init_completion(&wq->wq_dead);
190 init_completion(&wq->wq_resurrect);
191 wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER;
192 idxd_wq_set_max_batch_size(idxd->data->type, wq, WQ_DEFAULT_MAX_BATCH);
193 wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES;
194 wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev));
195 if (!wq->wqcfg) {
196 put_device(conf_dev);
197 rc = -ENOMEM;
198 goto err;
199 }
200
201 if (idxd->hw.wq_cap.op_config) {
202 wq->opcap_bmap = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL);
203 if (!wq->opcap_bmap) {
204 put_device(conf_dev);
205 rc = -ENOMEM;
206 goto err;
207 }
208 bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS);
209 }
210 mutex_init(&wq->uc_lock);
211 xa_init(&wq->upasid_xa);
212 idxd->wqs[i] = wq;
213 }
214
215 return 0;
216
217 err:
218 while (--i >= 0) {
219 wq = idxd->wqs[i];
220 conf_dev = wq_confdev(wq);
221 put_device(conf_dev);
222 }
223 return rc;
224 }
225
idxd_setup_engines(struct idxd_device * idxd)226 static int idxd_setup_engines(struct idxd_device *idxd)
227 {
228 struct idxd_engine *engine;
229 struct device *dev = &idxd->pdev->dev;
230 struct device *conf_dev;
231 int i, rc;
232
233 idxd->engines = kcalloc_node(idxd->max_engines, sizeof(struct idxd_engine *),
234 GFP_KERNEL, dev_to_node(dev));
235 if (!idxd->engines)
236 return -ENOMEM;
237
238 for (i = 0; i < idxd->max_engines; i++) {
239 engine = kzalloc_node(sizeof(*engine), GFP_KERNEL, dev_to_node(dev));
240 if (!engine) {
241 rc = -ENOMEM;
242 goto err;
243 }
244
245 idxd_dev_set_type(&engine->idxd_dev, IDXD_DEV_ENGINE);
246 conf_dev = engine_confdev(engine);
247 engine->id = i;
248 engine->idxd = idxd;
249 device_initialize(conf_dev);
250 conf_dev->parent = idxd_confdev(idxd);
251 conf_dev->bus = &dsa_bus_type;
252 conf_dev->type = &idxd_engine_device_type;
253 rc = dev_set_name(conf_dev, "engine%d.%d", idxd->id, engine->id);
254 if (rc < 0) {
255 put_device(conf_dev);
256 goto err;
257 }
258
259 idxd->engines[i] = engine;
260 }
261
262 return 0;
263
264 err:
265 while (--i >= 0) {
266 engine = idxd->engines[i];
267 conf_dev = engine_confdev(engine);
268 put_device(conf_dev);
269 }
270 return rc;
271 }
272
idxd_setup_groups(struct idxd_device * idxd)273 static int idxd_setup_groups(struct idxd_device *idxd)
274 {
275 struct device *dev = &idxd->pdev->dev;
276 struct device *conf_dev;
277 struct idxd_group *group;
278 int i, rc;
279
280 idxd->groups = kcalloc_node(idxd->max_groups, sizeof(struct idxd_group *),
281 GFP_KERNEL, dev_to_node(dev));
282 if (!idxd->groups)
283 return -ENOMEM;
284
285 for (i = 0; i < idxd->max_groups; i++) {
286 group = kzalloc_node(sizeof(*group), GFP_KERNEL, dev_to_node(dev));
287 if (!group) {
288 rc = -ENOMEM;
289 goto err;
290 }
291
292 idxd_dev_set_type(&group->idxd_dev, IDXD_DEV_GROUP);
293 conf_dev = group_confdev(group);
294 group->id = i;
295 group->idxd = idxd;
296 device_initialize(conf_dev);
297 conf_dev->parent = idxd_confdev(idxd);
298 conf_dev->bus = &dsa_bus_type;
299 conf_dev->type = &idxd_group_device_type;
300 rc = dev_set_name(conf_dev, "group%d.%d", idxd->id, group->id);
301 if (rc < 0) {
302 put_device(conf_dev);
303 goto err;
304 }
305
306 idxd->groups[i] = group;
307 if (idxd->hw.version <= DEVICE_VERSION_2 && !tc_override) {
308 group->tc_a = 1;
309 group->tc_b = 1;
310 } else {
311 group->tc_a = -1;
312 group->tc_b = -1;
313 }
314 /*
315 * The default value is the same as the value of
316 * total read buffers in GRPCAP.
317 */
318 group->rdbufs_allowed = idxd->max_rdbufs;
319 }
320
321 return 0;
322
323 err:
324 while (--i >= 0) {
325 group = idxd->groups[i];
326 put_device(group_confdev(group));
327 }
328 return rc;
329 }
330
idxd_cleanup_internals(struct idxd_device * idxd)331 static void idxd_cleanup_internals(struct idxd_device *idxd)
332 {
333 int i;
334
335 for (i = 0; i < idxd->max_groups; i++)
336 put_device(group_confdev(idxd->groups[i]));
337 for (i = 0; i < idxd->max_engines; i++)
338 put_device(engine_confdev(idxd->engines[i]));
339 for (i = 0; i < idxd->max_wqs; i++)
340 put_device(wq_confdev(idxd->wqs[i]));
341 destroy_workqueue(idxd->wq);
342 }
343
idxd_init_evl(struct idxd_device * idxd)344 static int idxd_init_evl(struct idxd_device *idxd)
345 {
346 struct device *dev = &idxd->pdev->dev;
347 unsigned int evl_cache_size;
348 struct idxd_evl *evl;
349 const char *idxd_name;
350
351 if (idxd->hw.gen_cap.evl_support == 0)
352 return 0;
353
354 evl = kzalloc_node(sizeof(*evl), GFP_KERNEL, dev_to_node(dev));
355 if (!evl)
356 return -ENOMEM;
357
358 mutex_init(&evl->lock);
359 evl->size = IDXD_EVL_SIZE_MIN;
360
361 idxd_name = dev_name(idxd_confdev(idxd));
362 evl_cache_size = sizeof(struct idxd_evl_fault) + evl_ent_size(idxd);
363 /*
364 * Since completion record in evl_cache will be copied to user
365 * when handling completion record page fault, need to create
366 * the cache suitable for user copy.
367 */
368 idxd->evl_cache = kmem_cache_create_usercopy(idxd_name, evl_cache_size,
369 0, 0, 0, evl_cache_size,
370 NULL);
371 if (!idxd->evl_cache) {
372 kfree(evl);
373 return -ENOMEM;
374 }
375
376 idxd->evl = evl;
377 return 0;
378 }
379
idxd_setup_internals(struct idxd_device * idxd)380 static int idxd_setup_internals(struct idxd_device *idxd)
381 {
382 struct device *dev = &idxd->pdev->dev;
383 int rc, i;
384
385 init_waitqueue_head(&idxd->cmd_waitq);
386
387 rc = idxd_setup_wqs(idxd);
388 if (rc < 0)
389 goto err_wqs;
390
391 rc = idxd_setup_engines(idxd);
392 if (rc < 0)
393 goto err_engine;
394
395 rc = idxd_setup_groups(idxd);
396 if (rc < 0)
397 goto err_group;
398
399 idxd->wq = create_workqueue(dev_name(dev));
400 if (!idxd->wq) {
401 rc = -ENOMEM;
402 goto err_wkq_create;
403 }
404
405 rc = idxd_init_evl(idxd);
406 if (rc < 0)
407 goto err_evl;
408
409 return 0;
410
411 err_evl:
412 destroy_workqueue(idxd->wq);
413 err_wkq_create:
414 for (i = 0; i < idxd->max_groups; i++)
415 put_device(group_confdev(idxd->groups[i]));
416 err_group:
417 for (i = 0; i < idxd->max_engines; i++)
418 put_device(engine_confdev(idxd->engines[i]));
419 err_engine:
420 for (i = 0; i < idxd->max_wqs; i++)
421 put_device(wq_confdev(idxd->wqs[i]));
422 err_wqs:
423 return rc;
424 }
425
idxd_read_table_offsets(struct idxd_device * idxd)426 static void idxd_read_table_offsets(struct idxd_device *idxd)
427 {
428 union offsets_reg offsets;
429 struct device *dev = &idxd->pdev->dev;
430
431 offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET);
432 offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64));
433 idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT;
434 dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset);
435 idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT;
436 dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset);
437 idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT;
438 dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset);
439 idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT;
440 dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset);
441 }
442
multi_u64_to_bmap(unsigned long * bmap,u64 * val,int count)443 void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count)
444 {
445 int i, j, nr;
446
447 for (i = 0, nr = 0; i < count; i++) {
448 for (j = 0; j < BITS_PER_LONG_LONG; j++) {
449 if (val[i] & BIT(j))
450 set_bit(nr, bmap);
451 nr++;
452 }
453 }
454 }
455
idxd_read_caps(struct idxd_device * idxd)456 static void idxd_read_caps(struct idxd_device *idxd)
457 {
458 struct device *dev = &idxd->pdev->dev;
459 int i;
460
461 /* reading generic capabilities */
462 idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET);
463 dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits);
464
465 if (idxd->hw.gen_cap.cmd_cap) {
466 idxd->hw.cmd_cap = ioread32(idxd->reg_base + IDXD_CMDCAP_OFFSET);
467 dev_dbg(dev, "cmd_cap: %#x\n", idxd->hw.cmd_cap);
468 }
469
470 /* reading command capabilities */
471 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE))
472 idxd->request_int_handles = true;
473
474 idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift;
475 dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes);
476 idxd_set_max_batch_size(idxd->data->type, idxd, 1U << idxd->hw.gen_cap.max_batch_shift);
477 dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size);
478 if (idxd->hw.gen_cap.config_en)
479 set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags);
480
481 /* reading group capabilities */
482 idxd->hw.group_cap.bits =
483 ioread64(idxd->reg_base + IDXD_GRPCAP_OFFSET);
484 dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits);
485 idxd->max_groups = idxd->hw.group_cap.num_groups;
486 dev_dbg(dev, "max groups: %u\n", idxd->max_groups);
487 idxd->max_rdbufs = idxd->hw.group_cap.total_rdbufs;
488 dev_dbg(dev, "max read buffers: %u\n", idxd->max_rdbufs);
489 idxd->nr_rdbufs = idxd->max_rdbufs;
490
491 /* read engine capabilities */
492 idxd->hw.engine_cap.bits =
493 ioread64(idxd->reg_base + IDXD_ENGCAP_OFFSET);
494 dev_dbg(dev, "engine_cap: %#llx\n", idxd->hw.engine_cap.bits);
495 idxd->max_engines = idxd->hw.engine_cap.num_engines;
496 dev_dbg(dev, "max engines: %u\n", idxd->max_engines);
497
498 /* read workqueue capabilities */
499 idxd->hw.wq_cap.bits = ioread64(idxd->reg_base + IDXD_WQCAP_OFFSET);
500 dev_dbg(dev, "wq_cap: %#llx\n", idxd->hw.wq_cap.bits);
501 idxd->max_wq_size = idxd->hw.wq_cap.total_wq_size;
502 dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size);
503 idxd->max_wqs = idxd->hw.wq_cap.num_wqs;
504 dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs);
505 idxd->wqcfg_size = 1 << (idxd->hw.wq_cap.wqcfg_size + IDXD_WQCFG_MIN);
506 dev_dbg(dev, "wqcfg size: %u\n", idxd->wqcfg_size);
507
508 /* reading operation capabilities */
509 for (i = 0; i < 4; i++) {
510 idxd->hw.opcap.bits[i] = ioread64(idxd->reg_base +
511 IDXD_OPCAP_OFFSET + i * sizeof(u64));
512 dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]);
513 }
514 multi_u64_to_bmap(idxd->opcap_bmap, &idxd->hw.opcap.bits[0], 4);
515
516 /* read iaa cap */
517 if (idxd->data->type == IDXD_TYPE_IAX && idxd->hw.version >= DEVICE_VERSION_2)
518 idxd->hw.iaa_cap.bits = ioread64(idxd->reg_base + IDXD_IAACAP_OFFSET);
519 }
520
idxd_alloc(struct pci_dev * pdev,struct idxd_driver_data * data)521 static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data)
522 {
523 struct device *dev = &pdev->dev;
524 struct device *conf_dev;
525 struct idxd_device *idxd;
526 int rc;
527
528 idxd = kzalloc_node(sizeof(*idxd), GFP_KERNEL, dev_to_node(dev));
529 if (!idxd)
530 return NULL;
531
532 conf_dev = idxd_confdev(idxd);
533 idxd->pdev = pdev;
534 idxd->data = data;
535 idxd_dev_set_type(&idxd->idxd_dev, idxd->data->type);
536 idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL);
537 if (idxd->id < 0)
538 return NULL;
539
540 idxd->opcap_bmap = bitmap_zalloc_node(IDXD_MAX_OPCAP_BITS, GFP_KERNEL, dev_to_node(dev));
541 if (!idxd->opcap_bmap) {
542 ida_free(&idxd_ida, idxd->id);
543 return NULL;
544 }
545
546 device_initialize(conf_dev);
547 conf_dev->parent = dev;
548 conf_dev->bus = &dsa_bus_type;
549 conf_dev->type = idxd->data->dev_type;
550 rc = dev_set_name(conf_dev, "%s%d", idxd->data->name_prefix, idxd->id);
551 if (rc < 0) {
552 put_device(conf_dev);
553 return NULL;
554 }
555
556 spin_lock_init(&idxd->dev_lock);
557 spin_lock_init(&idxd->cmd_lock);
558
559 return idxd;
560 }
561
idxd_enable_system_pasid(struct idxd_device * idxd)562 static int idxd_enable_system_pasid(struct idxd_device *idxd)
563 {
564 struct pci_dev *pdev = idxd->pdev;
565 struct device *dev = &pdev->dev;
566 struct iommu_domain *domain;
567 ioasid_t pasid;
568 int ret;
569
570 /*
571 * Attach a global PASID to the DMA domain so that we can use ENQCMDS
572 * to submit work on buffers mapped by DMA API.
573 */
574 domain = iommu_get_domain_for_dev(dev);
575 if (!domain)
576 return -EPERM;
577
578 pasid = iommu_alloc_global_pasid(dev);
579 if (pasid == IOMMU_PASID_INVALID)
580 return -ENOSPC;
581
582 /*
583 * DMA domain is owned by the driver, it should support all valid
584 * types such as DMA-FQ, identity, etc.
585 */
586 ret = iommu_attach_device_pasid(domain, dev, pasid);
587 if (ret) {
588 dev_err(dev, "failed to attach device pasid %d, domain type %d",
589 pasid, domain->type);
590 iommu_free_global_pasid(pasid);
591 return ret;
592 }
593
594 /* Since we set user privilege for kernel DMA, enable completion IRQ */
595 idxd_set_user_intr(idxd, 1);
596 idxd->pasid = pasid;
597
598 return ret;
599 }
600
idxd_disable_system_pasid(struct idxd_device * idxd)601 static void idxd_disable_system_pasid(struct idxd_device *idxd)
602 {
603 struct pci_dev *pdev = idxd->pdev;
604 struct device *dev = &pdev->dev;
605 struct iommu_domain *domain;
606
607 domain = iommu_get_domain_for_dev(dev);
608 if (!domain)
609 return;
610
611 iommu_detach_device_pasid(domain, dev, idxd->pasid);
612 iommu_free_global_pasid(idxd->pasid);
613
614 idxd_set_user_intr(idxd, 0);
615 idxd->sva = NULL;
616 idxd->pasid = IOMMU_PASID_INVALID;
617 }
618
idxd_enable_sva(struct pci_dev * pdev)619 static int idxd_enable_sva(struct pci_dev *pdev)
620 {
621 int ret;
622
623 ret = iommu_dev_enable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF);
624 if (ret)
625 return ret;
626
627 ret = iommu_dev_enable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA);
628 if (ret)
629 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF);
630
631 return ret;
632 }
633
idxd_disable_sva(struct pci_dev * pdev)634 static void idxd_disable_sva(struct pci_dev *pdev)
635 {
636 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA);
637 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF);
638 }
639
idxd_probe(struct idxd_device * idxd)640 static int idxd_probe(struct idxd_device *idxd)
641 {
642 struct pci_dev *pdev = idxd->pdev;
643 struct device *dev = &pdev->dev;
644 int rc;
645
646 dev_dbg(dev, "%s entered and resetting device\n", __func__);
647 rc = idxd_device_init_reset(idxd);
648 if (rc < 0)
649 return rc;
650
651 dev_dbg(dev, "IDXD reset complete\n");
652
653 if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) {
654 if (idxd_enable_sva(pdev)) {
655 dev_warn(dev, "Unable to turn on user SVA feature.\n");
656 } else {
657 set_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags);
658
659 rc = idxd_enable_system_pasid(idxd);
660 if (rc)
661 dev_warn(dev, "No in-kernel DMA with PASID. %d\n", rc);
662 else
663 set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
664 }
665 } else if (!sva) {
666 dev_warn(dev, "User forced SVA off via module param.\n");
667 }
668
669 idxd_read_caps(idxd);
670 idxd_read_table_offsets(idxd);
671
672 rc = idxd_setup_internals(idxd);
673 if (rc)
674 goto err;
675
676 /* If the configs are readonly, then load them from device */
677 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) {
678 dev_dbg(dev, "Loading RO device config\n");
679 rc = idxd_device_load_config(idxd);
680 if (rc < 0)
681 goto err_config;
682 }
683
684 rc = idxd_setup_interrupts(idxd);
685 if (rc)
686 goto err_config;
687
688 idxd->major = idxd_cdev_get_major(idxd);
689
690 rc = perfmon_pmu_init(idxd);
691 if (rc < 0)
692 dev_warn(dev, "Failed to initialize perfmon. No PMU support: %d\n", rc);
693
694 dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id);
695 return 0;
696
697 err_config:
698 idxd_cleanup_internals(idxd);
699 err:
700 if (device_pasid_enabled(idxd))
701 idxd_disable_system_pasid(idxd);
702 if (device_user_pasid_enabled(idxd))
703 idxd_disable_sva(pdev);
704 return rc;
705 }
706
idxd_cleanup(struct idxd_device * idxd)707 static void idxd_cleanup(struct idxd_device *idxd)
708 {
709 perfmon_pmu_remove(idxd);
710 idxd_cleanup_interrupts(idxd);
711 idxd_cleanup_internals(idxd);
712 if (device_pasid_enabled(idxd))
713 idxd_disable_system_pasid(idxd);
714 if (device_user_pasid_enabled(idxd))
715 idxd_disable_sva(idxd->pdev);
716 }
717
idxd_pci_probe(struct pci_dev * pdev,const struct pci_device_id * id)718 static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
719 {
720 struct device *dev = &pdev->dev;
721 struct idxd_device *idxd;
722 struct idxd_driver_data *data = (struct idxd_driver_data *)id->driver_data;
723 int rc;
724
725 rc = pci_enable_device(pdev);
726 if (rc)
727 return rc;
728
729 dev_dbg(dev, "Alloc IDXD context\n");
730 idxd = idxd_alloc(pdev, data);
731 if (!idxd) {
732 rc = -ENOMEM;
733 goto err_idxd_alloc;
734 }
735
736 dev_dbg(dev, "Mapping BARs\n");
737 idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0);
738 if (!idxd->reg_base) {
739 rc = -ENOMEM;
740 goto err_iomap;
741 }
742
743 dev_dbg(dev, "Set DMA masks\n");
744 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
745 if (rc)
746 goto err;
747
748 dev_dbg(dev, "Set PCI master\n");
749 pci_set_master(pdev);
750 pci_set_drvdata(pdev, idxd);
751
752 idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET);
753 rc = idxd_probe(idxd);
754 if (rc) {
755 dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n");
756 goto err;
757 }
758
759 rc = idxd_register_devices(idxd);
760 if (rc) {
761 dev_err(dev, "IDXD sysfs setup failed\n");
762 goto err_dev_register;
763 }
764
765 rc = idxd_device_init_debugfs(idxd);
766 if (rc)
767 dev_warn(dev, "IDXD debugfs failed to setup\n");
768
769 dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n",
770 idxd->hw.version);
771
772 idxd->user_submission_safe = data->user_submission_safe;
773
774 return 0;
775
776 err_dev_register:
777 idxd_cleanup(idxd);
778 err:
779 pci_iounmap(pdev, idxd->reg_base);
780 err_iomap:
781 put_device(idxd_confdev(idxd));
782 err_idxd_alloc:
783 pci_disable_device(pdev);
784 return rc;
785 }
786
idxd_wqs_quiesce(struct idxd_device * idxd)787 void idxd_wqs_quiesce(struct idxd_device *idxd)
788 {
789 struct idxd_wq *wq;
790 int i;
791
792 for (i = 0; i < idxd->max_wqs; i++) {
793 wq = idxd->wqs[i];
794 if (wq->state == IDXD_WQ_ENABLED && wq->type == IDXD_WQT_KERNEL)
795 idxd_wq_quiesce(wq);
796 }
797 }
798
idxd_shutdown(struct pci_dev * pdev)799 static void idxd_shutdown(struct pci_dev *pdev)
800 {
801 struct idxd_device *idxd = pci_get_drvdata(pdev);
802 struct idxd_irq_entry *irq_entry;
803 int rc;
804
805 rc = idxd_device_disable(idxd);
806 if (rc)
807 dev_err(&pdev->dev, "Disabling device failed\n");
808
809 irq_entry = &idxd->ie;
810 synchronize_irq(irq_entry->vector);
811 idxd_mask_error_interrupts(idxd);
812 flush_workqueue(idxd->wq);
813 }
814
idxd_remove(struct pci_dev * pdev)815 static void idxd_remove(struct pci_dev *pdev)
816 {
817 struct idxd_device *idxd = pci_get_drvdata(pdev);
818 struct idxd_irq_entry *irq_entry;
819
820 idxd_unregister_devices(idxd);
821 /*
822 * When ->release() is called for the idxd->conf_dev, it frees all the memory related
823 * to the idxd context. The driver still needs those bits in order to do the rest of
824 * the cleanup. However, we do need to unbound the idxd sub-driver. So take a ref
825 * on the device here to hold off the freeing while allowing the idxd sub-driver
826 * to unbind.
827 */
828 get_device(idxd_confdev(idxd));
829 device_unregister(idxd_confdev(idxd));
830 idxd_shutdown(pdev);
831 if (device_pasid_enabled(idxd))
832 idxd_disable_system_pasid(idxd);
833 idxd_device_remove_debugfs(idxd);
834
835 irq_entry = idxd_get_ie(idxd, 0);
836 free_irq(irq_entry->vector, irq_entry);
837 pci_free_irq_vectors(pdev);
838 pci_iounmap(pdev, idxd->reg_base);
839 if (device_user_pasid_enabled(idxd))
840 idxd_disable_sva(pdev);
841 pci_disable_device(pdev);
842 destroy_workqueue(idxd->wq);
843 perfmon_pmu_remove(idxd);
844 put_device(idxd_confdev(idxd));
845 }
846
847 static struct pci_driver idxd_pci_driver = {
848 .name = DRV_NAME,
849 .id_table = idxd_pci_tbl,
850 .probe = idxd_pci_probe,
851 .remove = idxd_remove,
852 .shutdown = idxd_shutdown,
853 };
854
idxd_init_module(void)855 static int __init idxd_init_module(void)
856 {
857 int err;
858
859 /*
860 * If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in
861 * enumerating the device. We can not utilize it.
862 */
863 if (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) {
864 pr_warn("idxd driver failed to load without MOVDIR64B.\n");
865 return -ENODEV;
866 }
867
868 if (!cpu_feature_enabled(X86_FEATURE_ENQCMD))
869 pr_warn("Platform does not have ENQCMD(S) support.\n");
870 else
871 support_enqcmd = true;
872
873 perfmon_init();
874
875 err = idxd_driver_register(&idxd_drv);
876 if (err < 0)
877 goto err_idxd_driver_register;
878
879 err = idxd_driver_register(&idxd_dmaengine_drv);
880 if (err < 0)
881 goto err_idxd_dmaengine_driver_register;
882
883 err = idxd_driver_register(&idxd_user_drv);
884 if (err < 0)
885 goto err_idxd_user_driver_register;
886
887 err = idxd_cdev_register();
888 if (err)
889 goto err_cdev_register;
890
891 err = idxd_init_debugfs();
892 if (err)
893 goto err_debugfs;
894
895 err = pci_register_driver(&idxd_pci_driver);
896 if (err)
897 goto err_pci_register;
898
899 return 0;
900
901 err_pci_register:
902 idxd_remove_debugfs();
903 err_debugfs:
904 idxd_cdev_remove();
905 err_cdev_register:
906 idxd_driver_unregister(&idxd_user_drv);
907 err_idxd_user_driver_register:
908 idxd_driver_unregister(&idxd_dmaengine_drv);
909 err_idxd_dmaengine_driver_register:
910 idxd_driver_unregister(&idxd_drv);
911 err_idxd_driver_register:
912 return err;
913 }
914 module_init(idxd_init_module);
915
idxd_exit_module(void)916 static void __exit idxd_exit_module(void)
917 {
918 idxd_driver_unregister(&idxd_user_drv);
919 idxd_driver_unregister(&idxd_dmaengine_drv);
920 idxd_driver_unregister(&idxd_drv);
921 pci_unregister_driver(&idxd_pci_driver);
922 idxd_cdev_remove();
923 perfmon_exit();
924 idxd_remove_debugfs();
925 }
926 module_exit(idxd_exit_module);
927