1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Intel(R) Trace Hub driver core
4  *
5  * Copyright (C) 2014-2015 Intel Corporation.
6  */
7 
8 #define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
9 
10 #include <linux/types.h>
11 #include <linux/module.h>
12 #include <linux/device.h>
13 #include <linux/sysfs.h>
14 #include <linux/kdev_t.h>
15 #include <linux/debugfs.h>
16 #include <linux/idr.h>
17 #include <linux/pci.h>
18 #include <linux/pm_runtime.h>
19 #include <linux/dma-mapping.h>
20 
21 #include "intel_th.h"
22 #include "debug.h"
23 
24 static bool host_mode __read_mostly;
25 module_param(host_mode, bool, 0444);
26 
27 static DEFINE_IDA(intel_th_ida);
28 
29 static int intel_th_match(struct device *dev, struct device_driver *driver)
30 {
31 	struct intel_th_driver *thdrv = to_intel_th_driver(driver);
32 	struct intel_th_device *thdev = to_intel_th_device(dev);
33 
34 	if (thdev->type == INTEL_TH_SWITCH &&
35 	    (!thdrv->enable || !thdrv->disable))
36 		return 0;
37 
38 	return !strcmp(thdev->name, driver->name);
39 }
40 
41 static int intel_th_child_remove(struct device *dev, void *data)
42 {
43 	device_release_driver(dev);
44 
45 	return 0;
46 }
47 
48 static int intel_th_probe(struct device *dev)
49 {
50 	struct intel_th_driver *thdrv = to_intel_th_driver(dev->driver);
51 	struct intel_th_device *thdev = to_intel_th_device(dev);
52 	struct intel_th_driver *hubdrv;
53 	struct intel_th_device *hub = NULL;
54 	int ret;
55 
56 	if (thdev->type == INTEL_TH_SWITCH)
57 		hub = thdev;
58 	else if (dev->parent)
59 		hub = to_intel_th_device(dev->parent);
60 
61 	if (!hub || !hub->dev.driver)
62 		return -EPROBE_DEFER;
63 
64 	hubdrv = to_intel_th_driver(hub->dev.driver);
65 
66 	pm_runtime_set_active(dev);
67 	pm_runtime_no_callbacks(dev);
68 	pm_runtime_enable(dev);
69 
70 	ret = thdrv->probe(to_intel_th_device(dev));
71 	if (ret)
72 		goto out_pm;
73 
74 	if (thdrv->attr_group) {
75 		ret = sysfs_create_group(&thdev->dev.kobj, thdrv->attr_group);
76 		if (ret)
77 			goto out;
78 	}
79 
80 	if (thdev->type == INTEL_TH_OUTPUT &&
81 	    !intel_th_output_assigned(thdev))
82 		/* does not talk to hardware */
83 		ret = hubdrv->assign(hub, thdev);
84 
85 out:
86 	if (ret)
87 		thdrv->remove(thdev);
88 
89 out_pm:
90 	if (ret)
91 		pm_runtime_disable(dev);
92 
93 	return ret;
94 }
95 
96 static void intel_th_device_remove(struct intel_th_device *thdev);
97 
98 static void intel_th_remove(struct device *dev)
99 {
100 	struct intel_th_driver *thdrv = to_intel_th_driver(dev->driver);
101 	struct intel_th_device *thdev = to_intel_th_device(dev);
102 	struct intel_th_device *hub = to_intel_th_hub(thdev);
103 
104 	if (thdev->type == INTEL_TH_SWITCH) {
105 		struct intel_th *th = to_intel_th(hub);
106 		int i, lowest;
107 
108 		/*
109 		 * disconnect outputs
110 		 *
111 		 * intel_th_child_remove returns 0 unconditionally, so there is
112 		 * no need to check the return value of device_for_each_child.
113 		 */
114 		device_for_each_child(dev, thdev, intel_th_child_remove);
115 
116 		/*
117 		 * Remove outputs, that is, hub's children: they are created
118 		 * at hub's probe time by having the hub call
119 		 * intel_th_output_enable() for each of them.
120 		 */
121 		for (i = 0, lowest = -1; i < th->num_thdevs; i++) {
122 			/*
123 			 * Move the non-output devices from higher up the
124 			 * th->thdev[] array to lower positions to maintain
125 			 * a contiguous array.
126 			 */
127 			if (th->thdev[i]->type != INTEL_TH_OUTPUT) {
128 				if (lowest >= 0) {
129 					th->thdev[lowest] = th->thdev[i];
130 					th->thdev[i] = NULL;
131 					++lowest;
132 				}
133 
134 				continue;
135 			}
136 
137 			if (lowest == -1)
138 				lowest = i;
139 
140 			intel_th_device_remove(th->thdev[i]);
141 			th->thdev[i] = NULL;
142 		}
143 
144 		if (lowest >= 0)
145 			th->num_thdevs = lowest;
146 	}
147 
148 	if (thdrv->attr_group)
149 		sysfs_remove_group(&thdev->dev.kobj, thdrv->attr_group);
150 
151 	pm_runtime_get_sync(dev);
152 
153 	thdrv->remove(thdev);
154 
155 	if (intel_th_output_assigned(thdev)) {
156 		struct intel_th_driver *hubdrv =
157 			to_intel_th_driver(dev->parent->driver);
158 
159 		if (hub->dev.driver)
160 			/* does not talk to hardware */
161 			hubdrv->unassign(hub, thdev);
162 	}
163 
164 	pm_runtime_disable(dev);
165 	pm_runtime_set_active(dev);
166 	pm_runtime_enable(dev);
167 }
168 
169 static struct bus_type intel_th_bus = {
170 	.name		= "intel_th",
171 	.match		= intel_th_match,
172 	.probe		= intel_th_probe,
173 	.remove		= intel_th_remove,
174 };
175 
176 static void intel_th_device_free(struct intel_th_device *thdev);
177 
178 static void intel_th_device_release(struct device *dev)
179 {
180 	intel_th_device_free(to_intel_th_device(dev));
181 }
182 
183 static struct device_type intel_th_source_device_type = {
184 	.name		= "intel_th_source_device",
185 	.release	= intel_th_device_release,
186 };
187 
188 static char *intel_th_output_devnode(struct device *dev, umode_t *mode,
189 				     kuid_t *uid, kgid_t *gid)
190 {
191 	struct intel_th_device *thdev = to_intel_th_device(dev);
192 	struct intel_th *th = to_intel_th(thdev);
193 	char *node;
194 
195 	if (thdev->id >= 0)
196 		node = kasprintf(GFP_KERNEL, "intel_th%d/%s%d", th->id,
197 				 thdev->name, thdev->id);
198 	else
199 		node = kasprintf(GFP_KERNEL, "intel_th%d/%s", th->id,
200 				 thdev->name);
201 
202 	return node;
203 }
204 
205 static ssize_t port_show(struct device *dev, struct device_attribute *attr,
206 			 char *buf)
207 {
208 	struct intel_th_device *thdev = to_intel_th_device(dev);
209 
210 	if (thdev->output.port >= 0)
211 		return scnprintf(buf, PAGE_SIZE, "%u\n", thdev->output.port);
212 
213 	return scnprintf(buf, PAGE_SIZE, "unassigned\n");
214 }
215 
216 static DEVICE_ATTR_RO(port);
217 
218 static void intel_th_trace_prepare(struct intel_th_device *thdev)
219 {
220 	struct intel_th_device *hub = to_intel_th_hub(thdev);
221 	struct intel_th_driver *hubdrv = to_intel_th_driver(hub->dev.driver);
222 
223 	if (hub->type != INTEL_TH_SWITCH)
224 		return;
225 
226 	if (thdev->type != INTEL_TH_OUTPUT)
227 		return;
228 
229 	pm_runtime_get_sync(&thdev->dev);
230 	hubdrv->prepare(hub, &thdev->output);
231 	pm_runtime_put(&thdev->dev);
232 }
233 
234 static int intel_th_output_activate(struct intel_th_device *thdev)
235 {
236 	struct intel_th_driver *thdrv =
237 		to_intel_th_driver_or_null(thdev->dev.driver);
238 	struct intel_th *th = to_intel_th(thdev);
239 	int ret = 0;
240 
241 	if (!thdrv)
242 		return -ENODEV;
243 
244 	if (!try_module_get(thdrv->driver.owner))
245 		return -ENODEV;
246 
247 	pm_runtime_get_sync(&thdev->dev);
248 
249 	if (th->activate)
250 		ret = th->activate(th);
251 	if (ret)
252 		goto fail_put;
253 
254 	intel_th_trace_prepare(thdev);
255 	if (thdrv->activate)
256 		ret = thdrv->activate(thdev);
257 	else
258 		intel_th_trace_enable(thdev);
259 
260 	if (ret)
261 		goto fail_deactivate;
262 
263 	return 0;
264 
265 fail_deactivate:
266 	if (th->deactivate)
267 		th->deactivate(th);
268 
269 fail_put:
270 	pm_runtime_put(&thdev->dev);
271 	module_put(thdrv->driver.owner);
272 
273 	return ret;
274 }
275 
276 static void intel_th_output_deactivate(struct intel_th_device *thdev)
277 {
278 	struct intel_th_driver *thdrv =
279 		to_intel_th_driver_or_null(thdev->dev.driver);
280 	struct intel_th *th = to_intel_th(thdev);
281 
282 	if (!thdrv)
283 		return;
284 
285 	if (thdrv->deactivate)
286 		thdrv->deactivate(thdev);
287 	else
288 		intel_th_trace_disable(thdev);
289 
290 	if (th->deactivate)
291 		th->deactivate(th);
292 
293 	pm_runtime_put(&thdev->dev);
294 	module_put(thdrv->driver.owner);
295 }
296 
297 static ssize_t active_show(struct device *dev, struct device_attribute *attr,
298 			   char *buf)
299 {
300 	struct intel_th_device *thdev = to_intel_th_device(dev);
301 
302 	return scnprintf(buf, PAGE_SIZE, "%d\n", thdev->output.active);
303 }
304 
305 static ssize_t active_store(struct device *dev, struct device_attribute *attr,
306 			    const char *buf, size_t size)
307 {
308 	struct intel_th_device *thdev = to_intel_th_device(dev);
309 	unsigned long val;
310 	int ret;
311 
312 	ret = kstrtoul(buf, 10, &val);
313 	if (ret)
314 		return ret;
315 
316 	if (!!val != thdev->output.active) {
317 		if (val)
318 			ret = intel_th_output_activate(thdev);
319 		else
320 			intel_th_output_deactivate(thdev);
321 	}
322 
323 	return ret ? ret : size;
324 }
325 
326 static DEVICE_ATTR_RW(active);
327 
328 static struct attribute *intel_th_output_attrs[] = {
329 	&dev_attr_port.attr,
330 	&dev_attr_active.attr,
331 	NULL,
332 };
333 
334 ATTRIBUTE_GROUPS(intel_th_output);
335 
336 static struct device_type intel_th_output_device_type = {
337 	.name		= "intel_th_output_device",
338 	.groups		= intel_th_output_groups,
339 	.release	= intel_th_device_release,
340 	.devnode	= intel_th_output_devnode,
341 };
342 
343 static struct device_type intel_th_switch_device_type = {
344 	.name		= "intel_th_switch_device",
345 	.release	= intel_th_device_release,
346 };
347 
348 static struct device_type *intel_th_device_type[] = {
349 	[INTEL_TH_SOURCE]	= &intel_th_source_device_type,
350 	[INTEL_TH_OUTPUT]	= &intel_th_output_device_type,
351 	[INTEL_TH_SWITCH]	= &intel_th_switch_device_type,
352 };
353 
354 int intel_th_driver_register(struct intel_th_driver *thdrv)
355 {
356 	if (!thdrv->probe || !thdrv->remove)
357 		return -EINVAL;
358 
359 	thdrv->driver.bus = &intel_th_bus;
360 
361 	return driver_register(&thdrv->driver);
362 }
363 EXPORT_SYMBOL_GPL(intel_th_driver_register);
364 
365 void intel_th_driver_unregister(struct intel_th_driver *thdrv)
366 {
367 	driver_unregister(&thdrv->driver);
368 }
369 EXPORT_SYMBOL_GPL(intel_th_driver_unregister);
370 
371 static struct intel_th_device *
372 intel_th_device_alloc(struct intel_th *th, unsigned int type, const char *name,
373 		      int id)
374 {
375 	struct device *parent;
376 	struct intel_th_device *thdev;
377 
378 	if (type == INTEL_TH_OUTPUT)
379 		parent = &th->hub->dev;
380 	else
381 		parent = th->dev;
382 
383 	thdev = kzalloc(sizeof(*thdev) + strlen(name) + 1, GFP_KERNEL);
384 	if (!thdev)
385 		return NULL;
386 
387 	thdev->id = id;
388 	thdev->type = type;
389 
390 	strcpy(thdev->name, name);
391 	device_initialize(&thdev->dev);
392 	thdev->dev.bus = &intel_th_bus;
393 	thdev->dev.type = intel_th_device_type[type];
394 	thdev->dev.parent = parent;
395 	thdev->dev.dma_mask = parent->dma_mask;
396 	thdev->dev.dma_parms = parent->dma_parms;
397 	dma_set_coherent_mask(&thdev->dev, parent->coherent_dma_mask);
398 	if (id >= 0)
399 		dev_set_name(&thdev->dev, "%d-%s%d", th->id, name, id);
400 	else
401 		dev_set_name(&thdev->dev, "%d-%s", th->id, name);
402 
403 	return thdev;
404 }
405 
406 static int intel_th_device_add_resources(struct intel_th_device *thdev,
407 					 struct resource *res, int nres)
408 {
409 	struct resource *r;
410 
411 	r = kmemdup(res, sizeof(*res) * nres, GFP_KERNEL);
412 	if (!r)
413 		return -ENOMEM;
414 
415 	thdev->resource = r;
416 	thdev->num_resources = nres;
417 
418 	return 0;
419 }
420 
421 static void intel_th_device_remove(struct intel_th_device *thdev)
422 {
423 	device_del(&thdev->dev);
424 	put_device(&thdev->dev);
425 }
426 
427 static void intel_th_device_free(struct intel_th_device *thdev)
428 {
429 	kfree(thdev->resource);
430 	kfree(thdev);
431 }
432 
433 /*
434  * Intel(R) Trace Hub subdevices
435  */
436 static const struct intel_th_subdevice {
437 	const char		*name;
438 	struct resource		res[3];
439 	unsigned		nres;
440 	unsigned		type;
441 	unsigned		otype;
442 	bool			mknode;
443 	unsigned		scrpd;
444 	int			id;
445 } intel_th_subdevices[] = {
446 	{
447 		.nres	= 1,
448 		.res	= {
449 			{
450 				/* Handle TSCU and CTS from GTH driver */
451 				.start	= REG_GTH_OFFSET,
452 				.end	= REG_CTS_OFFSET + REG_CTS_LENGTH - 1,
453 				.flags	= IORESOURCE_MEM,
454 			},
455 		},
456 		.name	= "gth",
457 		.type	= INTEL_TH_SWITCH,
458 		.id	= -1,
459 	},
460 	{
461 		.nres	= 2,
462 		.res	= {
463 			{
464 				.start	= REG_MSU_OFFSET,
465 				.end	= REG_MSU_OFFSET + REG_MSU_LENGTH - 1,
466 				.flags	= IORESOURCE_MEM,
467 			},
468 			{
469 				.start	= BUF_MSU_OFFSET,
470 				.end	= BUF_MSU_OFFSET + BUF_MSU_LENGTH - 1,
471 				.flags	= IORESOURCE_MEM,
472 			},
473 		},
474 		.name	= "msc",
475 		.id	= 0,
476 		.type	= INTEL_TH_OUTPUT,
477 		.mknode	= true,
478 		.otype	= GTH_MSU,
479 		.scrpd	= SCRPD_MEM_IS_PRIM_DEST | SCRPD_MSC0_IS_ENABLED,
480 	},
481 	{
482 		.nres	= 2,
483 		.res	= {
484 			{
485 				.start	= REG_MSU_OFFSET,
486 				.end	= REG_MSU_OFFSET + REG_MSU_LENGTH - 1,
487 				.flags	= IORESOURCE_MEM,
488 			},
489 			{
490 				.start	= BUF_MSU_OFFSET,
491 				.end	= BUF_MSU_OFFSET + BUF_MSU_LENGTH - 1,
492 				.flags	= IORESOURCE_MEM,
493 			},
494 		},
495 		.name	= "msc",
496 		.id	= 1,
497 		.type	= INTEL_TH_OUTPUT,
498 		.mknode	= true,
499 		.otype	= GTH_MSU,
500 		.scrpd	= SCRPD_MEM_IS_PRIM_DEST | SCRPD_MSC1_IS_ENABLED,
501 	},
502 	{
503 		.nres	= 2,
504 		.res	= {
505 			{
506 				.start	= REG_STH_OFFSET,
507 				.end	= REG_STH_OFFSET + REG_STH_LENGTH - 1,
508 				.flags	= IORESOURCE_MEM,
509 			},
510 			{
511 				.start	= TH_MMIO_SW,
512 				.end	= 0,
513 				.flags	= IORESOURCE_MEM,
514 			},
515 		},
516 		.id	= -1,
517 		.name	= "sth",
518 		.type	= INTEL_TH_SOURCE,
519 	},
520 	{
521 		.nres	= 2,
522 		.res	= {
523 			{
524 				.start	= REG_STH_OFFSET,
525 				.end	= REG_STH_OFFSET + REG_STH_LENGTH - 1,
526 				.flags	= IORESOURCE_MEM,
527 			},
528 			{
529 				.start	= TH_MMIO_RTIT,
530 				.end	= 0,
531 				.flags	= IORESOURCE_MEM,
532 			},
533 		},
534 		.id	= -1,
535 		.name	= "rtit",
536 		.type	= INTEL_TH_SOURCE,
537 	},
538 	{
539 		.nres	= 1,
540 		.res	= {
541 			{
542 				.start	= REG_PTI_OFFSET,
543 				.end	= REG_PTI_OFFSET + REG_PTI_LENGTH - 1,
544 				.flags	= IORESOURCE_MEM,
545 			},
546 		},
547 		.id	= -1,
548 		.name	= "pti",
549 		.type	= INTEL_TH_OUTPUT,
550 		.otype	= GTH_PTI,
551 		.scrpd	= SCRPD_PTI_IS_PRIM_DEST,
552 	},
553 	{
554 		.nres	= 1,
555 		.res	= {
556 			{
557 				.start	= REG_PTI_OFFSET,
558 				.end	= REG_PTI_OFFSET + REG_PTI_LENGTH - 1,
559 				.flags	= IORESOURCE_MEM,
560 			},
561 		},
562 		.id	= -1,
563 		.name	= "lpp",
564 		.type	= INTEL_TH_OUTPUT,
565 		.otype	= GTH_LPP,
566 		.scrpd	= SCRPD_PTI_IS_PRIM_DEST,
567 	},
568 	{
569 		.nres	= 1,
570 		.res	= {
571 			{
572 				.start	= REG_DCIH_OFFSET,
573 				.end	= REG_DCIH_OFFSET + REG_DCIH_LENGTH - 1,
574 				.flags	= IORESOURCE_MEM,
575 			},
576 		},
577 		.id	= -1,
578 		.name	= "dcih",
579 		.type	= INTEL_TH_OUTPUT,
580 	},
581 };
582 
583 #ifdef CONFIG_MODULES
584 static void __intel_th_request_hub_module(struct work_struct *work)
585 {
586 	struct intel_th *th = container_of(work, struct intel_th,
587 					   request_module_work);
588 
589 	request_module("intel_th_%s", th->hub->name);
590 }
591 
592 static int intel_th_request_hub_module(struct intel_th *th)
593 {
594 	INIT_WORK(&th->request_module_work, __intel_th_request_hub_module);
595 	schedule_work(&th->request_module_work);
596 
597 	return 0;
598 }
599 
600 static void intel_th_request_hub_module_flush(struct intel_th *th)
601 {
602 	flush_work(&th->request_module_work);
603 }
604 #else
605 static inline int intel_th_request_hub_module(struct intel_th *th)
606 {
607 	return -EINVAL;
608 }
609 
610 static inline void intel_th_request_hub_module_flush(struct intel_th *th)
611 {
612 }
613 #endif /* CONFIG_MODULES */
614 
615 static struct intel_th_device *
616 intel_th_subdevice_alloc(struct intel_th *th,
617 			 const struct intel_th_subdevice *subdev)
618 {
619 	struct intel_th_device *thdev;
620 	struct resource res[3];
621 	unsigned int req = 0;
622 	int r, err;
623 
624 	thdev = intel_th_device_alloc(th, subdev->type, subdev->name,
625 				      subdev->id);
626 	if (!thdev)
627 		return ERR_PTR(-ENOMEM);
628 
629 	thdev->drvdata = th->drvdata;
630 
631 	memcpy(res, subdev->res,
632 	       sizeof(struct resource) * subdev->nres);
633 
634 	for (r = 0; r < subdev->nres; r++) {
635 		struct resource *devres = th->resource;
636 		int bar = TH_MMIO_CONFIG;
637 
638 		/*
639 		 * Take .end == 0 to mean 'take the whole bar',
640 		 * .start then tells us which bar it is. Default to
641 		 * TH_MMIO_CONFIG.
642 		 */
643 		if (!res[r].end && res[r].flags == IORESOURCE_MEM) {
644 			bar = res[r].start;
645 			err = -ENODEV;
646 			if (bar >= th->num_resources)
647 				goto fail_put_device;
648 			res[r].start = 0;
649 			res[r].end = resource_size(&devres[bar]) - 1;
650 		}
651 
652 		if (res[r].flags & IORESOURCE_MEM) {
653 			res[r].start	+= devres[bar].start;
654 			res[r].end	+= devres[bar].start;
655 
656 			dev_dbg(th->dev, "%s:%d @ %pR\n",
657 				subdev->name, r, &res[r]);
658 		} else if (res[r].flags & IORESOURCE_IRQ) {
659 			/*
660 			 * Only pass on the IRQ if we have useful interrupts:
661 			 * the ones that can be configured via MINTCTL.
662 			 */
663 			if (INTEL_TH_CAP(th, has_mintctl) && th->irq != -1)
664 				res[r].start = th->irq;
665 		}
666 	}
667 
668 	err = intel_th_device_add_resources(thdev, res, subdev->nres);
669 	if (err)
670 		goto fail_put_device;
671 
672 	if (subdev->type == INTEL_TH_OUTPUT) {
673 		if (subdev->mknode)
674 			thdev->dev.devt = MKDEV(th->major, th->num_thdevs);
675 		thdev->output.type = subdev->otype;
676 		thdev->output.port = -1;
677 		thdev->output.scratchpad = subdev->scrpd;
678 	} else if (subdev->type == INTEL_TH_SWITCH) {
679 		thdev->host_mode =
680 			INTEL_TH_CAP(th, host_mode_only) ? true : host_mode;
681 		th->hub = thdev;
682 	}
683 
684 	err = device_add(&thdev->dev);
685 	if (err)
686 		goto fail_free_res;
687 
688 	/* need switch driver to be loaded to enumerate the rest */
689 	if (subdev->type == INTEL_TH_SWITCH && !req) {
690 		err = intel_th_request_hub_module(th);
691 		if (!err)
692 			req++;
693 	}
694 
695 	return thdev;
696 
697 fail_free_res:
698 	kfree(thdev->resource);
699 
700 fail_put_device:
701 	put_device(&thdev->dev);
702 
703 	return ERR_PTR(err);
704 }
705 
706 /**
707  * intel_th_output_enable() - find and enable a device for a given output type
708  * @th:		Intel TH instance
709  * @otype:	output type
710  *
711  * Go through the unallocated output devices, find the first one whos type
712  * matches @otype and instantiate it. These devices are removed when the hub
713  * device is removed, see intel_th_remove().
714  */
715 int intel_th_output_enable(struct intel_th *th, unsigned int otype)
716 {
717 	struct intel_th_device *thdev;
718 	int src = 0, dst = 0;
719 
720 	for (src = 0, dst = 0; dst <= th->num_thdevs; src++, dst++) {
721 		for (; src < ARRAY_SIZE(intel_th_subdevices); src++) {
722 			if (intel_th_subdevices[src].type != INTEL_TH_OUTPUT)
723 				continue;
724 
725 			if (intel_th_subdevices[src].otype != otype)
726 				continue;
727 
728 			break;
729 		}
730 
731 		/* no unallocated matching subdevices */
732 		if (src == ARRAY_SIZE(intel_th_subdevices))
733 			return -ENODEV;
734 
735 		for (; dst < th->num_thdevs; dst++) {
736 			if (th->thdev[dst]->type != INTEL_TH_OUTPUT)
737 				continue;
738 
739 			if (th->thdev[dst]->output.type != otype)
740 				continue;
741 
742 			break;
743 		}
744 
745 		/*
746 		 * intel_th_subdevices[src] matches our requirements and is
747 		 * not matched in th::thdev[]
748 		 */
749 		if (dst == th->num_thdevs)
750 			goto found;
751 	}
752 
753 	return -ENODEV;
754 
755 found:
756 	thdev = intel_th_subdevice_alloc(th, &intel_th_subdevices[src]);
757 	if (IS_ERR(thdev))
758 		return PTR_ERR(thdev);
759 
760 	th->thdev[th->num_thdevs++] = thdev;
761 
762 	return 0;
763 }
764 EXPORT_SYMBOL_GPL(intel_th_output_enable);
765 
766 static int intel_th_populate(struct intel_th *th)
767 {
768 	int src;
769 
770 	/* create devices for each intel_th_subdevice */
771 	for (src = 0; src < ARRAY_SIZE(intel_th_subdevices); src++) {
772 		const struct intel_th_subdevice *subdev =
773 			&intel_th_subdevices[src];
774 		struct intel_th_device *thdev;
775 
776 		/* only allow SOURCE and SWITCH devices in host mode */
777 		if ((INTEL_TH_CAP(th, host_mode_only) || host_mode) &&
778 		    subdev->type == INTEL_TH_OUTPUT)
779 			continue;
780 
781 		/*
782 		 * don't enable port OUTPUTs in this path; SWITCH enables them
783 		 * via intel_th_output_enable()
784 		 */
785 		if (subdev->type == INTEL_TH_OUTPUT &&
786 		    subdev->otype != GTH_NONE)
787 			continue;
788 
789 		thdev = intel_th_subdevice_alloc(th, subdev);
790 		/* note: caller should free subdevices from th::thdev[] */
791 		if (IS_ERR(thdev)) {
792 			/* ENODEV for individual subdevices is allowed */
793 			if (PTR_ERR(thdev) == -ENODEV)
794 				continue;
795 
796 			return PTR_ERR(thdev);
797 		}
798 
799 		th->thdev[th->num_thdevs++] = thdev;
800 	}
801 
802 	return 0;
803 }
804 
805 static int intel_th_output_open(struct inode *inode, struct file *file)
806 {
807 	const struct file_operations *fops;
808 	struct intel_th_driver *thdrv;
809 	struct device *dev;
810 	int err;
811 
812 	dev = bus_find_device_by_devt(&intel_th_bus, inode->i_rdev);
813 	if (!dev || !dev->driver)
814 		return -ENODEV;
815 
816 	thdrv = to_intel_th_driver(dev->driver);
817 	fops = fops_get(thdrv->fops);
818 	if (!fops)
819 		return -ENODEV;
820 
821 	replace_fops(file, fops);
822 
823 	file->private_data = to_intel_th_device(dev);
824 
825 	if (file->f_op->open) {
826 		err = file->f_op->open(inode, file);
827 		return err;
828 	}
829 
830 	return 0;
831 }
832 
833 static const struct file_operations intel_th_output_fops = {
834 	.open	= intel_th_output_open,
835 	.llseek	= noop_llseek,
836 };
837 
838 static irqreturn_t intel_th_irq(int irq, void *data)
839 {
840 	struct intel_th *th = data;
841 	irqreturn_t ret = IRQ_NONE;
842 	struct intel_th_driver *d;
843 	int i;
844 
845 	for (i = 0; i < th->num_thdevs; i++) {
846 		if (th->thdev[i]->type != INTEL_TH_OUTPUT)
847 			continue;
848 
849 		d = to_intel_th_driver(th->thdev[i]->dev.driver);
850 		if (d && d->irq)
851 			ret |= d->irq(th->thdev[i]);
852 	}
853 
854 	return ret;
855 }
856 
857 /**
858  * intel_th_alloc() - allocate a new Intel TH device and its subdevices
859  * @dev:	parent device
860  * @devres:	resources indexed by th_mmio_idx
861  * @irq:	irq number
862  */
863 struct intel_th *
864 intel_th_alloc(struct device *dev, const struct intel_th_drvdata *drvdata,
865 	       struct resource *devres, unsigned int ndevres)
866 {
867 	int err, r, nr_mmios = 0;
868 	struct intel_th *th;
869 
870 	th = kzalloc(sizeof(*th), GFP_KERNEL);
871 	if (!th)
872 		return ERR_PTR(-ENOMEM);
873 
874 	th->id = ida_simple_get(&intel_th_ida, 0, 0, GFP_KERNEL);
875 	if (th->id < 0) {
876 		err = th->id;
877 		goto err_alloc;
878 	}
879 
880 	th->major = __register_chrdev(0, 0, TH_POSSIBLE_OUTPUTS,
881 				      "intel_th/output", &intel_th_output_fops);
882 	if (th->major < 0) {
883 		err = th->major;
884 		goto err_ida;
885 	}
886 	th->irq = -1;
887 	th->dev = dev;
888 	th->drvdata = drvdata;
889 
890 	for (r = 0; r < ndevres; r++)
891 		switch (devres[r].flags & IORESOURCE_TYPE_BITS) {
892 		case IORESOURCE_MEM:
893 			th->resource[nr_mmios++] = devres[r];
894 			break;
895 		case IORESOURCE_IRQ:
896 			err = devm_request_irq(dev, devres[r].start,
897 					       intel_th_irq, IRQF_SHARED,
898 					       dev_name(dev), th);
899 			if (err)
900 				goto err_chrdev;
901 
902 			if (th->irq == -1)
903 				th->irq = devres[r].start;
904 			th->num_irqs++;
905 			break;
906 		default:
907 			dev_warn(dev, "Unknown resource type %lx\n",
908 				 devres[r].flags);
909 			break;
910 		}
911 
912 	th->num_resources = nr_mmios;
913 
914 	dev_set_drvdata(dev, th);
915 
916 	pm_runtime_no_callbacks(dev);
917 	pm_runtime_put(dev);
918 	pm_runtime_allow(dev);
919 
920 	err = intel_th_populate(th);
921 	if (err) {
922 		/* free the subdevices and undo everything */
923 		intel_th_free(th);
924 		return ERR_PTR(err);
925 	}
926 
927 	return th;
928 
929 err_chrdev:
930 	__unregister_chrdev(th->major, 0, TH_POSSIBLE_OUTPUTS,
931 			    "intel_th/output");
932 
933 err_ida:
934 	ida_simple_remove(&intel_th_ida, th->id);
935 
936 err_alloc:
937 	kfree(th);
938 
939 	return ERR_PTR(err);
940 }
941 EXPORT_SYMBOL_GPL(intel_th_alloc);
942 
943 void intel_th_free(struct intel_th *th)
944 {
945 	int i;
946 
947 	intel_th_request_hub_module_flush(th);
948 
949 	intel_th_device_remove(th->hub);
950 	for (i = 0; i < th->num_thdevs; i++) {
951 		if (th->thdev[i] != th->hub)
952 			intel_th_device_remove(th->thdev[i]);
953 		th->thdev[i] = NULL;
954 	}
955 
956 	th->num_thdevs = 0;
957 
958 	for (i = 0; i < th->num_irqs; i++)
959 		devm_free_irq(th->dev, th->irq + i, th);
960 
961 	pm_runtime_get_sync(th->dev);
962 	pm_runtime_forbid(th->dev);
963 
964 	__unregister_chrdev(th->major, 0, TH_POSSIBLE_OUTPUTS,
965 			    "intel_th/output");
966 
967 	ida_simple_remove(&intel_th_ida, th->id);
968 
969 	kfree(th);
970 }
971 EXPORT_SYMBOL_GPL(intel_th_free);
972 
973 /**
974  * intel_th_trace_enable() - enable tracing for an output device
975  * @thdev:	output device that requests tracing be enabled
976  */
977 int intel_th_trace_enable(struct intel_th_device *thdev)
978 {
979 	struct intel_th_device *hub = to_intel_th_device(thdev->dev.parent);
980 	struct intel_th_driver *hubdrv = to_intel_th_driver(hub->dev.driver);
981 
982 	if (WARN_ON_ONCE(hub->type != INTEL_TH_SWITCH))
983 		return -EINVAL;
984 
985 	if (WARN_ON_ONCE(thdev->type != INTEL_TH_OUTPUT))
986 		return -EINVAL;
987 
988 	pm_runtime_get_sync(&thdev->dev);
989 	hubdrv->enable(hub, &thdev->output);
990 
991 	return 0;
992 }
993 EXPORT_SYMBOL_GPL(intel_th_trace_enable);
994 
995 /**
996  * intel_th_trace_switch() - execute a switch sequence
997  * @thdev:	output device that requests tracing switch
998  */
999 int intel_th_trace_switch(struct intel_th_device *thdev)
1000 {
1001 	struct intel_th_device *hub = to_intel_th_device(thdev->dev.parent);
1002 	struct intel_th_driver *hubdrv = to_intel_th_driver(hub->dev.driver);
1003 
1004 	if (WARN_ON_ONCE(hub->type != INTEL_TH_SWITCH))
1005 		return -EINVAL;
1006 
1007 	if (WARN_ON_ONCE(thdev->type != INTEL_TH_OUTPUT))
1008 		return -EINVAL;
1009 
1010 	hubdrv->trig_switch(hub, &thdev->output);
1011 
1012 	return 0;
1013 }
1014 EXPORT_SYMBOL_GPL(intel_th_trace_switch);
1015 
1016 /**
1017  * intel_th_trace_disable() - disable tracing for an output device
1018  * @thdev:	output device that requests tracing be disabled
1019  */
1020 int intel_th_trace_disable(struct intel_th_device *thdev)
1021 {
1022 	struct intel_th_device *hub = to_intel_th_device(thdev->dev.parent);
1023 	struct intel_th_driver *hubdrv = to_intel_th_driver(hub->dev.driver);
1024 
1025 	WARN_ON_ONCE(hub->type != INTEL_TH_SWITCH);
1026 	if (WARN_ON_ONCE(thdev->type != INTEL_TH_OUTPUT))
1027 		return -EINVAL;
1028 
1029 	hubdrv->disable(hub, &thdev->output);
1030 	pm_runtime_put(&thdev->dev);
1031 
1032 	return 0;
1033 }
1034 EXPORT_SYMBOL_GPL(intel_th_trace_disable);
1035 
1036 int intel_th_set_output(struct intel_th_device *thdev,
1037 			unsigned int master)
1038 {
1039 	struct intel_th_device *hub = to_intel_th_hub(thdev);
1040 	struct intel_th_driver *hubdrv = to_intel_th_driver(hub->dev.driver);
1041 	int ret;
1042 
1043 	/* In host mode, this is up to the external debugger, do nothing. */
1044 	if (hub->host_mode)
1045 		return 0;
1046 
1047 	/*
1048 	 * hub is instantiated together with the source device that
1049 	 * calls here, so guaranteed to be present.
1050 	 */
1051 	hubdrv = to_intel_th_driver(hub->dev.driver);
1052 	if (!hubdrv || !try_module_get(hubdrv->driver.owner))
1053 		return -EINVAL;
1054 
1055 	if (!hubdrv->set_output) {
1056 		ret = -ENOTSUPP;
1057 		goto out;
1058 	}
1059 
1060 	ret = hubdrv->set_output(hub, master);
1061 
1062 out:
1063 	module_put(hubdrv->driver.owner);
1064 	return ret;
1065 }
1066 EXPORT_SYMBOL_GPL(intel_th_set_output);
1067 
1068 static int __init intel_th_init(void)
1069 {
1070 	intel_th_debug_init();
1071 
1072 	return bus_register(&intel_th_bus);
1073 }
1074 subsys_initcall(intel_th_init);
1075 
1076 static void __exit intel_th_exit(void)
1077 {
1078 	intel_th_debug_done();
1079 
1080 	bus_unregister(&intel_th_bus);
1081 }
1082 module_exit(intel_th_exit);
1083 
1084 MODULE_LICENSE("GPL v2");
1085 MODULE_DESCRIPTION("Intel(R) Trace Hub controller driver");
1086 MODULE_AUTHOR("Alexander Shishkin <alexander.shishkin@linux.intel.com>");
1087