1 /*
2  * Intel(R) Trace Hub driver core
3  *
4  * Copyright (C) 2014-2015 Intel Corporation.
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms and conditions of the GNU General Public License,
8  * version 2, as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope it will be useful, but WITHOUT
11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
13  * more details.
14  */
15 
16 #define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
17 
18 #include <linux/types.h>
19 #include <linux/module.h>
20 #include <linux/device.h>
21 #include <linux/sysfs.h>
22 #include <linux/kdev_t.h>
23 #include <linux/debugfs.h>
24 #include <linux/idr.h>
25 #include <linux/pci.h>
26 #include <linux/pm_runtime.h>
27 #include <linux/dma-mapping.h>
28 
29 #include "intel_th.h"
30 #include "debug.h"
31 
32 static bool host_mode __read_mostly;
33 module_param(host_mode, bool, 0444);
34 
35 static DEFINE_IDA(intel_th_ida);
36 
37 static int intel_th_match(struct device *dev, struct device_driver *driver)
38 {
39 	struct intel_th_driver *thdrv = to_intel_th_driver(driver);
40 	struct intel_th_device *thdev = to_intel_th_device(dev);
41 
42 	if (thdev->type == INTEL_TH_SWITCH &&
43 	    (!thdrv->enable || !thdrv->disable))
44 		return 0;
45 
46 	return !strcmp(thdev->name, driver->name);
47 }
48 
49 static int intel_th_child_remove(struct device *dev, void *data)
50 {
51 	device_release_driver(dev);
52 
53 	return 0;
54 }
55 
56 static int intel_th_probe(struct device *dev)
57 {
58 	struct intel_th_driver *thdrv = to_intel_th_driver(dev->driver);
59 	struct intel_th_device *thdev = to_intel_th_device(dev);
60 	struct intel_th_driver *hubdrv;
61 	struct intel_th_device *hub = NULL;
62 	int ret;
63 
64 	if (thdev->type == INTEL_TH_SWITCH)
65 		hub = thdev;
66 	else if (dev->parent)
67 		hub = to_intel_th_device(dev->parent);
68 
69 	if (!hub || !hub->dev.driver)
70 		return -EPROBE_DEFER;
71 
72 	hubdrv = to_intel_th_driver(hub->dev.driver);
73 
74 	pm_runtime_set_active(dev);
75 	pm_runtime_no_callbacks(dev);
76 	pm_runtime_enable(dev);
77 
78 	ret = thdrv->probe(to_intel_th_device(dev));
79 	if (ret)
80 		goto out_pm;
81 
82 	if (thdrv->attr_group) {
83 		ret = sysfs_create_group(&thdev->dev.kobj, thdrv->attr_group);
84 		if (ret)
85 			goto out;
86 	}
87 
88 	if (thdev->type == INTEL_TH_OUTPUT &&
89 	    !intel_th_output_assigned(thdev))
90 		/* does not talk to hardware */
91 		ret = hubdrv->assign(hub, thdev);
92 
93 out:
94 	if (ret)
95 		thdrv->remove(thdev);
96 
97 out_pm:
98 	if (ret)
99 		pm_runtime_disable(dev);
100 
101 	return ret;
102 }
103 
104 static void intel_th_device_remove(struct intel_th_device *thdev);
105 
106 static int intel_th_remove(struct device *dev)
107 {
108 	struct intel_th_driver *thdrv = to_intel_th_driver(dev->driver);
109 	struct intel_th_device *thdev = to_intel_th_device(dev);
110 	struct intel_th_device *hub = to_intel_th_hub(thdev);
111 	int err;
112 
113 	if (thdev->type == INTEL_TH_SWITCH) {
114 		struct intel_th *th = to_intel_th(hub);
115 		int i, lowest;
116 
117 		/* disconnect outputs */
118 		err = device_for_each_child(dev, thdev, intel_th_child_remove);
119 		if (err)
120 			return err;
121 
122 		/*
123 		 * Remove outputs, that is, hub's children: they are created
124 		 * at hub's probe time by having the hub call
125 		 * intel_th_output_enable() for each of them.
126 		 */
127 		for (i = 0, lowest = -1; i < th->num_thdevs; i++) {
128 			/*
129 			 * Move the non-output devices from higher up the
130 			 * th->thdev[] array to lower positions to maintain
131 			 * a contiguous array.
132 			 */
133 			if (th->thdev[i]->type != INTEL_TH_OUTPUT) {
134 				if (lowest >= 0) {
135 					th->thdev[lowest] = th->thdev[i];
136 					th->thdev[i] = NULL;
137 					++lowest;
138 				}
139 
140 				continue;
141 			}
142 
143 			if (lowest == -1)
144 				lowest = i;
145 
146 			intel_th_device_remove(th->thdev[i]);
147 			th->thdev[i] = NULL;
148 		}
149 
150 		th->num_thdevs = lowest;
151 	}
152 
153 	if (thdrv->attr_group)
154 		sysfs_remove_group(&thdev->dev.kobj, thdrv->attr_group);
155 
156 	pm_runtime_get_sync(dev);
157 
158 	thdrv->remove(thdev);
159 
160 	if (intel_th_output_assigned(thdev)) {
161 		struct intel_th_driver *hubdrv =
162 			to_intel_th_driver(dev->parent->driver);
163 
164 		if (hub->dev.driver)
165 			/* does not talk to hardware */
166 			hubdrv->unassign(hub, thdev);
167 	}
168 
169 	pm_runtime_disable(dev);
170 	pm_runtime_set_active(dev);
171 	pm_runtime_enable(dev);
172 
173 	return 0;
174 }
175 
176 static struct bus_type intel_th_bus = {
177 	.name		= "intel_th",
178 	.match		= intel_th_match,
179 	.probe		= intel_th_probe,
180 	.remove		= intel_th_remove,
181 };
182 
183 static void intel_th_device_free(struct intel_th_device *thdev);
184 
185 static void intel_th_device_release(struct device *dev)
186 {
187 	intel_th_device_free(to_intel_th_device(dev));
188 }
189 
190 static struct device_type intel_th_source_device_type = {
191 	.name		= "intel_th_source_device",
192 	.release	= intel_th_device_release,
193 };
194 
195 static char *intel_th_output_devnode(struct device *dev, umode_t *mode,
196 				     kuid_t *uid, kgid_t *gid)
197 {
198 	struct intel_th_device *thdev = to_intel_th_device(dev);
199 	struct intel_th *th = to_intel_th(thdev);
200 	char *node;
201 
202 	if (thdev->id >= 0)
203 		node = kasprintf(GFP_KERNEL, "intel_th%d/%s%d", th->id,
204 				 thdev->name, thdev->id);
205 	else
206 		node = kasprintf(GFP_KERNEL, "intel_th%d/%s", th->id,
207 				 thdev->name);
208 
209 	return node;
210 }
211 
212 static ssize_t port_show(struct device *dev, struct device_attribute *attr,
213 			 char *buf)
214 {
215 	struct intel_th_device *thdev = to_intel_th_device(dev);
216 
217 	if (thdev->output.port >= 0)
218 		return scnprintf(buf, PAGE_SIZE, "%u\n", thdev->output.port);
219 
220 	return scnprintf(buf, PAGE_SIZE, "unassigned\n");
221 }
222 
223 static DEVICE_ATTR_RO(port);
224 
225 static int intel_th_output_activate(struct intel_th_device *thdev)
226 {
227 	struct intel_th_driver *thdrv =
228 		to_intel_th_driver_or_null(thdev->dev.driver);
229 	struct intel_th *th = to_intel_th(thdev);
230 	int ret = 0;
231 
232 	if (!thdrv)
233 		return -ENODEV;
234 
235 	if (!try_module_get(thdrv->driver.owner))
236 		return -ENODEV;
237 
238 	pm_runtime_get_sync(&thdev->dev);
239 
240 	if (th->activate)
241 		ret = th->activate(th);
242 	if (ret)
243 		goto fail_put;
244 
245 	if (thdrv->activate)
246 		ret = thdrv->activate(thdev);
247 	else
248 		intel_th_trace_enable(thdev);
249 
250 	if (ret)
251 		goto fail_deactivate;
252 
253 	return 0;
254 
255 fail_deactivate:
256 	if (th->deactivate)
257 		th->deactivate(th);
258 
259 fail_put:
260 	pm_runtime_put(&thdev->dev);
261 	module_put(thdrv->driver.owner);
262 
263 	return ret;
264 }
265 
266 static void intel_th_output_deactivate(struct intel_th_device *thdev)
267 {
268 	struct intel_th_driver *thdrv =
269 		to_intel_th_driver_or_null(thdev->dev.driver);
270 	struct intel_th *th = to_intel_th(thdev);
271 
272 	if (!thdrv)
273 		return;
274 
275 	if (thdrv->deactivate)
276 		thdrv->deactivate(thdev);
277 	else
278 		intel_th_trace_disable(thdev);
279 
280 	if (th->deactivate)
281 		th->deactivate(th);
282 
283 	pm_runtime_put(&thdev->dev);
284 	module_put(thdrv->driver.owner);
285 }
286 
287 static ssize_t active_show(struct device *dev, struct device_attribute *attr,
288 			   char *buf)
289 {
290 	struct intel_th_device *thdev = to_intel_th_device(dev);
291 
292 	return scnprintf(buf, PAGE_SIZE, "%d\n", thdev->output.active);
293 }
294 
295 static ssize_t active_store(struct device *dev, struct device_attribute *attr,
296 			    const char *buf, size_t size)
297 {
298 	struct intel_th_device *thdev = to_intel_th_device(dev);
299 	unsigned long val;
300 	int ret;
301 
302 	ret = kstrtoul(buf, 10, &val);
303 	if (ret)
304 		return ret;
305 
306 	if (!!val != thdev->output.active) {
307 		if (val)
308 			ret = intel_th_output_activate(thdev);
309 		else
310 			intel_th_output_deactivate(thdev);
311 	}
312 
313 	return ret ? ret : size;
314 }
315 
316 static DEVICE_ATTR_RW(active);
317 
318 static struct attribute *intel_th_output_attrs[] = {
319 	&dev_attr_port.attr,
320 	&dev_attr_active.attr,
321 	NULL,
322 };
323 
324 ATTRIBUTE_GROUPS(intel_th_output);
325 
326 static struct device_type intel_th_output_device_type = {
327 	.name		= "intel_th_output_device",
328 	.groups		= intel_th_output_groups,
329 	.release	= intel_th_device_release,
330 	.devnode	= intel_th_output_devnode,
331 };
332 
333 static struct device_type intel_th_switch_device_type = {
334 	.name		= "intel_th_switch_device",
335 	.release	= intel_th_device_release,
336 };
337 
338 static struct device_type *intel_th_device_type[] = {
339 	[INTEL_TH_SOURCE]	= &intel_th_source_device_type,
340 	[INTEL_TH_OUTPUT]	= &intel_th_output_device_type,
341 	[INTEL_TH_SWITCH]	= &intel_th_switch_device_type,
342 };
343 
344 int intel_th_driver_register(struct intel_th_driver *thdrv)
345 {
346 	if (!thdrv->probe || !thdrv->remove)
347 		return -EINVAL;
348 
349 	thdrv->driver.bus = &intel_th_bus;
350 
351 	return driver_register(&thdrv->driver);
352 }
353 EXPORT_SYMBOL_GPL(intel_th_driver_register);
354 
355 void intel_th_driver_unregister(struct intel_th_driver *thdrv)
356 {
357 	driver_unregister(&thdrv->driver);
358 }
359 EXPORT_SYMBOL_GPL(intel_th_driver_unregister);
360 
361 static struct intel_th_device *
362 intel_th_device_alloc(struct intel_th *th, unsigned int type, const char *name,
363 		      int id)
364 {
365 	struct device *parent;
366 	struct intel_th_device *thdev;
367 
368 	if (type == INTEL_TH_OUTPUT)
369 		parent = &th->hub->dev;
370 	else
371 		parent = th->dev;
372 
373 	thdev = kzalloc(sizeof(*thdev) + strlen(name) + 1, GFP_KERNEL);
374 	if (!thdev)
375 		return NULL;
376 
377 	thdev->id = id;
378 	thdev->type = type;
379 
380 	strcpy(thdev->name, name);
381 	device_initialize(&thdev->dev);
382 	thdev->dev.bus = &intel_th_bus;
383 	thdev->dev.type = intel_th_device_type[type];
384 	thdev->dev.parent = parent;
385 	thdev->dev.dma_mask = parent->dma_mask;
386 	thdev->dev.dma_parms = parent->dma_parms;
387 	dma_set_coherent_mask(&thdev->dev, parent->coherent_dma_mask);
388 	if (id >= 0)
389 		dev_set_name(&thdev->dev, "%d-%s%d", th->id, name, id);
390 	else
391 		dev_set_name(&thdev->dev, "%d-%s", th->id, name);
392 
393 	return thdev;
394 }
395 
396 static int intel_th_device_add_resources(struct intel_th_device *thdev,
397 					 struct resource *res, int nres)
398 {
399 	struct resource *r;
400 
401 	r = kmemdup(res, sizeof(*res) * nres, GFP_KERNEL);
402 	if (!r)
403 		return -ENOMEM;
404 
405 	thdev->resource = r;
406 	thdev->num_resources = nres;
407 
408 	return 0;
409 }
410 
411 static void intel_th_device_remove(struct intel_th_device *thdev)
412 {
413 	device_del(&thdev->dev);
414 	put_device(&thdev->dev);
415 }
416 
417 static void intel_th_device_free(struct intel_th_device *thdev)
418 {
419 	kfree(thdev->resource);
420 	kfree(thdev);
421 }
422 
423 /*
424  * Intel(R) Trace Hub subdevices
425  */
426 static const struct intel_th_subdevice {
427 	const char		*name;
428 	struct resource		res[3];
429 	unsigned		nres;
430 	unsigned		type;
431 	unsigned		otype;
432 	unsigned		scrpd;
433 	int			id;
434 } intel_th_subdevices[] = {
435 	{
436 		.nres	= 1,
437 		.res	= {
438 			{
439 				/* Handle TSCU from GTH driver */
440 				.start	= REG_GTH_OFFSET,
441 				.end	= REG_TSCU_OFFSET + REG_TSCU_LENGTH - 1,
442 				.flags	= IORESOURCE_MEM,
443 			},
444 		},
445 		.name	= "gth",
446 		.type	= INTEL_TH_SWITCH,
447 		.id	= -1,
448 	},
449 	{
450 		.nres	= 2,
451 		.res	= {
452 			{
453 				.start	= REG_MSU_OFFSET,
454 				.end	= REG_MSU_OFFSET + REG_MSU_LENGTH - 1,
455 				.flags	= IORESOURCE_MEM,
456 			},
457 			{
458 				.start	= BUF_MSU_OFFSET,
459 				.end	= BUF_MSU_OFFSET + BUF_MSU_LENGTH - 1,
460 				.flags	= IORESOURCE_MEM,
461 			},
462 		},
463 		.name	= "msc",
464 		.id	= 0,
465 		.type	= INTEL_TH_OUTPUT,
466 		.otype	= GTH_MSU,
467 		.scrpd	= SCRPD_MEM_IS_PRIM_DEST | SCRPD_MSC0_IS_ENABLED,
468 	},
469 	{
470 		.nres	= 2,
471 		.res	= {
472 			{
473 				.start	= REG_MSU_OFFSET,
474 				.end	= REG_MSU_OFFSET + REG_MSU_LENGTH - 1,
475 				.flags	= IORESOURCE_MEM,
476 			},
477 			{
478 				.start	= BUF_MSU_OFFSET,
479 				.end	= BUF_MSU_OFFSET + BUF_MSU_LENGTH - 1,
480 				.flags	= IORESOURCE_MEM,
481 			},
482 		},
483 		.name	= "msc",
484 		.id	= 1,
485 		.type	= INTEL_TH_OUTPUT,
486 		.otype	= GTH_MSU,
487 		.scrpd	= SCRPD_MEM_IS_PRIM_DEST | SCRPD_MSC1_IS_ENABLED,
488 	},
489 	{
490 		.nres	= 2,
491 		.res	= {
492 			{
493 				.start	= REG_STH_OFFSET,
494 				.end	= REG_STH_OFFSET + REG_STH_LENGTH - 1,
495 				.flags	= IORESOURCE_MEM,
496 			},
497 			{
498 				.start	= TH_MMIO_SW,
499 				.end	= 0,
500 				.flags	= IORESOURCE_MEM,
501 			},
502 		},
503 		.id	= -1,
504 		.name	= "sth",
505 		.type	= INTEL_TH_SOURCE,
506 	},
507 	{
508 		.nres	= 1,
509 		.res	= {
510 			{
511 				.start	= REG_PTI_OFFSET,
512 				.end	= REG_PTI_OFFSET + REG_PTI_LENGTH - 1,
513 				.flags	= IORESOURCE_MEM,
514 			},
515 		},
516 		.id	= -1,
517 		.name	= "pti",
518 		.type	= INTEL_TH_OUTPUT,
519 		.otype	= GTH_PTI,
520 		.scrpd	= SCRPD_PTI_IS_PRIM_DEST,
521 	},
522 	{
523 		.nres	= 1,
524 		.res	= {
525 			{
526 				.start	= REG_PTI_OFFSET,
527 				.end	= REG_PTI_OFFSET + REG_PTI_LENGTH - 1,
528 				.flags	= IORESOURCE_MEM,
529 			},
530 		},
531 		.id	= -1,
532 		.name	= "lpp",
533 		.type	= INTEL_TH_OUTPUT,
534 		.otype	= GTH_LPP,
535 		.scrpd	= SCRPD_PTI_IS_PRIM_DEST,
536 	},
537 	{
538 		.nres	= 1,
539 		.res	= {
540 			{
541 				.start	= REG_DCIH_OFFSET,
542 				.end	= REG_DCIH_OFFSET + REG_DCIH_LENGTH - 1,
543 				.flags	= IORESOURCE_MEM,
544 			},
545 		},
546 		.id	= -1,
547 		.name	= "dcih",
548 		.type	= INTEL_TH_OUTPUT,
549 	},
550 };
551 
552 #ifdef CONFIG_MODULES
553 static void __intel_th_request_hub_module(struct work_struct *work)
554 {
555 	struct intel_th *th = container_of(work, struct intel_th,
556 					   request_module_work);
557 
558 	request_module("intel_th_%s", th->hub->name);
559 }
560 
561 static int intel_th_request_hub_module(struct intel_th *th)
562 {
563 	INIT_WORK(&th->request_module_work, __intel_th_request_hub_module);
564 	schedule_work(&th->request_module_work);
565 
566 	return 0;
567 }
568 
569 static void intel_th_request_hub_module_flush(struct intel_th *th)
570 {
571 	flush_work(&th->request_module_work);
572 }
573 #else
574 static inline int intel_th_request_hub_module(struct intel_th *th)
575 {
576 	return -EINVAL;
577 }
578 
579 static inline void intel_th_request_hub_module_flush(struct intel_th *th)
580 {
581 }
582 #endif /* CONFIG_MODULES */
583 
584 static struct intel_th_device *
585 intel_th_subdevice_alloc(struct intel_th *th,
586 			 const struct intel_th_subdevice *subdev)
587 {
588 	struct intel_th_device *thdev;
589 	struct resource res[3];
590 	unsigned int req = 0;
591 	int r, err;
592 
593 	thdev = intel_th_device_alloc(th, subdev->type, subdev->name,
594 				      subdev->id);
595 	if (!thdev)
596 		return ERR_PTR(-ENOMEM);
597 
598 	thdev->drvdata = th->drvdata;
599 
600 	memcpy(res, subdev->res,
601 	       sizeof(struct resource) * subdev->nres);
602 
603 	for (r = 0; r < subdev->nres; r++) {
604 		struct resource *devres = th->resource;
605 		int bar = TH_MMIO_CONFIG;
606 
607 		/*
608 		 * Take .end == 0 to mean 'take the whole bar',
609 		 * .start then tells us which bar it is. Default to
610 		 * TH_MMIO_CONFIG.
611 		 */
612 		if (!res[r].end && res[r].flags == IORESOURCE_MEM) {
613 			bar = res[r].start;
614 			res[r].start = 0;
615 			res[r].end = resource_size(&devres[bar]) - 1;
616 		}
617 
618 		if (res[r].flags & IORESOURCE_MEM) {
619 			res[r].start	+= devres[bar].start;
620 			res[r].end	+= devres[bar].start;
621 
622 			dev_dbg(th->dev, "%s:%d @ %pR\n",
623 				subdev->name, r, &res[r]);
624 		} else if (res[r].flags & IORESOURCE_IRQ) {
625 			res[r].start	= th->irq;
626 		}
627 	}
628 
629 	err = intel_th_device_add_resources(thdev, res, subdev->nres);
630 	if (err) {
631 		put_device(&thdev->dev);
632 		goto fail_put_device;
633 	}
634 
635 	if (subdev->type == INTEL_TH_OUTPUT) {
636 		thdev->dev.devt = MKDEV(th->major, th->num_thdevs);
637 		thdev->output.type = subdev->otype;
638 		thdev->output.port = -1;
639 		thdev->output.scratchpad = subdev->scrpd;
640 	} else if (subdev->type == INTEL_TH_SWITCH) {
641 		thdev->host_mode = host_mode;
642 		th->hub = thdev;
643 	}
644 
645 	err = device_add(&thdev->dev);
646 	if (err) {
647 		put_device(&thdev->dev);
648 		goto fail_free_res;
649 	}
650 
651 	/* need switch driver to be loaded to enumerate the rest */
652 	if (subdev->type == INTEL_TH_SWITCH && !req) {
653 		err = intel_th_request_hub_module(th);
654 		if (!err)
655 			req++;
656 	}
657 
658 	return thdev;
659 
660 fail_free_res:
661 	kfree(thdev->resource);
662 
663 fail_put_device:
664 	put_device(&thdev->dev);
665 
666 	return ERR_PTR(err);
667 }
668 
669 /**
670  * intel_th_output_enable() - find and enable a device for a given output type
671  * @th:		Intel TH instance
672  * @otype:	output type
673  *
674  * Go through the unallocated output devices, find the first one whos type
675  * matches @otype and instantiate it. These devices are removed when the hub
676  * device is removed, see intel_th_remove().
677  */
678 int intel_th_output_enable(struct intel_th *th, unsigned int otype)
679 {
680 	struct intel_th_device *thdev;
681 	int src = 0, dst = 0;
682 
683 	for (src = 0, dst = 0; dst <= th->num_thdevs; src++, dst++) {
684 		for (; src < ARRAY_SIZE(intel_th_subdevices); src++) {
685 			if (intel_th_subdevices[src].type != INTEL_TH_OUTPUT)
686 				continue;
687 
688 			if (intel_th_subdevices[src].otype != otype)
689 				continue;
690 
691 			break;
692 		}
693 
694 		/* no unallocated matching subdevices */
695 		if (src == ARRAY_SIZE(intel_th_subdevices))
696 			return -ENODEV;
697 
698 		for (; dst < th->num_thdevs; dst++) {
699 			if (th->thdev[dst]->type != INTEL_TH_OUTPUT)
700 				continue;
701 
702 			if (th->thdev[dst]->output.type != otype)
703 				continue;
704 
705 			break;
706 		}
707 
708 		/*
709 		 * intel_th_subdevices[src] matches our requirements and is
710 		 * not matched in th::thdev[]
711 		 */
712 		if (dst == th->num_thdevs)
713 			goto found;
714 	}
715 
716 	return -ENODEV;
717 
718 found:
719 	thdev = intel_th_subdevice_alloc(th, &intel_th_subdevices[src]);
720 	if (IS_ERR(thdev))
721 		return PTR_ERR(thdev);
722 
723 	th->thdev[th->num_thdevs++] = thdev;
724 
725 	return 0;
726 }
727 EXPORT_SYMBOL_GPL(intel_th_output_enable);
728 
729 static int intel_th_populate(struct intel_th *th)
730 {
731 	int src;
732 
733 	/* create devices for each intel_th_subdevice */
734 	for (src = 0; src < ARRAY_SIZE(intel_th_subdevices); src++) {
735 		const struct intel_th_subdevice *subdev =
736 			&intel_th_subdevices[src];
737 		struct intel_th_device *thdev;
738 
739 		/* only allow SOURCE and SWITCH devices in host mode */
740 		if (host_mode && subdev->type == INTEL_TH_OUTPUT)
741 			continue;
742 
743 		/*
744 		 * don't enable port OUTPUTs in this path; SWITCH enables them
745 		 * via intel_th_output_enable()
746 		 */
747 		if (subdev->type == INTEL_TH_OUTPUT &&
748 		    subdev->otype != GTH_NONE)
749 			continue;
750 
751 		thdev = intel_th_subdevice_alloc(th, subdev);
752 		/* note: caller should free subdevices from th::thdev[] */
753 		if (IS_ERR(thdev))
754 			return PTR_ERR(thdev);
755 
756 		th->thdev[th->num_thdevs++] = thdev;
757 	}
758 
759 	return 0;
760 }
761 
762 static int match_devt(struct device *dev, void *data)
763 {
764 	dev_t devt = (dev_t)(unsigned long)data;
765 
766 	return dev->devt == devt;
767 }
768 
769 static int intel_th_output_open(struct inode *inode, struct file *file)
770 {
771 	const struct file_operations *fops;
772 	struct intel_th_driver *thdrv;
773 	struct device *dev;
774 	int err;
775 
776 	dev = bus_find_device(&intel_th_bus, NULL,
777 			      (void *)(unsigned long)inode->i_rdev,
778 			      match_devt);
779 	if (!dev || !dev->driver)
780 		return -ENODEV;
781 
782 	thdrv = to_intel_th_driver(dev->driver);
783 	fops = fops_get(thdrv->fops);
784 	if (!fops)
785 		return -ENODEV;
786 
787 	replace_fops(file, fops);
788 
789 	file->private_data = to_intel_th_device(dev);
790 
791 	if (file->f_op->open) {
792 		err = file->f_op->open(inode, file);
793 		return err;
794 	}
795 
796 	return 0;
797 }
798 
799 static const struct file_operations intel_th_output_fops = {
800 	.open	= intel_th_output_open,
801 	.llseek	= noop_llseek,
802 };
803 
804 /**
805  * intel_th_alloc() - allocate a new Intel TH device and its subdevices
806  * @dev:	parent device
807  * @devres:	parent's resources
808  * @ndevres:	number of resources
809  * @irq:	irq number
810  */
811 struct intel_th *
812 intel_th_alloc(struct device *dev, struct intel_th_drvdata *drvdata,
813 	       struct resource *devres, unsigned int ndevres, int irq)
814 {
815 	struct intel_th *th;
816 	int err;
817 
818 	th = kzalloc(sizeof(*th), GFP_KERNEL);
819 	if (!th)
820 		return ERR_PTR(-ENOMEM);
821 
822 	th->id = ida_simple_get(&intel_th_ida, 0, 0, GFP_KERNEL);
823 	if (th->id < 0) {
824 		err = th->id;
825 		goto err_alloc;
826 	}
827 
828 	th->major = __register_chrdev(0, 0, TH_POSSIBLE_OUTPUTS,
829 				      "intel_th/output", &intel_th_output_fops);
830 	if (th->major < 0) {
831 		err = th->major;
832 		goto err_ida;
833 	}
834 	th->dev = dev;
835 	th->drvdata = drvdata;
836 
837 	th->resource = devres;
838 	th->num_resources = ndevres;
839 	th->irq = irq;
840 
841 	dev_set_drvdata(dev, th);
842 
843 	pm_runtime_no_callbacks(dev);
844 	pm_runtime_put(dev);
845 	pm_runtime_allow(dev);
846 
847 	err = intel_th_populate(th);
848 	if (err) {
849 		/* free the subdevices and undo everything */
850 		intel_th_free(th);
851 		return ERR_PTR(err);
852 	}
853 
854 	return th;
855 
856 err_ida:
857 	ida_simple_remove(&intel_th_ida, th->id);
858 
859 err_alloc:
860 	kfree(th);
861 
862 	return ERR_PTR(err);
863 }
864 EXPORT_SYMBOL_GPL(intel_th_alloc);
865 
866 void intel_th_free(struct intel_th *th)
867 {
868 	int i;
869 
870 	intel_th_request_hub_module_flush(th);
871 
872 	intel_th_device_remove(th->hub);
873 	for (i = 0; i < th->num_thdevs; i++) {
874 		if (th->thdev[i] != th->hub)
875 			intel_th_device_remove(th->thdev[i]);
876 		th->thdev[i] = NULL;
877 	}
878 
879 	th->num_thdevs = 0;
880 
881 	pm_runtime_get_sync(th->dev);
882 	pm_runtime_forbid(th->dev);
883 
884 	__unregister_chrdev(th->major, 0, TH_POSSIBLE_OUTPUTS,
885 			    "intel_th/output");
886 
887 	ida_simple_remove(&intel_th_ida, th->id);
888 
889 	kfree(th);
890 }
891 EXPORT_SYMBOL_GPL(intel_th_free);
892 
893 /**
894  * intel_th_trace_enable() - enable tracing for an output device
895  * @thdev:	output device that requests tracing be enabled
896  */
897 int intel_th_trace_enable(struct intel_th_device *thdev)
898 {
899 	struct intel_th_device *hub = to_intel_th_device(thdev->dev.parent);
900 	struct intel_th_driver *hubdrv = to_intel_th_driver(hub->dev.driver);
901 
902 	if (WARN_ON_ONCE(hub->type != INTEL_TH_SWITCH))
903 		return -EINVAL;
904 
905 	if (WARN_ON_ONCE(thdev->type != INTEL_TH_OUTPUT))
906 		return -EINVAL;
907 
908 	pm_runtime_get_sync(&thdev->dev);
909 	hubdrv->enable(hub, &thdev->output);
910 
911 	return 0;
912 }
913 EXPORT_SYMBOL_GPL(intel_th_trace_enable);
914 
915 /**
916  * intel_th_trace_disable() - disable tracing for an output device
917  * @thdev:	output device that requests tracing be disabled
918  */
919 int intel_th_trace_disable(struct intel_th_device *thdev)
920 {
921 	struct intel_th_device *hub = to_intel_th_device(thdev->dev.parent);
922 	struct intel_th_driver *hubdrv = to_intel_th_driver(hub->dev.driver);
923 
924 	WARN_ON_ONCE(hub->type != INTEL_TH_SWITCH);
925 	if (WARN_ON_ONCE(thdev->type != INTEL_TH_OUTPUT))
926 		return -EINVAL;
927 
928 	hubdrv->disable(hub, &thdev->output);
929 	pm_runtime_put(&thdev->dev);
930 
931 	return 0;
932 }
933 EXPORT_SYMBOL_GPL(intel_th_trace_disable);
934 
935 int intel_th_set_output(struct intel_th_device *thdev,
936 			unsigned int master)
937 {
938 	struct intel_th_device *hub = to_intel_th_device(thdev->dev.parent);
939 	struct intel_th_driver *hubdrv = to_intel_th_driver(hub->dev.driver);
940 
941 	if (!hubdrv->set_output)
942 		return -ENOTSUPP;
943 
944 	return hubdrv->set_output(hub, master);
945 }
946 EXPORT_SYMBOL_GPL(intel_th_set_output);
947 
948 static int __init intel_th_init(void)
949 {
950 	intel_th_debug_init();
951 
952 	return bus_register(&intel_th_bus);
953 }
954 subsys_initcall(intel_th_init);
955 
956 static void __exit intel_th_exit(void)
957 {
958 	intel_th_debug_done();
959 
960 	bus_unregister(&intel_th_bus);
961 }
962 module_exit(intel_th_exit);
963 
964 MODULE_LICENSE("GPL v2");
965 MODULE_DESCRIPTION("Intel(R) Trace Hub controller driver");
966 MODULE_AUTHOR("Alexander Shishkin <alexander.shishkin@linux.intel.com>");
967