xref: /openbmc/linux/drivers/interconnect/core.c (revision 4507a8b9)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Interconnect framework core driver
4  *
5  * Copyright (c) 2017-2019, Linaro Ltd.
6  * Author: Georgi Djakov <georgi.djakov@linaro.org>
7  */
8 
9 #include <linux/debugfs.h>
10 #include <linux/device.h>
11 #include <linux/idr.h>
12 #include <linux/init.h>
13 #include <linux/interconnect.h>
14 #include <linux/interconnect-provider.h>
15 #include <linux/list.h>
16 #include <linux/mutex.h>
17 #include <linux/slab.h>
18 #include <linux/of.h>
19 #include <linux/overflow.h>
20 
21 #include "internal.h"
22 
23 #define CREATE_TRACE_POINTS
24 #include "trace.h"
25 
26 static DEFINE_IDR(icc_idr);
27 static LIST_HEAD(icc_providers);
28 static int providers_count;
29 static bool synced_state;
30 static DEFINE_MUTEX(icc_lock);
31 static DEFINE_MUTEX(icc_bw_lock);
32 static struct dentry *icc_debugfs_dir;
33 
34 static void icc_summary_show_one(struct seq_file *s, struct icc_node *n)
35 {
36 	if (!n)
37 		return;
38 
39 	seq_printf(s, "%-42s %12u %12u\n",
40 		   n->name, n->avg_bw, n->peak_bw);
41 }
42 
43 static int icc_summary_show(struct seq_file *s, void *data)
44 {
45 	struct icc_provider *provider;
46 
47 	seq_puts(s, " node                                  tag          avg         peak\n");
48 	seq_puts(s, "--------------------------------------------------------------------\n");
49 
50 	mutex_lock(&icc_lock);
51 
52 	list_for_each_entry(provider, &icc_providers, provider_list) {
53 		struct icc_node *n;
54 
55 		list_for_each_entry(n, &provider->nodes, node_list) {
56 			struct icc_req *r;
57 
58 			icc_summary_show_one(s, n);
59 			hlist_for_each_entry(r, &n->req_list, req_node) {
60 				u32 avg_bw = 0, peak_bw = 0;
61 
62 				if (!r->dev)
63 					continue;
64 
65 				if (r->enabled) {
66 					avg_bw = r->avg_bw;
67 					peak_bw = r->peak_bw;
68 				}
69 
70 				seq_printf(s, "  %-27s %12u %12u %12u\n",
71 					   dev_name(r->dev), r->tag, avg_bw, peak_bw);
72 			}
73 		}
74 	}
75 
76 	mutex_unlock(&icc_lock);
77 
78 	return 0;
79 }
80 DEFINE_SHOW_ATTRIBUTE(icc_summary);
81 
82 static void icc_graph_show_link(struct seq_file *s, int level,
83 				struct icc_node *n, struct icc_node *m)
84 {
85 	seq_printf(s, "%s\"%d:%s\" -> \"%d:%s\"\n",
86 		   level == 2 ? "\t\t" : "\t",
87 		   n->id, n->name, m->id, m->name);
88 }
89 
90 static void icc_graph_show_node(struct seq_file *s, struct icc_node *n)
91 {
92 	seq_printf(s, "\t\t\"%d:%s\" [label=\"%d:%s",
93 		   n->id, n->name, n->id, n->name);
94 	seq_printf(s, "\n\t\t\t|avg_bw=%ukBps", n->avg_bw);
95 	seq_printf(s, "\n\t\t\t|peak_bw=%ukBps", n->peak_bw);
96 	seq_puts(s, "\"]\n");
97 }
98 
99 static int icc_graph_show(struct seq_file *s, void *data)
100 {
101 	struct icc_provider *provider;
102 	struct icc_node *n;
103 	int cluster_index = 0;
104 	int i;
105 
106 	seq_puts(s, "digraph {\n\trankdir = LR\n\tnode [shape = record]\n");
107 	mutex_lock(&icc_lock);
108 
109 	/* draw providers as cluster subgraphs */
110 	cluster_index = 0;
111 	list_for_each_entry(provider, &icc_providers, provider_list) {
112 		seq_printf(s, "\tsubgraph cluster_%d {\n", ++cluster_index);
113 		if (provider->dev)
114 			seq_printf(s, "\t\tlabel = \"%s\"\n",
115 				   dev_name(provider->dev));
116 
117 		/* draw nodes */
118 		list_for_each_entry(n, &provider->nodes, node_list)
119 			icc_graph_show_node(s, n);
120 
121 		/* draw internal links */
122 		list_for_each_entry(n, &provider->nodes, node_list)
123 			for (i = 0; i < n->num_links; ++i)
124 				if (n->provider == n->links[i]->provider)
125 					icc_graph_show_link(s, 2, n,
126 							    n->links[i]);
127 
128 		seq_puts(s, "\t}\n");
129 	}
130 
131 	/* draw external links */
132 	list_for_each_entry(provider, &icc_providers, provider_list)
133 		list_for_each_entry(n, &provider->nodes, node_list)
134 			for (i = 0; i < n->num_links; ++i)
135 				if (n->provider != n->links[i]->provider)
136 					icc_graph_show_link(s, 1, n,
137 							    n->links[i]);
138 
139 	mutex_unlock(&icc_lock);
140 	seq_puts(s, "}");
141 
142 	return 0;
143 }
144 DEFINE_SHOW_ATTRIBUTE(icc_graph);
145 
146 static struct icc_node *node_find(const int id)
147 {
148 	return idr_find(&icc_idr, id);
149 }
150 
151 static struct icc_node *node_find_by_name(const char *name)
152 {
153 	struct icc_provider *provider;
154 	struct icc_node *n;
155 
156 	list_for_each_entry(provider, &icc_providers, provider_list) {
157 		list_for_each_entry(n, &provider->nodes, node_list) {
158 			if (!strcmp(n->name, name))
159 				return n;
160 		}
161 	}
162 
163 	return NULL;
164 }
165 
166 static struct icc_path *path_init(struct device *dev, struct icc_node *dst,
167 				  ssize_t num_nodes)
168 {
169 	struct icc_node *node = dst;
170 	struct icc_path *path;
171 	int i;
172 
173 	path = kzalloc(struct_size(path, reqs, num_nodes), GFP_KERNEL);
174 	if (!path)
175 		return ERR_PTR(-ENOMEM);
176 
177 	path->num_nodes = num_nodes;
178 
179 	mutex_lock(&icc_bw_lock);
180 
181 	for (i = num_nodes - 1; i >= 0; i--) {
182 		node->provider->users++;
183 		hlist_add_head(&path->reqs[i].req_node, &node->req_list);
184 		path->reqs[i].node = node;
185 		path->reqs[i].dev = dev;
186 		path->reqs[i].enabled = true;
187 		/* reference to previous node was saved during path traversal */
188 		node = node->reverse;
189 	}
190 
191 	mutex_unlock(&icc_bw_lock);
192 
193 	return path;
194 }
195 
196 static struct icc_path *path_find(struct device *dev, struct icc_node *src,
197 				  struct icc_node *dst)
198 {
199 	struct icc_path *path = ERR_PTR(-EPROBE_DEFER);
200 	struct icc_node *n, *node = NULL;
201 	struct list_head traverse_list;
202 	struct list_head edge_list;
203 	struct list_head visited_list;
204 	size_t i, depth = 1;
205 	bool found = false;
206 
207 	INIT_LIST_HEAD(&traverse_list);
208 	INIT_LIST_HEAD(&edge_list);
209 	INIT_LIST_HEAD(&visited_list);
210 
211 	list_add(&src->search_list, &traverse_list);
212 	src->reverse = NULL;
213 
214 	do {
215 		list_for_each_entry_safe(node, n, &traverse_list, search_list) {
216 			if (node == dst) {
217 				found = true;
218 				list_splice_init(&edge_list, &visited_list);
219 				list_splice_init(&traverse_list, &visited_list);
220 				break;
221 			}
222 			for (i = 0; i < node->num_links; i++) {
223 				struct icc_node *tmp = node->links[i];
224 
225 				if (!tmp) {
226 					path = ERR_PTR(-ENOENT);
227 					goto out;
228 				}
229 
230 				if (tmp->is_traversed)
231 					continue;
232 
233 				tmp->is_traversed = true;
234 				tmp->reverse = node;
235 				list_add_tail(&tmp->search_list, &edge_list);
236 			}
237 		}
238 
239 		if (found)
240 			break;
241 
242 		list_splice_init(&traverse_list, &visited_list);
243 		list_splice_init(&edge_list, &traverse_list);
244 
245 		/* count the hops including the source */
246 		depth++;
247 
248 	} while (!list_empty(&traverse_list));
249 
250 out:
251 
252 	/* reset the traversed state */
253 	list_for_each_entry_reverse(n, &visited_list, search_list)
254 		n->is_traversed = false;
255 
256 	if (found)
257 		path = path_init(dev, dst, depth);
258 
259 	return path;
260 }
261 
262 /*
263  * We want the path to honor all bandwidth requests, so the average and peak
264  * bandwidth requirements from each consumer are aggregated at each node.
265  * The aggregation is platform specific, so each platform can customize it by
266  * implementing its own aggregate() function.
267  */
268 
269 static int aggregate_requests(struct icc_node *node)
270 {
271 	struct icc_provider *p = node->provider;
272 	struct icc_req *r;
273 	u32 avg_bw, peak_bw;
274 
275 	node->avg_bw = 0;
276 	node->peak_bw = 0;
277 
278 	if (p->pre_aggregate)
279 		p->pre_aggregate(node);
280 
281 	hlist_for_each_entry(r, &node->req_list, req_node) {
282 		if (r->enabled) {
283 			avg_bw = r->avg_bw;
284 			peak_bw = r->peak_bw;
285 		} else {
286 			avg_bw = 0;
287 			peak_bw = 0;
288 		}
289 		p->aggregate(node, r->tag, avg_bw, peak_bw,
290 			     &node->avg_bw, &node->peak_bw);
291 
292 		/* during boot use the initial bandwidth as a floor value */
293 		if (!synced_state) {
294 			node->avg_bw = max(node->avg_bw, node->init_avg);
295 			node->peak_bw = max(node->peak_bw, node->init_peak);
296 		}
297 	}
298 
299 	return 0;
300 }
301 
302 static int apply_constraints(struct icc_path *path)
303 {
304 	struct icc_node *next, *prev = NULL;
305 	struct icc_provider *p;
306 	int ret = -EINVAL;
307 	int i;
308 
309 	for (i = 0; i < path->num_nodes; i++) {
310 		next = path->reqs[i].node;
311 		p = next->provider;
312 
313 		/* both endpoints should be valid master-slave pairs */
314 		if (!prev || (p != prev->provider && !p->inter_set)) {
315 			prev = next;
316 			continue;
317 		}
318 
319 		/* set the constraints */
320 		ret = p->set(prev, next);
321 		if (ret)
322 			goto out;
323 
324 		prev = next;
325 	}
326 out:
327 	return ret;
328 }
329 
330 int icc_std_aggregate(struct icc_node *node, u32 tag, u32 avg_bw,
331 		      u32 peak_bw, u32 *agg_avg, u32 *agg_peak)
332 {
333 	*agg_avg += avg_bw;
334 	*agg_peak = max(*agg_peak, peak_bw);
335 
336 	return 0;
337 }
338 EXPORT_SYMBOL_GPL(icc_std_aggregate);
339 
340 /* of_icc_xlate_onecell() - Translate function using a single index.
341  * @spec: OF phandle args to map into an interconnect node.
342  * @data: private data (pointer to struct icc_onecell_data)
343  *
344  * This is a generic translate function that can be used to model simple
345  * interconnect providers that have one device tree node and provide
346  * multiple interconnect nodes. A single cell is used as an index into
347  * an array of icc nodes specified in the icc_onecell_data struct when
348  * registering the provider.
349  */
350 struct icc_node *of_icc_xlate_onecell(struct of_phandle_args *spec,
351 				      void *data)
352 {
353 	struct icc_onecell_data *icc_data = data;
354 	unsigned int idx = spec->args[0];
355 
356 	if (idx >= icc_data->num_nodes) {
357 		pr_err("%s: invalid index %u\n", __func__, idx);
358 		return ERR_PTR(-EINVAL);
359 	}
360 
361 	return icc_data->nodes[idx];
362 }
363 EXPORT_SYMBOL_GPL(of_icc_xlate_onecell);
364 
365 /**
366  * of_icc_get_from_provider() - Look-up interconnect node
367  * @spec: OF phandle args to use for look-up
368  *
369  * Looks for interconnect provider under the node specified by @spec and if
370  * found, uses xlate function of the provider to map phandle args to node.
371  *
372  * Returns a valid pointer to struct icc_node_data on success or ERR_PTR()
373  * on failure.
374  */
375 struct icc_node_data *of_icc_get_from_provider(struct of_phandle_args *spec)
376 {
377 	struct icc_node *node = ERR_PTR(-EPROBE_DEFER);
378 	struct icc_node_data *data = NULL;
379 	struct icc_provider *provider;
380 
381 	if (!spec)
382 		return ERR_PTR(-EINVAL);
383 
384 	mutex_lock(&icc_lock);
385 	list_for_each_entry(provider, &icc_providers, provider_list) {
386 		if (provider->dev->of_node == spec->np) {
387 			if (provider->xlate_extended) {
388 				data = provider->xlate_extended(spec, provider->data);
389 				if (!IS_ERR(data)) {
390 					node = data->node;
391 					break;
392 				}
393 			} else {
394 				node = provider->xlate(spec, provider->data);
395 				if (!IS_ERR(node))
396 					break;
397 			}
398 		}
399 	}
400 	mutex_unlock(&icc_lock);
401 
402 	if (!node)
403 		return ERR_PTR(-EINVAL);
404 
405 	if (IS_ERR(node))
406 		return ERR_CAST(node);
407 
408 	if (!data) {
409 		data = kzalloc(sizeof(*data), GFP_KERNEL);
410 		if (!data)
411 			return ERR_PTR(-ENOMEM);
412 		data->node = node;
413 	}
414 
415 	return data;
416 }
417 EXPORT_SYMBOL_GPL(of_icc_get_from_provider);
418 
419 static void devm_icc_release(struct device *dev, void *res)
420 {
421 	icc_put(*(struct icc_path **)res);
422 }
423 
424 struct icc_path *devm_of_icc_get(struct device *dev, const char *name)
425 {
426 	struct icc_path **ptr, *path;
427 
428 	ptr = devres_alloc(devm_icc_release, sizeof(*ptr), GFP_KERNEL);
429 	if (!ptr)
430 		return ERR_PTR(-ENOMEM);
431 
432 	path = of_icc_get(dev, name);
433 	if (!IS_ERR(path)) {
434 		*ptr = path;
435 		devres_add(dev, ptr);
436 	} else {
437 		devres_free(ptr);
438 	}
439 
440 	return path;
441 }
442 EXPORT_SYMBOL_GPL(devm_of_icc_get);
443 
444 /**
445  * of_icc_get_by_index() - get a path handle from a DT node based on index
446  * @dev: device pointer for the consumer device
447  * @idx: interconnect path index
448  *
449  * This function will search for a path between two endpoints and return an
450  * icc_path handle on success. Use icc_put() to release constraints when they
451  * are not needed anymore.
452  * If the interconnect API is disabled, NULL is returned and the consumer
453  * drivers will still build. Drivers are free to handle this specifically,
454  * but they don't have to.
455  *
456  * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
457  * when the API is disabled or the "interconnects" DT property is missing.
458  */
459 struct icc_path *of_icc_get_by_index(struct device *dev, int idx)
460 {
461 	struct icc_path *path;
462 	struct icc_node_data *src_data, *dst_data;
463 	struct device_node *np;
464 	struct of_phandle_args src_args, dst_args;
465 	int ret;
466 
467 	if (!dev || !dev->of_node)
468 		return ERR_PTR(-ENODEV);
469 
470 	np = dev->of_node;
471 
472 	/*
473 	 * When the consumer DT node do not have "interconnects" property
474 	 * return a NULL path to skip setting constraints.
475 	 */
476 	if (!of_property_present(np, "interconnects"))
477 		return NULL;
478 
479 	/*
480 	 * We use a combination of phandle and specifier for endpoint. For now
481 	 * lets support only global ids and extend this in the future if needed
482 	 * without breaking DT compatibility.
483 	 */
484 	ret = of_parse_phandle_with_args(np, "interconnects",
485 					 "#interconnect-cells", idx * 2,
486 					 &src_args);
487 	if (ret)
488 		return ERR_PTR(ret);
489 
490 	of_node_put(src_args.np);
491 
492 	ret = of_parse_phandle_with_args(np, "interconnects",
493 					 "#interconnect-cells", idx * 2 + 1,
494 					 &dst_args);
495 	if (ret)
496 		return ERR_PTR(ret);
497 
498 	of_node_put(dst_args.np);
499 
500 	src_data = of_icc_get_from_provider(&src_args);
501 
502 	if (IS_ERR(src_data)) {
503 		dev_err_probe(dev, PTR_ERR(src_data), "error finding src node\n");
504 		return ERR_CAST(src_data);
505 	}
506 
507 	dst_data = of_icc_get_from_provider(&dst_args);
508 
509 	if (IS_ERR(dst_data)) {
510 		dev_err_probe(dev, PTR_ERR(dst_data), "error finding dst node\n");
511 		kfree(src_data);
512 		return ERR_CAST(dst_data);
513 	}
514 
515 	mutex_lock(&icc_lock);
516 	path = path_find(dev, src_data->node, dst_data->node);
517 	mutex_unlock(&icc_lock);
518 	if (IS_ERR(path)) {
519 		dev_err(dev, "%s: invalid path=%ld\n", __func__, PTR_ERR(path));
520 		goto free_icc_data;
521 	}
522 
523 	if (src_data->tag && src_data->tag == dst_data->tag)
524 		icc_set_tag(path, src_data->tag);
525 
526 	path->name = kasprintf(GFP_KERNEL, "%s-%s",
527 			       src_data->node->name, dst_data->node->name);
528 	if (!path->name) {
529 		kfree(path);
530 		path = ERR_PTR(-ENOMEM);
531 	}
532 
533 free_icc_data:
534 	kfree(src_data);
535 	kfree(dst_data);
536 	return path;
537 }
538 EXPORT_SYMBOL_GPL(of_icc_get_by_index);
539 
540 /**
541  * of_icc_get() - get a path handle from a DT node based on name
542  * @dev: device pointer for the consumer device
543  * @name: interconnect path name
544  *
545  * This function will search for a path between two endpoints and return an
546  * icc_path handle on success. Use icc_put() to release constraints when they
547  * are not needed anymore.
548  * If the interconnect API is disabled, NULL is returned and the consumer
549  * drivers will still build. Drivers are free to handle this specifically,
550  * but they don't have to.
551  *
552  * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
553  * when the API is disabled or the "interconnects" DT property is missing.
554  */
555 struct icc_path *of_icc_get(struct device *dev, const char *name)
556 {
557 	struct device_node *np;
558 	int idx = 0;
559 
560 	if (!dev || !dev->of_node)
561 		return ERR_PTR(-ENODEV);
562 
563 	np = dev->of_node;
564 
565 	/*
566 	 * When the consumer DT node do not have "interconnects" property
567 	 * return a NULL path to skip setting constraints.
568 	 */
569 	if (!of_property_present(np, "interconnects"))
570 		return NULL;
571 
572 	/*
573 	 * We use a combination of phandle and specifier for endpoint. For now
574 	 * lets support only global ids and extend this in the future if needed
575 	 * without breaking DT compatibility.
576 	 */
577 	if (name) {
578 		idx = of_property_match_string(np, "interconnect-names", name);
579 		if (idx < 0)
580 			return ERR_PTR(idx);
581 	}
582 
583 	return of_icc_get_by_index(dev, idx);
584 }
585 EXPORT_SYMBOL_GPL(of_icc_get);
586 
587 /**
588  * icc_get() - get a path handle between two endpoints
589  * @dev: device pointer for the consumer device
590  * @src: source node name
591  * @dst: destination node name
592  *
593  * This function will search for a path between two endpoints and return an
594  * icc_path handle on success. Use icc_put() to release constraints when they
595  * are not needed anymore.
596  *
597  * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
598  * when the API is disabled.
599  */
600 struct icc_path *icc_get(struct device *dev, const char *src, const char *dst)
601 {
602 	struct icc_node *src_node, *dst_node;
603 	struct icc_path *path = ERR_PTR(-EPROBE_DEFER);
604 
605 	mutex_lock(&icc_lock);
606 
607 	src_node = node_find_by_name(src);
608 	if (!src_node) {
609 		dev_err(dev, "%s: invalid src=%s\n", __func__, src);
610 		goto out;
611 	}
612 
613 	dst_node = node_find_by_name(dst);
614 	if (!dst_node) {
615 		dev_err(dev, "%s: invalid dst=%s\n", __func__, dst);
616 		goto out;
617 	}
618 
619 	path = path_find(dev, src_node, dst_node);
620 	if (IS_ERR(path)) {
621 		dev_err(dev, "%s: invalid path=%ld\n", __func__, PTR_ERR(path));
622 		goto out;
623 	}
624 
625 	path->name = kasprintf(GFP_KERNEL, "%s-%s", src_node->name, dst_node->name);
626 	if (!path->name) {
627 		kfree(path);
628 		path = ERR_PTR(-ENOMEM);
629 	}
630 out:
631 	mutex_unlock(&icc_lock);
632 	return path;
633 }
634 
635 /**
636  * icc_set_tag() - set an optional tag on a path
637  * @path: the path we want to tag
638  * @tag: the tag value
639  *
640  * This function allows consumers to append a tag to the requests associated
641  * with a path, so that a different aggregation could be done based on this tag.
642  */
643 void icc_set_tag(struct icc_path *path, u32 tag)
644 {
645 	int i;
646 
647 	if (!path)
648 		return;
649 
650 	mutex_lock(&icc_lock);
651 
652 	for (i = 0; i < path->num_nodes; i++)
653 		path->reqs[i].tag = tag;
654 
655 	mutex_unlock(&icc_lock);
656 }
657 EXPORT_SYMBOL_GPL(icc_set_tag);
658 
659 /**
660  * icc_get_name() - Get name of the icc path
661  * @path: interconnect path
662  *
663  * This function is used by an interconnect consumer to get the name of the icc
664  * path.
665  *
666  * Returns a valid pointer on success, or NULL otherwise.
667  */
668 const char *icc_get_name(struct icc_path *path)
669 {
670 	if (!path)
671 		return NULL;
672 
673 	return path->name;
674 }
675 EXPORT_SYMBOL_GPL(icc_get_name);
676 
677 /**
678  * icc_set_bw() - set bandwidth constraints on an interconnect path
679  * @path: interconnect path
680  * @avg_bw: average bandwidth in kilobytes per second
681  * @peak_bw: peak bandwidth in kilobytes per second
682  *
683  * This function is used by an interconnect consumer to express its own needs
684  * in terms of bandwidth for a previously requested path between two endpoints.
685  * The requests are aggregated and each node is updated accordingly. The entire
686  * path is locked by a mutex to ensure that the set() is completed.
687  * The @path can be NULL when the "interconnects" DT properties is missing,
688  * which will mean that no constraints will be set.
689  *
690  * Returns 0 on success, or an appropriate error code otherwise.
691  */
692 int icc_set_bw(struct icc_path *path, u32 avg_bw, u32 peak_bw)
693 {
694 	struct icc_node *node;
695 	u32 old_avg, old_peak;
696 	size_t i;
697 	int ret;
698 
699 	if (!path)
700 		return 0;
701 
702 	if (WARN_ON(IS_ERR(path) || !path->num_nodes))
703 		return -EINVAL;
704 
705 	mutex_lock(&icc_bw_lock);
706 
707 	old_avg = path->reqs[0].avg_bw;
708 	old_peak = path->reqs[0].peak_bw;
709 
710 	for (i = 0; i < path->num_nodes; i++) {
711 		node = path->reqs[i].node;
712 
713 		/* update the consumer request for this path */
714 		path->reqs[i].avg_bw = avg_bw;
715 		path->reqs[i].peak_bw = peak_bw;
716 
717 		/* aggregate requests for this node */
718 		aggregate_requests(node);
719 
720 		trace_icc_set_bw(path, node, i, avg_bw, peak_bw);
721 	}
722 
723 	ret = apply_constraints(path);
724 	if (ret) {
725 		pr_debug("interconnect: error applying constraints (%d)\n",
726 			 ret);
727 
728 		for (i = 0; i < path->num_nodes; i++) {
729 			node = path->reqs[i].node;
730 			path->reqs[i].avg_bw = old_avg;
731 			path->reqs[i].peak_bw = old_peak;
732 			aggregate_requests(node);
733 		}
734 		apply_constraints(path);
735 	}
736 
737 	mutex_unlock(&icc_bw_lock);
738 
739 	trace_icc_set_bw_end(path, ret);
740 
741 	return ret;
742 }
743 EXPORT_SYMBOL_GPL(icc_set_bw);
744 
745 static int __icc_enable(struct icc_path *path, bool enable)
746 {
747 	int i;
748 
749 	if (!path)
750 		return 0;
751 
752 	if (WARN_ON(IS_ERR(path) || !path->num_nodes))
753 		return -EINVAL;
754 
755 	mutex_lock(&icc_lock);
756 
757 	for (i = 0; i < path->num_nodes; i++)
758 		path->reqs[i].enabled = enable;
759 
760 	mutex_unlock(&icc_lock);
761 
762 	return icc_set_bw(path, path->reqs[0].avg_bw,
763 			  path->reqs[0].peak_bw);
764 }
765 
766 int icc_enable(struct icc_path *path)
767 {
768 	return __icc_enable(path, true);
769 }
770 EXPORT_SYMBOL_GPL(icc_enable);
771 
772 int icc_disable(struct icc_path *path)
773 {
774 	return __icc_enable(path, false);
775 }
776 EXPORT_SYMBOL_GPL(icc_disable);
777 
778 /**
779  * icc_put() - release the reference to the icc_path
780  * @path: interconnect path
781  *
782  * Use this function to release the constraints on a path when the path is
783  * no longer needed. The constraints will be re-aggregated.
784  */
785 void icc_put(struct icc_path *path)
786 {
787 	struct icc_node *node;
788 	size_t i;
789 	int ret;
790 
791 	if (!path || WARN_ON(IS_ERR(path)))
792 		return;
793 
794 	ret = icc_set_bw(path, 0, 0);
795 	if (ret)
796 		pr_err("%s: error (%d)\n", __func__, ret);
797 
798 	mutex_lock(&icc_lock);
799 	mutex_lock(&icc_bw_lock);
800 
801 	for (i = 0; i < path->num_nodes; i++) {
802 		node = path->reqs[i].node;
803 		hlist_del(&path->reqs[i].req_node);
804 		if (!WARN_ON(!node->provider->users))
805 			node->provider->users--;
806 	}
807 
808 	mutex_unlock(&icc_bw_lock);
809 	mutex_unlock(&icc_lock);
810 
811 	kfree_const(path->name);
812 	kfree(path);
813 }
814 EXPORT_SYMBOL_GPL(icc_put);
815 
816 static struct icc_node *icc_node_create_nolock(int id)
817 {
818 	struct icc_node *node;
819 
820 	/* check if node already exists */
821 	node = node_find(id);
822 	if (node)
823 		return node;
824 
825 	node = kzalloc(sizeof(*node), GFP_KERNEL);
826 	if (!node)
827 		return ERR_PTR(-ENOMEM);
828 
829 	id = idr_alloc(&icc_idr, node, id, id + 1, GFP_KERNEL);
830 	if (id < 0) {
831 		WARN(1, "%s: couldn't get idr\n", __func__);
832 		kfree(node);
833 		return ERR_PTR(id);
834 	}
835 
836 	node->id = id;
837 
838 	return node;
839 }
840 
841 /**
842  * icc_node_create() - create a node
843  * @id: node id
844  *
845  * Return: icc_node pointer on success, or ERR_PTR() on error
846  */
847 struct icc_node *icc_node_create(int id)
848 {
849 	struct icc_node *node;
850 
851 	mutex_lock(&icc_lock);
852 
853 	node = icc_node_create_nolock(id);
854 
855 	mutex_unlock(&icc_lock);
856 
857 	return node;
858 }
859 EXPORT_SYMBOL_GPL(icc_node_create);
860 
861 /**
862  * icc_node_destroy() - destroy a node
863  * @id: node id
864  */
865 void icc_node_destroy(int id)
866 {
867 	struct icc_node *node;
868 
869 	mutex_lock(&icc_lock);
870 
871 	node = node_find(id);
872 	if (node) {
873 		idr_remove(&icc_idr, node->id);
874 		WARN_ON(!hlist_empty(&node->req_list));
875 	}
876 
877 	mutex_unlock(&icc_lock);
878 
879 	if (!node)
880 		return;
881 
882 	kfree(node->links);
883 	kfree(node);
884 }
885 EXPORT_SYMBOL_GPL(icc_node_destroy);
886 
887 /**
888  * icc_link_create() - create a link between two nodes
889  * @node: source node id
890  * @dst_id: destination node id
891  *
892  * Create a link between two nodes. The nodes might belong to different
893  * interconnect providers and the @dst_id node might not exist (if the
894  * provider driver has not probed yet). So just create the @dst_id node
895  * and when the actual provider driver is probed, the rest of the node
896  * data is filled.
897  *
898  * Return: 0 on success, or an error code otherwise
899  */
900 int icc_link_create(struct icc_node *node, const int dst_id)
901 {
902 	struct icc_node *dst;
903 	struct icc_node **new;
904 	int ret = 0;
905 
906 	if (!node->provider)
907 		return -EINVAL;
908 
909 	mutex_lock(&icc_lock);
910 
911 	dst = node_find(dst_id);
912 	if (!dst) {
913 		dst = icc_node_create_nolock(dst_id);
914 
915 		if (IS_ERR(dst)) {
916 			ret = PTR_ERR(dst);
917 			goto out;
918 		}
919 	}
920 
921 	new = krealloc(node->links,
922 		       (node->num_links + 1) * sizeof(*node->links),
923 		       GFP_KERNEL);
924 	if (!new) {
925 		ret = -ENOMEM;
926 		goto out;
927 	}
928 
929 	node->links = new;
930 	node->links[node->num_links++] = dst;
931 
932 out:
933 	mutex_unlock(&icc_lock);
934 
935 	return ret;
936 }
937 EXPORT_SYMBOL_GPL(icc_link_create);
938 
939 /**
940  * icc_node_add() - add interconnect node to interconnect provider
941  * @node: pointer to the interconnect node
942  * @provider: pointer to the interconnect provider
943  */
944 void icc_node_add(struct icc_node *node, struct icc_provider *provider)
945 {
946 	if (WARN_ON(node->provider))
947 		return;
948 
949 	mutex_lock(&icc_lock);
950 	mutex_lock(&icc_bw_lock);
951 
952 	node->provider = provider;
953 	list_add_tail(&node->node_list, &provider->nodes);
954 
955 	/* get the initial bandwidth values and sync them with hardware */
956 	if (provider->get_bw) {
957 		provider->get_bw(node, &node->init_avg, &node->init_peak);
958 	} else {
959 		node->init_avg = INT_MAX;
960 		node->init_peak = INT_MAX;
961 	}
962 	node->avg_bw = node->init_avg;
963 	node->peak_bw = node->init_peak;
964 
965 	if (node->avg_bw || node->peak_bw) {
966 		if (provider->pre_aggregate)
967 			provider->pre_aggregate(node);
968 
969 		if (provider->aggregate)
970 			provider->aggregate(node, 0, node->init_avg, node->init_peak,
971 					    &node->avg_bw, &node->peak_bw);
972 		if (provider->set)
973 			provider->set(node, node);
974 	}
975 
976 	node->avg_bw = 0;
977 	node->peak_bw = 0;
978 
979 	mutex_unlock(&icc_bw_lock);
980 	mutex_unlock(&icc_lock);
981 }
982 EXPORT_SYMBOL_GPL(icc_node_add);
983 
984 /**
985  * icc_node_del() - delete interconnect node from interconnect provider
986  * @node: pointer to the interconnect node
987  */
988 void icc_node_del(struct icc_node *node)
989 {
990 	mutex_lock(&icc_lock);
991 
992 	list_del(&node->node_list);
993 
994 	mutex_unlock(&icc_lock);
995 }
996 EXPORT_SYMBOL_GPL(icc_node_del);
997 
998 /**
999  * icc_nodes_remove() - remove all previously added nodes from provider
1000  * @provider: the interconnect provider we are removing nodes from
1001  *
1002  * Return: 0 on success, or an error code otherwise
1003  */
1004 int icc_nodes_remove(struct icc_provider *provider)
1005 {
1006 	struct icc_node *n, *tmp;
1007 
1008 	if (WARN_ON(IS_ERR_OR_NULL(provider)))
1009 		return -EINVAL;
1010 
1011 	list_for_each_entry_safe_reverse(n, tmp, &provider->nodes, node_list) {
1012 		icc_node_del(n);
1013 		icc_node_destroy(n->id);
1014 	}
1015 
1016 	return 0;
1017 }
1018 EXPORT_SYMBOL_GPL(icc_nodes_remove);
1019 
1020 /**
1021  * icc_provider_init() - initialize a new interconnect provider
1022  * @provider: the interconnect provider to initialize
1023  *
1024  * Must be called before adding nodes to the provider.
1025  */
1026 void icc_provider_init(struct icc_provider *provider)
1027 {
1028 	WARN_ON(!provider->set);
1029 
1030 	INIT_LIST_HEAD(&provider->nodes);
1031 }
1032 EXPORT_SYMBOL_GPL(icc_provider_init);
1033 
1034 /**
1035  * icc_provider_register() - register a new interconnect provider
1036  * @provider: the interconnect provider to register
1037  *
1038  * Return: 0 on success, or an error code otherwise
1039  */
1040 int icc_provider_register(struct icc_provider *provider)
1041 {
1042 	if (WARN_ON(!provider->xlate && !provider->xlate_extended))
1043 		return -EINVAL;
1044 
1045 	mutex_lock(&icc_lock);
1046 	list_add_tail(&provider->provider_list, &icc_providers);
1047 	mutex_unlock(&icc_lock);
1048 
1049 	dev_dbg(provider->dev, "interconnect provider registered\n");
1050 
1051 	return 0;
1052 }
1053 EXPORT_SYMBOL_GPL(icc_provider_register);
1054 
1055 /**
1056  * icc_provider_deregister() - deregister an interconnect provider
1057  * @provider: the interconnect provider to deregister
1058  */
1059 void icc_provider_deregister(struct icc_provider *provider)
1060 {
1061 	mutex_lock(&icc_lock);
1062 	WARN_ON(provider->users);
1063 
1064 	list_del(&provider->provider_list);
1065 	mutex_unlock(&icc_lock);
1066 }
1067 EXPORT_SYMBOL_GPL(icc_provider_deregister);
1068 
1069 static const struct of_device_id __maybe_unused ignore_list[] = {
1070 	{ .compatible = "qcom,sc7180-ipa-virt" },
1071 	{ .compatible = "qcom,sc8180x-ipa-virt" },
1072 	{ .compatible = "qcom,sdx55-ipa-virt" },
1073 	{ .compatible = "qcom,sm8150-ipa-virt" },
1074 	{ .compatible = "qcom,sm8250-ipa-virt" },
1075 	{}
1076 };
1077 
1078 static int of_count_icc_providers(struct device_node *np)
1079 {
1080 	struct device_node *child;
1081 	int count = 0;
1082 
1083 	for_each_available_child_of_node(np, child) {
1084 		if (of_property_read_bool(child, "#interconnect-cells") &&
1085 		    likely(!of_match_node(ignore_list, child)))
1086 			count++;
1087 		count += of_count_icc_providers(child);
1088 	}
1089 
1090 	return count;
1091 }
1092 
1093 void icc_sync_state(struct device *dev)
1094 {
1095 	struct icc_provider *p;
1096 	struct icc_node *n;
1097 	static int count;
1098 
1099 	count++;
1100 
1101 	if (count < providers_count)
1102 		return;
1103 
1104 	mutex_lock(&icc_lock);
1105 	mutex_lock(&icc_bw_lock);
1106 	synced_state = true;
1107 	list_for_each_entry(p, &icc_providers, provider_list) {
1108 		dev_dbg(p->dev, "interconnect provider is in synced state\n");
1109 		list_for_each_entry(n, &p->nodes, node_list) {
1110 			if (n->init_avg || n->init_peak) {
1111 				n->init_avg = 0;
1112 				n->init_peak = 0;
1113 				aggregate_requests(n);
1114 				p->set(n, n);
1115 			}
1116 		}
1117 	}
1118 	mutex_unlock(&icc_bw_lock);
1119 	mutex_unlock(&icc_lock);
1120 }
1121 EXPORT_SYMBOL_GPL(icc_sync_state);
1122 
1123 static int __init icc_init(void)
1124 {
1125 	struct device_node *root;
1126 
1127 	/* Teach lockdep about lock ordering wrt. shrinker: */
1128 	fs_reclaim_acquire(GFP_KERNEL);
1129 	might_lock(&icc_bw_lock);
1130 	fs_reclaim_release(GFP_KERNEL);
1131 
1132 	root = of_find_node_by_path("/");
1133 
1134 	providers_count = of_count_icc_providers(root);
1135 	of_node_put(root);
1136 
1137 	icc_debugfs_dir = debugfs_create_dir("interconnect", NULL);
1138 	debugfs_create_file("interconnect_summary", 0444,
1139 			    icc_debugfs_dir, NULL, &icc_summary_fops);
1140 	debugfs_create_file("interconnect_graph", 0444,
1141 			    icc_debugfs_dir, NULL, &icc_graph_fops);
1142 
1143 	icc_debugfs_client_init(icc_debugfs_dir);
1144 
1145 	return 0;
1146 }
1147 
1148 device_initcall(icc_init);
1149