Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Interconnect framework core driver
4 *
5 * Copyright (c) 2017-2019, Linaro Ltd.
6 * Author: Georgi Djakov <georgi.djakov@linaro.org>
7 */
8
9#include <linux/debugfs.h>
10#include <linux/device.h>
11#include <linux/idr.h>
12#include <linux/init.h>
13#include <linux/interconnect.h>
14#include <linux/interconnect-provider.h>
15#include <linux/list.h>
16#include <linux/mutex.h>
17#include <linux/slab.h>
18#include <linux/of.h>
19#include <linux/overflow.h>
20
21#include "internal.h"
22
23#define CREATE_TRACE_POINTS
24#include "trace.h"
25
26static DEFINE_IDR(icc_idr);
27static LIST_HEAD(icc_providers);
28static int providers_count;
29static bool synced_state;
30static DEFINE_MUTEX(icc_lock);
31static DEFINE_MUTEX(icc_bw_lock);
32static struct dentry *icc_debugfs_dir;
33
34static void icc_summary_show_one(struct seq_file *s, struct icc_node *n)
35{
36 if (!n)
37 return;
38
39 seq_printf(s, "%-42s %12u %12u\n",
40 n->name, n->avg_bw, n->peak_bw);
41}
42
43static int icc_summary_show(struct seq_file *s, void *data)
44{
45 struct icc_provider *provider;
46
47 seq_puts(s, " node tag avg peak\n");
48 seq_puts(s, "--------------------------------------------------------------------\n");
49
50 mutex_lock(&icc_lock);
51
52 list_for_each_entry(provider, &icc_providers, provider_list) {
53 struct icc_node *n;
54
55 list_for_each_entry(n, &provider->nodes, node_list) {
56 struct icc_req *r;
57
58 icc_summary_show_one(s, n);
59 hlist_for_each_entry(r, &n->req_list, req_node) {
60 u32 avg_bw = 0, peak_bw = 0;
61
62 if (!r->dev)
63 continue;
64
65 if (r->enabled) {
66 avg_bw = r->avg_bw;
67 peak_bw = r->peak_bw;
68 }
69
70 seq_printf(s, " %-27s %12u %12u %12u\n",
71 dev_name(r->dev), r->tag, avg_bw, peak_bw);
72 }
73 }
74 }
75
76 mutex_unlock(&icc_lock);
77
78 return 0;
79}
80DEFINE_SHOW_ATTRIBUTE(icc_summary);
81
82static void icc_graph_show_link(struct seq_file *s, int level,
83 struct icc_node *n, struct icc_node *m)
84{
85 seq_printf(s, "%s\"%d:%s\" -> \"%d:%s\"\n",
86 level == 2 ? "\t\t" : "\t",
87 n->id, n->name, m->id, m->name);
88}
89
90static void icc_graph_show_node(struct seq_file *s, struct icc_node *n)
91{
92 seq_printf(s, "\t\t\"%d:%s\" [label=\"%d:%s",
93 n->id, n->name, n->id, n->name);
94 seq_printf(s, "\n\t\t\t|avg_bw=%ukBps", n->avg_bw);
95 seq_printf(s, "\n\t\t\t|peak_bw=%ukBps", n->peak_bw);
96 seq_puts(s, "\"]\n");
97}
98
99static int icc_graph_show(struct seq_file *s, void *data)
100{
101 struct icc_provider *provider;
102 struct icc_node *n;
103 int cluster_index = 0;
104 int i;
105
106 seq_puts(s, "digraph {\n\trankdir = LR\n\tnode [shape = record]\n");
107 mutex_lock(&icc_lock);
108
109 /* draw providers as cluster subgraphs */
110 cluster_index = 0;
111 list_for_each_entry(provider, &icc_providers, provider_list) {
112 seq_printf(s, "\tsubgraph cluster_%d {\n", ++cluster_index);
113 if (provider->dev)
114 seq_printf(s, "\t\tlabel = \"%s\"\n",
115 dev_name(provider->dev));
116
117 /* draw nodes */
118 list_for_each_entry(n, &provider->nodes, node_list)
119 icc_graph_show_node(s, n);
120
121 /* draw internal links */
122 list_for_each_entry(n, &provider->nodes, node_list)
123 for (i = 0; i < n->num_links; ++i)
124 if (n->provider == n->links[i]->provider)
125 icc_graph_show_link(s, 2, n,
126 n->links[i]);
127
128 seq_puts(s, "\t}\n");
129 }
130
131 /* draw external links */
132 list_for_each_entry(provider, &icc_providers, provider_list)
133 list_for_each_entry(n, &provider->nodes, node_list)
134 for (i = 0; i < n->num_links; ++i)
135 if (n->provider != n->links[i]->provider)
136 icc_graph_show_link(s, 1, n,
137 n->links[i]);
138
139 mutex_unlock(&icc_lock);
140 seq_puts(s, "}");
141
142 return 0;
143}
144DEFINE_SHOW_ATTRIBUTE(icc_graph);
145
146static struct icc_node *node_find(const int id)
147{
148 return idr_find(&icc_idr, id);
149}
150
151static struct icc_node *node_find_by_name(const char *name)
152{
153 struct icc_provider *provider;
154 struct icc_node *n;
155
156 list_for_each_entry(provider, &icc_providers, provider_list) {
157 list_for_each_entry(n, &provider->nodes, node_list) {
158 if (!strcmp(n->name, name))
159 return n;
160 }
161 }
162
163 return NULL;
164}
165
166static struct icc_path *path_init(struct device *dev, struct icc_node *dst,
167 ssize_t num_nodes)
168{
169 struct icc_node *node = dst;
170 struct icc_path *path;
171 int i;
172
173 path = kzalloc(struct_size(path, reqs, num_nodes), GFP_KERNEL);
174 if (!path)
175 return ERR_PTR(-ENOMEM);
176
177 path->num_nodes = num_nodes;
178
179 mutex_lock(&icc_bw_lock);
180
181 for (i = num_nodes - 1; i >= 0; i--) {
182 node->provider->users++;
183 hlist_add_head(&path->reqs[i].req_node, &node->req_list);
184 path->reqs[i].node = node;
185 path->reqs[i].dev = dev;
186 path->reqs[i].enabled = true;
187 /* reference to previous node was saved during path traversal */
188 node = node->reverse;
189 }
190
191 mutex_unlock(&icc_bw_lock);
192
193 return path;
194}
195
196static struct icc_path *path_find(struct device *dev, struct icc_node *src,
197 struct icc_node *dst)
198{
199 struct icc_path *path = ERR_PTR(-EPROBE_DEFER);
200 struct icc_node *n, *node = NULL;
201 struct list_head traverse_list;
202 struct list_head edge_list;
203 struct list_head visited_list;
204 size_t i, depth = 1;
205 bool found = false;
206
207 INIT_LIST_HEAD(&traverse_list);
208 INIT_LIST_HEAD(&edge_list);
209 INIT_LIST_HEAD(&visited_list);
210
211 list_add(&src->search_list, &traverse_list);
212 src->reverse = NULL;
213
214 do {
215 list_for_each_entry_safe(node, n, &traverse_list, search_list) {
216 if (node == dst) {
217 found = true;
218 list_splice_init(&edge_list, &visited_list);
219 list_splice_init(&traverse_list, &visited_list);
220 break;
221 }
222 for (i = 0; i < node->num_links; i++) {
223 struct icc_node *tmp = node->links[i];
224
225 if (!tmp) {
226 path = ERR_PTR(-ENOENT);
227 goto out;
228 }
229
230 if (tmp->is_traversed)
231 continue;
232
233 tmp->is_traversed = true;
234 tmp->reverse = node;
235 list_add_tail(&tmp->search_list, &edge_list);
236 }
237 }
238
239 if (found)
240 break;
241
242 list_splice_init(&traverse_list, &visited_list);
243 list_splice_init(&edge_list, &traverse_list);
244
245 /* count the hops including the source */
246 depth++;
247
248 } while (!list_empty(&traverse_list));
249
250out:
251
252 /* reset the traversed state */
253 list_for_each_entry_reverse(n, &visited_list, search_list)
254 n->is_traversed = false;
255
256 if (found)
257 path = path_init(dev, dst, depth);
258
259 return path;
260}
261
262/*
263 * We want the path to honor all bandwidth requests, so the average and peak
264 * bandwidth requirements from each consumer are aggregated at each node.
265 * The aggregation is platform specific, so each platform can customize it by
266 * implementing its own aggregate() function.
267 */
268
269static int aggregate_requests(struct icc_node *node)
270{
271 struct icc_provider *p = node->provider;
272 struct icc_req *r;
273 u32 avg_bw, peak_bw;
274
275 node->avg_bw = 0;
276 node->peak_bw = 0;
277
278 if (p->pre_aggregate)
279 p->pre_aggregate(node);
280
281 hlist_for_each_entry(r, &node->req_list, req_node) {
282 if (r->enabled) {
283 avg_bw = r->avg_bw;
284 peak_bw = r->peak_bw;
285 } else {
286 avg_bw = 0;
287 peak_bw = 0;
288 }
289 p->aggregate(node, r->tag, avg_bw, peak_bw,
290 &node->avg_bw, &node->peak_bw);
291
292 /* during boot use the initial bandwidth as a floor value */
293 if (!synced_state) {
294 node->avg_bw = max(node->avg_bw, node->init_avg);
295 node->peak_bw = max(node->peak_bw, node->init_peak);
296 }
297 }
298
299 return 0;
300}
301
302static int apply_constraints(struct icc_path *path)
303{
304 struct icc_node *next, *prev = NULL;
305 struct icc_provider *p;
306 int ret = -EINVAL;
307 int i;
308
309 for (i = 0; i < path->num_nodes; i++) {
310 next = path->reqs[i].node;
311 p = next->provider;
312
313 /* both endpoints should be valid master-slave pairs */
314 if (!prev || (p != prev->provider && !p->inter_set)) {
315 prev = next;
316 continue;
317 }
318
319 /* set the constraints */
320 ret = p->set(prev, next);
321 if (ret)
322 goto out;
323
324 prev = next;
325 }
326out:
327 return ret;
328}
329
330int icc_std_aggregate(struct icc_node *node, u32 tag, u32 avg_bw,
331 u32 peak_bw, u32 *agg_avg, u32 *agg_peak)
332{
333 *agg_avg += avg_bw;
334 *agg_peak = max(*agg_peak, peak_bw);
335
336 return 0;
337}
338EXPORT_SYMBOL_GPL(icc_std_aggregate);
339
340/* of_icc_xlate_onecell() - Translate function using a single index.
341 * @spec: OF phandle args to map into an interconnect node.
342 * @data: private data (pointer to struct icc_onecell_data)
343 *
344 * This is a generic translate function that can be used to model simple
345 * interconnect providers that have one device tree node and provide
346 * multiple interconnect nodes. A single cell is used as an index into
347 * an array of icc nodes specified in the icc_onecell_data struct when
348 * registering the provider.
349 */
350struct icc_node *of_icc_xlate_onecell(const struct of_phandle_args *spec,
351 void *data)
352{
353 struct icc_onecell_data *icc_data = data;
354 unsigned int idx = spec->args[0];
355
356 if (idx >= icc_data->num_nodes) {
357 pr_err("%s: invalid index %u\n", __func__, idx);
358 return ERR_PTR(-EINVAL);
359 }
360
361 return icc_data->nodes[idx];
362}
363EXPORT_SYMBOL_GPL(of_icc_xlate_onecell);
364
365/**
366 * of_icc_get_from_provider() - Look-up interconnect node
367 * @spec: OF phandle args to use for look-up
368 *
369 * Looks for interconnect provider under the node specified by @spec and if
370 * found, uses xlate function of the provider to map phandle args to node.
371 *
372 * Returns a valid pointer to struct icc_node_data on success or ERR_PTR()
373 * on failure.
374 */
375struct icc_node_data *of_icc_get_from_provider(const struct of_phandle_args *spec)
376{
377 struct icc_node *node = ERR_PTR(-EPROBE_DEFER);
378 struct icc_node_data *data = NULL;
379 struct icc_provider *provider;
380
381 if (!spec)
382 return ERR_PTR(-EINVAL);
383
384 mutex_lock(&icc_lock);
385 list_for_each_entry(provider, &icc_providers, provider_list) {
386 if (provider->dev->of_node == spec->np) {
387 if (provider->xlate_extended) {
388 data = provider->xlate_extended(spec, provider->data);
389 if (!IS_ERR(data)) {
390 node = data->node;
391 break;
392 }
393 } else {
394 node = provider->xlate(spec, provider->data);
395 if (!IS_ERR(node))
396 break;
397 }
398 }
399 }
400 mutex_unlock(&icc_lock);
401
402 if (!node)
403 return ERR_PTR(-EINVAL);
404
405 if (IS_ERR(node))
406 return ERR_CAST(node);
407
408 if (!data) {
409 data = kzalloc(sizeof(*data), GFP_KERNEL);
410 if (!data)
411 return ERR_PTR(-ENOMEM);
412 data->node = node;
413 }
414
415 return data;
416}
417EXPORT_SYMBOL_GPL(of_icc_get_from_provider);
418
419static void devm_icc_release(struct device *dev, void *res)
420{
421 icc_put(*(struct icc_path **)res);
422}
423
424struct icc_path *devm_of_icc_get(struct device *dev, const char *name)
425{
426 struct icc_path **ptr, *path;
427
428 ptr = devres_alloc(devm_icc_release, sizeof(*ptr), GFP_KERNEL);
429 if (!ptr)
430 return ERR_PTR(-ENOMEM);
431
432 path = of_icc_get(dev, name);
433 if (!IS_ERR(path)) {
434 *ptr = path;
435 devres_add(dev, ptr);
436 } else {
437 devres_free(ptr);
438 }
439
440 return path;
441}
442EXPORT_SYMBOL_GPL(devm_of_icc_get);
443
444/**
445 * of_icc_get_by_index() - get a path handle from a DT node based on index
446 * @dev: device pointer for the consumer device
447 * @idx: interconnect path index
448 *
449 * This function will search for a path between two endpoints and return an
450 * icc_path handle on success. Use icc_put() to release constraints when they
451 * are not needed anymore.
452 * If the interconnect API is disabled, NULL is returned and the consumer
453 * drivers will still build. Drivers are free to handle this specifically,
454 * but they don't have to.
455 *
456 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
457 * when the API is disabled or the "interconnects" DT property is missing.
458 */
459struct icc_path *of_icc_get_by_index(struct device *dev, int idx)
460{
461 struct icc_path *path;
462 struct icc_node_data *src_data, *dst_data;
463 struct device_node *np;
464 struct of_phandle_args src_args, dst_args;
465 int ret;
466
467 if (!dev || !dev->of_node)
468 return ERR_PTR(-ENODEV);
469
470 np = dev->of_node;
471
472 /*
473 * When the consumer DT node do not have "interconnects" property
474 * return a NULL path to skip setting constraints.
475 */
476 if (!of_property_present(np, "interconnects"))
477 return NULL;
478
479 /*
480 * We use a combination of phandle and specifier for endpoint. For now
481 * lets support only global ids and extend this in the future if needed
482 * without breaking DT compatibility.
483 */
484 ret = of_parse_phandle_with_args(np, "interconnects",
485 "#interconnect-cells", idx * 2,
486 &src_args);
487 if (ret)
488 return ERR_PTR(ret);
489
490 of_node_put(src_args.np);
491
492 ret = of_parse_phandle_with_args(np, "interconnects",
493 "#interconnect-cells", idx * 2 + 1,
494 &dst_args);
495 if (ret)
496 return ERR_PTR(ret);
497
498 of_node_put(dst_args.np);
499
500 src_data = of_icc_get_from_provider(&src_args);
501
502 if (IS_ERR(src_data)) {
503 dev_err_probe(dev, PTR_ERR(src_data), "error finding src node\n");
504 return ERR_CAST(src_data);
505 }
506
507 dst_data = of_icc_get_from_provider(&dst_args);
508
509 if (IS_ERR(dst_data)) {
510 dev_err_probe(dev, PTR_ERR(dst_data), "error finding dst node\n");
511 kfree(src_data);
512 return ERR_CAST(dst_data);
513 }
514
515 mutex_lock(&icc_lock);
516 path = path_find(dev, src_data->node, dst_data->node);
517 mutex_unlock(&icc_lock);
518 if (IS_ERR(path)) {
519 dev_err(dev, "%s: invalid path=%ld\n", __func__, PTR_ERR(path));
520 goto free_icc_data;
521 }
522
523 if (src_data->tag && src_data->tag == dst_data->tag)
524 icc_set_tag(path, src_data->tag);
525
526 path->name = kasprintf(GFP_KERNEL, "%s-%s",
527 src_data->node->name, dst_data->node->name);
528 if (!path->name) {
529 kfree(path);
530 path = ERR_PTR(-ENOMEM);
531 }
532
533free_icc_data:
534 kfree(src_data);
535 kfree(dst_data);
536 return path;
537}
538EXPORT_SYMBOL_GPL(of_icc_get_by_index);
539
540/**
541 * of_icc_get() - get a path handle from a DT node based on name
542 * @dev: device pointer for the consumer device
543 * @name: interconnect path name
544 *
545 * This function will search for a path between two endpoints and return an
546 * icc_path handle on success. Use icc_put() to release constraints when they
547 * are not needed anymore.
548 * If the interconnect API is disabled, NULL is returned and the consumer
549 * drivers will still build. Drivers are free to handle this specifically,
550 * but they don't have to.
551 *
552 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
553 * when the API is disabled or the "interconnects" DT property is missing.
554 */
555struct icc_path *of_icc_get(struct device *dev, const char *name)
556{
557 struct device_node *np;
558 int idx = 0;
559
560 if (!dev || !dev->of_node)
561 return ERR_PTR(-ENODEV);
562
563 np = dev->of_node;
564
565 /*
566 * When the consumer DT node do not have "interconnects" property
567 * return a NULL path to skip setting constraints.
568 */
569 if (!of_property_present(np, "interconnects"))
570 return NULL;
571
572 /*
573 * We use a combination of phandle and specifier for endpoint. For now
574 * lets support only global ids and extend this in the future if needed
575 * without breaking DT compatibility.
576 */
577 if (name) {
578 idx = of_property_match_string(np, "interconnect-names", name);
579 if (idx < 0)
580 return ERR_PTR(idx);
581 }
582
583 return of_icc_get_by_index(dev, idx);
584}
585EXPORT_SYMBOL_GPL(of_icc_get);
586
587/**
588 * icc_get() - get a path handle between two endpoints
589 * @dev: device pointer for the consumer device
590 * @src: source node name
591 * @dst: destination node name
592 *
593 * This function will search for a path between two endpoints and return an
594 * icc_path handle on success. Use icc_put() to release constraints when they
595 * are not needed anymore.
596 *
597 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
598 * when the API is disabled.
599 */
600struct icc_path *icc_get(struct device *dev, const char *src, const char *dst)
601{
602 struct icc_node *src_node, *dst_node;
603 struct icc_path *path = ERR_PTR(-EPROBE_DEFER);
604
605 mutex_lock(&icc_lock);
606
607 src_node = node_find_by_name(src);
608 if (!src_node) {
609 dev_err(dev, "%s: invalid src=%s\n", __func__, src);
610 goto out;
611 }
612
613 dst_node = node_find_by_name(dst);
614 if (!dst_node) {
615 dev_err(dev, "%s: invalid dst=%s\n", __func__, dst);
616 goto out;
617 }
618
619 path = path_find(dev, src_node, dst_node);
620 if (IS_ERR(path)) {
621 dev_err(dev, "%s: invalid path=%ld\n", __func__, PTR_ERR(path));
622 goto out;
623 }
624
625 path->name = kasprintf(GFP_KERNEL, "%s-%s", src_node->name, dst_node->name);
626 if (!path->name) {
627 kfree(path);
628 path = ERR_PTR(-ENOMEM);
629 }
630out:
631 mutex_unlock(&icc_lock);
632 return path;
633}
634
635/**
636 * icc_set_tag() - set an optional tag on a path
637 * @path: the path we want to tag
638 * @tag: the tag value
639 *
640 * This function allows consumers to append a tag to the requests associated
641 * with a path, so that a different aggregation could be done based on this tag.
642 */
643void icc_set_tag(struct icc_path *path, u32 tag)
644{
645 int i;
646
647 if (!path)
648 return;
649
650 mutex_lock(&icc_lock);
651
652 for (i = 0; i < path->num_nodes; i++)
653 path->reqs[i].tag = tag;
654
655 mutex_unlock(&icc_lock);
656}
657EXPORT_SYMBOL_GPL(icc_set_tag);
658
659/**
660 * icc_get_name() - Get name of the icc path
661 * @path: interconnect path
662 *
663 * This function is used by an interconnect consumer to get the name of the icc
664 * path.
665 *
666 * Returns a valid pointer on success, or NULL otherwise.
667 */
668const char *icc_get_name(struct icc_path *path)
669{
670 if (!path)
671 return NULL;
672
673 return path->name;
674}
675EXPORT_SYMBOL_GPL(icc_get_name);
676
677/**
678 * icc_set_bw() - set bandwidth constraints on an interconnect path
679 * @path: interconnect path
680 * @avg_bw: average bandwidth in kilobytes per second
681 * @peak_bw: peak bandwidth in kilobytes per second
682 *
683 * This function is used by an interconnect consumer to express its own needs
684 * in terms of bandwidth for a previously requested path between two endpoints.
685 * The requests are aggregated and each node is updated accordingly. The entire
686 * path is locked by a mutex to ensure that the set() is completed.
687 * The @path can be NULL when the "interconnects" DT properties is missing,
688 * which will mean that no constraints will be set.
689 *
690 * Returns 0 on success, or an appropriate error code otherwise.
691 */
692int icc_set_bw(struct icc_path *path, u32 avg_bw, u32 peak_bw)
693{
694 struct icc_node *node;
695 u32 old_avg, old_peak;
696 size_t i;
697 int ret;
698
699 if (!path)
700 return 0;
701
702 if (WARN_ON(IS_ERR(path) || !path->num_nodes))
703 return -EINVAL;
704
705 mutex_lock(&icc_bw_lock);
706
707 old_avg = path->reqs[0].avg_bw;
708 old_peak = path->reqs[0].peak_bw;
709
710 for (i = 0; i < path->num_nodes; i++) {
711 node = path->reqs[i].node;
712
713 /* update the consumer request for this path */
714 path->reqs[i].avg_bw = avg_bw;
715 path->reqs[i].peak_bw = peak_bw;
716
717 /* aggregate requests for this node */
718 aggregate_requests(node);
719
720 trace_icc_set_bw(path, node, i, avg_bw, peak_bw);
721 }
722
723 ret = apply_constraints(path);
724 if (ret) {
725 pr_debug("interconnect: error applying constraints (%d)\n",
726 ret);
727
728 for (i = 0; i < path->num_nodes; i++) {
729 node = path->reqs[i].node;
730 path->reqs[i].avg_bw = old_avg;
731 path->reqs[i].peak_bw = old_peak;
732 aggregate_requests(node);
733 }
734 apply_constraints(path);
735 }
736
737 mutex_unlock(&icc_bw_lock);
738
739 trace_icc_set_bw_end(path, ret);
740
741 return ret;
742}
743EXPORT_SYMBOL_GPL(icc_set_bw);
744
745static int __icc_enable(struct icc_path *path, bool enable)
746{
747 int i;
748
749 if (!path)
750 return 0;
751
752 if (WARN_ON(IS_ERR(path) || !path->num_nodes))
753 return -EINVAL;
754
755 mutex_lock(&icc_lock);
756
757 for (i = 0; i < path->num_nodes; i++)
758 path->reqs[i].enabled = enable;
759
760 mutex_unlock(&icc_lock);
761
762 return icc_set_bw(path, path->reqs[0].avg_bw,
763 path->reqs[0].peak_bw);
764}
765
766int icc_enable(struct icc_path *path)
767{
768 return __icc_enable(path, true);
769}
770EXPORT_SYMBOL_GPL(icc_enable);
771
772int icc_disable(struct icc_path *path)
773{
774 return __icc_enable(path, false);
775}
776EXPORT_SYMBOL_GPL(icc_disable);
777
778/**
779 * icc_put() - release the reference to the icc_path
780 * @path: interconnect path
781 *
782 * Use this function to release the constraints on a path when the path is
783 * no longer needed. The constraints will be re-aggregated.
784 */
785void icc_put(struct icc_path *path)
786{
787 struct icc_node *node;
788 size_t i;
789 int ret;
790
791 if (!path || WARN_ON(IS_ERR(path)))
792 return;
793
794 ret = icc_set_bw(path, 0, 0);
795 if (ret)
796 pr_err("%s: error (%d)\n", __func__, ret);
797
798 mutex_lock(&icc_lock);
799 mutex_lock(&icc_bw_lock);
800
801 for (i = 0; i < path->num_nodes; i++) {
802 node = path->reqs[i].node;
803 hlist_del(&path->reqs[i].req_node);
804 if (!WARN_ON(!node->provider->users))
805 node->provider->users--;
806 }
807
808 mutex_unlock(&icc_bw_lock);
809 mutex_unlock(&icc_lock);
810
811 kfree_const(path->name);
812 kfree(path);
813}
814EXPORT_SYMBOL_GPL(icc_put);
815
816static struct icc_node *icc_node_create_nolock(int id)
817{
818 struct icc_node *node;
819
820 /* check if node already exists */
821 node = node_find(id);
822 if (node)
823 return node;
824
825 node = kzalloc(sizeof(*node), GFP_KERNEL);
826 if (!node)
827 return ERR_PTR(-ENOMEM);
828
829 id = idr_alloc(&icc_idr, node, id, id + 1, GFP_KERNEL);
830 if (id < 0) {
831 WARN(1, "%s: couldn't get idr\n", __func__);
832 kfree(node);
833 return ERR_PTR(id);
834 }
835
836 node->id = id;
837
838 return node;
839}
840
841/**
842 * icc_node_create() - create a node
843 * @id: node id
844 *
845 * Return: icc_node pointer on success, or ERR_PTR() on error
846 */
847struct icc_node *icc_node_create(int id)
848{
849 struct icc_node *node;
850
851 mutex_lock(&icc_lock);
852
853 node = icc_node_create_nolock(id);
854
855 mutex_unlock(&icc_lock);
856
857 return node;
858}
859EXPORT_SYMBOL_GPL(icc_node_create);
860
861/**
862 * icc_node_destroy() - destroy a node
863 * @id: node id
864 */
865void icc_node_destroy(int id)
866{
867 struct icc_node *node;
868
869 mutex_lock(&icc_lock);
870
871 node = node_find(id);
872 if (node) {
873 idr_remove(&icc_idr, node->id);
874 WARN_ON(!hlist_empty(&node->req_list));
875 }
876
877 mutex_unlock(&icc_lock);
878
879 if (!node)
880 return;
881
882 kfree(node->links);
883 kfree(node);
884}
885EXPORT_SYMBOL_GPL(icc_node_destroy);
886
887/**
888 * icc_link_create() - create a link between two nodes
889 * @node: source node id
890 * @dst_id: destination node id
891 *
892 * Create a link between two nodes. The nodes might belong to different
893 * interconnect providers and the @dst_id node might not exist (if the
894 * provider driver has not probed yet). So just create the @dst_id node
895 * and when the actual provider driver is probed, the rest of the node
896 * data is filled.
897 *
898 * Return: 0 on success, or an error code otherwise
899 */
900int icc_link_create(struct icc_node *node, const int dst_id)
901{
902 struct icc_node *dst;
903 struct icc_node **new;
904 int ret = 0;
905
906 if (!node->provider)
907 return -EINVAL;
908
909 mutex_lock(&icc_lock);
910
911 dst = node_find(dst_id);
912 if (!dst) {
913 dst = icc_node_create_nolock(dst_id);
914
915 if (IS_ERR(dst)) {
916 ret = PTR_ERR(dst);
917 goto out;
918 }
919 }
920
921 new = krealloc(node->links,
922 (node->num_links + 1) * sizeof(*node->links),
923 GFP_KERNEL);
924 if (!new) {
925 ret = -ENOMEM;
926 goto out;
927 }
928
929 node->links = new;
930 node->links[node->num_links++] = dst;
931
932out:
933 mutex_unlock(&icc_lock);
934
935 return ret;
936}
937EXPORT_SYMBOL_GPL(icc_link_create);
938
939/**
940 * icc_node_add() - add interconnect node to interconnect provider
941 * @node: pointer to the interconnect node
942 * @provider: pointer to the interconnect provider
943 */
944void icc_node_add(struct icc_node *node, struct icc_provider *provider)
945{
946 if (WARN_ON(node->provider))
947 return;
948
949 mutex_lock(&icc_lock);
950 mutex_lock(&icc_bw_lock);
951
952 node->provider = provider;
953 list_add_tail(&node->node_list, &provider->nodes);
954
955 /* get the initial bandwidth values and sync them with hardware */
956 if (provider->get_bw) {
957 provider->get_bw(node, &node->init_avg, &node->init_peak);
958 } else {
959 node->init_avg = INT_MAX;
960 node->init_peak = INT_MAX;
961 }
962 node->avg_bw = node->init_avg;
963 node->peak_bw = node->init_peak;
964
965 if (node->avg_bw || node->peak_bw) {
966 if (provider->pre_aggregate)
967 provider->pre_aggregate(node);
968
969 if (provider->aggregate)
970 provider->aggregate(node, 0, node->init_avg, node->init_peak,
971 &node->avg_bw, &node->peak_bw);
972 if (provider->set)
973 provider->set(node, node);
974 }
975
976 node->avg_bw = 0;
977 node->peak_bw = 0;
978
979 mutex_unlock(&icc_bw_lock);
980 mutex_unlock(&icc_lock);
981}
982EXPORT_SYMBOL_GPL(icc_node_add);
983
984/**
985 * icc_node_del() - delete interconnect node from interconnect provider
986 * @node: pointer to the interconnect node
987 */
988void icc_node_del(struct icc_node *node)
989{
990 mutex_lock(&icc_lock);
991
992 list_del(&node->node_list);
993
994 mutex_unlock(&icc_lock);
995}
996EXPORT_SYMBOL_GPL(icc_node_del);
997
998/**
999 * icc_nodes_remove() - remove all previously added nodes from provider
1000 * @provider: the interconnect provider we are removing nodes from
1001 *
1002 * Return: 0 on success, or an error code otherwise
1003 */
1004int icc_nodes_remove(struct icc_provider *provider)
1005{
1006 struct icc_node *n, *tmp;
1007
1008 if (WARN_ON(IS_ERR_OR_NULL(provider)))
1009 return -EINVAL;
1010
1011 list_for_each_entry_safe_reverse(n, tmp, &provider->nodes, node_list) {
1012 icc_node_del(n);
1013 icc_node_destroy(n->id);
1014 }
1015
1016 return 0;
1017}
1018EXPORT_SYMBOL_GPL(icc_nodes_remove);
1019
1020/**
1021 * icc_provider_init() - initialize a new interconnect provider
1022 * @provider: the interconnect provider to initialize
1023 *
1024 * Must be called before adding nodes to the provider.
1025 */
1026void icc_provider_init(struct icc_provider *provider)
1027{
1028 WARN_ON(!provider->set);
1029
1030 INIT_LIST_HEAD(&provider->nodes);
1031}
1032EXPORT_SYMBOL_GPL(icc_provider_init);
1033
1034/**
1035 * icc_provider_register() - register a new interconnect provider
1036 * @provider: the interconnect provider to register
1037 *
1038 * Return: 0 on success, or an error code otherwise
1039 */
1040int icc_provider_register(struct icc_provider *provider)
1041{
1042 if (WARN_ON(!provider->xlate && !provider->xlate_extended))
1043 return -EINVAL;
1044
1045 mutex_lock(&icc_lock);
1046 list_add_tail(&provider->provider_list, &icc_providers);
1047 mutex_unlock(&icc_lock);
1048
1049 dev_dbg(provider->dev, "interconnect provider registered\n");
1050
1051 return 0;
1052}
1053EXPORT_SYMBOL_GPL(icc_provider_register);
1054
1055/**
1056 * icc_provider_deregister() - deregister an interconnect provider
1057 * @provider: the interconnect provider to deregister
1058 */
1059void icc_provider_deregister(struct icc_provider *provider)
1060{
1061 mutex_lock(&icc_lock);
1062 WARN_ON(provider->users);
1063
1064 list_del(&provider->provider_list);
1065 mutex_unlock(&icc_lock);
1066}
1067EXPORT_SYMBOL_GPL(icc_provider_deregister);
1068
1069static const struct of_device_id __maybe_unused ignore_list[] = {
1070 { .compatible = "qcom,sc7180-ipa-virt" },
1071 { .compatible = "qcom,sc8180x-ipa-virt" },
1072 { .compatible = "qcom,sdx55-ipa-virt" },
1073 { .compatible = "qcom,sm8150-ipa-virt" },
1074 { .compatible = "qcom,sm8250-ipa-virt" },
1075 {}
1076};
1077
1078static int of_count_icc_providers(struct device_node *np)
1079{
1080 struct device_node *child;
1081 int count = 0;
1082
1083 for_each_available_child_of_node(np, child) {
1084 if (of_property_read_bool(child, "#interconnect-cells") &&
1085 likely(!of_match_node(ignore_list, child)))
1086 count++;
1087 count += of_count_icc_providers(child);
1088 }
1089
1090 return count;
1091}
1092
1093void icc_sync_state(struct device *dev)
1094{
1095 struct icc_provider *p;
1096 struct icc_node *n;
1097 static int count;
1098
1099 count++;
1100
1101 if (count < providers_count)
1102 return;
1103
1104 mutex_lock(&icc_lock);
1105 mutex_lock(&icc_bw_lock);
1106 synced_state = true;
1107 list_for_each_entry(p, &icc_providers, provider_list) {
1108 dev_dbg(p->dev, "interconnect provider is in synced state\n");
1109 list_for_each_entry(n, &p->nodes, node_list) {
1110 if (n->init_avg || n->init_peak) {
1111 n->init_avg = 0;
1112 n->init_peak = 0;
1113 aggregate_requests(n);
1114 p->set(n, n);
1115 }
1116 }
1117 }
1118 mutex_unlock(&icc_bw_lock);
1119 mutex_unlock(&icc_lock);
1120}
1121EXPORT_SYMBOL_GPL(icc_sync_state);
1122
1123static int __init icc_init(void)
1124{
1125 struct device_node *root;
1126
1127 /* Teach lockdep about lock ordering wrt. shrinker: */
1128 fs_reclaim_acquire(GFP_KERNEL);
1129 might_lock(&icc_bw_lock);
1130 fs_reclaim_release(GFP_KERNEL);
1131
1132 root = of_find_node_by_path("/");
1133
1134 providers_count = of_count_icc_providers(root);
1135 of_node_put(root);
1136
1137 icc_debugfs_dir = debugfs_create_dir("interconnect", NULL);
1138 debugfs_create_file("interconnect_summary", 0444,
1139 icc_debugfs_dir, NULL, &icc_summary_fops);
1140 debugfs_create_file("interconnect_graph", 0444,
1141 icc_debugfs_dir, NULL, &icc_graph_fops);
1142
1143 icc_debugfs_client_init(icc_debugfs_dir);
1144
1145 return 0;
1146}
1147
1148device_initcall(icc_init);