Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Interconnect framework core driver
4 *
5 * Copyright (c) 2017-2019, Linaro Ltd.
6 * Author: Georgi Djakov <georgi.djakov@linaro.org>
7 */
8
9#include <linux/debugfs.h>
10#include <linux/device.h>
11#include <linux/idr.h>
12#include <linux/init.h>
13#include <linux/interconnect.h>
14#include <linux/interconnect-provider.h>
15#include <linux/list.h>
16#include <linux/mutex.h>
17#include <linux/slab.h>
18#include <linux/of.h>
19#include <linux/overflow.h>
20
21#include "internal.h"
22
23#define CREATE_TRACE_POINTS
24#include "trace.h"
25
26static DEFINE_IDR(icc_idr);
27static LIST_HEAD(icc_providers);
28static int providers_count;
29static bool synced_state;
30static DEFINE_MUTEX(icc_lock);
31static DEFINE_MUTEX(icc_bw_lock);
32static struct dentry *icc_debugfs_dir;
33
34static void icc_summary_show_one(struct seq_file *s, struct icc_node *n)
35{
36 if (!n)
37 return;
38
39 seq_printf(s, "%-42s %12u %12u\n",
40 n->name, n->avg_bw, n->peak_bw);
41}
42
43static int icc_summary_show(struct seq_file *s, void *data)
44{
45 struct icc_provider *provider;
46
47 seq_puts(s, " node tag avg peak\n");
48 seq_puts(s, "--------------------------------------------------------------------\n");
49
50 mutex_lock(&icc_lock);
51
52 list_for_each_entry(provider, &icc_providers, provider_list) {
53 struct icc_node *n;
54
55 list_for_each_entry(n, &provider->nodes, node_list) {
56 struct icc_req *r;
57
58 icc_summary_show_one(s, n);
59 hlist_for_each_entry(r, &n->req_list, req_node) {
60 u32 avg_bw = 0, peak_bw = 0;
61
62 if (!r->dev)
63 continue;
64
65 if (r->enabled) {
66 avg_bw = r->avg_bw;
67 peak_bw = r->peak_bw;
68 }
69
70 seq_printf(s, " %-27s %12u %12u %12u\n",
71 dev_name(r->dev), r->tag, avg_bw, peak_bw);
72 }
73 }
74 }
75
76 mutex_unlock(&icc_lock);
77
78 return 0;
79}
80DEFINE_SHOW_ATTRIBUTE(icc_summary);
81
82static void icc_graph_show_link(struct seq_file *s, int level,
83 struct icc_node *n, struct icc_node *m)
84{
85 seq_printf(s, "%s\"%d:%s\" -> \"%d:%s\"\n",
86 level == 2 ? "\t\t" : "\t",
87 n->id, n->name, m->id, m->name);
88}
89
90static void icc_graph_show_node(struct seq_file *s, struct icc_node *n)
91{
92 seq_printf(s, "\t\t\"%d:%s\" [label=\"%d:%s",
93 n->id, n->name, n->id, n->name);
94 seq_printf(s, "\n\t\t\t|avg_bw=%ukBps", n->avg_bw);
95 seq_printf(s, "\n\t\t\t|peak_bw=%ukBps", n->peak_bw);
96 seq_puts(s, "\"]\n");
97}
98
99static int icc_graph_show(struct seq_file *s, void *data)
100{
101 struct icc_provider *provider;
102 struct icc_node *n;
103 int cluster_index = 0;
104 int i;
105
106 seq_puts(s, "digraph {\n\trankdir = LR\n\tnode [shape = record]\n");
107 mutex_lock(&icc_lock);
108
109 /* draw providers as cluster subgraphs */
110 cluster_index = 0;
111 list_for_each_entry(provider, &icc_providers, provider_list) {
112 seq_printf(s, "\tsubgraph cluster_%d {\n", ++cluster_index);
113 if (provider->dev)
114 seq_printf(s, "\t\tlabel = \"%s\"\n",
115 dev_name(provider->dev));
116
117 /* draw nodes */
118 list_for_each_entry(n, &provider->nodes, node_list)
119 icc_graph_show_node(s, n);
120
121 /* draw internal links */
122 list_for_each_entry(n, &provider->nodes, node_list)
123 for (i = 0; i < n->num_links; ++i)
124 if (n->provider == n->links[i]->provider)
125 icc_graph_show_link(s, 2, n,
126 n->links[i]);
127
128 seq_puts(s, "\t}\n");
129 }
130
131 /* draw external links */
132 list_for_each_entry(provider, &icc_providers, provider_list)
133 list_for_each_entry(n, &provider->nodes, node_list)
134 for (i = 0; i < n->num_links; ++i)
135 if (n->provider != n->links[i]->provider)
136 icc_graph_show_link(s, 1, n,
137 n->links[i]);
138
139 mutex_unlock(&icc_lock);
140 seq_puts(s, "}");
141
142 return 0;
143}
144DEFINE_SHOW_ATTRIBUTE(icc_graph);
145
146static struct icc_node *node_find(const int id)
147{
148 return idr_find(&icc_idr, id);
149}
150
151static struct icc_node *node_find_by_name(const char *name)
152{
153 struct icc_provider *provider;
154 struct icc_node *n;
155
156 list_for_each_entry(provider, &icc_providers, provider_list) {
157 list_for_each_entry(n, &provider->nodes, node_list) {
158 if (!strcmp(n->name, name))
159 return n;
160 }
161 }
162
163 return NULL;
164}
165
166static struct icc_path *path_init(struct device *dev, struct icc_node *dst,
167 ssize_t num_nodes)
168{
169 struct icc_node *node = dst;
170 struct icc_path *path;
171 int i;
172
173 path = kzalloc(struct_size(path, reqs, num_nodes), GFP_KERNEL);
174 if (!path)
175 return ERR_PTR(-ENOMEM);
176
177 path->num_nodes = num_nodes;
178
179 for (i = num_nodes - 1; i >= 0; i--) {
180 node->provider->users++;
181 hlist_add_head(&path->reqs[i].req_node, &node->req_list);
182 path->reqs[i].node = node;
183 path->reqs[i].dev = dev;
184 path->reqs[i].enabled = true;
185 /* reference to previous node was saved during path traversal */
186 node = node->reverse;
187 }
188
189 return path;
190}
191
192static struct icc_path *path_find(struct device *dev, struct icc_node *src,
193 struct icc_node *dst)
194{
195 struct icc_path *path = ERR_PTR(-EPROBE_DEFER);
196 struct icc_node *n, *node = NULL;
197 struct list_head traverse_list;
198 struct list_head edge_list;
199 struct list_head visited_list;
200 size_t i, depth = 1;
201 bool found = false;
202
203 INIT_LIST_HEAD(&traverse_list);
204 INIT_LIST_HEAD(&edge_list);
205 INIT_LIST_HEAD(&visited_list);
206
207 list_add(&src->search_list, &traverse_list);
208 src->reverse = NULL;
209
210 do {
211 list_for_each_entry_safe(node, n, &traverse_list, search_list) {
212 if (node == dst) {
213 found = true;
214 list_splice_init(&edge_list, &visited_list);
215 list_splice_init(&traverse_list, &visited_list);
216 break;
217 }
218 for (i = 0; i < node->num_links; i++) {
219 struct icc_node *tmp = node->links[i];
220
221 if (!tmp) {
222 path = ERR_PTR(-ENOENT);
223 goto out;
224 }
225
226 if (tmp->is_traversed)
227 continue;
228
229 tmp->is_traversed = true;
230 tmp->reverse = node;
231 list_add_tail(&tmp->search_list, &edge_list);
232 }
233 }
234
235 if (found)
236 break;
237
238 list_splice_init(&traverse_list, &visited_list);
239 list_splice_init(&edge_list, &traverse_list);
240
241 /* count the hops including the source */
242 depth++;
243
244 } while (!list_empty(&traverse_list));
245
246out:
247
248 /* reset the traversed state */
249 list_for_each_entry_reverse(n, &visited_list, search_list)
250 n->is_traversed = false;
251
252 if (found)
253 path = path_init(dev, dst, depth);
254
255 return path;
256}
257
258/*
259 * We want the path to honor all bandwidth requests, so the average and peak
260 * bandwidth requirements from each consumer are aggregated at each node.
261 * The aggregation is platform specific, so each platform can customize it by
262 * implementing its own aggregate() function.
263 */
264
265static int aggregate_requests(struct icc_node *node)
266{
267 struct icc_provider *p = node->provider;
268 struct icc_req *r;
269 u32 avg_bw, peak_bw;
270
271 node->avg_bw = 0;
272 node->peak_bw = 0;
273
274 if (p->pre_aggregate)
275 p->pre_aggregate(node);
276
277 hlist_for_each_entry(r, &node->req_list, req_node) {
278 if (r->enabled) {
279 avg_bw = r->avg_bw;
280 peak_bw = r->peak_bw;
281 } else {
282 avg_bw = 0;
283 peak_bw = 0;
284 }
285 p->aggregate(node, r->tag, avg_bw, peak_bw,
286 &node->avg_bw, &node->peak_bw);
287
288 /* during boot use the initial bandwidth as a floor value */
289 if (!synced_state) {
290 node->avg_bw = max(node->avg_bw, node->init_avg);
291 node->peak_bw = max(node->peak_bw, node->init_peak);
292 }
293 }
294
295 return 0;
296}
297
298static int apply_constraints(struct icc_path *path)
299{
300 struct icc_node *next, *prev = NULL;
301 struct icc_provider *p;
302 int ret = -EINVAL;
303 int i;
304
305 for (i = 0; i < path->num_nodes; i++) {
306 next = path->reqs[i].node;
307 p = next->provider;
308
309 /* both endpoints should be valid master-slave pairs */
310 if (!prev || (p != prev->provider && !p->inter_set)) {
311 prev = next;
312 continue;
313 }
314
315 /* set the constraints */
316 ret = p->set(prev, next);
317 if (ret)
318 goto out;
319
320 prev = next;
321 }
322out:
323 return ret;
324}
325
326int icc_std_aggregate(struct icc_node *node, u32 tag, u32 avg_bw,
327 u32 peak_bw, u32 *agg_avg, u32 *agg_peak)
328{
329 *agg_avg += avg_bw;
330 *agg_peak = max(*agg_peak, peak_bw);
331
332 return 0;
333}
334EXPORT_SYMBOL_GPL(icc_std_aggregate);
335
336/* of_icc_xlate_onecell() - Translate function using a single index.
337 * @spec: OF phandle args to map into an interconnect node.
338 * @data: private data (pointer to struct icc_onecell_data)
339 *
340 * This is a generic translate function that can be used to model simple
341 * interconnect providers that have one device tree node and provide
342 * multiple interconnect nodes. A single cell is used as an index into
343 * an array of icc nodes specified in the icc_onecell_data struct when
344 * registering the provider.
345 */
346struct icc_node *of_icc_xlate_onecell(struct of_phandle_args *spec,
347 void *data)
348{
349 struct icc_onecell_data *icc_data = data;
350 unsigned int idx = spec->args[0];
351
352 if (idx >= icc_data->num_nodes) {
353 pr_err("%s: invalid index %u\n", __func__, idx);
354 return ERR_PTR(-EINVAL);
355 }
356
357 return icc_data->nodes[idx];
358}
359EXPORT_SYMBOL_GPL(of_icc_xlate_onecell);
360
361/**
362 * of_icc_get_from_provider() - Look-up interconnect node
363 * @spec: OF phandle args to use for look-up
364 *
365 * Looks for interconnect provider under the node specified by @spec and if
366 * found, uses xlate function of the provider to map phandle args to node.
367 *
368 * Returns a valid pointer to struct icc_node_data on success or ERR_PTR()
369 * on failure.
370 */
371struct icc_node_data *of_icc_get_from_provider(struct of_phandle_args *spec)
372{
373 struct icc_node *node = ERR_PTR(-EPROBE_DEFER);
374 struct icc_node_data *data = NULL;
375 struct icc_provider *provider;
376
377 if (!spec)
378 return ERR_PTR(-EINVAL);
379
380 mutex_lock(&icc_lock);
381 list_for_each_entry(provider, &icc_providers, provider_list) {
382 if (provider->dev->of_node == spec->np) {
383 if (provider->xlate_extended) {
384 data = provider->xlate_extended(spec, provider->data);
385 if (!IS_ERR(data)) {
386 node = data->node;
387 break;
388 }
389 } else {
390 node = provider->xlate(spec, provider->data);
391 if (!IS_ERR(node))
392 break;
393 }
394 }
395 }
396 mutex_unlock(&icc_lock);
397
398 if (!node)
399 return ERR_PTR(-EINVAL);
400
401 if (IS_ERR(node))
402 return ERR_CAST(node);
403
404 if (!data) {
405 data = kzalloc(sizeof(*data), GFP_KERNEL);
406 if (!data)
407 return ERR_PTR(-ENOMEM);
408 data->node = node;
409 }
410
411 return data;
412}
413EXPORT_SYMBOL_GPL(of_icc_get_from_provider);
414
415static void devm_icc_release(struct device *dev, void *res)
416{
417 icc_put(*(struct icc_path **)res);
418}
419
420struct icc_path *devm_of_icc_get(struct device *dev, const char *name)
421{
422 struct icc_path **ptr, *path;
423
424 ptr = devres_alloc(devm_icc_release, sizeof(*ptr), GFP_KERNEL);
425 if (!ptr)
426 return ERR_PTR(-ENOMEM);
427
428 path = of_icc_get(dev, name);
429 if (!IS_ERR(path)) {
430 *ptr = path;
431 devres_add(dev, ptr);
432 } else {
433 devres_free(ptr);
434 }
435
436 return path;
437}
438EXPORT_SYMBOL_GPL(devm_of_icc_get);
439
440/**
441 * of_icc_get_by_index() - get a path handle from a DT node based on index
442 * @dev: device pointer for the consumer device
443 * @idx: interconnect path index
444 *
445 * This function will search for a path between two endpoints and return an
446 * icc_path handle on success. Use icc_put() to release constraints when they
447 * are not needed anymore.
448 * If the interconnect API is disabled, NULL is returned and the consumer
449 * drivers will still build. Drivers are free to handle this specifically,
450 * but they don't have to.
451 *
452 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
453 * when the API is disabled or the "interconnects" DT property is missing.
454 */
455struct icc_path *of_icc_get_by_index(struct device *dev, int idx)
456{
457 struct icc_path *path;
458 struct icc_node_data *src_data, *dst_data;
459 struct device_node *np;
460 struct of_phandle_args src_args, dst_args;
461 int ret;
462
463 if (!dev || !dev->of_node)
464 return ERR_PTR(-ENODEV);
465
466 np = dev->of_node;
467
468 /*
469 * When the consumer DT node do not have "interconnects" property
470 * return a NULL path to skip setting constraints.
471 */
472 if (!of_property_present(np, "interconnects"))
473 return NULL;
474
475 /*
476 * We use a combination of phandle and specifier for endpoint. For now
477 * lets support only global ids and extend this in the future if needed
478 * without breaking DT compatibility.
479 */
480 ret = of_parse_phandle_with_args(np, "interconnects",
481 "#interconnect-cells", idx * 2,
482 &src_args);
483 if (ret)
484 return ERR_PTR(ret);
485
486 of_node_put(src_args.np);
487
488 ret = of_parse_phandle_with_args(np, "interconnects",
489 "#interconnect-cells", idx * 2 + 1,
490 &dst_args);
491 if (ret)
492 return ERR_PTR(ret);
493
494 of_node_put(dst_args.np);
495
496 src_data = of_icc_get_from_provider(&src_args);
497
498 if (IS_ERR(src_data)) {
499 dev_err_probe(dev, PTR_ERR(src_data), "error finding src node\n");
500 return ERR_CAST(src_data);
501 }
502
503 dst_data = of_icc_get_from_provider(&dst_args);
504
505 if (IS_ERR(dst_data)) {
506 dev_err_probe(dev, PTR_ERR(dst_data), "error finding dst node\n");
507 kfree(src_data);
508 return ERR_CAST(dst_data);
509 }
510
511 mutex_lock(&icc_lock);
512 path = path_find(dev, src_data->node, dst_data->node);
513 mutex_unlock(&icc_lock);
514 if (IS_ERR(path)) {
515 dev_err(dev, "%s: invalid path=%ld\n", __func__, PTR_ERR(path));
516 goto free_icc_data;
517 }
518
519 if (src_data->tag && src_data->tag == dst_data->tag)
520 icc_set_tag(path, src_data->tag);
521
522 path->name = kasprintf(GFP_KERNEL, "%s-%s",
523 src_data->node->name, dst_data->node->name);
524 if (!path->name) {
525 kfree(path);
526 path = ERR_PTR(-ENOMEM);
527 }
528
529free_icc_data:
530 kfree(src_data);
531 kfree(dst_data);
532 return path;
533}
534EXPORT_SYMBOL_GPL(of_icc_get_by_index);
535
536/**
537 * of_icc_get() - get a path handle from a DT node based on name
538 * @dev: device pointer for the consumer device
539 * @name: interconnect path name
540 *
541 * This function will search for a path between two endpoints and return an
542 * icc_path handle on success. Use icc_put() to release constraints when they
543 * are not needed anymore.
544 * If the interconnect API is disabled, NULL is returned and the consumer
545 * drivers will still build. Drivers are free to handle this specifically,
546 * but they don't have to.
547 *
548 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
549 * when the API is disabled or the "interconnects" DT property is missing.
550 */
551struct icc_path *of_icc_get(struct device *dev, const char *name)
552{
553 struct device_node *np;
554 int idx = 0;
555
556 if (!dev || !dev->of_node)
557 return ERR_PTR(-ENODEV);
558
559 np = dev->of_node;
560
561 /*
562 * When the consumer DT node do not have "interconnects" property
563 * return a NULL path to skip setting constraints.
564 */
565 if (!of_property_present(np, "interconnects"))
566 return NULL;
567
568 /*
569 * We use a combination of phandle and specifier for endpoint. For now
570 * lets support only global ids and extend this in the future if needed
571 * without breaking DT compatibility.
572 */
573 if (name) {
574 idx = of_property_match_string(np, "interconnect-names", name);
575 if (idx < 0)
576 return ERR_PTR(idx);
577 }
578
579 return of_icc_get_by_index(dev, idx);
580}
581EXPORT_SYMBOL_GPL(of_icc_get);
582
583/**
584 * icc_get() - get a path handle between two endpoints
585 * @dev: device pointer for the consumer device
586 * @src: source node name
587 * @dst: destination node name
588 *
589 * This function will search for a path between two endpoints and return an
590 * icc_path handle on success. Use icc_put() to release constraints when they
591 * are not needed anymore.
592 *
593 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
594 * when the API is disabled.
595 */
596struct icc_path *icc_get(struct device *dev, const char *src, const char *dst)
597{
598 struct icc_node *src_node, *dst_node;
599 struct icc_path *path = ERR_PTR(-EPROBE_DEFER);
600
601 mutex_lock(&icc_lock);
602
603 src_node = node_find_by_name(src);
604 if (!src_node) {
605 dev_err(dev, "%s: invalid src=%s\n", __func__, src);
606 goto out;
607 }
608
609 dst_node = node_find_by_name(dst);
610 if (!dst_node) {
611 dev_err(dev, "%s: invalid dst=%s\n", __func__, dst);
612 goto out;
613 }
614
615 path = path_find(dev, src_node, dst_node);
616 if (IS_ERR(path)) {
617 dev_err(dev, "%s: invalid path=%ld\n", __func__, PTR_ERR(path));
618 goto out;
619 }
620
621 path->name = kasprintf(GFP_KERNEL, "%s-%s", src_node->name, dst_node->name);
622 if (!path->name) {
623 kfree(path);
624 path = ERR_PTR(-ENOMEM);
625 }
626out:
627 mutex_unlock(&icc_lock);
628 return path;
629}
630
631/**
632 * icc_set_tag() - set an optional tag on a path
633 * @path: the path we want to tag
634 * @tag: the tag value
635 *
636 * This function allows consumers to append a tag to the requests associated
637 * with a path, so that a different aggregation could be done based on this tag.
638 */
639void icc_set_tag(struct icc_path *path, u32 tag)
640{
641 int i;
642
643 if (!path)
644 return;
645
646 mutex_lock(&icc_lock);
647
648 for (i = 0; i < path->num_nodes; i++)
649 path->reqs[i].tag = tag;
650
651 mutex_unlock(&icc_lock);
652}
653EXPORT_SYMBOL_GPL(icc_set_tag);
654
655/**
656 * icc_get_name() - Get name of the icc path
657 * @path: interconnect path
658 *
659 * This function is used by an interconnect consumer to get the name of the icc
660 * path.
661 *
662 * Returns a valid pointer on success, or NULL otherwise.
663 */
664const char *icc_get_name(struct icc_path *path)
665{
666 if (!path)
667 return NULL;
668
669 return path->name;
670}
671EXPORT_SYMBOL_GPL(icc_get_name);
672
673/**
674 * icc_set_bw() - set bandwidth constraints on an interconnect path
675 * @path: interconnect path
676 * @avg_bw: average bandwidth in kilobytes per second
677 * @peak_bw: peak bandwidth in kilobytes per second
678 *
679 * This function is used by an interconnect consumer to express its own needs
680 * in terms of bandwidth for a previously requested path between two endpoints.
681 * The requests are aggregated and each node is updated accordingly. The entire
682 * path is locked by a mutex to ensure that the set() is completed.
683 * The @path can be NULL when the "interconnects" DT properties is missing,
684 * which will mean that no constraints will be set.
685 *
686 * Returns 0 on success, or an appropriate error code otherwise.
687 */
688int icc_set_bw(struct icc_path *path, u32 avg_bw, u32 peak_bw)
689{
690 struct icc_node *node;
691 u32 old_avg, old_peak;
692 size_t i;
693 int ret;
694
695 if (!path)
696 return 0;
697
698 if (WARN_ON(IS_ERR(path) || !path->num_nodes))
699 return -EINVAL;
700
701 mutex_lock(&icc_bw_lock);
702
703 old_avg = path->reqs[0].avg_bw;
704 old_peak = path->reqs[0].peak_bw;
705
706 for (i = 0; i < path->num_nodes; i++) {
707 node = path->reqs[i].node;
708
709 /* update the consumer request for this path */
710 path->reqs[i].avg_bw = avg_bw;
711 path->reqs[i].peak_bw = peak_bw;
712
713 /* aggregate requests for this node */
714 aggregate_requests(node);
715
716 trace_icc_set_bw(path, node, i, avg_bw, peak_bw);
717 }
718
719 ret = apply_constraints(path);
720 if (ret) {
721 pr_debug("interconnect: error applying constraints (%d)\n",
722 ret);
723
724 for (i = 0; i < path->num_nodes; i++) {
725 node = path->reqs[i].node;
726 path->reqs[i].avg_bw = old_avg;
727 path->reqs[i].peak_bw = old_peak;
728 aggregate_requests(node);
729 }
730 apply_constraints(path);
731 }
732
733 mutex_unlock(&icc_bw_lock);
734
735 trace_icc_set_bw_end(path, ret);
736
737 return ret;
738}
739EXPORT_SYMBOL_GPL(icc_set_bw);
740
741static int __icc_enable(struct icc_path *path, bool enable)
742{
743 int i;
744
745 if (!path)
746 return 0;
747
748 if (WARN_ON(IS_ERR(path) || !path->num_nodes))
749 return -EINVAL;
750
751 mutex_lock(&icc_lock);
752
753 for (i = 0; i < path->num_nodes; i++)
754 path->reqs[i].enabled = enable;
755
756 mutex_unlock(&icc_lock);
757
758 return icc_set_bw(path, path->reqs[0].avg_bw,
759 path->reqs[0].peak_bw);
760}
761
762int icc_enable(struct icc_path *path)
763{
764 return __icc_enable(path, true);
765}
766EXPORT_SYMBOL_GPL(icc_enable);
767
768int icc_disable(struct icc_path *path)
769{
770 return __icc_enable(path, false);
771}
772EXPORT_SYMBOL_GPL(icc_disable);
773
774/**
775 * icc_put() - release the reference to the icc_path
776 * @path: interconnect path
777 *
778 * Use this function to release the constraints on a path when the path is
779 * no longer needed. The constraints will be re-aggregated.
780 */
781void icc_put(struct icc_path *path)
782{
783 struct icc_node *node;
784 size_t i;
785 int ret;
786
787 if (!path || WARN_ON(IS_ERR(path)))
788 return;
789
790 ret = icc_set_bw(path, 0, 0);
791 if (ret)
792 pr_err("%s: error (%d)\n", __func__, ret);
793
794 mutex_lock(&icc_lock);
795 for (i = 0; i < path->num_nodes; i++) {
796 node = path->reqs[i].node;
797 hlist_del(&path->reqs[i].req_node);
798 if (!WARN_ON(!node->provider->users))
799 node->provider->users--;
800 }
801 mutex_unlock(&icc_lock);
802
803 kfree_const(path->name);
804 kfree(path);
805}
806EXPORT_SYMBOL_GPL(icc_put);
807
808static struct icc_node *icc_node_create_nolock(int id)
809{
810 struct icc_node *node;
811
812 /* check if node already exists */
813 node = node_find(id);
814 if (node)
815 return node;
816
817 node = kzalloc(sizeof(*node), GFP_KERNEL);
818 if (!node)
819 return ERR_PTR(-ENOMEM);
820
821 id = idr_alloc(&icc_idr, node, id, id + 1, GFP_KERNEL);
822 if (id < 0) {
823 WARN(1, "%s: couldn't get idr\n", __func__);
824 kfree(node);
825 return ERR_PTR(id);
826 }
827
828 node->id = id;
829
830 return node;
831}
832
833/**
834 * icc_node_create() - create a node
835 * @id: node id
836 *
837 * Return: icc_node pointer on success, or ERR_PTR() on error
838 */
839struct icc_node *icc_node_create(int id)
840{
841 struct icc_node *node;
842
843 mutex_lock(&icc_lock);
844
845 node = icc_node_create_nolock(id);
846
847 mutex_unlock(&icc_lock);
848
849 return node;
850}
851EXPORT_SYMBOL_GPL(icc_node_create);
852
853/**
854 * icc_node_destroy() - destroy a node
855 * @id: node id
856 */
857void icc_node_destroy(int id)
858{
859 struct icc_node *node;
860
861 mutex_lock(&icc_lock);
862
863 node = node_find(id);
864 if (node) {
865 idr_remove(&icc_idr, node->id);
866 WARN_ON(!hlist_empty(&node->req_list));
867 }
868
869 mutex_unlock(&icc_lock);
870
871 if (!node)
872 return;
873
874 kfree(node->links);
875 kfree(node);
876}
877EXPORT_SYMBOL_GPL(icc_node_destroy);
878
879/**
880 * icc_link_create() - create a link between two nodes
881 * @node: source node id
882 * @dst_id: destination node id
883 *
884 * Create a link between two nodes. The nodes might belong to different
885 * interconnect providers and the @dst_id node might not exist (if the
886 * provider driver has not probed yet). So just create the @dst_id node
887 * and when the actual provider driver is probed, the rest of the node
888 * data is filled.
889 *
890 * Return: 0 on success, or an error code otherwise
891 */
892int icc_link_create(struct icc_node *node, const int dst_id)
893{
894 struct icc_node *dst;
895 struct icc_node **new;
896 int ret = 0;
897
898 if (!node->provider)
899 return -EINVAL;
900
901 mutex_lock(&icc_lock);
902
903 dst = node_find(dst_id);
904 if (!dst) {
905 dst = icc_node_create_nolock(dst_id);
906
907 if (IS_ERR(dst)) {
908 ret = PTR_ERR(dst);
909 goto out;
910 }
911 }
912
913 new = krealloc(node->links,
914 (node->num_links + 1) * sizeof(*node->links),
915 GFP_KERNEL);
916 if (!new) {
917 ret = -ENOMEM;
918 goto out;
919 }
920
921 node->links = new;
922 node->links[node->num_links++] = dst;
923
924out:
925 mutex_unlock(&icc_lock);
926
927 return ret;
928}
929EXPORT_SYMBOL_GPL(icc_link_create);
930
931/**
932 * icc_node_add() - add interconnect node to interconnect provider
933 * @node: pointer to the interconnect node
934 * @provider: pointer to the interconnect provider
935 */
936void icc_node_add(struct icc_node *node, struct icc_provider *provider)
937{
938 if (WARN_ON(node->provider))
939 return;
940
941 mutex_lock(&icc_lock);
942 mutex_lock(&icc_bw_lock);
943
944 node->provider = provider;
945 list_add_tail(&node->node_list, &provider->nodes);
946
947 /* get the initial bandwidth values and sync them with hardware */
948 if (provider->get_bw) {
949 provider->get_bw(node, &node->init_avg, &node->init_peak);
950 } else {
951 node->init_avg = INT_MAX;
952 node->init_peak = INT_MAX;
953 }
954 node->avg_bw = node->init_avg;
955 node->peak_bw = node->init_peak;
956
957 if (node->avg_bw || node->peak_bw) {
958 if (provider->pre_aggregate)
959 provider->pre_aggregate(node);
960
961 if (provider->aggregate)
962 provider->aggregate(node, 0, node->init_avg, node->init_peak,
963 &node->avg_bw, &node->peak_bw);
964 if (provider->set)
965 provider->set(node, node);
966 }
967
968 node->avg_bw = 0;
969 node->peak_bw = 0;
970
971 mutex_unlock(&icc_bw_lock);
972 mutex_unlock(&icc_lock);
973}
974EXPORT_SYMBOL_GPL(icc_node_add);
975
976/**
977 * icc_node_del() - delete interconnect node from interconnect provider
978 * @node: pointer to the interconnect node
979 */
980void icc_node_del(struct icc_node *node)
981{
982 mutex_lock(&icc_lock);
983
984 list_del(&node->node_list);
985
986 mutex_unlock(&icc_lock);
987}
988EXPORT_SYMBOL_GPL(icc_node_del);
989
990/**
991 * icc_nodes_remove() - remove all previously added nodes from provider
992 * @provider: the interconnect provider we are removing nodes from
993 *
994 * Return: 0 on success, or an error code otherwise
995 */
996int icc_nodes_remove(struct icc_provider *provider)
997{
998 struct icc_node *n, *tmp;
999
1000 if (WARN_ON(IS_ERR_OR_NULL(provider)))
1001 return -EINVAL;
1002
1003 list_for_each_entry_safe_reverse(n, tmp, &provider->nodes, node_list) {
1004 icc_node_del(n);
1005 icc_node_destroy(n->id);
1006 }
1007
1008 return 0;
1009}
1010EXPORT_SYMBOL_GPL(icc_nodes_remove);
1011
1012/**
1013 * icc_provider_init() - initialize a new interconnect provider
1014 * @provider: the interconnect provider to initialize
1015 *
1016 * Must be called before adding nodes to the provider.
1017 */
1018void icc_provider_init(struct icc_provider *provider)
1019{
1020 WARN_ON(!provider->set);
1021
1022 INIT_LIST_HEAD(&provider->nodes);
1023}
1024EXPORT_SYMBOL_GPL(icc_provider_init);
1025
1026/**
1027 * icc_provider_register() - register a new interconnect provider
1028 * @provider: the interconnect provider to register
1029 *
1030 * Return: 0 on success, or an error code otherwise
1031 */
1032int icc_provider_register(struct icc_provider *provider)
1033{
1034 if (WARN_ON(!provider->xlate && !provider->xlate_extended))
1035 return -EINVAL;
1036
1037 mutex_lock(&icc_lock);
1038 list_add_tail(&provider->provider_list, &icc_providers);
1039 mutex_unlock(&icc_lock);
1040
1041 dev_dbg(provider->dev, "interconnect provider registered\n");
1042
1043 return 0;
1044}
1045EXPORT_SYMBOL_GPL(icc_provider_register);
1046
1047/**
1048 * icc_provider_deregister() - deregister an interconnect provider
1049 * @provider: the interconnect provider to deregister
1050 */
1051void icc_provider_deregister(struct icc_provider *provider)
1052{
1053 mutex_lock(&icc_lock);
1054 WARN_ON(provider->users);
1055
1056 list_del(&provider->provider_list);
1057 mutex_unlock(&icc_lock);
1058}
1059EXPORT_SYMBOL_GPL(icc_provider_deregister);
1060
1061static const struct of_device_id __maybe_unused ignore_list[] = {
1062 { .compatible = "qcom,sc7180-ipa-virt" },
1063 { .compatible = "qcom,sc8180x-ipa-virt" },
1064 { .compatible = "qcom,sdx55-ipa-virt" },
1065 { .compatible = "qcom,sm8150-ipa-virt" },
1066 { .compatible = "qcom,sm8250-ipa-virt" },
1067 {}
1068};
1069
1070static int of_count_icc_providers(struct device_node *np)
1071{
1072 struct device_node *child;
1073 int count = 0;
1074
1075 for_each_available_child_of_node(np, child) {
1076 if (of_property_read_bool(child, "#interconnect-cells") &&
1077 likely(!of_match_node(ignore_list, child)))
1078 count++;
1079 count += of_count_icc_providers(child);
1080 }
1081
1082 return count;
1083}
1084
1085void icc_sync_state(struct device *dev)
1086{
1087 struct icc_provider *p;
1088 struct icc_node *n;
1089 static int count;
1090
1091 count++;
1092
1093 if (count < providers_count)
1094 return;
1095
1096 mutex_lock(&icc_lock);
1097 mutex_lock(&icc_bw_lock);
1098 synced_state = true;
1099 list_for_each_entry(p, &icc_providers, provider_list) {
1100 dev_dbg(p->dev, "interconnect provider is in synced state\n");
1101 list_for_each_entry(n, &p->nodes, node_list) {
1102 if (n->init_avg || n->init_peak) {
1103 n->init_avg = 0;
1104 n->init_peak = 0;
1105 aggregate_requests(n);
1106 p->set(n, n);
1107 }
1108 }
1109 }
1110 mutex_unlock(&icc_bw_lock);
1111 mutex_unlock(&icc_lock);
1112}
1113EXPORT_SYMBOL_GPL(icc_sync_state);
1114
1115static int __init icc_init(void)
1116{
1117 struct device_node *root;
1118
1119 /* Teach lockdep about lock ordering wrt. shrinker: */
1120 fs_reclaim_acquire(GFP_KERNEL);
1121 might_lock(&icc_bw_lock);
1122 fs_reclaim_release(GFP_KERNEL);
1123
1124 root = of_find_node_by_path("/");
1125
1126 providers_count = of_count_icc_providers(root);
1127 of_node_put(root);
1128
1129 icc_debugfs_dir = debugfs_create_dir("interconnect", NULL);
1130 debugfs_create_file("interconnect_summary", 0444,
1131 icc_debugfs_dir, NULL, &icc_summary_fops);
1132 debugfs_create_file("interconnect_graph", 0444,
1133 icc_debugfs_dir, NULL, &icc_graph_fops);
1134
1135 icc_debugfs_client_init(icc_debugfs_dir);
1136
1137 return 0;
1138}
1139
1140device_initcall(icc_init);
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Interconnect framework core driver
4 *
5 * Copyright (c) 2017-2019, Linaro Ltd.
6 * Author: Georgi Djakov <georgi.djakov@linaro.org>
7 */
8
9#include <linux/debugfs.h>
10#include <linux/device.h>
11#include <linux/idr.h>
12#include <linux/init.h>
13#include <linux/interconnect.h>
14#include <linux/interconnect-provider.h>
15#include <linux/list.h>
16#include <linux/module.h>
17#include <linux/mutex.h>
18#include <linux/slab.h>
19#include <linux/of.h>
20#include <linux/overflow.h>
21
22#include "internal.h"
23
24#define CREATE_TRACE_POINTS
25#include "trace.h"
26
27static DEFINE_IDR(icc_idr);
28static LIST_HEAD(icc_providers);
29static DEFINE_MUTEX(icc_lock);
30static struct dentry *icc_debugfs_dir;
31
32static void icc_summary_show_one(struct seq_file *s, struct icc_node *n)
33{
34 if (!n)
35 return;
36
37 seq_printf(s, "%-42s %12u %12u\n",
38 n->name, n->avg_bw, n->peak_bw);
39}
40
41static int icc_summary_show(struct seq_file *s, void *data)
42{
43 struct icc_provider *provider;
44
45 seq_puts(s, " node tag avg peak\n");
46 seq_puts(s, "--------------------------------------------------------------------\n");
47
48 mutex_lock(&icc_lock);
49
50 list_for_each_entry(provider, &icc_providers, provider_list) {
51 struct icc_node *n;
52
53 list_for_each_entry(n, &provider->nodes, node_list) {
54 struct icc_req *r;
55
56 icc_summary_show_one(s, n);
57 hlist_for_each_entry(r, &n->req_list, req_node) {
58 u32 avg_bw = 0, peak_bw = 0;
59
60 if (!r->dev)
61 continue;
62
63 if (r->enabled) {
64 avg_bw = r->avg_bw;
65 peak_bw = r->peak_bw;
66 }
67
68 seq_printf(s, " %-27s %12u %12u %12u\n",
69 dev_name(r->dev), r->tag, avg_bw, peak_bw);
70 }
71 }
72 }
73
74 mutex_unlock(&icc_lock);
75
76 return 0;
77}
78DEFINE_SHOW_ATTRIBUTE(icc_summary);
79
80static void icc_graph_show_link(struct seq_file *s, int level,
81 struct icc_node *n, struct icc_node *m)
82{
83 seq_printf(s, "%s\"%d:%s\" -> \"%d:%s\"\n",
84 level == 2 ? "\t\t" : "\t",
85 n->id, n->name, m->id, m->name);
86}
87
88static void icc_graph_show_node(struct seq_file *s, struct icc_node *n)
89{
90 seq_printf(s, "\t\t\"%d:%s\" [label=\"%d:%s",
91 n->id, n->name, n->id, n->name);
92 seq_printf(s, "\n\t\t\t|avg_bw=%ukBps", n->avg_bw);
93 seq_printf(s, "\n\t\t\t|peak_bw=%ukBps", n->peak_bw);
94 seq_puts(s, "\"]\n");
95}
96
97static int icc_graph_show(struct seq_file *s, void *data)
98{
99 struct icc_provider *provider;
100 struct icc_node *n;
101 int cluster_index = 0;
102 int i;
103
104 seq_puts(s, "digraph {\n\trankdir = LR\n\tnode [shape = record]\n");
105 mutex_lock(&icc_lock);
106
107 /* draw providers as cluster subgraphs */
108 cluster_index = 0;
109 list_for_each_entry(provider, &icc_providers, provider_list) {
110 seq_printf(s, "\tsubgraph cluster_%d {\n", ++cluster_index);
111 if (provider->dev)
112 seq_printf(s, "\t\tlabel = \"%s\"\n",
113 dev_name(provider->dev));
114
115 /* draw nodes */
116 list_for_each_entry(n, &provider->nodes, node_list)
117 icc_graph_show_node(s, n);
118
119 /* draw internal links */
120 list_for_each_entry(n, &provider->nodes, node_list)
121 for (i = 0; i < n->num_links; ++i)
122 if (n->provider == n->links[i]->provider)
123 icc_graph_show_link(s, 2, n,
124 n->links[i]);
125
126 seq_puts(s, "\t}\n");
127 }
128
129 /* draw external links */
130 list_for_each_entry(provider, &icc_providers, provider_list)
131 list_for_each_entry(n, &provider->nodes, node_list)
132 for (i = 0; i < n->num_links; ++i)
133 if (n->provider != n->links[i]->provider)
134 icc_graph_show_link(s, 1, n,
135 n->links[i]);
136
137 mutex_unlock(&icc_lock);
138 seq_puts(s, "}");
139
140 return 0;
141}
142DEFINE_SHOW_ATTRIBUTE(icc_graph);
143
144static struct icc_node *node_find(const int id)
145{
146 return idr_find(&icc_idr, id);
147}
148
149static struct icc_path *path_init(struct device *dev, struct icc_node *dst,
150 ssize_t num_nodes)
151{
152 struct icc_node *node = dst;
153 struct icc_path *path;
154 int i;
155
156 path = kzalloc(struct_size(path, reqs, num_nodes), GFP_KERNEL);
157 if (!path)
158 return ERR_PTR(-ENOMEM);
159
160 path->num_nodes = num_nodes;
161
162 for (i = num_nodes - 1; i >= 0; i--) {
163 node->provider->users++;
164 hlist_add_head(&path->reqs[i].req_node, &node->req_list);
165 path->reqs[i].node = node;
166 path->reqs[i].dev = dev;
167 path->reqs[i].enabled = true;
168 /* reference to previous node was saved during path traversal */
169 node = node->reverse;
170 }
171
172 return path;
173}
174
175static struct icc_path *path_find(struct device *dev, struct icc_node *src,
176 struct icc_node *dst)
177{
178 struct icc_path *path = ERR_PTR(-EPROBE_DEFER);
179 struct icc_node *n, *node = NULL;
180 struct list_head traverse_list;
181 struct list_head edge_list;
182 struct list_head visited_list;
183 size_t i, depth = 1;
184 bool found = false;
185
186 INIT_LIST_HEAD(&traverse_list);
187 INIT_LIST_HEAD(&edge_list);
188 INIT_LIST_HEAD(&visited_list);
189
190 list_add(&src->search_list, &traverse_list);
191 src->reverse = NULL;
192
193 do {
194 list_for_each_entry_safe(node, n, &traverse_list, search_list) {
195 if (node == dst) {
196 found = true;
197 list_splice_init(&edge_list, &visited_list);
198 list_splice_init(&traverse_list, &visited_list);
199 break;
200 }
201 for (i = 0; i < node->num_links; i++) {
202 struct icc_node *tmp = node->links[i];
203
204 if (!tmp) {
205 path = ERR_PTR(-ENOENT);
206 goto out;
207 }
208
209 if (tmp->is_traversed)
210 continue;
211
212 tmp->is_traversed = true;
213 tmp->reverse = node;
214 list_add_tail(&tmp->search_list, &edge_list);
215 }
216 }
217
218 if (found)
219 break;
220
221 list_splice_init(&traverse_list, &visited_list);
222 list_splice_init(&edge_list, &traverse_list);
223
224 /* count the hops including the source */
225 depth++;
226
227 } while (!list_empty(&traverse_list));
228
229out:
230
231 /* reset the traversed state */
232 list_for_each_entry_reverse(n, &visited_list, search_list)
233 n->is_traversed = false;
234
235 if (found)
236 path = path_init(dev, dst, depth);
237
238 return path;
239}
240
241/*
242 * We want the path to honor all bandwidth requests, so the average and peak
243 * bandwidth requirements from each consumer are aggregated at each node.
244 * The aggregation is platform specific, so each platform can customize it by
245 * implementing its own aggregate() function.
246 */
247
248static int aggregate_requests(struct icc_node *node)
249{
250 struct icc_provider *p = node->provider;
251 struct icc_req *r;
252 u32 avg_bw, peak_bw;
253
254 node->avg_bw = 0;
255 node->peak_bw = 0;
256
257 if (p->pre_aggregate)
258 p->pre_aggregate(node);
259
260 hlist_for_each_entry(r, &node->req_list, req_node) {
261 if (r->enabled) {
262 avg_bw = r->avg_bw;
263 peak_bw = r->peak_bw;
264 } else {
265 avg_bw = 0;
266 peak_bw = 0;
267 }
268 p->aggregate(node, r->tag, avg_bw, peak_bw,
269 &node->avg_bw, &node->peak_bw);
270 }
271
272 return 0;
273}
274
275static int apply_constraints(struct icc_path *path)
276{
277 struct icc_node *next, *prev = NULL;
278 struct icc_provider *p;
279 int ret = -EINVAL;
280 int i;
281
282 for (i = 0; i < path->num_nodes; i++) {
283 next = path->reqs[i].node;
284 p = next->provider;
285
286 /* both endpoints should be valid master-slave pairs */
287 if (!prev || (p != prev->provider && !p->inter_set)) {
288 prev = next;
289 continue;
290 }
291
292 /* set the constraints */
293 ret = p->set(prev, next);
294 if (ret)
295 goto out;
296
297 prev = next;
298 }
299out:
300 return ret;
301}
302
303int icc_std_aggregate(struct icc_node *node, u32 tag, u32 avg_bw,
304 u32 peak_bw, u32 *agg_avg, u32 *agg_peak)
305{
306 *agg_avg += avg_bw;
307 *agg_peak = max(*agg_peak, peak_bw);
308
309 return 0;
310}
311EXPORT_SYMBOL_GPL(icc_std_aggregate);
312
313/* of_icc_xlate_onecell() - Translate function using a single index.
314 * @spec: OF phandle args to map into an interconnect node.
315 * @data: private data (pointer to struct icc_onecell_data)
316 *
317 * This is a generic translate function that can be used to model simple
318 * interconnect providers that have one device tree node and provide
319 * multiple interconnect nodes. A single cell is used as an index into
320 * an array of icc nodes specified in the icc_onecell_data struct when
321 * registering the provider.
322 */
323struct icc_node *of_icc_xlate_onecell(struct of_phandle_args *spec,
324 void *data)
325{
326 struct icc_onecell_data *icc_data = data;
327 unsigned int idx = spec->args[0];
328
329 if (idx >= icc_data->num_nodes) {
330 pr_err("%s: invalid index %u\n", __func__, idx);
331 return ERR_PTR(-EINVAL);
332 }
333
334 return icc_data->nodes[idx];
335}
336EXPORT_SYMBOL_GPL(of_icc_xlate_onecell);
337
338/**
339 * of_icc_get_from_provider() - Look-up interconnect node
340 * @spec: OF phandle args to use for look-up
341 *
342 * Looks for interconnect provider under the node specified by @spec and if
343 * found, uses xlate function of the provider to map phandle args to node.
344 *
345 * Returns a valid pointer to struct icc_node on success or ERR_PTR()
346 * on failure.
347 */
348struct icc_node *of_icc_get_from_provider(struct of_phandle_args *spec)
349{
350 struct icc_node *node = ERR_PTR(-EPROBE_DEFER);
351 struct icc_provider *provider;
352
353 if (!spec)
354 return ERR_PTR(-EINVAL);
355
356 mutex_lock(&icc_lock);
357 list_for_each_entry(provider, &icc_providers, provider_list) {
358 if (provider->dev->of_node == spec->np)
359 node = provider->xlate(spec, provider->data);
360 if (!IS_ERR(node))
361 break;
362 }
363 mutex_unlock(&icc_lock);
364
365 return node;
366}
367EXPORT_SYMBOL_GPL(of_icc_get_from_provider);
368
369static void devm_icc_release(struct device *dev, void *res)
370{
371 icc_put(*(struct icc_path **)res);
372}
373
374struct icc_path *devm_of_icc_get(struct device *dev, const char *name)
375{
376 struct icc_path **ptr, *path;
377
378 ptr = devres_alloc(devm_icc_release, sizeof(**ptr), GFP_KERNEL);
379 if (!ptr)
380 return ERR_PTR(-ENOMEM);
381
382 path = of_icc_get(dev, name);
383 if (!IS_ERR(path)) {
384 *ptr = path;
385 devres_add(dev, ptr);
386 } else {
387 devres_free(ptr);
388 }
389
390 return path;
391}
392EXPORT_SYMBOL_GPL(devm_of_icc_get);
393
394/**
395 * of_icc_get_by_index() - get a path handle from a DT node based on index
396 * @dev: device pointer for the consumer device
397 * @idx: interconnect path index
398 *
399 * This function will search for a path between two endpoints and return an
400 * icc_path handle on success. Use icc_put() to release constraints when they
401 * are not needed anymore.
402 * If the interconnect API is disabled, NULL is returned and the consumer
403 * drivers will still build. Drivers are free to handle this specifically,
404 * but they don't have to.
405 *
406 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
407 * when the API is disabled or the "interconnects" DT property is missing.
408 */
409struct icc_path *of_icc_get_by_index(struct device *dev, int idx)
410{
411 struct icc_path *path;
412 struct icc_node *src_node, *dst_node;
413 struct device_node *np;
414 struct of_phandle_args src_args, dst_args;
415 int ret;
416
417 if (!dev || !dev->of_node)
418 return ERR_PTR(-ENODEV);
419
420 np = dev->of_node;
421
422 /*
423 * When the consumer DT node do not have "interconnects" property
424 * return a NULL path to skip setting constraints.
425 */
426 if (!of_find_property(np, "interconnects", NULL))
427 return NULL;
428
429 /*
430 * We use a combination of phandle and specifier for endpoint. For now
431 * lets support only global ids and extend this in the future if needed
432 * without breaking DT compatibility.
433 */
434 ret = of_parse_phandle_with_args(np, "interconnects",
435 "#interconnect-cells", idx * 2,
436 &src_args);
437 if (ret)
438 return ERR_PTR(ret);
439
440 of_node_put(src_args.np);
441
442 ret = of_parse_phandle_with_args(np, "interconnects",
443 "#interconnect-cells", idx * 2 + 1,
444 &dst_args);
445 if (ret)
446 return ERR_PTR(ret);
447
448 of_node_put(dst_args.np);
449
450 src_node = of_icc_get_from_provider(&src_args);
451
452 if (IS_ERR(src_node)) {
453 if (PTR_ERR(src_node) != -EPROBE_DEFER)
454 dev_err(dev, "error finding src node: %ld\n",
455 PTR_ERR(src_node));
456 return ERR_CAST(src_node);
457 }
458
459 dst_node = of_icc_get_from_provider(&dst_args);
460
461 if (IS_ERR(dst_node)) {
462 if (PTR_ERR(dst_node) != -EPROBE_DEFER)
463 dev_err(dev, "error finding dst node: %ld\n",
464 PTR_ERR(dst_node));
465 return ERR_CAST(dst_node);
466 }
467
468 mutex_lock(&icc_lock);
469 path = path_find(dev, src_node, dst_node);
470 mutex_unlock(&icc_lock);
471 if (IS_ERR(path)) {
472 dev_err(dev, "%s: invalid path=%ld\n", __func__, PTR_ERR(path));
473 return path;
474 }
475
476 path->name = kasprintf(GFP_KERNEL, "%s-%s",
477 src_node->name, dst_node->name);
478 if (!path->name) {
479 kfree(path);
480 return ERR_PTR(-ENOMEM);
481 }
482
483 return path;
484}
485EXPORT_SYMBOL_GPL(of_icc_get_by_index);
486
487/**
488 * of_icc_get() - get a path handle from a DT node based on name
489 * @dev: device pointer for the consumer device
490 * @name: interconnect path name
491 *
492 * This function will search for a path between two endpoints and return an
493 * icc_path handle on success. Use icc_put() to release constraints when they
494 * are not needed anymore.
495 * If the interconnect API is disabled, NULL is returned and the consumer
496 * drivers will still build. Drivers are free to handle this specifically,
497 * but they don't have to.
498 *
499 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
500 * when the API is disabled or the "interconnects" DT property is missing.
501 */
502struct icc_path *of_icc_get(struct device *dev, const char *name)
503{
504 struct device_node *np;
505 int idx = 0;
506
507 if (!dev || !dev->of_node)
508 return ERR_PTR(-ENODEV);
509
510 np = dev->of_node;
511
512 /*
513 * When the consumer DT node do not have "interconnects" property
514 * return a NULL path to skip setting constraints.
515 */
516 if (!of_find_property(np, "interconnects", NULL))
517 return NULL;
518
519 /*
520 * We use a combination of phandle and specifier for endpoint. For now
521 * lets support only global ids and extend this in the future if needed
522 * without breaking DT compatibility.
523 */
524 if (name) {
525 idx = of_property_match_string(np, "interconnect-names", name);
526 if (idx < 0)
527 return ERR_PTR(idx);
528 }
529
530 return of_icc_get_by_index(dev, idx);
531}
532EXPORT_SYMBOL_GPL(of_icc_get);
533
534/**
535 * icc_set_tag() - set an optional tag on a path
536 * @path: the path we want to tag
537 * @tag: the tag value
538 *
539 * This function allows consumers to append a tag to the requests associated
540 * with a path, so that a different aggregation could be done based on this tag.
541 */
542void icc_set_tag(struct icc_path *path, u32 tag)
543{
544 int i;
545
546 if (!path)
547 return;
548
549 mutex_lock(&icc_lock);
550
551 for (i = 0; i < path->num_nodes; i++)
552 path->reqs[i].tag = tag;
553
554 mutex_unlock(&icc_lock);
555}
556EXPORT_SYMBOL_GPL(icc_set_tag);
557
558/**
559 * icc_get_name() - Get name of the icc path
560 * @path: reference to the path returned by icc_get()
561 *
562 * This function is used by an interconnect consumer to get the name of the icc
563 * path.
564 *
565 * Returns a valid pointer on success, or NULL otherwise.
566 */
567const char *icc_get_name(struct icc_path *path)
568{
569 if (!path)
570 return NULL;
571
572 return path->name;
573}
574EXPORT_SYMBOL_GPL(icc_get_name);
575
576/**
577 * icc_set_bw() - set bandwidth constraints on an interconnect path
578 * @path: reference to the path returned by icc_get()
579 * @avg_bw: average bandwidth in kilobytes per second
580 * @peak_bw: peak bandwidth in kilobytes per second
581 *
582 * This function is used by an interconnect consumer to express its own needs
583 * in terms of bandwidth for a previously requested path between two endpoints.
584 * The requests are aggregated and each node is updated accordingly. The entire
585 * path is locked by a mutex to ensure that the set() is completed.
586 * The @path can be NULL when the "interconnects" DT properties is missing,
587 * which will mean that no constraints will be set.
588 *
589 * Returns 0 on success, or an appropriate error code otherwise.
590 */
591int icc_set_bw(struct icc_path *path, u32 avg_bw, u32 peak_bw)
592{
593 struct icc_node *node;
594 u32 old_avg, old_peak;
595 size_t i;
596 int ret;
597
598 if (!path)
599 return 0;
600
601 if (WARN_ON(IS_ERR(path) || !path->num_nodes))
602 return -EINVAL;
603
604 mutex_lock(&icc_lock);
605
606 old_avg = path->reqs[0].avg_bw;
607 old_peak = path->reqs[0].peak_bw;
608
609 for (i = 0; i < path->num_nodes; i++) {
610 node = path->reqs[i].node;
611
612 /* update the consumer request for this path */
613 path->reqs[i].avg_bw = avg_bw;
614 path->reqs[i].peak_bw = peak_bw;
615
616 /* aggregate requests for this node */
617 aggregate_requests(node);
618
619 trace_icc_set_bw(path, node, i, avg_bw, peak_bw);
620 }
621
622 ret = apply_constraints(path);
623 if (ret) {
624 pr_debug("interconnect: error applying constraints (%d)\n",
625 ret);
626
627 for (i = 0; i < path->num_nodes; i++) {
628 node = path->reqs[i].node;
629 path->reqs[i].avg_bw = old_avg;
630 path->reqs[i].peak_bw = old_peak;
631 aggregate_requests(node);
632 }
633 apply_constraints(path);
634 }
635
636 mutex_unlock(&icc_lock);
637
638 trace_icc_set_bw_end(path, ret);
639
640 return ret;
641}
642EXPORT_SYMBOL_GPL(icc_set_bw);
643
644static int __icc_enable(struct icc_path *path, bool enable)
645{
646 int i;
647
648 if (!path)
649 return 0;
650
651 if (WARN_ON(IS_ERR(path) || !path->num_nodes))
652 return -EINVAL;
653
654 mutex_lock(&icc_lock);
655
656 for (i = 0; i < path->num_nodes; i++)
657 path->reqs[i].enabled = enable;
658
659 mutex_unlock(&icc_lock);
660
661 return icc_set_bw(path, path->reqs[0].avg_bw,
662 path->reqs[0].peak_bw);
663}
664
665int icc_enable(struct icc_path *path)
666{
667 return __icc_enable(path, true);
668}
669EXPORT_SYMBOL_GPL(icc_enable);
670
671int icc_disable(struct icc_path *path)
672{
673 return __icc_enable(path, false);
674}
675EXPORT_SYMBOL_GPL(icc_disable);
676
677/**
678 * icc_get() - return a handle for path between two endpoints
679 * @dev: the device requesting the path
680 * @src_id: source device port id
681 * @dst_id: destination device port id
682 *
683 * This function will search for a path between two endpoints and return an
684 * icc_path handle on success. Use icc_put() to release
685 * constraints when they are not needed anymore.
686 * If the interconnect API is disabled, NULL is returned and the consumer
687 * drivers will still build. Drivers are free to handle this specifically,
688 * but they don't have to.
689 *
690 * Return: icc_path pointer on success, ERR_PTR() on error or NULL if the
691 * interconnect API is disabled.
692 */
693struct icc_path *icc_get(struct device *dev, const int src_id, const int dst_id)
694{
695 struct icc_node *src, *dst;
696 struct icc_path *path = ERR_PTR(-EPROBE_DEFER);
697
698 mutex_lock(&icc_lock);
699
700 src = node_find(src_id);
701 if (!src)
702 goto out;
703
704 dst = node_find(dst_id);
705 if (!dst)
706 goto out;
707
708 path = path_find(dev, src, dst);
709 if (IS_ERR(path)) {
710 dev_err(dev, "%s: invalid path=%ld\n", __func__, PTR_ERR(path));
711 goto out;
712 }
713
714 path->name = kasprintf(GFP_KERNEL, "%s-%s", src->name, dst->name);
715 if (!path->name) {
716 kfree(path);
717 path = ERR_PTR(-ENOMEM);
718 }
719out:
720 mutex_unlock(&icc_lock);
721 return path;
722}
723EXPORT_SYMBOL_GPL(icc_get);
724
725/**
726 * icc_put() - release the reference to the icc_path
727 * @path: interconnect path
728 *
729 * Use this function to release the constraints on a path when the path is
730 * no longer needed. The constraints will be re-aggregated.
731 */
732void icc_put(struct icc_path *path)
733{
734 struct icc_node *node;
735 size_t i;
736 int ret;
737
738 if (!path || WARN_ON(IS_ERR(path)))
739 return;
740
741 ret = icc_set_bw(path, 0, 0);
742 if (ret)
743 pr_err("%s: error (%d)\n", __func__, ret);
744
745 mutex_lock(&icc_lock);
746 for (i = 0; i < path->num_nodes; i++) {
747 node = path->reqs[i].node;
748 hlist_del(&path->reqs[i].req_node);
749 if (!WARN_ON(!node->provider->users))
750 node->provider->users--;
751 }
752 mutex_unlock(&icc_lock);
753
754 kfree_const(path->name);
755 kfree(path);
756}
757EXPORT_SYMBOL_GPL(icc_put);
758
759static struct icc_node *icc_node_create_nolock(int id)
760{
761 struct icc_node *node;
762
763 /* check if node already exists */
764 node = node_find(id);
765 if (node)
766 return node;
767
768 node = kzalloc(sizeof(*node), GFP_KERNEL);
769 if (!node)
770 return ERR_PTR(-ENOMEM);
771
772 id = idr_alloc(&icc_idr, node, id, id + 1, GFP_KERNEL);
773 if (id < 0) {
774 WARN(1, "%s: couldn't get idr\n", __func__);
775 kfree(node);
776 return ERR_PTR(id);
777 }
778
779 node->id = id;
780
781 return node;
782}
783
784/**
785 * icc_node_create() - create a node
786 * @id: node id
787 *
788 * Return: icc_node pointer on success, or ERR_PTR() on error
789 */
790struct icc_node *icc_node_create(int id)
791{
792 struct icc_node *node;
793
794 mutex_lock(&icc_lock);
795
796 node = icc_node_create_nolock(id);
797
798 mutex_unlock(&icc_lock);
799
800 return node;
801}
802EXPORT_SYMBOL_GPL(icc_node_create);
803
804/**
805 * icc_node_destroy() - destroy a node
806 * @id: node id
807 */
808void icc_node_destroy(int id)
809{
810 struct icc_node *node;
811
812 mutex_lock(&icc_lock);
813
814 node = node_find(id);
815 if (node) {
816 idr_remove(&icc_idr, node->id);
817 WARN_ON(!hlist_empty(&node->req_list));
818 }
819
820 mutex_unlock(&icc_lock);
821
822 kfree(node);
823}
824EXPORT_SYMBOL_GPL(icc_node_destroy);
825
826/**
827 * icc_link_create() - create a link between two nodes
828 * @node: source node id
829 * @dst_id: destination node id
830 *
831 * Create a link between two nodes. The nodes might belong to different
832 * interconnect providers and the @dst_id node might not exist (if the
833 * provider driver has not probed yet). So just create the @dst_id node
834 * and when the actual provider driver is probed, the rest of the node
835 * data is filled.
836 *
837 * Return: 0 on success, or an error code otherwise
838 */
839int icc_link_create(struct icc_node *node, const int dst_id)
840{
841 struct icc_node *dst;
842 struct icc_node **new;
843 int ret = 0;
844
845 if (!node->provider)
846 return -EINVAL;
847
848 mutex_lock(&icc_lock);
849
850 dst = node_find(dst_id);
851 if (!dst) {
852 dst = icc_node_create_nolock(dst_id);
853
854 if (IS_ERR(dst)) {
855 ret = PTR_ERR(dst);
856 goto out;
857 }
858 }
859
860 new = krealloc(node->links,
861 (node->num_links + 1) * sizeof(*node->links),
862 GFP_KERNEL);
863 if (!new) {
864 ret = -ENOMEM;
865 goto out;
866 }
867
868 node->links = new;
869 node->links[node->num_links++] = dst;
870
871out:
872 mutex_unlock(&icc_lock);
873
874 return ret;
875}
876EXPORT_SYMBOL_GPL(icc_link_create);
877
878/**
879 * icc_link_destroy() - destroy a link between two nodes
880 * @src: pointer to source node
881 * @dst: pointer to destination node
882 *
883 * Return: 0 on success, or an error code otherwise
884 */
885int icc_link_destroy(struct icc_node *src, struct icc_node *dst)
886{
887 struct icc_node **new;
888 size_t slot;
889 int ret = 0;
890
891 if (IS_ERR_OR_NULL(src))
892 return -EINVAL;
893
894 if (IS_ERR_OR_NULL(dst))
895 return -EINVAL;
896
897 mutex_lock(&icc_lock);
898
899 for (slot = 0; slot < src->num_links; slot++)
900 if (src->links[slot] == dst)
901 break;
902
903 if (WARN_ON(slot == src->num_links)) {
904 ret = -ENXIO;
905 goto out;
906 }
907
908 src->links[slot] = src->links[--src->num_links];
909
910 new = krealloc(src->links, src->num_links * sizeof(*src->links),
911 GFP_KERNEL);
912 if (new)
913 src->links = new;
914
915out:
916 mutex_unlock(&icc_lock);
917
918 return ret;
919}
920EXPORT_SYMBOL_GPL(icc_link_destroy);
921
922/**
923 * icc_node_add() - add interconnect node to interconnect provider
924 * @node: pointer to the interconnect node
925 * @provider: pointer to the interconnect provider
926 */
927void icc_node_add(struct icc_node *node, struct icc_provider *provider)
928{
929 mutex_lock(&icc_lock);
930
931 node->provider = provider;
932 list_add_tail(&node->node_list, &provider->nodes);
933
934 mutex_unlock(&icc_lock);
935}
936EXPORT_SYMBOL_GPL(icc_node_add);
937
938/**
939 * icc_node_del() - delete interconnect node from interconnect provider
940 * @node: pointer to the interconnect node
941 */
942void icc_node_del(struct icc_node *node)
943{
944 mutex_lock(&icc_lock);
945
946 list_del(&node->node_list);
947
948 mutex_unlock(&icc_lock);
949}
950EXPORT_SYMBOL_GPL(icc_node_del);
951
952/**
953 * icc_nodes_remove() - remove all previously added nodes from provider
954 * @provider: the interconnect provider we are removing nodes from
955 *
956 * Return: 0 on success, or an error code otherwise
957 */
958int icc_nodes_remove(struct icc_provider *provider)
959{
960 struct icc_node *n, *tmp;
961
962 if (WARN_ON(IS_ERR_OR_NULL(provider)))
963 return -EINVAL;
964
965 list_for_each_entry_safe_reverse(n, tmp, &provider->nodes, node_list) {
966 icc_node_del(n);
967 icc_node_destroy(n->id);
968 }
969
970 return 0;
971}
972EXPORT_SYMBOL_GPL(icc_nodes_remove);
973
974/**
975 * icc_provider_add() - add a new interconnect provider
976 * @provider: the interconnect provider that will be added into topology
977 *
978 * Return: 0 on success, or an error code otherwise
979 */
980int icc_provider_add(struct icc_provider *provider)
981{
982 if (WARN_ON(!provider->set))
983 return -EINVAL;
984 if (WARN_ON(!provider->xlate))
985 return -EINVAL;
986
987 mutex_lock(&icc_lock);
988
989 INIT_LIST_HEAD(&provider->nodes);
990 list_add_tail(&provider->provider_list, &icc_providers);
991
992 mutex_unlock(&icc_lock);
993
994 dev_dbg(provider->dev, "interconnect provider added to topology\n");
995
996 return 0;
997}
998EXPORT_SYMBOL_GPL(icc_provider_add);
999
1000/**
1001 * icc_provider_del() - delete previously added interconnect provider
1002 * @provider: the interconnect provider that will be removed from topology
1003 *
1004 * Return: 0 on success, or an error code otherwise
1005 */
1006int icc_provider_del(struct icc_provider *provider)
1007{
1008 mutex_lock(&icc_lock);
1009 if (provider->users) {
1010 pr_warn("interconnect provider still has %d users\n",
1011 provider->users);
1012 mutex_unlock(&icc_lock);
1013 return -EBUSY;
1014 }
1015
1016 if (!list_empty(&provider->nodes)) {
1017 pr_warn("interconnect provider still has nodes\n");
1018 mutex_unlock(&icc_lock);
1019 return -EBUSY;
1020 }
1021
1022 list_del(&provider->provider_list);
1023 mutex_unlock(&icc_lock);
1024
1025 return 0;
1026}
1027EXPORT_SYMBOL_GPL(icc_provider_del);
1028
1029static int __init icc_init(void)
1030{
1031 icc_debugfs_dir = debugfs_create_dir("interconnect", NULL);
1032 debugfs_create_file("interconnect_summary", 0444,
1033 icc_debugfs_dir, NULL, &icc_summary_fops);
1034 debugfs_create_file("interconnect_graph", 0444,
1035 icc_debugfs_dir, NULL, &icc_graph_fops);
1036 return 0;
1037}
1038
1039device_initcall(icc_init);
1040
1041MODULE_AUTHOR("Georgi Djakov <georgi.djakov@linaro.org>");
1042MODULE_DESCRIPTION("Interconnect Driver Core");
1043MODULE_LICENSE("GPL v2");