Loading...
Note: File does not exist in v4.10.11.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * System Control and Management Interface (SCMI) Performance Protocol
4 *
5 * Copyright (C) 2018-2023 ARM Ltd.
6 */
7
8#define pr_fmt(fmt) "SCMI Notifications PERF - " fmt
9
10#include <linux/bits.h>
11#include <linux/hashtable.h>
12#include <linux/io.h>
13#include <linux/log2.h>
14#include <linux/module.h>
15#include <linux/of.h>
16#include <linux/platform_device.h>
17#include <linux/pm_opp.h>
18#include <linux/scmi_protocol.h>
19#include <linux/sort.h>
20#include <linux/xarray.h>
21
22#include <trace/events/scmi.h>
23
24#include "protocols.h"
25#include "notify.h"
26
27/* Updated only after ALL the mandatory features for that version are merged */
28#define SCMI_PROTOCOL_SUPPORTED_VERSION 0x40000
29
30#define MAX_OPPS 32
31
32enum scmi_performance_protocol_cmd {
33 PERF_DOMAIN_ATTRIBUTES = 0x3,
34 PERF_DESCRIBE_LEVELS = 0x4,
35 PERF_LIMITS_SET = 0x5,
36 PERF_LIMITS_GET = 0x6,
37 PERF_LEVEL_SET = 0x7,
38 PERF_LEVEL_GET = 0x8,
39 PERF_NOTIFY_LIMITS = 0x9,
40 PERF_NOTIFY_LEVEL = 0xa,
41 PERF_DESCRIBE_FASTCHANNEL = 0xb,
42 PERF_DOMAIN_NAME_GET = 0xc,
43};
44
45enum {
46 PERF_FC_LEVEL,
47 PERF_FC_LIMIT,
48 PERF_FC_MAX,
49};
50
51struct scmi_opp {
52 u32 perf;
53 u32 power;
54 u32 trans_latency_us;
55 u32 indicative_freq;
56 u32 level_index;
57 struct hlist_node hash;
58};
59
60struct scmi_msg_resp_perf_attributes {
61 __le16 num_domains;
62 __le16 flags;
63#define POWER_SCALE_IN_MILLIWATT(x) ((x) & BIT(0))
64#define POWER_SCALE_IN_MICROWATT(x) ((x) & BIT(1))
65 __le32 stats_addr_low;
66 __le32 stats_addr_high;
67 __le32 stats_size;
68};
69
70struct scmi_msg_resp_perf_domain_attributes {
71 __le32 flags;
72#define SUPPORTS_SET_LIMITS(x) ((x) & BIT(31))
73#define SUPPORTS_SET_PERF_LVL(x) ((x) & BIT(30))
74#define SUPPORTS_PERF_LIMIT_NOTIFY(x) ((x) & BIT(29))
75#define SUPPORTS_PERF_LEVEL_NOTIFY(x) ((x) & BIT(28))
76#define SUPPORTS_PERF_FASTCHANNELS(x) ((x) & BIT(27))
77#define SUPPORTS_EXTENDED_NAMES(x) ((x) & BIT(26))
78#define SUPPORTS_LEVEL_INDEXING(x) ((x) & BIT(25))
79 __le32 rate_limit_us;
80 __le32 sustained_freq_khz;
81 __le32 sustained_perf_level;
82 u8 name[SCMI_SHORT_NAME_MAX_SIZE];
83};
84
85struct scmi_msg_perf_describe_levels {
86 __le32 domain;
87 __le32 level_index;
88};
89
90struct scmi_perf_set_limits {
91 __le32 domain;
92 __le32 max_level;
93 __le32 min_level;
94};
95
96struct scmi_perf_get_limits {
97 __le32 max_level;
98 __le32 min_level;
99};
100
101struct scmi_perf_set_level {
102 __le32 domain;
103 __le32 level;
104};
105
106struct scmi_perf_notify_level_or_limits {
107 __le32 domain;
108 __le32 notify_enable;
109};
110
111struct scmi_perf_limits_notify_payld {
112 __le32 agent_id;
113 __le32 domain_id;
114 __le32 range_max;
115 __le32 range_min;
116};
117
118struct scmi_perf_level_notify_payld {
119 __le32 agent_id;
120 __le32 domain_id;
121 __le32 performance_level;
122};
123
124struct scmi_msg_resp_perf_describe_levels {
125 __le16 num_returned;
126 __le16 num_remaining;
127 struct {
128 __le32 perf_val;
129 __le32 power;
130 __le16 transition_latency_us;
131 __le16 reserved;
132 } opp[];
133};
134
135struct scmi_msg_resp_perf_describe_levels_v4 {
136 __le16 num_returned;
137 __le16 num_remaining;
138 struct {
139 __le32 perf_val;
140 __le32 power;
141 __le16 transition_latency_us;
142 __le16 reserved;
143 __le32 indicative_freq;
144 __le32 level_index;
145 } opp[];
146};
147
148struct perf_dom_info {
149 u32 id;
150 bool set_limits;
151 bool perf_limit_notify;
152 bool perf_level_notify;
153 bool perf_fastchannels;
154 bool level_indexing_mode;
155 u32 opp_count;
156 u32 rate_limit_us;
157 u32 sustained_freq_khz;
158 u32 sustained_perf_level;
159 unsigned long mult_factor;
160 struct scmi_perf_domain_info info;
161 struct scmi_opp opp[MAX_OPPS];
162 struct scmi_fc_info *fc_info;
163 struct xarray opps_by_idx;
164 struct xarray opps_by_lvl;
165 DECLARE_HASHTABLE(opps_by_freq, ilog2(MAX_OPPS));
166};
167
168#define LOOKUP_BY_FREQ(__htp, __freq) \
169({ \
170 /* u32 cast is needed to pick right hash func */ \
171 u32 f_ = (u32)(__freq); \
172 struct scmi_opp *_opp; \
173 \
174 hash_for_each_possible((__htp), _opp, hash, f_) \
175 if (_opp->indicative_freq == f_) \
176 break; \
177 _opp; \
178})
179
180struct scmi_perf_info {
181 u32 version;
182 u16 num_domains;
183 enum scmi_power_scale power_scale;
184 u64 stats_addr;
185 u32 stats_size;
186 bool notify_lvl_cmd;
187 bool notify_lim_cmd;
188 struct perf_dom_info *dom_info;
189};
190
191static enum scmi_performance_protocol_cmd evt_2_cmd[] = {
192 PERF_NOTIFY_LIMITS,
193 PERF_NOTIFY_LEVEL,
194};
195
196static int scmi_perf_attributes_get(const struct scmi_protocol_handle *ph,
197 struct scmi_perf_info *pi)
198{
199 int ret;
200 struct scmi_xfer *t;
201 struct scmi_msg_resp_perf_attributes *attr;
202
203 ret = ph->xops->xfer_get_init(ph, PROTOCOL_ATTRIBUTES, 0,
204 sizeof(*attr), &t);
205 if (ret)
206 return ret;
207
208 attr = t->rx.buf;
209
210 ret = ph->xops->do_xfer(ph, t);
211 if (!ret) {
212 u16 flags = le16_to_cpu(attr->flags);
213
214 pi->num_domains = le16_to_cpu(attr->num_domains);
215
216 if (POWER_SCALE_IN_MILLIWATT(flags))
217 pi->power_scale = SCMI_POWER_MILLIWATTS;
218 if (PROTOCOL_REV_MAJOR(pi->version) >= 0x3)
219 if (POWER_SCALE_IN_MICROWATT(flags))
220 pi->power_scale = SCMI_POWER_MICROWATTS;
221
222 pi->stats_addr = le32_to_cpu(attr->stats_addr_low) |
223 (u64)le32_to_cpu(attr->stats_addr_high) << 32;
224 pi->stats_size = le32_to_cpu(attr->stats_size);
225 }
226
227 ph->xops->xfer_put(ph, t);
228
229 if (!ret) {
230 if (!ph->hops->protocol_msg_check(ph, PERF_NOTIFY_LEVEL, NULL))
231 pi->notify_lvl_cmd = true;
232
233 if (!ph->hops->protocol_msg_check(ph, PERF_NOTIFY_LIMITS, NULL))
234 pi->notify_lim_cmd = true;
235 }
236
237 return ret;
238}
239
240static void scmi_perf_xa_destroy(void *data)
241{
242 int domain;
243 struct scmi_perf_info *pinfo = data;
244
245 for (domain = 0; domain < pinfo->num_domains; domain++) {
246 xa_destroy(&((pinfo->dom_info + domain)->opps_by_idx));
247 xa_destroy(&((pinfo->dom_info + domain)->opps_by_lvl));
248 }
249}
250
251static int
252scmi_perf_domain_attributes_get(const struct scmi_protocol_handle *ph,
253 struct perf_dom_info *dom_info,
254 bool notify_lim_cmd, bool notify_lvl_cmd,
255 u32 version)
256{
257 int ret;
258 u32 flags;
259 struct scmi_xfer *t;
260 struct scmi_msg_resp_perf_domain_attributes *attr;
261
262 ret = ph->xops->xfer_get_init(ph, PERF_DOMAIN_ATTRIBUTES,
263 sizeof(dom_info->id), sizeof(*attr), &t);
264 if (ret)
265 return ret;
266
267 put_unaligned_le32(dom_info->id, t->tx.buf);
268 attr = t->rx.buf;
269
270 ret = ph->xops->do_xfer(ph, t);
271 if (!ret) {
272 flags = le32_to_cpu(attr->flags);
273
274 dom_info->set_limits = SUPPORTS_SET_LIMITS(flags);
275 dom_info->info.set_perf = SUPPORTS_SET_PERF_LVL(flags);
276 if (notify_lim_cmd)
277 dom_info->perf_limit_notify =
278 SUPPORTS_PERF_LIMIT_NOTIFY(flags);
279 if (notify_lvl_cmd)
280 dom_info->perf_level_notify =
281 SUPPORTS_PERF_LEVEL_NOTIFY(flags);
282 dom_info->perf_fastchannels = SUPPORTS_PERF_FASTCHANNELS(flags);
283 if (PROTOCOL_REV_MAJOR(version) >= 0x4)
284 dom_info->level_indexing_mode =
285 SUPPORTS_LEVEL_INDEXING(flags);
286 dom_info->rate_limit_us = le32_to_cpu(attr->rate_limit_us) &
287 GENMASK(19, 0);
288 dom_info->sustained_freq_khz =
289 le32_to_cpu(attr->sustained_freq_khz);
290 dom_info->sustained_perf_level =
291 le32_to_cpu(attr->sustained_perf_level);
292 /*
293 * sustained_freq_khz = mult_factor * sustained_perf_level
294 * mult_factor must be non zero positive integer(not fraction)
295 */
296 if (!dom_info->sustained_freq_khz ||
297 !dom_info->sustained_perf_level ||
298 dom_info->level_indexing_mode) {
299 /* CPUFreq converts to kHz, hence default 1000 */
300 dom_info->mult_factor = 1000;
301 } else {
302 dom_info->mult_factor =
303 (dom_info->sustained_freq_khz * 1000UL)
304 / dom_info->sustained_perf_level;
305 if ((dom_info->sustained_freq_khz * 1000UL) %
306 dom_info->sustained_perf_level)
307 dev_warn(ph->dev,
308 "multiplier for domain %d rounded\n",
309 dom_info->id);
310 }
311 if (!dom_info->mult_factor)
312 dev_warn(ph->dev,
313 "Wrong sustained perf/frequency(domain %d)\n",
314 dom_info->id);
315
316 strscpy(dom_info->info.name, attr->name,
317 SCMI_SHORT_NAME_MAX_SIZE);
318 }
319
320 ph->xops->xfer_put(ph, t);
321
322 /*
323 * If supported overwrite short name with the extended one;
324 * on error just carry on and use already provided short name.
325 */
326 if (!ret && PROTOCOL_REV_MAJOR(version) >= 0x3 &&
327 SUPPORTS_EXTENDED_NAMES(flags))
328 ph->hops->extended_name_get(ph, PERF_DOMAIN_NAME_GET,
329 dom_info->id, NULL, dom_info->info.name,
330 SCMI_MAX_STR_SIZE);
331
332 xa_init(&dom_info->opps_by_lvl);
333 if (dom_info->level_indexing_mode) {
334 xa_init(&dom_info->opps_by_idx);
335 hash_init(dom_info->opps_by_freq);
336 }
337
338 return ret;
339}
340
341static int opp_cmp_func(const void *opp1, const void *opp2)
342{
343 const struct scmi_opp *t1 = opp1, *t2 = opp2;
344
345 return t1->perf - t2->perf;
346}
347
348struct scmi_perf_ipriv {
349 u32 version;
350 struct perf_dom_info *perf_dom;
351};
352
353static void iter_perf_levels_prepare_message(void *message,
354 unsigned int desc_index,
355 const void *priv)
356{
357 struct scmi_msg_perf_describe_levels *msg = message;
358 const struct scmi_perf_ipriv *p = priv;
359
360 msg->domain = cpu_to_le32(p->perf_dom->id);
361 /* Set the number of OPPs to be skipped/already read */
362 msg->level_index = cpu_to_le32(desc_index);
363}
364
365static int iter_perf_levels_update_state(struct scmi_iterator_state *st,
366 const void *response, void *priv)
367{
368 const struct scmi_msg_resp_perf_describe_levels *r = response;
369
370 st->num_returned = le16_to_cpu(r->num_returned);
371 st->num_remaining = le16_to_cpu(r->num_remaining);
372
373 return 0;
374}
375
376static inline int
377process_response_opp(struct device *dev, struct perf_dom_info *dom,
378 struct scmi_opp *opp, unsigned int loop_idx,
379 const struct scmi_msg_resp_perf_describe_levels *r)
380{
381 int ret;
382
383 opp->perf = le32_to_cpu(r->opp[loop_idx].perf_val);
384 opp->power = le32_to_cpu(r->opp[loop_idx].power);
385 opp->trans_latency_us =
386 le16_to_cpu(r->opp[loop_idx].transition_latency_us);
387
388 ret = xa_insert(&dom->opps_by_lvl, opp->perf, opp, GFP_KERNEL);
389 if (ret) {
390 dev_info(dev, FW_BUG "Failed to add opps_by_lvl at %d for %s - ret:%d\n",
391 opp->perf, dom->info.name, ret);
392 return ret;
393 }
394
395 return 0;
396}
397
398static inline int
399process_response_opp_v4(struct device *dev, struct perf_dom_info *dom,
400 struct scmi_opp *opp, unsigned int loop_idx,
401 const struct scmi_msg_resp_perf_describe_levels_v4 *r)
402{
403 int ret;
404
405 opp->perf = le32_to_cpu(r->opp[loop_idx].perf_val);
406 opp->power = le32_to_cpu(r->opp[loop_idx].power);
407 opp->trans_latency_us =
408 le16_to_cpu(r->opp[loop_idx].transition_latency_us);
409
410 ret = xa_insert(&dom->opps_by_lvl, opp->perf, opp, GFP_KERNEL);
411 if (ret) {
412 dev_info(dev, FW_BUG "Failed to add opps_by_lvl at %d for %s - ret:%d\n",
413 opp->perf, dom->info.name, ret);
414 return ret;
415 }
416
417 /* Note that PERF v4 reports always five 32-bit words */
418 opp->indicative_freq = le32_to_cpu(r->opp[loop_idx].indicative_freq);
419 if (dom->level_indexing_mode) {
420 opp->level_index = le32_to_cpu(r->opp[loop_idx].level_index);
421
422 ret = xa_insert(&dom->opps_by_idx, opp->level_index, opp,
423 GFP_KERNEL);
424 if (ret) {
425 dev_warn(dev,
426 "Failed to add opps_by_idx at %d for %s - ret:%d\n",
427 opp->level_index, dom->info.name, ret);
428
429 /* Cleanup by_lvl too */
430 xa_erase(&dom->opps_by_lvl, opp->perf);
431
432 return ret;
433 }
434
435 hash_add(dom->opps_by_freq, &opp->hash, opp->indicative_freq);
436 }
437
438 return 0;
439}
440
441static int
442iter_perf_levels_process_response(const struct scmi_protocol_handle *ph,
443 const void *response,
444 struct scmi_iterator_state *st, void *priv)
445{
446 int ret;
447 struct scmi_opp *opp;
448 struct scmi_perf_ipriv *p = priv;
449
450 opp = &p->perf_dom->opp[p->perf_dom->opp_count];
451 if (PROTOCOL_REV_MAJOR(p->version) <= 0x3)
452 ret = process_response_opp(ph->dev, p->perf_dom, opp,
453 st->loop_idx, response);
454 else
455 ret = process_response_opp_v4(ph->dev, p->perf_dom, opp,
456 st->loop_idx, response);
457
458 /* Skip BAD duplicates received from firmware */
459 if (ret)
460 return ret == -EBUSY ? 0 : ret;
461
462 p->perf_dom->opp_count++;
463
464 dev_dbg(ph->dev, "Level %d Power %d Latency %dus Ifreq %d Index %d\n",
465 opp->perf, opp->power, opp->trans_latency_us,
466 opp->indicative_freq, opp->level_index);
467
468 return 0;
469}
470
471static int
472scmi_perf_describe_levels_get(const struct scmi_protocol_handle *ph,
473 struct perf_dom_info *perf_dom, u32 version)
474{
475 int ret;
476 void *iter;
477 struct scmi_iterator_ops ops = {
478 .prepare_message = iter_perf_levels_prepare_message,
479 .update_state = iter_perf_levels_update_state,
480 .process_response = iter_perf_levels_process_response,
481 };
482 struct scmi_perf_ipriv ppriv = {
483 .version = version,
484 .perf_dom = perf_dom,
485 };
486
487 iter = ph->hops->iter_response_init(ph, &ops, MAX_OPPS,
488 PERF_DESCRIBE_LEVELS,
489 sizeof(struct scmi_msg_perf_describe_levels),
490 &ppriv);
491 if (IS_ERR(iter))
492 return PTR_ERR(iter);
493
494 ret = ph->hops->iter_response_run(iter);
495 if (ret)
496 return ret;
497
498 if (perf_dom->opp_count)
499 sort(perf_dom->opp, perf_dom->opp_count,
500 sizeof(struct scmi_opp), opp_cmp_func, NULL);
501
502 return ret;
503}
504
505static int scmi_perf_num_domains_get(const struct scmi_protocol_handle *ph)
506{
507 struct scmi_perf_info *pi = ph->get_priv(ph);
508
509 return pi->num_domains;
510}
511
512static inline struct perf_dom_info *
513scmi_perf_domain_lookup(const struct scmi_protocol_handle *ph, u32 domain)
514{
515 struct scmi_perf_info *pi = ph->get_priv(ph);
516
517 if (domain >= pi->num_domains)
518 return ERR_PTR(-EINVAL);
519
520 return pi->dom_info + domain;
521}
522
523static const struct scmi_perf_domain_info *
524scmi_perf_info_get(const struct scmi_protocol_handle *ph, u32 domain)
525{
526 struct perf_dom_info *dom;
527
528 dom = scmi_perf_domain_lookup(ph, domain);
529 if (IS_ERR(dom))
530 return ERR_PTR(-EINVAL);
531
532 return &dom->info;
533}
534
535static int scmi_perf_msg_limits_set(const struct scmi_protocol_handle *ph,
536 u32 domain, u32 max_perf, u32 min_perf)
537{
538 int ret;
539 struct scmi_xfer *t;
540 struct scmi_perf_set_limits *limits;
541
542 ret = ph->xops->xfer_get_init(ph, PERF_LIMITS_SET,
543 sizeof(*limits), 0, &t);
544 if (ret)
545 return ret;
546
547 limits = t->tx.buf;
548 limits->domain = cpu_to_le32(domain);
549 limits->max_level = cpu_to_le32(max_perf);
550 limits->min_level = cpu_to_le32(min_perf);
551
552 ret = ph->xops->do_xfer(ph, t);
553
554 ph->xops->xfer_put(ph, t);
555 return ret;
556}
557
558static int __scmi_perf_limits_set(const struct scmi_protocol_handle *ph,
559 struct perf_dom_info *dom, u32 max_perf,
560 u32 min_perf)
561{
562 if (dom->fc_info && dom->fc_info[PERF_FC_LIMIT].set_addr) {
563 struct scmi_fc_info *fci = &dom->fc_info[PERF_FC_LIMIT];
564
565 trace_scmi_fc_call(SCMI_PROTOCOL_PERF, PERF_LIMITS_SET,
566 dom->id, min_perf, max_perf);
567 iowrite32(max_perf, fci->set_addr);
568 iowrite32(min_perf, fci->set_addr + 4);
569 ph->hops->fastchannel_db_ring(fci->set_db);
570 return 0;
571 }
572
573 return scmi_perf_msg_limits_set(ph, dom->id, max_perf, min_perf);
574}
575
576static int scmi_perf_limits_set(const struct scmi_protocol_handle *ph,
577 u32 domain, u32 max_perf, u32 min_perf)
578{
579 struct scmi_perf_info *pi = ph->get_priv(ph);
580 struct perf_dom_info *dom;
581
582 dom = scmi_perf_domain_lookup(ph, domain);
583 if (IS_ERR(dom))
584 return PTR_ERR(dom);
585
586 if (!dom->set_limits)
587 return -EOPNOTSUPP;
588
589 if (PROTOCOL_REV_MAJOR(pi->version) >= 0x3 && !max_perf && !min_perf)
590 return -EINVAL;
591
592 if (dom->level_indexing_mode) {
593 struct scmi_opp *opp;
594
595 if (min_perf) {
596 opp = xa_load(&dom->opps_by_lvl, min_perf);
597 if (!opp)
598 return -EIO;
599
600 min_perf = opp->level_index;
601 }
602
603 if (max_perf) {
604 opp = xa_load(&dom->opps_by_lvl, max_perf);
605 if (!opp)
606 return -EIO;
607
608 max_perf = opp->level_index;
609 }
610 }
611
612 return __scmi_perf_limits_set(ph, dom, max_perf, min_perf);
613}
614
615static int scmi_perf_msg_limits_get(const struct scmi_protocol_handle *ph,
616 u32 domain, u32 *max_perf, u32 *min_perf)
617{
618 int ret;
619 struct scmi_xfer *t;
620 struct scmi_perf_get_limits *limits;
621
622 ret = ph->xops->xfer_get_init(ph, PERF_LIMITS_GET,
623 sizeof(__le32), 0, &t);
624 if (ret)
625 return ret;
626
627 put_unaligned_le32(domain, t->tx.buf);
628
629 ret = ph->xops->do_xfer(ph, t);
630 if (!ret) {
631 limits = t->rx.buf;
632
633 *max_perf = le32_to_cpu(limits->max_level);
634 *min_perf = le32_to_cpu(limits->min_level);
635 }
636
637 ph->xops->xfer_put(ph, t);
638 return ret;
639}
640
641static int __scmi_perf_limits_get(const struct scmi_protocol_handle *ph,
642 struct perf_dom_info *dom, u32 *max_perf,
643 u32 *min_perf)
644{
645 if (dom->fc_info && dom->fc_info[PERF_FC_LIMIT].get_addr) {
646 struct scmi_fc_info *fci = &dom->fc_info[PERF_FC_LIMIT];
647
648 *max_perf = ioread32(fci->get_addr);
649 *min_perf = ioread32(fci->get_addr + 4);
650 trace_scmi_fc_call(SCMI_PROTOCOL_PERF, PERF_LIMITS_GET,
651 dom->id, *min_perf, *max_perf);
652 return 0;
653 }
654
655 return scmi_perf_msg_limits_get(ph, dom->id, max_perf, min_perf);
656}
657
658static int scmi_perf_limits_get(const struct scmi_protocol_handle *ph,
659 u32 domain, u32 *max_perf, u32 *min_perf)
660{
661 int ret;
662 struct perf_dom_info *dom;
663
664 dom = scmi_perf_domain_lookup(ph, domain);
665 if (IS_ERR(dom))
666 return PTR_ERR(dom);
667
668 ret = __scmi_perf_limits_get(ph, dom, max_perf, min_perf);
669 if (ret)
670 return ret;
671
672 if (dom->level_indexing_mode) {
673 struct scmi_opp *opp;
674
675 opp = xa_load(&dom->opps_by_idx, *min_perf);
676 if (!opp)
677 return -EIO;
678
679 *min_perf = opp->perf;
680
681 opp = xa_load(&dom->opps_by_idx, *max_perf);
682 if (!opp)
683 return -EIO;
684
685 *max_perf = opp->perf;
686 }
687
688 return 0;
689}
690
691static int scmi_perf_msg_level_set(const struct scmi_protocol_handle *ph,
692 u32 domain, u32 level, bool poll)
693{
694 int ret;
695 struct scmi_xfer *t;
696 struct scmi_perf_set_level *lvl;
697
698 ret = ph->xops->xfer_get_init(ph, PERF_LEVEL_SET, sizeof(*lvl), 0, &t);
699 if (ret)
700 return ret;
701
702 t->hdr.poll_completion = poll;
703 lvl = t->tx.buf;
704 lvl->domain = cpu_to_le32(domain);
705 lvl->level = cpu_to_le32(level);
706
707 ret = ph->xops->do_xfer(ph, t);
708
709 ph->xops->xfer_put(ph, t);
710 return ret;
711}
712
713static int __scmi_perf_level_set(const struct scmi_protocol_handle *ph,
714 struct perf_dom_info *dom, u32 level,
715 bool poll)
716{
717 if (dom->fc_info && dom->fc_info[PERF_FC_LEVEL].set_addr) {
718 struct scmi_fc_info *fci = &dom->fc_info[PERF_FC_LEVEL];
719
720 trace_scmi_fc_call(SCMI_PROTOCOL_PERF, PERF_LEVEL_SET,
721 dom->id, level, 0);
722 iowrite32(level, fci->set_addr);
723 ph->hops->fastchannel_db_ring(fci->set_db);
724 return 0;
725 }
726
727 return scmi_perf_msg_level_set(ph, dom->id, level, poll);
728}
729
730static int scmi_perf_level_set(const struct scmi_protocol_handle *ph,
731 u32 domain, u32 level, bool poll)
732{
733 struct perf_dom_info *dom;
734
735 dom = scmi_perf_domain_lookup(ph, domain);
736 if (IS_ERR(dom))
737 return PTR_ERR(dom);
738
739 if (!dom->info.set_perf)
740 return -EOPNOTSUPP;
741
742 if (dom->level_indexing_mode) {
743 struct scmi_opp *opp;
744
745 opp = xa_load(&dom->opps_by_lvl, level);
746 if (!opp)
747 return -EIO;
748
749 level = opp->level_index;
750 }
751
752 return __scmi_perf_level_set(ph, dom, level, poll);
753}
754
755static int scmi_perf_msg_level_get(const struct scmi_protocol_handle *ph,
756 u32 domain, u32 *level, bool poll)
757{
758 int ret;
759 struct scmi_xfer *t;
760
761 ret = ph->xops->xfer_get_init(ph, PERF_LEVEL_GET,
762 sizeof(u32), sizeof(u32), &t);
763 if (ret)
764 return ret;
765
766 t->hdr.poll_completion = poll;
767 put_unaligned_le32(domain, t->tx.buf);
768
769 ret = ph->xops->do_xfer(ph, t);
770 if (!ret)
771 *level = get_unaligned_le32(t->rx.buf);
772
773 ph->xops->xfer_put(ph, t);
774 return ret;
775}
776
777static int __scmi_perf_level_get(const struct scmi_protocol_handle *ph,
778 struct perf_dom_info *dom, u32 *level,
779 bool poll)
780{
781 if (dom->fc_info && dom->fc_info[PERF_FC_LEVEL].get_addr) {
782 *level = ioread32(dom->fc_info[PERF_FC_LEVEL].get_addr);
783 trace_scmi_fc_call(SCMI_PROTOCOL_PERF, PERF_LEVEL_GET,
784 dom->id, *level, 0);
785 return 0;
786 }
787
788 return scmi_perf_msg_level_get(ph, dom->id, level, poll);
789}
790
791static int scmi_perf_level_get(const struct scmi_protocol_handle *ph,
792 u32 domain, u32 *level, bool poll)
793{
794 int ret;
795 struct perf_dom_info *dom;
796
797 dom = scmi_perf_domain_lookup(ph, domain);
798 if (IS_ERR(dom))
799 return PTR_ERR(dom);
800
801 ret = __scmi_perf_level_get(ph, dom, level, poll);
802 if (ret)
803 return ret;
804
805 if (dom->level_indexing_mode) {
806 struct scmi_opp *opp;
807
808 opp = xa_load(&dom->opps_by_idx, *level);
809 if (!opp)
810 return -EIO;
811
812 *level = opp->perf;
813 }
814
815 return 0;
816}
817
818static int scmi_perf_level_limits_notify(const struct scmi_protocol_handle *ph,
819 u32 domain, int message_id,
820 bool enable)
821{
822 int ret;
823 struct scmi_xfer *t;
824 struct scmi_perf_notify_level_or_limits *notify;
825
826 ret = ph->xops->xfer_get_init(ph, message_id, sizeof(*notify), 0, &t);
827 if (ret)
828 return ret;
829
830 notify = t->tx.buf;
831 notify->domain = cpu_to_le32(domain);
832 notify->notify_enable = enable ? cpu_to_le32(BIT(0)) : 0;
833
834 ret = ph->xops->do_xfer(ph, t);
835
836 ph->xops->xfer_put(ph, t);
837 return ret;
838}
839
840static void scmi_perf_domain_init_fc(const struct scmi_protocol_handle *ph,
841 struct perf_dom_info *dom)
842{
843 struct scmi_fc_info *fc;
844
845 fc = devm_kcalloc(ph->dev, PERF_FC_MAX, sizeof(*fc), GFP_KERNEL);
846 if (!fc)
847 return;
848
849 ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL,
850 PERF_LEVEL_GET, 4, dom->id,
851 &fc[PERF_FC_LEVEL].get_addr, NULL,
852 &fc[PERF_FC_LEVEL].rate_limit);
853
854 ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL,
855 PERF_LIMITS_GET, 8, dom->id,
856 &fc[PERF_FC_LIMIT].get_addr, NULL,
857 &fc[PERF_FC_LIMIT].rate_limit);
858
859 if (dom->info.set_perf)
860 ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL,
861 PERF_LEVEL_SET, 4, dom->id,
862 &fc[PERF_FC_LEVEL].set_addr,
863 &fc[PERF_FC_LEVEL].set_db,
864 &fc[PERF_FC_LEVEL].rate_limit);
865
866 if (dom->set_limits)
867 ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL,
868 PERF_LIMITS_SET, 8, dom->id,
869 &fc[PERF_FC_LIMIT].set_addr,
870 &fc[PERF_FC_LIMIT].set_db,
871 &fc[PERF_FC_LIMIT].rate_limit);
872
873 dom->fc_info = fc;
874}
875
876static int scmi_dvfs_device_opps_add(const struct scmi_protocol_handle *ph,
877 struct device *dev, u32 domain)
878{
879 int idx, ret;
880 unsigned long freq;
881 struct dev_pm_opp_data data = {};
882 struct perf_dom_info *dom;
883
884 dom = scmi_perf_domain_lookup(ph, domain);
885 if (IS_ERR(dom))
886 return PTR_ERR(dom);
887
888 for (idx = 0; idx < dom->opp_count; idx++) {
889 if (!dom->level_indexing_mode)
890 freq = dom->opp[idx].perf * dom->mult_factor;
891 else
892 freq = dom->opp[idx].indicative_freq * dom->mult_factor;
893
894 /* All OPPs above the sustained frequency are treated as turbo */
895 data.turbo = freq > dom->sustained_freq_khz * 1000;
896
897 data.level = dom->opp[idx].perf;
898 data.freq = freq;
899
900 ret = dev_pm_opp_add_dynamic(dev, &data);
901 if (ret) {
902 dev_warn(dev, "[%d][%s]: Failed to add OPP[%d] %lu\n",
903 domain, dom->info.name, idx, freq);
904 dev_pm_opp_remove_all_dynamic(dev);
905 return ret;
906 }
907
908 dev_dbg(dev, "[%d][%s]:: Registered OPP[%d] %lu\n",
909 domain, dom->info.name, idx, freq);
910 }
911 return 0;
912}
913
914static int
915scmi_dvfs_transition_latency_get(const struct scmi_protocol_handle *ph,
916 u32 domain)
917{
918 struct perf_dom_info *dom;
919
920 dom = scmi_perf_domain_lookup(ph, domain);
921 if (IS_ERR(dom))
922 return PTR_ERR(dom);
923
924 /* uS to nS */
925 return dom->opp[dom->opp_count - 1].trans_latency_us * 1000;
926}
927
928static int
929scmi_dvfs_rate_limit_get(const struct scmi_protocol_handle *ph,
930 u32 domain, u32 *rate_limit)
931{
932 struct perf_dom_info *dom;
933
934 if (!rate_limit)
935 return -EINVAL;
936
937 dom = scmi_perf_domain_lookup(ph, domain);
938 if (IS_ERR(dom))
939 return PTR_ERR(dom);
940
941 *rate_limit = dom->rate_limit_us;
942 return 0;
943}
944
945static int scmi_dvfs_freq_set(const struct scmi_protocol_handle *ph, u32 domain,
946 unsigned long freq, bool poll)
947{
948 unsigned int level;
949 struct perf_dom_info *dom;
950
951 dom = scmi_perf_domain_lookup(ph, domain);
952 if (IS_ERR(dom))
953 return PTR_ERR(dom);
954
955 if (!dom->level_indexing_mode) {
956 level = freq / dom->mult_factor;
957 } else {
958 struct scmi_opp *opp;
959
960 opp = LOOKUP_BY_FREQ(dom->opps_by_freq,
961 freq / dom->mult_factor);
962 if (!opp)
963 return -EIO;
964
965 level = opp->level_index;
966 }
967
968 return __scmi_perf_level_set(ph, dom, level, poll);
969}
970
971static int scmi_dvfs_freq_get(const struct scmi_protocol_handle *ph, u32 domain,
972 unsigned long *freq, bool poll)
973{
974 int ret;
975 u32 level;
976 struct perf_dom_info *dom;
977
978 dom = scmi_perf_domain_lookup(ph, domain);
979 if (IS_ERR(dom))
980 return PTR_ERR(dom);
981
982 ret = __scmi_perf_level_get(ph, dom, &level, poll);
983 if (ret)
984 return ret;
985
986 if (!dom->level_indexing_mode) {
987 *freq = level * dom->mult_factor;
988 } else {
989 struct scmi_opp *opp;
990
991 opp = xa_load(&dom->opps_by_idx, level);
992 if (!opp)
993 return -EIO;
994
995 *freq = opp->indicative_freq * dom->mult_factor;
996 }
997
998 return ret;
999}
1000
1001static int scmi_dvfs_est_power_get(const struct scmi_protocol_handle *ph,
1002 u32 domain, unsigned long *freq,
1003 unsigned long *power)
1004{
1005 struct perf_dom_info *dom;
1006 unsigned long opp_freq;
1007 int idx, ret = -EINVAL;
1008 struct scmi_opp *opp;
1009
1010 dom = scmi_perf_domain_lookup(ph, domain);
1011 if (IS_ERR(dom))
1012 return PTR_ERR(dom);
1013
1014 for (opp = dom->opp, idx = 0; idx < dom->opp_count; idx++, opp++) {
1015 if (!dom->level_indexing_mode)
1016 opp_freq = opp->perf * dom->mult_factor;
1017 else
1018 opp_freq = opp->indicative_freq * dom->mult_factor;
1019
1020 if (opp_freq < *freq)
1021 continue;
1022
1023 *freq = opp_freq;
1024 *power = opp->power;
1025 ret = 0;
1026 break;
1027 }
1028
1029 return ret;
1030}
1031
1032static bool scmi_fast_switch_possible(const struct scmi_protocol_handle *ph,
1033 u32 domain)
1034{
1035 struct perf_dom_info *dom;
1036
1037 dom = scmi_perf_domain_lookup(ph, domain);
1038 if (IS_ERR(dom))
1039 return false;
1040
1041 return dom->fc_info && dom->fc_info[PERF_FC_LEVEL].set_addr;
1042}
1043
1044static int scmi_fast_switch_rate_limit(const struct scmi_protocol_handle *ph,
1045 u32 domain, u32 *rate_limit)
1046{
1047 struct perf_dom_info *dom;
1048
1049 if (!rate_limit)
1050 return -EINVAL;
1051
1052 dom = scmi_perf_domain_lookup(ph, domain);
1053 if (IS_ERR(dom))
1054 return PTR_ERR(dom);
1055
1056 if (!dom->fc_info)
1057 return -EINVAL;
1058
1059 *rate_limit = dom->fc_info[PERF_FC_LEVEL].rate_limit;
1060 return 0;
1061}
1062
1063static enum scmi_power_scale
1064scmi_power_scale_get(const struct scmi_protocol_handle *ph)
1065{
1066 struct scmi_perf_info *pi = ph->get_priv(ph);
1067
1068 return pi->power_scale;
1069}
1070
1071static const struct scmi_perf_proto_ops perf_proto_ops = {
1072 .num_domains_get = scmi_perf_num_domains_get,
1073 .info_get = scmi_perf_info_get,
1074 .limits_set = scmi_perf_limits_set,
1075 .limits_get = scmi_perf_limits_get,
1076 .level_set = scmi_perf_level_set,
1077 .level_get = scmi_perf_level_get,
1078 .transition_latency_get = scmi_dvfs_transition_latency_get,
1079 .rate_limit_get = scmi_dvfs_rate_limit_get,
1080 .device_opps_add = scmi_dvfs_device_opps_add,
1081 .freq_set = scmi_dvfs_freq_set,
1082 .freq_get = scmi_dvfs_freq_get,
1083 .est_power_get = scmi_dvfs_est_power_get,
1084 .fast_switch_possible = scmi_fast_switch_possible,
1085 .fast_switch_rate_limit = scmi_fast_switch_rate_limit,
1086 .power_scale_get = scmi_power_scale_get,
1087};
1088
1089static bool scmi_perf_notify_supported(const struct scmi_protocol_handle *ph,
1090 u8 evt_id, u32 src_id)
1091{
1092 bool supported;
1093 struct perf_dom_info *dom;
1094
1095 if (evt_id >= ARRAY_SIZE(evt_2_cmd))
1096 return false;
1097
1098 dom = scmi_perf_domain_lookup(ph, src_id);
1099 if (IS_ERR(dom))
1100 return false;
1101
1102 if (evt_id == SCMI_EVENT_PERFORMANCE_LIMITS_CHANGED)
1103 supported = dom->perf_limit_notify;
1104 else
1105 supported = dom->perf_level_notify;
1106
1107 return supported;
1108}
1109
1110static int scmi_perf_set_notify_enabled(const struct scmi_protocol_handle *ph,
1111 u8 evt_id, u32 src_id, bool enable)
1112{
1113 int ret, cmd_id;
1114
1115 if (evt_id >= ARRAY_SIZE(evt_2_cmd))
1116 return -EINVAL;
1117
1118 cmd_id = evt_2_cmd[evt_id];
1119 ret = scmi_perf_level_limits_notify(ph, src_id, cmd_id, enable);
1120 if (ret)
1121 pr_debug("FAIL_ENABLED - evt[%X] dom[%d] - ret:%d\n",
1122 evt_id, src_id, ret);
1123
1124 return ret;
1125}
1126
1127static int
1128scmi_perf_xlate_opp_to_freq(struct perf_dom_info *dom,
1129 unsigned int index, unsigned long *freq)
1130{
1131 struct scmi_opp *opp;
1132
1133 if (!dom || !freq)
1134 return -EINVAL;
1135
1136 if (!dom->level_indexing_mode) {
1137 opp = xa_load(&dom->opps_by_lvl, index);
1138 if (!opp)
1139 return -ENODEV;
1140
1141 *freq = opp->perf * dom->mult_factor;
1142 } else {
1143 opp = xa_load(&dom->opps_by_idx, index);
1144 if (!opp)
1145 return -ENODEV;
1146
1147 *freq = opp->indicative_freq * dom->mult_factor;
1148 }
1149
1150 return 0;
1151}
1152
1153static void *scmi_perf_fill_custom_report(const struct scmi_protocol_handle *ph,
1154 u8 evt_id, ktime_t timestamp,
1155 const void *payld, size_t payld_sz,
1156 void *report, u32 *src_id)
1157{
1158 int ret;
1159 void *rep = NULL;
1160 struct perf_dom_info *dom;
1161
1162 switch (evt_id) {
1163 case SCMI_EVENT_PERFORMANCE_LIMITS_CHANGED:
1164 {
1165 const struct scmi_perf_limits_notify_payld *p = payld;
1166 struct scmi_perf_limits_report *r = report;
1167 unsigned long freq_min, freq_max;
1168
1169 if (sizeof(*p) != payld_sz)
1170 break;
1171
1172 r->timestamp = timestamp;
1173 r->agent_id = le32_to_cpu(p->agent_id);
1174 r->domain_id = le32_to_cpu(p->domain_id);
1175 r->range_max = le32_to_cpu(p->range_max);
1176 r->range_min = le32_to_cpu(p->range_min);
1177 /* Check if the reported domain exist at all */
1178 dom = scmi_perf_domain_lookup(ph, r->domain_id);
1179 if (IS_ERR(dom))
1180 break;
1181 /*
1182 * Event will be reported from this point on...
1183 * ...even if, later, xlated frequencies were not retrieved.
1184 */
1185 *src_id = r->domain_id;
1186 rep = r;
1187
1188 ret = scmi_perf_xlate_opp_to_freq(dom, r->range_max, &freq_max);
1189 if (ret)
1190 break;
1191
1192 ret = scmi_perf_xlate_opp_to_freq(dom, r->range_min, &freq_min);
1193 if (ret)
1194 break;
1195
1196 /* Report translated freqs ONLY if both available */
1197 r->range_max_freq = freq_max;
1198 r->range_min_freq = freq_min;
1199
1200 break;
1201 }
1202 case SCMI_EVENT_PERFORMANCE_LEVEL_CHANGED:
1203 {
1204 const struct scmi_perf_level_notify_payld *p = payld;
1205 struct scmi_perf_level_report *r = report;
1206 unsigned long freq;
1207
1208 if (sizeof(*p) != payld_sz)
1209 break;
1210
1211 r->timestamp = timestamp;
1212 r->agent_id = le32_to_cpu(p->agent_id);
1213 r->domain_id = le32_to_cpu(p->domain_id);
1214 /* Report translated freqs ONLY if available */
1215 r->performance_level = le32_to_cpu(p->performance_level);
1216 /* Check if the reported domain exist at all */
1217 dom = scmi_perf_domain_lookup(ph, r->domain_id);
1218 if (IS_ERR(dom))
1219 break;
1220 /*
1221 * Event will be reported from this point on...
1222 * ...even if, later, xlated frequencies were not retrieved.
1223 */
1224 *src_id = r->domain_id;
1225 rep = r;
1226
1227 /* Report translated freqs ONLY if available */
1228 ret = scmi_perf_xlate_opp_to_freq(dom, r->performance_level,
1229 &freq);
1230 if (ret)
1231 break;
1232
1233 r->performance_level_freq = freq;
1234
1235 break;
1236 }
1237 default:
1238 break;
1239 }
1240
1241 return rep;
1242}
1243
1244static int scmi_perf_get_num_sources(const struct scmi_protocol_handle *ph)
1245{
1246 struct scmi_perf_info *pi = ph->get_priv(ph);
1247
1248 if (!pi)
1249 return -EINVAL;
1250
1251 return pi->num_domains;
1252}
1253
1254static const struct scmi_event perf_events[] = {
1255 {
1256 .id = SCMI_EVENT_PERFORMANCE_LIMITS_CHANGED,
1257 .max_payld_sz = sizeof(struct scmi_perf_limits_notify_payld),
1258 .max_report_sz = sizeof(struct scmi_perf_limits_report),
1259 },
1260 {
1261 .id = SCMI_EVENT_PERFORMANCE_LEVEL_CHANGED,
1262 .max_payld_sz = sizeof(struct scmi_perf_level_notify_payld),
1263 .max_report_sz = sizeof(struct scmi_perf_level_report),
1264 },
1265};
1266
1267static const struct scmi_event_ops perf_event_ops = {
1268 .is_notify_supported = scmi_perf_notify_supported,
1269 .get_num_sources = scmi_perf_get_num_sources,
1270 .set_notify_enabled = scmi_perf_set_notify_enabled,
1271 .fill_custom_report = scmi_perf_fill_custom_report,
1272};
1273
1274static const struct scmi_protocol_events perf_protocol_events = {
1275 .queue_sz = SCMI_PROTO_QUEUE_SZ,
1276 .ops = &perf_event_ops,
1277 .evts = perf_events,
1278 .num_events = ARRAY_SIZE(perf_events),
1279};
1280
1281static int scmi_perf_protocol_init(const struct scmi_protocol_handle *ph)
1282{
1283 int domain, ret;
1284 u32 version;
1285 struct scmi_perf_info *pinfo;
1286
1287 ret = ph->xops->version_get(ph, &version);
1288 if (ret)
1289 return ret;
1290
1291 dev_dbg(ph->dev, "Performance Version %d.%d\n",
1292 PROTOCOL_REV_MAJOR(version), PROTOCOL_REV_MINOR(version));
1293
1294 pinfo = devm_kzalloc(ph->dev, sizeof(*pinfo), GFP_KERNEL);
1295 if (!pinfo)
1296 return -ENOMEM;
1297
1298 pinfo->version = version;
1299
1300 ret = scmi_perf_attributes_get(ph, pinfo);
1301 if (ret)
1302 return ret;
1303
1304 pinfo->dom_info = devm_kcalloc(ph->dev, pinfo->num_domains,
1305 sizeof(*pinfo->dom_info), GFP_KERNEL);
1306 if (!pinfo->dom_info)
1307 return -ENOMEM;
1308
1309 for (domain = 0; domain < pinfo->num_domains; domain++) {
1310 struct perf_dom_info *dom = pinfo->dom_info + domain;
1311
1312 dom->id = domain;
1313 scmi_perf_domain_attributes_get(ph, dom, pinfo->notify_lim_cmd,
1314 pinfo->notify_lvl_cmd, version);
1315 scmi_perf_describe_levels_get(ph, dom, version);
1316
1317 if (dom->perf_fastchannels)
1318 scmi_perf_domain_init_fc(ph, dom);
1319 }
1320
1321 ret = devm_add_action_or_reset(ph->dev, scmi_perf_xa_destroy, pinfo);
1322 if (ret)
1323 return ret;
1324
1325 return ph->set_priv(ph, pinfo, version);
1326}
1327
1328static const struct scmi_protocol scmi_perf = {
1329 .id = SCMI_PROTOCOL_PERF,
1330 .owner = THIS_MODULE,
1331 .instance_init = &scmi_perf_protocol_init,
1332 .ops = &perf_proto_ops,
1333 .events = &perf_protocol_events,
1334 .supported_version = SCMI_PROTOCOL_SUPPORTED_VERSION,
1335};
1336
1337DEFINE_SCMI_PROTOCOL_REGISTER_UNREGISTER(perf, scmi_perf)