Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2012 NVIDIA CORPORATION. All rights reserved.
4 */
5
6#include <linux/bitfield.h>
7#include <linux/delay.h>
8#include <linux/mutex.h>
9#include <linux/of_device.h>
10#include <linux/slab.h>
11#include <linux/string.h>
12
13#include <dt-bindings/memory/tegra20-mc.h>
14
15#include "mc.h"
16
17#define MC_STAT_CONTROL 0x90
18#define MC_STAT_EMC_CLOCK_LIMIT 0xa0
19#define MC_STAT_EMC_CLOCKS 0xa4
20#define MC_STAT_EMC_CONTROL_0 0xa8
21#define MC_STAT_EMC_CONTROL_1 0xac
22#define MC_STAT_EMC_COUNT_0 0xb8
23#define MC_STAT_EMC_COUNT_1 0xbc
24
25#define MC_STAT_CONTROL_CLIENT_ID GENMASK(13, 8)
26#define MC_STAT_CONTROL_EVENT GENMASK(23, 16)
27#define MC_STAT_CONTROL_PRI_EVENT GENMASK(25, 24)
28#define MC_STAT_CONTROL_FILTER_CLIENT_ENABLE GENMASK(26, 26)
29#define MC_STAT_CONTROL_FILTER_PRI GENMASK(29, 28)
30
31#define MC_STAT_CONTROL_PRI_EVENT_HP 0
32#define MC_STAT_CONTROL_PRI_EVENT_TM 1
33#define MC_STAT_CONTROL_PRI_EVENT_BW 2
34
35#define MC_STAT_CONTROL_FILTER_PRI_DISABLE 0
36#define MC_STAT_CONTROL_FILTER_PRI_NO 1
37#define MC_STAT_CONTROL_FILTER_PRI_YES 2
38
39#define MC_STAT_CONTROL_EVENT_QUALIFIED 0
40#define MC_STAT_CONTROL_EVENT_ANY_READ 1
41#define MC_STAT_CONTROL_EVENT_ANY_WRITE 2
42#define MC_STAT_CONTROL_EVENT_RD_WR_CHANGE 3
43#define MC_STAT_CONTROL_EVENT_SUCCESSIVE 4
44#define MC_STAT_CONTROL_EVENT_ARB_BANK_AA 5
45#define MC_STAT_CONTROL_EVENT_ARB_BANK_BB 6
46#define MC_STAT_CONTROL_EVENT_PAGE_MISS 7
47#define MC_STAT_CONTROL_EVENT_AUTO_PRECHARGE 8
48
49#define EMC_GATHER_RST (0 << 8)
50#define EMC_GATHER_CLEAR (1 << 8)
51#define EMC_GATHER_DISABLE (2 << 8)
52#define EMC_GATHER_ENABLE (3 << 8)
53
54#define MC_STAT_SAMPLE_TIME_USEC 16000
55
56/* we store collected statistics as a fixed point values */
57#define MC_FX_FRAC_SCALE 100
58
59static DEFINE_MUTEX(tegra20_mc_stat_lock);
60
61struct tegra20_mc_stat_gather {
62 unsigned int pri_filter;
63 unsigned int pri_event;
64 unsigned int result;
65 unsigned int client;
66 unsigned int event;
67 bool client_enb;
68};
69
70struct tegra20_mc_stat {
71 struct tegra20_mc_stat_gather gather0;
72 struct tegra20_mc_stat_gather gather1;
73 unsigned int sample_time_usec;
74 const struct tegra_mc *mc;
75};
76
77struct tegra20_mc_client_stat {
78 unsigned int events;
79 unsigned int arb_high_prio;
80 unsigned int arb_timeout;
81 unsigned int arb_bandwidth;
82 unsigned int rd_wr_change;
83 unsigned int successive;
84 unsigned int page_miss;
85 unsigned int auto_precharge;
86 unsigned int arb_bank_aa;
87 unsigned int arb_bank_bb;
88};
89
90static const struct tegra_mc_client tegra20_mc_clients[] = {
91 {
92 .id = 0x00,
93 .name = "display0a",
94 }, {
95 .id = 0x01,
96 .name = "display0ab",
97 }, {
98 .id = 0x02,
99 .name = "display0b",
100 }, {
101 .id = 0x03,
102 .name = "display0bb",
103 }, {
104 .id = 0x04,
105 .name = "display0c",
106 }, {
107 .id = 0x05,
108 .name = "display0cb",
109 }, {
110 .id = 0x06,
111 .name = "display1b",
112 }, {
113 .id = 0x07,
114 .name = "display1bb",
115 }, {
116 .id = 0x08,
117 .name = "eppup",
118 }, {
119 .id = 0x09,
120 .name = "g2pr",
121 }, {
122 .id = 0x0a,
123 .name = "g2sr",
124 }, {
125 .id = 0x0b,
126 .name = "mpeunifbr",
127 }, {
128 .id = 0x0c,
129 .name = "viruv",
130 }, {
131 .id = 0x0d,
132 .name = "avpcarm7r",
133 }, {
134 .id = 0x0e,
135 .name = "displayhc",
136 }, {
137 .id = 0x0f,
138 .name = "displayhcb",
139 }, {
140 .id = 0x10,
141 .name = "fdcdrd",
142 }, {
143 .id = 0x11,
144 .name = "g2dr",
145 }, {
146 .id = 0x12,
147 .name = "host1xdmar",
148 }, {
149 .id = 0x13,
150 .name = "host1xr",
151 }, {
152 .id = 0x14,
153 .name = "idxsrd",
154 }, {
155 .id = 0x15,
156 .name = "mpcorer",
157 }, {
158 .id = 0x16,
159 .name = "mpe_ipred",
160 }, {
161 .id = 0x17,
162 .name = "mpeamemrd",
163 }, {
164 .id = 0x18,
165 .name = "mpecsrd",
166 }, {
167 .id = 0x19,
168 .name = "ppcsahbdmar",
169 }, {
170 .id = 0x1a,
171 .name = "ppcsahbslvr",
172 }, {
173 .id = 0x1b,
174 .name = "texsrd",
175 }, {
176 .id = 0x1c,
177 .name = "vdebsevr",
178 }, {
179 .id = 0x1d,
180 .name = "vdember",
181 }, {
182 .id = 0x1e,
183 .name = "vdemcer",
184 }, {
185 .id = 0x1f,
186 .name = "vdetper",
187 }, {
188 .id = 0x20,
189 .name = "eppu",
190 }, {
191 .id = 0x21,
192 .name = "eppv",
193 }, {
194 .id = 0x22,
195 .name = "eppy",
196 }, {
197 .id = 0x23,
198 .name = "mpeunifbw",
199 }, {
200 .id = 0x24,
201 .name = "viwsb",
202 }, {
203 .id = 0x25,
204 .name = "viwu",
205 }, {
206 .id = 0x26,
207 .name = "viwv",
208 }, {
209 .id = 0x27,
210 .name = "viwy",
211 }, {
212 .id = 0x28,
213 .name = "g2dw",
214 }, {
215 .id = 0x29,
216 .name = "avpcarm7w",
217 }, {
218 .id = 0x2a,
219 .name = "fdcdwr",
220 }, {
221 .id = 0x2b,
222 .name = "host1xw",
223 }, {
224 .id = 0x2c,
225 .name = "ispw",
226 }, {
227 .id = 0x2d,
228 .name = "mpcorew",
229 }, {
230 .id = 0x2e,
231 .name = "mpecswr",
232 }, {
233 .id = 0x2f,
234 .name = "ppcsahbdmaw",
235 }, {
236 .id = 0x30,
237 .name = "ppcsahbslvw",
238 }, {
239 .id = 0x31,
240 .name = "vdebsevw",
241 }, {
242 .id = 0x32,
243 .name = "vdembew",
244 }, {
245 .id = 0x33,
246 .name = "vdetpmw",
247 },
248};
249
250#define TEGRA20_MC_RESET(_name, _control, _status, _reset, _bit) \
251 { \
252 .name = #_name, \
253 .id = TEGRA20_MC_RESET_##_name, \
254 .control = _control, \
255 .status = _status, \
256 .reset = _reset, \
257 .bit = _bit, \
258 }
259
260static const struct tegra_mc_reset tegra20_mc_resets[] = {
261 TEGRA20_MC_RESET(AVPC, 0x100, 0x140, 0x104, 0),
262 TEGRA20_MC_RESET(DC, 0x100, 0x144, 0x104, 1),
263 TEGRA20_MC_RESET(DCB, 0x100, 0x148, 0x104, 2),
264 TEGRA20_MC_RESET(EPP, 0x100, 0x14c, 0x104, 3),
265 TEGRA20_MC_RESET(2D, 0x100, 0x150, 0x104, 4),
266 TEGRA20_MC_RESET(HC, 0x100, 0x154, 0x104, 5),
267 TEGRA20_MC_RESET(ISP, 0x100, 0x158, 0x104, 6),
268 TEGRA20_MC_RESET(MPCORE, 0x100, 0x15c, 0x104, 7),
269 TEGRA20_MC_RESET(MPEA, 0x100, 0x160, 0x104, 8),
270 TEGRA20_MC_RESET(MPEB, 0x100, 0x164, 0x104, 9),
271 TEGRA20_MC_RESET(MPEC, 0x100, 0x168, 0x104, 10),
272 TEGRA20_MC_RESET(3D, 0x100, 0x16c, 0x104, 11),
273 TEGRA20_MC_RESET(PPCS, 0x100, 0x170, 0x104, 12),
274 TEGRA20_MC_RESET(VDE, 0x100, 0x174, 0x104, 13),
275 TEGRA20_MC_RESET(VI, 0x100, 0x178, 0x104, 14),
276};
277
278static int tegra20_mc_hotreset_assert(struct tegra_mc *mc,
279 const struct tegra_mc_reset *rst)
280{
281 unsigned long flags;
282 u32 value;
283
284 spin_lock_irqsave(&mc->lock, flags);
285
286 value = mc_readl(mc, rst->reset);
287 mc_writel(mc, value & ~BIT(rst->bit), rst->reset);
288
289 spin_unlock_irqrestore(&mc->lock, flags);
290
291 return 0;
292}
293
294static int tegra20_mc_hotreset_deassert(struct tegra_mc *mc,
295 const struct tegra_mc_reset *rst)
296{
297 unsigned long flags;
298 u32 value;
299
300 spin_lock_irqsave(&mc->lock, flags);
301
302 value = mc_readl(mc, rst->reset);
303 mc_writel(mc, value | BIT(rst->bit), rst->reset);
304
305 spin_unlock_irqrestore(&mc->lock, flags);
306
307 return 0;
308}
309
310static int tegra20_mc_block_dma(struct tegra_mc *mc,
311 const struct tegra_mc_reset *rst)
312{
313 unsigned long flags;
314 u32 value;
315
316 spin_lock_irqsave(&mc->lock, flags);
317
318 value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
319 mc_writel(mc, value, rst->control);
320
321 spin_unlock_irqrestore(&mc->lock, flags);
322
323 return 0;
324}
325
326static bool tegra20_mc_dma_idling(struct tegra_mc *mc,
327 const struct tegra_mc_reset *rst)
328{
329 return mc_readl(mc, rst->status) == 0;
330}
331
332static int tegra20_mc_reset_status(struct tegra_mc *mc,
333 const struct tegra_mc_reset *rst)
334{
335 return (mc_readl(mc, rst->reset) & BIT(rst->bit)) == 0;
336}
337
338static int tegra20_mc_unblock_dma(struct tegra_mc *mc,
339 const struct tegra_mc_reset *rst)
340{
341 unsigned long flags;
342 u32 value;
343
344 spin_lock_irqsave(&mc->lock, flags);
345
346 value = mc_readl(mc, rst->control) | BIT(rst->bit);
347 mc_writel(mc, value, rst->control);
348
349 spin_unlock_irqrestore(&mc->lock, flags);
350
351 return 0;
352}
353
354static const struct tegra_mc_reset_ops tegra20_mc_reset_ops = {
355 .hotreset_assert = tegra20_mc_hotreset_assert,
356 .hotreset_deassert = tegra20_mc_hotreset_deassert,
357 .block_dma = tegra20_mc_block_dma,
358 .dma_idling = tegra20_mc_dma_idling,
359 .unblock_dma = tegra20_mc_unblock_dma,
360 .reset_status = tegra20_mc_reset_status,
361};
362
363static int tegra20_mc_icc_set(struct icc_node *src, struct icc_node *dst)
364{
365 /*
366 * It should be possible to tune arbitration knobs here, but the
367 * default values are known to work well on all devices. Hence
368 * nothing to do here so far.
369 */
370 return 0;
371}
372
373static int tegra20_mc_icc_aggreate(struct icc_node *node, u32 tag, u32 avg_bw,
374 u32 peak_bw, u32 *agg_avg, u32 *agg_peak)
375{
376 /*
377 * ISO clients need to reserve extra bandwidth up-front because
378 * there could be high bandwidth pressure during initial filling
379 * of the client's FIFO buffers. Secondly, we need to take into
380 * account impurities of the memory subsystem.
381 */
382 if (tag & TEGRA_MC_ICC_TAG_ISO)
383 peak_bw = tegra_mc_scale_percents(peak_bw, 300);
384
385 *agg_avg += avg_bw;
386 *agg_peak = max(*agg_peak, peak_bw);
387
388 return 0;
389}
390
391static struct icc_node_data *
392tegra20_mc_of_icc_xlate_extended(struct of_phandle_args *spec, void *data)
393{
394 struct tegra_mc *mc = icc_provider_to_tegra_mc(data);
395 unsigned int i, idx = spec->args[0];
396 struct icc_node_data *ndata;
397 struct icc_node *node;
398
399 list_for_each_entry(node, &mc->provider.nodes, node_list) {
400 if (node->id != idx)
401 continue;
402
403 ndata = kzalloc(sizeof(*ndata), GFP_KERNEL);
404 if (!ndata)
405 return ERR_PTR(-ENOMEM);
406
407 ndata->node = node;
408
409 /* these clients are isochronous by default */
410 if (strstarts(node->name, "display") ||
411 strstarts(node->name, "vi"))
412 ndata->tag = TEGRA_MC_ICC_TAG_ISO;
413 else
414 ndata->tag = TEGRA_MC_ICC_TAG_DEFAULT;
415
416 return ndata;
417 }
418
419 for (i = 0; i < mc->soc->num_clients; i++) {
420 if (mc->soc->clients[i].id == idx)
421 return ERR_PTR(-EPROBE_DEFER);
422 }
423
424 dev_err(mc->dev, "invalid ICC client ID %u\n", idx);
425
426 return ERR_PTR(-EINVAL);
427}
428
429static const struct tegra_mc_icc_ops tegra20_mc_icc_ops = {
430 .xlate_extended = tegra20_mc_of_icc_xlate_extended,
431 .aggregate = tegra20_mc_icc_aggreate,
432 .set = tegra20_mc_icc_set,
433};
434
435static u32 tegra20_mc_stat_gather_control(const struct tegra20_mc_stat_gather *g)
436{
437 u32 control;
438
439 control = FIELD_PREP(MC_STAT_CONTROL_EVENT, g->event);
440 control |= FIELD_PREP(MC_STAT_CONTROL_CLIENT_ID, g->client);
441 control |= FIELD_PREP(MC_STAT_CONTROL_PRI_EVENT, g->pri_event);
442 control |= FIELD_PREP(MC_STAT_CONTROL_FILTER_PRI, g->pri_filter);
443 control |= FIELD_PREP(MC_STAT_CONTROL_FILTER_CLIENT_ENABLE, g->client_enb);
444
445 return control;
446}
447
448static void tegra20_mc_stat_gather(struct tegra20_mc_stat *stat)
449{
450 u32 clocks, count0, count1, control_0, control_1;
451 const struct tegra_mc *mc = stat->mc;
452
453 control_0 = tegra20_mc_stat_gather_control(&stat->gather0);
454 control_1 = tegra20_mc_stat_gather_control(&stat->gather1);
455
456 /*
457 * Reset statistic gathers state, select statistics collection mode
458 * and set clocks counter saturation limit to maximum.
459 */
460 mc_writel(mc, 0x00000000, MC_STAT_CONTROL);
461 mc_writel(mc, control_0, MC_STAT_EMC_CONTROL_0);
462 mc_writel(mc, control_1, MC_STAT_EMC_CONTROL_1);
463 mc_writel(mc, 0xffffffff, MC_STAT_EMC_CLOCK_LIMIT);
464
465 mc_writel(mc, EMC_GATHER_ENABLE, MC_STAT_CONTROL);
466 fsleep(stat->sample_time_usec);
467 mc_writel(mc, EMC_GATHER_DISABLE, MC_STAT_CONTROL);
468
469 count0 = mc_readl(mc, MC_STAT_EMC_COUNT_0);
470 count1 = mc_readl(mc, MC_STAT_EMC_COUNT_1);
471 clocks = mc_readl(mc, MC_STAT_EMC_CLOCKS);
472 clocks = max(clocks / 100 / MC_FX_FRAC_SCALE, 1u);
473
474 stat->gather0.result = DIV_ROUND_UP(count0, clocks);
475 stat->gather1.result = DIV_ROUND_UP(count1, clocks);
476}
477
478static void tegra20_mc_stat_events(const struct tegra_mc *mc,
479 const struct tegra_mc_client *client0,
480 const struct tegra_mc_client *client1,
481 unsigned int pri_filter,
482 unsigned int pri_event,
483 unsigned int event,
484 unsigned int *result0,
485 unsigned int *result1)
486{
487 struct tegra20_mc_stat stat = {};
488
489 stat.gather0.client = client0 ? client0->id : 0;
490 stat.gather0.pri_filter = pri_filter;
491 stat.gather0.client_enb = !!client0;
492 stat.gather0.pri_event = pri_event;
493 stat.gather0.event = event;
494
495 stat.gather1.client = client1 ? client1->id : 0;
496 stat.gather1.pri_filter = pri_filter;
497 stat.gather1.client_enb = !!client1;
498 stat.gather1.pri_event = pri_event;
499 stat.gather1.event = event;
500
501 stat.sample_time_usec = MC_STAT_SAMPLE_TIME_USEC;
502 stat.mc = mc;
503
504 tegra20_mc_stat_gather(&stat);
505
506 *result0 = stat.gather0.result;
507 *result1 = stat.gather1.result;
508}
509
510static void tegra20_mc_collect_stats(const struct tegra_mc *mc,
511 struct tegra20_mc_client_stat *stats)
512{
513 const struct tegra_mc_client *client0, *client1;
514 unsigned int i;
515
516 /* collect memory controller utilization percent for each client */
517 for (i = 0; i < mc->soc->num_clients; i += 2) {
518 client0 = &mc->soc->clients[i];
519 client1 = &mc->soc->clients[i + 1];
520
521 if (i + 1 == mc->soc->num_clients)
522 client1 = NULL;
523
524 tegra20_mc_stat_events(mc, client0, client1,
525 MC_STAT_CONTROL_FILTER_PRI_DISABLE,
526 MC_STAT_CONTROL_PRI_EVENT_HP,
527 MC_STAT_CONTROL_EVENT_QUALIFIED,
528 &stats[i + 0].events,
529 &stats[i + 1].events);
530 }
531
532 /* collect more info from active clients */
533 for (i = 0; i < mc->soc->num_clients; i++) {
534 unsigned int clienta, clientb = mc->soc->num_clients;
535
536 for (client0 = NULL; i < mc->soc->num_clients; i++) {
537 if (stats[i].events) {
538 client0 = &mc->soc->clients[i];
539 clienta = i++;
540 break;
541 }
542 }
543
544 for (client1 = NULL; i < mc->soc->num_clients; i++) {
545 if (stats[i].events) {
546 client1 = &mc->soc->clients[i];
547 clientb = i;
548 break;
549 }
550 }
551
552 if (!client0 && !client1)
553 break;
554
555 tegra20_mc_stat_events(mc, client0, client1,
556 MC_STAT_CONTROL_FILTER_PRI_YES,
557 MC_STAT_CONTROL_PRI_EVENT_HP,
558 MC_STAT_CONTROL_EVENT_QUALIFIED,
559 &stats[clienta].arb_high_prio,
560 &stats[clientb].arb_high_prio);
561
562 tegra20_mc_stat_events(mc, client0, client1,
563 MC_STAT_CONTROL_FILTER_PRI_YES,
564 MC_STAT_CONTROL_PRI_EVENT_TM,
565 MC_STAT_CONTROL_EVENT_QUALIFIED,
566 &stats[clienta].arb_timeout,
567 &stats[clientb].arb_timeout);
568
569 tegra20_mc_stat_events(mc, client0, client1,
570 MC_STAT_CONTROL_FILTER_PRI_YES,
571 MC_STAT_CONTROL_PRI_EVENT_BW,
572 MC_STAT_CONTROL_EVENT_QUALIFIED,
573 &stats[clienta].arb_bandwidth,
574 &stats[clientb].arb_bandwidth);
575
576 tegra20_mc_stat_events(mc, client0, client1,
577 MC_STAT_CONTROL_FILTER_PRI_DISABLE,
578 MC_STAT_CONTROL_PRI_EVENT_HP,
579 MC_STAT_CONTROL_EVENT_RD_WR_CHANGE,
580 &stats[clienta].rd_wr_change,
581 &stats[clientb].rd_wr_change);
582
583 tegra20_mc_stat_events(mc, client0, client1,
584 MC_STAT_CONTROL_FILTER_PRI_DISABLE,
585 MC_STAT_CONTROL_PRI_EVENT_HP,
586 MC_STAT_CONTROL_EVENT_SUCCESSIVE,
587 &stats[clienta].successive,
588 &stats[clientb].successive);
589
590 tegra20_mc_stat_events(mc, client0, client1,
591 MC_STAT_CONTROL_FILTER_PRI_DISABLE,
592 MC_STAT_CONTROL_PRI_EVENT_HP,
593 MC_STAT_CONTROL_EVENT_PAGE_MISS,
594 &stats[clienta].page_miss,
595 &stats[clientb].page_miss);
596 }
597}
598
599static void tegra20_mc_printf_percents(struct seq_file *s,
600 const char *fmt,
601 unsigned int percents_fx)
602{
603 char percents_str[8];
604
605 snprintf(percents_str, ARRAY_SIZE(percents_str), "%3u.%02u%%",
606 percents_fx / MC_FX_FRAC_SCALE, percents_fx % MC_FX_FRAC_SCALE);
607
608 seq_printf(s, fmt, percents_str);
609}
610
611static int tegra20_mc_stats_show(struct seq_file *s, void *unused)
612{
613 const struct tegra_mc *mc = dev_get_drvdata(s->private);
614 struct tegra20_mc_client_stat *stats;
615 unsigned int i;
616
617 stats = kcalloc(mc->soc->num_clients + 1, sizeof(*stats), GFP_KERNEL);
618 if (!stats)
619 return -ENOMEM;
620
621 mutex_lock(&tegra20_mc_stat_lock);
622
623 tegra20_mc_collect_stats(mc, stats);
624
625 mutex_unlock(&tegra20_mc_stat_lock);
626
627 seq_puts(s, "Memory client Events Timeout High priority Bandwidth ARB RW change Successive Page miss\n");
628 seq_puts(s, "-----------------------------------------------------------------------------------------------------\n");
629
630 for (i = 0; i < mc->soc->num_clients; i++) {
631 seq_printf(s, "%-14s ", mc->soc->clients[i].name);
632
633 /* An event is generated when client performs R/W request. */
634 tegra20_mc_printf_percents(s, "%-9s", stats[i].events);
635
636 /*
637 * An event is generated based on the timeout (TM) signal
638 * accompanying a request for arbitration.
639 */
640 tegra20_mc_printf_percents(s, "%-10s", stats[i].arb_timeout);
641
642 /*
643 * An event is generated based on the high-priority (HP) signal
644 * accompanying a request for arbitration.
645 */
646 tegra20_mc_printf_percents(s, "%-16s", stats[i].arb_high_prio);
647
648 /*
649 * An event is generated based on the bandwidth (BW) signal
650 * accompanying a request for arbitration.
651 */
652 tegra20_mc_printf_percents(s, "%-16s", stats[i].arb_bandwidth);
653
654 /*
655 * An event is generated when the memory controller switches
656 * between making a read request to making a write request.
657 */
658 tegra20_mc_printf_percents(s, "%-12s", stats[i].rd_wr_change);
659
660 /*
661 * An even generated when the chosen client has wins arbitration
662 * when it was also the winner at the previous request. If a
663 * client makes N requests in a row that are honored, SUCCESSIVE
664 * will be counted (N-1) times. Large values for this event
665 * imply that if we were patient enough, all of those requests
666 * could have been coalesced.
667 */
668 tegra20_mc_printf_percents(s, "%-13s", stats[i].successive);
669
670 /*
671 * An event is generated when the memory controller detects a
672 * page miss for the current request.
673 */
674 tegra20_mc_printf_percents(s, "%-12s\n", stats[i].page_miss);
675 }
676
677 kfree(stats);
678
679 return 0;
680}
681
682static int tegra20_mc_probe(struct tegra_mc *mc)
683{
684 debugfs_create_devm_seqfile(mc->dev, "stats", mc->debugfs.root,
685 tegra20_mc_stats_show);
686
687 return 0;
688}
689
690static int tegra20_mc_suspend(struct tegra_mc *mc)
691{
692 int err;
693
694 if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) {
695 err = tegra_gart_suspend(mc->gart);
696 if (err < 0)
697 return err;
698 }
699
700 return 0;
701}
702
703static int tegra20_mc_resume(struct tegra_mc *mc)
704{
705 int err;
706
707 if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) {
708 err = tegra_gart_resume(mc->gart);
709 if (err < 0)
710 return err;
711 }
712
713 return 0;
714}
715
716static irqreturn_t tegra20_mc_handle_irq(int irq, void *data)
717{
718 struct tegra_mc *mc = data;
719 unsigned long status;
720 unsigned int bit;
721
722 /* mask all interrupts to avoid flooding */
723 status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
724 if (!status)
725 return IRQ_NONE;
726
727 for_each_set_bit(bit, &status, 32) {
728 const char *error = tegra_mc_status_names[bit];
729 const char *direction = "read", *secure = "";
730 const char *client, *desc;
731 phys_addr_t addr;
732 u32 value, reg;
733 u8 id, type;
734
735 switch (BIT(bit)) {
736 case MC_INT_DECERR_EMEM:
737 reg = MC_DECERR_EMEM_OTHERS_STATUS;
738 value = mc_readl(mc, reg);
739
740 id = value & mc->soc->client_id_mask;
741 desc = tegra_mc_error_names[2];
742
743 if (value & BIT(31))
744 direction = "write";
745 break;
746
747 case MC_INT_INVALID_GART_PAGE:
748 reg = MC_GART_ERROR_REQ;
749 value = mc_readl(mc, reg);
750
751 id = (value >> 1) & mc->soc->client_id_mask;
752 desc = tegra_mc_error_names[2];
753
754 if (value & BIT(0))
755 direction = "write";
756 break;
757
758 case MC_INT_SECURITY_VIOLATION:
759 reg = MC_SECURITY_VIOLATION_STATUS;
760 value = mc_readl(mc, reg);
761
762 id = value & mc->soc->client_id_mask;
763 type = (value & BIT(30)) ? 4 : 3;
764 desc = tegra_mc_error_names[type];
765 secure = "secure ";
766
767 if (value & BIT(31))
768 direction = "write";
769 break;
770
771 default:
772 continue;
773 }
774
775 client = mc->soc->clients[id].name;
776 addr = mc_readl(mc, reg + sizeof(u32));
777
778 dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s)\n",
779 client, secure, direction, &addr, error,
780 desc);
781 }
782
783 /* clear interrupts */
784 mc_writel(mc, status, MC_INTSTATUS);
785
786 return IRQ_HANDLED;
787}
788
789static const struct tegra_mc_ops tegra20_mc_ops = {
790 .probe = tegra20_mc_probe,
791 .suspend = tegra20_mc_suspend,
792 .resume = tegra20_mc_resume,
793 .handle_irq = tegra20_mc_handle_irq,
794};
795
796const struct tegra_mc_soc tegra20_mc_soc = {
797 .clients = tegra20_mc_clients,
798 .num_clients = ARRAY_SIZE(tegra20_mc_clients),
799 .num_address_bits = 32,
800 .client_id_mask = 0x3f,
801 .intmask = MC_INT_SECURITY_VIOLATION | MC_INT_INVALID_GART_PAGE |
802 MC_INT_DECERR_EMEM,
803 .reset_ops = &tegra20_mc_reset_ops,
804 .resets = tegra20_mc_resets,
805 .num_resets = ARRAY_SIZE(tegra20_mc_resets),
806 .icc_ops = &tegra20_mc_icc_ops,
807 .ops = &tegra20_mc_ops,
808};
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2012 NVIDIA CORPORATION. All rights reserved.
4 */
5
6#include <linux/bitfield.h>
7#include <linux/delay.h>
8#include <linux/device.h>
9#include <linux/mutex.h>
10#include <linux/of.h>
11#include <linux/slab.h>
12#include <linux/string.h>
13
14#include <dt-bindings/memory/tegra20-mc.h>
15
16#include "mc.h"
17
18#define MC_STAT_CONTROL 0x90
19#define MC_STAT_EMC_CLOCK_LIMIT 0xa0
20#define MC_STAT_EMC_CLOCKS 0xa4
21#define MC_STAT_EMC_CONTROL_0 0xa8
22#define MC_STAT_EMC_CONTROL_1 0xac
23#define MC_STAT_EMC_COUNT_0 0xb8
24#define MC_STAT_EMC_COUNT_1 0xbc
25
26#define MC_STAT_CONTROL_CLIENT_ID GENMASK(13, 8)
27#define MC_STAT_CONTROL_EVENT GENMASK(23, 16)
28#define MC_STAT_CONTROL_PRI_EVENT GENMASK(25, 24)
29#define MC_STAT_CONTROL_FILTER_CLIENT_ENABLE GENMASK(26, 26)
30#define MC_STAT_CONTROL_FILTER_PRI GENMASK(29, 28)
31
32#define MC_STAT_CONTROL_PRI_EVENT_HP 0
33#define MC_STAT_CONTROL_PRI_EVENT_TM 1
34#define MC_STAT_CONTROL_PRI_EVENT_BW 2
35
36#define MC_STAT_CONTROL_FILTER_PRI_DISABLE 0
37#define MC_STAT_CONTROL_FILTER_PRI_NO 1
38#define MC_STAT_CONTROL_FILTER_PRI_YES 2
39
40#define MC_STAT_CONTROL_EVENT_QUALIFIED 0
41#define MC_STAT_CONTROL_EVENT_ANY_READ 1
42#define MC_STAT_CONTROL_EVENT_ANY_WRITE 2
43#define MC_STAT_CONTROL_EVENT_RD_WR_CHANGE 3
44#define MC_STAT_CONTROL_EVENT_SUCCESSIVE 4
45#define MC_STAT_CONTROL_EVENT_ARB_BANK_AA 5
46#define MC_STAT_CONTROL_EVENT_ARB_BANK_BB 6
47#define MC_STAT_CONTROL_EVENT_PAGE_MISS 7
48#define MC_STAT_CONTROL_EVENT_AUTO_PRECHARGE 8
49
50#define EMC_GATHER_RST (0 << 8)
51#define EMC_GATHER_CLEAR (1 << 8)
52#define EMC_GATHER_DISABLE (2 << 8)
53#define EMC_GATHER_ENABLE (3 << 8)
54
55#define MC_STAT_SAMPLE_TIME_USEC 16000
56
57/* we store collected statistics as a fixed point values */
58#define MC_FX_FRAC_SCALE 100
59
60static DEFINE_MUTEX(tegra20_mc_stat_lock);
61
62struct tegra20_mc_stat_gather {
63 unsigned int pri_filter;
64 unsigned int pri_event;
65 unsigned int result;
66 unsigned int client;
67 unsigned int event;
68 bool client_enb;
69};
70
71struct tegra20_mc_stat {
72 struct tegra20_mc_stat_gather gather0;
73 struct tegra20_mc_stat_gather gather1;
74 unsigned int sample_time_usec;
75 const struct tegra_mc *mc;
76};
77
78struct tegra20_mc_client_stat {
79 unsigned int events;
80 unsigned int arb_high_prio;
81 unsigned int arb_timeout;
82 unsigned int arb_bandwidth;
83 unsigned int rd_wr_change;
84 unsigned int successive;
85 unsigned int page_miss;
86 unsigned int auto_precharge;
87 unsigned int arb_bank_aa;
88 unsigned int arb_bank_bb;
89};
90
91static const struct tegra_mc_client tegra20_mc_clients[] = {
92 {
93 .id = 0x00,
94 .name = "display0a",
95 }, {
96 .id = 0x01,
97 .name = "display0ab",
98 }, {
99 .id = 0x02,
100 .name = "display0b",
101 }, {
102 .id = 0x03,
103 .name = "display0bb",
104 }, {
105 .id = 0x04,
106 .name = "display0c",
107 }, {
108 .id = 0x05,
109 .name = "display0cb",
110 }, {
111 .id = 0x06,
112 .name = "display1b",
113 }, {
114 .id = 0x07,
115 .name = "display1bb",
116 }, {
117 .id = 0x08,
118 .name = "eppup",
119 }, {
120 .id = 0x09,
121 .name = "g2pr",
122 }, {
123 .id = 0x0a,
124 .name = "g2sr",
125 }, {
126 .id = 0x0b,
127 .name = "mpeunifbr",
128 }, {
129 .id = 0x0c,
130 .name = "viruv",
131 }, {
132 .id = 0x0d,
133 .name = "avpcarm7r",
134 }, {
135 .id = 0x0e,
136 .name = "displayhc",
137 }, {
138 .id = 0x0f,
139 .name = "displayhcb",
140 }, {
141 .id = 0x10,
142 .name = "fdcdrd",
143 }, {
144 .id = 0x11,
145 .name = "g2dr",
146 }, {
147 .id = 0x12,
148 .name = "host1xdmar",
149 }, {
150 .id = 0x13,
151 .name = "host1xr",
152 }, {
153 .id = 0x14,
154 .name = "idxsrd",
155 }, {
156 .id = 0x15,
157 .name = "mpcorer",
158 }, {
159 .id = 0x16,
160 .name = "mpe_ipred",
161 }, {
162 .id = 0x17,
163 .name = "mpeamemrd",
164 }, {
165 .id = 0x18,
166 .name = "mpecsrd",
167 }, {
168 .id = 0x19,
169 .name = "ppcsahbdmar",
170 }, {
171 .id = 0x1a,
172 .name = "ppcsahbslvr",
173 }, {
174 .id = 0x1b,
175 .name = "texsrd",
176 }, {
177 .id = 0x1c,
178 .name = "vdebsevr",
179 }, {
180 .id = 0x1d,
181 .name = "vdember",
182 }, {
183 .id = 0x1e,
184 .name = "vdemcer",
185 }, {
186 .id = 0x1f,
187 .name = "vdetper",
188 }, {
189 .id = 0x20,
190 .name = "eppu",
191 }, {
192 .id = 0x21,
193 .name = "eppv",
194 }, {
195 .id = 0x22,
196 .name = "eppy",
197 }, {
198 .id = 0x23,
199 .name = "mpeunifbw",
200 }, {
201 .id = 0x24,
202 .name = "viwsb",
203 }, {
204 .id = 0x25,
205 .name = "viwu",
206 }, {
207 .id = 0x26,
208 .name = "viwv",
209 }, {
210 .id = 0x27,
211 .name = "viwy",
212 }, {
213 .id = 0x28,
214 .name = "g2dw",
215 }, {
216 .id = 0x29,
217 .name = "avpcarm7w",
218 }, {
219 .id = 0x2a,
220 .name = "fdcdwr",
221 }, {
222 .id = 0x2b,
223 .name = "host1xw",
224 }, {
225 .id = 0x2c,
226 .name = "ispw",
227 }, {
228 .id = 0x2d,
229 .name = "mpcorew",
230 }, {
231 .id = 0x2e,
232 .name = "mpecswr",
233 }, {
234 .id = 0x2f,
235 .name = "ppcsahbdmaw",
236 }, {
237 .id = 0x30,
238 .name = "ppcsahbslvw",
239 }, {
240 .id = 0x31,
241 .name = "vdebsevw",
242 }, {
243 .id = 0x32,
244 .name = "vdembew",
245 }, {
246 .id = 0x33,
247 .name = "vdetpmw",
248 },
249};
250
251#define TEGRA20_MC_RESET(_name, _control, _status, _reset, _bit) \
252 { \
253 .name = #_name, \
254 .id = TEGRA20_MC_RESET_##_name, \
255 .control = _control, \
256 .status = _status, \
257 .reset = _reset, \
258 .bit = _bit, \
259 }
260
261static const struct tegra_mc_reset tegra20_mc_resets[] = {
262 TEGRA20_MC_RESET(AVPC, 0x100, 0x140, 0x104, 0),
263 TEGRA20_MC_RESET(DC, 0x100, 0x144, 0x104, 1),
264 TEGRA20_MC_RESET(DCB, 0x100, 0x148, 0x104, 2),
265 TEGRA20_MC_RESET(EPP, 0x100, 0x14c, 0x104, 3),
266 TEGRA20_MC_RESET(2D, 0x100, 0x150, 0x104, 4),
267 TEGRA20_MC_RESET(HC, 0x100, 0x154, 0x104, 5),
268 TEGRA20_MC_RESET(ISP, 0x100, 0x158, 0x104, 6),
269 TEGRA20_MC_RESET(MPCORE, 0x100, 0x15c, 0x104, 7),
270 TEGRA20_MC_RESET(MPEA, 0x100, 0x160, 0x104, 8),
271 TEGRA20_MC_RESET(MPEB, 0x100, 0x164, 0x104, 9),
272 TEGRA20_MC_RESET(MPEC, 0x100, 0x168, 0x104, 10),
273 TEGRA20_MC_RESET(3D, 0x100, 0x16c, 0x104, 11),
274 TEGRA20_MC_RESET(PPCS, 0x100, 0x170, 0x104, 12),
275 TEGRA20_MC_RESET(VDE, 0x100, 0x174, 0x104, 13),
276 TEGRA20_MC_RESET(VI, 0x100, 0x178, 0x104, 14),
277};
278
279static int tegra20_mc_hotreset_assert(struct tegra_mc *mc,
280 const struct tegra_mc_reset *rst)
281{
282 unsigned long flags;
283 u32 value;
284
285 spin_lock_irqsave(&mc->lock, flags);
286
287 value = mc_readl(mc, rst->reset);
288 mc_writel(mc, value & ~BIT(rst->bit), rst->reset);
289
290 spin_unlock_irqrestore(&mc->lock, flags);
291
292 return 0;
293}
294
295static int tegra20_mc_hotreset_deassert(struct tegra_mc *mc,
296 const struct tegra_mc_reset *rst)
297{
298 unsigned long flags;
299 u32 value;
300
301 spin_lock_irqsave(&mc->lock, flags);
302
303 value = mc_readl(mc, rst->reset);
304 mc_writel(mc, value | BIT(rst->bit), rst->reset);
305
306 spin_unlock_irqrestore(&mc->lock, flags);
307
308 return 0;
309}
310
311static int tegra20_mc_block_dma(struct tegra_mc *mc,
312 const struct tegra_mc_reset *rst)
313{
314 unsigned long flags;
315 u32 value;
316
317 spin_lock_irqsave(&mc->lock, flags);
318
319 value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
320 mc_writel(mc, value, rst->control);
321
322 spin_unlock_irqrestore(&mc->lock, flags);
323
324 return 0;
325}
326
327static bool tegra20_mc_dma_idling(struct tegra_mc *mc,
328 const struct tegra_mc_reset *rst)
329{
330 return mc_readl(mc, rst->status) == 0;
331}
332
333static int tegra20_mc_reset_status(struct tegra_mc *mc,
334 const struct tegra_mc_reset *rst)
335{
336 return (mc_readl(mc, rst->reset) & BIT(rst->bit)) == 0;
337}
338
339static int tegra20_mc_unblock_dma(struct tegra_mc *mc,
340 const struct tegra_mc_reset *rst)
341{
342 unsigned long flags;
343 u32 value;
344
345 spin_lock_irqsave(&mc->lock, flags);
346
347 value = mc_readl(mc, rst->control) | BIT(rst->bit);
348 mc_writel(mc, value, rst->control);
349
350 spin_unlock_irqrestore(&mc->lock, flags);
351
352 return 0;
353}
354
355static const struct tegra_mc_reset_ops tegra20_mc_reset_ops = {
356 .hotreset_assert = tegra20_mc_hotreset_assert,
357 .hotreset_deassert = tegra20_mc_hotreset_deassert,
358 .block_dma = tegra20_mc_block_dma,
359 .dma_idling = tegra20_mc_dma_idling,
360 .unblock_dma = tegra20_mc_unblock_dma,
361 .reset_status = tegra20_mc_reset_status,
362};
363
364static int tegra20_mc_icc_set(struct icc_node *src, struct icc_node *dst)
365{
366 /*
367 * It should be possible to tune arbitration knobs here, but the
368 * default values are known to work well on all devices. Hence
369 * nothing to do here so far.
370 */
371 return 0;
372}
373
374static int tegra20_mc_icc_aggreate(struct icc_node *node, u32 tag, u32 avg_bw,
375 u32 peak_bw, u32 *agg_avg, u32 *agg_peak)
376{
377 /*
378 * ISO clients need to reserve extra bandwidth up-front because
379 * there could be high bandwidth pressure during initial filling
380 * of the client's FIFO buffers. Secondly, we need to take into
381 * account impurities of the memory subsystem.
382 */
383 if (tag & TEGRA_MC_ICC_TAG_ISO)
384 peak_bw = tegra_mc_scale_percents(peak_bw, 300);
385
386 *agg_avg += avg_bw;
387 *agg_peak = max(*agg_peak, peak_bw);
388
389 return 0;
390}
391
392static struct icc_node_data *
393tegra20_mc_of_icc_xlate_extended(const struct of_phandle_args *spec, void *data)
394{
395 struct tegra_mc *mc = icc_provider_to_tegra_mc(data);
396 unsigned int i, idx = spec->args[0];
397 struct icc_node_data *ndata;
398 struct icc_node *node;
399
400 list_for_each_entry(node, &mc->provider.nodes, node_list) {
401 if (node->id != idx)
402 continue;
403
404 ndata = kzalloc(sizeof(*ndata), GFP_KERNEL);
405 if (!ndata)
406 return ERR_PTR(-ENOMEM);
407
408 ndata->node = node;
409
410 /* these clients are isochronous by default */
411 if (strstarts(node->name, "display") ||
412 strstarts(node->name, "vi"))
413 ndata->tag = TEGRA_MC_ICC_TAG_ISO;
414 else
415 ndata->tag = TEGRA_MC_ICC_TAG_DEFAULT;
416
417 return ndata;
418 }
419
420 for (i = 0; i < mc->soc->num_clients; i++) {
421 if (mc->soc->clients[i].id == idx)
422 return ERR_PTR(-EPROBE_DEFER);
423 }
424
425 dev_err(mc->dev, "invalid ICC client ID %u\n", idx);
426
427 return ERR_PTR(-EINVAL);
428}
429
430static const struct tegra_mc_icc_ops tegra20_mc_icc_ops = {
431 .xlate_extended = tegra20_mc_of_icc_xlate_extended,
432 .aggregate = tegra20_mc_icc_aggreate,
433 .set = tegra20_mc_icc_set,
434};
435
436static u32 tegra20_mc_stat_gather_control(const struct tegra20_mc_stat_gather *g)
437{
438 u32 control;
439
440 control = FIELD_PREP(MC_STAT_CONTROL_EVENT, g->event);
441 control |= FIELD_PREP(MC_STAT_CONTROL_CLIENT_ID, g->client);
442 control |= FIELD_PREP(MC_STAT_CONTROL_PRI_EVENT, g->pri_event);
443 control |= FIELD_PREP(MC_STAT_CONTROL_FILTER_PRI, g->pri_filter);
444 control |= FIELD_PREP(MC_STAT_CONTROL_FILTER_CLIENT_ENABLE, g->client_enb);
445
446 return control;
447}
448
449static void tegra20_mc_stat_gather(struct tegra20_mc_stat *stat)
450{
451 u32 clocks, count0, count1, control_0, control_1;
452 const struct tegra_mc *mc = stat->mc;
453
454 control_0 = tegra20_mc_stat_gather_control(&stat->gather0);
455 control_1 = tegra20_mc_stat_gather_control(&stat->gather1);
456
457 /*
458 * Reset statistic gathers state, select statistics collection mode
459 * and set clocks counter saturation limit to maximum.
460 */
461 mc_writel(mc, 0x00000000, MC_STAT_CONTROL);
462 mc_writel(mc, control_0, MC_STAT_EMC_CONTROL_0);
463 mc_writel(mc, control_1, MC_STAT_EMC_CONTROL_1);
464 mc_writel(mc, 0xffffffff, MC_STAT_EMC_CLOCK_LIMIT);
465
466 mc_writel(mc, EMC_GATHER_ENABLE, MC_STAT_CONTROL);
467 fsleep(stat->sample_time_usec);
468 mc_writel(mc, EMC_GATHER_DISABLE, MC_STAT_CONTROL);
469
470 count0 = mc_readl(mc, MC_STAT_EMC_COUNT_0);
471 count1 = mc_readl(mc, MC_STAT_EMC_COUNT_1);
472 clocks = mc_readl(mc, MC_STAT_EMC_CLOCKS);
473 clocks = max(clocks / 100 / MC_FX_FRAC_SCALE, 1u);
474
475 stat->gather0.result = DIV_ROUND_UP(count0, clocks);
476 stat->gather1.result = DIV_ROUND_UP(count1, clocks);
477}
478
479static void tegra20_mc_stat_events(const struct tegra_mc *mc,
480 const struct tegra_mc_client *client0,
481 const struct tegra_mc_client *client1,
482 unsigned int pri_filter,
483 unsigned int pri_event,
484 unsigned int event,
485 unsigned int *result0,
486 unsigned int *result1)
487{
488 struct tegra20_mc_stat stat = {};
489
490 stat.gather0.client = client0 ? client0->id : 0;
491 stat.gather0.pri_filter = pri_filter;
492 stat.gather0.client_enb = !!client0;
493 stat.gather0.pri_event = pri_event;
494 stat.gather0.event = event;
495
496 stat.gather1.client = client1 ? client1->id : 0;
497 stat.gather1.pri_filter = pri_filter;
498 stat.gather1.client_enb = !!client1;
499 stat.gather1.pri_event = pri_event;
500 stat.gather1.event = event;
501
502 stat.sample_time_usec = MC_STAT_SAMPLE_TIME_USEC;
503 stat.mc = mc;
504
505 tegra20_mc_stat_gather(&stat);
506
507 *result0 = stat.gather0.result;
508 *result1 = stat.gather1.result;
509}
510
511static void tegra20_mc_collect_stats(const struct tegra_mc *mc,
512 struct tegra20_mc_client_stat *stats)
513{
514 const struct tegra_mc_client *client0, *client1;
515 unsigned int i;
516
517 /* collect memory controller utilization percent for each client */
518 for (i = 0; i < mc->soc->num_clients; i += 2) {
519 client0 = &mc->soc->clients[i];
520 client1 = &mc->soc->clients[i + 1];
521
522 if (i + 1 == mc->soc->num_clients)
523 client1 = NULL;
524
525 tegra20_mc_stat_events(mc, client0, client1,
526 MC_STAT_CONTROL_FILTER_PRI_DISABLE,
527 MC_STAT_CONTROL_PRI_EVENT_HP,
528 MC_STAT_CONTROL_EVENT_QUALIFIED,
529 &stats[i + 0].events,
530 &stats[i + 1].events);
531 }
532
533 /* collect more info from active clients */
534 for (i = 0; i < mc->soc->num_clients; i++) {
535 unsigned int clienta, clientb = mc->soc->num_clients;
536
537 for (client0 = NULL; i < mc->soc->num_clients; i++) {
538 if (stats[i].events) {
539 client0 = &mc->soc->clients[i];
540 clienta = i++;
541 break;
542 }
543 }
544
545 for (client1 = NULL; i < mc->soc->num_clients; i++) {
546 if (stats[i].events) {
547 client1 = &mc->soc->clients[i];
548 clientb = i;
549 break;
550 }
551 }
552
553 if (!client0 && !client1)
554 break;
555
556 tegra20_mc_stat_events(mc, client0, client1,
557 MC_STAT_CONTROL_FILTER_PRI_YES,
558 MC_STAT_CONTROL_PRI_EVENT_HP,
559 MC_STAT_CONTROL_EVENT_QUALIFIED,
560 &stats[clienta].arb_high_prio,
561 &stats[clientb].arb_high_prio);
562
563 tegra20_mc_stat_events(mc, client0, client1,
564 MC_STAT_CONTROL_FILTER_PRI_YES,
565 MC_STAT_CONTROL_PRI_EVENT_TM,
566 MC_STAT_CONTROL_EVENT_QUALIFIED,
567 &stats[clienta].arb_timeout,
568 &stats[clientb].arb_timeout);
569
570 tegra20_mc_stat_events(mc, client0, client1,
571 MC_STAT_CONTROL_FILTER_PRI_YES,
572 MC_STAT_CONTROL_PRI_EVENT_BW,
573 MC_STAT_CONTROL_EVENT_QUALIFIED,
574 &stats[clienta].arb_bandwidth,
575 &stats[clientb].arb_bandwidth);
576
577 tegra20_mc_stat_events(mc, client0, client1,
578 MC_STAT_CONTROL_FILTER_PRI_DISABLE,
579 MC_STAT_CONTROL_PRI_EVENT_HP,
580 MC_STAT_CONTROL_EVENT_RD_WR_CHANGE,
581 &stats[clienta].rd_wr_change,
582 &stats[clientb].rd_wr_change);
583
584 tegra20_mc_stat_events(mc, client0, client1,
585 MC_STAT_CONTROL_FILTER_PRI_DISABLE,
586 MC_STAT_CONTROL_PRI_EVENT_HP,
587 MC_STAT_CONTROL_EVENT_SUCCESSIVE,
588 &stats[clienta].successive,
589 &stats[clientb].successive);
590
591 tegra20_mc_stat_events(mc, client0, client1,
592 MC_STAT_CONTROL_FILTER_PRI_DISABLE,
593 MC_STAT_CONTROL_PRI_EVENT_HP,
594 MC_STAT_CONTROL_EVENT_PAGE_MISS,
595 &stats[clienta].page_miss,
596 &stats[clientb].page_miss);
597 }
598}
599
600static void tegra20_mc_printf_percents(struct seq_file *s,
601 const char *fmt,
602 unsigned int percents_fx)
603{
604 char percents_str[8];
605
606 snprintf(percents_str, ARRAY_SIZE(percents_str), "%3u.%02u%%",
607 percents_fx / MC_FX_FRAC_SCALE, percents_fx % MC_FX_FRAC_SCALE);
608
609 seq_printf(s, fmt, percents_str);
610}
611
612static int tegra20_mc_stats_show(struct seq_file *s, void *unused)
613{
614 const struct tegra_mc *mc = dev_get_drvdata(s->private);
615 struct tegra20_mc_client_stat *stats;
616 unsigned int i;
617
618 stats = kcalloc(mc->soc->num_clients + 1, sizeof(*stats), GFP_KERNEL);
619 if (!stats)
620 return -ENOMEM;
621
622 mutex_lock(&tegra20_mc_stat_lock);
623
624 tegra20_mc_collect_stats(mc, stats);
625
626 mutex_unlock(&tegra20_mc_stat_lock);
627
628 seq_puts(s, "Memory client Events Timeout High priority Bandwidth ARB RW change Successive Page miss\n");
629 seq_puts(s, "-----------------------------------------------------------------------------------------------------\n");
630
631 for (i = 0; i < mc->soc->num_clients; i++) {
632 seq_printf(s, "%-14s ", mc->soc->clients[i].name);
633
634 /* An event is generated when client performs R/W request. */
635 tegra20_mc_printf_percents(s, "%-9s", stats[i].events);
636
637 /*
638 * An event is generated based on the timeout (TM) signal
639 * accompanying a request for arbitration.
640 */
641 tegra20_mc_printf_percents(s, "%-10s", stats[i].arb_timeout);
642
643 /*
644 * An event is generated based on the high-priority (HP) signal
645 * accompanying a request for arbitration.
646 */
647 tegra20_mc_printf_percents(s, "%-16s", stats[i].arb_high_prio);
648
649 /*
650 * An event is generated based on the bandwidth (BW) signal
651 * accompanying a request for arbitration.
652 */
653 tegra20_mc_printf_percents(s, "%-16s", stats[i].arb_bandwidth);
654
655 /*
656 * An event is generated when the memory controller switches
657 * between making a read request to making a write request.
658 */
659 tegra20_mc_printf_percents(s, "%-12s", stats[i].rd_wr_change);
660
661 /*
662 * An even generated when the chosen client has wins arbitration
663 * when it was also the winner at the previous request. If a
664 * client makes N requests in a row that are honored, SUCCESSIVE
665 * will be counted (N-1) times. Large values for this event
666 * imply that if we were patient enough, all of those requests
667 * could have been coalesced.
668 */
669 tegra20_mc_printf_percents(s, "%-13s", stats[i].successive);
670
671 /*
672 * An event is generated when the memory controller detects a
673 * page miss for the current request.
674 */
675 tegra20_mc_printf_percents(s, "%-12s\n", stats[i].page_miss);
676 }
677
678 kfree(stats);
679
680 return 0;
681}
682
683static int tegra20_mc_probe(struct tegra_mc *mc)
684{
685 debugfs_create_devm_seqfile(mc->dev, "stats", mc->debugfs.root,
686 tegra20_mc_stats_show);
687
688 return 0;
689}
690
691static irqreturn_t tegra20_mc_handle_irq(int irq, void *data)
692{
693 struct tegra_mc *mc = data;
694 unsigned long status;
695 unsigned int bit;
696
697 /* mask all interrupts to avoid flooding */
698 status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
699 if (!status)
700 return IRQ_NONE;
701
702 for_each_set_bit(bit, &status, 32) {
703 const char *error = tegra_mc_status_names[bit];
704 const char *direction = "read", *secure = "";
705 const char *client, *desc;
706 phys_addr_t addr;
707 u32 value, reg;
708 u8 id, type;
709
710 switch (BIT(bit)) {
711 case MC_INT_DECERR_EMEM:
712 reg = MC_DECERR_EMEM_OTHERS_STATUS;
713 value = mc_readl(mc, reg);
714
715 id = value & mc->soc->client_id_mask;
716 desc = tegra_mc_error_names[2];
717
718 if (value & BIT(31))
719 direction = "write";
720 break;
721
722 case MC_INT_INVALID_GART_PAGE:
723 reg = MC_GART_ERROR_REQ;
724 value = mc_readl(mc, reg);
725
726 id = (value >> 1) & mc->soc->client_id_mask;
727 desc = tegra_mc_error_names[2];
728
729 if (value & BIT(0))
730 direction = "write";
731 break;
732
733 case MC_INT_SECURITY_VIOLATION:
734 reg = MC_SECURITY_VIOLATION_STATUS;
735 value = mc_readl(mc, reg);
736
737 id = value & mc->soc->client_id_mask;
738 type = (value & BIT(30)) ? 4 : 3;
739 desc = tegra_mc_error_names[type];
740 secure = "secure ";
741
742 if (value & BIT(31))
743 direction = "write";
744 break;
745
746 default:
747 continue;
748 }
749
750 client = mc->soc->clients[id].name;
751 addr = mc_readl(mc, reg + sizeof(u32));
752
753 dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s)\n",
754 client, secure, direction, &addr, error,
755 desc);
756 }
757
758 /* clear interrupts */
759 mc_writel(mc, status, MC_INTSTATUS);
760
761 return IRQ_HANDLED;
762}
763
764static const struct tegra_mc_ops tegra20_mc_ops = {
765 .probe = tegra20_mc_probe,
766 .handle_irq = tegra20_mc_handle_irq,
767};
768
769const struct tegra_mc_soc tegra20_mc_soc = {
770 .clients = tegra20_mc_clients,
771 .num_clients = ARRAY_SIZE(tegra20_mc_clients),
772 .num_address_bits = 32,
773 .client_id_mask = 0x3f,
774 .intmask = MC_INT_SECURITY_VIOLATION | MC_INT_INVALID_GART_PAGE |
775 MC_INT_DECERR_EMEM,
776 .reset_ops = &tegra20_mc_reset_ops,
777 .resets = tegra20_mc_resets,
778 .num_resets = ARRAY_SIZE(tegra20_mc_resets),
779 .icc_ops = &tegra20_mc_icc_ops,
780 .ops = &tegra20_mc_ops,
781};