Loading...
1#!/usr/bin/env python3
2# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
3"""Convert directories of JSON events to C code."""
4import argparse
5import csv
6import json
7import metric
8import os
9import sys
10from typing import (Callable, Dict, Optional, Sequence, Set, Tuple)
11import collections
12
13# Global command line arguments.
14_args = None
15# List of event tables generated from "/sys" directories.
16_sys_event_tables = []
17# Map from an event name to an architecture standard
18# JsonEvent. Architecture standard events are in json files in the top
19# f'{_args.starting_dir}/{_args.arch}' directory.
20_arch_std_events = {}
21# Track whether an events table is currently being defined and needs closing.
22_close_table = False
23# Events to write out when the table is closed
24_pending_events = []
25# Global BigCString shared by all structures.
26_bcs = None
27# Order specific JsonEvent attributes will be visited.
28_json_event_attributes = [
29 # cmp_sevent related attributes.
30 'name', 'pmu', 'topic', 'desc', 'metric_name', 'metric_group',
31 # Seems useful, put it early.
32 'event',
33 # Short things in alphabetical order.
34 'aggr_mode', 'compat', 'deprecated', 'perpkg', 'unit',
35 # Longer things (the last won't be iterated over during decompress).
36 'metric_constraint', 'metric_expr', 'long_desc'
37]
38
39
40def removesuffix(s: str, suffix: str) -> str:
41 """Remove the suffix from a string
42
43 The removesuffix function is added to str in Python 3.9. We aim for 3.6
44 compatibility and so provide our own function here.
45 """
46 return s[0:-len(suffix)] if s.endswith(suffix) else s
47
48
49def file_name_to_table_name(parents: Sequence[str], dirname: str) -> str:
50 """Generate a C table name from directory names."""
51 tblname = 'pme'
52 for p in parents:
53 tblname += '_' + p
54 tblname += '_' + dirname
55 return tblname.replace('-', '_')
56
57def c_len(s: str) -> int:
58 """Return the length of s a C string
59
60 This doesn't handle all escape characters properly. It first assumes
61 all \ are for escaping, it then adjusts as it will have over counted
62 \\. The code uses \000 rather than \0 as a terminator as an adjacent
63 number would be folded into a string of \0 (ie. "\0" + "5" doesn't
64 equal a terminator followed by the number 5 but the escape of
65 \05). The code adjusts for \000 but not properly for all octal, hex
66 or unicode values.
67 """
68 try:
69 utf = s.encode(encoding='utf-8',errors='strict')
70 except:
71 print(f'broken string {s}')
72 raise
73 return len(utf) - utf.count(b'\\') + utf.count(b'\\\\') - (utf.count(b'\\000') * 2)
74
75class BigCString:
76 """A class to hold many strings concatenated together.
77
78 Generating a large number of stand-alone C strings creates a large
79 number of relocations in position independent code. The BigCString
80 is a helper for this case. It builds a single string which within it
81 are all the other C strings (to avoid memory issues the string
82 itself is held as a list of strings). The offsets within the big
83 string are recorded and when stored to disk these don't need
84 relocation. To reduce the size of the string further, identical
85 strings are merged. If a longer string ends-with the same value as a
86 shorter string, these entries are also merged.
87 """
88 strings: Set[str]
89 big_string: Sequence[str]
90 offsets: Dict[str, int]
91
92 def __init__(self):
93 self.strings = set()
94
95 def add(self, s: str) -> None:
96 """Called to add to the big string."""
97 self.strings.add(s)
98
99 def compute(self) -> None:
100 """Called once all strings are added to compute the string and offsets."""
101
102 folded_strings = {}
103 # Determine if two strings can be folded, ie. let 1 string use the
104 # end of another. First reverse all strings and sort them.
105 sorted_reversed_strings = sorted([x[::-1] for x in self.strings])
106
107 # Strings 'xyz' and 'yz' will now be [ 'zy', 'zyx' ]. Scan forward
108 # for each string to see if there is a better candidate to fold it
109 # into, in the example rather than using 'yz' we can use'xyz' at
110 # an offset of 1. We record which string can be folded into which
111 # in folded_strings, we don't need to record the offset as it is
112 # trivially computed from the string lengths.
113 for pos,s in enumerate(sorted_reversed_strings):
114 best_pos = pos
115 for check_pos in range(pos + 1, len(sorted_reversed_strings)):
116 if sorted_reversed_strings[check_pos].startswith(s):
117 best_pos = check_pos
118 else:
119 break
120 if pos != best_pos:
121 folded_strings[s[::-1]] = sorted_reversed_strings[best_pos][::-1]
122
123 # Compute reverse mappings for debugging.
124 fold_into_strings = collections.defaultdict(set)
125 for key, val in folded_strings.items():
126 if key != val:
127 fold_into_strings[val].add(key)
128
129 # big_string_offset is the current location within the C string
130 # being appended to - comments, etc. don't count. big_string is
131 # the string contents represented as a list. Strings are immutable
132 # in Python and so appending to one causes memory issues, while
133 # lists are mutable.
134 big_string_offset = 0
135 self.big_string = []
136 self.offsets = {}
137
138 # Emit all strings that aren't folded in a sorted manner.
139 for s in sorted(self.strings):
140 if s not in folded_strings:
141 self.offsets[s] = big_string_offset
142 self.big_string.append(f'/* offset={big_string_offset} */ "')
143 self.big_string.append(s)
144 self.big_string.append('"')
145 if s in fold_into_strings:
146 self.big_string.append(' /* also: ' + ', '.join(fold_into_strings[s]) + ' */')
147 self.big_string.append('\n')
148 big_string_offset += c_len(s)
149 continue
150
151 # Compute the offsets of the folded strings.
152 for s in folded_strings.keys():
153 assert s not in self.offsets
154 folded_s = folded_strings[s]
155 self.offsets[s] = self.offsets[folded_s] + c_len(folded_s) - c_len(s)
156
157_bcs = BigCString()
158
159class JsonEvent:
160 """Representation of an event loaded from a json file dictionary."""
161
162 def __init__(self, jd: dict):
163 """Constructor passed the dictionary of parsed json values."""
164
165 def llx(x: int) -> str:
166 """Convert an int to a string similar to a printf modifier of %#llx."""
167 return '0' if x == 0 else hex(x)
168
169 def fixdesc(s: str) -> str:
170 """Fix formatting issue for the desc string."""
171 if s is None:
172 return None
173 return removesuffix(removesuffix(removesuffix(s, '. '),
174 '. '), '.').replace('\n', '\\n').replace(
175 '\"', '\\"').replace('\r', '\\r')
176
177 def convert_aggr_mode(aggr_mode: str) -> Optional[str]:
178 """Returns the aggr_mode_class enum value associated with the JSON string."""
179 if not aggr_mode:
180 return None
181 aggr_mode_to_enum = {
182 'PerChip': '1',
183 'PerCore': '2',
184 }
185 return aggr_mode_to_enum[aggr_mode]
186
187 def lookup_msr(num: str) -> Optional[str]:
188 """Converts the msr number, or first in a list to the appropriate event field."""
189 if not num:
190 return None
191 msrmap = {
192 0x3F6: 'ldlat=',
193 0x1A6: 'offcore_rsp=',
194 0x1A7: 'offcore_rsp=',
195 0x3F7: 'frontend=',
196 }
197 return msrmap[int(num.split(',', 1)[0], 0)]
198
199 def real_event(name: str, event: str) -> Optional[str]:
200 """Convert well known event names to an event string otherwise use the event argument."""
201 fixed = {
202 'inst_retired.any': 'event=0xc0,period=2000003',
203 'inst_retired.any_p': 'event=0xc0,period=2000003',
204 'cpu_clk_unhalted.ref': 'event=0x0,umask=0x03,period=2000003',
205 'cpu_clk_unhalted.thread': 'event=0x3c,period=2000003',
206 'cpu_clk_unhalted.core': 'event=0x3c,period=2000003',
207 'cpu_clk_unhalted.thread_any': 'event=0x3c,any=1,period=2000003',
208 }
209 if not name:
210 return None
211 if name.lower() in fixed:
212 return fixed[name.lower()]
213 return event
214
215 def unit_to_pmu(unit: str) -> Optional[str]:
216 """Convert a JSON Unit to Linux PMU name."""
217 if not unit:
218 return None
219 # Comment brought over from jevents.c:
220 # it's not realistic to keep adding these, we need something more scalable ...
221 table = {
222 'CBO': 'uncore_cbox',
223 'QPI LL': 'uncore_qpi',
224 'SBO': 'uncore_sbox',
225 'iMPH-U': 'uncore_arb',
226 'CPU-M-CF': 'cpum_cf',
227 'CPU-M-SF': 'cpum_sf',
228 'PAI-CRYPTO' : 'pai_crypto',
229 'UPI LL': 'uncore_upi',
230 'hisi_sicl,cpa': 'hisi_sicl,cpa',
231 'hisi_sccl,ddrc': 'hisi_sccl,ddrc',
232 'hisi_sccl,hha': 'hisi_sccl,hha',
233 'hisi_sccl,l3c': 'hisi_sccl,l3c',
234 'imx8_ddr': 'imx8_ddr',
235 'L3PMC': 'amd_l3',
236 'DFPMC': 'amd_df',
237 'cpu_core': 'cpu_core',
238 'cpu_atom': 'cpu_atom',
239 }
240 return table[unit] if unit in table else f'uncore_{unit.lower()}'
241
242 eventcode = 0
243 if 'EventCode' in jd:
244 eventcode = int(jd['EventCode'].split(',', 1)[0], 0)
245 if 'ExtSel' in jd:
246 eventcode |= int(jd['ExtSel']) << 8
247 configcode = int(jd['ConfigCode'], 0) if 'ConfigCode' in jd else None
248 self.name = jd['EventName'].lower() if 'EventName' in jd else None
249 self.topic = ''
250 self.compat = jd.get('Compat')
251 self.desc = fixdesc(jd.get('BriefDescription'))
252 self.long_desc = fixdesc(jd.get('PublicDescription'))
253 precise = jd.get('PEBS')
254 msr = lookup_msr(jd.get('MSRIndex'))
255 msrval = jd.get('MSRValue')
256 extra_desc = ''
257 if 'Data_LA' in jd:
258 extra_desc += ' Supports address when precise'
259 if 'Errata' in jd:
260 extra_desc += '.'
261 if 'Errata' in jd:
262 extra_desc += ' Spec update: ' + jd['Errata']
263 self.pmu = unit_to_pmu(jd.get('Unit'))
264 filter = jd.get('Filter')
265 self.unit = jd.get('ScaleUnit')
266 self.perpkg = jd.get('PerPkg')
267 self.aggr_mode = convert_aggr_mode(jd.get('AggregationMode'))
268 self.deprecated = jd.get('Deprecated')
269 self.metric_name = jd.get('MetricName')
270 self.metric_group = jd.get('MetricGroup')
271 self.metric_constraint = jd.get('MetricConstraint')
272 self.metric_expr = None
273 if 'MetricExpr' in jd:
274 self.metric_expr = metric.ParsePerfJson(jd['MetricExpr']).Simplify()
275
276 arch_std = jd.get('ArchStdEvent')
277 if precise and self.desc and '(Precise Event)' not in self.desc:
278 extra_desc += ' (Must be precise)' if precise == '2' else (' (Precise '
279 'event)')
280 event = f'config={llx(configcode)}' if configcode is not None else f'event={llx(eventcode)}'
281 event_fields = [
282 ('AnyThread', 'any='),
283 ('PortMask', 'ch_mask='),
284 ('CounterMask', 'cmask='),
285 ('EdgeDetect', 'edge='),
286 ('FCMask', 'fc_mask='),
287 ('Invert', 'inv='),
288 ('SampleAfterValue', 'period='),
289 ('UMask', 'umask='),
290 ]
291 for key, value in event_fields:
292 if key in jd and jd[key] != '0':
293 event += ',' + value + jd[key]
294 if filter:
295 event += f',{filter}'
296 if msr:
297 event += f',{msr}{msrval}'
298 if self.desc and extra_desc:
299 self.desc += extra_desc
300 if self.long_desc and extra_desc:
301 self.long_desc += extra_desc
302 if self.pmu:
303 if self.desc and not self.desc.endswith('. '):
304 self.desc += '. '
305 self.desc = (self.desc if self.desc else '') + ('Unit: ' + self.pmu + ' ')
306 if arch_std and arch_std.lower() in _arch_std_events:
307 event = _arch_std_events[arch_std.lower()].event
308 # Copy from the architecture standard event to self for undefined fields.
309 for attr, value in _arch_std_events[arch_std.lower()].__dict__.items():
310 if hasattr(self, attr) and not getattr(self, attr):
311 setattr(self, attr, value)
312
313 self.event = real_event(self.name, event)
314
315 def __repr__(self) -> str:
316 """String representation primarily for debugging."""
317 s = '{\n'
318 for attr, value in self.__dict__.items():
319 if value:
320 s += f'\t{attr} = {value},\n'
321 return s + '}'
322
323 def build_c_string(self) -> str:
324 s = ''
325 for attr in _json_event_attributes:
326 x = getattr(self, attr)
327 if x and attr == 'metric_expr':
328 # Convert parsed metric expressions into a string. Slashes
329 # must be doubled in the file.
330 x = x.ToPerfJson().replace('\\', '\\\\')
331 s += f'{x}\\000' if x else '\\000'
332 return s
333
334 def to_c_string(self) -> str:
335 """Representation of the event as a C struct initializer."""
336
337 s = self.build_c_string()
338 return f'{{ { _bcs.offsets[s] } }}, /* {s} */\n'
339
340
341def read_json_events(path: str, topic: str) -> Sequence[JsonEvent]:
342 """Read json events from the specified file."""
343
344 try:
345 result = json.load(open(path), object_hook=JsonEvent)
346 except BaseException as err:
347 print(f"Exception processing {path}")
348 raise
349 for event in result:
350 event.topic = topic
351 return result
352
353def preprocess_arch_std_files(archpath: str) -> None:
354 """Read in all architecture standard events."""
355 global _arch_std_events
356 for item in os.scandir(archpath):
357 if item.is_file() and item.name.endswith('.json'):
358 for event in read_json_events(item.path, topic=''):
359 if event.name:
360 _arch_std_events[event.name.lower()] = event
361
362
363def print_events_table_prefix(tblname: str) -> None:
364 """Called when a new events table is started."""
365 global _close_table
366 if _close_table:
367 raise IOError('Printing table prefix but last table has no suffix')
368 _args.output_file.write(f'static const struct compact_pmu_event {tblname}[] = {{\n')
369 _close_table = True
370
371
372def add_events_table_entries(item: os.DirEntry, topic: str) -> None:
373 """Add contents of file to _pending_events table."""
374 if not _close_table:
375 raise IOError('Table entries missing prefix')
376 for e in read_json_events(item.path, topic):
377 _pending_events.append(e)
378
379
380def print_events_table_suffix() -> None:
381 """Optionally close events table."""
382
383 def event_cmp_key(j: JsonEvent) -> Tuple[bool, str, str, str, str]:
384 def fix_none(s: Optional[str]) -> str:
385 if s is None:
386 return ''
387 return s
388
389 return (j.desc is not None, fix_none(j.topic), fix_none(j.name), fix_none(j.pmu),
390 fix_none(j.metric_name))
391
392 global _close_table
393 if not _close_table:
394 return
395
396 global _pending_events
397 for event in sorted(_pending_events, key=event_cmp_key):
398 _args.output_file.write(event.to_c_string())
399 _pending_events = []
400
401 _args.output_file.write('};\n\n')
402 _close_table = False
403
404def get_topic(topic: str) -> str:
405 if topic.endswith('metrics.json'):
406 return 'metrics'
407 return removesuffix(topic, '.json').replace('-', ' ')
408
409def preprocess_one_file(parents: Sequence[str], item: os.DirEntry) -> None:
410
411 if item.is_dir():
412 return
413
414 # base dir or too deep
415 level = len(parents)
416 if level == 0 or level > 4:
417 return
418
419 # Ignore other directories. If the file name does not have a .json
420 # extension, ignore it. It could be a readme.txt for instance.
421 if not item.is_file() or not item.name.endswith('.json'):
422 return
423
424 topic = get_topic(item.name)
425 for event in read_json_events(item.path, topic):
426 _bcs.add(event.build_c_string())
427
428def process_one_file(parents: Sequence[str], item: os.DirEntry) -> None:
429 """Process a JSON file during the main walk."""
430 global _sys_event_tables
431
432 def is_leaf_dir(path: str) -> bool:
433 for item in os.scandir(path):
434 if item.is_dir():
435 return False
436 return True
437
438 # model directory, reset topic
439 if item.is_dir() and is_leaf_dir(item.path):
440 print_events_table_suffix()
441
442 tblname = file_name_to_table_name(parents, item.name)
443 if item.name == 'sys':
444 _sys_event_tables.append(tblname)
445 print_events_table_prefix(tblname)
446 return
447
448 # base dir or too deep
449 level = len(parents)
450 if level == 0 or level > 4:
451 return
452
453 # Ignore other directories. If the file name does not have a .json
454 # extension, ignore it. It could be a readme.txt for instance.
455 if not item.is_file() or not item.name.endswith('.json'):
456 return
457
458 add_events_table_entries(item, get_topic(item.name))
459
460
461def print_mapping_table(archs: Sequence[str]) -> None:
462 """Read the mapfile and generate the struct from cpuid string to event table."""
463 _args.output_file.write("""
464/* Struct used to make the PMU event table implementation opaque to callers. */
465struct pmu_events_table {
466 const struct compact_pmu_event *entries;
467 size_t length;
468};
469
470/*
471 * Map a CPU to its table of PMU events. The CPU is identified by the
472 * cpuid field, which is an arch-specific identifier for the CPU.
473 * The identifier specified in tools/perf/pmu-events/arch/xxx/mapfile
474 * must match the get_cpuid_str() in tools/perf/arch/xxx/util/header.c)
475 *
476 * The cpuid can contain any character other than the comma.
477 */
478struct pmu_events_map {
479 const char *arch;
480 const char *cpuid;
481 struct pmu_events_table table;
482};
483
484/*
485 * Global table mapping each known CPU for the architecture to its
486 * table of PMU events.
487 */
488const struct pmu_events_map pmu_events_map[] = {
489""")
490 for arch in archs:
491 if arch == 'test':
492 _args.output_file.write("""{
493\t.arch = "testarch",
494\t.cpuid = "testcpu",
495\t.table = {
496\t.entries = pme_test_soc_cpu,
497\t.length = ARRAY_SIZE(pme_test_soc_cpu),
498\t}
499},
500""")
501 else:
502 with open(f'{_args.starting_dir}/{arch}/mapfile.csv') as csvfile:
503 table = csv.reader(csvfile)
504 first = True
505 for row in table:
506 # Skip the first row or any row beginning with #.
507 if not first and len(row) > 0 and not row[0].startswith('#'):
508 tblname = file_name_to_table_name([], row[2].replace('/', '_'))
509 cpuid = row[0].replace('\\', '\\\\')
510 _args.output_file.write(f"""{{
511\t.arch = "{arch}",
512\t.cpuid = "{cpuid}",
513\t.table = {{
514\t\t.entries = {tblname},
515\t\t.length = ARRAY_SIZE({tblname})
516\t}}
517}},
518""")
519 first = False
520
521 _args.output_file.write("""{
522\t.arch = 0,
523\t.cpuid = 0,
524\t.table = { 0, 0 },
525}
526};
527""")
528
529
530def print_system_mapping_table() -> None:
531 """C struct mapping table array for tables from /sys directories."""
532 _args.output_file.write("""
533struct pmu_sys_events {
534\tconst char *name;
535\tstruct pmu_events_table table;
536};
537
538static const struct pmu_sys_events pmu_sys_event_tables[] = {
539""")
540 for tblname in _sys_event_tables:
541 _args.output_file.write(f"""\t{{
542\t\t.table = {{
543\t\t\t.entries = {tblname},
544\t\t\t.length = ARRAY_SIZE({tblname})
545\t\t}},
546\t\t.name = \"{tblname}\",
547\t}},
548""")
549 _args.output_file.write("""\t{
550\t\t.table = { 0, 0 }
551\t},
552};
553
554static void decompress(int offset, struct pmu_event *pe)
555{
556\tconst char *p = &big_c_string[offset];
557""")
558 for attr in _json_event_attributes:
559 _args.output_file.write(f"""
560\tpe->{attr} = (*p == '\\0' ? NULL : p);
561""")
562 if attr == _json_event_attributes[-1]:
563 continue
564 _args.output_file.write('\twhile (*p++);')
565 _args.output_file.write("""}
566
567int pmu_events_table_for_each_event(const struct pmu_events_table *table,
568 pmu_event_iter_fn fn,
569 void *data)
570{
571 for (size_t i = 0; i < table->length; i++) {
572 struct pmu_event pe;
573 int ret;
574
575 decompress(table->entries[i].offset, &pe);
576 ret = fn(&pe, table, data);
577 if (ret)
578 return ret;
579 }
580 return 0;
581}
582
583const struct pmu_events_table *perf_pmu__find_table(struct perf_pmu *pmu)
584{
585 const struct pmu_events_table *table = NULL;
586 char *cpuid = perf_pmu__getcpuid(pmu);
587 int i;
588
589 /* on some platforms which uses cpus map, cpuid can be NULL for
590 * PMUs other than CORE PMUs.
591 */
592 if (!cpuid)
593 return NULL;
594
595 i = 0;
596 for (;;) {
597 const struct pmu_events_map *map = &pmu_events_map[i++];
598 if (!map->arch)
599 break;
600
601 if (!strcmp_cpuid_str(map->cpuid, cpuid)) {
602 table = &map->table;
603 break;
604 }
605 }
606 free(cpuid);
607 return table;
608}
609
610const struct pmu_events_table *find_core_events_table(const char *arch, const char *cpuid)
611{
612 for (const struct pmu_events_map *tables = &pmu_events_map[0];
613 tables->arch;
614 tables++) {
615 if (!strcmp(tables->arch, arch) && !strcmp_cpuid_str(tables->cpuid, cpuid))
616 return &tables->table;
617 }
618 return NULL;
619}
620
621int pmu_for_each_core_event(pmu_event_iter_fn fn, void *data)
622{
623 for (const struct pmu_events_map *tables = &pmu_events_map[0];
624 tables->arch;
625 tables++) {
626 int ret = pmu_events_table_for_each_event(&tables->table, fn, data);
627
628 if (ret)
629 return ret;
630 }
631 return 0;
632}
633
634const struct pmu_events_table *find_sys_events_table(const char *name)
635{
636 for (const struct pmu_sys_events *tables = &pmu_sys_event_tables[0];
637 tables->name;
638 tables++) {
639 if (!strcmp(tables->name, name))
640 return &tables->table;
641 }
642 return NULL;
643}
644
645int pmu_for_each_sys_event(pmu_event_iter_fn fn, void *data)
646{
647 for (const struct pmu_sys_events *tables = &pmu_sys_event_tables[0];
648 tables->name;
649 tables++) {
650 int ret = pmu_events_table_for_each_event(&tables->table, fn, data);
651
652 if (ret)
653 return ret;
654 }
655 return 0;
656}
657""")
658
659
660def main() -> None:
661 global _args
662
663 def dir_path(path: str) -> str:
664 """Validate path is a directory for argparse."""
665 if os.path.isdir(path):
666 return path
667 raise argparse.ArgumentTypeError(f'\'{path}\' is not a valid directory')
668
669 def ftw(path: str, parents: Sequence[str],
670 action: Callable[[Sequence[str], os.DirEntry], None]) -> None:
671 """Replicate the directory/file walking behavior of C's file tree walk."""
672 for item in os.scandir(path):
673 action(parents, item)
674 if item.is_dir():
675 ftw(item.path, parents + [item.name], action)
676
677 ap = argparse.ArgumentParser()
678 ap.add_argument('arch', help='Architecture name like x86')
679 ap.add_argument(
680 'starting_dir',
681 type=dir_path,
682 help='Root of tree containing architecture directories containing json files'
683 )
684 ap.add_argument(
685 'output_file', type=argparse.FileType('w', encoding='utf-8'), nargs='?', default=sys.stdout)
686 _args = ap.parse_args()
687
688 _args.output_file.write("""
689#include "pmu-events/pmu-events.h"
690#include "util/header.h"
691#include "util/pmu.h"
692#include <string.h>
693#include <stddef.h>
694
695struct compact_pmu_event {
696 int offset;
697};
698
699""")
700 archs = []
701 for item in os.scandir(_args.starting_dir):
702 if not item.is_dir():
703 continue
704 if item.name == _args.arch or _args.arch == 'all' or item.name == 'test':
705 archs.append(item.name)
706
707 if len(archs) < 2:
708 raise IOError(f'Missing architecture directory \'{_args.arch}\'')
709
710 archs.sort()
711 for arch in archs:
712 arch_path = f'{_args.starting_dir}/{arch}'
713 preprocess_arch_std_files(arch_path)
714 ftw(arch_path, [], preprocess_one_file)
715
716 _bcs.compute()
717 _args.output_file.write('static const char *const big_c_string =\n')
718 for s in _bcs.big_string:
719 _args.output_file.write(s)
720 _args.output_file.write(';\n\n')
721 for arch in archs:
722 arch_path = f'{_args.starting_dir}/{arch}'
723 ftw(arch_path, [], process_one_file)
724 print_events_table_suffix()
725
726 print_mapping_table(archs)
727 print_system_mapping_table()
728
729
730if __name__ == '__main__':
731 main()
1#!/usr/bin/env python3
2# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
3"""Convert directories of JSON events to C code."""
4import argparse
5import csv
6from functools import lru_cache
7import json
8import metric
9import os
10import sys
11from typing import (Callable, Dict, Optional, Sequence, Set, Tuple)
12import collections
13
14# Global command line arguments.
15_args = None
16# List of regular event tables.
17_event_tables = []
18# List of event tables generated from "/sys" directories.
19_sys_event_tables = []
20# List of regular metric tables.
21_metric_tables = []
22# List of metric tables generated from "/sys" directories.
23_sys_metric_tables = []
24# Mapping between sys event table names and sys metric table names.
25_sys_event_table_to_metric_table_mapping = {}
26# Map from an event name to an architecture standard
27# JsonEvent. Architecture standard events are in json files in the top
28# f'{_args.starting_dir}/{_args.arch}' directory.
29_arch_std_events = {}
30# Events to write out when the table is closed
31_pending_events = []
32# Name of events table to be written out
33_pending_events_tblname = None
34# Metrics to write out when the table is closed
35_pending_metrics = []
36# Name of metrics table to be written out
37_pending_metrics_tblname = None
38# Global BigCString shared by all structures.
39_bcs = None
40# Map from the name of a metric group to a description of the group.
41_metricgroups = {}
42# Order specific JsonEvent attributes will be visited.
43_json_event_attributes = [
44 # cmp_sevent related attributes.
45 'name', 'topic', 'desc',
46 # Seems useful, put it early.
47 'event',
48 # Short things in alphabetical order.
49 'compat', 'deprecated', 'perpkg', 'unit',
50 # Longer things (the last won't be iterated over during decompress).
51 'long_desc'
52]
53
54# Attributes that are in pmu_metric rather than pmu_event.
55_json_metric_attributes = [
56 'metric_name', 'metric_group', 'metric_expr', 'metric_threshold',
57 'desc', 'long_desc', 'unit', 'compat', 'metricgroup_no_group',
58 'default_metricgroup_name', 'aggr_mode', 'event_grouping'
59]
60# Attributes that are bools or enum int values, encoded as '0', '1',...
61_json_enum_attributes = ['aggr_mode', 'deprecated', 'event_grouping', 'perpkg']
62
63def removesuffix(s: str, suffix: str) -> str:
64 """Remove the suffix from a string
65
66 The removesuffix function is added to str in Python 3.9. We aim for 3.6
67 compatibility and so provide our own function here.
68 """
69 return s[0:-len(suffix)] if s.endswith(suffix) else s
70
71
72def file_name_to_table_name(prefix: str, parents: Sequence[str],
73 dirname: str) -> str:
74 """Generate a C table name from directory names."""
75 tblname = prefix
76 for p in parents:
77 tblname += '_' + p
78 tblname += '_' + dirname
79 return tblname.replace('-', '_')
80
81
82def c_len(s: str) -> int:
83 """Return the length of s a C string
84
85 This doesn't handle all escape characters properly. It first assumes
86 all \\ are for escaping, it then adjusts as it will have over counted
87 \\. The code uses \000 rather than \0 as a terminator as an adjacent
88 number would be folded into a string of \0 (ie. "\0" + "5" doesn't
89 equal a terminator followed by the number 5 but the escape of
90 \05). The code adjusts for \000 but not properly for all octal, hex
91 or unicode values.
92 """
93 try:
94 utf = s.encode(encoding='utf-8',errors='strict')
95 except:
96 print(f'broken string {s}')
97 raise
98 return len(utf) - utf.count(b'\\') + utf.count(b'\\\\') - (utf.count(b'\\000') * 2)
99
100class BigCString:
101 """A class to hold many strings concatenated together.
102
103 Generating a large number of stand-alone C strings creates a large
104 number of relocations in position independent code. The BigCString
105 is a helper for this case. It builds a single string which within it
106 are all the other C strings (to avoid memory issues the string
107 itself is held as a list of strings). The offsets within the big
108 string are recorded and when stored to disk these don't need
109 relocation. To reduce the size of the string further, identical
110 strings are merged. If a longer string ends-with the same value as a
111 shorter string, these entries are also merged.
112 """
113 strings: Set[str]
114 big_string: Sequence[str]
115 offsets: Dict[str, int]
116 insert_number: int
117 insert_point: Dict[str, int]
118 metrics: Set[str]
119
120 def __init__(self):
121 self.strings = set()
122 self.insert_number = 0;
123 self.insert_point = {}
124 self.metrics = set()
125
126 def add(self, s: str, metric: bool) -> None:
127 """Called to add to the big string."""
128 if s not in self.strings:
129 self.strings.add(s)
130 self.insert_point[s] = self.insert_number
131 self.insert_number += 1
132 if metric:
133 self.metrics.add(s)
134
135 def compute(self) -> None:
136 """Called once all strings are added to compute the string and offsets."""
137
138 folded_strings = {}
139 # Determine if two strings can be folded, ie. let 1 string use the
140 # end of another. First reverse all strings and sort them.
141 sorted_reversed_strings = sorted([x[::-1] for x in self.strings])
142
143 # Strings 'xyz' and 'yz' will now be [ 'zy', 'zyx' ]. Scan forward
144 # for each string to see if there is a better candidate to fold it
145 # into, in the example rather than using 'yz' we can use'xyz' at
146 # an offset of 1. We record which string can be folded into which
147 # in folded_strings, we don't need to record the offset as it is
148 # trivially computed from the string lengths.
149 for pos,s in enumerate(sorted_reversed_strings):
150 best_pos = pos
151 for check_pos in range(pos + 1, len(sorted_reversed_strings)):
152 if sorted_reversed_strings[check_pos].startswith(s):
153 best_pos = check_pos
154 else:
155 break
156 if pos != best_pos:
157 folded_strings[s[::-1]] = sorted_reversed_strings[best_pos][::-1]
158
159 # Compute reverse mappings for debugging.
160 fold_into_strings = collections.defaultdict(set)
161 for key, val in folded_strings.items():
162 if key != val:
163 fold_into_strings[val].add(key)
164
165 # big_string_offset is the current location within the C string
166 # being appended to - comments, etc. don't count. big_string is
167 # the string contents represented as a list. Strings are immutable
168 # in Python and so appending to one causes memory issues, while
169 # lists are mutable.
170 big_string_offset = 0
171 self.big_string = []
172 self.offsets = {}
173
174 def string_cmp_key(s: str) -> Tuple[bool, int, str]:
175 return (s in self.metrics, self.insert_point[s], s)
176
177 # Emit all strings that aren't folded in a sorted manner.
178 for s in sorted(self.strings, key=string_cmp_key):
179 if s not in folded_strings:
180 self.offsets[s] = big_string_offset
181 self.big_string.append(f'/* offset={big_string_offset} */ "')
182 self.big_string.append(s)
183 self.big_string.append('"')
184 if s in fold_into_strings:
185 self.big_string.append(' /* also: ' + ', '.join(fold_into_strings[s]) + ' */')
186 self.big_string.append('\n')
187 big_string_offset += c_len(s)
188 continue
189
190 # Compute the offsets of the folded strings.
191 for s in folded_strings.keys():
192 assert s not in self.offsets
193 folded_s = folded_strings[s]
194 self.offsets[s] = self.offsets[folded_s] + c_len(folded_s) - c_len(s)
195
196_bcs = BigCString()
197
198class JsonEvent:
199 """Representation of an event loaded from a json file dictionary."""
200
201 def __init__(self, jd: dict):
202 """Constructor passed the dictionary of parsed json values."""
203
204 def llx(x: int) -> str:
205 """Convert an int to a string similar to a printf modifier of %#llx."""
206 return '0' if x == 0 else hex(x)
207
208 def fixdesc(s: str) -> str:
209 """Fix formatting issue for the desc string."""
210 if s is None:
211 return None
212 return removesuffix(removesuffix(removesuffix(s, '. '),
213 '. '), '.').replace('\n', '\\n').replace(
214 '\"', '\\"').replace('\r', '\\r')
215
216 def convert_aggr_mode(aggr_mode: str) -> Optional[str]:
217 """Returns the aggr_mode_class enum value associated with the JSON string."""
218 if not aggr_mode:
219 return None
220 aggr_mode_to_enum = {
221 'PerChip': '1',
222 'PerCore': '2',
223 }
224 return aggr_mode_to_enum[aggr_mode]
225
226 def convert_metric_constraint(metric_constraint: str) -> Optional[str]:
227 """Returns the metric_event_groups enum value associated with the JSON string."""
228 if not metric_constraint:
229 return None
230 metric_constraint_to_enum = {
231 'NO_GROUP_EVENTS': '1',
232 'NO_GROUP_EVENTS_NMI': '2',
233 'NO_NMI_WATCHDOG': '2',
234 'NO_GROUP_EVENTS_SMT': '3',
235 }
236 return metric_constraint_to_enum[metric_constraint]
237
238 def lookup_msr(num: str) -> Optional[str]:
239 """Converts the msr number, or first in a list to the appropriate event field."""
240 if not num:
241 return None
242 msrmap = {
243 0x3F6: 'ldlat=',
244 0x1A6: 'offcore_rsp=',
245 0x1A7: 'offcore_rsp=',
246 0x3F7: 'frontend=',
247 }
248 return msrmap[int(num.split(',', 1)[0], 0)]
249
250 def real_event(name: str, event: str) -> Optional[str]:
251 """Convert well known event names to an event string otherwise use the event argument."""
252 fixed = {
253 'inst_retired.any': 'event=0xc0,period=2000003',
254 'inst_retired.any_p': 'event=0xc0,period=2000003',
255 'cpu_clk_unhalted.ref': 'event=0x0,umask=0x03,period=2000003',
256 'cpu_clk_unhalted.thread': 'event=0x3c,period=2000003',
257 'cpu_clk_unhalted.core': 'event=0x3c,period=2000003',
258 'cpu_clk_unhalted.thread_any': 'event=0x3c,any=1,period=2000003',
259 }
260 if not name:
261 return None
262 if name.lower() in fixed:
263 return fixed[name.lower()]
264 return event
265
266 def unit_to_pmu(unit: str) -> Optional[str]:
267 """Convert a JSON Unit to Linux PMU name."""
268 if not unit:
269 return 'default_core'
270 # Comment brought over from jevents.c:
271 # it's not realistic to keep adding these, we need something more scalable ...
272 table = {
273 'CBO': 'uncore_cbox',
274 'QPI LL': 'uncore_qpi',
275 'SBO': 'uncore_sbox',
276 'iMPH-U': 'uncore_arb',
277 'CPU-M-CF': 'cpum_cf',
278 'CPU-M-SF': 'cpum_sf',
279 'PAI-CRYPTO' : 'pai_crypto',
280 'PAI-EXT' : 'pai_ext',
281 'UPI LL': 'uncore_upi',
282 'hisi_sicl,cpa': 'hisi_sicl,cpa',
283 'hisi_sccl,ddrc': 'hisi_sccl,ddrc',
284 'hisi_sccl,hha': 'hisi_sccl,hha',
285 'hisi_sccl,l3c': 'hisi_sccl,l3c',
286 'imx8_ddr': 'imx8_ddr',
287 'L3PMC': 'amd_l3',
288 'DFPMC': 'amd_df',
289 'UMCPMC': 'amd_umc',
290 'cpu_core': 'cpu_core',
291 'cpu_atom': 'cpu_atom',
292 'ali_drw': 'ali_drw',
293 'arm_cmn': 'arm_cmn',
294 }
295 return table[unit] if unit in table else f'uncore_{unit.lower()}'
296
297 eventcode = 0
298 if 'EventCode' in jd:
299 eventcode = int(jd['EventCode'].split(',', 1)[0], 0)
300 if 'ExtSel' in jd:
301 eventcode |= int(jd['ExtSel']) << 8
302 configcode = int(jd['ConfigCode'], 0) if 'ConfigCode' in jd else None
303 eventidcode = int(jd['EventidCode'], 0) if 'EventidCode' in jd else None
304 self.name = jd['EventName'].lower() if 'EventName' in jd else None
305 self.topic = ''
306 self.compat = jd.get('Compat')
307 self.desc = fixdesc(jd.get('BriefDescription'))
308 self.long_desc = fixdesc(jd.get('PublicDescription'))
309 precise = jd.get('PEBS')
310 msr = lookup_msr(jd.get('MSRIndex'))
311 msrval = jd.get('MSRValue')
312 extra_desc = ''
313 if 'Data_LA' in jd:
314 extra_desc += ' Supports address when precise'
315 if 'Errata' in jd:
316 extra_desc += '.'
317 if 'Errata' in jd:
318 extra_desc += ' Spec update: ' + jd['Errata']
319 self.pmu = unit_to_pmu(jd.get('Unit'))
320 filter = jd.get('Filter')
321 self.unit = jd.get('ScaleUnit')
322 self.perpkg = jd.get('PerPkg')
323 self.aggr_mode = convert_aggr_mode(jd.get('AggregationMode'))
324 self.deprecated = jd.get('Deprecated')
325 self.metric_name = jd.get('MetricName')
326 self.metric_group = jd.get('MetricGroup')
327 self.metricgroup_no_group = jd.get('MetricgroupNoGroup')
328 self.default_metricgroup_name = jd.get('DefaultMetricgroupName')
329 self.event_grouping = convert_metric_constraint(jd.get('MetricConstraint'))
330 self.metric_expr = None
331 if 'MetricExpr' in jd:
332 self.metric_expr = metric.ParsePerfJson(jd['MetricExpr']).Simplify()
333 # Note, the metric formula for the threshold isn't parsed as the &
334 # and > have incorrect precedence.
335 self.metric_threshold = jd.get('MetricThreshold')
336
337 arch_std = jd.get('ArchStdEvent')
338 if precise and self.desc and '(Precise Event)' not in self.desc:
339 extra_desc += ' (Must be precise)' if precise == '2' else (' (Precise '
340 'event)')
341 event = None
342 if configcode is not None:
343 event = f'config={llx(configcode)}'
344 elif eventidcode is not None:
345 event = f'eventid={llx(eventidcode)}'
346 else:
347 event = f'event={llx(eventcode)}'
348 event_fields = [
349 ('AnyThread', 'any='),
350 ('PortMask', 'ch_mask='),
351 ('CounterMask', 'cmask='),
352 ('EdgeDetect', 'edge='),
353 ('FCMask', 'fc_mask='),
354 ('Invert', 'inv='),
355 ('SampleAfterValue', 'period='),
356 ('UMask', 'umask='),
357 ('NodeType', 'type='),
358 ('RdWrMask', 'rdwrmask='),
359 ]
360 for key, value in event_fields:
361 if key in jd and jd[key] != '0':
362 event += ',' + value + jd[key]
363 if filter:
364 event += f',{filter}'
365 if msr:
366 event += f',{msr}{msrval}'
367 if self.desc and extra_desc:
368 self.desc += extra_desc
369 if self.long_desc and extra_desc:
370 self.long_desc += extra_desc
371 if arch_std:
372 if arch_std.lower() in _arch_std_events:
373 event = _arch_std_events[arch_std.lower()].event
374 # Copy from the architecture standard event to self for undefined fields.
375 for attr, value in _arch_std_events[arch_std.lower()].__dict__.items():
376 if hasattr(self, attr) and not getattr(self, attr):
377 setattr(self, attr, value)
378 else:
379 raise argparse.ArgumentTypeError('Cannot find arch std event:', arch_std)
380
381 self.event = real_event(self.name, event)
382
383 def __repr__(self) -> str:
384 """String representation primarily for debugging."""
385 s = '{\n'
386 for attr, value in self.__dict__.items():
387 if value:
388 s += f'\t{attr} = {value},\n'
389 return s + '}'
390
391 def build_c_string(self, metric: bool) -> str:
392 s = ''
393 for attr in _json_metric_attributes if metric else _json_event_attributes:
394 x = getattr(self, attr)
395 if metric and x and attr == 'metric_expr':
396 # Convert parsed metric expressions into a string. Slashes
397 # must be doubled in the file.
398 x = x.ToPerfJson().replace('\\', '\\\\')
399 if metric and x and attr == 'metric_threshold':
400 x = x.replace('\\', '\\\\')
401 if attr in _json_enum_attributes:
402 s += x if x else '0'
403 else:
404 s += f'{x}\\000' if x else '\\000'
405 return s
406
407 def to_c_string(self, metric: bool) -> str:
408 """Representation of the event as a C struct initializer."""
409
410 s = self.build_c_string(metric)
411 return f'{{ { _bcs.offsets[s] } }}, /* {s} */\n'
412
413
414@lru_cache(maxsize=None)
415def read_json_events(path: str, topic: str) -> Sequence[JsonEvent]:
416 """Read json events from the specified file."""
417 try:
418 events = json.load(open(path), object_hook=JsonEvent)
419 except BaseException as err:
420 print(f"Exception processing {path}")
421 raise
422 metrics: list[Tuple[str, str, metric.Expression]] = []
423 for event in events:
424 event.topic = topic
425 if event.metric_name and '-' not in event.metric_name:
426 metrics.append((event.pmu, event.metric_name, event.metric_expr))
427 updates = metric.RewriteMetricsInTermsOfOthers(metrics)
428 if updates:
429 for event in events:
430 if event.metric_name in updates:
431 # print(f'Updated {event.metric_name} from\n"{event.metric_expr}"\n'
432 # f'to\n"{updates[event.metric_name]}"')
433 event.metric_expr = updates[event.metric_name]
434
435 return events
436
437def preprocess_arch_std_files(archpath: str) -> None:
438 """Read in all architecture standard events."""
439 global _arch_std_events
440 for item in os.scandir(archpath):
441 if item.is_file() and item.name.endswith('.json'):
442 for event in read_json_events(item.path, topic=''):
443 if event.name:
444 _arch_std_events[event.name.lower()] = event
445 if event.metric_name:
446 _arch_std_events[event.metric_name.lower()] = event
447
448
449def add_events_table_entries(item: os.DirEntry, topic: str) -> None:
450 """Add contents of file to _pending_events table."""
451 for e in read_json_events(item.path, topic):
452 if e.name:
453 _pending_events.append(e)
454 if e.metric_name:
455 _pending_metrics.append(e)
456
457
458def print_pending_events() -> None:
459 """Optionally close events table."""
460
461 def event_cmp_key(j: JsonEvent) -> Tuple[str, str, bool, str, str]:
462 def fix_none(s: Optional[str]) -> str:
463 if s is None:
464 return ''
465 return s
466
467 return (fix_none(j.pmu).replace(',','_'), fix_none(j.name), j.desc is not None, fix_none(j.topic),
468 fix_none(j.metric_name))
469
470 global _pending_events
471 if not _pending_events:
472 return
473
474 global _pending_events_tblname
475 if _pending_events_tblname.endswith('_sys'):
476 global _sys_event_tables
477 _sys_event_tables.append(_pending_events_tblname)
478 else:
479 global event_tables
480 _event_tables.append(_pending_events_tblname)
481
482 first = True
483 last_pmu = None
484 pmus = set()
485 for event in sorted(_pending_events, key=event_cmp_key):
486 if event.pmu != last_pmu:
487 if not first:
488 _args.output_file.write('};\n')
489 pmu_name = event.pmu.replace(',', '_')
490 _args.output_file.write(
491 f'static const struct compact_pmu_event {_pending_events_tblname}_{pmu_name}[] = {{\n')
492 first = False
493 last_pmu = event.pmu
494 pmus.add((event.pmu, pmu_name))
495
496 _args.output_file.write(event.to_c_string(metric=False))
497 _pending_events = []
498
499 _args.output_file.write(f"""
500}};
501
502const struct pmu_table_entry {_pending_events_tblname}[] = {{
503""")
504 for (pmu, tbl_pmu) in sorted(pmus):
505 pmu_name = f"{pmu}\\000"
506 _args.output_file.write(f"""{{
507 .entries = {_pending_events_tblname}_{tbl_pmu},
508 .num_entries = ARRAY_SIZE({_pending_events_tblname}_{tbl_pmu}),
509 .pmu_name = {{ {_bcs.offsets[pmu_name]} /* {pmu_name} */ }},
510}},
511""")
512 _args.output_file.write('};\n\n')
513
514def print_pending_metrics() -> None:
515 """Optionally close metrics table."""
516
517 def metric_cmp_key(j: JsonEvent) -> Tuple[bool, str, str]:
518 def fix_none(s: Optional[str]) -> str:
519 if s is None:
520 return ''
521 return s
522
523 return (j.desc is not None, fix_none(j.pmu), fix_none(j.metric_name))
524
525 global _pending_metrics
526 if not _pending_metrics:
527 return
528
529 global _pending_metrics_tblname
530 if _pending_metrics_tblname.endswith('_sys'):
531 global _sys_metric_tables
532 _sys_metric_tables.append(_pending_metrics_tblname)
533 else:
534 global metric_tables
535 _metric_tables.append(_pending_metrics_tblname)
536
537 first = True
538 last_pmu = None
539 pmus = set()
540 for metric in sorted(_pending_metrics, key=metric_cmp_key):
541 if metric.pmu != last_pmu:
542 if not first:
543 _args.output_file.write('};\n')
544 pmu_name = metric.pmu.replace(',', '_')
545 _args.output_file.write(
546 f'static const struct compact_pmu_event {_pending_metrics_tblname}_{pmu_name}[] = {{\n')
547 first = False
548 last_pmu = metric.pmu
549 pmus.add((metric.pmu, pmu_name))
550
551 _args.output_file.write(metric.to_c_string(metric=True))
552 _pending_metrics = []
553
554 _args.output_file.write(f"""
555}};
556
557const struct pmu_table_entry {_pending_metrics_tblname}[] = {{
558""")
559 for (pmu, tbl_pmu) in sorted(pmus):
560 pmu_name = f"{pmu}\\000"
561 _args.output_file.write(f"""{{
562 .entries = {_pending_metrics_tblname}_{tbl_pmu},
563 .num_entries = ARRAY_SIZE({_pending_metrics_tblname}_{tbl_pmu}),
564 .pmu_name = {{ {_bcs.offsets[pmu_name]} /* {pmu_name} */ }},
565}},
566""")
567 _args.output_file.write('};\n\n')
568
569def get_topic(topic: str) -> str:
570 if topic.endswith('metrics.json'):
571 return 'metrics'
572 return removesuffix(topic, '.json').replace('-', ' ')
573
574def preprocess_one_file(parents: Sequence[str], item: os.DirEntry) -> None:
575
576 if item.is_dir():
577 return
578
579 # base dir or too deep
580 level = len(parents)
581 if level == 0 or level > 4:
582 return
583
584 # Ignore other directories. If the file name does not have a .json
585 # extension, ignore it. It could be a readme.txt for instance.
586 if not item.is_file() or not item.name.endswith('.json'):
587 return
588
589 if item.name == 'metricgroups.json':
590 metricgroup_descriptions = json.load(open(item.path))
591 for mgroup in metricgroup_descriptions:
592 assert len(mgroup) > 1, parents
593 description = f"{metricgroup_descriptions[mgroup]}\\000"
594 mgroup = f"{mgroup}\\000"
595 _bcs.add(mgroup, metric=True)
596 _bcs.add(description, metric=True)
597 _metricgroups[mgroup] = description
598 return
599
600 topic = get_topic(item.name)
601 for event in read_json_events(item.path, topic):
602 pmu_name = f"{event.pmu}\\000"
603 if event.name:
604 _bcs.add(pmu_name, metric=False)
605 _bcs.add(event.build_c_string(metric=False), metric=False)
606 if event.metric_name:
607 _bcs.add(pmu_name, metric=True)
608 _bcs.add(event.build_c_string(metric=True), metric=True)
609
610def process_one_file(parents: Sequence[str], item: os.DirEntry) -> None:
611 """Process a JSON file during the main walk."""
612 def is_leaf_dir(path: str) -> bool:
613 for item in os.scandir(path):
614 if item.is_dir():
615 return False
616 return True
617
618 # model directory, reset topic
619 if item.is_dir() and is_leaf_dir(item.path):
620 print_pending_events()
621 print_pending_metrics()
622
623 global _pending_events_tblname
624 _pending_events_tblname = file_name_to_table_name('pmu_events_', parents, item.name)
625 global _pending_metrics_tblname
626 _pending_metrics_tblname = file_name_to_table_name('pmu_metrics_', parents, item.name)
627
628 if item.name == 'sys':
629 _sys_event_table_to_metric_table_mapping[_pending_events_tblname] = _pending_metrics_tblname
630 return
631
632 # base dir or too deep
633 level = len(parents)
634 if level == 0 or level > 4:
635 return
636
637 # Ignore other directories. If the file name does not have a .json
638 # extension, ignore it. It could be a readme.txt for instance.
639 if not item.is_file() or not item.name.endswith('.json') or item.name == 'metricgroups.json':
640 return
641
642 add_events_table_entries(item, get_topic(item.name))
643
644
645def print_mapping_table(archs: Sequence[str]) -> None:
646 """Read the mapfile and generate the struct from cpuid string to event table."""
647 _args.output_file.write("""
648/* Struct used to make the PMU event table implementation opaque to callers. */
649struct pmu_events_table {
650 const struct pmu_table_entry *pmus;
651 uint32_t num_pmus;
652};
653
654/* Struct used to make the PMU metric table implementation opaque to callers. */
655struct pmu_metrics_table {
656 const struct pmu_table_entry *pmus;
657 uint32_t num_pmus;
658};
659
660/*
661 * Map a CPU to its table of PMU events. The CPU is identified by the
662 * cpuid field, which is an arch-specific identifier for the CPU.
663 * The identifier specified in tools/perf/pmu-events/arch/xxx/mapfile
664 * must match the get_cpuid_str() in tools/perf/arch/xxx/util/header.c)
665 *
666 * The cpuid can contain any character other than the comma.
667 */
668struct pmu_events_map {
669 const char *arch;
670 const char *cpuid;
671 struct pmu_events_table event_table;
672 struct pmu_metrics_table metric_table;
673};
674
675/*
676 * Global table mapping each known CPU for the architecture to its
677 * table of PMU events.
678 */
679const struct pmu_events_map pmu_events_map[] = {
680""")
681 for arch in archs:
682 if arch == 'test':
683 _args.output_file.write("""{
684\t.arch = "testarch",
685\t.cpuid = "testcpu",
686\t.event_table = {
687\t\t.pmus = pmu_events__test_soc_cpu,
688\t\t.num_pmus = ARRAY_SIZE(pmu_events__test_soc_cpu),
689\t},
690\t.metric_table = {
691\t\t.pmus = pmu_metrics__test_soc_cpu,
692\t\t.num_pmus = ARRAY_SIZE(pmu_metrics__test_soc_cpu),
693\t}
694},
695""")
696 else:
697 with open(f'{_args.starting_dir}/{arch}/mapfile.csv') as csvfile:
698 table = csv.reader(csvfile)
699 first = True
700 for row in table:
701 # Skip the first row or any row beginning with #.
702 if not first and len(row) > 0 and not row[0].startswith('#'):
703 event_tblname = file_name_to_table_name('pmu_events_', [], row[2].replace('/', '_'))
704 if event_tblname in _event_tables:
705 event_size = f'ARRAY_SIZE({event_tblname})'
706 else:
707 event_tblname = 'NULL'
708 event_size = '0'
709 metric_tblname = file_name_to_table_name('pmu_metrics_', [], row[2].replace('/', '_'))
710 if metric_tblname in _metric_tables:
711 metric_size = f'ARRAY_SIZE({metric_tblname})'
712 else:
713 metric_tblname = 'NULL'
714 metric_size = '0'
715 if event_size == '0' and metric_size == '0':
716 continue
717 cpuid = row[0].replace('\\', '\\\\')
718 _args.output_file.write(f"""{{
719\t.arch = "{arch}",
720\t.cpuid = "{cpuid}",
721\t.event_table = {{
722\t\t.pmus = {event_tblname},
723\t\t.num_pmus = {event_size}
724\t}},
725\t.metric_table = {{
726\t\t.pmus = {metric_tblname},
727\t\t.num_pmus = {metric_size}
728\t}}
729}},
730""")
731 first = False
732
733 _args.output_file.write("""{
734\t.arch = 0,
735\t.cpuid = 0,
736\t.event_table = { 0, 0 },
737\t.metric_table = { 0, 0 },
738}
739};
740""")
741
742
743def print_system_mapping_table() -> None:
744 """C struct mapping table array for tables from /sys directories."""
745 _args.output_file.write("""
746struct pmu_sys_events {
747\tconst char *name;
748\tstruct pmu_events_table event_table;
749\tstruct pmu_metrics_table metric_table;
750};
751
752static const struct pmu_sys_events pmu_sys_event_tables[] = {
753""")
754 printed_metric_tables = []
755 for tblname in _sys_event_tables:
756 _args.output_file.write(f"""\t{{
757\t\t.event_table = {{
758\t\t\t.pmus = {tblname},
759\t\t\t.num_pmus = ARRAY_SIZE({tblname})
760\t\t}},""")
761 metric_tblname = _sys_event_table_to_metric_table_mapping[tblname]
762 if metric_tblname in _sys_metric_tables:
763 _args.output_file.write(f"""
764\t\t.metric_table = {{
765\t\t\t.pmus = {metric_tblname},
766\t\t\t.num_pmus = ARRAY_SIZE({metric_tblname})
767\t\t}},""")
768 printed_metric_tables.append(metric_tblname)
769 _args.output_file.write(f"""
770\t\t.name = \"{tblname}\",
771\t}},
772""")
773 for tblname in _sys_metric_tables:
774 if tblname in printed_metric_tables:
775 continue
776 _args.output_file.write(f"""\t{{
777\t\t.metric_table = {{
778\t\t\t.pmus = {tblname},
779\t\t\t.num_pmus = ARRAY_SIZE({tblname})
780\t\t}},
781\t\t.name = \"{tblname}\",
782\t}},
783""")
784 _args.output_file.write("""\t{
785\t\t.event_table = { 0, 0 },
786\t\t.metric_table = { 0, 0 },
787\t},
788};
789
790static void decompress_event(int offset, struct pmu_event *pe)
791{
792\tconst char *p = &big_c_string[offset];
793""")
794 for attr in _json_event_attributes:
795 _args.output_file.write(f'\n\tpe->{attr} = ')
796 if attr in _json_enum_attributes:
797 _args.output_file.write("*p - '0';\n")
798 else:
799 _args.output_file.write("(*p == '\\0' ? NULL : p);\n")
800 if attr == _json_event_attributes[-1]:
801 continue
802 if attr in _json_enum_attributes:
803 _args.output_file.write('\tp++;')
804 else:
805 _args.output_file.write('\twhile (*p++);')
806 _args.output_file.write("""}
807
808static void decompress_metric(int offset, struct pmu_metric *pm)
809{
810\tconst char *p = &big_c_string[offset];
811""")
812 for attr in _json_metric_attributes:
813 _args.output_file.write(f'\n\tpm->{attr} = ')
814 if attr in _json_enum_attributes:
815 _args.output_file.write("*p - '0';\n")
816 else:
817 _args.output_file.write("(*p == '\\0' ? NULL : p);\n")
818 if attr == _json_metric_attributes[-1]:
819 continue
820 if attr in _json_enum_attributes:
821 _args.output_file.write('\tp++;')
822 else:
823 _args.output_file.write('\twhile (*p++);')
824 _args.output_file.write("""}
825
826static int pmu_events_table__for_each_event_pmu(const struct pmu_events_table *table,
827 const struct pmu_table_entry *pmu,
828 pmu_event_iter_fn fn,
829 void *data)
830{
831 int ret;
832 struct pmu_event pe = {
833 .pmu = &big_c_string[pmu->pmu_name.offset],
834 };
835
836 for (uint32_t i = 0; i < pmu->num_entries; i++) {
837 decompress_event(pmu->entries[i].offset, &pe);
838 if (!pe.name)
839 continue;
840 ret = fn(&pe, table, data);
841 if (ret)
842 return ret;
843 }
844 return 0;
845 }
846
847static int pmu_events_table__find_event_pmu(const struct pmu_events_table *table,
848 const struct pmu_table_entry *pmu,
849 const char *name,
850 pmu_event_iter_fn fn,
851 void *data)
852{
853 struct pmu_event pe = {
854 .pmu = &big_c_string[pmu->pmu_name.offset],
855 };
856 int low = 0, high = pmu->num_entries - 1;
857
858 while (low <= high) {
859 int cmp, mid = (low + high) / 2;
860
861 decompress_event(pmu->entries[mid].offset, &pe);
862
863 if (!pe.name && !name)
864 goto do_call;
865
866 if (!pe.name && name) {
867 low = mid + 1;
868 continue;
869 }
870 if (pe.name && !name) {
871 high = mid - 1;
872 continue;
873 }
874
875 cmp = strcasecmp(pe.name, name);
876 if (cmp < 0) {
877 low = mid + 1;
878 continue;
879 }
880 if (cmp > 0) {
881 high = mid - 1;
882 continue;
883 }
884 do_call:
885 return fn ? fn(&pe, table, data) : 0;
886 }
887 return -1000;
888}
889
890int pmu_events_table__for_each_event(const struct pmu_events_table *table,
891 struct perf_pmu *pmu,
892 pmu_event_iter_fn fn,
893 void *data)
894{
895 for (size_t i = 0; i < table->num_pmus; i++) {
896 const struct pmu_table_entry *table_pmu = &table->pmus[i];
897 const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset];
898 int ret;
899
900 if (pmu && !pmu__name_match(pmu, pmu_name))
901 continue;
902
903 ret = pmu_events_table__for_each_event_pmu(table, table_pmu, fn, data);
904 if (pmu || ret)
905 return ret;
906 }
907 return 0;
908}
909
910int pmu_events_table__find_event(const struct pmu_events_table *table,
911 struct perf_pmu *pmu,
912 const char *name,
913 pmu_event_iter_fn fn,
914 void *data)
915{
916 for (size_t i = 0; i < table->num_pmus; i++) {
917 const struct pmu_table_entry *table_pmu = &table->pmus[i];
918 const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset];
919 int ret;
920
921 if (!pmu__name_match(pmu, pmu_name))
922 continue;
923
924 ret = pmu_events_table__find_event_pmu(table, table_pmu, name, fn, data);
925 if (ret != -1000)
926 return ret;
927 }
928 return -1000;
929}
930
931size_t pmu_events_table__num_events(const struct pmu_events_table *table,
932 struct perf_pmu *pmu)
933{
934 size_t count = 0;
935
936 for (size_t i = 0; i < table->num_pmus; i++) {
937 const struct pmu_table_entry *table_pmu = &table->pmus[i];
938 const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset];
939
940 if (pmu__name_match(pmu, pmu_name))
941 count += table_pmu->num_entries;
942 }
943 return count;
944}
945
946static int pmu_metrics_table__for_each_metric_pmu(const struct pmu_metrics_table *table,
947 const struct pmu_table_entry *pmu,
948 pmu_metric_iter_fn fn,
949 void *data)
950{
951 int ret;
952 struct pmu_metric pm = {
953 .pmu = &big_c_string[pmu->pmu_name.offset],
954 };
955
956 for (uint32_t i = 0; i < pmu->num_entries; i++) {
957 decompress_metric(pmu->entries[i].offset, &pm);
958 if (!pm.metric_expr)
959 continue;
960 ret = fn(&pm, table, data);
961 if (ret)
962 return ret;
963 }
964 return 0;
965}
966
967int pmu_metrics_table__for_each_metric(const struct pmu_metrics_table *table,
968 pmu_metric_iter_fn fn,
969 void *data)
970{
971 for (size_t i = 0; i < table->num_pmus; i++) {
972 int ret = pmu_metrics_table__for_each_metric_pmu(table, &table->pmus[i],
973 fn, data);
974
975 if (ret)
976 return ret;
977 }
978 return 0;
979}
980
981static const struct pmu_events_map *map_for_pmu(struct perf_pmu *pmu)
982{
983 static struct {
984 const struct pmu_events_map *map;
985 struct perf_pmu *pmu;
986 } last_result;
987 static struct {
988 const struct pmu_events_map *map;
989 char *cpuid;
990 } last_map_search;
991 static bool has_last_result, has_last_map_search;
992 const struct pmu_events_map *map = NULL;
993 char *cpuid = NULL;
994 size_t i;
995
996 if (has_last_result && last_result.pmu == pmu)
997 return last_result.map;
998
999 cpuid = perf_pmu__getcpuid(pmu);
1000
1001 /*
1002 * On some platforms which uses cpus map, cpuid can be NULL for
1003 * PMUs other than CORE PMUs.
1004 */
1005 if (!cpuid)
1006 goto out_update_last_result;
1007
1008 if (has_last_map_search && !strcmp(last_map_search.cpuid, cpuid)) {
1009 map = last_map_search.map;
1010 free(cpuid);
1011 } else {
1012 i = 0;
1013 for (;;) {
1014 map = &pmu_events_map[i++];
1015
1016 if (!map->arch) {
1017 map = NULL;
1018 break;
1019 }
1020
1021 if (!strcmp_cpuid_str(map->cpuid, cpuid))
1022 break;
1023 }
1024 free(last_map_search.cpuid);
1025 last_map_search.cpuid = cpuid;
1026 last_map_search.map = map;
1027 has_last_map_search = true;
1028 }
1029out_update_last_result:
1030 last_result.pmu = pmu;
1031 last_result.map = map;
1032 has_last_result = true;
1033 return map;
1034}
1035
1036const struct pmu_events_table *perf_pmu__find_events_table(struct perf_pmu *pmu)
1037{
1038 const struct pmu_events_map *map = map_for_pmu(pmu);
1039
1040 if (!map)
1041 return NULL;
1042
1043 if (!pmu)
1044 return &map->event_table;
1045
1046 for (size_t i = 0; i < map->event_table.num_pmus; i++) {
1047 const struct pmu_table_entry *table_pmu = &map->event_table.pmus[i];
1048 const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset];
1049
1050 if (pmu__name_match(pmu, pmu_name))
1051 return &map->event_table;
1052 }
1053 return NULL;
1054}
1055
1056const struct pmu_metrics_table *perf_pmu__find_metrics_table(struct perf_pmu *pmu)
1057{
1058 const struct pmu_events_map *map = map_for_pmu(pmu);
1059
1060 if (!map)
1061 return NULL;
1062
1063 if (!pmu)
1064 return &map->metric_table;
1065
1066 for (size_t i = 0; i < map->metric_table.num_pmus; i++) {
1067 const struct pmu_table_entry *table_pmu = &map->metric_table.pmus[i];
1068 const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset];
1069
1070 if (pmu__name_match(pmu, pmu_name))
1071 return &map->metric_table;
1072 }
1073 return NULL;
1074}
1075
1076const struct pmu_events_table *find_core_events_table(const char *arch, const char *cpuid)
1077{
1078 for (const struct pmu_events_map *tables = &pmu_events_map[0];
1079 tables->arch;
1080 tables++) {
1081 if (!strcmp(tables->arch, arch) && !strcmp_cpuid_str(tables->cpuid, cpuid))
1082 return &tables->event_table;
1083 }
1084 return NULL;
1085}
1086
1087const struct pmu_metrics_table *find_core_metrics_table(const char *arch, const char *cpuid)
1088{
1089 for (const struct pmu_events_map *tables = &pmu_events_map[0];
1090 tables->arch;
1091 tables++) {
1092 if (!strcmp(tables->arch, arch) && !strcmp_cpuid_str(tables->cpuid, cpuid))
1093 return &tables->metric_table;
1094 }
1095 return NULL;
1096}
1097
1098int pmu_for_each_core_event(pmu_event_iter_fn fn, void *data)
1099{
1100 for (const struct pmu_events_map *tables = &pmu_events_map[0];
1101 tables->arch;
1102 tables++) {
1103 int ret = pmu_events_table__for_each_event(&tables->event_table,
1104 /*pmu=*/ NULL, fn, data);
1105
1106 if (ret)
1107 return ret;
1108 }
1109 return 0;
1110}
1111
1112int pmu_for_each_core_metric(pmu_metric_iter_fn fn, void *data)
1113{
1114 for (const struct pmu_events_map *tables = &pmu_events_map[0];
1115 tables->arch;
1116 tables++) {
1117 int ret = pmu_metrics_table__for_each_metric(&tables->metric_table, fn, data);
1118
1119 if (ret)
1120 return ret;
1121 }
1122 return 0;
1123}
1124
1125const struct pmu_events_table *find_sys_events_table(const char *name)
1126{
1127 for (const struct pmu_sys_events *tables = &pmu_sys_event_tables[0];
1128 tables->name;
1129 tables++) {
1130 if (!strcmp(tables->name, name))
1131 return &tables->event_table;
1132 }
1133 return NULL;
1134}
1135
1136int pmu_for_each_sys_event(pmu_event_iter_fn fn, void *data)
1137{
1138 for (const struct pmu_sys_events *tables = &pmu_sys_event_tables[0];
1139 tables->name;
1140 tables++) {
1141 int ret = pmu_events_table__for_each_event(&tables->event_table,
1142 /*pmu=*/ NULL, fn, data);
1143
1144 if (ret)
1145 return ret;
1146 }
1147 return 0;
1148}
1149
1150int pmu_for_each_sys_metric(pmu_metric_iter_fn fn, void *data)
1151{
1152 for (const struct pmu_sys_events *tables = &pmu_sys_event_tables[0];
1153 tables->name;
1154 tables++) {
1155 int ret = pmu_metrics_table__for_each_metric(&tables->metric_table, fn, data);
1156
1157 if (ret)
1158 return ret;
1159 }
1160 return 0;
1161}
1162""")
1163
1164def print_metricgroups() -> None:
1165 _args.output_file.write("""
1166static const int metricgroups[][2] = {
1167""")
1168 for mgroup in sorted(_metricgroups):
1169 description = _metricgroups[mgroup]
1170 _args.output_file.write(
1171 f'\t{{ {_bcs.offsets[mgroup]}, {_bcs.offsets[description]} }}, /* {mgroup} => {description} */\n'
1172 )
1173 _args.output_file.write("""
1174};
1175
1176const char *describe_metricgroup(const char *group)
1177{
1178 int low = 0, high = (int)ARRAY_SIZE(metricgroups) - 1;
1179
1180 while (low <= high) {
1181 int mid = (low + high) / 2;
1182 const char *mgroup = &big_c_string[metricgroups[mid][0]];
1183 int cmp = strcmp(mgroup, group);
1184
1185 if (cmp == 0) {
1186 return &big_c_string[metricgroups[mid][1]];
1187 } else if (cmp < 0) {
1188 low = mid + 1;
1189 } else {
1190 high = mid - 1;
1191 }
1192 }
1193 return NULL;
1194}
1195""")
1196
1197def main() -> None:
1198 global _args
1199
1200 def dir_path(path: str) -> str:
1201 """Validate path is a directory for argparse."""
1202 if os.path.isdir(path):
1203 return path
1204 raise argparse.ArgumentTypeError(f'\'{path}\' is not a valid directory')
1205
1206 def ftw(path: str, parents: Sequence[str],
1207 action: Callable[[Sequence[str], os.DirEntry], None]) -> None:
1208 """Replicate the directory/file walking behavior of C's file tree walk."""
1209 for item in sorted(os.scandir(path), key=lambda e: e.name):
1210 if _args.model != 'all' and item.is_dir():
1211 # Check if the model matches one in _args.model.
1212 if len(parents) == _args.model.split(',')[0].count('/'):
1213 # We're testing the correct directory.
1214 item_path = '/'.join(parents) + ('/' if len(parents) > 0 else '') + item.name
1215 if 'test' not in item_path and item_path not in _args.model.split(','):
1216 continue
1217 action(parents, item)
1218 if item.is_dir():
1219 ftw(item.path, parents + [item.name], action)
1220
1221 ap = argparse.ArgumentParser()
1222 ap.add_argument('arch', help='Architecture name like x86')
1223 ap.add_argument('model', help='''Select a model such as skylake to
1224reduce the code size. Normally set to "all". For architectures like
1225ARM64 with an implementor/model, the model must include the implementor
1226such as "arm/cortex-a34".''',
1227 default='all')
1228 ap.add_argument(
1229 'starting_dir',
1230 type=dir_path,
1231 help='Root of tree containing architecture directories containing json files'
1232 )
1233 ap.add_argument(
1234 'output_file', type=argparse.FileType('w', encoding='utf-8'), nargs='?', default=sys.stdout)
1235 _args = ap.parse_args()
1236
1237 _args.output_file.write("""
1238#include <pmu-events/pmu-events.h>
1239#include "util/header.h"
1240#include "util/pmu.h"
1241#include <string.h>
1242#include <stddef.h>
1243
1244struct compact_pmu_event {
1245 int offset;
1246};
1247
1248struct pmu_table_entry {
1249 const struct compact_pmu_event *entries;
1250 uint32_t num_entries;
1251 struct compact_pmu_event pmu_name;
1252};
1253
1254""")
1255 archs = []
1256 for item in os.scandir(_args.starting_dir):
1257 if not item.is_dir():
1258 continue
1259 if item.name == _args.arch or _args.arch == 'all' or item.name == 'test':
1260 archs.append(item.name)
1261
1262 if len(archs) < 2:
1263 raise IOError(f'Missing architecture directory \'{_args.arch}\'')
1264
1265 archs.sort()
1266 for arch in archs:
1267 arch_path = f'{_args.starting_dir}/{arch}'
1268 preprocess_arch_std_files(arch_path)
1269 ftw(arch_path, [], preprocess_one_file)
1270
1271 _bcs.compute()
1272 _args.output_file.write('static const char *const big_c_string =\n')
1273 for s in _bcs.big_string:
1274 _args.output_file.write(s)
1275 _args.output_file.write(';\n\n')
1276 for arch in archs:
1277 arch_path = f'{_args.starting_dir}/{arch}'
1278 ftw(arch_path, [], process_one_file)
1279 print_pending_events()
1280 print_pending_metrics()
1281
1282 print_mapping_table(archs)
1283 print_system_mapping_table()
1284 print_metricgroups()
1285
1286if __name__ == '__main__':
1287 main()