Loading...
1# SPDX-License-Identifier: GPL-2.0
2
3from __future__ import print_function
4
5import os
6import sys
7import glob
8import optparse
9import tempfile
10import logging
11import shutil
12
13try:
14 import configparser
15except ImportError:
16 import ConfigParser as configparser
17
18def data_equal(a, b):
19 # Allow multiple values in assignment separated by '|'
20 a_list = a.split('|')
21 b_list = b.split('|')
22
23 for a_item in a_list:
24 for b_item in b_list:
25 if (a_item == b_item):
26 return True
27 elif (a_item == '*') or (b_item == '*'):
28 return True
29
30 return False
31
32class Fail(Exception):
33 def __init__(self, test, msg):
34 self.msg = msg
35 self.test = test
36 def getMsg(self):
37 return '\'%s\' - %s' % (self.test.path, self.msg)
38
39class Notest(Exception):
40 def __init__(self, test, arch):
41 self.arch = arch
42 self.test = test
43 def getMsg(self):
44 return '[%s] \'%s\'' % (self.arch, self.test.path)
45
46class Unsup(Exception):
47 def __init__(self, test):
48 self.test = test
49 def getMsg(self):
50 return '\'%s\'' % self.test.path
51
52class Event(dict):
53 terms = [
54 'cpu',
55 'flags',
56 'type',
57 'size',
58 'config',
59 'sample_period',
60 'sample_type',
61 'read_format',
62 'disabled',
63 'inherit',
64 'pinned',
65 'exclusive',
66 'exclude_user',
67 'exclude_kernel',
68 'exclude_hv',
69 'exclude_idle',
70 'mmap',
71 'comm',
72 'freq',
73 'inherit_stat',
74 'enable_on_exec',
75 'task',
76 'watermark',
77 'precise_ip',
78 'mmap_data',
79 'sample_id_all',
80 'exclude_host',
81 'exclude_guest',
82 'exclude_callchain_kernel',
83 'exclude_callchain_user',
84 'wakeup_events',
85 'bp_type',
86 'config1',
87 'config2',
88 'branch_sample_type',
89 'sample_regs_user',
90 'sample_stack_user',
91 ]
92
93 def add(self, data):
94 for key, val in data:
95 log.debug(" %s = %s" % (key, val))
96 self[key] = val
97
98 def __init__(self, name, data, base):
99 log.debug(" Event %s" % name);
100 self.name = name;
101 self.group = ''
102 self.add(base)
103 self.add(data)
104
105 def equal(self, other):
106 for t in Event.terms:
107 log.debug(" [%s] %s %s" % (t, self[t], other[t]));
108 if t not in self or t not in other:
109 return False
110 if not data_equal(self[t], other[t]):
111 return False
112 return True
113
114 def optional(self):
115 if 'optional' in self and self['optional'] == '1':
116 return True
117 return False
118
119 def diff(self, other):
120 for t in Event.terms:
121 if t not in self or t not in other:
122 continue
123 if not data_equal(self[t], other[t]):
124 log.warning("expected %s=%s, got %s" % (t, self[t], other[t]))
125
126# Test file description needs to have following sections:
127# [config]
128# - just single instance in file
129# - needs to specify:
130# 'command' - perf command name
131# 'args' - special command arguments
132# 'ret' - expected command return value (0 by default)
133# 'arch' - architecture specific test (optional)
134# comma separated list, ! at the beginning
135# negates it.
136#
137# [eventX:base]
138# - one or multiple instances in file
139# - expected values assignments
140class Test(object):
141 def __init__(self, path, options):
142 parser = configparser.SafeConfigParser()
143 parser.read(path)
144
145 log.warning("running '%s'" % path)
146
147 self.path = path
148 self.test_dir = options.test_dir
149 self.perf = options.perf
150 self.command = parser.get('config', 'command')
151 self.args = parser.get('config', 'args')
152
153 try:
154 self.ret = parser.get('config', 'ret')
155 except:
156 self.ret = 0
157
158 try:
159 self.arch = parser.get('config', 'arch')
160 log.warning("test limitation '%s'" % self.arch)
161 except:
162 self.arch = ''
163
164 self.expect = {}
165 self.result = {}
166 log.debug(" loading expected events");
167 self.load_events(path, self.expect)
168
169 def is_event(self, name):
170 if name.find("event") == -1:
171 return False
172 else:
173 return True
174
175 def skip_test(self, myarch):
176 # If architecture not set always run test
177 if self.arch == '':
178 # log.warning("test for arch %s is ok" % myarch)
179 return False
180
181 # Allow multiple values in assignment separated by ','
182 arch_list = self.arch.split(',')
183
184 # Handle negated list such as !s390x,ppc
185 if arch_list[0][0] == '!':
186 arch_list[0] = arch_list[0][1:]
187 log.warning("excluded architecture list %s" % arch_list)
188 for arch_item in arch_list:
189 # log.warning("test for %s arch is %s" % (arch_item, myarch))
190 if arch_item == myarch:
191 return True
192 return False
193
194 for arch_item in arch_list:
195 # log.warning("test for architecture '%s' current '%s'" % (arch_item, myarch))
196 if arch_item == myarch:
197 return False
198 return True
199
200 def load_events(self, path, events):
201 parser_event = configparser.SafeConfigParser()
202 parser_event.read(path)
203
204 # The event record section header contains 'event' word,
205 # optionaly followed by ':' allowing to load 'parent
206 # event' first as a base
207 for section in filter(self.is_event, parser_event.sections()):
208
209 parser_items = parser_event.items(section);
210 base_items = {}
211
212 # Read parent event if there's any
213 if (':' in section):
214 base = section[section.index(':') + 1:]
215 parser_base = configparser.SafeConfigParser()
216 parser_base.read(self.test_dir + '/' + base)
217 base_items = parser_base.items('event')
218
219 e = Event(section, parser_items, base_items)
220 events[section] = e
221
222 def run_cmd(self, tempdir):
223 junk1, junk2, junk3, junk4, myarch = (os.uname())
224
225 if self.skip_test(myarch):
226 raise Notest(self, myarch)
227
228 cmd = "PERF_TEST_ATTR=%s %s %s -o %s/perf.data %s" % (tempdir,
229 self.perf, self.command, tempdir, self.args)
230 ret = os.WEXITSTATUS(os.system(cmd))
231
232 log.info(" '%s' ret '%s', expected '%s'" % (cmd, str(ret), str(self.ret)))
233
234 if not data_equal(str(ret), str(self.ret)):
235 raise Unsup(self)
236
237 def compare(self, expect, result):
238 match = {}
239
240 log.debug(" compare");
241
242 # For each expected event find all matching
243 # events in result. Fail if there's not any.
244 for exp_name, exp_event in expect.items():
245 exp_list = []
246 res_event = {}
247 log.debug(" matching [%s]" % exp_name)
248 for res_name, res_event in result.items():
249 log.debug(" to [%s]" % res_name)
250 if (exp_event.equal(res_event)):
251 exp_list.append(res_name)
252 log.debug(" ->OK")
253 else:
254 log.debug(" ->FAIL");
255
256 log.debug(" match: [%s] matches %s" % (exp_name, str(exp_list)))
257
258 # we did not any matching event - fail
259 if not exp_list:
260 if exp_event.optional():
261 log.debug(" %s does not match, but is optional" % exp_name)
262 else:
263 if not res_event:
264 log.debug(" res_event is empty");
265 else:
266 exp_event.diff(res_event)
267 raise Fail(self, 'match failure');
268
269 match[exp_name] = exp_list
270
271 # For each defined group in the expected events
272 # check we match the same group in the result.
273 for exp_name, exp_event in expect.items():
274 group = exp_event.group
275
276 if (group == ''):
277 continue
278
279 for res_name in match[exp_name]:
280 res_group = result[res_name].group
281 if res_group not in match[group]:
282 raise Fail(self, 'group failure')
283
284 log.debug(" group: [%s] matches group leader %s" %
285 (exp_name, str(match[group])))
286
287 log.debug(" matched")
288
289 def resolve_groups(self, events):
290 for name, event in events.items():
291 group_fd = event['group_fd'];
292 if group_fd == '-1':
293 continue;
294
295 for iname, ievent in events.items():
296 if (ievent['fd'] == group_fd):
297 event.group = iname
298 log.debug('[%s] has group leader [%s]' % (name, iname))
299 break;
300
301 def run(self):
302 tempdir = tempfile.mkdtemp();
303
304 try:
305 # run the test script
306 self.run_cmd(tempdir);
307
308 # load events expectation for the test
309 log.debug(" loading result events");
310 for f in glob.glob(tempdir + '/event*'):
311 self.load_events(f, self.result);
312
313 # resolve group_fd to event names
314 self.resolve_groups(self.expect);
315 self.resolve_groups(self.result);
316
317 # do the expectation - results matching - both ways
318 self.compare(self.expect, self.result)
319 self.compare(self.result, self.expect)
320
321 finally:
322 # cleanup
323 shutil.rmtree(tempdir)
324
325
326def run_tests(options):
327 for f in glob.glob(options.test_dir + '/' + options.test):
328 try:
329 Test(f, options).run()
330 except Unsup as obj:
331 log.warning("unsupp %s" % obj.getMsg())
332 except Notest as obj:
333 log.warning("skipped %s" % obj.getMsg())
334
335def setup_log(verbose):
336 global log
337 level = logging.CRITICAL
338
339 if verbose == 1:
340 level = logging.WARNING
341 if verbose == 2:
342 level = logging.INFO
343 if verbose >= 3:
344 level = logging.DEBUG
345
346 log = logging.getLogger('test')
347 log.setLevel(level)
348 ch = logging.StreamHandler()
349 ch.setLevel(level)
350 formatter = logging.Formatter('%(message)s')
351 ch.setFormatter(formatter)
352 log.addHandler(ch)
353
354USAGE = '''%s [OPTIONS]
355 -d dir # tests dir
356 -p path # perf binary
357 -t test # single test
358 -v # verbose level
359''' % sys.argv[0]
360
361def main():
362 parser = optparse.OptionParser(usage=USAGE)
363
364 parser.add_option("-t", "--test",
365 action="store", type="string", dest="test")
366 parser.add_option("-d", "--test-dir",
367 action="store", type="string", dest="test_dir")
368 parser.add_option("-p", "--perf",
369 action="store", type="string", dest="perf")
370 parser.add_option("-v", "--verbose",
371 default=0, action="count", dest="verbose")
372
373 options, args = parser.parse_args()
374 if args:
375 parser.error('FAILED wrong arguments %s' % ' '.join(args))
376 return -1
377
378 setup_log(options.verbose)
379
380 if not options.test_dir:
381 print('FAILED no -d option specified')
382 sys.exit(-1)
383
384 if not options.test:
385 options.test = 'test*'
386
387 try:
388 run_tests(options)
389
390 except Fail as obj:
391 print("FAILED %s" % obj.getMsg())
392 sys.exit(-1)
393
394 sys.exit(0)
395
396if __name__ == '__main__':
397 main()
1# SPDX-License-Identifier: GPL-2.0
2
3from __future__ import print_function
4
5import os
6import sys
7import glob
8import optparse
9import platform
10import tempfile
11import logging
12import re
13import shutil
14import subprocess
15
16try:
17 import configparser
18except ImportError:
19 import ConfigParser as configparser
20
21def data_equal(a, b):
22 # Allow multiple values in assignment separated by '|'
23 a_list = a.split('|')
24 b_list = b.split('|')
25
26 for a_item in a_list:
27 for b_item in b_list:
28 if (a_item == b_item):
29 return True
30 elif (a_item == '*') or (b_item == '*'):
31 return True
32
33 return False
34
35class Fail(Exception):
36 def __init__(self, test, msg):
37 self.msg = msg
38 self.test = test
39 def getMsg(self):
40 return '\'%s\' - %s' % (self.test.path, self.msg)
41
42class Notest(Exception):
43 def __init__(self, test, arch):
44 self.arch = arch
45 self.test = test
46 def getMsg(self):
47 return '[%s] \'%s\'' % (self.arch, self.test.path)
48
49class Unsup(Exception):
50 def __init__(self, test):
51 self.test = test
52 def getMsg(self):
53 return '\'%s\'' % self.test.path
54
55class Event(dict):
56 terms = [
57 'cpu',
58 'flags',
59 'type',
60 'size',
61 'config',
62 'sample_period',
63 'sample_type',
64 'read_format',
65 'disabled',
66 'inherit',
67 'pinned',
68 'exclusive',
69 'exclude_user',
70 'exclude_kernel',
71 'exclude_hv',
72 'exclude_idle',
73 'mmap',
74 'comm',
75 'freq',
76 'inherit_stat',
77 'enable_on_exec',
78 'task',
79 'watermark',
80 'precise_ip',
81 'mmap_data',
82 'sample_id_all',
83 'exclude_host',
84 'exclude_guest',
85 'exclude_callchain_kernel',
86 'exclude_callchain_user',
87 'wakeup_events',
88 'bp_type',
89 'config1',
90 'config2',
91 'branch_sample_type',
92 'sample_regs_user',
93 'sample_stack_user',
94 ]
95
96 def add(self, data):
97 for key, val in data:
98 log.debug(" %s = %s" % (key, val))
99 self[key] = val
100
101 def __init__(self, name, data, base):
102 log.debug(" Event %s" % name);
103 self.name = name;
104 self.group = ''
105 self.add(base)
106 self.add(data)
107
108 def equal(self, other):
109 for t in Event.terms:
110 log.debug(" [%s] %s %s" % (t, self[t], other[t]));
111 if t not in self or t not in other:
112 return False
113 if not data_equal(self[t], other[t]):
114 return False
115 return True
116
117 def optional(self):
118 if 'optional' in self and self['optional'] == '1':
119 return True
120 return False
121
122 def diff(self, other):
123 for t in Event.terms:
124 if t not in self or t not in other:
125 continue
126 if not data_equal(self[t], other[t]):
127 log.warning("expected %s=%s, got %s" % (t, self[t], other[t]))
128
129def parse_version(version):
130 if not version:
131 return None
132 return [int(v) for v in version.split(".")[0:2]]
133
134# Test file description needs to have following sections:
135# [config]
136# - just single instance in file
137# - needs to specify:
138# 'command' - perf command name
139# 'args' - special command arguments
140# 'ret' - Skip test if Perf doesn't exit with this value (0 by default)
141# 'test_ret'- If set to 'true', fail test instead of skipping for 'ret' argument
142# 'arch' - architecture specific test (optional)
143# comma separated list, ! at the beginning
144# negates it.
145# 'auxv' - Truthy statement that is evaled in the scope of the auxv map. When false,
146# the test is skipped. For example 'auxv["AT_HWCAP"] == 10'. (optional)
147# 'kernel_since' - Inclusive kernel version from which the test will start running. Only the
148# first two values are supported, for example "6.1" (optional)
149# 'kernel_until' - Exclusive kernel version from which the test will stop running. (optional)
150# [eventX:base]
151# - one or multiple instances in file
152# - expected values assignments
153class Test(object):
154 def __init__(self, path, options):
155 parser = configparser.SafeConfigParser()
156 parser.read(path)
157
158 log.warning("running '%s'" % path)
159
160 self.path = path
161 self.test_dir = options.test_dir
162 self.perf = options.perf
163 self.command = parser.get('config', 'command')
164 self.args = parser.get('config', 'args')
165
166 try:
167 self.ret = parser.get('config', 'ret')
168 except:
169 self.ret = 0
170
171 self.test_ret = parser.getboolean('config', 'test_ret', fallback=False)
172
173 try:
174 self.arch = parser.get('config', 'arch')
175 log.warning("test limitation '%s'" % self.arch)
176 except:
177 self.arch = ''
178
179 self.auxv = parser.get('config', 'auxv', fallback=None)
180 self.kernel_since = parse_version(parser.get('config', 'kernel_since', fallback=None))
181 self.kernel_until = parse_version(parser.get('config', 'kernel_until', fallback=None))
182 self.expect = {}
183 self.result = {}
184 log.debug(" loading expected events");
185 self.load_events(path, self.expect)
186
187 def is_event(self, name):
188 if name.find("event") == -1:
189 return False
190 else:
191 return True
192
193 def skip_test_kernel_since(self):
194 if not self.kernel_since:
195 return False
196 return not self.kernel_since <= parse_version(platform.release())
197
198 def skip_test_kernel_until(self):
199 if not self.kernel_until:
200 return False
201 return not parse_version(platform.release()) < self.kernel_until
202
203 def skip_test_auxv(self):
204 def new_auxv(a, pattern):
205 items = list(filter(None, pattern.split(a)))
206 # AT_HWCAP is hex but doesn't have a prefix, so special case it
207 if items[0] == "AT_HWCAP":
208 value = int(items[-1], 16)
209 else:
210 try:
211 value = int(items[-1], 0)
212 except:
213 value = items[-1]
214 return (items[0], value)
215
216 if not self.auxv:
217 return False
218 auxv = subprocess.check_output("LD_SHOW_AUXV=1 sleep 0", shell=True) \
219 .decode(sys.stdout.encoding)
220 pattern = re.compile(r"[: ]+")
221 auxv = dict([new_auxv(a, pattern) for a in auxv.splitlines()])
222 return not eval(self.auxv)
223
224 def skip_test_arch(self, myarch):
225 # If architecture not set always run test
226 if self.arch == '':
227 # log.warning("test for arch %s is ok" % myarch)
228 return False
229
230 # Allow multiple values in assignment separated by ','
231 arch_list = self.arch.split(',')
232
233 # Handle negated list such as !s390x,ppc
234 if arch_list[0][0] == '!':
235 arch_list[0] = arch_list[0][1:]
236 log.warning("excluded architecture list %s" % arch_list)
237 for arch_item in arch_list:
238 # log.warning("test for %s arch is %s" % (arch_item, myarch))
239 if arch_item == myarch:
240 return True
241 return False
242
243 for arch_item in arch_list:
244 # log.warning("test for architecture '%s' current '%s'" % (arch_item, myarch))
245 if arch_item == myarch:
246 return False
247 return True
248
249 def load_events(self, path, events):
250 parser_event = configparser.SafeConfigParser()
251 parser_event.read(path)
252
253 # The event record section header contains 'event' word,
254 # optionaly followed by ':' allowing to load 'parent
255 # event' first as a base
256 for section in filter(self.is_event, parser_event.sections()):
257
258 parser_items = parser_event.items(section);
259 base_items = {}
260
261 # Read parent event if there's any
262 if (':' in section):
263 base = section[section.index(':') + 1:]
264 parser_base = configparser.SafeConfigParser()
265 parser_base.read(self.test_dir + '/' + base)
266 base_items = parser_base.items('event')
267
268 e = Event(section, parser_items, base_items)
269 events[section] = e
270
271 def run_cmd(self, tempdir):
272 junk1, junk2, junk3, junk4, myarch = (os.uname())
273
274 if self.skip_test_arch(myarch):
275 raise Notest(self, myarch)
276
277 if self.skip_test_auxv():
278 raise Notest(self, "auxv skip")
279
280 if self.skip_test_kernel_since():
281 raise Notest(self, "old kernel skip")
282
283 if self.skip_test_kernel_until():
284 raise Notest(self, "new kernel skip")
285
286 cmd = "PERF_TEST_ATTR=%s %s %s -o %s/perf.data %s" % (tempdir,
287 self.perf, self.command, tempdir, self.args)
288 ret = os.WEXITSTATUS(os.system(cmd))
289
290 log.info(" '%s' ret '%s', expected '%s'" % (cmd, str(ret), str(self.ret)))
291
292 if not data_equal(str(ret), str(self.ret)):
293 if self.test_ret:
294 raise Fail(self, "Perf exit code failure")
295 else:
296 raise Unsup(self)
297
298 def compare(self, expect, result):
299 match = {}
300
301 log.debug(" compare");
302
303 # For each expected event find all matching
304 # events in result. Fail if there's not any.
305 for exp_name, exp_event in expect.items():
306 exp_list = []
307 res_event = {}
308 log.debug(" matching [%s]" % exp_name)
309 for res_name, res_event in result.items():
310 log.debug(" to [%s]" % res_name)
311 if (exp_event.equal(res_event)):
312 exp_list.append(res_name)
313 log.debug(" ->OK")
314 else:
315 log.debug(" ->FAIL");
316
317 log.debug(" match: [%s] matches %s" % (exp_name, str(exp_list)))
318
319 # we did not any matching event - fail
320 if not exp_list:
321 if exp_event.optional():
322 log.debug(" %s does not match, but is optional" % exp_name)
323 else:
324 if not res_event:
325 log.debug(" res_event is empty");
326 else:
327 exp_event.diff(res_event)
328 raise Fail(self, 'match failure');
329
330 match[exp_name] = exp_list
331
332 # For each defined group in the expected events
333 # check we match the same group in the result.
334 for exp_name, exp_event in expect.items():
335 group = exp_event.group
336
337 if (group == ''):
338 continue
339
340 for res_name in match[exp_name]:
341 res_group = result[res_name].group
342 if res_group not in match[group]:
343 raise Fail(self, 'group failure')
344
345 log.debug(" group: [%s] matches group leader %s" %
346 (exp_name, str(match[group])))
347
348 log.debug(" matched")
349
350 def resolve_groups(self, events):
351 for name, event in events.items():
352 group_fd = event['group_fd'];
353 if group_fd == '-1':
354 continue;
355
356 for iname, ievent in events.items():
357 if (ievent['fd'] == group_fd):
358 event.group = iname
359 log.debug('[%s] has group leader [%s]' % (name, iname))
360 break;
361
362 def run(self):
363 tempdir = tempfile.mkdtemp();
364
365 try:
366 # run the test script
367 self.run_cmd(tempdir);
368
369 # load events expectation for the test
370 log.debug(" loading result events");
371 for f in glob.glob(tempdir + '/event*'):
372 self.load_events(f, self.result);
373
374 # resolve group_fd to event names
375 self.resolve_groups(self.expect);
376 self.resolve_groups(self.result);
377
378 # do the expectation - results matching - both ways
379 self.compare(self.expect, self.result)
380 self.compare(self.result, self.expect)
381
382 finally:
383 # cleanup
384 shutil.rmtree(tempdir)
385
386
387def run_tests(options):
388 for f in glob.glob(options.test_dir + '/' + options.test):
389 try:
390 Test(f, options).run()
391 except Unsup as obj:
392 log.warning("unsupp %s" % obj.getMsg())
393 except Notest as obj:
394 log.warning("skipped %s" % obj.getMsg())
395
396def setup_log(verbose):
397 global log
398 level = logging.CRITICAL
399
400 if verbose == 1:
401 level = logging.WARNING
402 if verbose == 2:
403 level = logging.INFO
404 if verbose >= 3:
405 level = logging.DEBUG
406
407 log = logging.getLogger('test')
408 log.setLevel(level)
409 ch = logging.StreamHandler()
410 ch.setLevel(level)
411 formatter = logging.Formatter('%(message)s')
412 ch.setFormatter(formatter)
413 log.addHandler(ch)
414
415USAGE = '''%s [OPTIONS]
416 -d dir # tests dir
417 -p path # perf binary
418 -t test # single test
419 -v # verbose level
420''' % sys.argv[0]
421
422def main():
423 parser = optparse.OptionParser(usage=USAGE)
424
425 parser.add_option("-t", "--test",
426 action="store", type="string", dest="test")
427 parser.add_option("-d", "--test-dir",
428 action="store", type="string", dest="test_dir")
429 parser.add_option("-p", "--perf",
430 action="store", type="string", dest="perf")
431 parser.add_option("-v", "--verbose",
432 default=0, action="count", dest="verbose")
433
434 options, args = parser.parse_args()
435 if args:
436 parser.error('FAILED wrong arguments %s' % ' '.join(args))
437 return -1
438
439 setup_log(options.verbose)
440
441 if not options.test_dir:
442 print('FAILED no -d option specified')
443 sys.exit(-1)
444
445 if not options.test:
446 options.test = 'test*'
447
448 try:
449 run_tests(options)
450
451 except Fail as obj:
452 print("FAILED %s" % obj.getMsg())
453 sys.exit(-1)
454
455 sys.exit(0)
456
457if __name__ == '__main__':
458 main()