Loading...
1#! /usr/bin/python
2
3import os
4import sys
5import glob
6import optparse
7import tempfile
8import logging
9import shutil
10import ConfigParser
11
12class Fail(Exception):
13 def __init__(self, test, msg):
14 self.msg = msg
15 self.test = test
16 def getMsg(self):
17 return '\'%s\' - %s' % (self.test.path, self.msg)
18
19class Unsup(Exception):
20 def __init__(self, test):
21 self.test = test
22 def getMsg(self):
23 return '\'%s\'' % self.test.path
24
25class Event(dict):
26 terms = [
27 'cpu',
28 'flags',
29 'type',
30 'size',
31 'config',
32 'sample_period',
33 'sample_type',
34 'read_format',
35 'disabled',
36 'inherit',
37 'pinned',
38 'exclusive',
39 'exclude_user',
40 'exclude_kernel',
41 'exclude_hv',
42 'exclude_idle',
43 'mmap',
44 'comm',
45 'freq',
46 'inherit_stat',
47 'enable_on_exec',
48 'task',
49 'watermark',
50 'precise_ip',
51 'mmap_data',
52 'sample_id_all',
53 'exclude_host',
54 'exclude_guest',
55 'exclude_callchain_kernel',
56 'exclude_callchain_user',
57 'wakeup_events',
58 'bp_type',
59 'config1',
60 'config2',
61 'branch_sample_type',
62 'sample_regs_user',
63 'sample_stack_user',
64 ]
65
66 def add(self, data):
67 for key, val in data:
68 log.debug(" %s = %s" % (key, val))
69 self[key] = val
70
71 def __init__(self, name, data, base):
72 log.debug(" Event %s" % name);
73 self.name = name;
74 self.group = ''
75 self.add(base)
76 self.add(data)
77
78 def compare_data(self, a, b):
79 # Allow multiple values in assignment separated by '|'
80 a_list = a.split('|')
81 b_list = b.split('|')
82
83 for a_item in a_list:
84 for b_item in b_list:
85 if (a_item == b_item):
86 return True
87 elif (a_item == '*') or (b_item == '*'):
88 return True
89
90 return False
91
92 def equal(self, other):
93 for t in Event.terms:
94 log.debug(" [%s] %s %s" % (t, self[t], other[t]));
95 if not self.has_key(t) or not other.has_key(t):
96 return False
97 if not self.compare_data(self[t], other[t]):
98 return False
99 return True
100
101 def diff(self, other):
102 for t in Event.terms:
103 if not self.has_key(t) or not other.has_key(t):
104 continue
105 if not self.compare_data(self[t], other[t]):
106 log.warning("expected %s=%s, got %s" % (t, self[t], other[t]))
107
108
109# Test file description needs to have following sections:
110# [config]
111# - just single instance in file
112# - needs to specify:
113# 'command' - perf command name
114# 'args' - special command arguments
115# 'ret' - expected command return value (0 by default)
116#
117# [eventX:base]
118# - one or multiple instances in file
119# - expected values assignments
120class Test(object):
121 def __init__(self, path, options):
122 parser = ConfigParser.SafeConfigParser()
123 parser.read(path)
124
125 log.warning("running '%s'" % path)
126
127 self.path = path
128 self.test_dir = options.test_dir
129 self.perf = options.perf
130 self.command = parser.get('config', 'command')
131 self.args = parser.get('config', 'args')
132
133 try:
134 self.ret = parser.get('config', 'ret')
135 except:
136 self.ret = 0
137
138 self.expect = {}
139 self.result = {}
140 log.debug(" loading expected events");
141 self.load_events(path, self.expect)
142
143 def is_event(self, name):
144 if name.find("event") == -1:
145 return False
146 else:
147 return True
148
149 def load_events(self, path, events):
150 parser_event = ConfigParser.SafeConfigParser()
151 parser_event.read(path)
152
153 # The event record section header contains 'event' word,
154 # optionaly followed by ':' allowing to load 'parent
155 # event' first as a base
156 for section in filter(self.is_event, parser_event.sections()):
157
158 parser_items = parser_event.items(section);
159 base_items = {}
160
161 # Read parent event if there's any
162 if (':' in section):
163 base = section[section.index(':') + 1:]
164 parser_base = ConfigParser.SafeConfigParser()
165 parser_base.read(self.test_dir + '/' + base)
166 base_items = parser_base.items('event')
167
168 e = Event(section, parser_items, base_items)
169 events[section] = e
170
171 def run_cmd(self, tempdir):
172 cmd = "PERF_TEST_ATTR=%s %s %s -o %s/perf.data %s" % (tempdir,
173 self.perf, self.command, tempdir, self.args)
174 ret = os.WEXITSTATUS(os.system(cmd))
175
176 log.info(" '%s' ret %d " % (cmd, ret))
177
178 if ret != int(self.ret):
179 raise Unsup(self)
180
181 def compare(self, expect, result):
182 match = {}
183
184 log.debug(" compare");
185
186 # For each expected event find all matching
187 # events in result. Fail if there's not any.
188 for exp_name, exp_event in expect.items():
189 exp_list = []
190 log.debug(" matching [%s]" % exp_name)
191 for res_name, res_event in result.items():
192 log.debug(" to [%s]" % res_name)
193 if (exp_event.equal(res_event)):
194 exp_list.append(res_name)
195 log.debug(" ->OK")
196 else:
197 log.debug(" ->FAIL");
198
199 log.debug(" match: [%s] matches %s" % (exp_name, str(exp_list)))
200
201 # we did not any matching event - fail
202 if (not exp_list):
203 exp_event.diff(res_event)
204 raise Fail(self, 'match failure');
205
206 match[exp_name] = exp_list
207
208 # For each defined group in the expected events
209 # check we match the same group in the result.
210 for exp_name, exp_event in expect.items():
211 group = exp_event.group
212
213 if (group == ''):
214 continue
215
216 for res_name in match[exp_name]:
217 res_group = result[res_name].group
218 if res_group not in match[group]:
219 raise Fail(self, 'group failure')
220
221 log.debug(" group: [%s] matches group leader %s" %
222 (exp_name, str(match[group])))
223
224 log.debug(" matched")
225
226 def resolve_groups(self, events):
227 for name, event in events.items():
228 group_fd = event['group_fd'];
229 if group_fd == '-1':
230 continue;
231
232 for iname, ievent in events.items():
233 if (ievent['fd'] == group_fd):
234 event.group = iname
235 log.debug('[%s] has group leader [%s]' % (name, iname))
236 break;
237
238 def run(self):
239 tempdir = tempfile.mkdtemp();
240
241 try:
242 # run the test script
243 self.run_cmd(tempdir);
244
245 # load events expectation for the test
246 log.debug(" loading result events");
247 for f in glob.glob(tempdir + '/event*'):
248 self.load_events(f, self.result);
249
250 # resolve group_fd to event names
251 self.resolve_groups(self.expect);
252 self.resolve_groups(self.result);
253
254 # do the expectation - results matching - both ways
255 self.compare(self.expect, self.result)
256 self.compare(self.result, self.expect)
257
258 finally:
259 # cleanup
260 shutil.rmtree(tempdir)
261
262
263def run_tests(options):
264 for f in glob.glob(options.test_dir + '/' + options.test):
265 try:
266 Test(f, options).run()
267 except Unsup, obj:
268 log.warning("unsupp %s" % obj.getMsg())
269
270def setup_log(verbose):
271 global log
272 level = logging.CRITICAL
273
274 if verbose == 1:
275 level = logging.WARNING
276 if verbose == 2:
277 level = logging.INFO
278 if verbose >= 3:
279 level = logging.DEBUG
280
281 log = logging.getLogger('test')
282 log.setLevel(level)
283 ch = logging.StreamHandler()
284 ch.setLevel(level)
285 formatter = logging.Formatter('%(message)s')
286 ch.setFormatter(formatter)
287 log.addHandler(ch)
288
289USAGE = '''%s [OPTIONS]
290 -d dir # tests dir
291 -p path # perf binary
292 -t test # single test
293 -v # verbose level
294''' % sys.argv[0]
295
296def main():
297 parser = optparse.OptionParser(usage=USAGE)
298
299 parser.add_option("-t", "--test",
300 action="store", type="string", dest="test")
301 parser.add_option("-d", "--test-dir",
302 action="store", type="string", dest="test_dir")
303 parser.add_option("-p", "--perf",
304 action="store", type="string", dest="perf")
305 parser.add_option("-v", "--verbose",
306 action="count", dest="verbose")
307
308 options, args = parser.parse_args()
309 if args:
310 parser.error('FAILED wrong arguments %s' % ' '.join(args))
311 return -1
312
313 setup_log(options.verbose)
314
315 if not options.test_dir:
316 print 'FAILED no -d option specified'
317 sys.exit(-1)
318
319 if not options.test:
320 options.test = 'test*'
321
322 try:
323 run_tests(options)
324
325 except Fail, obj:
326 print "FAILED %s" % obj.getMsg();
327 sys.exit(-1)
328
329 sys.exit(0)
330
331if __name__ == '__main__':
332 main()
1#! /usr/bin/python
2# SPDX-License-Identifier: GPL-2.0
3
4import os
5import sys
6import glob
7import optparse
8import tempfile
9import logging
10import shutil
11import ConfigParser
12
13def data_equal(a, b):
14 # Allow multiple values in assignment separated by '|'
15 a_list = a.split('|')
16 b_list = b.split('|')
17
18 for a_item in a_list:
19 for b_item in b_list:
20 if (a_item == b_item):
21 return True
22 elif (a_item == '*') or (b_item == '*'):
23 return True
24
25 return False
26
27class Fail(Exception):
28 def __init__(self, test, msg):
29 self.msg = msg
30 self.test = test
31 def getMsg(self):
32 return '\'%s\' - %s' % (self.test.path, self.msg)
33
34class Notest(Exception):
35 def __init__(self, test, arch):
36 self.arch = arch
37 self.test = test
38 def getMsg(self):
39 return '[%s] \'%s\'' % (self.arch, self.test.path)
40
41class Unsup(Exception):
42 def __init__(self, test):
43 self.test = test
44 def getMsg(self):
45 return '\'%s\'' % self.test.path
46
47class Event(dict):
48 terms = [
49 'cpu',
50 'flags',
51 'type',
52 'size',
53 'config',
54 'sample_period',
55 'sample_type',
56 'read_format',
57 'disabled',
58 'inherit',
59 'pinned',
60 'exclusive',
61 'exclude_user',
62 'exclude_kernel',
63 'exclude_hv',
64 'exclude_idle',
65 'mmap',
66 'comm',
67 'freq',
68 'inherit_stat',
69 'enable_on_exec',
70 'task',
71 'watermark',
72 'precise_ip',
73 'mmap_data',
74 'sample_id_all',
75 'exclude_host',
76 'exclude_guest',
77 'exclude_callchain_kernel',
78 'exclude_callchain_user',
79 'wakeup_events',
80 'bp_type',
81 'config1',
82 'config2',
83 'branch_sample_type',
84 'sample_regs_user',
85 'sample_stack_user',
86 ]
87
88 def add(self, data):
89 for key, val in data:
90 log.debug(" %s = %s" % (key, val))
91 self[key] = val
92
93 def __init__(self, name, data, base):
94 log.debug(" Event %s" % name);
95 self.name = name;
96 self.group = ''
97 self.add(base)
98 self.add(data)
99
100 def equal(self, other):
101 for t in Event.terms:
102 log.debug(" [%s] %s %s" % (t, self[t], other[t]));
103 if not self.has_key(t) or not other.has_key(t):
104 return False
105 if not data_equal(self[t], other[t]):
106 return False
107 return True
108
109 def optional(self):
110 if self.has_key('optional') and self['optional'] == '1':
111 return True
112 return False
113
114 def diff(self, other):
115 for t in Event.terms:
116 if not self.has_key(t) or not other.has_key(t):
117 continue
118 if not data_equal(self[t], other[t]):
119 log.warning("expected %s=%s, got %s" % (t, self[t], other[t]))
120
121# Test file description needs to have following sections:
122# [config]
123# - just single instance in file
124# - needs to specify:
125# 'command' - perf command name
126# 'args' - special command arguments
127# 'ret' - expected command return value (0 by default)
128# 'arch' - architecture specific test (optional)
129# comma separated list, ! at the beginning
130# negates it.
131#
132# [eventX:base]
133# - one or multiple instances in file
134# - expected values assignments
135class Test(object):
136 def __init__(self, path, options):
137 parser = ConfigParser.SafeConfigParser()
138 parser.read(path)
139
140 log.warning("running '%s'" % path)
141
142 self.path = path
143 self.test_dir = options.test_dir
144 self.perf = options.perf
145 self.command = parser.get('config', 'command')
146 self.args = parser.get('config', 'args')
147
148 try:
149 self.ret = parser.get('config', 'ret')
150 except:
151 self.ret = 0
152
153 try:
154 self.arch = parser.get('config', 'arch')
155 log.warning("test limitation '%s'" % self.arch)
156 except:
157 self.arch = ''
158
159 self.expect = {}
160 self.result = {}
161 log.debug(" loading expected events");
162 self.load_events(path, self.expect)
163
164 def is_event(self, name):
165 if name.find("event") == -1:
166 return False
167 else:
168 return True
169
170 def skip_test(self, myarch):
171 # If architecture not set always run test
172 if self.arch == '':
173 # log.warning("test for arch %s is ok" % myarch)
174 return False
175
176 # Allow multiple values in assignment separated by ','
177 arch_list = self.arch.split(',')
178
179 # Handle negated list such as !s390x,ppc
180 if arch_list[0][0] == '!':
181 arch_list[0] = arch_list[0][1:]
182 log.warning("excluded architecture list %s" % arch_list)
183 for arch_item in arch_list:
184 # log.warning("test for %s arch is %s" % (arch_item, myarch))
185 if arch_item == myarch:
186 return True
187 return False
188
189 for arch_item in arch_list:
190 # log.warning("test for architecture '%s' current '%s'" % (arch_item, myarch))
191 if arch_item == myarch:
192 return False
193 return True
194
195 def load_events(self, path, events):
196 parser_event = ConfigParser.SafeConfigParser()
197 parser_event.read(path)
198
199 # The event record section header contains 'event' word,
200 # optionaly followed by ':' allowing to load 'parent
201 # event' first as a base
202 for section in filter(self.is_event, parser_event.sections()):
203
204 parser_items = parser_event.items(section);
205 base_items = {}
206
207 # Read parent event if there's any
208 if (':' in section):
209 base = section[section.index(':') + 1:]
210 parser_base = ConfigParser.SafeConfigParser()
211 parser_base.read(self.test_dir + '/' + base)
212 base_items = parser_base.items('event')
213
214 e = Event(section, parser_items, base_items)
215 events[section] = e
216
217 def run_cmd(self, tempdir):
218 junk1, junk2, junk3, junk4, myarch = (os.uname())
219
220 if self.skip_test(myarch):
221 raise Notest(self, myarch)
222
223 cmd = "PERF_TEST_ATTR=%s %s %s -o %s/perf.data %s" % (tempdir,
224 self.perf, self.command, tempdir, self.args)
225 ret = os.WEXITSTATUS(os.system(cmd))
226
227 log.info(" '%s' ret '%s', expected '%s'" % (cmd, str(ret), str(self.ret)))
228
229 if not data_equal(str(ret), str(self.ret)):
230 raise Unsup(self)
231
232 def compare(self, expect, result):
233 match = {}
234
235 log.debug(" compare");
236
237 # For each expected event find all matching
238 # events in result. Fail if there's not any.
239 for exp_name, exp_event in expect.items():
240 exp_list = []
241 res_event = {}
242 log.debug(" matching [%s]" % exp_name)
243 for res_name, res_event in result.items():
244 log.debug(" to [%s]" % res_name)
245 if (exp_event.equal(res_event)):
246 exp_list.append(res_name)
247 log.debug(" ->OK")
248 else:
249 log.debug(" ->FAIL");
250
251 log.debug(" match: [%s] matches %s" % (exp_name, str(exp_list)))
252
253 # we did not any matching event - fail
254 if not exp_list:
255 if exp_event.optional():
256 log.debug(" %s does not match, but is optional" % exp_name)
257 else:
258 if not res_event:
259 log.debug(" res_event is empty");
260 else:
261 exp_event.diff(res_event)
262 raise Fail(self, 'match failure');
263
264 match[exp_name] = exp_list
265
266 # For each defined group in the expected events
267 # check we match the same group in the result.
268 for exp_name, exp_event in expect.items():
269 group = exp_event.group
270
271 if (group == ''):
272 continue
273
274 for res_name in match[exp_name]:
275 res_group = result[res_name].group
276 if res_group not in match[group]:
277 raise Fail(self, 'group failure')
278
279 log.debug(" group: [%s] matches group leader %s" %
280 (exp_name, str(match[group])))
281
282 log.debug(" matched")
283
284 def resolve_groups(self, events):
285 for name, event in events.items():
286 group_fd = event['group_fd'];
287 if group_fd == '-1':
288 continue;
289
290 for iname, ievent in events.items():
291 if (ievent['fd'] == group_fd):
292 event.group = iname
293 log.debug('[%s] has group leader [%s]' % (name, iname))
294 break;
295
296 def run(self):
297 tempdir = tempfile.mkdtemp();
298
299 try:
300 # run the test script
301 self.run_cmd(tempdir);
302
303 # load events expectation for the test
304 log.debug(" loading result events");
305 for f in glob.glob(tempdir + '/event*'):
306 self.load_events(f, self.result);
307
308 # resolve group_fd to event names
309 self.resolve_groups(self.expect);
310 self.resolve_groups(self.result);
311
312 # do the expectation - results matching - both ways
313 self.compare(self.expect, self.result)
314 self.compare(self.result, self.expect)
315
316 finally:
317 # cleanup
318 shutil.rmtree(tempdir)
319
320
321def run_tests(options):
322 for f in glob.glob(options.test_dir + '/' + options.test):
323 try:
324 Test(f, options).run()
325 except Unsup, obj:
326 log.warning("unsupp %s" % obj.getMsg())
327 except Notest, obj:
328 log.warning("skipped %s" % obj.getMsg())
329
330def setup_log(verbose):
331 global log
332 level = logging.CRITICAL
333
334 if verbose == 1:
335 level = logging.WARNING
336 if verbose == 2:
337 level = logging.INFO
338 if verbose >= 3:
339 level = logging.DEBUG
340
341 log = logging.getLogger('test')
342 log.setLevel(level)
343 ch = logging.StreamHandler()
344 ch.setLevel(level)
345 formatter = logging.Formatter('%(message)s')
346 ch.setFormatter(formatter)
347 log.addHandler(ch)
348
349USAGE = '''%s [OPTIONS]
350 -d dir # tests dir
351 -p path # perf binary
352 -t test # single test
353 -v # verbose level
354''' % sys.argv[0]
355
356def main():
357 parser = optparse.OptionParser(usage=USAGE)
358
359 parser.add_option("-t", "--test",
360 action="store", type="string", dest="test")
361 parser.add_option("-d", "--test-dir",
362 action="store", type="string", dest="test_dir")
363 parser.add_option("-p", "--perf",
364 action="store", type="string", dest="perf")
365 parser.add_option("-v", "--verbose",
366 action="count", dest="verbose")
367
368 options, args = parser.parse_args()
369 if args:
370 parser.error('FAILED wrong arguments %s' % ' '.join(args))
371 return -1
372
373 setup_log(options.verbose)
374
375 if not options.test_dir:
376 print 'FAILED no -d option specified'
377 sys.exit(-1)
378
379 if not options.test:
380 options.test = 'test*'
381
382 try:
383 run_tests(options)
384
385 except Fail, obj:
386 print "FAILED %s" % obj.getMsg();
387 sys.exit(-1)
388
389 sys.exit(0)
390
391if __name__ == '__main__':
392 main()