1#!/usr/bin/env python3
   2# SPDX-License-Identifier: GPL-2.0
   3
   4"""
   5tdc.py - Linux tc (Traffic Control) unit test driver
   6
   7Copyright (C) 2017 Lucas Bates <lucasb@mojatatu.com>
   8"""
   9
  10import re
  11import os
  12import sys
  13import argparse
  14import importlib
  15import json
  16import subprocess
  17import time
  18import traceback
  19import random
  20from multiprocessing import Pool
  21from collections import OrderedDict
  22from string import Template
  23
  24from tdc_config import *
  25from tdc_helper import *
  26
  27import TdcPlugin
  28from TdcResults import *
  29
  30class PluginDependencyException(Exception):
  31    def __init__(self, missing_pg):
  32        self.missing_pg = missing_pg
  33
  34class PluginMgrTestFail(Exception):
  35    def __init__(self, stage, output, message):
  36        self.stage = stage
  37        self.output = output
  38        self.message = message
  39
  40class PluginMgr:
  41    def __init__(self, argparser):
  42        super().__init__()
  43        self.plugins = set()
  44        self.plugin_instances = []
  45        self.failed_plugins = {}
  46        self.argparser = argparser
  47
  48        plugindir = os.getenv('TDC_PLUGIN_DIR', './plugins')
  49        for dirpath, dirnames, filenames in os.walk(plugindir):
  50            for fn in filenames:
  51                if (fn.endswith('.py') and
  52                    not fn == '__init__.py' and
  53                    not fn.startswith('#') and
  54                    not fn.startswith('.#')):
  55                    mn = fn[0:-3]
  56                    foo = importlib.import_module('plugins.' + mn)
  57                    self.plugins.add(mn)
  58                    self.plugin_instances[mn] = foo.SubPlugin()
  59
  60    def load_plugin(self, pgdir, pgname):
  61        pgname = pgname[0:-3]
  62        self.plugins.add(pgname)
  63
  64        foo = importlib.import_module('{}.{}'.format(pgdir, pgname))
  65
  66        # nsPlugin must always be the first one
  67        if pgname == "nsPlugin":
  68            self.plugin_instances.insert(0, (pgname, foo.SubPlugin()))
  69            self.plugin_instances[0][1].check_args(self.args, None)
  70        else:
  71            self.plugin_instances.append((pgname, foo.SubPlugin()))
  72            self.plugin_instances[-1][1].check_args(self.args, None)
  73
  74    def get_required_plugins(self, testlist):
  75        '''
  76        Get all required plugins from the list of test cases and return
  77        all unique items.
  78        '''
  79        reqs = set()
  80        for t in testlist:
  81            try:
  82                if 'requires' in t['plugins']:
  83                    if isinstance(t['plugins']['requires'], list):
  84                        reqs.update(set(t['plugins']['requires']))
  85                    else:
  86                        reqs.add(t['plugins']['requires'])
  87                    t['plugins'] = t['plugins']['requires']
  88                else:
  89                    t['plugins'] = []
  90            except KeyError:
  91                t['plugins'] = []
  92                continue
  93
  94        return reqs
  95
  96    def load_required_plugins(self, reqs, parser, args, remaining):
  97        '''
  98        Get all required plugins from the list of test cases and load any plugin
  99        that is not already enabled.
 100        '''
 101        pgd = ['plugin-lib', 'plugin-lib-custom']
 102        pnf = []
 103
 104        for r in reqs:
 105            if r not in self.plugins:
 106                fname = '{}.py'.format(r)
 107                source_path = []
 108                for d in pgd:
 109                    pgpath = '{}/{}'.format(d, fname)
 110                    if os.path.isfile(pgpath):
 111                        source_path.append(pgpath)
 112                if len(source_path) == 0:
 113                    print('ERROR: unable to find required plugin {}'.format(r))
 114                    pnf.append(fname)
 115                    continue
 116                elif len(source_path) > 1:
 117                    print('WARNING: multiple copies of plugin {} found, using version found')
 118                    print('at {}'.format(source_path[0]))
 119                pgdir = source_path[0]
 120                pgdir = pgdir.split('/')[0]
 121                self.load_plugin(pgdir, fname)
 122        if len(pnf) > 0:
 123            raise PluginDependencyException(pnf)
 124
 125        parser = self.call_add_args(parser)
 126        (args, remaining) = parser.parse_known_args(args=remaining, namespace=args)
 127        return args
 128
 129    def call_pre_suite(self, testcount, testidlist):
 130        for (_, pgn_inst) in self.plugin_instances:
 131            pgn_inst.pre_suite(testcount, testidlist)
 132
 133    def call_post_suite(self, index):
 134        for (_, pgn_inst) in reversed(self.plugin_instances):
 135            pgn_inst.post_suite(index)
 136
 137    def call_pre_case(self, caseinfo, *, test_skip=False):
 138        for (pgn, pgn_inst) in self.plugin_instances:
 139            if pgn not in caseinfo['plugins']:
 140                continue
 141            try:
 142                pgn_inst.pre_case(caseinfo, test_skip)
 143            except Exception as ee:
 144                print('exception {} in call to pre_case for {} plugin'.
 145                      format(ee, pgn_inst.__class__))
 
 146                print('testid is {}'.format(caseinfo['id']))
 147                raise
 148
 149    def call_post_case(self, caseinfo):
 150        for (pgn, pgn_inst) in reversed(self.plugin_instances):
 151            if pgn not in caseinfo['plugins']:
 152                continue
 153            pgn_inst.post_case()
 154
 155    def call_pre_execute(self, caseinfo):
 156        for (pgn, pgn_inst) in self.plugin_instances:
 157            if pgn not in caseinfo['plugins']:
 158                continue
 159            pgn_inst.pre_execute()
 160
 161    def call_post_execute(self, caseinfo):
 162        for (pgn, pgn_inst) in reversed(self.plugin_instances):
 163            if pgn not in caseinfo['plugins']:
 164                continue
 165            pgn_inst.post_execute()
 166
 167    def call_add_args(self, parser):
 168        for (pgn, pgn_inst) in self.plugin_instances:
 169            parser = pgn_inst.add_args(parser)
 170        return parser
 171
 172    def call_check_args(self, args, remaining):
 173        for (pgn, pgn_inst) in self.plugin_instances:
 174            pgn_inst.check_args(args, remaining)
 175
 176    def call_adjust_command(self, caseinfo, stage, command):
 177        for (pgn, pgn_inst) in self.plugin_instances:
 178            if pgn not in caseinfo['plugins']:
 179                continue
 180            command = pgn_inst.adjust_command(stage, command)
 181        return command
 182
 183    def set_args(self, args):
 184        self.args = args
 185
 186    @staticmethod
 187    def _make_argparser(args):
 188        self.argparser = argparse.ArgumentParser(
 189            description='Linux TC unit tests')
 190
 191def replace_keywords(cmd):
 192    """
 193    For a given executable command, substitute any known
 194    variables contained within NAMES with the correct values
 195    """
 196    tcmd = Template(cmd)
 197    subcmd = tcmd.safe_substitute(NAMES)
 198    return subcmd
 199
 200
 201def exec_cmd(caseinfo, args, pm, stage, command):
 202    """
 203    Perform any required modifications on an executable command, then run
 204    it in a subprocess and return the results.
 205    """
 206    if len(command.strip()) == 0:
 207        return None, None
 208    if '$' in command:
 209        command = replace_keywords(command)
 210
 211    command = pm.call_adjust_command(caseinfo, stage, command)
 212    if args.verbose > 0:
 213        print('command "{}"'.format(command))
 214
 215    proc = subprocess.Popen(command,
 216        shell=True,
 217        stdout=subprocess.PIPE,
 218        stderr=subprocess.PIPE,
 219        env=ENVIR)
 220
 221    try:
 222        (rawout, serr) = proc.communicate(timeout=NAMES['TIMEOUT'])
 223        if proc.returncode != 0 and len(serr) > 0:
 224            foutput = serr.decode("utf-8", errors="ignore")
 225        else:
 226            foutput = rawout.decode("utf-8", errors="ignore")
 227    except subprocess.TimeoutExpired:
 228        foutput = "Command \"{}\" timed out\n".format(command)
 229        proc.returncode = 255
 230
 231    proc.stdout.close()
 232    proc.stderr.close()
 233    return proc, foutput
 234
 235
 236def prepare_env(caseinfo, args, pm, stage, prefix, cmdlist, output = None):
 237    """
 238    Execute the setup/teardown commands for a test case.
 239    Optionally terminate test execution if the command fails.
 240    """
 241    if args.verbose > 0:
 242        print('{}'.format(prefix))
 243    for cmdinfo in cmdlist:
 244        if isinstance(cmdinfo, list):
 245            exit_codes = cmdinfo[1:]
 246            cmd = cmdinfo[0]
 247        else:
 248            exit_codes = [0]
 249            cmd = cmdinfo
 250
 251        if not cmd:
 252            continue
 253
 254        (proc, foutput) = exec_cmd(caseinfo, args, pm, stage, cmd)
 255
 256        if proc and (proc.returncode not in exit_codes):
 257            print('', file=sys.stderr)
 258            print("{} *** Could not execute: \"{}\"".format(prefix, cmd),
 259                  file=sys.stderr)
 260            print("\n{} *** Error message: \"{}\"".format(prefix, foutput),
 261                  file=sys.stderr)
 262            print("returncode {}; expected {}".format(proc.returncode,
 263                                                      exit_codes))
 264            print("\n{} *** Aborting test run.".format(prefix), file=sys.stderr)
 265            print("\n\n{} *** stdout ***".format(proc.stdout), file=sys.stderr)
 266            print("\n\n{} *** stderr ***".format(proc.stderr), file=sys.stderr)
 267            raise PluginMgrTestFail(
 268                stage, output,
 269                '"{}" did not complete successfully'.format(prefix))
 270
 271def verify_by_json(procout, res, tidx, args, pm):
 272    try:
 273        outputJSON = json.loads(procout)
 274    except json.JSONDecodeError:
 275        res.set_result(ResultState.fail)
 276        res.set_failmsg('Cannot decode verify command\'s output. Is it JSON?')
 277        return res
 278
 279    matchJSON = json.loads(json.dumps(tidx['matchJSON']))
 280
 281    if type(outputJSON) != type(matchJSON):
 282        failmsg = 'Original output and matchJSON value are not the same type: output: {} != matchJSON: {} '
 283        failmsg = failmsg.format(type(outputJSON).__name__, type(matchJSON).__name__)
 284        res.set_result(ResultState.fail)
 285        res.set_failmsg(failmsg)
 286        return res
 287
 288    if len(matchJSON) > len(outputJSON):
 289        failmsg = "Your matchJSON value is an array, and it contains more elements than the command under test\'s output:\ncommand output (length: {}):\n{}\nmatchJSON value (length: {}):\n{}"
 290        failmsg = failmsg.format(len(outputJSON), outputJSON, len(matchJSON), matchJSON)
 291        res.set_result(ResultState.fail)
 292        res.set_failmsg(failmsg)
 293        return res
 294    res = find_in_json(res, outputJSON, matchJSON, 0)
 295
 296    return res
 297
 298def find_in_json(res, outputJSONVal, matchJSONVal, matchJSONKey=None):
 299    if res.get_result() == ResultState.fail:
 300        return res
 301
 302    if type(matchJSONVal) == list:
 303        res = find_in_json_list(res, outputJSONVal, matchJSONVal, matchJSONKey)
 304
 305    elif type(matchJSONVal) == dict:
 306        res = find_in_json_dict(res, outputJSONVal, matchJSONVal)
 307    else:
 308        res = find_in_json_other(res, outputJSONVal, matchJSONVal, matchJSONKey)
 309
 310    if res.get_result() != ResultState.fail:
 311        res.set_result(ResultState.success)
 312        return res
 313
 314    return res
 315
 316def find_in_json_list(res, outputJSONVal, matchJSONVal, matchJSONKey=None):
 317    if (type(matchJSONVal) != type(outputJSONVal)):
 318        failmsg = 'Original output and matchJSON value are not the same type: output: {} != matchJSON: {}'
 319        failmsg = failmsg.format(outputJSONVal, matchJSONVal)
 320        res.set_result(ResultState.fail)
 321        res.set_failmsg(failmsg)
 322        return res
 323
 324    if len(matchJSONVal) > len(outputJSONVal):
 325        failmsg = "Your matchJSON value is an array, and it contains more elements than the command under test\'s output:\ncommand output (length: {}):\n{}\nmatchJSON value (length: {}):\n{}"
 326        failmsg = failmsg.format(len(outputJSONVal), outputJSONVal, len(matchJSONVal), matchJSONVal)
 327        res.set_result(ResultState.fail)
 328        res.set_failmsg(failmsg)
 329        return res
 330
 331    for matchJSONIdx, matchJSONVal in enumerate(matchJSONVal):
 332        res = find_in_json(res, outputJSONVal[matchJSONIdx], matchJSONVal,
 333                           matchJSONKey)
 334    return res
 335
 336def find_in_json_dict(res, outputJSONVal, matchJSONVal):
 337    for matchJSONKey, matchJSONVal in matchJSONVal.items():
 338        if type(outputJSONVal) == dict:
 339            if matchJSONKey not in outputJSONVal:
 340                failmsg = 'Key not found in json output: {}: {}\nMatching against output: {}'
 341                failmsg = failmsg.format(matchJSONKey, matchJSONVal, outputJSONVal)
 342                res.set_result(ResultState.fail)
 343                res.set_failmsg(failmsg)
 344                return res
 345
 346        else:
 347            failmsg = 'Original output and matchJSON value are not the same type: output: {} != matchJSON: {}'
 348            failmsg = failmsg.format(type(outputJSON).__name__, type(matchJSON).__name__)
 349            res.set_result(ResultState.fail)
 350            res.set_failmsg(failmsg)
 351            return rest
 352
 353        if type(outputJSONVal) == dict and (type(outputJSONVal[matchJSONKey]) == dict or
 354                type(outputJSONVal[matchJSONKey]) == list):
 355            if len(matchJSONVal) > 0:
 356                res = find_in_json(res, outputJSONVal[matchJSONKey], matchJSONVal, matchJSONKey)
 357            # handling corner case where matchJSONVal == [] or matchJSONVal == {}
 358            else:
 359                res = find_in_json_other(res, outputJSONVal, matchJSONVal, matchJSONKey)
 360        else:
 361            res = find_in_json(res, outputJSONVal, matchJSONVal, matchJSONKey)
 362    return res
 363
 364def find_in_json_other(res, outputJSONVal, matchJSONVal, matchJSONKey=None):
 365    if matchJSONKey in outputJSONVal:
 366        if matchJSONVal != outputJSONVal[matchJSONKey]:
 367            failmsg = 'Value doesn\'t match: {}: {} != {}\nMatching against output: {}'
 368            failmsg = failmsg.format(matchJSONKey, matchJSONVal, outputJSONVal[matchJSONKey], outputJSONVal)
 369            res.set_result(ResultState.fail)
 370            res.set_failmsg(failmsg)
 371            return res
 372
 373    return res
 374
 375def run_one_test(pm, args, index, tidx):
 376    global NAMES
 377    ns = NAMES['NS']
 378    dev0 = NAMES['DEV0']
 379    dev1 = NAMES['DEV1']
 380    dummy = NAMES['DUMMY']
 381    result = True
 382    tresult = ""
 383    tap = ""
 384    res = TestResult(tidx['id'], tidx['name'])
 385    if args.verbose > 0:
 386        print("\t====================\n=====> ", end="")
 387    print("Test " + tidx["id"] + ": " + tidx["name"])
 388
 389    if 'skip' in tidx:
 390        if tidx['skip'] == 'yes':
 391            res = TestResult(tidx['id'], tidx['name'])
 392            res.set_result(ResultState.skip)
 393            res.set_errormsg('Test case designated as skipped.')
 394            pm.call_pre_case(tidx, test_skip=True)
 395            pm.call_post_execute(tidx)
 396            return res
 397
 398    if 'dependsOn' in tidx:
 399        if (args.verbose > 0):
 400            print('probe command for test skip')
 401        (p, procout) = exec_cmd(tidx, args, pm, 'execute', tidx['dependsOn'])
 402        if p:
 403            if (p.returncode != 0):
 404                res = TestResult(tidx['id'], tidx['name'])
 405                res.set_result(ResultState.skip)
 406                res.set_errormsg('probe command: test skipped.')
 407                pm.call_pre_case(tidx, test_skip=True)
 408                pm.call_post_execute(tidx)
 409                return res
 410
 411    # populate NAMES with TESTID for this test
 412    NAMES['TESTID'] = tidx['id']
 413    NAMES['NS'] = '{}-{}'.format(NAMES['NS'], tidx['random'])
 414    NAMES['DEV0'] = '{}id{}'.format(NAMES['DEV0'], tidx['id'])
 415    NAMES['DEV1'] = '{}id{}'.format(NAMES['DEV1'], tidx['id'])
 416    NAMES['DUMMY'] = '{}id{}'.format(NAMES['DUMMY'], tidx['id'])
 417
 418    pm.call_pre_case(tidx)
 419    prepare_env(tidx, args, pm, 'setup', "-----> prepare stage", tidx["setup"])
 420
 421    if (args.verbose > 0):
 422        print('-----> execute stage')
 423    pm.call_pre_execute(tidx)
 424    (p, procout) = exec_cmd(tidx, args, pm, 'execute', tidx["cmdUnderTest"])
 425    if p:
 426        exit_code = p.returncode
 427    else:
 428        exit_code = None
 429
 430    pm.call_post_execute(tidx)
 431
 432    if (exit_code is None or exit_code != int(tidx["expExitCode"])):
 433        print("exit: {!r}".format(exit_code))
 434        print("exit: {}".format(int(tidx["expExitCode"])))
 435        #print("exit: {!r} {}".format(exit_code, int(tidx["expExitCode"])))
 436        res.set_result(ResultState.fail)
 437        res.set_failmsg('Command exited with {}, expected {}\n{}'.format(exit_code, tidx["expExitCode"], procout))
 438        print(procout)
 439    else:
 440        if args.verbose > 0:
 441            print('-----> verify stage')
 442        (p, procout) = exec_cmd(tidx, args, pm, 'verify', tidx["verifyCmd"])
 443        if procout:
 444            if 'matchJSON' in tidx:
 445                verify_by_json(procout, res, tidx, args, pm)
 446            elif 'matchPattern' in tidx:
 447                match_pattern = re.compile(
 448                    str(tidx["matchPattern"]), re.DOTALL | re.MULTILINE)
 449                match_index = re.findall(match_pattern, procout)
 450                if len(match_index) != int(tidx["matchCount"]):
 451                    res.set_result(ResultState.fail)
 452                    res.set_failmsg('Could not match regex pattern. Verify command output:\n{}'.format(procout))
 453                else:
 454                    res.set_result(ResultState.success)
 455            else:
 456                res.set_result(ResultState.fail)
 457                res.set_failmsg('Must specify a match option: matchJSON or matchPattern\n{}'.format(procout))
 458        elif int(tidx["matchCount"]) != 0:
 459            res.set_result(ResultState.fail)
 460            res.set_failmsg('No output generated by verify command.')
 461        else:
 462            res.set_result(ResultState.success)
 463
 464    prepare_env(tidx, args, pm, 'teardown', '-----> teardown stage', tidx['teardown'], procout)
 465    pm.call_post_case(tidx)
 466
 467    index += 1
 468
 469    # remove TESTID from NAMES
 470    del(NAMES['TESTID'])
 471
 472    # Restore names
 473    NAMES['NS'] = ns
 474    NAMES['DEV0'] = dev0
 475    NAMES['DEV1'] = dev1
 476    NAMES['DUMMY'] = dummy
 477
 478    return res
 479
 480def prepare_run(pm, args, testlist):
 481    tcount = len(testlist)
 482    emergency_exit = False
 483    emergency_exit_message = ''
 484
 485    try:
 486        pm.call_pre_suite(tcount, testlist)
 487    except Exception as ee:
 488        ex_type, ex, ex_tb = sys.exc_info()
 489        print('Exception {} {} (caught in pre_suite).'.
 490              format(ex_type, ex))
 491        traceback.print_tb(ex_tb)
 492        emergency_exit_message = 'EMERGENCY EXIT, call_pre_suite failed with exception {} {}\n'.format(ex_type, ex)
 493        emergency_exit = True
 494
 495    if emergency_exit:
 496        pm.call_post_suite(1)
 497        return emergency_exit_message
 498
 499def purge_run(pm, index):
 500    pm.call_post_suite(index)
 501
 502def test_runner(pm, args, filtered_tests):
 503    """
 504    Driver function for the unit tests.
 505
 506    Prints information about the tests being run, executes the setup and
 507    teardown commands and the command under test itself. Also determines
 508    success/failure based on the information in the test case and generates
 509    TAP output accordingly.
 510    """
 511    testlist = filtered_tests
 512    tcount = len(testlist)
 513    index = 1
 514    tap = ''
 515    badtest = None
 516    stage = None
 517
 518    tsr = TestSuiteReport()
 519
 520    for tidx in testlist:
 521        if "flower" in tidx["category"] and args.device == None:
 522            errmsg = "Tests using the DEV2 variable must define the name of a "
 523            errmsg += "physical NIC with the -d option when running tdc.\n"
 524            errmsg += "Test has been skipped."
 525            if args.verbose > 1:
 526                print(errmsg)
 527            res = TestResult(tidx['id'], tidx['name'])
 528            res.set_result(ResultState.skip)
 529            res.set_errormsg(errmsg)
 530            tsr.add_resultdata(res)
 531            index += 1
 532            continue
 533        try:
 534            badtest = tidx  # in case it goes bad
 535            res = run_one_test(pm, args, index, tidx)
 536            tsr.add_resultdata(res)
 537        except PluginMgrTestFail as pmtf:
 538            ex_type, ex, ex_tb = sys.exc_info()
 539            stage = pmtf.stage
 540            message = pmtf.message
 541            output = pmtf.output
 542            res = TestResult(tidx['id'], tidx['name'])
 543            res.set_result(ResultState.fail)
 544            res.set_errormsg(pmtf.message)
 545            res.set_failmsg(pmtf.output)
 546            tsr.add_resultdata(res)
 547            index += 1
 548            print(message)
 549            print('Exception {} {} (caught in test_runner, running test {} {} {} stage {})'.
 550                  format(ex_type, ex, index, tidx['id'], tidx['name'], stage))
 551            print('---------------')
 552            print('traceback')
 553            traceback.print_tb(ex_tb)
 554            print('---------------')
 555            if stage == 'teardown':
 556                print('accumulated output for this test:')
 557                if pmtf.output:
 558                    print(pmtf.output)
 559            print('---------------')
 560            break
 561        index += 1
 562
 563    # if we failed in setup or teardown,
 564    # fill in the remaining tests with ok-skipped
 565    count = index
 566
 567    if tcount + 1 != count:
 568        for tidx in testlist[count - 1:]:
 569            res = TestResult(tidx['id'], tidx['name'])
 570            res.set_result(ResultState.skip)
 571            msg = 'skipped - previous {} failed {} {}'.format(stage,
 572                index, badtest.get('id', '--Unknown--'))
 573            res.set_errormsg(msg)
 574            tsr.add_resultdata(res)
 575            count += 1
 576
 577    if args.pause:
 578        print('Want to pause\nPress enter to continue ...')
 579        if input(sys.stdin):
 580            print('got something on stdin')
 581
 582    return (index, tsr)
 583
 584def mp_bins(alltests):
 585    serial = []
 586    parallel = []
 587
 588    for test in alltests:
 589        if 'nsPlugin' not in test['plugins']:
 590            serial.append(test)
 591        else:
 592            # We can only create one netdevsim device at a time
 593            if 'netdevsim/new_device' in str(test['setup']):
 594                serial.append(test)
 595            else:
 596                parallel.append(test)
 597
 598    return (serial, parallel)
 599
 600def __mp_runner(tests):
 601    (_, tsr) = test_runner(mp_pm, mp_args, tests)
 602    return tsr._testsuite
 603
 604def test_runner_mp(pm, args, alltests):
 605    prepare_run(pm, args, alltests)
 606
 607    (serial, parallel) = mp_bins(alltests)
 608
 609    batches = [parallel[n : n + 32] for n in range(0, len(parallel), 32)]
 610    batches.insert(0, serial)
 611
 612    print("Executing {} tests in parallel and {} in serial".format(len(parallel), len(serial)))
 613    print("Using {} batches and {} workers".format(len(batches), args.mp))
 614
 615    # We can't pickle these objects so workaround them
 616    global mp_pm
 617    mp_pm = pm
 618
 619    global mp_args
 620    mp_args = args
 621
 622    with Pool(args.mp) as p:
 623        pres = p.map(__mp_runner, batches)
 624
 625    tsr = TestSuiteReport()
 626    for trs in pres:
 627        for res in trs:
 628            tsr.add_resultdata(res)
 629
 630    # Passing an index is not useful in MP
 631    purge_run(pm, None)
 632
 633    return tsr
 634
 635def test_runner_serial(pm, args, alltests):
 636    prepare_run(pm, args, alltests)
 637
 638    if args.verbose:
 639        print("Executing {} tests in serial".format(len(alltests)))
 640
 641    (index, tsr) = test_runner(pm, args, alltests)
 642
 643    purge_run(pm, index)
 644
 645    return tsr
 646
 647def has_blank_ids(idlist):
 648    """
 649    Search the list for empty ID fields and return true/false accordingly.
 650    """
 651    return not(all(k for k in idlist))
 652
 653
 654def load_from_file(filename):
 655    """
 656    Open the JSON file containing the test cases and return them
 657    as list of ordered dictionary objects.
 658    """
 659    try:
 660        with open(filename) as test_data:
 661            testlist = json.load(test_data, object_pairs_hook=OrderedDict)
 662    except json.JSONDecodeError as jde:
 663        print('IGNORING test case file {}\n\tBECAUSE:  {}'.format(filename, jde))
 664        testlist = list()
 665    else:
 666        idlist = get_id_list(testlist)
 667        if (has_blank_ids(idlist)):
 668            for k in testlist:
 669                k['filename'] = filename
 670    return testlist
 671
 672def identity(string):
 673    return string
 674
 675def args_parse():
 676    """
 677    Create the argument parser.
 678    """
 679    parser = argparse.ArgumentParser(description='Linux TC unit tests')
 680    parser.register('type', None, identity)
 681    return parser
 682
 683
 684def set_args(parser):
 685    """
 686    Set the command line arguments for tdc.
 687    """
 688    parser.add_argument(
 689        '--outfile', type=str,
 690        help='Path to the file in which results should be saved. ' +
 691        'Default target is the current directory.')
 692    parser.add_argument(
 693        '-p', '--path', type=str,
 694        help='The full path to the tc executable to use')
 695    sg = parser.add_argument_group(
 696        'selection', 'select which test cases: ' +
 697        'files plus directories; filtered by categories plus testids')
 698    ag = parser.add_argument_group(
 699        'action', 'select action to perform on selected test cases')
 700
 701    sg.add_argument(
 702        '-D', '--directory', nargs='+', metavar='DIR',
 703        help='Collect tests from the specified directory(ies) ' +
 704        '(default [tc-tests])')
 705    sg.add_argument(
 706        '-f', '--file', nargs='+', metavar='FILE',
 707        help='Run tests from the specified file(s)')
 708    sg.add_argument(
 709        '-c', '--category', nargs='*', metavar='CATG', default=['+c'],
 710        help='Run tests only from the specified category/ies, ' +
 711        'or if no category/ies is/are specified, list known categories.')
 712    sg.add_argument(
 713        '-e', '--execute', nargs='+', metavar='ID',
 714        help='Execute the specified test cases with specified IDs')
 715    ag.add_argument(
 716        '-l', '--list', action='store_true',
 717        help='List all test cases, or those only within the specified category')
 718    ag.add_argument(
 719        '-s', '--show', action='store_true', dest='showID',
 720        help='Display the selected test cases')
 721    ag.add_argument(
 722        '-i', '--id', action='store_true', dest='gen_id',
 723        help='Generate ID numbers for new test cases')
 724    parser.add_argument(
 725        '-v', '--verbose', action='count', default=0,
 726        help='Show the commands that are being run')
 727    parser.add_argument(
 728        '--format', default='tap', const='tap', nargs='?',
 729        choices=['none', 'xunit', 'tap'],
 730        help='Specify the format for test results. (Default: TAP)')
 731    parser.add_argument('-d', '--device',
 732                        help='Execute test cases that use a physical device, ' +
 733                        'where DEVICE is its name. (If not defined, tests ' +
 734                        'that require a physical device will be skipped)')
 735    parser.add_argument(
 736        '-P', '--pause', action='store_true',
 737        help='Pause execution just before post-suite stage')
 738    parser.add_argument(
 739        '-J', '--multiprocess', type=int, default=1, dest='mp',
 740        help='Run tests in parallel whenever possible')
 741    return parser
 742
 743
 744def check_default_settings(args, remaining, pm):
 745    """
 746    Process any arguments overriding the default settings,
 747    and ensure the settings are correct.
 748    """
 749    # Allow for overriding specific settings
 750    global NAMES
 751
 752    if args.path != None:
 753        NAMES['TC'] = args.path
 754    if args.device != None:
 755        NAMES['DEV2'] = args.device
 756    if 'TIMEOUT' not in NAMES:
 757        NAMES['TIMEOUT'] = None
 758    if not os.path.isfile(NAMES['TC']):
 759        print("The specified tc path " + NAMES['TC'] + " does not exist.")
 760        exit(1)
 761
 762    pm.call_check_args(args, remaining)
 763
 764
 765def get_id_list(alltests):
 766    """
 767    Generate a list of all IDs in the test cases.
 768    """
 769    return [x["id"] for x in alltests]
 770
 771def check_case_id(alltests):
 772    """
 773    Check for duplicate test case IDs.
 774    """
 775    idl = get_id_list(alltests)
 776    return [x for x in idl if idl.count(x) > 1]
 777
 778
 779def does_id_exist(alltests, newid):
 780    """
 781    Check if a given ID already exists in the list of test cases.
 782    """
 783    idl = get_id_list(alltests)
 784    return (any(newid == x for x in idl))
 785
 786
 787def generate_case_ids(alltests):
 788    """
 789    If a test case has a blank ID field, generate a random hex ID for it
 790    and then write the test cases back to disk.
 791    """
 792    for c in alltests:
 793        if (c["id"] == ""):
 794            while True:
 795                newid = str('{:04x}'.format(random.randrange(16**4)))
 796                if (does_id_exist(alltests, newid)):
 797                    continue
 798                else:
 799                    c['id'] = newid
 800                    break
 801
 802    ufilename = []
 803    for c in alltests:
 804        if ('filename' in c):
 805            ufilename.append(c['filename'])
 806    ufilename = get_unique_item(ufilename)
 807    for f in ufilename:
 808        testlist = []
 809        for t in alltests:
 810            if 'filename' in t:
 811                if t['filename'] == f:
 812                    del t['filename']
 813                    testlist.append(t)
 814        outfile = open(f, "w")
 815        json.dump(testlist, outfile, indent=4)
 816        outfile.write("\n")
 817        outfile.close()
 818
 819def filter_tests_by_id(args, testlist):
 820    '''
 821    Remove tests from testlist that are not in the named id list.
 822    If id list is empty, return empty list.
 823    '''
 824    newlist = list()
 825    if testlist and args.execute:
 826        target_ids = args.execute
 827
 828        if isinstance(target_ids, list) and (len(target_ids) > 0):
 829            newlist = list(filter(lambda x: x['id'] in target_ids, testlist))
 830    return newlist
 831
 832def filter_tests_by_category(args, testlist):
 833    '''
 834    Remove tests from testlist that are not in a named category.
 835    '''
 836    answer = list()
 837    if args.category and testlist:
 838        test_ids = list()
 839        for catg in set(args.category):
 840            if catg == '+c':
 841                continue
 842            print('considering category {}'.format(catg))
 843            for tc in testlist:
 844                if catg in tc['category'] and tc['id'] not in test_ids:
 845                    answer.append(tc)
 846                    test_ids.append(tc['id'])
 847
 848    return answer
 849
 850def set_random(alltests):
 851    for tidx in alltests:
 852        tidx['random'] = random.getrandbits(32)
 853
 854def get_test_cases(args):
 855    """
 856    If a test case file is specified, retrieve tests from that file.
 857    Otherwise, glob for all json files in subdirectories and load from
 858    each one.
 859    Also, if requested, filter by category, and add tests matching
 860    certain ids.
 861    """
 862    import fnmatch
 863
 864    flist = []
 865    testdirs = ['tc-tests']
 866
 867    if args.file:
 868        # at least one file was specified - remove the default directory
 869        testdirs = []
 870
 871        for ff in args.file:
 872            if not os.path.isfile(ff):
 873                print("IGNORING file " + ff + "\n\tBECAUSE does not exist.")
 874            else:
 875                flist.append(os.path.abspath(ff))
 876
 877    if args.directory:
 878        testdirs = args.directory
 879
 880    for testdir in testdirs:
 881        for root, dirnames, filenames in os.walk(testdir):
 882            for filename in fnmatch.filter(filenames, '*.json'):
 883                candidate = os.path.abspath(os.path.join(root, filename))
 884                if candidate not in testdirs:
 885                    flist.append(candidate)
 886
 887    alltestcases = list()
 888    for casefile in flist:
 889        alltestcases = alltestcases + (load_from_file(casefile))
 890
 891    allcatlist = get_test_categories(alltestcases)
 892    allidlist = get_id_list(alltestcases)
 893
 894    testcases_by_cats = get_categorized_testlist(alltestcases, allcatlist)
 895    idtestcases = filter_tests_by_id(args, alltestcases)
 896    cattestcases = filter_tests_by_category(args, alltestcases)
 897
 898    cat_ids = [x['id'] for x in cattestcases]
 899    if args.execute:
 900        if args.category:
 901            alltestcases = cattestcases + [x for x in idtestcases if x['id'] not in cat_ids]
 902        else:
 903            alltestcases = idtestcases
 904    else:
 905        if cat_ids:
 906            alltestcases = cattestcases
 907        else:
 908            # just accept the existing value of alltestcases,
 909            # which has been filtered by file/directory
 910            pass
 911
 912    return allcatlist, allidlist, testcases_by_cats, alltestcases
 913
 914
 915def set_operation_mode(pm, parser, args, remaining):
 916    """
 917    Load the test case data and process remaining arguments to determine
 918    what the script should do for this run, and call the appropriate
 919    function.
 920    """
 921    ucat, idlist, testcases, alltests = get_test_cases(args)
 922
 923    if args.gen_id:
 924        if (has_blank_ids(idlist)):
 925            alltests = generate_case_ids(alltests)
 926        else:
 927            print("No empty ID fields found in test files.")
 928        exit(0)
 929
 930    duplicate_ids = check_case_id(alltests)
 931    if (len(duplicate_ids) > 0):
 932        print("The following test case IDs are not unique:")
 933        print(str(set(duplicate_ids)))
 934        print("Please correct them before continuing.")
 935        exit(1)
 936
 937    if args.showID:
 938        for atest in alltests:
 939            print_test_case(atest)
 940        exit(0)
 941
 942    if isinstance(args.category, list) and (len(args.category) == 0):
 943        print("Available categories:")
 944        print_sll(ucat)
 945        exit(0)
 946
 947    if args.list:
 948        list_test_cases(alltests)
 949        exit(0)
 950
 951    set_random(alltests)
 952
 953    exit_code = 0 # KSFT_PASS
 954    if len(alltests):
 955        req_plugins = pm.get_required_plugins(alltests)
 956        try:
 957            args = pm.load_required_plugins(req_plugins, parser, args, remaining)
 958        except PluginDependencyException as pde:
 959            print('The following plugins were not found:')
 960            print('{}'.format(pde.missing_pg))
 961
 962        if args.mp > 1:
 963            catresults = test_runner_mp(pm, args, alltests)
 964        else:
 965            catresults = test_runner_serial(pm, args, alltests)
 966
 967        if catresults.count_failures() != 0:
 968            exit_code = 1 # KSFT_FAIL
 969        if args.format == 'none':
 970            print('Test results output suppression requested\n')
 971        else:
 972            print('\nAll test results: \n')
 973            if args.format == 'xunit':
 974                suffix = 'xml'
 975                res = catresults.format_xunit()
 976            elif args.format == 'tap':
 977                suffix = 'tap'
 978                res = catresults.format_tap()
 979            print(res)
 980            print('\n\n')
 981            if not args.outfile:
 982                fname = 'test-results.{}'.format(suffix)
 983            else:
 984                fname = args.outfile
 985            with open(fname, 'w') as fh:
 986                fh.write(res)
 987                fh.close()
 988                if os.getenv('SUDO_UID') is not None:
 989                    os.chown(fname, uid=int(os.getenv('SUDO_UID')),
 990                        gid=int(os.getenv('SUDO_GID')))
 991    else:
 992        print('No tests found\n')
 993        exit_code = 4 # KSFT_SKIP
 994    exit(exit_code)
 995
 996def main():
 997    """
 998    Start of execution; set up argument parser and get the arguments,
 999    and start operations.
1000    """
1001    import resource
1002
1003    if sys.version_info.major < 3 or sys.version_info.minor < 8:
1004        sys.exit("tdc requires at least python 3.8")
1005
1006    resource.setrlimit(resource.RLIMIT_NOFILE, (1048576, 1048576))
1007
1008    parser = args_parse()
1009    parser = set_args(parser)
1010    pm = PluginMgr(parser)
1011    parser = pm.call_add_args(parser)
1012    (args, remaining) = parser.parse_known_args()
1013    args.NAMES = NAMES
1014    args.mp = min(args.mp, 4)
1015    pm.set_args(args)
1016    check_default_settings(args, remaining, pm)
1017    if args.verbose > 2:
1018        print('args is {}'.format(args))
1019
1020    try:
1021        set_operation_mode(pm, parser, args, remaining)
1022    except KeyboardInterrupt:
1023        # Cleanup on Ctrl-C
1024        pm.call_post_suite(None)
1025
1026if __name__ == "__main__":
1027    main()