diff options
Diffstat (limited to 'libs/assimp/test/regression/run.py')
| -rwxr-xr-x | libs/assimp/test/regression/run.py | 320 |
1 files changed, 320 insertions, 0 deletions
diff --git a/libs/assimp/test/regression/run.py b/libs/assimp/test/regression/run.py new file mode 100755 index 0000000..151d7a8 --- /dev/null +++ b/libs/assimp/test/regression/run.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +# -*- Coding: UTF-8 -*- + +# --------------------------------------------------------------------------- +# Open Asset Import Library (ASSIMP) +# --------------------------------------------------------------------------- +# +# Copyright (c) 2006-2020, ASSIMP Development Team +# +# All rights reserved. +# +# Redistribution and use of this software in source and binary forms, +# with or without modification, are permitted provided that the following +# conditions are met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the +# following disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the +# following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# * Neither the name of the ASSIMP team, nor the names of its +# contributors may be used to endorse or promote products +# derived from this software without specific prior +# written permission of the ASSIMP Development Team. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# --------------------------------------------------------------------------- + +""" +Run the regression test suite using settings from settings.py. + +The assimp_cmd (or assimp) binary to use is specified by the first +command line argument and defaults to ``assimp``. + +To build, set ``ASSIMP_BUILD_ASSIMP_TOOLS=ON`` in CMake. If generating +configs for an IDE, make sure to build the assimp_cmd project. + +On Windows, use ``py run.py <path to assimp>`` to make sure the command +line parameter is forwarded to the script. +""" + +import sys +import os +import subprocess +import zipfile +import collections +import multiprocessing + +import settings +import utils + +# ------------------------------------------------------------------------------- +EXPECTED_FAILURE_NOT_MET, DATABASE_LENGTH_MISMATCH, \ +DATABASE_VALUE_MISMATCH, IMPORT_FAILURE, \ +FILE_NOT_READABLE, COMPARE_SUCCESS, EXPECTED_FAILURE = range(7) + +messages = collections.defaultdict(lambda: "<unknown", { + EXPECTED_FAILURE_NOT_MET: +"""Unexpected success during import\n\ +\tReturn code was 0""", + + DATABASE_LENGTH_MISMATCH: +"""Database mismatch: lengths don't match\n\ +\tExpected: {0} Actual: {1}""", + + DATABASE_VALUE_MISMATCH: +"""Database mismatch: """, + + IMPORT_FAILURE: +"""Unexpected failure during import\n\ +\tReturn code was {0}""", + + FILE_NOT_READABLE: +"""Unexpected failure reading file""", + + COMPARE_SUCCESS: +"""Results match archived reference dump in database\n\ +\tNumber of bytes compared: {0}""", + + EXPECTED_FAILURE: +"""Expected failure was met.""", +}) + +outfilename_output = "run_regression_suite_output.txt" +outfilename_failur = "run_regression_suite_failures.csv" +Environment = {} + +# ------------------------------------------------------------------------------- +class results: + + """ Handle formatting of results""" + + def __init__(self, zipin): + """Init, given a ZIPed database """ + self.failures = [] + self.success = [] + self.zipin = zipin + + + def fail(self, failfile, filename_expect, pp, msg, *args): + """ + Report failure of a sub-test + + File f failed a test for pp config pp, failure notice is msg, + *args is format()ting args for msg + + """ + print("[FAILURE] " + messages[msg].format(*args)) + self.failures.append((failfile, filename_expect, pp)) + + + def ok(self, f, pp, msg, *args): + """ + Report success of a sub-test + + File f passed the test, msg is a happy success note, + *args is format()ing args for msg. + + """ + print("[SUCCESS] " + messages[msg].format(*args)) + self.success.append(f) + + + def report_results(self): + """Write results to ../results/run_regression_suite_failures.txt""" + + count_success = len(self.success) + count_fail = len(self.failures) + percent_good = float(count_success) / (count_success + count_fail) + print("\n" + ('='*60) + "\n" + "SUCCESS: {0}\nFAILURE: {1}\nPercentage good: {2}".format( + count_success, count_fail, percent_good) + + "\n" + ('='*60) + "\n") + + with open(os.path.join('..', 'results',outfilename_failur), "wt") as f: + f.write("ORIGINAL FILE;EXPECTED DUMP\n") + f.writelines(map( + lambda x: x[0] + ' ' + x[2] + ";" + x[1] + "\n", self.failures)) + + if self.failures: + print("\nSee " + settings.results + "\\" + outfilename_failur + + " for more details\n\n") + + def hasFailures( self ): + """ Return True, if any failures there. """ + return 0 != len( self.failures ) + +# ------------------------------------------------------------------------------- +def setEnvVar( var, value ): + print ( "set var " + var +" to" + value) + Environment[ var ] = value + +# ------------------------------------------------------------------------------- +def getEnvVar( var ): + if var in Environment: + return Environment[ var ] + else: + print ( "Error: cannot find " + var ) + return "" + +# ------------------------------------------------------------------------------- +def prepare_output_dir(fullpath, myhash, app): + outfile = os.path.join(settings.results, "tmp", os.path.split(fullpath)[1] + "_" + myhash) + try: + os.mkdir(outfile) + except OSError: + pass + + outfile = os.path.join(outfile, app) + return outfile + +# ------------------------------------------------------------------------------- +def process_dir(d, outfile_results, zipin, result ): + shellparams = {'stdout':outfile_results, 'stderr':outfile_results, 'shell':False} + + print("Processing directory " + d) + all = "" + for f in sorted(os.listdir(d)): + fullpath = os.path.join(d, f) + if os.path.isdir(fullpath) and not f[:1] == '.': + process_dir(fullpath, outfile_results, zipin, result) + continue + + if f in settings.files_to_ignore or os.path.splitext(f)[1] in settings.exclude_extensions: + print("Ignoring " + f) + return + + for pppreset in settings.pp_configs_to_test: + filehash = utils.hashing(fullpath, pppreset) + failure = False + + try: + input_expected = zipin.open(filehash, "r").read() + # empty dump files indicate 'expected import failure' + if not len(input_expected): + failure = True + except KeyError: + # TODO(acgessler): Keep track of this and report as error in the end. + print("Didn't find "+fullpath+" (Hash is "+filehash+") in database. Outdated "+\ + "regression database? Use gen_db.zip to re-generate.") + continue + + print("-"*60 + "\n " + os.path.realpath(fullpath) + " pp: " + pppreset) + + outfile_actual = prepare_output_dir(fullpath, filehash, "ACTUAL") + outfile_expect = prepare_output_dir(fullpath, filehash, "EXPECT") + outfile_results.write("assimp dump "+"-"*80+"\n") + outfile_results.flush() + assimp_bin_path = getEnvVar("assimp_path") + command = [assimp_bin_path, + "dump", + fullpath, outfile_actual, "-b", "-s", "-l" ] +\ + pppreset.split() + print( "command = " + str( command ) ) + r = subprocess.call(command, **shellparams) + outfile_results.flush() + + if r and not failure: + result.fail(fullpath, outfile_expect, pppreset, IMPORT_FAILURE, r) + outfile_results.write("Failed to import\n") + continue + elif failure and not r: + result.fail(fullpath, outfile_expect, pppreset, EXPECTED_FAILURE_NOT_MET) + outfile_results.write("Expected import to fail\n") + continue + elif failure and r: + result.ok(fullpath, pppreset, EXPECTED_FAILURE) + outfile_results.write("Failed as expected, skipping.\n") + continue + + with open(outfile_expect, "wb") as s: + s.write(input_expected) + + try: + with open(outfile_actual, "rb") as s: + input_actual = s.read() + except IOError: + continue + + outfile_results.write("Expected data length: {0}\n".format(len(input_expected))) + outfile_results.write("Actual data length: {0}\n".format(len(input_actual))) + failed = False + if len(input_expected) != len(input_actual): + result.fail(fullpath, outfile_expect, pppreset, DATABASE_LENGTH_MISMATCH, + len(input_expected), len(input_actual)) + # Still compare the dumps to see what the difference is + failed = True + + outfile_results.write("assimp cmpdump "+"-"*80+"\n") + outfile_results.flush() + command = [ assimp_bin_path, 'cmpdump', outfile_actual, outfile_expect ] + if subprocess.call(command, **shellparams) != 0: + if not failed: + result.fail(fullpath, outfile_expect, pppreset, DATABASE_VALUE_MISMATCH) + continue + + result.ok(fullpath, pppreset, COMPARE_SUCCESS, len(input_expected)) + +# ------------------------------------------------------------------------------- +def del_folder_with_contents(folder): + for root, dirs, files in os.walk(folder, topdown=False): + for name in files: + os.remove(os.path.join(root, name)) + for name in dirs: + os.rmdir(os.path.join(root, name)) + + +# ------------------------------------------------------------------------------- +def run_test(): + tmp_target_path = os.path.join(settings.results, "tmp") + try: + print( "try to make " + tmp_target_path ) + os.mkdir(tmp_target_path) + except OSError as oerr: + # clear contents if tmp folder exists already + del_folder_with_contents(tmp_target_path) + + try: + zipin = zipfile.ZipFile(settings.database_name + ".zip", + "r", zipfile.ZIP_STORED) + except IOError: + print("Regression database ", settings.database_name, + ".zip was not found") + return + + res = results(zipin) + with open(os.path.join(settings.results, outfilename_output), "wt") as outfile: + for tp in settings.model_directories: + process_dir(tp, outfile, zipin, res) + + res.report_results() + if res.hasFailures(): + return 1 + + return 0 + +# ------------------------------------------------------------------------------- +if __name__ == "__main__": + if len(sys.argv) > 1: + assimp_bin_path = sys.argv[1] + else: + assimp_bin_path = 'assimp' + setEnvVar("assimp_path", assimp_bin_path) + print('Using assimp binary: ' + assimp_bin_path) + sys.exit( run_test() ) + +# vim: ai ts=4 sts=4 et sw=4 |