From fe077ef3bfcef2d65034c19377c7ed6f4b7945df Mon Sep 17 00:00:00 2001 From: deadwood Date: Tue, 28 Mar 2023 15:23:59 +0200 Subject: [PATCH] Fixes to unit tests scripts 1) Delete -Results.xml that could have copied from build directory to make sure results are always fresh 2) Delete contents of logs directory not to have stalled contents 3) Check names of suites that ran against names of suites that should run 4) Display statistics about failures, errors and tests run --- scripts/merge_junit_results.py | 33 ++++++++---- scripts/nightly/autotest/User-Startup.cunit | 58 ++++++++++----------- scripts/runtests.sh | 15 ++++++ 3 files changed, 66 insertions(+), 40 deletions(-) diff --git a/scripts/merge_junit_results.py b/scripts/merge_junit_results.py index d58fe82dc4..bd00735985 100755 --- a/scripts/merge_junit_results.py +++ b/scripts/merge_junit_results.py @@ -51,36 +51,47 @@ def main(): if f.endswith("-Results.xml"): xml_files.append(os.path.join(args[0], f)) - merge_results(xml_files) + test_suite_names = [] + + merge_results(xml_files, test_suite_names) + + with open(os.path.join(args[0], "all-test-suites-out"), "w") as f: + for line in test_suite_names: + f.write(line + "\n") -def merge_results(xml_files): +def merge_results(xml_files, test_suite_names): failures = 0 tests = 0 errors = 0 time = 0.0 - cases = [] + testsuites = [] for file_name in xml_files: tree = ET.parse(file_name) - test_suite = tree.getroot() - failures += int(test_suite.attrib['failures']) - tests += int(test_suite.attrib['tests']) - errors += int(test_suite.attrib['errors']) - time += float(test_suite.attrib['time']) - cases.append(list(test_suite)) + root = tree.getroot() + failures += int(root.attrib['failures']) + tests += int(root.attrib['tests']) + errors += int(root.attrib['errors']) + time += float(root.attrib['time']) + testsuites.append(list(root)) + + for testsuite in root: + test_suite_names.append(testsuite.attrib['name']) new_root = ET.Element('testsuites') new_root.attrib['failures'] = '%s' % failures new_root.attrib['tests'] = '%s' % tests new_root.attrib['errors'] = '%s' % errors new_root.attrib['time'] = '%s' % time - for case in cases: - new_root.extend(case) + for testsuite in testsuites: + new_root.extend(testsuite) new_tree = ET.ElementTree(new_root) ET.dump(new_tree) + + def usage(): this_file = os.path.basename(__file__) print ('Usage: %s directory' % this_file) diff --git a/scripts/nightly/autotest/User-Startup.cunit b/scripts/nightly/autotest/User-Startup.cunit index 3d0a915a7b..d7ff129fcd 100644 --- a/scripts/nightly/autotest/User-Startup.cunit +++ b/scripts/nightly/autotest/User-Startup.cunit @@ -4,18 +4,18 @@ If NOT WARN Echo "## Unit Test Output is redirected to DEBUG:" >DEBUG: ; Perform basic "Exec" type unit tests... - Execute S/Test cunit/exec/cunit-exec-types + Execute S/Test cunit/exec/cunit-exec-types ;BasicTypes_Suite ; Peform basic 'C' type unit tests.. - Execute S/Test cunit/crt/stdc/cunit-crt-types + Execute S/Test cunit/crt/stdc/cunit-crt-types ;StandardTypes_Suite ; Peform basic 'genmodule' unit tests.. If EXISTS "SYS:Development/Debug/Tests/Library/Libs" Assign "LIBS:" "SYS:Development/Debug/Tests/Library/Libs" ADD Copy SYS:Development/Debug/Tests/Library/extlib1.seg SYS: - Execute S/Test cunit/genmodule/library/cunit-genmodule-autoopen - Execute S/Test cunit/genmodule/library/cunit-genmodule-library - Execute S/Test cunit/genmodule/library/cunit-genmodule-relbase + Execute S/Test cunit/genmodule/library/cunit-genmodule-autoopen ;AutoOpen_Suite + Execute S/Test cunit/genmodule/library/cunit-genmodule-library ;Library_Suite + Execute S/Test cunit/genmodule/library/cunit-genmodule-relbase ;Relbase_Suite EndIf ; Perform basic "exec.library" unit tests... @@ -23,39 +23,39 @@ If NOT WARN ; Perform basic "utility.library" unit tests... ; Perform basic "dos.library" unit tests... - Execute S/Test cunit/dos/cunit-dos-createnewproc - Execute S/Test cunit/dos/cunit-dos-fileseek - Execute S/Test cunit/dos/cunit-dos-readargs - Execute S/Test cunit/dos/cunit-dos-smallcodemodel - Execute S/Test cunit/dos/cunit-dos-systemtags + Execute S/Test cunit/dos/cunit-dos-createnewproc ;CreateNewProc_Suite + Execute S/Test cunit/dos/cunit-dos-fileseek ;FileSeek_Suite + Execute S/Test cunit/dos/cunit-dos-readargs ;ReadArgs_Suite + Execute S/Test cunit/dos/cunit-dos-smallcodemodel ;SmallCodeModel_Suite + Execute S/Test cunit/dos/cunit-dos-systemtags ;SystemTags_Suite ; Perform basic "locale.library" unit tests... - Execute S/Test cunit/locale/cunit-locale-formatstring + Execute S/Test cunit/locale/cunit-locale-formatstring ;FormatString_Suite ; Perform basic "gfx.hidd" unit tests... - Execute S/Test cunit/hidds/gfx/cunit-convertpixels + Execute S/Test cunit/hidds/gfx/cunit-convertpixels ;ConvertPixels_Suite ; Perform basic C library unit tests... - Execute S/Test cunit/crt/stdc/math/cunit-crt-math - Execute S/Test cunit/crt/stdc/cunit-crt-fileseek - Execute S/Test cunit/crt/stdc/cunit-crt-sscanf - Execute S/Test cunit/crt/stdc/cunit-crt-sxprintf - Execute S/Test cunit/crt/posix/cunit-crt-fread + Execute S/Test cunit/crt/stdc/math/cunit-crt-math ;CrtMath_Suite + Execute S/Test cunit/crt/stdc/cunit-crt-fileseek ;FileSeek_Suite + Execute S/Test cunit/crt/stdc/cunit-crt-sscanf ;sscanf_Suite + Execute S/Test cunit/crt/stdc/cunit-crt-sxprintf ;sxprintf_Suite + Execute S/Test cunit/crt/posix/cunit-crt-fread ;fread_Suite Copy SYS:Development/Debug/Tests/crt/posix/execl2_slave SYS: - Execute S/Test cunit/crt/posix/cunit-crt-vfork - Execute S/Test cunit/crt/posix/cunit-crt-types64 - Execute S/Test cunit/crt/posix/cunit-crt-types64-lfs - Execute S/Test cunit/crt/posix/cunit-crt-types64-lfs-fob + Execute S/Test cunit/crt/posix/cunit-crt-vfork ;vfork_Suite + Execute S/Test cunit/crt/posix/cunit-crt-types64 ;types64_Suite + Execute S/Test cunit/crt/posix/cunit-crt-types64-lfs ;types64-lfs_Suite + Execute S/Test cunit/crt/posix/cunit-crt-types64-lfs-fob ;types64-lfs-fob_Suite ; Perform basic "Zune" unit tests... - Execute S/Test cunit/zune/cunit-zune-handleevent - Execute S/Test cunit/zune/cunit-zune-handleevent-2 - Execute S/Test cunit/zune/cunit-zune-handleevent-3 - Execute S/Test cunit/zune/cunit-zune-notify - Execute S/Test cunit/zune/cunit-zune-list - Execute S/Test cunit/zune/cunit-zune-showhide - Execute S/Test cunit/zune/cunit-zune-sleep - Execute S/Test cunit/zune/cunit-zune-window + Execute S/Test cunit/zune/cunit-zune-handleevent ;MUIM_HandleEvent_Suite + Execute S/Test cunit/zune/cunit-zune-handleevent-2 ;MUIM_HandleEvent_2_Suite + Execute S/Test cunit/zune/cunit-zune-handleevent-3 ;MUIM_HandleEvent_3_Suite + Execute S/Test cunit/zune/cunit-zune-notify ;MUIM_Notify_Suite + Execute S/Test cunit/zune/cunit-zune-list ;MUIC_List_Suite + Execute S/Test cunit/zune/cunit-zune-showhide ;MUIA_ShowMe_Suite + Execute S/Test cunit/zune/cunit-zune-sleep ;MUIA_Sleep_Suite + Execute S/Test cunit/zune/cunit-zune-window ;MUIC_Window_Suite Endif diff --git a/scripts/runtests.sh b/scripts/runtests.sh index d00907171d..9e601d3a86 100755 --- a/scripts/runtests.sh +++ b/scripts/runtests.sh @@ -22,7 +22,9 @@ then rm -rf $AROSBUILDDIR fi +# Preparing run environment mkdir -p $AROSRUNDIR +rm -rf $AROSLOGDIR mkdir -p $AROSLOGDIR rsync -r -t -l --info=progress2 -s ./core-linux-x86_64-d/bin/linux-x86_64/AROS/* $AROSRUNDIR cp ./core-linux-x86_64-d/bin/linux-x86_64/AROS/.gdb_ignore_errors.py $AROSRUNDIR @@ -35,10 +37,23 @@ mv $AROSRUNDIR/Devs/Monitors/X1* $AROSRUNDIR/Storage/Monitors sed -i "s|Startup = 0|Startup = 1|g" $AROSRUNDIR/Devs/DOSDrivers/DEBUG sed -i "s|module Devs/Drivers/gfx.hidd|&\nmodule Storage/Drivers/headlessgfx.hidd|g" $AROSRUNDIR/boot/linux/AROSBootstrap.conf sed -i "/module Devs\/Drivers\/x11gfx.hidd/d" $AROSRUNDIR/boot/linux/AROSBootstrap.conf +find $AROSRUNDIR/ -name "*-Results.xml" -exec rm {} \; + +# Running tests cd $AROSRUNDIR ./boot/linux/AROSBootstrap -m 640 + +# Generating results cd ../../../../ find $AROSRUNDIR/ -name "*-Results.xml" -exec cp {} $AROSLOGDIR/ \; $AROSSRCDIR/scripts/merge_junit_results.py $AROSLOGDIR > $AROSLOGDIR/all-unittests.xml junit2html $AROSLOGDIR/all-unittests.xml $AROSLOGDIR/all-unittests.html +# Checking if all suites were executed +sed -En "s/.*;(.*Suite).*/\1/gp" $AROSRUNDIR/S/User-Startup > $AROSLOGDIR/all-test-suites-in +sort -o $AROSLOGDIR/all-test-suites-in $AROSLOGDIR/all-test-suites-in +sort -o $AROSLOGDIR/all-test-suites-out $AROSLOGDIR/all-test-suites-out + +# Print out results +sed -En "s/^