Fixes to unit tests scripts

1) Delete -Results.xml that could have copied from build directory to make
sure results are always fresh
2) Delete contents of logs directory not to have stalled contents
3) Check names of suites that ran against names of suites that should run
4) Display statistics about failures, errors and tests run

scripts/merge_junit_results.py

@@ -51,36 +51,47 @@ def main():
         if f.endswith("-Results.xml"):
             xml_files.append(os.path.join(args[0], f))
 
-    merge_results(xml_files)
+    test_suite_names = []
+
+    merge_results(xml_files, test_suite_names)
+
+    with open(os.path.join(args[0], "all-test-suites-out"), "w") as f:
+        for line in test_suite_names:
+            f.write(line + "\n")
 
 
-def merge_results(xml_files):
+def merge_results(xml_files, test_suite_names):
     failures = 0
     tests = 0
     errors = 0
     time = 0.0
-    cases = []
+    testsuites = []
 
     for file_name in xml_files:
         tree = ET.parse(file_name)
-        test_suite = tree.getroot()
+        root = tree.getroot()
-        failures += int(test_suite.attrib['failures'])
+        failures    += int(root.attrib['failures'])
-        tests += int(test_suite.attrib['tests'])
+        tests       += int(root.attrib['tests'])
-        errors += int(test_suite.attrib['errors'])
+        errors      += int(root.attrib['errors'])
-        time += float(test_suite.attrib['time'])
+        time        += float(root.attrib['time'])
-        cases.append(list(test_suite))
+        testsuites.append(list(root))
+
+        for testsuite in root:
+            test_suite_names.append(testsuite.attrib['name'])
 
     new_root = ET.Element('testsuites')
     new_root.attrib['failures'] = '%s' % failures
     new_root.attrib['tests'] = '%s' % tests
     new_root.attrib['errors'] = '%s' % errors
     new_root.attrib['time'] = '%s' % time
-    for case in cases:
+    for testsuite in testsuites:
-        new_root.extend(case)
+        new_root.extend(testsuite)
     new_tree = ET.ElementTree(new_root)
     ET.dump(new_tree)
 
 
+
+
 def usage():
     this_file = os.path.basename(__file__)
     print ('Usage:  %s directory' % this_file)

scripts/nightly/autotest/User-Startup.cunit

@@ -4,18 +4,18 @@ If NOT WARN
     Echo "## Unit Test Output is redirected to DEBUG:" >DEBUG:
 
     ; Perform basic "Exec" type unit tests...
-    Execute S/Test cunit/exec/cunit-exec-types
+    Execute S/Test cunit/exec/cunit-exec-types                  ;BasicTypes_Suite
 
     ; Peform basic 'C' type unit tests..
-    Execute S/Test cunit/crt/stdc/cunit-crt-types
+    Execute S/Test cunit/crt/stdc/cunit-crt-types               ;StandardTypes_Suite
 
     ; Peform basic 'genmodule' unit tests..
     If EXISTS "SYS:Development/Debug/Tests/Library/Libs"
         Assign "LIBS:" "SYS:Development/Debug/Tests/Library/Libs" ADD
         Copy SYS:Development/Debug/Tests/Library/extlib1.seg SYS:
-        Execute S/Test cunit/genmodule/library/cunit-genmodule-autoopen
+        Execute S/Test cunit/genmodule/library/cunit-genmodule-autoopen ;AutoOpen_Suite
-        Execute S/Test cunit/genmodule/library/cunit-genmodule-library
+        Execute S/Test cunit/genmodule/library/cunit-genmodule-library  ;Library_Suite
-        Execute S/Test cunit/genmodule/library/cunit-genmodule-relbase
+        Execute S/Test cunit/genmodule/library/cunit-genmodule-relbase  ;Relbase_Suite
     EndIf
 
     ; Perform basic "exec.library" unit tests...
@@ -23,39 +23,39 @@ If NOT WARN
     ; Perform basic "utility.library" unit tests...
 
     ; Perform basic "dos.library" unit tests...
-    Execute S/Test cunit/dos/cunit-dos-createnewproc
+    Execute S/Test cunit/dos/cunit-dos-createnewproc            ;CreateNewProc_Suite
-    Execute S/Test cunit/dos/cunit-dos-fileseek
+    Execute S/Test cunit/dos/cunit-dos-fileseek                 ;FileSeek_Suite
-    Execute S/Test cunit/dos/cunit-dos-readargs
+    Execute S/Test cunit/dos/cunit-dos-readargs                 ;ReadArgs_Suite
-    Execute S/Test cunit/dos/cunit-dos-smallcodemodel
+    Execute S/Test cunit/dos/cunit-dos-smallcodemodel           ;SmallCodeModel_Suite
-    Execute S/Test cunit/dos/cunit-dos-systemtags
+    Execute S/Test cunit/dos/cunit-dos-systemtags               ;SystemTags_Suite
 
     ; Perform basic "locale.library" unit tests...
-    Execute S/Test cunit/locale/cunit-locale-formatstring
+    Execute S/Test cunit/locale/cunit-locale-formatstring       ;FormatString_Suite
 
     ; Perform basic "gfx.hidd" unit tests...
-    Execute S/Test cunit/hidds/gfx/cunit-convertpixels
+    Execute S/Test cunit/hidds/gfx/cunit-convertpixels          ;ConvertPixels_Suite
 
     ; Perform basic C library unit tests...
-    Execute S/Test cunit/crt/stdc/math/cunit-crt-math
+    Execute S/Test cunit/crt/stdc/math/cunit-crt-math           ;CrtMath_Suite
-    Execute S/Test cunit/crt/stdc/cunit-crt-fileseek
+    Execute S/Test cunit/crt/stdc/cunit-crt-fileseek            ;FileSeek_Suite
-    Execute S/Test cunit/crt/stdc/cunit-crt-sscanf
+    Execute S/Test cunit/crt/stdc/cunit-crt-sscanf              ;sscanf_Suite
-    Execute S/Test cunit/crt/stdc/cunit-crt-sxprintf
+    Execute S/Test cunit/crt/stdc/cunit-crt-sxprintf            ;sxprintf_Suite
-    Execute S/Test cunit/crt/posix/cunit-crt-fread
+    Execute S/Test cunit/crt/posix/cunit-crt-fread              ;fread_Suite
     Copy SYS:Development/Debug/Tests/crt/posix/execl2_slave SYS:
-    Execute S/Test cunit/crt/posix/cunit-crt-vfork
+    Execute S/Test cunit/crt/posix/cunit-crt-vfork              ;vfork_Suite
-    Execute S/Test cunit/crt/posix/cunit-crt-types64
+    Execute S/Test cunit/crt/posix/cunit-crt-types64            ;types64_Suite
-    Execute S/Test cunit/crt/posix/cunit-crt-types64-lfs
+    Execute S/Test cunit/crt/posix/cunit-crt-types64-lfs        ;types64-lfs_Suite
-    Execute S/Test cunit/crt/posix/cunit-crt-types64-lfs-fob
+    Execute S/Test cunit/crt/posix/cunit-crt-types64-lfs-fob    ;types64-lfs-fob_Suite
 
     ; Perform basic "Zune" unit tests...
-    Execute S/Test cunit/zune/cunit-zune-handleevent
+    Execute S/Test cunit/zune/cunit-zune-handleevent            ;MUIM_HandleEvent_Suite
-    Execute S/Test cunit/zune/cunit-zune-handleevent-2
+    Execute S/Test cunit/zune/cunit-zune-handleevent-2          ;MUIM_HandleEvent_2_Suite
-    Execute S/Test cunit/zune/cunit-zune-handleevent-3
+    Execute S/Test cunit/zune/cunit-zune-handleevent-3          ;MUIM_HandleEvent_3_Suite
-    Execute S/Test cunit/zune/cunit-zune-notify
+    Execute S/Test cunit/zune/cunit-zune-notify                 ;MUIM_Notify_Suite
-    Execute S/Test cunit/zune/cunit-zune-list
+    Execute S/Test cunit/zune/cunit-zune-list                   ;MUIC_List_Suite
-    Execute S/Test cunit/zune/cunit-zune-showhide
+    Execute S/Test cunit/zune/cunit-zune-showhide               ;MUIA_ShowMe_Suite
-    Execute S/Test cunit/zune/cunit-zune-sleep
+    Execute S/Test cunit/zune/cunit-zune-sleep                  ;MUIA_Sleep_Suite
-    Execute S/Test cunit/zune/cunit-zune-window
+    Execute S/Test cunit/zune/cunit-zune-window                 ;MUIC_Window_Suite
 
 Endif
 

scripts/runtests.sh

@@ -22,7 +22,9 @@ then
 rm -rf $AROSBUILDDIR
 fi
 
+# Preparing run environment
 mkdir -p $AROSRUNDIR
+rm -rf $AROSLOGDIR
 mkdir -p $AROSLOGDIR
 rsync -r -t -l --info=progress2 -s ./core-linux-x86_64-d/bin/linux-x86_64/AROS/* $AROSRUNDIR
 cp ./core-linux-x86_64-d/bin/linux-x86_64/AROS/.gdb_ignore_errors.py $AROSRUNDIR
@@ -35,10 +37,23 @@ mv $AROSRUNDIR/Devs/Monitors/X1* $AROSRUNDIR/Storage/Monitors
 sed -i "s|Startup = 0|Startup = 1|g" $AROSRUNDIR/Devs/DOSDrivers/DEBUG
 sed -i "s|module Devs/Drivers/gfx.hidd|&\nmodule Storage/Drivers/headlessgfx.hidd|g" $AROSRUNDIR/boot/linux/AROSBootstrap.conf
 sed -i "/module Devs\/Drivers\/x11gfx.hidd/d" $AROSRUNDIR/boot/linux/AROSBootstrap.conf
+find $AROSRUNDIR/ -name "*-Results.xml" -exec rm {} \;
+
+# Running tests
 cd $AROSRUNDIR
 ./boot/linux/AROSBootstrap -m 640
+
+# Generating results
 cd ../../../../
 find $AROSRUNDIR/ -name "*-Results.xml" -exec cp {} $AROSLOGDIR/ \;
 $AROSSRCDIR/scripts/merge_junit_results.py $AROSLOGDIR > $AROSLOGDIR/all-unittests.xml
 junit2html $AROSLOGDIR/all-unittests.xml $AROSLOGDIR/all-unittests.html
 
+# Checking if all suites were executed
+sed -En "s/.*;(.*Suite).*/\1/gp" $AROSRUNDIR/S/User-Startup > $AROSLOGDIR/all-test-suites-in
+sort -o $AROSLOGDIR/all-test-suites-in  $AROSLOGDIR/all-test-suites-in
+sort -o $AROSLOGDIR/all-test-suites-out $AROSLOGDIR/all-test-suites-out
+
+# Print out results
+sed -En "s/^<testsuites.*(fail.*).*><test.*/\1/gp" $AROSLOGDIR/all-unittests.xml
+diff -ruN $AROSLOGDIR/all-test-suites-in $AROSLOGDIR/all-test-suites-out