Add kola-junit script to generate JUnit XML files

build.sh

@@ -44,6 +44,9 @@ install_rpms() {
 
     frozendeps=""
 
+    # We freeze the version for now since we're carrying patches.
+    frozendeps+=" $(echo osbuild{,-ostree,-selinux,-tools}-100-1.fc39.noarch)"
+
     # First, a general update; this is best practice.  We also hit an issue recently
     # where qemu implicitly depended on an updated libusbx but didn't have a versioned
     # requires https://bugzilla.redhat.com/show_bug.cgi?id=1625641

mantle/cmd/kola/testiso.go

@@ -30,6 +30,8 @@ import (
 	"time"
 
 	"github.com/coreos/coreos-assembler/mantle/harness"
+	"github.com/coreos/coreos-assembler/mantle/harness/reporters"
+	"github.com/coreos/coreos-assembler/mantle/harness/testresult"
 	"github.com/coreos/coreos-assembler/mantle/platform/conf"
 	"github.com/coreos/coreos-assembler/mantle/util"
 	coreosarch "github.com/coreos/stream-metadata-go/arch"
@@ -448,8 +450,7 @@ func filterTests(tests []string, patterns []string) ([]string, error) {
 	return r, nil
 }
 
-func runTestIso(cmd *cobra.Command, args []string) error {
-	var err error
+func runTestIso(cmd *cobra.Command, args []string) (err error) {
 	if kola.CosaBuild == nil {
 		return fmt.Errorf("Must provide --build")
 	}
@@ -491,6 +492,19 @@ func runTestIso(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	// see similar code in suite.go
+	reportDir := filepath.Join(outputDir, "reports")
+	if err := os.Mkdir(reportDir, 0777); err != nil {
+		return err
+	}
+
+	reporter := reporters.NewJSONReporter("report.json", "testiso", "")
+	defer func() {
+		if reportErr := reporter.Output(reportDir); reportErr != nil && err != nil {
+			err = reportErr
+		}
+	}()
+
 	baseInst := platform.Install{
 		CosaBuild:       kola.CosaBuild,
 		PxeAppendRootfs: pxeAppendRootfs,
@@ -576,12 +590,22 @@ func runTestIso(cmd *cobra.Command, args []string) error {
 		default:
 			plog.Fatalf("Unknown test name:%s", test)
 		}
+
+		result := testresult.Pass
+		output := []byte{}
+		if err != nil {
+			result = testresult.Fail
+			output = []byte(err.Error())
+		}
+		reporter.ReportTest(test, []string{}, result, duration, output)
 		if printResult(test, duration, err) {
 			atLeastOneFailed = true
 		}
 	}
 
+	reporter.SetResult(testresult.Pass)
 	if atLeastOneFailed {
+		reporter.SetResult(testresult.Fail)
 		return harness.SuiteFailed
 	}
 

src/deps.txt

@@ -73,6 +73,9 @@ python3-resultsdb_conventions
 python3-resultsdb_conventions-fedora
 python3-fedfind
 
+# For creating JUnit test results for Jenkins
+python3-junit_xml
+
 # For debugging running processes in the pipelines
 strace
 

src/kola-junit

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+
+'''
+    This script is used in Jenkins to convert kola's `report.json` to JUnit
+    XML, which Jenkins knows how to display natively.
+'''
+
+import argparse
+import json
+import os
+import re
+import sys
+
+from junit_xml import TestSuite, TestCase
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--classname", required=True)
+parser.add_argument("--koladir", required=True)
+parser.add_argument("--output", required=True)
+parser.add_argument("--with-passed-tests", action='store_true')
+args = parser.parse_args()
+
+with open(os.path.join(args.koladir, "reports/report.json")) as f:
+    report = json.load(f)
+
+
+# https://stackoverflow.com/a/14693789
+def strip_ansi(text):
+    ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
+    return ansi_escape.sub('', text)
+
+
+# In a {kola_outputdir}/{testname}/ dir, there can be multiple directories; one
+# per host that was brought up. e.g. the Tang test brings up a Tang server in a
+# separate host. We can only report one host, so we choose the one whose dir
+# was created most recently since that's more likely to be the one with the
+# error.
+def get_most_recent_host_dir(parent_dir):
+    # special case for testiso tests, where there's no nested dir per host
+    if os.path.exists(os.path.join(parent_dir, "console.txt")):
+        return parent_dir
+
+    most_recent = None
+    most_recent_ts = 0
+    for dirent in os.listdir(parent_dir):
+        path = os.path.join(parent_dir, dirent)
+        if not os.path.isdir(path):
+            continue
+        ts = os.path.getctime(path)
+        if ts > most_recent_ts:
+            most_recent = path
+            most_recent_ts = ts
+    return most_recent
+
+
+test_cases = []
+for test in report['tests']:
+    # by default, we don't include passed tests in the report
+    if test['result'] == 'PASS' and not args.with_passed_tests:
+        continue
+
+    has_bucket_prefix = test['name'].startswith('non-exclusive-test-bucket-')
+
+    if '/' in test['name']:
+        # a subtest; we don't normally report them, except bucketed tests since
+        # they're really full tests of their own
+        main_test, subtest = test['name'].split('/')
+        if not has_bucket_prefix:
+            continue
+        dirname = main_test
+        test['name'] = subtest
+    elif has_bucket_prefix:
+        # it's the bucketed test itself; we don't report that one since we
+        # directly report on the subtests that failed
+        continue
+    else:
+        dirname = test['name']
+
+    host_dir = get_most_recent_host_dir(f"{args.koladir}/{dirname}")
+
+    ext_test_out = None
+    if host_dir and test['name'].startswith('ext.'):
+        ext_test_out_fn = os.path.join(host_dir, f"{test['name']}.txt")
+        if os.path.isfile(ext_test_out_fn):
+            with open(ext_test_out_fn, encoding='utf-8') as f:
+                ext_test_out = f.read()
+
+    # for external tests; we append the kola-runext.service output
+    if ext_test_out is not None:
+        test['output'] += f"\n---\n{ext_test_out}"
+    test['output'] = strip_ansi(test['output'])
+
+    if test['result'] == 'PASS':
+        tc = TestCase(test['name'], args.classname + ".tests",
+                      test['duration'] / 10**9, test["output"])
+    else:
+        # we only add console/journal for failing tests
+        console_txt = None
+        journal_txt = None
+        if host_dir:
+            console_fn = os.path.join(host_dir, "console.txt")
+            if os.path.isfile(console_fn):
+                with open(console_fn, encoding='utf-8') as f:
+                    console_txt = strip_ansi(f.read())
+            journal_fn = os.path.join(host_dir, "journal.txt")
+            if os.path.isfile(journal_fn):
+                with open(journal_fn, encoding='utf-8') as f:
+                    journal_txt = f.read()
+
+        tc = TestCase(test['name'], args.classname + ".tests",
+                      test['duration'] / 10**9, console_txt, journal_txt)
+
+        if test["result"] == 'FAIL':
+            tc.add_failure_info(message="Test failed", output=test["output"])
+        elif test["result"] == 'SKIP':
+            tc.add_skipped_info(message="Test skipped", output=test["output"])
+
+    test_cases.append(tc)
+
+ts = TestSuite(args.classname, test_cases)
+if args.output == '-':
+    TestSuite.to_file(sys.stdout, [ts])
+else:
+    with open(args.outputxml, "w", encoding='utf-8') as f:
+        TestSuite.to_file(f, [ts])