diff --git a/src/main/scala/uk/org/floop/sparqlTestRunner/Run.scala b/src/main/scala/uk/org/floop/sparqlTestRunner/Run.scala
index dda9557..1b46704 100644
--- a/src/main/scala/uk/org/floop/sparqlTestRunner/Run.scala
+++ b/src/main/scala/uk/org/floop/sparqlTestRunner/Run.scala
@@ -141,35 +141,44 @@
case other => other
}
}
- val (error, results) = runTestsUnder(config.dir, config.params, config.limit, queryExecution, config.dir.toPath)
+ // https://llg.cubic.org/docs/junit/ gives details of JUnit XML format, specifically for Jenkins.
+ val timeSuiteStart = System.currentTimeMillis()
+ val (errors, failures, tests, results) = runTestsUnder(config.dir, config.params, config.limit, queryExecution, config.dir.toPath)
for (dir <- Option(config.report.getParentFile)) {
dir.mkdirs
}
val pp = new PrettyPrinter(80, 2)
val pw = new PrintWriter(config.report)
- pw.write(pp.format({results}))
+ pw.write(pp.format(
+
+ {results}
+ ))
pw.close()
- System.exit(if (error && !config.ignoreFail) 1 else 0)
+ System.exit(if (!config.ignoreFail && (errors > 0 || failures > 0)) 1 else 0)
case None =>
}
def runTestsUnder(dir: File, params: Map[String, String], limit: Option[Int],
- queryExecution: Query => QueryExecution, root: Path): (Boolean, NodeSeq) = {
+ queryExecution: Query => QueryExecution, root: Path): (Int, Int, Int, NodeSeq) = {
var testSuites = NodeSeq.Empty
var testCases = NodeSeq.Empty
- var overallError = false
var errors = 0
+ var failures = 0
var skipped = 0
var tests = 0
val timeSuiteStart = System.currentTimeMillis()
- var subSuiteTimes = 0L
for (f <- dir.listFiles) yield {
if (f.isDirectory) {
val subSuiteStart = System.currentTimeMillis()
- val (error, subSuites) = runTestsUnder(f, params, limit, queryExecution, root)
+ val (subErrors, subFailures, subTests, subSuites) = runTestsUnder(f, params, limit, queryExecution, root)
testSuites ++= subSuites
- overallError |= error
- subSuiteTimes += (System.currentTimeMillis() - subSuiteStart)
+ errors += subErrors
+ failures += subFailures
+ tests += subTests
} else if (f.isFile && f.getName.endsWith(".sparql")) {
val timeTestStart = System.currentTimeMillis()
val relativePath = root.relativize(f.toPath).toString
@@ -205,7 +214,7 @@
var results = exec.execSelect()
val nonEmptyResults = results.hasNext
val timeTaken = (System.currentTimeMillis() - timeTestStart).toFloat / 1000
- testCases = testCases ++
+ testCases = testCases ++
{
val out = new ByteArrayOutputStream
ResultSetFormatter.outputAsCSV(out, results)
@@ -214,14 +223,14 @@
if (expect.exists && expect.isFile) {
val expectedResults = new String(Files.readAllBytes(expect.toPath), StandardCharsets.UTF_8)
if (actualResults != expectedResults) {
- errors += 1
+ failures += 1
System.err.println(s"Testcase $comment\nExpected:\n$expectedResults\nActual:\n$actualResults")
-
+
}
} else {
// assume there should be no results
if (nonEmptyResults) {
- errors += 1
+ failures += 1
System.err.println(s"Testcase $comment\nExpected empty result set, got:\n$actualResults")
}
@@ -231,33 +240,31 @@
} else if (query.isAskType) {
val result = exec.execAsk()
val timeTaken = (System.currentTimeMillis() - timeTestStart).toFloat / 1000
- testCases = testCases ++ {
+ testCases = testCases ++ {
if (result) {
- errors += 1
+ failures += 1
System.err.println(s"Testcase $comment\nExpected ASK query to return FALSE")
}}
} else {
skipped += 1
System.out.println(s"Skipped testcase $comment")
- testCases = testCases ++
+ testCases = testCases ++
}
} catch {
case e: Exception =>
testCases = testCases ++
-
+ errors += 1
}
}
}
- if (errors > 0) {
- overallError = true
- }
- val testSuiteTime = (System.currentTimeMillis() - timeSuiteStart - subSuiteTimes).toFloat / 1000
+ val testSuiteTime = (System.currentTimeMillis() - timeSuiteStart).toFloat / 1000
val suiteName = {
val relativeName = root.toAbsolutePath.getParent.relativize(dir.toPath.toAbsolutePath).toString
if (relativeName.length == 0) {
@@ -266,8 +273,13 @@
relativeName
}
}
- (overallError, testSuites ++
+ (errors, failures, tests, testSuites ++
+
{testCases}
)
}