From b833e251674724e88ca191842dd954b2e87abe31 Mon Sep 17 00:00:00 2001 From: Gunther Rademacher Date: Wed, 11 Nov 2026 11:48:19 +0100 Subject: [PATCH] add JUnit test runner for QT4 test suite --- basex-tests/pom.xml | 18 ++ .../main/java/org/basex/tests/w3c/QT3TS.java | 246 ++++++++++++++---- .../java/org/basex/tests/w3c/QT4Test.java | 136 ++++++++++ 3 files changed, 356 insertions(+), 44 deletions(-) create mode 100644 basex-tests/src/test/java/org/basex/tests/w3c/QT4Test.java diff --git a/basex-tests/pom.xml b/basex-tests/pom.xml index 472e11cf7b..c6de7a0a1a 100644 --- a/basex-tests/pom.xml +++ b/basex-tests/pom.xml @@ -57,6 +57,24 @@ org.eclipse.jetty.compression jetty-compression-gzip + + nu.validator + htmlparser + + + de.bottlecaps + markup-blitz + + + com.ibm.icu + icu4j + runtime + + + net.sf.saxon + Saxon-HE + runtime + org.apache.commons commons-math3 diff --git a/basex-tests/src/main/java/org/basex/tests/w3c/QT3TS.java b/basex-tests/src/main/java/org/basex/tests/w3c/QT3TS.java index a5dfc0204d..8b0a4ebce7 100644 --- a/basex-tests/src/main/java/org/basex/tests/w3c/QT3TS.java +++ b/basex-tests/src/main/java/org/basex/tests/w3c/QT3TS.java @@ -74,6 +74,8 @@ public final class QT3TS extends Main { /** Number of ignored queries. */ private int ignored; + /** Current test set. */ + private TestSet current; /** Current base URI. */ private String baseURI; /** Current base directory. */ @@ -126,6 +128,20 @@ public static void main(final String... args) throws Exception { sandpit = SANDPIT; } + /** + * Constructor for running single test cases (e.g., from JUnit). + * @param path path to the test suite + * @param sandbox sandbox directory + * @throws IOException I/O exception + */ + QT3TS(final String path, final String sandbox) throws IOException { + super(new String[0]); + sandpit = new IOFile(sandbox); + basePath = new File(path).getCanonicalPath(); + init(); + catalog(); + } + /** * Constructor for a worker that runs test sets in parallel. * @param main main instance @@ -168,24 +184,14 @@ private void run() throws Exception { init(); parseArgs(); - // output directory of fn:put tests (results/ is reserved for test runs) - final IOFile results = new IOFile(file(false, "results/sandpit")); - results.delete(); - results.md(); - final Performance perf = new Performance(); - final XdmValue doc = new XQuery("doc('" + file(false, CATALOG) + "')", ctx).value(); + final XdmValue doc = catalog(); final String version = asString("*:catalog/@version", doc); Util.println(NL + "QT Test Suite " + version); Util.println("Test directory: " + file(false, ".")); Util.print("Parsing queries"); - for(final XdmItem ienv : new XQuery("*:catalog/*:environment", ctx).context(doc)) - genvs.add(new QT3Env(ctx, ienv)); - - for(final XdmItem item : new XQuery("for $f in //*:test-set/@file return string($f)", - ctx).context(doc)) setFiles.add(item.getString()); // the report is built in the order of the tests if(threads == 1 || report != null) { for(final String file : setFiles) testSet(file); @@ -236,8 +242,50 @@ private void run() throws Exception { } } + close(); + } + + /** + * Closes the context and removes the sandbox. + */ + void close() { ctx.close(); - SANDPIT.delete(); + sandpit.delete(); + } + + /** + * Opens the catalog and loads its global environments and test set files. + * @return catalog document + * @throws BaseXException database exception + */ + private XdmValue catalog() throws BaseXException { + // output directory of fn:put tests (results/ is reserved for test runs) + final IOFile results = new IOFile(file(false, "results/sandpit")); + results.delete(); + results.md(); + final XdmValue doc = new XQuery("doc('" + file(false, CATALOG) + "')", ctx).value(); + for(final XdmItem ienv : new XQuery("*:catalog/*:environment", ctx).context(doc)) + genvs.add(new QT3Env(ctx, ienv)); + + for(final XdmItem item : new XQuery("for $f in //*:test-set/@file return string($f)", + ctx).context(doc)) setFiles.add(item.getString()); + return doc; + } + + /** + * Returns the path to the test suite. + * @return path + */ + String path() { + return basePath; + } + + /** + * Returns the test set files. + * @return files + */ + List testSetFiles() { + return List.of(setFiles.toArray()); } /** @@ -290,32 +338,18 @@ private void parallel() throws Exception { * @throws Exception exception */ private void testSet(final String name) throws Exception { - // release test sets shared by the previous test set - setDatas = null; - sourceDatas.clear(); - final XdmValue doc = new XQuery("doc(' " + file(false, name) + "')", ctx).value(); - final XdmValue set = new XQuery("*:test-set", ctx).context(doc).value(); - final IO base = IO.get(doc.getBaseURI()); - baseURI = base.path(); - baseDir = base.dir(); - - if(!supported(set)) { - final long n = new XQuery("count(*:test-case)", ctx).context(set).value().getInteger(); + final TestSet ts = open(name); + if(!ts.supported) { + final long n = new XQuery("count(*:test-case)", ctx).context(ts.set).value().getInteger(); total += n; ignored += n; } else { - // parse environment of test-set - final ArrayList envs = new ArrayList<>(); - for(final XdmItem ienv : new XQuery("*:environment", ctx).context(set)) - envs.add(new QT3Env(ctx, ienv)); - - if(report != null) report.addSet(asString("@name", set)); + if(report != null) report.addSet(asString("@name", ts.set)); // run all test cases - final Set unsupported = unsupported(set); - for(final XdmItem its : new XQuery("*:test-case", ctx).context(set)) { + for(final XdmItem its : ts.cases()) { try { - testCase(its, envs, unsupported); + testCase(its, ts.envs, ts.unsupported); } catch(final IOException ex) { Util.debug(ex); } @@ -323,6 +357,101 @@ private void testSet(final String name) throws Exception { } } + /** + * A parsed test set. + */ + final class TestSet { + /** Test set. */ + final XdmValue set; + /** Base URI. */ + final String uri; + /** Base directory. */ + final String dir; + /** Supported flag. */ + final boolean supported; + /** Environments (empty if unsupported). */ + final ArrayList envs = new ArrayList<>(); + /** Unsupported test cases (empty if the test set is unsupported). */ + final Set unsupported; + + /** + * Constructor. + * @param doc test set document + * @throws BaseXException database exception + */ + private TestSet(final XdmValue doc) throws BaseXException { + set = new XQuery("*:test-set", ctx).context(doc).value(); + final IO base = IO.get(doc.getBaseURI()); + uri = base.path(); + dir = base.dir(); + supported = supported(set); + if(supported) { + for(final XdmItem ienv : new XQuery("*:environment", ctx).context(set)) + envs.add(new QT3Env(ctx, ienv)); + unsupported = unsupported(set); + } else { + unsupported = Set.of(); + } + } + + /** + * Returns the test cases. + * @return test cases + */ + ArrayList cases() { + final ArrayList list = new ArrayList<>(); + for(final XdmItem item : new XQuery("*:test-case", ctx).context(set)) list.add(item); + return list; + } + + /** + * Checks if a test case is ignored. + * @param test test case + * @return result of check + */ + boolean ignored(final XdmItem test) { + return !supported || unsupported.contains(attribute(test, "name")); + } + + /** + * Runs a test case. + * @param test test case + * @return outcome + * @throws Exception exception + */ + Outcome run(final XdmItem test) throws Exception { + activate(this); + return execute(test, envs); + } + } + + /** + * Opens a test set. + * @param name name of test set + * @return test set + * @throws BaseXException database exception + */ + TestSet open(final String name) throws BaseXException { + final XdmValue doc = new XQuery("doc(' " + file(false, name) + "')", ctx).value(); + final TestSet ts = new TestSet(doc); + activate(ts); + return ts; + } + + /** + * Makes a test set current and releases the data of the previous one. + * @param ts test set + */ + private void activate(final TestSet ts) { + if(current != ts) { + current = ts; + setDatas = null; + sourceDatas.clear(); + } + baseURI = ts.uri; + baseDir = ts.dir; + } + /** * Returns the parsed test sets. * @return databases @@ -359,6 +488,35 @@ private void testCase(final XdmItem test, final ArrayList envs, tested++; + final Outcome outcome = execute(test, envs); + if(slow != null && outcome.nanos > 100000000) slow.put(-outcome.nanos, name); + if(outcome.expected == null) { + right.add(outcome.text).add(NL); + correct++; + } else { + wrong.add(outcome.text).add(NL); + } + if(report != null) report.addTest(name, outcome.expected == null); + } + + /** + * Test case outcome. + * @param text name, query, result (and expected result if wrong) + * @param expected expected result ({@code null} if correct) + * @param nanos run time + */ + record Outcome(String text, String expected, long nanos) { } + + /** + * Runs a test case. + * @param test node + * @param envs environments + * @return outcome + * @throws Exception exception + */ + private Outcome execute(final XdmItem test, final ArrayList envs) throws Exception { + final String name = attribute(test, "name"); + // expected result final XdmItem result = child(test, "result"); final XdmItem first = result != null ? child(result, null) : null; @@ -399,10 +557,7 @@ private void testCase(final XdmItem test, final ArrayList envs, } final String string = queries.toString(); - if(slow != null) { - final long l = perf.nanoRuntime(); - if(l > 100000000) slow.put(-l, name); - } + final long nanos = perf.nanoRuntime(); final String exp = test(returned, expected); final TokenBuilder tmp = new TokenBuilder(); @@ -428,14 +583,8 @@ private void testCase(final XdmItem test, final ArrayList envs, } tmp.add(err ? "Error : " : "Result: ").add(normSpecial(res)).add(NL); - if(exp == null) { - tmp.add(NL); - right.add(tmp.finish()); - correct++; - } else { - wrong.add(tmp.add("Expect: ").add(normSpecial(exp)).add(NL).add(NL).finish()); - } - if(report != null) report.addTest(name, exp == null); + if(exp != null) tmp.add("Expect: ").add(normSpecial(exp)).add(NL); + return new Outcome(tmp.toString(), exp, nanos); } /** @@ -756,6 +905,15 @@ private static String attribute(final XdmItem node, final String name) { return value != null ? string(value) : null; } + /** + * Returns the name of a test case. + * @param test test case + * @return name + */ + static String name(final XdmItem test) { + return attribute(test, "name"); + } + /** * Returns child elements. * @param node node diff --git a/basex-tests/src/test/java/org/basex/tests/w3c/QT4Test.java b/basex-tests/src/test/java/org/basex/tests/w3c/QT4Test.java new file mode 100644 index 0000000000..19e95ab141 --- /dev/null +++ b/basex-tests/src/test/java/org/basex/tests/w3c/QT4Test.java @@ -0,0 +1,136 @@ +package org.basex.tests.w3c; + +import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assumptions.*; + +import java.io.*; +import java.nio.file.*; +import java.util.*; +import java.util.stream.*; + +import org.basex.tests.bxapi.xdm.*; +import org.junit.jupiter.api.*; +import org.junit.jupiter.api.parallel.*; + +/** + * Runs the QT4 test suite: one dynamic container per test set, one test per test case. + * The suite is taken from {@code QT4_PATH}, or from a {@code qt4tests} directory next to the + * root project (the tests are skipped if neither exists). + * For XSLT 4.0 and XSD 1.1, put Saxon-EE before Saxon-HE on the run configuration's classpath. + * All tests share one driver and must run sequentially. + * + *

Run with {@code mvn -f basex-tests/pom.xml -Dtest=QT4Test test}. + * Use {@code -Dqt4.test-set=fn/concat.xml} to select a test set, or a glob such as + * {@code -Dqt4.test-set=fn/*.xml} to select several. Paths are relative to the suite directory. + * A filter that matches no catalog entries fails the test factory.

+ * + * @author BaseX Team, BSD License + */ +@Execution(ExecutionMode.SAME_THREAD) +public class QT4Test { + /** Environment variable with the suite path. */ + private static final String PATH_ENV = "QT4_PATH"; + /** Name of the suite directory next to the root project. */ + private static final String SUITE_DIR = "qt4tests"; + /** System property with a test set glob. */ + private static final String TEST_SET = "qt4.test-set"; + + /** Test suite driver. */ + private static QT3TS driver; + + /** + * Creates the driver; skips all tests if the suite is missing. + * @throws IOException I/O exception + */ + @BeforeAll + public static void setUp() throws IOException { + final String env = System.getenv(PATH_ENV); + final Path path; + if(env != null && !env.isBlank()) { + path = Path.of(env); + assertTrue(isSuite(path), PATH_ENV + " does not point to the QT4 test suite: " + path); + } else { + path = sibling(); + assumeTrue(path != null, "QT4 test suite not found. Set " + PATH_ENV + ", or clone " + + "https://github.com/qt4cg/qt4tests next to the root project."); + } + driver = new QT3TS(path.toString(), Path.of("target", "qt4").toAbsolutePath().toString()); + } + + /** + * Checks if a directory contains the test suite. + * @param dir directory + * @return result of check + */ + private static boolean isSuite(final Path dir) { + return Files.isRegularFile(dir.resolve("catalog.xml")); + } + + /** + * Looks for the suite in the ancestors of the working directory (the working directory is + * the root project or a module; worktrees inside the project are covered as well). + * @return suite directory, or {@code null} + */ + private static Path sibling() { + for(Path dir = Path.of("").toAbsolutePath().getParent(); dir != null; dir = dir.getParent()) { + final Path path = dir.resolve(SUITE_DIR); + if(isSuite(path)) return path; + } + return null; + } + + /** + * Closes the driver. + */ + @AfterAll + public static void tearDown() { + if(driver != null) { + driver.close(); + driver = null; + } + } + + /** + * Creates a container per selected test set. Children are read when the container is executed. + * @return containers + */ + @TestFactory + public Stream tests() { + final String filter = System.getProperty(TEST_SET, "").trim(); + final PathMatcher matcher = filter.isEmpty() ? null : + FileSystems.getDefault().getPathMatcher("glob:" + filter); + final List files = driver.testSetFiles().stream(). + filter(file -> matcher == null || matcher.matches(Path.of(file))).sorted().toList(); + assertFalse(files.isEmpty(), "No QT4 test sets match " + TEST_SET + '=' + filter); + return files.stream().map(file -> DynamicContainer.dynamicContainer(file, + Path.of(driver.path(), file).toUri(), Stream.of(file).flatMap(QT4Test::cases))); + } + + /** + * Creates the tests of a test set, including unsupported cases (aborted via an assumption). + * @param file test set file + * @return tests + */ + private static Stream cases(final String file) { + try { + final QT3TS.TestSet set = driver.open(file); + return set.cases().stream().map(item -> DynamicTest.dynamicTest(QT3TS.name(item), + () -> run(set, item))); + } catch(final IOException ex) { + throw new UncheckedIOException(ex); + } + } + + /** + * Runs a test case. + * @param set test set + * @param item test case + * @throws Exception exception + */ + private static void run(final QT3TS.TestSet set, final XdmItem item) throws Exception { + assumeFalse(set.ignored(item), "Not supported by BaseX."); + final QT3TS.Outcome outcome = set.run(item); + if(outcome.expected() != null) fail(outcome.text() + "File: " + set.uri); + } + +}