diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ad9b767e8b..c9b03bf692 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -74,6 +74,9 @@ jobs: - { dc: ldc-master, do_test: true } # Test on ARM64 - { os: macOS-14, dc: ldc-latest, do_test: true } + # ice when building tests: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=119817 + - { os: ubuntu-24.04, dc: gdc-13, do_test: false } + - { os: ubuntu-24.04, dc: gdc-14, do_test: true } exclude: # Error with those versions: # ld: multiple errors: symbol count from symbol table and dynamic symbol table differ in [.../dub.o]; address=0x0 points to section(2) with no content in '[...]/osx/lib/libphobos2.a[3177](config_a68_4c3.o)' @@ -96,14 +99,29 @@ jobs: - name: '[Linux] Install dependencies' if: runner.os == 'Linux' run: | - sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev netcat + sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev netcat-openbsd # Compiler to test with - name: Prepare compiler - uses: dlang-community/setup-dlang@v1 + uses: dlang-community/setup-dlang@v2 with: compiler: ${{ matrix.dc }} + - name: Set environment variables + shell: bash + run: | + for name in DC DMD; do + var=${!name} + var=$(basename "${var}") + var=${var%.exe} # strip the extension + export "${name}=${var}" + tee -a ${GITHUB_ENV} <<<"${name}=${var}" + done + + if [[ ${{ matrix.dc }} == gdc-13 ]]; then + tee -a ${GITHUB_ENV} <<<"DFLAGS=-Wno-error" + fi + # Checkout the repository - name: Checkout uses: actions/checkout@v4 @@ -140,6 +158,8 @@ jobs: rm -rf test/use-c-sources fi test/run-unittest.sh + + dub run --root test/run_unittest -- -v fi shell: bash diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine index 1ca99412e5..286d06afbf 100644 --- a/docker/Dockerfile.alpine +++ b/docker/Dockerfile.alpine @@ -7,7 +7,7 @@ ARG DCBIN # Build dub (and install tests dependencies in the process) WORKDIR /root/build/ -RUN apk add --no-cache bash build-base curl curl-dev dtools dub git grep rsync $DCPKG +RUN apk add --no-cache bash build-base curl curl-dev dtools dub git grep rsync lld $DCPKG ADD . /root/build/ RUN dub test --compiler=$DCBIN && dub build --compiler=$DCBIN @@ -20,4 +20,4 @@ ENV DC=$DCBIN # Finally, just run the test-suite WORKDIR /root/build/test/ -ENTRYPOINT [ "/root/build/test/run-unittest.sh" ] +ENTRYPOINT [ "/root/build/bin/dub", "--root", "run_unittest", "--" ] diff --git a/scripts/ci/ci.sh b/scripts/ci/ci.sh index bccfd811e7..4bc8b1c3dd 100755 --- a/scripts/ci/ci.sh +++ b/scripts/ci/ci.sh @@ -2,21 +2,41 @@ set -v -e -o pipefail -vibe_ver=$(jq -r '.versions | .["vibe-d"]' < dub.selections.json) -dub fetch vibe-d@$vibe_ver # get optional dependency -dub test --compiler=${DC} -c library-nonet +testLibraryNonet=1 +if [[ ${DC} =~ gdc|gdmd ]]; then + # ICE with gdc-14 + testLibraryNonet= +fi + +if [[ ${testLibraryNonet} ]]; then + vibe_ver=$(jq -r '.versions | .["vibe-d"]' < dub.selections.json) + dub fetch vibe-d@$vibe_ver # get optional dependency + dub test --compiler=${DC} -c library-nonet --build=unittest +fi export DMD="$(command -v $DMD)" -./build.d -preview=in -w -g -debug +"${DMD}" -run build.d -preview=in -w -g -debug + +if [[ ${testLibraryNoNet} ]]; then + dub test --compiler=${DC} -b unittest-cov +fi if [ "$COVERAGE" = true ]; then # library-nonet fails to build with coverage (Issue 13742) - dub test --compiler=${DC} -b unittest-cov - ./build.d -cov + "${DMD}" -run build.d -cov else - dub test --compiler=${DC} -b unittest-cov - ./build.d + "${DMD}" -run build.d +fi + +# force the creation of the coverage dir +bin/dub --version + +# let the runner add the needed flags, in the case of gdmd +unset DFLAGS +DC=${DMD} dub run --root test/run_unittest -- -v + +if [[ ! ${DC} =~ gdc|gdmd ]]; then + DUB=`pwd`/bin/dub DC=${DC} dub --single ./test/run-unittest.d + DUB=`pwd`/bin/dub DC=${DC} test/run-unittest.sh fi -DUB=`pwd`/bin/dub DC=${DC} dub --single ./test/run-unittest.d -DUB=`pwd`/bin/dub DC=${DC} test/run-unittest.sh diff --git a/test/.gitignore b/test/.gitignore index cfe8d612fc..bc2a8bc176 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -67,3 +67,5 @@ version-filters-source-dep/version-filters-source-dep version-filters/version-filters version-spec/newfoo/foo-test-application version-spec/oldfoo/foo-test-application + +!new_tests/* diff --git a/test/new_tests/.gitignore b/test/new_tests/.gitignore new file mode 100644 index 0000000000..5a6d94c310 --- /dev/null +++ b/test/new_tests/.gitignore @@ -0,0 +1,11 @@ +test.log +*/* +!*/dub.json +!*/dub.sdl +!*/package.json +!*/run.d +!*/run.sh +!*/source +!*/.gitignore +!*/test.config +!extra/* diff --git a/test/new_tests/.no_build b/test/new_tests/.no_build new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/new_tests/0-init-fail-json/dub.sdl b/test/new_tests/0-init-fail-json/dub.sdl new file mode 100644 index 0000000000..66dff5fd4a --- /dev/null +++ b/test/new_tests/0-init-fail-json/dub.sdl @@ -0,0 +1,2 @@ +name "0-init-fail-json" +dependency "common" path="../common" diff --git a/test/new_tests/0-init-fail-json/source/app.d b/test/new_tests/0-init-fail-json/source/app.d new file mode 100644 index 0000000000..3879a868aa --- /dev/null +++ b/test/new_tests/0-init-fail-json/source/app.d @@ -0,0 +1,21 @@ +import std.file : exists, remove; +import std.path : buildPath; +import std.process : environment, spawnProcess, wait; + +import common; + +void main() +{ + enum packname = "0-init-fail-pack"; + immutable deps = ["logger", "PACKAGE_DONT_EXIST"]; // would be very unlucky if it does exist... + + if (!spawnProcess([dub, "init", "-n", packname] ~ deps ~ [ "-f", "json"]).wait) + die("Init with unknown non-existing dependency expected to fail"); + + const filepath = buildPath(packname, "dub.json"); + if (filepath.exists) + { + remove(packname); + die(filepath, " was not created"); + } +} diff --git a/test/new_tests/0-init-fail/dub.sdl b/test/new_tests/0-init-fail/dub.sdl new file mode 100644 index 0000000000..465201c13c --- /dev/null +++ b/test/new_tests/0-init-fail/dub.sdl @@ -0,0 +1,2 @@ +name "0-init-fail" +dependency "common" path="../common" diff --git a/test/new_tests/0-init-fail/source/app.d b/test/new_tests/0-init-fail/source/app.d new file mode 100644 index 0000000000..dfe8c7a1a3 --- /dev/null +++ b/test/new_tests/0-init-fail/source/app.d @@ -0,0 +1,21 @@ +import std.file : exists, remove; +import std.path : buildPath; +import std.process : environment, spawnProcess, wait; + +import common; + +void main() +{ + enum packname = "0-init-fail-pack"; + immutable deps = ["logger", "PACKAGE_DONT_EXIST"]; // would be very unlucky if it does exist... + + if (!spawnProcess([dub, "init", "-n", packname] ~ deps).wait) + die("Init with unknown non-existing dependency expected to fail"); + + const filepath = buildPath(packname, "dub.sdl"); + if (filepath.exists) + { + remove(packname); + die(filepath ~ " was not created"); + } +} diff --git a/test/new_tests/0-init-interactive/.gitignore b/test/new_tests/0-init-interactive/.gitignore new file mode 100644 index 0000000000..4f2f404e33 --- /dev/null +++ b/test/new_tests/0-init-interactive/.gitignore @@ -0,0 +1,2 @@ +!/exp +!/exp/* \ No newline at end of file diff --git a/test/new_tests/0-init-interactive/dub.sdl b/test/new_tests/0-init-interactive/dub.sdl new file mode 100644 index 0000000000..ccd2866ae8 --- /dev/null +++ b/test/new_tests/0-init-interactive/dub.sdl @@ -0,0 +1,2 @@ +name "0-init-interactive" +dependency "common" path="../common" diff --git a/test/new_tests/0-init-interactive/exp/default_name.dub.sdl b/test/new_tests/0-init-interactive/exp/default_name.dub.sdl new file mode 100644 index 0000000000..34f3c864f1 --- /dev/null +++ b/test/new_tests/0-init-interactive/exp/default_name.dub.sdl @@ -0,0 +1,5 @@ +name "new-package" +description "desc" +authors "author" +copyright "copy" +license "gpl" diff --git a/test/new_tests/0-init-interactive/exp/dub.json b/test/new_tests/0-init-interactive/exp/dub.json new file mode 100644 index 0000000000..0901ce4fb6 --- /dev/null +++ b/test/new_tests/0-init-interactive/exp/dub.json @@ -0,0 +1,9 @@ +{ + "description": "desc", + "license": "gpl", + "authors": [ + "author" + ], + "copyright": "copy", + "name": "test" +} \ No newline at end of file diff --git a/test/new_tests/0-init-interactive/exp/dub.sdl b/test/new_tests/0-init-interactive/exp/dub.sdl new file mode 100644 index 0000000000..3eaf63ce6b --- /dev/null +++ b/test/new_tests/0-init-interactive/exp/dub.sdl @@ -0,0 +1,5 @@ +name "test" +description "desc" +authors "author" +copyright "copy" +license "gpl" diff --git a/test/new_tests/0-init-interactive/exp/license_gpl3.dub.sdl b/test/new_tests/0-init-interactive/exp/license_gpl3.dub.sdl new file mode 100644 index 0000000000..8b2979980c --- /dev/null +++ b/test/new_tests/0-init-interactive/exp/license_gpl3.dub.sdl @@ -0,0 +1,5 @@ +name "test" +description "desc" +authors "author" +copyright "copy" +license "GPL-3.0-only" diff --git a/test/new_tests/0-init-interactive/exp/license_mpl2.dub.sdl b/test/new_tests/0-init-interactive/exp/license_mpl2.dub.sdl new file mode 100644 index 0000000000..b2a5ee4221 --- /dev/null +++ b/test/new_tests/0-init-interactive/exp/license_mpl2.dub.sdl @@ -0,0 +1,5 @@ +name "test" +description "desc" +authors "author" +copyright "copy" +license "MPL-2.0" diff --git a/test/new_tests/0-init-interactive/exp/license_proprietary.dub.sdl b/test/new_tests/0-init-interactive/exp/license_proprietary.dub.sdl new file mode 100644 index 0000000000..166cc6c709 --- /dev/null +++ b/test/new_tests/0-init-interactive/exp/license_proprietary.dub.sdl @@ -0,0 +1,5 @@ +name "test" +description "desc" +authors "author" +copyright "copy" +license "proprietary" diff --git a/test/new_tests/0-init-interactive/source/app.d b/test/new_tests/0-init-interactive/source/app.d new file mode 100644 index 0000000000..740e2b9f4b --- /dev/null +++ b/test/new_tests/0-init-interactive/source/app.d @@ -0,0 +1,63 @@ +import common; + +void main() +{ + runTest("1\ntest\ndesc\nauthor\ngpl\ncopy\n\n", "dub.sdl"); + runTest("3\n1\ntest\ndesc\nauthor\ngpl\ncopy\n\n", "dub.sdl"); + runTest("sdl\ntest\ndesc\nauthor\ngpl\ncopy\n\n", "dub.sdl"); + runTest("sdlf\n1\ntest\ndesc\nauthor\ngpl\ncopy\n\n", "dub.sdl"); + runTest("1\n\ndesc\nauthor\ngpl\ncopy\n\n", "default_name.dub.sdl"); + runTest("2\ntest\ndesc\nauthor\ngpl\ncopy\n\n", "dub.json"); + runTest("\ntest\ndesc\nauthor\ngpl\ncopy\n\n", "dub.json"); + runTest("1\ntest\ndesc\nauthor\n6\n3\ncopy\n\n", "license_gpl3.dub.sdl"); + runTest("1\ntest\ndesc\nauthor\n9\n3\ncopy\n\n", "license_mpl2.dub.sdl"); + runTest("1\ntest\ndesc\nauthor\n21\n6\n3\ncopy\n\n", "license_gpl3.dub.sdl"); + runTest("1\ntest\ndesc\nauthor\n\ncopy\n\n", "license_proprietary.dub.sdl"); +} + +void runTest(string input, string expectedPath) { + import std.array; + import std.algorithm; + import std.range; + import std.process; + import std.file; + import std.path; + import std.string; + + immutable test = baseName(expectedPath); + immutable dub_ext = expectedPath[expectedPath.lastIndexOf(".") + 1 .. $]; + + const dir = "new-package"; + + if (dir.exists) rmdirRecurse(dir); + auto pipes = pipeProcess([dub, "init", dir], + Redirect.stdin | Redirect.stdout | Redirect.stderrToStdout); + scope(success) rmdirRecurse(dir); + + immutable escapedInput = format("%(%s%)", [input]); + pipes.stdin.writeln(input); + pipes.stdin.close(); + if (pipes.pid.wait != 0) { + die("Dub failed to generate init file for " ~ escapedInput); + } + + scope(failure) { + logError("You can find the generated files in ", absolutePath(dir)); + } + + if (!exists(dir ~ "/dub." ~ dub_ext)) { + logError("No dub." ~ dub_ext ~ " file has been generated for test " ~ test); + logError("with input " ~ escapedInput ~ ". Output:"); + foreach (line; pipes.stdout.byLine) + logError(line); + die("No dub." ~ dub_ext ~ " file has been found"); + } + + immutable got = readText(dir ~ "/dub." ~ dub_ext).replace("\r\n", "\n"); + immutable expPath = "exp/" ~ expectedPath; + immutable exp = expPath.readText.replace("\r\n", "\n"); + + if (got != exp) { + die("Contents of generated dub." ~ dub_ext ~ " does not match " ~ expPath); + } +} diff --git a/test/new_tests/0-init-multi-json/dub.sdl b/test/new_tests/0-init-multi-json/dub.sdl new file mode 100644 index 0000000000..6de210c108 --- /dev/null +++ b/test/new_tests/0-init-multi-json/dub.sdl @@ -0,0 +1,2 @@ +name "0-init-multi-json" +dependency "common" path="../common" diff --git a/test/new_tests/0-init-multi-json/source/app.d b/test/new_tests/0-init-multi-json/source/app.d new file mode 100644 index 0000000000..8d1a828f7b --- /dev/null +++ b/test/new_tests/0-init-multi-json/source/app.d @@ -0,0 +1,27 @@ +import std.file : exists, readText, rmdirRecurse; +import std.path : buildPath; +import std.process : environment, spawnProcess, wait; + +import common; + +void main() +{ + enum packname = "test-package"; + immutable deps = ["openssl", "logger"]; + enum type = "vibe.d"; + + if(packname.exists) rmdirRecurse(packname); + spawnProcess([dub, "init", "-n", packname ] ~ deps ~ [ "--type", type, "-f", "json"]).wait; + + const filepath = buildPath(packname, "dub.json"); + if (!filepath.exists) + die("dub.json not created"); + + immutable got = readText(filepath); + foreach (dep; deps ~ type) { + import std.algorithm; + if (got.count(dep) != 1) { + die(dep, " not in " ~ filepath); + } + } +} diff --git a/test/new_tests/0-init-multi/dub.sdl b/test/new_tests/0-init-multi/dub.sdl new file mode 100644 index 0000000000..57395d2e1e --- /dev/null +++ b/test/new_tests/0-init-multi/dub.sdl @@ -0,0 +1,2 @@ +name "0-init-multi" +dependency "common" path="../common" diff --git a/test/new_tests/0-init-multi/source/app.d b/test/new_tests/0-init-multi/source/app.d new file mode 100644 index 0000000000..faf4a4206e --- /dev/null +++ b/test/new_tests/0-init-multi/source/app.d @@ -0,0 +1,27 @@ +import std.file : exists, readText, rmdirRecurse; +import std.path : buildPath; +import std.process : environment, spawnProcess, wait; + +import common; + +void main() +{ + enum packname = "test-package"; + immutable deps = ["openssl", "logger"]; + enum type = "vibe.d"; + + if(packname.exists) rmdirRecurse(packname); + spawnProcess([dub, "init", "-n", packname ] ~ deps ~ [ "--type", type, "-f", "sdl"]).wait; + + const filepath = buildPath(packname, "dub.sdl"); + if (!filepath.exists) + die("dub.sdl not created"); + + immutable got = readText(filepath); + foreach (dep; deps ~ type) { + import std.algorithm; + if (got.count(dep) != 1) { + die(dep, " not in " ~ filepath); + } + } +} diff --git a/test/new_tests/0-init-simple-json/dub.sdl b/test/new_tests/0-init-simple-json/dub.sdl new file mode 100644 index 0000000000..205bf62051 --- /dev/null +++ b/test/new_tests/0-init-simple-json/dub.sdl @@ -0,0 +1,2 @@ +name "0-init-simple-json" +dependency "common" path="../common" diff --git a/test/new_tests/0-init-simple-json/source/app.d b/test/new_tests/0-init-simple-json/source/app.d new file mode 100644 index 0000000000..de0d500d2e --- /dev/null +++ b/test/new_tests/0-init-simple-json/source/app.d @@ -0,0 +1,17 @@ +import std.file : exists, readText, rmdirRecurse; +import std.path : buildPath; +import std.process : environment, spawnProcess, wait; + +import common; + +void main() +{ + enum packname = "test-package"; + + if(packname.exists) rmdirRecurse(packname); + spawnProcess([dub, "init", "-n", packname, "-f", "json"]).wait; + + const filepath = buildPath(packname, "dub.json"); + if (!filepath.exists) + die("dub.json not created"); +} diff --git a/test/new_tests/0-init-simple/dub.sdl b/test/new_tests/0-init-simple/dub.sdl new file mode 100644 index 0000000000..b353c56ece --- /dev/null +++ b/test/new_tests/0-init-simple/dub.sdl @@ -0,0 +1,2 @@ +name "0-init-simple" +dependency "common" path="../common" diff --git a/test/new_tests/0-init-simple/source/app.d b/test/new_tests/0-init-simple/source/app.d new file mode 100644 index 0000000000..61fc953fcc --- /dev/null +++ b/test/new_tests/0-init-simple/source/app.d @@ -0,0 +1,17 @@ +import std.file : exists, readText, rmdirRecurse; +import std.path : buildPath; +import std.process : environment, spawnProcess, wait; + +import common; + +void main() +{ + enum packname = "test-package"; + + if(packname.exists) rmdirRecurse(packname); + spawnProcess([dub, "init", "-n", packname, "--format", "sdl"]).wait; + + const filepath = buildPath(packname, "dub.sdl"); + if (!filepath.exists) + die("dub.sdl not created"); +} diff --git a/test/new_tests/README.md b/test/new_tests/README.md new file mode 100644 index 0000000000..e3a424809c --- /dev/null +++ b/test/new_tests/README.md @@ -0,0 +1,305 @@ +# For test writers + +The short version of the process is: + +1. Make a new directory under test +2. (optionally) write a test.config if you need +3. Invoke `dub` in your code with any arguments/paths you need and check the generated files or output; or anything else you need to test. + +Feel free to peek at other tests for inspiration. + +# For test runners + +Run: +``` +DC= ./bin/dub run --root test/run_unittest -- +``` + +Where: +- `` is replaced by your desired D compiler. + The supported variants are: + - `dmd` + - `ldc2` + - `ldmd2` + - `gdmd` (only very recent of the `gdmd` script work, the underlying `gdc` can be older) + + `gdc` is not supported. + +- `` + Can contain the switches: + - `-j` for how many tests to run in parallel + - `-v` in order to show the full output of each test to the console, rather than only *starting* and *finished* lines. The full output is still saved to the `test.log` file so you can safely pass this switch and still have the full output available in case of a failure. + - `--color[=]` To turn on/off color output for log lines and the dub invocations. Note that this leads to color output being saved in the `test.log` file + + You can also pass any amount of glob patterns in order to select which tests to run. + It is an error not to select any tests so if you misspell the pattern the runner will complain. + +As an example, the following invocation: +``` +DC=/usr/bin/dmd-2.111 ./bin/dub run --root test/run_unittest -- -j4 --color=false -v '1-exec-*' +``` +runs the all the tests that match `1-exec-*` (currently those are `1-exec-simple` and `1-exec-simple-package-json`), without color but with full output, with 4 threads. + +# Advanced test writing + +## The `test.config` file + +A summary of all the settings: +``` +# Test requirements +os = [linux, windows, osx] +dc_backend = [gdc, dmd, ldc] +dlang_fe_version_min = 2108 + +# CLI switches passed to dub when running the test +dub_command = build +dub_config = myconfig +dub_build_type = mybuild +extra_dub_args = [ --, -my-app-flag-1 ] + +# Synchronization +locks = [ 1-dynLib-simple ] +must_be_run_alone = true + +# Misc +expect_nonzero = false +``` + +The syntax is very basic: +- empty or whitespace-only lines are ignored +- comments start with `#` and can only be placed at the start of a line +- Other lines are treated as `key = value` assignments. + +The value can be an array denoted by the `[` and `]` characters, with elements separated by commas. +The value can also be a simple string. +Quotes are not supported, nor can you span an array multiple lines. + +The accepted keys are the members of the `TestConfig` struct in [test_config.d](/test/run_unittest/source/run_unittest/test_config.d) + +The accepted values for each setting are based on their D type: +- `enum` accepts any of the names of the `enum`'s members +- `string` accepts any value +- `bool` accept only `true` and `false` + +Arrays accept any number of their element's type. + +As a shorthand, if an array contains only one element, you can skip writing the `[]` around the value. +For example, the following two lines are equivalent: +``` +os = windows +os = [ windows ] +``` + +What follows are detailed descriptions for each setting key: + +#### `os` + +Restricts the test to only run on selected platforms. + +For example: +``` +os = [linux, osx] +``` +will only run the test of `Linux` and `MacOS` platforms. + +### `dc_backend` + +Required that the compiler backend be one of the listed values. + +For example: +``` +dc_backend = [dmd, ldc] +``` +will only run the test with `dmd`, `ldc2`, or `ldmd2`, but not with `gdmd`. + +If you need to disallow `ldc2` but not `ldmd2` then you will need to do so pragmatically inside your test code. +The `common.skip` helper function can be used for this purpose. + +### `dlang_fe_version_min` + +Restrict the compiler frontend version to be higher or equal to the passed value. +The frontend version is the version of the `dmd` code each compiler contains. +For example `gdmd-14` has a FE version of `2.108`. + +Example: +``` +dlang_fe_version_min = 2101 +``` + +Use this setting if you are testing a new feature of the compiler, otherwise try to make your test work with older compilers by not using very recent language features. + +### `dub_command` + +This selects how to run your test. +Possible values are: +- `build` +- `test` +- `run` + +Each value translates to a `dub build`, `dub test`, or, `dub run` invocation. + +This setting is an array so you can pass multiple of the above values, in case you need the test to be built multiple times. + +The default value is `run`. + +For example: +``` +dub_command = build +``` +will not run your test, it will only call `dub build` and interpret a zero exit status as success. + +### `dub_config` + +This selects the package configuration (the `--config` dub switch). + +By default, no value is selected and the switch is not passed to dub. + +For example: +``` +dub_config = myconfig +``` +will run your test with `dub run --config myconfig` + +### `dub_build_type` + +Similarly to `dub_config`, this selects what is passed to the `--build` switch. + +By default, no value is passed. + +For example: +``` +dub_build_type = release +``` +will result in your test being run as `dub run --build release` + +### `extra_dub_args` + +This is a catch-all setting for any specific switches you want to pass to dub. + +For example: +``` +extra_dub_args = [ --, --my-switch ] +``` +will run the test as `dub run -- --my-switch`. + +### `locks` + +This setting is used to prevent tests that use the same resource/dependency from running at the same time. +While the runner tries to isolate each test by passing a specific `DUB_HOME` directory in order to avoid concurrent build of the same (named) package this is not always possible. + +For example, if three tests depend on the same library in `extra/` those could not be run at the same time. +In that scenario, each of those three tests would need to have a `locks` setting with the same value, say `locks = extra/mydep`. +The value doesn't matter, so long as it matches between the three `test.config` files. +Do try, however, to use a self-explanatory name, in order to make it obvious why the tests can't be run in parallel. + +As a special case, the runner always adds the directory name of the test to the `locks` setting to facilitate the few cases in which a test depends on another test. + +For example, if you had two tests `1-lib` and `2-exec-dep-lib`, with `2-exec-dep-lib` having a dependency in its `dub.json` for `1-lib` then you can solve this with a single `test.config`. +It would be placed in the `2-exec-dep-lib` directory and contain: +``` +locks = [ 1-lib ] +``` + +### `must_be_run_alone` + +Similarly to `locks` this setting controls how a test is scheduled with regards to other tests. +It accepts only a `true` or `false` value and, if the value is `true`, like the name suggests, the test will only be run if no other tests are being run. + +It stands to reason that you should only use this setting as a last resort, in case the functionality you are testing actively interferes with the test setup. +An example of such an operation may involve renaming the `dub` executable back and forth. + +Example: +``` +must_be_run_alone = true +``` + +### `expect_nonzero` + +This setting controls the default behavior of deciding the test success/failure based on its exit status. +Normally a zero exit status means that the test completed successfully and a non-zero status means that something failed. +You can switch this behavior with this boolean setting and require that your test exits with a non-zero status in order to be declared successful. + +Note that it is still possible to explicitly fail a test by printing a `[FAIL]: ` line in the output of your program (which is what the `common.die` helper does). +In such a case the test is still marked as a failure, even if `expect_nonzero` is set to `true`. + +Example: +``` +expect_nonzero = true +``` + +## General guarantees + +- `DUB` exists in the environment +- `DC` exists in the environment +- Your test program's working directory is its test folder +- `DUB_HOME` being set and pointing to a test-specific directory. + This allows you to freely build/fetch/remove packages without affecting the user's setup or interfere with other tests. +- `CURR_DIR` exists in the environment and point to the [test](/test) directory + +## General requirements + +### Try to respect `DFLAGS` + +Try to respect the `DFLAGS` environment variable and not overwrite it, as it is meant for users to pass arguments possibly required by their setup. + +If you test fails with any `DFLAGS` then it is acceptable to delete its value. + +### Don't overwrite `etc/dub/settings.json` + +This path, relative to the root of this repository, is meant for users to control `dub` settings. + +### Avoid short names for packages + +Don't have top-level packages (i.e. directly inside [test](/test)) with short or common names. +If two test have the same name (for example `test`) they risk being built at the same time and trigger race conditions between compiler processes. +Use names like `issue1202-test` and `issue1404-test`. + +Note that it is fine to use names such as `test` when generating or building packages from inside your test, since at that point the test will have a separate `DUB_HOME` which will be local to your test so no conflicts can arise. + +## Other notes + +### Output format + +The test runner picks up lines that start with: +- `[INFO]: ` +- `[WARN]: ` +- `[ERROR]: ` +- `[FAIL]: ` +- `[SKIP]: ` + +and either prints them with possible color or it marks the test as failed or skipped. + +The `common` package provides convenience wrappers for these but you're free to print them directly if its easier. + +`[FAIL]:` and `[SKIP]:` use the remaining portion of the line to tell the user why the test was skipped so try to print something meaningful. + +### Directory structure + +The common pattern is that each test is a folder inside `/test/`. +If your test needs some static files they are usually placed inside `sample/`. +If your test dynamically generated some data it is usually placed in a local `test/` subdirectory (for example `/test/custom-unittest/test`). +A `dub` subdirectory inside each test directory is also generated and `DUB_HOME` is set to point to it when the test is run. + +### .gitignore usage + +The default policy is black-list all, white-list as needed. +Try to follow this when you unmask your test's files, which you probably have to do when adding anything other that a `dub.json` and a `source/` directory. + +### cleaning up garbage files + +It's fine if your tests leave temporary files laying around in git-ignored paths. +You don't have to explicitly clean up everything as the user is entrusted to run `git clean -fdx` if they want to get rid of all the junk. + +It is, however, important to perform all the necessary cleanup at the start of your test. +You can't assume that a previous invocation completed successfully or unsuccessfully so try to always start with a clean environment and manually reset all generated files or directories. + +# Advanced test running + +You can configure setting with either the `DFLAGS` environment variable or the `etc/dub/settings.json` file (relative to the root of this repository) + +If you change `DFLAGS` take a note that `gdmd` may fail to build some tests unless you pass it `-q,-Wno-error -allinst`, so be sure to also include these flags. + +The `dub/settings.json` file can be used to configure custom package registries which would allow you to run (some of) the tests without internet access. +It can also give you control of all the tests' inputs. +However, a few tests do fail without internet access and which packages would need to be manually downloaded is not clearly stated. +With some hacking it can be done but if you rely on this functionality feel free to open an issue if you want the situation to improve. diff --git a/test/new_tests/common/dub.json b/test/new_tests/common/dub.json new file mode 100644 index 0000000000..f0da51915d --- /dev/null +++ b/test/new_tests/common/dub.json @@ -0,0 +1,5 @@ +{ + "name": "common", + "license": "MIT", + "targetType": "sourceLibrary" +} diff --git a/test/new_tests/common/source/common.d b/test/new_tests/common/source/common.d new file mode 100644 index 0000000000..db73c5e12a --- /dev/null +++ b/test/new_tests/common/source/common.d @@ -0,0 +1,142 @@ +module common; + +import core.stdc.stdio; +import std.parallelism; +import std.process; +import std.stdio : File; +import std.string; + +void log (const(char)[][] args...) { + printImpl("INFO", args); +} + +void logError (const(char)[][] args...) { + printImpl("ERROR", args); +} + +void die (const(char)[][] args...) { + printImpl("FAIL", args); + throw new Exception("test failed"); +} + +void skip (const(char)[][] args...) { + printImpl("SKIP", args); + throw new Exception("test skipped"); +} + +version(Posix) +immutable DotExe = ""; +else +immutable DotExe = ".exe"; + +immutable string dub; +immutable string dubHome; + +shared static this() { + import std.file; + import std.path; + dub = environment["DUB"]; + dubHome = getcwd.buildPath("dub"); + environment["DUB_HOME"] = dubHome; +} + +struct ProcessT { + string stdout(){ + if (stdoutTask is null) + throw new Exception("Trying to access stdout but it wasn't redirected"); + return stdoutTask.yieldForce; + } + string stderr(){ + if (stderrTask is null) + throw new Exception("Trying to access stderr but it wasn't redirected"); + return stderrTask.yieldForce; + } + string[] stdoutLines() { + return stdout.splitLines(); + } + string[] stderrLines() { + return stderr.splitLines(); + } + File stdin() { return p.stdin; } + + int wait() { + return p.pid.wait; + } + + Pid pid() { return p.pid; } + + this(ProcessPipes p, Redirect redirect, bool quiet = false) { + this.p = p; + this.redirect = redirect; + this.quiet = quiet; + + if (redirect & Redirect.stdout) { + this.stdoutTask = task!linesImpl(p.stdout, quiet); + this.stdoutTask.executeInNewThread(); + } + if (redirect & Redirect.stderr) { + this.stderrTask = task!linesImpl(p.stderr, quiet); + this.stderrTask.executeInNewThread(); + } + } + + ~this() { + if (stdoutTask) + stdoutTask.yieldForce; + if (stderrTask) + stderrTask.yieldForce; + } + + ProcessPipes p; +private: + Task!(linesImpl, File, bool)* stdoutTask; + Task!(linesImpl, File, bool)* stderrTask; + + Redirect redirect; + bool quiet; + bool stdoutDone; + bool stderrDone; + + static string linesImpl(File file, bool quiet) { + import std.typecons; + + string result; + foreach (line; file.byLine(Yes.keepTerminator)) { + if (!quiet) + log(line.chomp); + result ~= line; + } + file.close(); + return result; + } +} + +ProcessT teeProcess( + const string[] args, + Redirect redirect = Redirect.all, + const string[string] env = null, + Config config = Config.none, + const char[] workDir = null, +) { + return ProcessT(pipeProcess(args, redirect, env, config, workDir), redirect); +} + +ProcessT teeProcessQuiet( + const string[] args, + Redirect redirect = Redirect.all, + const string[string] env = null, + Config config = Config.none, + const char[] workDir = null, +) { + return ProcessT(pipeProcess(args, redirect, env, config, workDir), redirect, true); +} + +private: + +void printImpl (string header, const(char)[][] args...) { + printf("[%.*s]: ", cast(int)header.length, header.ptr); + foreach (arg; args) + printf("%.*s", cast(int)arg.length, arg.ptr); + fputc('\n', stdout); + fflush(stdout); +} diff --git a/test/new_tests/extra/.gitignore b/test/new_tests/extra/.gitignore new file mode 100644 index 0000000000..7eb300fdb3 --- /dev/null +++ b/test/new_tests/extra/.gitignore @@ -0,0 +1,7 @@ +* +!/*.d +!/*/ +!/*/dub.json +!/*/dub.sdl +!/*/source/ +!/*/.gitignore \ No newline at end of file diff --git a/test/run_unittest/dub.json b/test/run_unittest/dub.json new file mode 100644 index 0000000000..0832001c31 --- /dev/null +++ b/test/run_unittest/dub.json @@ -0,0 +1,5 @@ +{ + "description": "Dub test runner", + "license": "MIT", + "name": "run_unittest" +} diff --git a/test/run_unittest/source/app.d b/test/run_unittest/source/app.d new file mode 100644 index 0000000000..a451e4b47c --- /dev/null +++ b/test/run_unittest/source/app.d @@ -0,0 +1,48 @@ +module app; + +import run_unittest.log; +import run_unittest.runner; + +import std.file; +import std.getopt; +import std.stdio; +import std.path; + +int main(string[] args) { + bool verbose; + bool color; + int jobs; + version(Posix) + color = true; + + auto help = getopt(args, + "v|verbose", &verbose, + "color", &color, + "j|jobs", &jobs, + ); + if (help.helpWanted) { + defaultGetoptPrinter(`run_unittest [-v|--verbose] [--color] [-j|--jobs] [...] + + are shell globs matching directory names under test/ +`, help.options); + return 0; + } + + auto testDir = __FILE_FULL_PATH__.dirName.dirName.dirName.buildPath("new_tests"); + chdir(testDir); + + ErrorSink sink; + { + ErrorSink fileSink = new FileSink("test.log"); + ErrorSink consoleSink = new ConsoleSink(color); + if (!verbose) + consoleSink = new NonVerboseSink(consoleSink); + sink = new GroupSink(fileSink, consoleSink); + } + auto config = generateRunnerConfig(sink); + config.color = color; + config.jobs = jobs; + + auto runner = Runner(config, sink); + return runner.run(args[1 .. $]); +} diff --git a/test/run_unittest/source/run_unittest/log.d b/test/run_unittest/source/run_unittest/log.d new file mode 100644 index 0000000000..fc6c9b516f --- /dev/null +++ b/test/run_unittest/source/run_unittest/log.d @@ -0,0 +1,197 @@ +module run_unittest.log; + +import std.conv; +import std.format; +import std.stdio; + +enum Severity { + Info, + Warning, + Error, + Status, +} + +string getName(Severity severity) { + final switch (severity) { + case Severity.Warning: + return "WARN"; + case Severity.Info: + return "INFO"; + case Severity.Error: + return "ERROR"; + case Severity.Status: + return "STAT"; + } +} + +abstract class ErrorSink { + void info (const(char)[] msg) { + log(Severity.Info, msg); + } + void warn (const(char)[] msg) { + log(Severity.Warning, msg); + } + void error (const(char)[] msg) { + log(Severity.Error, msg); + } + void status (const(char)[] msg) { + log(Severity.Status, msg); + } + + void info (Args...) (Args args) { + log(Severity.Info, text(args)); + } + void warn (Args...) (Args args) { + log(Severity.Warning, text(args)); + } + void error (Args...) (Args args) { + log(Severity.Error, text(args)); + } + void status (Args...) (Args args) { + log(Severity.Status, text(args)); + } + + abstract void log(Severity severity, const(char)[] msg); + void log (Args...) (Severity severity, Args args) { + log(severity, text(args)); + } +} + +class ConsoleSink : ErrorSink { + this(bool useColor) { + this.useColor = useColor; + } + + override void log (Severity severity, const(char)[] msg) { + immutable preamble = severity.getName; + immutable color = getColor(severity); + + immutable colorBegin = useColor ? "\033[0;" ~ color ~ "m" : ""; + immutable colorEnd = useColor ? "\033[0;m" : ""; + immutable str = format("%s[%5s]:%s %s", colorBegin, preamble, colorEnd, msg); + + stderr.writeln(str); + stderr.flush(); + } + +private: + bool useColor; + + enum AnsiColor { + Red = "31", + Green = "32", + Yellow = "33", + } + + string getColor(Severity severity) { + final switch (severity) { + case Severity.Warning: + return AnsiColor.Yellow; + case Severity.Info: + return AnsiColor.Green; + case Severity.Error: + return AnsiColor.Red; + case Severity.Status: + return AnsiColor.Green; + } + } +} + +class FileSink : ErrorSink { + this(string logFile) { + this.logFile = File(logFile, "w"); + } + + override void log (Severity severity, const(char)[] msg) { + immutable preamble = severity.getName; + immutable str = format("[%5s]: %s", preamble, msg); + logFile.writeln(str); + } + +private: + File logFile; +} + +class GroupSink : ErrorSink { + this (ErrorSink[] sinks...) { + this.sinks = sinks.dup; + } + + override void log (Severity severity, const(char)[] msg) { + foreach (sink; sinks) + sink.log(severity, msg); + } + +private: + ErrorSink[] sinks; +} + +class TestCaseSink : ErrorSink { + ErrorSink proxy; + string tc; + this(ErrorSink proxy, string testCase) { + this.proxy = proxy; + tc = testCase; + } + + override void log(Severity severity, const(char)[] msg) { + proxy.log(severity, tc, ": ", msg); + } +} + +class CaptureErrorSink : ErrorSink { + const(char)[][][Severity] capturedMessages; + + override void log (Severity severity, const(char)[] msg) { + capturedMessages[severity] ~= msg; + } + + override string toString() const { + import std.conv; + return text(capturedMessages); + } + + bool empty() { + int result = 0; + foreach (value; capturedMessages) + result += value.length; + return result == 0; + } + void clear() { + foreach (ref value; capturedMessages) value = []; + } + + const(char)[][] errors () { + return capturedMessages[Severity.Error]; + } + const(char)[] errorsBlock () { + return block(Severity.Error); + } + const(char)[] warningsBlock () { + return block(Severity.Warning); + } + const(char)[] infosBlock () { + return block(Severity.Info); + } + const(char)[] statusBlock () { + return block(Severity.Status); + } + +private: + const(char)[] block(Severity severity) { + import std.array; + return capturedMessages.get(severity, []).join(" "); + } +} + +class NonVerboseSink : ErrorSink { + public ErrorSink sink; + this(ErrorSink sink) { + this.sink = sink; + } + + override void log(Severity severity, const(char)[] msg) { + if (severity == Severity.Info) return; + sink.log(severity, msg); + } +} diff --git a/test/run_unittest/source/run_unittest/runner/config.d b/test/run_unittest/source/run_unittest/runner/config.d new file mode 100644 index 0000000000..eb3d865470 --- /dev/null +++ b/test/run_unittest/source/run_unittest/runner/config.d @@ -0,0 +1,185 @@ +module run_unittest.runner.config; + +import run_unittest.test_config; +import run_unittest.log; + +import core.sync.mutex; +import std.algorithm; + +struct RunnerConfig { + Os os; + DcBackend dc_backend; + FeVersion dlang_fe_version; + + string dubPath; + string dc; + bool color; + int jobs; +} + +RunnerConfig generateRunnerConfig(ErrorSink sink) { + import std.process; + + RunnerConfig config; + + version(linux) config.os = Os.linux; + else version(Windows) config.os = Os.windows; + else version(OSX) config.os = Os.osx; + else static assert(false, "Unknown target OS"); + + version(DigitalMars) config.dc_backend = DcBackend.dmd; + else version(LDC) config.dc_backend = DcBackend.ldc; + else version(GNU) config.dc_backend = DcBackend.gdc; + else static assert(false, "Unknown compiler"); + { + auto envDc = environment.get("DC"); + if (envDc.length == 0) { + sink.warn("The DC environment is empty. Defaulting to dmd"); + envDc = "dmd"; + } + + handleEnvironmentDc(envDc, config.dc_backend, sink); + config.dc = envDc; + } + + import std.format; + config.dlang_fe_version = __VERSION__; + { + const envFe = environment.get("FRONTEND"); + handleEnvironmentFrontend(envFe, sink); + } + + import std.path; + immutable fallbackPath = buildNormalizedPath(absolutePath("../../bin/dub")); + config.dubPath = environment.get("DUB", fallbackPath); + + return config; +} + + +private: + +void handleEnvironmentFrontend(string envFe, ErrorSink errorSink) { + if (envFe.length == 0) return; + + errorSink.warn("The FRONTEND environment variable is ignored and this script will compute it by itself"); + errorSink.warn("You can safely remove the variable from the environment"); +} + +unittest { + auto sink = new CaptureErrorSink(); + handleEnvironmentFrontend(null, sink); + assert(sink.empty()); +} + +unittest { + auto sink = new CaptureErrorSink(); + handleEnvironmentFrontend("", sink); + assert(sink.empty()); +} + +unittest { + auto sink = new CaptureErrorSink(); + handleEnvironmentFrontend("2109", sink); + assert(!sink.empty()); + assert(sink.warningsBlock.canFind("FRONTEND")); +} + +void handleEnvironmentDc(string envDc, DcBackend thisDc, ErrorSink sink) { + import std.path; + + const dcBasename = baseName(envDc); + DcBackend dcBackendGuess; + if (dcBasename.canFind("gdmd")) + dcBackendGuess = DcBackend.gdc; + else if (dcBasename.canFind("gdc")) { + sink.error("Running the testsuite with plain gdc is not supported."); + sink.error("Please use (an up-to-date) gdmd instead."); + throw new Exception("gdc is not supported. Use gdmd"); + } else if (dcBasename.canFind("ldc", "ldmd")) + dcBackendGuess = DcBackend.ldc; + else if (dcBasename.canFind("dmd")) + dcBackendGuess = DcBackend.dmd; + else { + // Dub will fail as well with this + throw new Exception("DC environment variable(" ~ envDc ~ ") does not seem to be a D compiler"); + } + + if (dcBackendGuess != thisDc) { + sink.error("The DC environment is not the same backend as the D compiler"); + sink.error("used to build this script: ", dcBackendGuess, " vs ", thisDc, '.'); + sink.error("If you invoke this script manually make sure you compile this"); + sink.error("script with the same compiler that you will run the tests with."); + + throw new Exception("$DC is not the same compiler as the one used to build this script"); + } +} + +CaptureErrorSink successfullDcTestCase(string env, DcBackend backend) { + auto sink = new CaptureErrorSink(); + try { + handleEnvironmentDc(env, backend, sink); + } catch (Exception e) { + assert(false, sink.toString()); + } + return sink; +} + +CaptureErrorSink unsuccessfullDcTestCase(string env, DcBackend backend) { + auto sink = new CaptureErrorSink(); + try { + handleEnvironmentDc(env, backend, sink); + } catch (Exception e) { + return sink; + } + assert(false, "handleEnvironemntDc did not fail as expected"); +} + +unittest { + auto sink = successfullDcTestCase("/usr/bin/dmd", DcBackend.dmd); + assert(sink.empty, sink.toString); +} +unittest { + auto sink = successfullDcTestCase("dmd", DcBackend.dmd); + assert(sink.empty); +} + +unittest { + successfullDcTestCase("/usr/bin/dmd-2.109", DcBackend.dmd); +} +unittest { + successfullDcTestCase("dmd-2.111", DcBackend.dmd); +} +unittest { + successfullDcTestCase("/bin/gdmd", DcBackend.gdc); +} +unittest { + successfullDcTestCase("x86_64-pc-linux-gnu-gdmd-15", DcBackend.gdc); +} +unittest { + successfullDcTestCase("ldc2", DcBackend.ldc); +} +unittest { + successfullDcTestCase("/usr/local/bin/ldc", DcBackend.ldc); +} +unittest { + successfullDcTestCase("ldmd2-1.37", DcBackend.ldc); +} + +unittest { + unsuccessfullDcTestCase("/usr/bin/true", DcBackend.dmd); +} + +unittest { + auto sink = unsuccessfullDcTestCase("dmd", DcBackend.gdc); + assert(sink.errorsBlock.canFind("dmd")); + assert(sink.errorsBlock.canFind("gdc")); + + assert(sink.errorsBlock.canFind("compile this script with the same compiler")); +} + +unittest { + auto sink = unsuccessfullDcTestCase("gdc-11", DcBackend.gdc); + assert(sink.errorsBlock.canFind("gdc")); + assert(sink.errorsBlock.canFind("gdmd")); +} diff --git a/test/run_unittest/source/run_unittest/runner/individual.d b/test/run_unittest/source/run_unittest/runner/individual.d new file mode 100644 index 0000000000..128187d7c1 --- /dev/null +++ b/test/run_unittest/source/run_unittest/runner/individual.d @@ -0,0 +1,260 @@ +module run_unittest.runner.individual; + +import run_unittest.log; +import run_unittest.runner; +import run_unittest.test_config; + +import std.algorithm; +import std.file; +import std.format; +import std.path; +import std.process; + +struct TestCaseRunner { + string tc; + ErrorSink sink; + Runner runner; + void delegate() testResultAction = null; + + void run() { + initConfig(); + ensureTestCanRun(); + + runner.acquireLocks(testConfig); + scope(exit) runner.releaseLocks(testConfig); + + scope(success) endTest(); + if (testConfig.dub_command.length != 0) { + foreach (dubConfig; ["dub.sdl", "dub.json", "package.json"]) + if (exists(buildPath(tc, dubConfig))) { + runDubTestCase(); + return; + } + } + } + +private: + TestConfig testConfig; + + void ensureTestCanRun() { + import std.array; + import std.format; + + string reason; + static foreach (member; ["dc_backend", "os"]) {{ + const testArray = __traits(getMember, testConfig, member); + const myValue = __traits(getMember, runner.config, member); + + if (testArray.length && !testArray.canFind(myValue)) { + reason = format("our %s (%s) is not in %s", member, myValue, testArray); + } + }} + + if (testConfig.dlang_fe_version_min != 0 + && testConfig.dlang_fe_version_min > runner.config.dlang_fe_version) + reason = format("our frontend version (%s) is lower than the minimum %s", + runner.config.dlang_fe_version, testConfig.dlang_fe_version_min); + + if (reason) + skipTest(reason); + } + + unittest { + import std.exception; + + auto sink = new CaptureErrorSink(); + TestCaseRunner tcr = TestCaseRunner("foo", sink, Runner()); + + tcr.runner.config.dc_backend = DcBackend.dmd; + tcr.runner.config.os = Os.linux; + tcr.testConfig.dc_backend = [ DcBackend.dmd ]; + tcr.testConfig.os = [ Os.linux ]; + + tcr.ensureTestCanRun(); + + tcr.testConfig.dc_backend = [ DcBackend.gdc, DcBackend.ldc, DcBackend.dmd]; + tcr.ensureTestCanRun(); + + tcr.testConfig.dc_backend = [ DcBackend.gdc, DcBackend.ldc ]; + assertThrown!TCSkip(tcr.ensureTestCanRun()); + assert(sink.statusBlock.canFind("dmd")); + assert(sink.statusBlock.canFind("dc_backend")); + sink.clear(); + + tcr.testConfig.dc_backend = [ DcBackend.dmd ]; + tcr.testConfig.os = [ Os.windows ]; + assertThrown!TCSkip(tcr.ensureTestCanRun()); + assert(sink.statusBlock.canFind("linux"), sink.toString()); + assert(sink.statusBlock.canFind("os")); + assert(sink.statusBlock.canFind("windows")); + sink.clear; + + tcr.testConfig.os = [ Os.linux ]; + tcr.testConfig.dlang_fe_version_min = 2100; + tcr.runner.config.dlang_fe_version = 2105; + tcr.ensureTestCanRun(); + + tcr.testConfig.dlang_fe_version_min = 2110; + assertThrown!TCSkip(tcr.ensureTestCanRun()); + assert(sink.statusBlock.canFind("2110")); + assert(sink.statusBlock.canFind("2105")); + sink.clear(); + } + + void initConfig() { + immutable testConfigPath = buildPath(tc, "test.config"); + if (testConfigPath.exists) { + immutable testConfigContents = readText(testConfigPath); + try + testConfig = parseConfig(testConfigContents, sink); + catch (Exception e) + failTest("Could not load test.config:", e); + } + + testConfig.locks ~= tc; + } + + void runDubTestCase() + in(testConfig.dub_command.length != 0) + { + foreach (cmd; getDubCmds()) { + auto env = [ + "DUB": runner.config.dubPath, + "DC": runner.config.dc, + "CURR_DIR": getcwd(), + ]; + immutable redirect = Redirect.stdout | Redirect.stderrToStdout; + + beginTest(cmd); + auto pipes = pipeProcess(cmd, redirect, env, Config.none, tc); + scope(exit) pipes.pid.wait; + + foreach (line; pipes.stdout.byLine) + passthrough(line); + + // Handle possible skips or explicit failures + if (testResultAction) + testResultAction(); + + immutable exitStatus = pipes.pid.wait; + if (testConfig.expect_nonzero) { + if (exitStatus == 0) + failTest("Expected non-0 exit status"); + } else + if (exitStatus != 0) + failTest("Expected 0 exit status"); + } + } + + string[][] getDubCmds() { + string[][] result; + foreach (dub_command; testConfig.dub_command) { + string dubVerb; + sw: final switch (dub_command) { + static foreach (member; __traits(allMembers, DubCommand)) { + case __traits(getMember, DubCommand, member): + dubVerb = member; + break sw; + } + } + + auto now = [runner.config.dubPath, dubVerb, "--force"]; + now ~= ["--color", runner.config.color ? "always" : "never"]; + if (testConfig.dub_build_type !is null) + now ~= ["--build", testConfig.dub_build_type]; + now ~= testConfig.extra_dub_args; + result ~= now; + } + return result; + } + + unittest { + auto tcr = TestCaseRunner(); + tcr.runner.config.dubPath = "dub"; + tcr.runner.config.color = true; + tcr.testConfig.dub_command = [ DubCommand.build ]; + import std.stdio; + assert(tcr.getDubCmds == [ ["dub", "build", "--force", "--color", "always"] ]); + } + + unittest { + auto tcr = TestCaseRunner(); + tcr.runner.config.dubPath = "dub"; + tcr.runner.config.color = false; + tcr.testConfig.dub_command = [ DubCommand.build, DubCommand.test ]; + assert(tcr.getDubCmds == [ + ["dub", "build", "--force", "--color", "never"], + ["dub", "test", "--force", "--color", "never"], + ]); + } + + unittest { + auto tcr = TestCaseRunner(); + tcr.runner.config.dubPath = "dub"; + tcr.runner.config.color = false; + tcr.testConfig.dub_command = [ DubCommand.run ]; + tcr.testConfig.extra_dub_args = [ "--", "--switch" ]; + assert(tcr.getDubCmds == [ + ["dub", "run", "--force", "--color", "never", "--", "--switch"], + ]); + } + + void passthrough(const(char)[] logLine) { + import std.typecons; + import std.string; + alias Tup = Tuple!(string, void delegate(const(char)[])); + auto actions = [ + Tup("ERROR", &sink.error), + Tup("INFO", &sink.info), + Tup("FAIL", (line) { + immutable cpy = line.idup; + testResultAction = () => failTest(cpy); + }), + Tup("SKIP", (line) { + immutable cpy = line.idup; + testResultAction = () => skipTest(cpy); + }), + ]; + + foreach (tup; actions) { + immutable match = "[" ~ tup[0] ~ "]: "; + if (logLine.startsWith(match)) { + const rest = logLine[match.length .. $]; + tup[1](rest); + return; + } + } + sink.info(logLine); + } + + void beginTest(const string[] cmd) { + import std.array; + sink.status("starting: ", cmd.join(" ")); + } + void endTest() { + sink.status("success"); + } + noreturn failTest(const(char)[] reason, in Throwable exception = null) { + sink.error("failed because: ", reason); + + if (exception) { + import std.conv; + sink.error("Error context:"); + foreach (trace; exception.info) + sink.error(trace); + } + + throw new TCFailure(); + } + noreturn skipTest(const(char)[] reason) { + sink.status("skipped because ", reason); + throw new TCSkip(); + } +} + + +class TestResult : Exception { + this() { super(""); } +} +class TCFailure : TestResult {} +class TCSkip : TestResult {} diff --git a/test/run_unittest/source/run_unittest/runner/package.d b/test/run_unittest/source/run_unittest/runner/package.d new file mode 100644 index 0000000000..43a607af87 --- /dev/null +++ b/test/run_unittest/source/run_unittest/runner/package.d @@ -0,0 +1,4 @@ +module run_unittest.runner; + +public import run_unittest.runner.runner; +public import run_unittest.runner.config; diff --git a/test/run_unittest/source/run_unittest/runner/runner.d b/test/run_unittest/source/run_unittest/runner/runner.d new file mode 100644 index 0000000000..a952eebd5f --- /dev/null +++ b/test/run_unittest/source/run_unittest/runner/runner.d @@ -0,0 +1,148 @@ +module run_unittest.runner.runner; + +import run_unittest.test_config; +import run_unittest.runner.config; +import run_unittest.runner.individual; +import run_unittest.log; + +import core.sync.rwmutex; +import core.sync.mutex; +import core.atomic; +import std.array; +import std.algorithm; +import std.file; +import std.format; +import std.path; + +struct Runner { + RunnerConfig config; + ErrorSink sink; + + shared int skippedTests; + shared int failedTests; + shared int successfulTests; + int totalTests () { + return skippedTests.atomicLoad + failedTests.atomicLoad + successfulTests.atomicLoad; + } + + this(RunnerConfig config, ErrorSink sink) { + this.config = config; + this.sink = sink; + + // Get around issue https://github.com/dlang/dmd/issues/17955 + // with older compilers + //locks = new typeof(locks); + locks[""] = null; + locksMutex = new typeof(locksMutex); + lockExclusive = new typeof(lockExclusive); + } + + int run (string[] patterns) { + if (config.dc_backend == DcBackend.gdc) { + import std.process; + if ("DFLAGS" !in environment) { + immutable defaultFlags = "-q,-Wno-error -allinst"; + sink.info("Adding ", defaultFlags, " to DFLAGS because gdmd will fail some tests without them"); + environment["DFLAGS"] = defaultFlags; + } + } + + const testCases = dirEntries(".", SpanMode.shallow) + .filter!`a.isDir` + .filter!(a => !canFind(["extra", "common"], a.baseName)) + .filter!(entry => entry.name.matches(patterns)) + .map!(a => a.name.baseName) + .array; + + void runTc(string tc) { + auto tcSink = new TestCaseSink(sink, tc); + auto tcRunner = TestCaseRunner(tc, tcSink, this); + try { + tcRunner.run(); + successfulTests.atomicOp!"+="(1); + } catch (TCFailure) { + failedTests.atomicOp!"+="(1); + } catch (TCSkip) { + skippedTests.atomicOp!"+="(1); + } catch (Exception e) { + tcSink.error("Unexpected exception was thrown: ", e); + failedTests.atomicOp!"+="(1); + } + } + + import std.parallelism; + if (config.jobs != 0) + defaultPoolThreads = config.jobs - 1; + foreach (tc; testCases.parallel) + runTc(tc); + + if (totalTests == 0) { + sink.error("No tests that match your search were found"); + throw new Exception("No tests were run"); + } + + sink.status(format("Summary %s total: %s successful %s failed and %s skipped", + totalTests, successfulTests.atomicLoad, failedTests.atomicLoad, skippedTests.atomicLoad)); + return failedTests != 0; + } + + void acquireLocks (const TestConfig testConfig) { + if (testConfig.must_be_run_alone) { + lockExclusive.writer.lock(); + return; + } + + lockExclusive.reader.lock(); + + const orderedKeys = testConfig.locks.dup.sort.release; + auto ourLocks = new shared(Mutex)[](orderedKeys.length); + + onLocks((locks) { + foreach (i, key; orderedKeys) + ourLocks[i].atomicStore(cast(shared)locks.require(key, new Mutex())); + }); + + foreach (i; 0 .. ourLocks.length) + ourLocks[i].lock; + } + + void releaseLocks (const TestConfig testConfig) { + if (testConfig.must_be_run_alone) { + lockExclusive.writer.unlock(); + return; + } + lockExclusive.reader.unlock(); + + onLocks((locks) { + testConfig.locks.each!(key => locks[key].unlock); + }); + } +private: + shared ReadWriteMutex lockExclusive; + shared Mutex[string] locks; + void onLocks(void delegate(Mutex[string] unsharedLocks) action) { + locksMutex.lock; + scope(exit) locksMutex.unlock; + action(cast(Mutex[string])locks); + } + + shared Mutex locksMutex; +} + +private: + +bool matches(string dir, string[] patterns) { + if (patterns.length == 0) return true; + + foreach (pat; patterns) + if (globMatch(baseName(dir), pat)) return true; + return false; +} + +unittest { + assert(matches("./foo", [])); + assert(matches("./foo", ["foo"])); + assert(matches("./foo", ["f*"])); + assert(!matches("./foo", ["bar"])); + assert(matches("./foo", ["b", "f*"])); +} diff --git a/test/run_unittest/source/run_unittest/test_config.d b/test/run_unittest/source/run_unittest/test_config.d new file mode 100644 index 0000000000..510dcad67f --- /dev/null +++ b/test/run_unittest/source/run_unittest/test_config.d @@ -0,0 +1,354 @@ +module run_unittest.test_config; + +import run_unittest.log; + +import std.algorithm; +import std.array; +import std.conv; +import std.string; + +enum Os { + linux, + windows, + osx, +} + +enum DcBackend { + gdc, + dmd, + ldc, +} + +enum DubCommand { + run, + test, + build, + none, +} + +struct FeVersion { + int value; + alias value this; +} + +struct TestConfig { + Os[] os; + DcBackend[] dc_backend; + FeVersion dlang_fe_version_min; + DubCommand[] dub_command = [ DubCommand.run ]; + string dub_config = null; + string dub_build_type = null; + string[] locks; + bool expect_nonzero = false; + + string[] extra_dub_args; + bool must_be_run_alone = false; +} + +TestConfig parseConfig(string content, ErrorSink errorSink) { + TestConfig result; + + bool any_errors = false; + foreach (line; content.lineSplitter) { + line = line.strip(); + if (line.empty) continue; + if (line[0] == '#') continue; + + const split = line.findSplit("="); + if (!split[1].length) { + errorSink.warn("Malformed config line '", line, "'. Missing ="); + continue; + } + + const key = split[0].strip(); + const value = split[2]; + + sw: switch (key) { + static foreach (idx, _; TestConfig.tupleof) { + case TestConfig.tupleof[idx].stringof: + if (!handle(result.tupleof[idx], key, value, errorSink)) + any_errors = true; + break sw; + } + default: + errorSink.error("Setting ", key, " is not recognized.", + " Available settings are: ", + join([__traits(allMembers, TestConfig)], ", ")); + any_errors = true; + break; + } + } + + if (any_errors) + throw new Exception("Config file is not in the correct format"); + + return result; +} + + +private: + +bool handle(T)(ref T field, string memberName, string value, ErrorSink errorSink) + if (is(T == string) || !is(T: V[], V)) +{ + value = value.strip(); + try { + alias TgtType = TargetType!T; + field = to!TgtType(value); + } catch (ConvException e) { + + errorSink.error("Setting ", memberName, " does not recognize value ", value, + ". Possible values are: ", PossibleValues!T); + return false; + } + + static if (is(T == FeVersion)) { + if (field < 2000 || field >= 3000) { + errorSink.error("The value ", value, " for setting ", memberName, " does not respect the format ", PossibleValues!T); + return false; + } + } + return true; +} + +bool handle(T)(ref T[] field, string memberName, string value, ErrorSink errorSink) +if (!is(immutable T == immutable char)) +{ + value = value.strip(); + if ((value[0] != '[') ^ (value[$ - 1] != ']')) { + errorSink.error("Setting ", memberName, " missmatch of [ and ]"); + return false; + } + + string[] values; + if (value[0] == '[') { + assert(value[$ - 1] == ']'); + value = value[1 .. $ - 1]; + values = value.split(','); + } else { + values = [ value ]; + } + + bool any_errors = false; + field = []; + foreach (singleValue; values) { + T thisValue; + if (!handle(thisValue, memberName, singleValue, errorSink)) { + any_errors = true; + continue; + } + field ~= thisValue; + } + + return !any_errors; +} + +template PossibleValues (T) { + static if (is(T == FeVersion)) { + enum PossibleValues = "2XXX"; + } else static if (is(T == string)) { + enum PossibleValues = ""; + } else static if (is(T == bool)) { + enum PossibleValues = "true or false"; + } else { + enum PossibleValues = (){ + string result; + alias members = __traits(allMembers, T); + + result ~= members[0]; + foreach(member; members[1..$]) { + result ~= ", "; + result ~= member; + } + + return result; + }(); + } +} + +template TargetType (T) { + static if (__traits(isScalar, T)) + alias TargetType = T; + else static if (is(T == FeVersion)) + alias TargetType = int; + else static if (is(T == string)) + alias TargetType = T; + else + static assert(false, "Unknown type " ~ T.stringof); +} + + +void parseSuccess(out TestConfig config, out CaptureErrorSink sink, string content) { + sink = new CaptureErrorSink(); + try { + config = parseConfig(content, sink); + } catch (Exception e) { + assert(false, "Parsing failed with error messages: " ~ sink.toString()); + } +} + +void parseFailure(out CaptureErrorSink sink, string content) { + sink = new CaptureErrorSink(); + try { + const _ = parseConfig(content, sink); + } catch (Exception e) { + return; + } + assert(false, "Parsing did not fail as expected"); +} + +unittest { + TestConfig config; + CaptureErrorSink sink; + parseSuccess(config, sink, ` + dub_command = test + os = [ linux,windows, osx] + dc_backend = [dmd, gdc,ldc] + dub_config = cappy-barry + + dlang_fe_version_min = 2108 + # A comment + # and one with spaces + locks=[XX,YY] + + dub_build_type = foo + + expect_nonzero = true + extra_dub_args = [ -f, -b ] + + must_be_run_alone = true + `); + + assert(config.dc_backend == [DcBackend.dmd, DcBackend.gdc, DcBackend.ldc]); + assert(config.os == [Os.linux, Os.windows, Os.osx]); + assert(config.dub_command == [ DubCommand.test ]); + assert(config.dlang_fe_version_min == 2108); + assert(config.dub_config == "cappy-barry"); + assert(config.locks == ["XX", "YY"]); + assert(config.dub_build_type == "foo"); + assert(config.expect_nonzero); + assert(config.extra_dub_args == ["-f", "-b"]); + assert(config.must_be_run_alone); + assert(sink.empty); +} + +unittest { + TestConfig config; + CaptureErrorSink sink; + parseSuccess(config, sink, ` +dub_command = build + +dc_backend = [gdc] +`); + + assert(config.dc_backend == [DcBackend.gdc]); + assert(config.dub_command == [ DubCommand.build ]); + assert(config.os == []); + assert(config.dlang_fe_version_min == 0); + assert(sink.empty); +} + +unittest { + CaptureErrorSink sink; + parseFailure(sink, `dub_command = foo_bar_baz`); + assert(sink.errors.length == 1); + assert(sink.errors[0].canFind("dub_command")); + assert(sink.errors[0].canFind("foo_bar_baz")); + + assert(sink.errors[0].canFind("run")); + assert(sink.errors[0].canFind("build")); + assert(sink.errors[0].canFind("test")); +} + +unittest { + CaptureErrorSink sink; + parseFailure(sink, `strace = [boo]`); + assert(sink.errors.length == 1); + assert(sink.errors[0].canFind("strace")); + + assert(sink.errors[0].canFind("dub_command")); + assert(sink.errors[0].canFind("dlang_fe_version_min")); + assert(sink.errors[0].canFind("os")); + assert(sink.errors[0].canFind("dc_backend")); +} + +unittest { + CaptureErrorSink sink; + parseFailure(sink, `os = linux]`); + assert(sink.errors.length == 1); + assert(sink.errors[0].canFind("os")); +} + +unittest { + CaptureErrorSink sink; + TestConfig config; + parseSuccess(config, sink, `os = [linux, linux]`); + + assert(sink.empty); + assert(config.os == [Os.linux, Os.linux]); +} + +unittest { + CaptureErrorSink sink; + parseFailure(sink, `dc_backend = [gdmd]`); + + assert(sink.errors.length == 1); + assert(sink.errors[0].canFind("dc_backend")); + assert(sink.errors[0].canFind("gdmd")); +} + +unittest { + CaptureErrorSink sink; + parseFailure(sink, `dc_backend = [ldmd2, gdmd, ldc2]`); + + assert(sink.errors.length == 3); + assert(sink.errors[0].canFind("dc_backend")); + assert(sink.errors[0].canFind("ldmd2")); + assert(sink.errors[1].canFind("dc_backend")); + assert(sink.errors[1].canFind("gdmd")); + assert(sink.errors[2].canFind("dc_backend")); + assert(sink.errors[2].canFind("ldc2")); +} + +unittest { + CaptureErrorSink sink; + parseFailure(sink, `dlang_fe_version_min = 2.109`); + + assert(sink.errors.length == 1); + assert(sink.errors[0].canFind("2.109")); + assert(sink.errors[0].canFind("2XXX")); +} + +unittest { + CaptureErrorSink sink; + parseFailure(sink, `dlang_fe_version_min = 2.foo`); + + assert(sink.errors.length == 1); + assert(sink.errors[0].canFind("2.foo")); +} + +unittest { + CaptureErrorSink sink; + parseFailure(sink, `dlang_fe_version_min = garbage`); + + assert(sink.errors.length == 1); + assert(sink.errors[0].canFind("garbage")); +} + +unittest { + TestConfig config; + CaptureErrorSink sink; + parseSuccess(config, sink, `dub_command = [build, test]`); + + assert(config.dub_command == [DubCommand.build, DubCommand.test]); +} + +unittest { + CaptureErrorSink sink; + parseFailure(sink, `expect_nonzero = 1`); + + assert(sink.errors.length == 1); + assert(sink.errors[0].canFind("1")); + assert(sink.errors[0].canFind("true")); + assert(sink.errors[0].canFind("false")); +}