Merge pull request #638 from pkgw/executable-kcov

tests: attempt to gather code coverage info in the executable tests

dist/azure-coverage.yml

@@ -28,17 +28,16 @@ steps:
     cargo install --force cargo-kcov
   displayName: Set up code coverage
 
+# As of Rust 1.44, test executables land in target/debug/deps/ instead of
+# target/debug/, which messes up current cargo-kcov (0.5.2) because it tries to
+# search for those executables. Work around with `cp`. One of the `tectonic-*`
+# binaries is the debug executable, which is hard-linked to
+# `target/debug/tectonic`. kcov will erroneously try to run this as a test if we
+# copy it, so we have to make not to do that, which we accomplish with a search
+# based on the hardlink count. Hacky and fragile but this should get us going.
+# Hopefully kcov will get fixed where this will become unneccessary anyway.
 - bash: |
     set -xeuo pipefail
-    # As of Rust 1.44, test executables land in target/debug/deps/ instead of
-    # target/debug/, which messes up current cargo-kcov (0.5.2) because it tries
-    # to search for those executables. Work around with `cp`. One of the
-    # `tectonic-*` binaries is the debug executable, which is hard-linked to
-    # `target/debug/tectonic`. kcov will erroneously try to run this as a test
-    # if we copy it, so we have to make not to do that, which we accomplish with
-    # a search based on the hardlink count. Hacky and fragile but this should
-    # get us going. Hopefully kcov will get fixed where this will become
-    # unneccessary anyway.
     cargo test --no-run
     find target/debug/deps/tectonic-???????????????? -links 2 -print -delete
     cp -vp target/debug/deps/*-???????????????? target/debug/
@@ -51,9 +50,34 @@ steps:
     git show HEAD
   displayName: Make release commit
 
-- bash: cargo kcov --no-clean-rebuild
+- bash: |
+    set -xeuo pipefail
+    p="$(pwd)"
+    cargo kcov --no-clean-rebuild -- --include-pattern="$p/src,$p/tectonic,$p/xdv"
   displayName: cargo kcov
 
+# Our "executable" test executes the actual Tectonic binary. In order to collect
+# coverage information about what happens in those executions, we have special
+# support in the test harness to wrap the invocations in `kcov` calls.
+- bash: |
+    set -xeuo pipefail
+    p="$(pwd)"
+    export TECTONIC_EXETEST_KCOV_RUNNER="kcov --include-pattern=$p/src,$p/tectonic,$p/xdv"
+    cargo test --test executable
+  displayName: Special-case executable tests
+
+# Now, merge all of the coverage reports. `cargo kcov` does this automatically,
+# but it uses an explicit list of coverage runs, so there's no way to get it to
+# include our special executable tests. We just glob everything, which means we
+# have to delete the preexisting merged report.
+- bash: |
+    set -xeou pipefail
+    cd target/cov
+    rm -rf amber.png bcov.css data glass.png index.html index.js* \
+      kcov-merged merged-kcov-output
+    kcov --merge . *
+  displayName: Merge coverage reports
+
 - bash: |
     set -xeuo pipefail
     bash <(curl -s https://codecov.io/bash)

tests/executable.rs

@@ -23,6 +23,7 @@ lazy_static! {
         root.push("tests");
         root
     };
+
     static ref TARGET_RUNNER_WORDS: Vec<String> = {
         // compile-time environment variable from build.rs:
         let target = env!("TARGET").to_owned();
@@ -36,6 +37,17 @@ lazy_static! {
             vec![]
         }
     };
+
+    // Special coverage-collection mode. This implementation is quite tuned for
+    // the Tectonic CI/CD system, so if you're trying to use it manually, expect
+    // some rough edges.
+    static ref KCOV_WORDS: Vec<String> = {
+        if let Ok(runtext) = env::var("TECTONIC_EXETEST_KCOV_RUNNER") {
+            runtext.split_whitespace().map(|x| x.to_owned()).collect()
+        } else {
+            vec![]
+        }
+    };
 }
 
 fn get_plain_format_arg() -> String {
@@ -61,10 +73,32 @@ fn prep_tectonic(cwd: &Path, args: &[&str]) -> Command {
     println!("using tectonic binary at {:?}", tectonic);
     println!("using cwd {:?}", cwd);
 
+    // We may need to wrap the Tectonic invocation. If we're cross-compiling, we
+    // might need to use something like QEMU to actually be able to run the
+    // executable. If we're collecting code coverage information with kcov, we
+    // need to wrap the invocation with that program.
     let mut command = if TARGET_RUNNER_WORDS.len() > 0 {
         let mut cmd = Command::new(&TARGET_RUNNER_WORDS[0]);
         cmd.args(&TARGET_RUNNER_WORDS[1..]).arg(tectonic);
         cmd
+    } else if KCOV_WORDS.len() > 0 {
+        let mut cmd = Command::new(&KCOV_WORDS[0]);
+        cmd.args(&KCOV_WORDS[1..]);
+
+        // Give kcov a directory into which to put its output. We use
+        // mktemp-like functionality to automatically create such directories
+        // uniquely so that we don't have to manually bookkeep. This does mean
+        // that successive runs will build up new data directories indefinitely.
+        let mut root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+        root.push("target");
+        root.push("cov");
+        root.push("exetest.");
+        let tempdir = tempfile::Builder::new().prefix(&root).tempdir().unwrap();
+        let tempdir = tempdir.into_path();
+        cmd.arg(tempdir);
+
+        cmd.arg(tectonic);
+        cmd
     } else {
         Command::new(tectonic)
     };