Add support for "pinned" tests

Whenever we bump the RPM version in cmake, certain tests also need
updating, e.g. those that compare package digests or RPM sonames as
those are subject to the version number.

Up until now, this had to be done manually by running the test-suite,
looking at the failed tests' diffs and fixing up the expected outputs by
hand.

No more!  Separate those tests and add the "make pinned" cmake target to
update them automatically.  This makes use of the new-ish atshell
feature which runs a shell/command (*.sh scripts in tests/pinned/ in
this case) in the standard test environment in order to get the new
outputs.

See the included README for more details.

tests/CMakeLists.txt

@@ -61,6 +61,8 @@ foreach(prg ${TESTPROGS})
 endforeach()
 string(REPLACE ";" " " TESTPROG_NAMES "${TESTPROGS}")
 
+set(PINNED_DIR ${CMAKE_CURRENT_SOURCE_DIR}/pinned)
+
 include(ProcessorCount)
 ProcessorCount(nproc)
 if (nproc GREATER 1)
@@ -127,6 +129,14 @@ add_custom_target(atshell
 	DEPENDS tree
 )
 
+add_custom_target(pinned
+	COMMAND ./mktree atshell /pinned/update
+	COMMAND cp pinned/*.txt ${PINNED_DIR}/
+	COMMAND git add ${PINNED_DIR}/*.txt
+	COMMAND git diff --staged ${PINNED_DIR}/*.txt
+	DEPENDS tree
+)
+
 add_custom_target(shell
 	COMMAND ./mktree shell ||:
 	DEPENDS tree

tests/local.at

@@ -44,6 +44,14 @@ snapshot umount
 AT_CLEANUP
 ])
 
+m4_define([RPMTEST_CHECK_PINNED],[
+RPMTEST_CHECK_UNQUOTED(
+[source /pinned/$1.sh],
+[0],
+[$(cat /pinned/$1.txt)
+])
+])
+
 m4_define([RPMPY_CHECK],[
 AT_SKIP_IF([$PYTHON_DISABLED])
 RPMTEST_CHECK([RPMPY_RUN([$1])], [], [$2], [$3])

tests/mktree.common

@@ -11,6 +11,7 @@ make_install()
     ldconfig -r $DESTDIR @CMAKE_INSTALL_FULL_LIBDIR@
 
     cp -r @CMAKE_CURRENT_SOURCE_DIR@/data $DESTDIR/
+    cp -r @CMAKE_CURRENT_SOURCE_DIR@/pinned $DESTDIR/
 
     mkdir -p $DESTDIR/usr/bin
     cp @TESTPROG_NAMES@ $DESTDIR/usr/bin/

tests/mktree.oci

@@ -83,7 +83,7 @@ case $CMD in
         rpmtests "$@"
     ;;
     atshell)
-        rpmtests --shell
+        rpmtests --shell "$@"
     ;;
     shell)
         rpmtests --shell snapshot shell

tests/pinned/README

@@ -0,0 +1,26 @@
+PINNED TESTS
+============
+
+This directory contains tests whose expected output needs to be "pinned" to a
+specific commit, typically one that bumps the RPM version number.  This output
+usually can't be generated by other means during test runtime so it has to be
+tracked in git alongside the sources and updated whenever a change in those
+triggers a change in the output.
+
+A typical example of such a test is one that covers reproducible package builds
+by clamping file mtimes to SOURCE_DATE_EPOCH and making sure the package digest
+is always the same.  However, since packages carry the version of RPM used to
+build them in a tag, the digests have to be updated whenever the version is.
+
+USAGE
+-----
+
+The tests are used via the m4 macro RPMTEST_CHECK_PINNED([<name>]) where <name>
+is the respective filename (without an extension) found in this directory.  The
+macro then sources the script <name>.sh and compares its standard output with
+the contents of <name>.txt.
+
+To pin the *.txt files to the current commit, run "make pinned" from the build
+directory.  This will run all the *.sh tests, capture their output and store it
+in the corresponding *.txt files in your source directory, then automatically
+git-stage the changes and print the diff (if any).

tests/pinned/rpmlibver.sh

@@ -0,0 +1,2 @@
+cd ${RPMTEST}/${RPMLIBDIR}
+ls lib{rpm,rpmbuild,rpmio,rpmsign}.so.*.*

tests/pinned/rpmlibver.txt

@@ -0,0 +1,4 @@
+librpm.so.10.0.0
+librpmbuild.so.10.0.0
+librpmio.so.10.0.0
+librpmsign.so.10.0.0

tests/pinned/rpmsigdig.sh

@@ -0,0 +1,14 @@
+runroot rpmbuild -bb --quiet \
+	--define "optflags -O2 -g" \
+	--define "_target_platform noarch-linux" \
+	--define "_binary_payload w.ufdio" \
+	--define "_buildhost localhost" \
+	--define "use_source_date_epoch_as_buildtime 1" \
+	--define "source_date_epoch_from_changelog 1" \
+	--define "clamp_mtime_to_source_date_epoch 1" \
+	--define "_use_weak_usergroup_deps 0" \
+	/data/SPECS/attrtest.spec
+for v in SHA256HEADER SHA1HEADER SIGMD5 PAYLOADDIGEST PAYLOADDIGESTALT; do
+    runroot rpm -q --qf "${v}: %{${v}}\n" /build/RPMS/noarch/attrtest-1.0-1.noarch.rpm
+done
+runroot rpmkeys -Kv /build/RPMS/noarch/attrtest-1.0-1.noarch.rpm

tests/pinned/rpmsigdig.txt

@@ -0,0 +1,11 @@
+SHA256HEADER: 8ab715bef152a18bef62df29805b9d1b8f2c5d92955a1a7fa2491967cc17c035
+SHA1HEADER: 7a8ad9bdb9b0910fa54724f7bdac8ac49d992d2f
+SIGMD5: 87da99d1993240e2d698f5b3dbcbd44c
+PAYLOADDIGEST: 116ce41ebb72f1877cda3d7dedaf5b78770e202d6389ade4e415d78548d703a8
+PAYLOADDIGESTALT: 116ce41ebb72f1877cda3d7dedaf5b78770e202d6389ade4e415d78548d703a8
+/build/RPMS/noarch/attrtest-1.0-1.noarch.rpm:
+    Header SHA256 digest: OK
+    Header SHA1 digest: OK
+    Payload SHA256 ALT digest: OK
+    Payload SHA256 digest: OK
+    MD5 digest: OK

tests/pinned/update

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+SCRIPT_DIR="$(dirname $0)"
+mkdir -p pinned
+
+for script in $SCRIPT_DIR/*.sh; do
+    name=$(basename $script)
+    $script > pinned/${name%.sh}.txt
+done

tests/rpmgeneral.at

@@ -39,16 +39,7 @@ RPMTEST_CLEANUP
 AT_SETUP([rpm library version])
 AT_KEYWORDS([basic])
 AT_SKIP_IF(test -f "${RPMTEST}/${RPMLIBDIR}/librpm.a")
-RPMTEST_CHECK([
-cd ${RPMTEST}/${RPMLIBDIR}
-ls lib{rpm,rpmbuild,rpmio,rpmsign}.so.*.*
-],
-[0],
-[librpm.so.10.0.0
-librpmbuild.so.10.0.0
-librpmio.so.10.0.0
-librpmsign.so.10.0.0
-])
+RPMTEST_CHECK_PINNED([rpmlibver])
 RPMTEST_CLEANUP
 
 # ------------------------------

tests/rpmsigdig.at

@@ -156,38 +156,8 @@ RPMTEST_CLEANUP
 # Reproducably build and verify a package
 AT_SETUP([rpmkeys -Kv <unsigned> 2])
 AT_KEYWORDS([rpmkeys digest])
-RPMTEST_CHECK([
 RPMDB_INIT
-
-runroot rpmbuild -bb --quiet \
-	--define "optflags -O2 -g" \
-	--define "_target_platform noarch-linux" \
-	--define "_binary_payload w.ufdio" \
-	--define "_buildhost localhost" \
-	--define "use_source_date_epoch_as_buildtime 1" \
-	--define "source_date_epoch_from_changelog 1" \
-	--define "clamp_mtime_to_source_date_epoch 1" \
-	--define "_use_weak_usergroup_deps 0" \
-	/data/SPECS/attrtest.spec 
-for v in SHA256HEADER SHA1HEADER SIGMD5 PAYLOADDIGEST PAYLOADDIGESTALT; do
-    runroot rpm -q --qf "${v}: %{${v}}\n" /build/RPMS/noarch/attrtest-1.0-1.noarch.rpm
-done
-runroot rpmkeys -Kv /build/RPMS/noarch/attrtest-1.0-1.noarch.rpm
-],
-[0],
-[SHA256HEADER: 8ab715bef152a18bef62df29805b9d1b8f2c5d92955a1a7fa2491967cc17c035
-SHA1HEADER: 7a8ad9bdb9b0910fa54724f7bdac8ac49d992d2f
-SIGMD5: 87da99d1993240e2d698f5b3dbcbd44c
-PAYLOADDIGEST: 116ce41ebb72f1877cda3d7dedaf5b78770e202d6389ade4e415d78548d703a8
-PAYLOADDIGESTALT: 116ce41ebb72f1877cda3d7dedaf5b78770e202d6389ade4e415d78548d703a8
-/build/RPMS/noarch/attrtest-1.0-1.noarch.rpm:
-    Header SHA256 digest: OK
-    Header SHA1 digest: OK
-    Payload SHA256 ALT digest: OK
-    Payload SHA256 digest: OK
-    MD5 digest: OK
-],
-[])
+RPMTEST_CHECK_PINNED([rpmsigdig])
 RPMTEST_CLEANUP
 
 # ------------------------------