Merge bitcoin#21619: ci: Run self-hosted ci

fa41a91 ci: Run self-hosted ci (MarcoFalke)
fa52a40 ci: Make cirrus cache folders relative to cirrus base dir (MarcoFalke)
fa27841 ci: Restart docker before run (MarcoFalke)
fad4f48 ci: [refactor] Create setting for ephemeral config in .cirrus.yml (MarcoFalke)

Pull request description:

  Due to our heavy use of the Cirrus CI community cluster, some tasks may take a long time to get scheduled. While it is possible to use "Compute Credits" to get immediately scheduled on the cluster, I couldn't find a sponsor that'd be willing to cover the total cost, if all tasks were paid for with credits.

  However, it is also possible to bring our own runners to Cirrus CI.

  For testing purposes, a single task will be transformed to run on the DrahtBot infrastructure. If all goes well, the other tasks can be moved, too.

ACKs for top commit:
  hebasto:
    ACK fa41a91, I have reviewed the code and it looks OK.

Tree-SHA512: 513738daec36da8cd48a8f11e687ff0b7dfaba1ae4ed2fa77e7b043f88fd52bf5c0dbad2768e13df88518323917f08348cb62be6376a423142921f8d2c59a938

.cirrus.yml

@@ -3,10 +3,19 @@
 env:
   PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y"
   MAKEJOBS: "-j4"
-  DANGER_RUN_CI_ON_HOST: "1"  # Containers will be discarded after the run, so there is no risk that the ci scripts modify the system
   TEST_RUNNER_PORT_MIN: "14000"  # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache
   CCACHE_SIZE: "200M"
   CCACHE_DIR: "/tmp/ccache_dir"
+  CCACHE_NOHASHDIR: "1"  # Debug info might contain a stale path if the build dir changes, but this is fine
+
+cirrus_ephemeral_worker_template_env: &CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
+  DANGER_RUN_CI_ON_HOST: "1"  # Containers will be discarded after the run, so there is no risk that the ci scripts modify the system
+
+persistent_worker_template_env: &PERSISTENT_WORKER_TEMPLATE_ENV
+  RESTART_CI_DOCKER_BEFORE_RUN: "1"
+
+persistent_worker_template: &PERSISTENT_WORKER_TEMPLATE
+  persistent_worker: {}  # https://cirrus-ci.org/guide/persistent-workers/
 
 # https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks
 base_template: &BASE_TEMPLATE
@@ -31,9 +40,9 @@ global_task_template: &GLOBAL_TASK_TEMPLATE
   ccache_cache:
     folder: "/tmp/ccache_dir"
   depends_built_cache:
-    folder: "/tmp/cirrus-ci-build/depends/built"
+    folder: "depends/built"
   depends_sdk_cache:
-    folder: "/tmp/cirrus-ci-build/depends/sdk-sources"
+    folder: "depends/sdk-sources"
   ci_script:
     - ./ci/test_run_all.sh
 
@@ -69,13 +78,16 @@ task:
   << : *CREDITS_TEMPLATE
   lint_script:
     - ./ci/lint_run_all.sh
+  env:
+    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
 
 task:
   name: 'ARM [unit tests, no functional tests] [buster]'
   << : *GLOBAL_TASK_TEMPLATE
   container:
     image: debian:buster
   env:
+    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
     FILE_ENV: "./ci/test/00_setup_env_arm.sh"
 
 task:
@@ -84,6 +96,7 @@ task:
   container:
     image: ubuntu:focal
   env:
+    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
     FILE_ENV: "./ci/test/00_setup_env_win64.sh"
 
 task:
@@ -92,19 +105,18 @@ task:
   container:
     image: centos:8
   env:
+    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
     PACKAGE_MANAGER_INSTALL: "yum install -y"
     FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh"
 
 task:
   name: '[previous releases, uses qt5 dev package and some depends packages] [unsigned char] [bionic]'
-  # For faster CI feedback, immediately schedule a task that compiles most modules
-  << : *CREDITS_TEMPLATE
   << : *GLOBAL_TASK_TEMPLATE
   depends_releases_cache:
-    folder: "/tmp/cirrus-ci-build/releases"
-  container:
-    image: ubuntu:bionic
+    folder: "releases"
+  << : *PERSISTENT_WORKER_TEMPLATE
   env:
+    << : *PERSISTENT_WORKER_TEMPLATE_ENV
     FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh"
 
 task:
@@ -115,6 +127,7 @@ task:
     cpu: 6  # Increase CPU and Memory to avoid timeout
     memory: 24G
   env:
+    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
     MAKEJOBS: "-j8"
     FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh"
 
@@ -124,6 +137,7 @@ task:
   container:
     image: ubuntu:focal
   env:
+    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
     FILE_ENV: "./ci/test/00_setup_env_native_msan.sh"
 
 task:
@@ -132,6 +146,7 @@ task:
   container:
     image: ubuntu:focal
   env:
+    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
     FILE_ENV: "./ci/test/00_setup_env_native_asan.sh"
 
 task:
@@ -140,6 +155,7 @@ task:
   container:
     image: ubuntu:focal
   env:
+    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
     FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh"
 
 task:
@@ -148,6 +164,7 @@ task:
   container:
     image: ubuntu:focal
   env:
+    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
     FILE_ENV: "./ci/test/00_setup_env_native_multiprocess.sh"
 
 task:
@@ -156,6 +173,7 @@ task:
   container:
     image: ubuntu:bionic
   env:
+    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
     FILE_ENV: "./ci/test/00_setup_env_native_nowallet.sh"
 
 task:
@@ -164,6 +182,7 @@ task:
   container:
     image: ubuntu:focal
   env:
+    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
     FILE_ENV: "./ci/test/00_setup_env_mac.sh"
 
 task:
@@ -175,17 +194,18 @@ task:
     # Use latest image, but hardcode version to avoid silent upgrades (and breaks)
     image: big-sur-xcode-12.4  # https://cirrus-ci.org/guide/macOS
   env:
-    DANGER_RUN_CI_ON_HOST: "true"
+    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
     CI_USE_APT_INSTALL: "no"
     PACKAGE_MANAGER_INSTALL: "echo"  # Nothing to do
     FILE_ENV: "./ci/test/00_setup_env_mac_host.sh"
 
 task:
   name: 'ARM64 Android APK [focal]'
   depends_sources_cache:
-    folder: "/tmp/cirrus-ci-build/depends/sources"
+    folder: "depends/sources"
   << : *GLOBAL_TASK_TEMPLATE
   container:
     image: ubuntu:focal
   env:
+    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
     FILE_ENV: "./ci/test/00_setup_env_android.sh"

ci/test/04_install.sh

@@ -33,7 +33,12 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
   echo "Creating $DOCKER_NAME_TAG container to run in"
   ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG"
 
-  DOCKER_ID=$(docker run $DOCKER_ADMIN -idt \
+  if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then
+    echo "Restart docker before run to stop and clear all containers started with --rm"
+    systemctl restart docker
+  fi
+
+  DOCKER_ID=$(docker run $DOCKER_ADMIN --rm --interactive --detach --tty \
                   --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \
                   --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \
                   --mount type=bind,src=$DEPENDS_DIR,dst=$DEPENDS_DIR \