From d45ed0191fb80f3549b22105e18b84d7a1fa2fbf Mon Sep 17 00:00:00 2001 From: GitGab19 Date: Thu, 5 Sep 2024 18:28:03 +0200 Subject: [PATCH] common-template-provider-builder addition + healthcheck fix + update-mainnet-chainstate.sh script addition --- containers-scripts/healthcheck.sh | 26 +++++ .../update-mainnet-chainstate.sh | 88 +++++++++++++++ docker-compose-config-a.yaml | 103 ++++++++---------- docker-compose-config-c.yaml | 80 ++++++-------- 4 files changed, 190 insertions(+), 107 deletions(-) create mode 100755 containers-scripts/healthcheck.sh create mode 100755 containers-scripts/update-mainnet-chainstate.sh diff --git a/containers-scripts/healthcheck.sh b/containers-scripts/healthcheck.sh new file mode 100755 index 0000000..9adb26b --- /dev/null +++ b/containers-scripts/healthcheck.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +URL=$1 + +# Esegui la chiamata e salva la risposta +RESPONSE=$(curl -s --user username:password --data-binary '{"jsonrpc":"1.0","id":"curltest","method":"getblockchaininfo","params":[]}' -H 'content-type:text/plain;' $URL) + +# Verifica se la risposta contiene un errore +if echo "$RESPONSE" | jq -e '.error != null' > /dev/null; then + ERROR_MESSAGE=$(echo "$RESPONSE" | jq -r '.error.message') + echo "Error: $ERROR_MESSAGE" + exit 1 +fi + +# Estrai il numero di headers e blocks +HEADERS=$(echo "$RESPONSE" | jq -r '.result.headers') +BLOCKS=$(echo "$RESPONSE" | jq -r '.result.blocks') + +# Verifica se il numero di blocchi รจ uguale al numero di headers +if [ "$HEADERS" -eq "$BLOCKS" ]; then + echo "Node is fully synchronized" + exit 0 +else + echo "Node is not fully synchronized" + exit 1 +fi diff --git a/containers-scripts/update-mainnet-chainstate.sh b/containers-scripts/update-mainnet-chainstate.sh new file mode 100755 index 0000000..eb3457d --- /dev/null +++ b/containers-scripts/update-mainnet-chainstate.sh @@ -0,0 +1,88 @@ +#!/bin/sh + +NETWORK=$1 +TIMESTAMP_FILE="/root/.bitcoin/last_download_timestamp" +BACKUP_BASE_URL="http://75.119.150.111/backup" +DOWNLOAD_INTERVAL_DAYS=1 # 1 day = 24 hours +CHAINSTATE_DIR="/root/.bitcoin/chainstate" +MAX_RETRIES=30 # Number of attempts (1 per minute for 30 minutes) +RETRY_INTERVAL=60 # 60 seconds (1 minute) between attempts + +if [ "$NETWORK" = "" ]; then + echo "Checking if mainnet snapshot update is needed..." + + # Check if the chainstate directory exists + if [ -d "$CHAINSTATE_DIR" ]; then + CHAINSTATE_MOD_TIME=$(stat -c %Y "$CHAINSTATE_DIR") # Last modification time of chainstate directory + CURRENT_TIME=$(date +%s) # Current timestamp + TIME_DIFF=$(( (CURRENT_TIME - CHAINSTATE_MOD_TIME) / 86400 )) # Difference in days + + # Check if the chainstate directory was updated more than 24 hours ago + if [ "$TIME_DIFF" -ge "$DOWNLOAD_INTERVAL_DAYS" ]; then + echo "It has been more than $DOWNLOAD_INTERVAL_DAYS day(s) since the last node run. Proceeding with download..." + else + echo "Node has run recently (within $DOWNLOAD_INTERVAL_DAYS day(s)). Skipping download." + exit 0 + fi + else + echo "Chainstate directory $CHAINSTATE_DIR does not exist. This is a new node. Proceeding with download." + fi + + # Get current UTC date and time in the format used by the backup files + CURRENT_UTC=$(date -u +"%Y-%m-%d_%H-UTC") + BACKUP_FILE_NAME="backup_mainnet_blocks_chainstate_$CURRENT_UTC.tar.gz" + BACKUP_URL="$BACKUP_BASE_URL/$BACKUP_FILE_NAME" + BACKUP_HASH_URL="$BACKUP_URL.sha256" + + # Paths for files + BACKUP_FILE="/tmp/$BACKUP_FILE_NAME" + BACKUP_HASH_FILE="/tmp/$BACKUP_FILE_NAME.sha256" + + retry_count=0 + success=0 + + # Retry downloading the backup every minute for up to MAX_RETRIES (30 minutes) + while [ $retry_count -lt $MAX_RETRIES ]; do + echo "Attempt $((retry_count + 1)) of $MAX_RETRIES to download the backup..." + + # Try to download the backup file + if wget $BACKUP_URL -O $BACKUP_FILE && wget $BACKUP_HASH_URL -O $BACKUP_HASH_FILE; then + echo "Download succeeded. Verifying the downloaded snapshot..." + + cd /tmp/ + # Check the hash + if sha256sum -c $BACKUP_HASH_FILE; then + echo "Hash verification succeeded." + success=1 + break + else + echo "Hash verification failed! Retrying..." + rm -f $BACKUP_FILE $BACKUP_HASH_FILE + fi + else + echo "Download failed! Retrying in $RETRY_INTERVAL seconds..." + fi + + retry_count=$((retry_count + 1)) + sleep $RETRY_INTERVAL + done + + if [ $success -eq 0 ]; then + echo "Failed to download the backup after $MAX_RETRIES attempts. Aborting." + exit 1 + fi + + echo "Extracting the snapshot into the bitcoin directory..." + + tar -xzvf $BACKUP_FILE -C /root/.bitcoin + + # Remove the downloaded files to save space + rm -f $BACKUP_FILE $BACKUP_HASH_FILE + + # Update the timestamp file to the current time + touch $TIMESTAMP_FILE + + echo "Update completed." +else + echo "NETWORK is not set to 'mainnet'. Skipping update." +fi diff --git a/docker-compose-config-a.yaml b/docker-compose-config-a.yaml index df05825..14683d6 100644 --- a/docker-compose-config-a.yaml +++ b/docker-compose-config-a.yaml @@ -17,6 +17,13 @@ networks: gateway: 10.5.0.1 services: + common-template-provider-builder: + build: + dockerfile: ./template-provider.dockerfile + container_name: common-template-provider-builder + image: template-provider-builder-image + command: echo "common-template-provider build completed" + sv2-roles-builder: build: dockerfile: ./sv2-roles.dockerfile @@ -53,116 +60,93 @@ services: command: echo "pools-latency-calculator build completed" template-provider-pool-side: - build: - dockerfile: ./template-provider.dockerfile labels: logging: "config-a" - image: sv2-template-provider - command: - [ - "/bitcoin/bin/bitcoind", - "-${NETWORK}", - "-sv2", - "-sv2port=8442", - "-sv2interval=${SV2_INTERVAL}", - "-sv2feedelta=0", - "-debug=sv2", - "-loglevel=sv2:trace", - "-sv2bind=0.0.0.0", - ] + image: template-provider-builder-image + entrypoint: ["/bin/sh", "-c", "./scripts/update-mainnet-chainstate.sh ${NETWORK} && /bitcoin/bin/bitcoind -sv2 -sv2port=8442 -sv2interval=${SV2_INTERVAL} -sv2feedelta=0 -debug=sv2 -loglevel=sv2:trace -sv2bind=0.0.0.0 -${NETWORK}"] ports: - "8442:8442" - "18333:48333" - "8332:8332" - "18332:18332" container_name: sv2-tp-pool-side + depends_on: + - common-template-provider-builder volumes: - bitcoin_pool_side_data:/root/.bitcoin - ./custom-configs/sri-roles/bitcoin-tp-pool.conf:/root/.bitcoin/bitcoin.conf + - ./containers-scripts:/scripts restart: unless-stopped networks: sv2-net: ipv4_address: 10.5.0.2 healthcheck: - test: ["CMD-SHELL", "curl --silent --user username:password --data-binary '{\"jsonrpc\":\"1.0\",\"id\":\"healthcheck\",\"method\":\"getblockchaininfo\",\"params\":[]}' -H 'Content-Type: application/json' http://10.5.0.2:18332/ | jq -e '.result.initialblockdownload == false' || exit 1"] - interval: 30s - timeout: 10s - retries: 10 - start_period: 10s + test: ["CMD", "./scripts/healthcheck.sh", "http://127.0.0.1:18332"] + interval: 2m + timeout: 30s + retries: 100 + start_period: 1m template-provider-miner-side: - build: - dockerfile: ./template-provider.dockerfile labels: logging: "config-a" - image: sv2-template-provider - command: - [ - "/bitcoin/bin/bitcoind", - "-${NETWORK}", - "-sv2", - "-sv2port=8443", - "-sv2interval=${SV2_INTERVAL}", - "-sv2feedelta=0", - "-debug=sv2", - "-loglevel=sv2:trace", - "-sv2bind=0.0.0.0", - ] + image: template-provider-builder-image + entrypoint: ["/bin/sh", "-c", "./scripts/update-mainnet-chainstate.sh ${NETWORK} && /bitcoin/bin/bitcoind -sv2 -sv2port=8443 -sv2interval=${SV2_INTERVAL} -sv2feedelta=0 -debug=sv2 -loglevel=sv2:trace -sv2bind=0.0.0.0 -${NETWORK}"] ports: - "8443:8443" - "28333:18333" container_name: sv2-tp-miner-side + depends_on: + - common-template-provider-builder volumes: - bitcoin_miner_side_data:/root/.bitcoin - ./custom-configs/sri-roles/bitcoin-tp-miner.conf:/root/.bitcoin/bitcoin.conf + - ./containers-scripts:/scripts restart: unless-stopped networks: sv2-net: ipv4_address: 10.5.0.3 healthcheck: - test: ["CMD-SHELL", "curl --silent --user username:password --data-binary '{\"jsonrpc\":\"1.0\",\"id\":\"healthcheck\",\"method\":\"getblockchaininfo\",\"params\":[]}' -H 'Content-Type: application/json' http://10.5.0.3:18332/ | jq -e '.result.initialblockdownload == false' || exit 1"] - interval: 30s - timeout: 10s - retries: 10 - start_period: 10s + test: ["CMD", "./scripts/healthcheck.sh", "http://127.0.0.1:18332"] + interval: 2m + timeout: 30s + retries: 100 + start_period: 1m sv1-node-pool-side: - build: - dockerfile: ./template-provider.dockerfile labels: logging: "config-a" - image: sv2-template-provider - command: ["/bitcoin/bin/bitcoind", "-${NETWORK}"] + image: template-provider-builder-image + entrypoint: ["/bin/sh", "-c", "./scripts/update-mainnet-chainstate.sh ${NETWORK} && /bitcoin/bin/bitcoind -${NETWORK}"] ports: - "38333:18333" - "28332:18332" - "18443:18443" container_name: sv1-node-pool-side + depends_on: + - common-template-provider-builder volumes: - bitcoin_sv1_pool_side_data:/root/.bitcoin - ./custom-configs/sri-roles/bitcoin-sv1-node-pool.conf:/root/.bitcoin/bitcoin.conf + - ./containers-scripts:/scripts restart: unless-stopped networks: sv2-net: ipv4_address: 10.5.0.16 + environment: + - NETWORK=${NETWORK} healthcheck: - test: ["CMD-SHELL", "curl --silent --user username:password --data-binary '{\"jsonrpc\":\"1.0\",\"id\":\"healthcheck\",\"method\":\"getblockchaininfo\",\"params\":[]}' -H 'Content-Type: application/json' http://10.5.0.16:18332/ | jq -e '.result.initialblockdownload == false' || exit 1"] - interval: 30s - timeout: 10s - retries: 10 - start_period: 10s + test: ["CMD", "./scripts/healthcheck.sh", "http://127.0.0.1:18332"] + interval: 2m + timeout: 30s + retries: 100 + start_period: 1m pool: image: sv2-roles-builder-image labels: logging: "config-a" command: ["/bin/bash", "-c", "/usr/local/bin/monitor_and_apply_latency.sh 10.5.0.6 2 & exec ./target/release/pool_sv2 -c pool/config-examples/pool-config-a-docker-example.toml"] - #command: - #[ - #"./target/release/pool_sv2", - #"-c", - #"pool/config-examples/pool-config-a-docker-example.toml", - #] ports: - "34254:34254" container_name: sv2-pool @@ -191,7 +175,6 @@ services: "-c", "/usr/local/bin/monitor_and_apply_latency.sh 10.5.0.6 2 & exec ./target/release/jd_server -c jd-server/config-examples/jds-config-a-docker-example.toml", ] - #command: ["./target/release/jd_server", "-c", "jd-server/config-examples/jds-config-a-docker-example.toml"] ports: - "34264:34264" container_name: sv2-jds @@ -432,7 +415,7 @@ services: - "--web.enable-lifecycle" - "--web.enable-admin-api" ports: - - 9090:9090 + - "9090:9090" container_name: prometheus depends_on: - cadvisor @@ -460,7 +443,7 @@ services: - --collector.filesystem.ignored-mount-points - "^/(sys|proc|dev|host|etc|rootfs/var/lib/docker/containers|rootfs/var/lib/docker/overlay2|rootfs/run/docker/netns|rootfs/var/lib/docker/aufs)($$|/)" ports: - - 9100:9100 + - "9100:9100" container_name: node-exporter networks: sv2-net: @@ -479,7 +462,7 @@ services: - /sys:/sys:ro - /var/lib/docker/:/var/lib/docker:ro ports: - - 8080:8080 + - "8080:8080" container_name: cadvisor networks: sv2-net: @@ -494,7 +477,7 @@ services: depends_on: - prometheus ports: - - 3000:3000 + - "3000:3000" environment: GF_USERS_ALLOW_SIGN_UP: "false" GF_AUTH_DISABLE_LOGIN_FORM: "true" diff --git a/docker-compose-config-c.yaml b/docker-compose-config-c.yaml index cf2d552..0f067bc 100644 --- a/docker-compose-config-c.yaml +++ b/docker-compose-config-c.yaml @@ -16,6 +16,13 @@ networks: gateway: 10.5.0.1 services: + common-template-provider-builder: + build: + dockerfile: ./template-provider.dockerfile + container_name: common-template-provider-builder + image: template-provider-builder-image + command: echo "common-template-provider build completed" + sv2-roles-builder: build: dockerfile: ./sv2-roles.dockerfile @@ -52,79 +59,64 @@ services: command: echo "pools-latency-calculator build completed" template-provider-pool-side: - build: - dockerfile: ./template-provider.dockerfile labels: logging: "config-c" - image: sv2-template-provider - command: - [ - "/bitcoin/bin/bitcoind", - "-${NETWORK}", - "-sv2", - "-sv2port=8442", - "-sv2interval=${SV2_INTERVAL}", - "-sv2feedelta=0", - "-debug=sv2", - "-loglevel=sv2:trace", - "-sv2bind=0.0.0.0", - ] + image: template-provider-builder-image + entrypoint: ["/bin/sh", "-c", "./scripts/update-mainnet-chainstate.sh ${NETWORK} && /bitcoin/bin/bitcoind -sv2 -sv2port=8442 -sv2interval=${SV2_INTERVAL} -sv2feedelta=0 -debug=sv2 -loglevel=sv2:trace -sv2bind=0.0.0.0 -${NETWORK}"] ports: - "8442:8442" - "18333:48333" - "8332:8332" - "18332:18332" container_name: sv2-tp-pool-side + depends_on: + - common-template-provider-builder volumes: - bitcoin_pool_side_data:/root/.bitcoin - ./custom-configs/sri-roles/bitcoin-tp-pool.conf:/root/.bitcoin/bitcoin.conf + - ./containers-scripts:/scripts restart: unless-stopped networks: sv2-net: ipv4_address: 10.5.0.2 healthcheck: - test: ["CMD-SHELL", "curl --silent --user username:password --data-binary '{\"jsonrpc\":\"1.0\",\"id\":\"healthcheck\",\"method\":\"getblockchaininfo\",\"params\":[]}' -H 'Content-Type: application/json' http://10.5.0.2:18332/ | jq -e '.result.initialblockdownload == false' || exit 1"] - interval: 30s - timeout: 10s - retries: 10 - start_period: 10s + test: [ "CMD", "./scripts/healthcheck.sh", "http://127.0.0.1:18332" ] + interval: 2m + timeout: 30s + retries: 100 + start_period: 1m sv1-node-pool-side: - build: - dockerfile: ./template-provider.dockerfile labels: - logging: "config-c" - image: sv2-template-provider - command: ["/bitcoin/bin/bitcoind", "-${NETWORK}"] + logging: "config-c" + image: template-provider-builder-image + entrypoint: ["/bin/sh", "-c", "./scripts/update-mainnet-chainstate.sh ${NETWORK} && /bitcoin/bin/bitcoind -${NETWORK}"] ports: - "38333:18333" - "28332:18332" - "18443:18443" container_name: sv1-node-pool-side + depends_on: + - common-template-provider-builder volumes: - bitcoin_sv1_pool_side_data:/root/.bitcoin - ./custom-configs/sri-roles/bitcoin-sv1-node-pool.conf:/root/.bitcoin/bitcoin.conf + - ./containers-scripts:/scripts restart: unless-stopped networks: sv2-net: ipv4_address: 10.5.0.16 healthcheck: - test: ["CMD-SHELL", "curl --silent --user username:password --data-binary '{\"jsonrpc\":\"1.0\",\"id\":\"healthcheck\",\"method\":\"getblockchaininfo\",\"params\":[]}' -H 'Content-Type: application/json' http://10.5.0.16:18332/ | jq -e '.result.initialblockdownload == false' || exit 1"] - interval: 30s - timeout: 10s - retries: 10 - start_period: 10s + test: [ "CMD", "./scripts/healthcheck.sh", "http://127.0.0.1:18332" ] + interval: 2m + timeout: 30s + retries: 100 + start_period: 1m pool: image: sv2-roles-builder-image labels: logging: "config-c" - #command: - #[ - #"/bin/bash", - #"-c", - #"/usr/local/bin/monitor_and_apply_latency.sh 10.5.0.17 1 & exec ./target/release/pool_sv2 -c pool/config-examples/pool-config-c-docker-example.toml", - #] command: [ "./target/release/pool_sv2", @@ -156,11 +148,6 @@ services: "-c", "/usr/local/bin/monitor_and_apply_latency.sh 10.5.0.23 2 & exec ./target/release/translator_sv2 -c translator/config-examples/tproxy-config-c-docker-example.toml", ] - #[ - #"./target/release/translator_sv2", - #"-c", - #"translator/config-examples/tproxy-config-c-docker-example.toml", - #] ports: - "34256:34256" container_name: sv2-translator @@ -178,8 +165,7 @@ services: sv2-pool-translator-proxy: image: sv2-custom-proxy-builder-image labels: - logging: "config-c" - #command: ["./target/release/sv2-custom-proxy"] + logging: "config-c" command: [ "/bin/bash", @@ -354,7 +340,7 @@ services: - "--web.enable-lifecycle" - "--web.enable-admin-api" ports: - - 9090:9090 + - "9090:9090" container_name: prometheus depends_on: - cadvisor @@ -382,7 +368,7 @@ services: - --collector.filesystem.ignored-mount-points - "^/(sys|proc|dev|host|etc|rootfs/var/lib/docker/containers|rootfs/var/lib/docker/overlay2|rootfs/run/docker/netns|rootfs/var/lib/docker/aufs)($$|/)" ports: - - 9100:9100 + - "9100:9100" container_name: node-exporter networks: sv2-net: @@ -401,7 +387,7 @@ services: - /sys:/sys:ro - /var/lib/docker/:/var/lib/docker:ro ports: - - 8080:8080 + - "8080:8080" container_name: cadvisor networks: sv2-net: @@ -417,7 +403,7 @@ services: - prometheus - loki ports: - - 3000:3000 + - "3000:3000" environment: GF_USERS_ALLOW_SIGN_UP: "false" GF_AUTH_DISABLE_LOGIN_FORM: "true"