Chore/prometheus

Makefile

@@ -105,6 +105,14 @@ genesis: check-not-running | build $(CHAINSTATE_DIR) /usr/bin/sudo
 # Secondary name to boot the genesis network
 up-genesis: genesis
 
+up-prom:
+	@echo "Starting $(PROJECT) prometheus monitoring"
+	docker compose -f docker/docker-compose.prom.yml --profile default -p monitoring up -d
+
+down-prom:
+	@echo "Shutting down $(PROJECT) prometheus monitoring"
+	docker compose -f docker/docker-compose.prom.yml --profile default -p monitoring down
+
 # Shut down the network (chainstate and logs will be preserved)
 down: backup-logs current-chainstate-dir
 	@echo "Shutting down $(PROJECT) network"

README.md

@@ -58,6 +58,7 @@ make log stacks-signer-1 -- -f
 make log stacks-signer-1
 ```
 
+### Container management
 #### Pause/Unpause service
 To pause all services on the network
 ```sh
@@ -88,6 +89,19 @@ Restart the stopped service
 make start <service name>
 ```
 
+#### Force stop the hacknet network
+If the network is in a "stuck" state where the Makefile targets are not stopping the services (i.e. the `.current-chainstate-dir` file was removed while network was running), `down-force` may be used to force stop the network.
+
+```sh
+make down-force
+```
+
+Additionally, `clean` target will call `down-force` *and also* delete any chainstates on disk in `./docker/chainstate/*`
+```sh
+make clean
+```
+
+### Additional Features
 #### Stress the CPU
 To simulate CPU load. Can be modified with:
 - `STRESS_CORES` to target how many worker threads (default will use all cores)
@@ -99,6 +113,12 @@ make stress
 STRESS_CORES=10 STRESS_TIMEOUT=60 make stress
 ```
 
+#### Monitor chain heights
+Run a script outputting the current chain heights of each miner
+```sh
+make monitor
+```
+
 #### Create a chainstate snapshot
 - Setting the env var `PAUSE_HEIGHT` is optional to pause the chain at a specific height, else a default of Bitcoin block `999999999999` is used.
 - Setting the env var `MINE_INTERVAL_EPOCH3` is recommended to reach the `PAUSE_HEIGHT` more quickly to create the snapshot
@@ -129,19 +149,17 @@ ex:
 CHAINSTATE_ARCHIVE=./docker/chainstate_new.tar.zstd make up
 ```
 
-#### Force stop the hacknet network
-If the network is in a "stuck" state where the Makefile targets are not stopping the services (i.e. the `.current-chainstate-dir` file was removed while network was running), `down-force` may be used to force stop the network.
-
+#### Prometheus sidecar
+##### Run prometheus and cadvisor
+Runs a prometheus container to record data collected by `cadvisor` for tracking host/container metrics
 ```sh
-make down-force
+make up-prom
 ```
-
-Additionally, `clean` target will call `down-force` *and also* delete any chainstates on disk in `./docker/chainstate/*`
+##### Stop prometheus and cadvisor
 ```sh
-make clean
+make down-prom
 ```
 
-
 ## Containers
 
 - **bitcoin**: Runs a bitcoin regtest node

docker/docker-compose.prom.yml

@@ -0,0 +1,74 @@
+x-common-vars:
+  - &DOCKER_NETWORK ${DOCKER_NETWORK:-stacks}
+  - &DOCKER_NETWORK_CIDR 10.0.0.0/24
+  - &DOCKER_NETWORK_GATEWAY 10.0.0.1
+  - &PROMETHEUS_IP 10.0.0.252
+  - &CADVISOR_IP 10.0.0.253
+
+name: monitoring
+
+configs:
+  # prometheus scraping config
+  prometheus.yml:
+    content: |
+      # Global config
+      global:
+        external_labels:
+          monitor: "hacknet"
+      # A scrape configuration containing Prometheus and Docker
+      scrape_configs:
+      - job_name: cadvisor
+        scrape_interval: 5s
+        static_configs:
+          - targets:
+            - cadvisor:8080
+        metric_relabel_configs:
+          - source_labels: [ __name__ ]
+            regex: '^(go_|cadvisor_version_info).+'
+            action: drop
+      - job_name: stacks
+        scrape_interval: 5s
+        static_configs:
+          - targets:
+            - stacks-miner-1:9153
+            - stacks-miner-2:9153
+            - stacks-miner-3:9153
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: prometheus
+    networks:
+      default:
+        ipv4_address: *PROMETHEUS_IP
+    configs:
+      - source: prometheus.yml
+        target: /etc/prometheus/prometheus.yml
+    depends_on:
+      - cadvisor
+    ports:
+      - '0.0.0.0:${PROM_HTTP:-9090}:${PROM_HTTP:-9090}'
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:latest
+    container_name: cadvisor
+    networks:
+      default:
+        ipv4_address: *CADVISOR_IP
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    ports:
+      - '0.0.0.0:8080:8080'
+    command: "--enable_metrics=cpu,cpuLoad,disk,diskIO,memory"
+
+networks:
+  default:
+    name: *DOCKER_NETWORK
+    ipam:
+      driver: default
+      config:
+        - subnet: *DOCKER_NETWORK_CIDR
+          gateway: *DOCKER_NETWORK_GATEWAY