Merge pull request #10 from NebulousLabs/docker-ports

Enable publishing all ants' HTTP API ports

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Sia Antfarm Docker Image Changelog
 
+## Nov 18, 2020:
+### v1.0.5
+**Key Updates**
+- Allow to publish all APIAddr ports by parsing config and setting socat port
+  forwarding automatically.
+
 ## Nov 13, 2020:
 ### v1.0.4
 **Key Updates**

README.md

@@ -11,17 +11,20 @@
 ### Latest
 * **latest**
 
+### v1.0.5
+* Allows publishing multiple ant HTTP API ports
+
 ### v1.0.4
-* **1.0.4**: Sia Ant Farm `v1.0.4` based on Sia `v.1.5.3`
+* Sia Ant Farm `v1.0.4` based on Sia `v.1.5.3`
 
 ### v1.0.3
-* **1.0.3**: Sia Ant Farm `v1.0.3` based on Sia `v.1.5.2`
+* Sia Ant Farm `v1.0.3` based on Sia `v.1.5.2`
 
 ### v1.0.2
-* **1.0.2**: Sia Ant Farm `v1.0.2` based on Sia `v.1.5.1`
+* Sia Ant Farm `v1.0.2` based on Sia `v.1.5.1`
 
 ### v1.0.1
-* **1.0.1**: Sia Ant Farm `v1.0.1` based on Sia `v.1.5.0`
+* Sia Ant Farm `v1.0.1` based on Sia `v.1.5.0`
 
 ## Running Ant Farm in Docker container
 
@@ -37,15 +40,20 @@ Port `127.0.0.1:9980` above is the renter API address which you can use to
 issue commands to the renter. For security reasons you should bind the port to
 localhost (see `127.0.0.1:9980` above).
 
-### Container internal port forwarding
-Note that the renter's API port set in config is `10980` (see
-`"APIAddr": "127.0.0.1:10980"`) in config, but renter's API is accessible from
-container internal port `9980` (not `10980`). This is because the internal
-container's port `10980` is bound only to container's internal localhost IP
-`127.0.0.1` and is not accessible from container's outbound IP. That is why
-container's `127.0.0.1:10980` had to be forwarded from container's localhost IP
-`127.0.0.1` via `socat` (done by `run.sh`) inside the container to accept calls
-from non localhost IP of container.
+### Custom Configuration
+By default the Sia Ant Farm docker image has a copy of
+`config/basic-renter-5-hosts-docker.json` configuration file.
+
+If you want to execute Ant Farm with a custom configuration, create your custom
+configuration e.g. `config/custom-config.json`, mount your config directory and
+set `CONFIG` environment variable to your custom configuration by executing:
+```
+docker run \
+    --publish 127.0.0.1:9980:9980 \
+    --volume $(pwd)/config:/sia-antfarm/config \
+    --env CONFIG=config/custom-config.json \
+    nebulouslabs/siaantfarm
+```
 
 ### Change Port
 To change port on which you can access the renter (e.g. to 39980) execute:
@@ -55,16 +63,45 @@ docker run \
     nebulouslabs/siaantfarm
 ```
 
-### Custom Configuration
-By default the Sia Ant Farm docker image has a copy of
-`config/basic-renter-5-hosts-docker.json` configuration file.
+### Open multiple ports
+In default configuration only renter's HTTP API port is accessible from outside
+of the docker container. If you want to configure access to more or all the
+ants, you need to use custom configuration file (described above) and each
+ant's HTTP API port needs to be set in 2 places:
+* In the configuration file
+* Pubished when starting docker container
 
-If you want to execute Ant Farm with a custom configuration, create your custom
-configuration e.g. `config/custom-config.json`, mount your config directory and
-set `CONFIG` environment variable to your custom configuration by executing:
+#### Specify port in configuration file
+`APIAddr` setting needs to be set in the configuration file same way as it is
+set for renter ant in default configuration file
+`config/basic-renter-5-hosts-docker.json`. Hostname part can only have one of
+two values: `127.0.0.1` or `localhost`.
+
+Example snippet:
+```
+        ...
+		{
+			"AllowHostLocalNetAddress": true,
+            "APIAddr": "127.0.0.1:10980",
+			"Name": "host1",
+			"Jobs": [
+				"host"
+			],
+			"DesiredCurrency": 100000
+		},
+        ...
+```
+
+#### Publish port when starting docker
+Once you have prepared configuration file, you can start the container. You
+need to set the path to custom configuration via `CONFIG` environment variable
+and publish each port via `--publish` flag.
+
+Example docker run command:
 ```
 docker run \
     --publish 127.0.0.1:9980:9980 \
+    --publish 127.0.0.1:10980:10980 \
     --volume $(pwd)/config:/sia-antfarm/config \
     --env CONFIG=config/custom-config.json \
     nebulouslabs/siaantfarm

build-test.sh

@@ -14,17 +14,38 @@ cleanup_test_containers() {
   fi
 }
 
+###############################################################################
+# Echo curl loop string to be used in 'timeout' command. The string when
+# executed executes curl command with the given url in the loop untill the curl
+# executes successfully.
+# Globals:
+#   none
+# Parameters:
+#   url, e.g.: http://localhost:9988/consensus
+# Outputs:
+#   Echoes curl loop string
+###############################################################################
+curl_loop() {
+  local url=$1
+  echo "until curl -A \"Sia-Agent\" --fail $url
+    do
+      sleep 1
+    done"
+}
+
 cleanup_test_containers
 
 for DIR in ./
 do
   docker build \
+    --no-cache \
     --tag sia-ant-farm-image-test \
     -f $DIR/Dockerfile \
     .
 
   export DUMMY_DATA_DIR=$(mktemp -d)
 
+  # Test with a single published port
   # Run container in detached state
   docker run \
     --detach \
@@ -35,10 +56,30 @@ do
 
   # Wait till API (/consensus) is accessible
   echo "Get consensus..."
-  timeout 120 bash -c 'until curl -A "Sia-Agent" --fail "http://localhost:9988/consensus"; do sleep 1; done'
+  curl_loop=$(curl_loop "http://localhost:9988/consensus")
+  timeout 120 sh -c "$curl_loop"
   echo "Got consensus successfully"
 
   docker rm -f sia-ant-farm-test-container
+
+  # Test with 2 published ports
+  docker run \
+    --detach \
+    --publish 127.0.0.1:9988:9980 \
+    --publish 127.0.0.1:10988:10980 \
+    --volume "${DUMMY_DATA_DIR}:/sia-antfarm/data" \
+    --volume "$(pwd)/config:/sia-antfarm/config" \
+    --env CONFIG=config/basic-renter-5-hosts-2-api-ports-docker.json \
+    --name sia-ant-farm-test-2-apis-container \
+    sia-ant-farm-image-test
+
+  # Wait till both APIs (/consensus) are accessible
+  echo "Get consensus..."
+  curl_loop2=$(curl_loop "http://localhost:10988/consensus")
+  timeout 120 sh -c "$curl_loop" && timeout 10 sh -c "$curl_loop2"
+  echo "Got consensus successfully"
+
+  docker rm -f sia-ant-farm-test-2-apis-container
 done
 
 cleanup_test_containers

changelog/changelog-tail.md

@@ -1,3 +1,9 @@
+## Nov 18, 2020:
+### v1.0.5
+**Key Updates**
+- Allow to publish all APIAddr ports by parsing config and setting socat port
+  forwarding automatically.
+
 ## Nov 13, 2020:
 ### v1.0.4
 **Key Updates**

config/basic-renter-5-hosts-2-api-ports-docker.json

@@ -0,0 +1,65 @@
+{
+	"AntConfigs": 
+	[ 
+		{
+			"AllowHostLocalNetAddress": true,
+			"Jobs": [
+				"gateway",
+				"miner"
+			]
+		},
+		{
+			"AllowHostLocalNetAddress": true,
+			"APIAddr": "127.0.0.1:10980",
+			"Name": "host1",
+			"Jobs": [
+				"host"
+			],
+			"DesiredCurrency": 100000
+		},
+		{
+			"AllowHostLocalNetAddress": true,
+			"Name": "host2",
+			"Jobs": [
+				"host"
+			],
+			"DesiredCurrency": 100000
+		},
+		{
+			"AllowHostLocalNetAddress": true,
+			"Name": "host3",
+			"Jobs": [
+				"host"
+			],
+			"DesiredCurrency": 100000
+		},
+		{
+			"AllowHostLocalNetAddress": true,
+			"Name": "host4",
+			"Jobs": [
+				"host"
+			],
+			"DesiredCurrency": 100000
+		},
+		{
+			"AllowHostLocalNetAddress": true,
+			"Name": "host5",
+			"Jobs": [
+				"host"
+			],
+			"DesiredCurrency": 100000
+		},
+		{
+			"AllowHostLocalNetAddress": true,
+			"RenterDisableIPViolationCheck": true,
+			"APIAddr": "127.0.0.1:9980",
+			"Name": "renter",
+			"Jobs": [
+				"renter"
+			],
+			"DesiredCurrency": 100000
+		}
+	],
+	"AutoConnect": true,
+	"WaitForSync": true
+}

config/basic-renter-5-hosts-docker.json

@@ -51,7 +51,7 @@
 		{
 			"AllowHostLocalNetAddress": true,
 			"RenterDisableIPViolationCheck": true,
-			"APIAddr": "127.0.0.1:10980",
+			"APIAddr": "127.0.0.1:9980",
 			"Name": "renter",
 			"Jobs": [
 				"renter"

run.sh

@@ -1,8 +1,22 @@
 #!/bin/sh
 
-# We are using socat in order ant API to be accessible outside of docker
-# container.
-socat tcp-listen:9980,reuseaddr,fork tcp:localhost:10980 &
+# We are using socat in order ant localhost http API to be accessible outside
+# of docker container.
+# Note: Default shell in Debian Slim is dash.
+
+# Get internal docker container ip for port forwarding.
+internal_ip_address=$(hostname -I)
+
+# Trim white space.
+IFS=' ' read internal_ip_address <<EOF
+$internal_ip_address
+EOF
+
+# Find all APIAddr ports in config, extract ports, set socat forwarding between
+# internal ip address and internal localhost ip address.
+grep -i apiaddr $CONFIG | \
+grep -oe '\([0-9]\{4,5\}\)' | \
+xargs -n1 -I % sh -c "socat tcp-listen:%,bind=$internal_ip_address,reuseaddr,fork tcp:localhost:%,bind=127.0.0.1 &"
 
 # Sia Antfarm deletes antfarm-data directory at startup, so this directory
 # itself can't be mounted as a volume, but an intermediary directory can be.
@@ -11,4 +25,4 @@ cd data
 # We are using `exec` to start Sia Ant Farm in order to ensure that it will be
 # run as PID 1. We need that in order to have Sia Ant Farm receive OS signals
 # (e.g. SIGTERM) on container shutdown, so it can exit gracefully.
-exec sia-antfarm -config=../${CONFIG}
+exec sia-antfarm -config=../$CONFIG