Merge pull request #50 from loopholelabs/heterogeneous-host-migrations

Add initial support for heterogeneous host migrations and running Kubernetes in the guest
loopholelabs · Nov 25, 2024 · 6e75057 · 6e75057
2 parents c8a842c + 3a826d7
commit 6e75057
Show file tree

Hide file tree

Showing 33 changed files with 1,280 additions and 8,623 deletions.
diff --git a/.github/workflows/hydrun.yaml b/.github/workflows/hydrun.yaml
@@ -106,68 +106,114 @@ jobs:
             dst: out/*
             runner: depot-ubuntu-22.04-32
 
-          # OS
-          - id: os.drafter-x86_64
+          # OCI OS
+          - id: os.drafteros-oci-x86_64
             src: .
-            os: fedora:40
+            os: fedora:41
             flags: -e '-v /tmp/ccache/go:/root/.cache/go-build -v /tmp/ccache/buildroot:/root/.buildroot-ccache'
-            cmd: ./Hydrunfile os drafteros-firecracker-x86_64_defconfig drafteros-x86_64.tar.zst
-            dst: out/drafteros-x86_64.tar.zst
+            cmd: ./Hydrunfile os drafteros-oci-firecracker-x86_64_defconfig drafteros-oci-x86_64.tar.zst
+            dst: out/drafteros-oci-x86_64.tar.zst
             runner: depot-ubuntu-22.04-32
-          - id: os.drafter-x86_64_pvm
+          - id: os.drafteros-oci-x86_64_pvm
             src: .
-            os: fedora:40
+            os: fedora:41
             flags: -e '-v /tmp/ccache/go:/root/.cache/go-build -v /tmp/ccache/buildroot:/root/.buildroot-ccache'
-            cmd: ./Hydrunfile os drafteros-firecracker-x86_64_pvm_defconfig drafteros-x86_64_pvm.tar.zst
-            dst: out/drafteros-x86_64_pvm.tar.zst
+            cmd: ./Hydrunfile os drafteros-oci-firecracker-x86_64_pvm_defconfig drafteros-oci-x86_64_pvm.tar.zst
+            dst: out/drafteros-oci-x86_64_pvm.tar.zst
             runner: depot-ubuntu-22.04-32
-          - id: os.drafter-aarch64
+          - id: os.drafteros-oci-aarch64
             src: .
-            os: fedora:40
+            os: fedora:41
             flags: -e '-v /tmp/ccache/go:/root/.cache/go-build -v /tmp/ccache/buildroot:/root/.buildroot-ccache'
-            cmd: ./Hydrunfile os drafteros-firecracker-aarch64_defconfig drafteros-aarch64.tar.zst
-            dst: out/drafteros-aarch64.tar.zst
+            cmd: ./Hydrunfile os drafteros-oci-firecracker-aarch64_defconfig drafteros-oci-aarch64.tar.zst
+            dst: out/drafteros-oci-aarch64.tar.zst
+            runner: depot-ubuntu-22.04-32
+
+          # k3s Server OS
+          - id: os.drafteros-k3s-server-x86_64
+            src: .
+            os: fedora:41
+            flags: -e '-v /tmp/ccache/go:/root/.cache/go-build -v /tmp/ccache/buildroot:/root/.buildroot-ccache'
+            cmd: ./Hydrunfile os drafteros-k3s-server-firecracker-x86_64_defconfig drafteros-k3s-server-x86_64.tar.zst
+            dst: out/drafteros-k3s-server-x86_64.tar.zst
+            runner: depot-ubuntu-22.04-32
+          - id: os.drafteros-k3s-server-x86_64_pvm
+            src: .
+            os: fedora:41
+            flags: -e '-v /tmp/ccache/go:/root/.cache/go-build -v /tmp/ccache/buildroot:/root/.buildroot-ccache'
+            cmd: ./Hydrunfile os drafteros-k3s-server-firecracker-x86_64_pvm_defconfig drafteros-k3s-server-x86_64_pvm.tar.zst
+            dst: out/drafteros-k3s-server-x86_64_pvm.tar.zst
+            runner: depot-ubuntu-22.04-32
+          - id: os.drafteros-k3s-server-aarch64
+            src: .
+            os: fedora:41
+            flags: -e '-v /tmp/ccache/go:/root/.cache/go-build -v /tmp/ccache/buildroot:/root/.buildroot-ccache'
+            cmd: ./Hydrunfile os drafteros-k3s-server-firecracker-aarch64_defconfig drafteros-k3s-server-aarch64.tar.zst
+            dst: out/drafteros-k3s-server-aarch64.tar.zst
+            runner: depot-ubuntu-22.04-32
+
+          # k3s Client OS
+          - id: os.drafteros-k3s-client-x86_64
+            src: .
+            os: fedora:41
+            flags: -e '-v /tmp/ccache/go:/root/.cache/go-build -v /tmp/ccache/buildroot:/root/.buildroot-ccache'
+            cmd: ./Hydrunfile os drafteros-k3s-client-firecracker-x86_64_defconfig drafteros-k3s-client-x86_64.tar.zst
+            dst: out/drafteros-k3s-client-x86_64.tar.zst
+            runner: depot-ubuntu-22.04-32
+          - id: os.drafteros-k3s-client-x86_64_pvm
+            src: .
+            os: fedora:41
+            flags: -e '-v /tmp/ccache/go:/root/.cache/go-build -v /tmp/ccache/buildroot:/root/.buildroot-ccache'
+            cmd: ./Hydrunfile os drafteros-k3s-client-firecracker-x86_64_pvm_defconfig drafteros-k3s-client-x86_64_pvm.tar.zst
+            dst: out/drafteros-k3s-client-x86_64_pvm.tar.zst
+            runner: depot-ubuntu-22.04-32
+          - id: os.drafteros-k3s-client-aarch64
+            src: .
+            os: fedora:41
+            flags: -e '-v /tmp/ccache/go:/root/.cache/go-build -v /tmp/ccache/buildroot:/root/.buildroot-ccache'
+            cmd: ./Hydrunfile os drafteros-k3s-client-firecracker-aarch64_defconfig drafteros-k3s-client-aarch64.tar.zst
+            dst: out/drafteros-k3s-client-aarch64.tar.zst
             runner: depot-ubuntu-22.04-32
 
           # OCI runtime bundles
           - id: oci.valkey-x86_64
             src: .
-            os: fedora:40
+            os: fedora:41
             flags: -e '-v /tmp/ccache/go:/root/.cache/go-build'
             cmd: ./Hydrunfile oci docker://valkey/valkey:latest amd64 oci-valkey-x86_64.tar.zst
             dst: out/oci-valkey-x86_64.tar.zst
             runner: depot-ubuntu-22.04-32
           - id: oci.valkey-aarch64
             src: .
-            os: fedora:40
+            os: fedora:41
             flags: -e '-v /tmp/ccache/go:/root/.cache/go-build'
             cmd: ./Hydrunfile oci docker://valkey/valkey:latest arm64 oci-valkey-aarch64.tar.zst
             dst: out/oci-valkey-aarch64.tar.zst
             runner: depot-ubuntu-22.04-32
           - id: oci.postgres-x86_64
             src: .
-            os: fedora:40
+            os: fedora:41
             flags: -e '-v /tmp/ccache/go:/root/.cache/go-build'
             cmd: ./Hydrunfile oci docker://postgres:latest amd64 oci-postgres-x86_64.tar.zst
             dst: out/oci-postgres-x86_64.tar.zst
             runner: depot-ubuntu-22.04-32
           - id: oci.postgres-aarch64
             src: .
-            os: fedora:40
+            os: fedora:41
             flags: -e '-v /tmp/ccache/go:/root/.cache/go-build'
             cmd: ./Hydrunfile oci docker://postgres:latest arm64 oci-postgres-aarch64.tar.zst
             dst: out/oci-postgres-aarch64.tar.zst
             runner: depot-ubuntu-22.04-32
           - id: oci.ollama-x86_64
             src: .
-            os: fedora:40
+            os: fedora:41
             flags: -e '-v /tmp/ccache/go:/root/.cache/go-build'
             cmd: ./Hydrunfile oci docker://ollama/ollama:latest amd64 oci-ollama-x86_64.tar.zst
             dst: out/oci-ollama-x86_64.tar.zst
             runner: depot-ubuntu-22.04-32
           - id: oci.ollama-aarch64
             src: .
-            os: fedora:40
+            os: fedora:41
             flags: -e '-v /tmp/ccache/go:/root/.cache/go-build'
             cmd: ./Hydrunfile oci docker://ollama/ollama:latest arm64 oci-ollama-aarch64.tar.zst
             dst: out/oci-ollama-aarch64.tar.zst

diff --git a/Hydrunfile b/Hydrunfile
@@ -41,8 +41,7 @@ fi
 # OS
 if [ "$1" = "os" ]; then
   # Install native dependencies
-  dnf group install -y "C Development Tools and Libraries" "Development Tools"
-  dnf install -y go curl make file cpio unzip rsync bc openssh-clients which wget perl
+  dnf install -y @c-development @development-tools go curl make file cpio unzip rsync bc openssh-clients which wget perl
 
   # Configure Git
   git config --global --add safe.directory '*'

diff --git a/Makefile b/Makefile
@@ -8,8 +8,9 @@ OCI_IMAGE_URI ?= docker://valkey/valkey:latest
 OCI_IMAGE_ARCHITECTURE ?= amd64
 OCI_IMAGE_HOSTNAME ?= drafterguest
 
-OS_URL ?= https://buildroot.org/downloads/buildroot-2024.08.tar.gz
-OS_DEFCONFIG ?= drafteros-firecracker-x86_64_defconfig
+# We're pinning to this specific commit since there isn't a release with Go 1.23+ yet, once there is use `OS_URL ?= https://buildroot.org/downloads/buildroot-2024.08.2.tar.gz` instead
+OS_URL ?= https://gitlab.com/buildroot.org/buildroot/-/archive/11ae90b0011abe3ad01d9953c7d41d444e689f5c/buildroot-11ae90b0011abe3ad01d9953c7d41d444e689f5c.tar.gz
+OS_DEFCONFIG ?= drafteros-oci-firecracker-x86_64_defconfig
 OS_BR2_EXTERNAL ?= ../../os
 
 # Private variables
@@ -32,7 +33,25 @@ build/oci:
 
 # Build OS
 build/os:
-	$(MAKE) -C $(OUTPUT_DIR)/buildroot BR2_EXTERNAL="$(OS_BR2_EXTERNAL)" drafter-liveness-reconfigure drafter-agent-reconfigure oci-runtime-bundle-reconfigure
+	# Common OS packages
+	if grep -q "BR2_PACKAGE_DRAFTER_AGENT=y" $(OUTPUT_DIR)/buildroot/.config; then \
+		$(MAKE) -C $(OUTPUT_DIR)/buildroot BR2_EXTERNAL="$(OS_BR2_EXTERNAL)" drafter-agent-reconfigure; \
+	fi
+	if grep -q "BR2_PACKAGE_DRAFTER_LIVENESS=y" $(OUTPUT_DIR)/buildroot/.config; then \
+		$(MAKE) -C $(OUTPUT_DIR)/buildroot BR2_EXTERNAL="$(OS_BR2_EXTERNAL)" drafter-liveness-reconfigure; \
+	fi	
+
+	# OCI OS packages
+	if grep -q "BR2_PACKAGE_OCI_RUNTIME_BUNDLE=y" $(OUTPUT_DIR)/buildroot/.config; then \
+		$(MAKE) -C $(OUTPUT_DIR)/buildroot BR2_EXTERNAL="$(OS_BR2_EXTERNAL)" oci-runtime-bundle-reconfigure; \
+	fi
+
+	# k3s OS packages
+	if grep -q "BR2_PACKAGE_K3S=y" $(OUTPUT_DIR)/buildroot/.config; then \
+		$(MAKE) -C $(OUTPUT_DIR)/buildroot BR2_EXTERNAL="$(OS_BR2_EXTERNAL)" k3s-dirclean; \
+		$(MAKE) -C $(OUTPUT_DIR)/buildroot BR2_EXTERNAL="$(OS_BR2_EXTERNAL)" k3s-reconfigure; \
+	fi
+
 	$(MAKE) -C $(OUTPUT_DIR)/buildroot BR2_EXTERNAL="$(OS_BR2_EXTERNAL)"
 
 	mkdir -p $(OUTPUT_DIR)/blueprint
@@ -43,10 +62,19 @@ build/os:
 config/os:
 	$(MAKE) -C $(OUTPUT_DIR)/buildroot BR2_EXTERNAL="$(OS_BR2_EXTERNAL)" menuconfig
 
+# Configure kernel
+config/kernel:
+	$(MAKE) -C $(OUTPUT_DIR)/buildroot BR2_EXTERNAL="$(OS_BR2_EXTERNAL)" linux-menuconfig
+
 # Save OS defconfig changes
 save/os:
 	$(MAKE) -C $(OUTPUT_DIR)/buildroot BR2_EXTERNAL="$(OS_BR2_EXTERNAL)" savedefconfig
 
+# Save kernel defconfig changes
+save/kernel:
+	$(MAKE) -C $(OUTPUT_DIR)/buildroot BR2_EXTERNAL="$(OS_BR2_EXTERNAL)" linux-savedefconfig
+	$(MAKE) -C $(OUTPUT_DIR)/buildroot BR2_EXTERNAL="$(OS_BR2_EXTERNAL)" linux-update-defconfig
+
 # Unpack OCI runtime bundle
 unpack/oci:
 	rm -rf $(OUTPUT_DIR)/oci-image

diff --git a/README.md b/README.md
@@ -91,14 +91,14 @@ To download the pre-built blueprints for your architecture, execute the followin
 
 ```shell
 $ mkdir -p out
-$ curl -Lo out/drafteros.tar.zst "https://github.com/loopholelabs/drafter/releases/latest/download/drafteros-$(uname -m).tar.zst" # Use `drafteros-$(uname -m)_pvm.tar.zst` if you're using PVM
+$ curl -Lo out/drafteros-oci.tar.zst "https://github.com/loopholelabs/drafter/releases/latest/download/drafteros-oci-$(uname -m).tar.zst" # Use `drafteros-oci-$(uname -m)_pvm.tar.zst` if you're using PVM
 $ curl -Lo out/oci-valkey.tar.zst "https://github.com/loopholelabs/drafter/releases/latest/download/oci-valkey-$(uname -m).tar.zst"
 ```
 
 Next, use `drafter-packager` to extract the blueprints:
 
 ```shell
-$ drafter-packager --package-path out/drafteros.tar.zst --extract --devices '[
+$ drafter-packager --package-path out/drafteros-oci.tar.zst --extract --devices '[
   {
     "name": "kernel",
     "path": "out/blueprint/vmlinux"
@@ -142,7 +142,9 @@ To build the blueprints locally, you can use the [included Makefile](./Makefile)
 
 ```shell
 # Build the DrafterOS blueprint
-$ make depend/os OS_DEFCONFIG=drafteros-firecracker-x86_64_defconfig # Use `drafteros-firecracker-x86_64_pvm_defconfig` if you're using PVM and `drafteros-firecracker-aarch64_defconfig` if you're on `aarch64`
+$ make depend/os OS_DEFCONFIG=drafteros-oci-firecracker-x86_64_defconfig # Use `drafteros-oci-firecracker-x86_64_pvm_defconfig` if you're using PVM and `drafteros-oci-firecracker-aarch64_defconfig` if you're on `aarch64`
+$ make config/kernel # Optional: Configure kernel
+$ make save/kernel # Optional: Write back the kernel configuration to the defconfig
 $ make config/os # Optional: Configure DrafterOS
 $ make save/os # Optional: Write back the DrafterOS configuration to the defconfig
 $ make build/os
@@ -166,7 +168,7 @@ out/
 You can optionally package the VM blueprint files using `drafter-packager` for distribution by running the following:
 
 ```shell
-$ drafter-packager --package-path out/drafteros.tar.zst --devices '[
+$ drafter-packager --package-path out/drafteros-oci.tar.zst --devices '[
   {
     "name": "kernel",
     "path": "out/blueprint/vmlinux"

diff --git a/cmd/drafter-mounter/main.go b/cmd/drafter-mounter/main.go
@@ -169,7 +169,7 @@ func main() {
 	raddr := flag.String("raddr", "localhost:1337", "Remote address to connect to (leave empty to disable)")
 	laddr := flag.String("laddr", "localhost:1337", "Local address to listen on (leave empty to disable)")
 
-	concurrency := flag.Int("concurrency", 4096, "Number of concurrent workers to use in migrations")
+	concurrency := flag.Int("concurrency", 1024, "Number of concurrent workers to use in migrations")
 
 	flag.Parse()
 

diff --git a/cmd/drafter-peer/main.go b/cmd/drafter-peer/main.go
@@ -204,7 +204,7 @@ func main() {
 	raddr := flag.String("raddr", "localhost:1337", "Remote address to connect to (leave empty to disable)")
 	laddr := flag.String("laddr", "localhost:1337", "Local address to listen on (leave empty to disable)")
 
-	concurrency := flag.Int("concurrency", 4096, "Number of concurrent workers to use in migrations")
+	concurrency := flag.Int("concurrency", 1024, "Number of concurrent workers to use in migrations")
 
 	flag.Parse()
 

diff --git a/cmd/drafter-registry/main.go b/cmd/drafter-registry/main.go
@@ -61,7 +61,7 @@ func main() {
 
 	laddr := flag.String("laddr", ":1600", "Address to listen on")
 
-	concurrency := flag.Int("concurrency", 4096, "Number of concurrent workers to use in migrations")
+	concurrency := flag.Int("concurrency", 1024, "Number of concurrent workers to use in migrations")
 
 	flag.Parse()
 

diff --git a/go.mod b/go.mod
@@ -1,28 +1,27 @@
 module github.com/loopholelabs/drafter
 
-go 1.22.5
+go 1.23
 
-toolchain go1.22.6
+toolchain go1.23.2
 
 require (
 	github.com/coreos/go-iptables v0.8.0
 	github.com/freddierice/go-losetup/v2 v2.0.1
 	github.com/klauspost/compress v1.17.11
 	github.com/lithammer/shortuuid/v4 v4.0.0
 	github.com/loopholelabs/goroutine-manager v0.1.1
-	github.com/loopholelabs/silo v0.1.0
-	github.com/metal-stack/go-ipam v1.14.0
-	github.com/pojntfx/panrpc/go v0.0.0-20240913062914-ea5ef6b07692
-	github.com/vishvananda/netlink v1.1.0
-	github.com/vishvananda/netns v0.0.4
-	golang.org/x/sys v0.26.0
-	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
+	github.com/loopholelabs/silo v0.1.2
+	github.com/metal-stack/go-ipam v1.14.7
+	github.com/pojntfx/panrpc/go v0.0.0-20241003051136-b93809e92a15
+	github.com/vishvananda/netlink v1.3.0
+	github.com/vishvananda/netns v0.0.5
+	golang.org/x/sys v0.27.0
+	k8s.io/utils v0.0.0-20241104163129-6fe5fd82f078
 )
 
 require (
-	github.com/Merovius/nbd v0.0.0-20231017152624-27b78b60d8da // indirect
-	github.com/agext/levenshtein v1.2.1 // indirect
-	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
+	github.com/Merovius/nbd v0.0.0-20240812113926-fd65a54c9949 // indirect
+	github.com/agext/levenshtein v1.2.3 // indirect
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
 	github.com/avast/retry-go/v4 v4.6.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
@@ -37,41 +36,41 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/hashicorp/hcl/v2 v2.21.0 // indirect
+	github.com/hashicorp/hcl/v2 v2.22.0 // indirect
 	github.com/jmoiron/sqlx v1.4.0 // indirect
 	github.com/josharian/native v1.1.0 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
 	github.com/lib/pq v1.10.9 // indirect
 	github.com/loopholelabs/logging v0.3.1 // indirect
 	github.com/mdlayher/genetlink v1.3.2 // indirect
 	github.com/mdlayher/netlink v1.7.2 // indirect
-	github.com/mdlayher/socket v0.4.1 // indirect
+	github.com/mdlayher/socket v0.5.1 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/minio/minio-go/v7 v7.0.77 // indirect
-	github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 // indirect
+	github.com/minio/minio-go/v7 v7.0.80 // indirect
+	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect
-	github.com/redis/go-redis/v9 v9.6.1 // indirect
+	github.com/redis/go-redis/v9 v9.7.0 // indirect
 	github.com/rs/xid v1.6.0 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
-	github.com/youmark/pkcs8 v0.0.0-20240424034433-3c2c7870ae76 // indirect
-	github.com/zclconf/go-cty v1.13.0 // indirect
-	go.etcd.io/etcd/api/v3 v3.5.14 // indirect
-	go.etcd.io/etcd/client/pkg/v3 v3.5.14 // indirect
-	go.etcd.io/etcd/client/v3 v3.5.14 // indirect
-	go.mongodb.org/mongo-driver v1.15.0 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
+	github.com/zclconf/go-cty v1.15.0 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.16 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.16 // indirect
+	go.etcd.io/etcd/client/v3 v3.5.16 // indirect
+	go.mongodb.org/mongo-driver v1.17.1 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
-	golang.org/x/crypto v0.26.0 // indirect
-	golang.org/x/mod v0.17.0 // indirect
-	golang.org/x/net v0.28.0 // indirect
-	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/text v0.17.0 // indirect
-	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect
-	google.golang.org/grpc v1.64.1 // indirect
-	google.golang.org/protobuf v1.34.1 // indirect
+	golang.org/x/crypto v0.29.0 // indirect
+	golang.org/x/mod v0.22.0 // indirect
+	golang.org/x/net v0.31.0 // indirect
+	golang.org/x/sync v0.9.0 // indirect
+	golang.org/x/text v0.20.0 // indirect
+	golang.org/x/tools v0.27.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect
+	google.golang.org/grpc v1.68.0 // indirect
+	google.golang.org/protobuf v1.35.1 // indirect
 )