feat: move to compressed qcow

loic-roux-404 · May 25, 2024 · 1817d1d · 1817d1d
1 parent 7b6e100
commit 1817d1d
Show file tree

Hide file tree

Showing 14 changed files with 119 additions and 103 deletions.
diff --git a/.github/workflows/release-nixos.yml b/.github/workflows/release-nixos.yml
@@ -6,6 +6,7 @@ on:
     paths:
       - 'nixos/**.nix'
       - 'nixos-options/**.nix'
+      - 'nixos-generators/**.nix'
       - flake.nix
       - flake.lock
       - '!**.md'
@@ -45,13 +46,7 @@ jobs:
 
       - name: Build
         id: build
-        run: nix build .#nixosConfigurations.x86_64-linux.contabo && ls -lah
-
-      # - name: Retrieve result path
-      #   id: nixos_result
-      #   run: |
-      #     result_path="$(readlink -f ./result)"
-      #     echo "::set-output name=path::$result_path"
+        run: nix build .#nixosConfigurations.x86_64-linux.contabo
 
       - name: Release
         uses: softprops/action-gh-release@v1

diff --git a/Makefile b/Makefile
@@ -1,7 +1,10 @@
 SHELL:=/usr/bin/env bash
 MAKEFLAGS += --no-builtin-rules --no-builtin-variables
 
+#### Nix
+
 BUILDER_EXEC:=
+NIXOS_CONFIG:=qcow
 
 ifeq ($(shell uname -s),Darwin)
    BUILDER_EXEC:=NIX_CONF_DIR=$(PWD)/bootstrap nix develop .\#builder --command
@@ -11,10 +14,10 @@ bootstrap:
 	@$(BUILDER_EXEC) echo "Started build environment"
 
 build:
-	@$(BUILDER_EXEC) nix build .#nixosConfigurations.aarch64-darwin.default --system aarch64-linux $(ARGS)
+	@$(BUILDER_EXEC) nix build .#nixosConfigurations.aarch64-darwin.$(NIXOS_CONFIG) --system aarch64-linux $(ARGS)
 
 build-x86:
-	@$(BUILDER_EXEC) nix build .#nixosConfigurations.x86_64-darwin.default --system x86_64-linux $(ARGS)
+	@$(BUILDER_EXEC) nix build .#nixosConfigurations.x86_64-darwin.$(NIXOS_CONFIG) --system x86_64-linux $(ARGS)
 
 #### Terraform
 
@@ -49,4 +52,14 @@ $(TF_ROOT_DIRS_VALIDATE):
 	@$(eval DIR:=$(subst -validate,,$@))
 	terraform -chdir=$(DIR) validate -no-color $(ARGS)
 
-.PHONY: fmt validate build build-x86 bootstrap init $(TF_ROOT_DIRS) $(TF_ROOT_DIRS_DESTROY) $(TF_ROOT_DIRS_INIT)
+#### Image server
+
+serve-iso:
+	@nohup python -m http.server -d result/iso &
+
+kill-iso-server:
+	@pkill -f "python -m http.server"
+
+.PHONY: fmt validate build build-x86 bootstrap init \
+  $(TF_ROOT_DIRS) $(TF_ROOT_DIRS_DESTROY) $(TF_ROOT_DIRS_INIT) \
+  serve-iso kill-iso-server
diff --git a/README.md b/README.md
@@ -8,7 +8,7 @@ Compatibility Matrix :
 | OS | Status |
 | --- | --- |
 | Darwin | OK |
-| Linux | missing builder tooling |
+| Linux | NO |
 
 ## New Nix system (beta)
 

diff --git a/flake.nix b/flake.nix
@@ -94,8 +94,8 @@
 
         # Need a bare darwinConfigurations.builder started before building this one.
         builder-docker = self.darwinConfigurations.builder.override {
-          extraModules = singleton {
-            nix.linux-builder.config = ./nixos-darwin/linux-builder-docker.nix;
+          extraModules = attrValues {
+            linux-docker-builder = ./nixos-darwin/linux-builder-docker.nix;
           };
         };
 
@@ -136,13 +136,17 @@
           };
         };
 
+        iso = self.nixosConfigurations.${system}.qcow.override {
+          format = "iso";
+        };
+
         contabo = self.nixosConfigurations.${system}.qcow.override {
           modules = attrValues self.nixosModules ++ [
             ./nixos/contabo.nix
           ];
         };
 
-        docker = self.nixosConfigurations.${system}.qcow.override {
+        container = self.nixosConfigurations.${system}.qcow.override {
           modules = attrValues self.nixosModules ++ [ 
             ./nixos/docker.nix
           ];
@@ -165,7 +169,7 @@
               inherit (pkgs) bashInteractive grpcurl jq coreutils e2fsprogs
               docker-client kubectl kubernetes-helm libvirt qemu
               tailscale pebble cntb
-              nil nix-tree;
+              nil nix-tree python3;
               inherit (stablePkgs) terraform waypoint;
             };
             shellHook = ''

diff --git a/nixos-darwin/configuration.nix b/nixos-darwin/configuration.nix
@@ -1,7 +1,9 @@
 { 
   pkgs,
-  config, 
-  ... }:
+  config,
+  lib,
+  ... 
+}:
 {
   programs.fish.enable = true;
   programs.bash.enable = true;
@@ -54,8 +56,8 @@
     unix_sock_admin_perms = "0770"
     auth_unix_ro = "none"
     auth_unix_rw = "none"
-    log_level = 1
-    log_outputs="1:stderr"
+    log_level = 3
+    log_outputs="3:stderr"
   '';
   environment.etc."libvirt/qemu.conf".text = ''
     security_driver = "none"
@@ -94,8 +96,12 @@
   nix.linux-builder = {
     enable = true;
     maxJobs = 8;
-    package = pkgs.darwin.linux-builder;
-    ephemeral = true;
+    package = lib.mkDefault pkgs.darwin.linux-builder-x86_64;
+    ephemeral = lib.mkDefault true;
+    config = lib.mkDefault ({ lib, ... }: { 
+      # WAITING FOR https://github.com/NixOS/nixpkgs/issues/313784
+      # nixpkgs.hostPlatform = lib.mkForce "x86_64-linux";
+    });
   };
   nix.configureBuildUsers = true;
   services.nix-daemon.enable = true;

diff --git a/nixos-darwin/linux-builder-docker.nix b/nixos-darwin/linux-builder-docker.nix
@@ -1,13 +1,17 @@
-{ lib, ... }: {
-  virtualisation.docker.enable = true;
-  virtualisation.docker.daemon.settings = {
-    hosts = [ "tcp://0.0.0.0:2375" ];
-  };
-  networking.firewall.enable = lib.mkForce false;
-  virtualisation.forwardPorts = lib.mkForce [
-    { from = "host"; guest.port = 22; host.port = 31022; }
-    { from = "host"; guest.port = 2375; host.port = 2375; }
-  ];
-  security.sudo.wheelNeedsPassword = false;
-  users.users.builder.extraGroups = lib.mkForce [ "docker" "wheel" ];
+{ pkgs, ... }: {
+  nix.linux-builder.package = pkgs.darwin.linux-builder;
+  nix.linux-builder.ephemeral = false;
+  nix.linux-builder.config = ({ lib, ... }: {
+    virtualisation.docker.enable = true;
+    virtualisation.docker.daemon.settings = {
+      hosts = [ "tcp://0.0.0.0:2375" ];
+    };
+    networking.firewall.enable = lib.mkForce false;
+    virtualisation.forwardPorts = lib.mkForce [
+      { from = "host"; guest.port = 22; host.port = 31022; }
+      { from = "host"; guest.port = 2375; host.port = 2375; }
+    ];
+    security.sudo.wheelNeedsPassword = false;
+    users.users.builder.extraGroups = lib.mkForce [ "docker" "wheel" ];
+  });
 }
diff --git a/nixos-options/default.nix b/nixos-options/default.nix
@@ -25,7 +25,7 @@
     };
 
     user.name = lib.mkOption {
-      default = "zizou";
+      default = "admin";
       type = lib.types.str;
       description = "User name";
     };

diff --git a/nixos/configuration.nix b/nixos/configuration.nix
@@ -3,40 +3,48 @@
   lib,
   pkgs,
   stableLegacyPackages,
+  modulesPath,
   ...
 }: 
 
 let
   dex_hostname = "https://dex.${config.k3s-paas.dns.name}";
-  certs =  builtins.map (cert: builtins.fetchurl { inherit (cert) url sha256; }) config.k3s-paas.certs;
+  certs = builtins.map (cert: builtins.fetchurl { inherit (cert) url sha256; }) config.k3s-paas.certs;
   certManagerCrds = builtins.fetchurl {
     url = "https://github.com/cert-manager/cert-manager/releases/download/v1.14.4/cert-manager.crds.yaml";
     sha256 = "060bn3gvrr5jphaig1g195prip5rn0x1s7qrp09q47719fgc6636";
   };
   manifests = builtins.filter (d: d != "") [certManagerCrds];
 in {
+
+  system.build.qcow = lib.mkForce (import "${toString modulesPath}/../lib/make-disk-image.nix" {
+    inherit lib config pkgs;
+    diskSize = "auto";
+    format = "qcow2-compressed";
+    partitionTableType = "hybrid";
+  });
+
   console = {
     earlySetup = true;
     keyMap = "fr";
   };
 
+  boot.tmp.cleanOnBoot = true;
   boot.kernelPackages = pkgs.linuxPackages_latest;
-  fileSystems."/".autoResize = true;
-  fileSystems."/boot" =
-    { device = "/dev/disk/by-label/boot";
-      fsType = "vfat";
-    };
+  boot.loader.systemd-boot.consoleMode = "auto";
 
-  swapDevices = [ {
-    device = "/var/lib/swapfile";
-    size = 16 * 1024;
-  } ];
+  zramSwap.algorithm  = "zstd";
 
-  boot.loader.systemd-boot.consoleMode = "auto";
+  # fileSystems = {
+  #   "/boot" = { 
+  #     device = "/dev/disk/by-label/boot";
+  #     fsType = "vfat";
+  #   };
+  # };
+
+  #services.cloud-init.enable = true;
 
   system.stateVersion = "23.05";
-  # FIXME: when branch is merged, uncomment the following line
-  # system.autoUpgrade.flake = "github:loic-roux-404/k3s-paas#nixosConfigurations.${pkgs.system}.default";
 
   time = {
     timeZone = lib.mkForce "Europe/Paris";
@@ -143,18 +151,18 @@ in {
   networking = {
     hostName = "k3s-paas";
     useNetworkd = true;
-    useDHCP = false;
+    useDHCP = true;
     firewall = {
       enable = true;
       allowedTCPPorts = lib.mkForce [80 443 22 6443];
     };
     nftables.enable = true;
-    networkmanager.enable = true;
+    networkmanager.enable = false;
     usePredictableInterfaceNames = true;
   };
 
   systemd.network = {
-    enable = true;
+    enable = lib.mkForce true;
     wait-online.anyInterface = true;
   };
 

diff --git a/nixos/contabo.nix b/nixos/contabo.nix
@@ -1,12 +1,17 @@
 { lib, ... }:
 {
-  # boot.loader.grub = {
-  #   efiSupport = true;
-  #   efiInstallAsRemovable = true;
-  #   device = "nodev";
-  # };
 
-  boot.initrd.kernelModules = lib.mkForce ["dm-snapshot"];
+  boot.loader.grub.device = lib.mkForce "/dev/sda";
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "virtio_pci" "virtio_scsi" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  swapDevices = [ ];
+
   k3s-paas.dns.name = "404-tools.xyz";
   k3s-paas.certs = [];
+  # system.autoUpgrade.flake = "github:loic-roux-404/k3s-paas#nixosConfigurations.${pkgs.system}.default";
 }
+
diff --git a/tf-root-contabo/main.tf b/tf-root-contabo/main.tf
@@ -60,21 +60,15 @@ locals {
   })
 }
 
-resource "contabo_secret" "paas_instance_ssh_key" {
-  name  = "paas_instance_ssh_key"
+resource "contabo_secret" "paas_instance_trusted_key" {
+  name  = "paas_instance_trusted_key"
   type  = "ssh"
   value = local.ssh_connection.public_key
 }
 
-resource "contabo_secret" "paas_instance_password" {
-  name  = "paas_instance_password"
-  type  = "password"
-  value = local.ssh_connection.password
-}
-
-resource "contabo_image" "paas_instance_qcow2" {
+resource "contabo_image" "paas_instance_image" {
   name        = "k3s"
-  image_url   = var.image_url
+  image_url   = format(var.image_url_format, var.image_version)
   os_type     = "Linux"
   version     = var.image_version
   description = "Generated PaaS vm image with packer"
@@ -87,14 +81,8 @@ data "contabo_instance" "paas_instance" {
 resource "contabo_instance" "paas_instance" {
   existing_instance_id = var.contabo_instance
   display_name         = "nixos-k3s-paas"
-  image_id             = contabo_image.paas_instance_qcow2.id
-  ssh_keys             = [contabo_secret.paas_instance_ssh_key.id]
-  user_data = sensitive(templatefile(
-    "${path.root}/user-data.yaml.tmpl",
-    {
-      tailscale_key = tailscale_tailnet_key.k3s_paas_node.key
-    }
-  ))
+  image_id             = contabo_image.paas_instance_image.id
+  ssh_keys             = [contabo_secret.paas_instance_trusted_key.id]
 }
 
 resource "terraform_data" "paas_instance_wait_bootstrap" {
@@ -109,10 +97,13 @@ resource "terraform_data" "paas_instance_wait_bootstrap" {
     host        = contabo_instance.paas_instance.ip_config[0].v4[0].ip
   }
 
+  # TODO move following code in a nix configuration to update machine with
+  # new user, ssh key password and tailscale link
   provisioner "remote-exec" {
     on_failure = fail
     inline = [
-      "sudo cloud-init status --wait && sudo cloud-init clean"
+      "echo ${contabo_instance.paas_instance.id}",
+      "tailscale, up, -authkey, '${tailscale_tailnet_key.k3s_paas_node.key}'"
     ]
   }
 }
diff --git a/tf-root-contabo/user-data.yaml.tmpl b/tf-root-contabo/user-data.yaml.tmpl