feat: finalizing paas and enforce security

.github/workflows/release-nixos.yml

@@ -39,7 +39,7 @@ jobs:
 
       - name: Build
         id: build
-        run: nix build .#nixosConfigurations.contabo
+        run: nix build .#nixosConfigurations.contabo-qcow
 
       - name: Release
         uses: softprops/action-gh-release@v1

Makefile

@@ -1,31 +1,30 @@
 SHELL:=/usr/bin/env bash
 MAKEFLAGS += --no-builtin-rules --no-builtin-variables
 TF_CMD:=apply -auto-approve
-VARIANT:=builder
+VARIANT=builder
 
 #### Nix
 
 BUILDER_EXEC:=
 NIXOS_CONFIG:=qcow
 
 ifeq ($(shell uname -s),Darwin)
-   BUILDER_EXEC:=NIX_CONF_DIR=$(PWD)/bootstrap nix develop .\#$(BUILDER) --command
+   BUILDER_EXEC:=NIX_CONF_DIR=$(PWD)/bootstrap nix develop .\#builder --command
 endif
 
 bootstrap:
 	@$(BUILDER_EXEC) echo "Started default build environment"
 
 bootstrap-x86:
-	@VARIANT=$(VARIANT)=builder-x86 $(BUILDER_EXEC) echo "Started x86 environment"
+	@VARIANT=builder-x86 $(BUILDER_EXEC) echo "Started x86 environment"
 
 nixos-local:
 	@$(BUILDER_EXEC) nix build .#nixosConfigurations.default --system aarch64-linux
 
 TERRAGRUNT_FILES:=$(shell find terragrunt -type d -name '.*' -prune -o -name 'terragrunt.hcl' -exec dirname {} \;)
 
 $(TERRAGRUNT_FILES):
-	@echo "Running apply in $@ directory"
-	@chmod -f -R 777 result/ || true
+	@chmod -fR 755 $@/.terragrunt-cache/ && chmod -fR 755 result || true
 	@cd $@ && terragrunt $(TF_CMD)
 
 release-stable:

README.md

@@ -134,6 +134,13 @@ make terragrunt/network/contabo
 make terragrunt/paas/contabo
 ```
 
+**Note** : Retrieve any config with :
+
+```bash
+make terragrunt/network/contabo TF_CMD='output -json k3s_config | yq -p json -o yaml'
+
+```
+
 ### Install in local
 
 Here is the command to set up the paas on a libvirt vm.
@@ -233,14 +240,6 @@ See any assets :
 kubectl get all -A
 ```
 
-### Tailscale
-
-Retrieve kubeconfig :
-
-```bash
-tailscale configure kubeconfig
-```
-
 ### Release
 
 ```bash

flake.nix

@@ -12,7 +12,7 @@
 
     # Environment/system management
     darwin.url = "github:LnL7/nix-darwin";
-    darwin.inputs.nixpkgs.follows = "srvos/nixpkgs";
+    darwin.inputs.nixpkgs.follows = "nixpkgs-stable-darwin";
 
     home-manager = { 
       url = "github:nix-community/home-manager/master"; 
@@ -169,7 +169,7 @@
           format = "qcow";
         };
 
-        contabo = self.packages.${system}.nixosConfigurations.qcow.override {
+        contabo-qcow = self.packages.${system}.nixosConfigurations.qcow.override {
           modules = self.nixosAllModules.contabo ++ [
             ./nixos/qcow-compressed.nix
           ];
@@ -227,7 +227,7 @@
             };
             shellHook = ''
               set -e
-              nix build .#darwinConfigurations.builder''${VARIANT:-builder}.system
+              nix build .#darwinConfigurations.''${VARIANT:-builder}.system
               ./result/sw/bin/darwin-rebuild switch --flake .#''${VARIANT:-builder}
             '';
           };

nixos-darwin/configuration.nix

@@ -17,34 +17,37 @@
   };
   services.tailscale.enable = true;
 
-  launchd.daemons."libvirt" = {
+  launchd.daemons.libvirt = {
     path = [ pkgs.gcc pkgs.qemu pkgs.dnsmasq pkgs.libvirt ];
     serviceConfig = {
       KeepAlive = true;
       RunAtLoad = true;
       ProgramArguments = [ 
         "${pkgs.libvirt}/bin/libvirtd" "-f" "/etc/libvirt/libvirtd.conf" "-v"
       ];
+      WorkingDirectory = "/var/lib/libvirt";
       StandardOutPath = "/var/log/libvirt/libvirt.log";
       StandardErrorPath = "/var/log/libvirt/libvirt-error.log";
     };
   };
-  launchd.daemons."virtlogd" = {
+  launchd.daemons.virtlogd = {
     path = [ pkgs.libvirt ];
     serviceConfig = {
       KeepAlive = true;
       RunAtLoad = true;
+      WorkingDirectory = "/var/lib/libvirt";
       ProgramArguments = [ "${pkgs.libvirt}/bin/virtlogd" "-d" ];
       StandardOutPath = "/var/log/libvirt/virtlogd.log";
       StandardErrorPath = "/var/log/libvirt/virtlogd-error.log";
     };
   };
-  launchd.daemons."pebble" = {
+  launchd.daemons.pebble = {
     path = [ pkgs.pebble ];
     serviceConfig = {
       KeepAlive = true;
       RunAtLoad = true;
       ProgramArguments = [ "${pkgs.pebble}/bin/pebble" "-config" "/etc/pebble/config.json" ];
+      WorkingDirectory = "/var/lib/pebble";
       StandardOutPath = "/var/log/pebble.log";
       StandardErrorPath = "/var/log/pebble-error.log";
     };

nixos/configuration.nix

@@ -72,6 +72,7 @@ in {
       role = "server";
       extraFlags = lib.strings.concatStringsSep " " (
         map (service: "--disable=${service}") k3s.disableServices
+        ++ ["--write-kubeconfig-mode=400" "--write-kubeconfig-user=${user.name}"]
         ++ k3s.serverExtraArgs
       );
     };
@@ -105,6 +106,8 @@ in {
     shells = [ pkgs.bashInteractive ];
     variables = {
       PAGER = "less -FirSwX";
+      KUBECONFIG = "/etc/rancher/k3s/k3s.yaml";
+      SYSTEMD_EDITOR = "vim";
     };
     systemPackages = with pkgs; [
       glibcLocales
@@ -192,7 +195,7 @@ in {
     firewall = {
       trustedInterfaces = [ "tailscale0" "cni0" ];
       enable = true;
-      allowedTCPPorts = lib.mkDefault [80 443 22 6443];
+      allowedTCPPorts = lib.mkDefault [ 80 443 22 ];
       allowedUDPPorts = [ config.services.tailscale.port ];
     };
     nftables.enable = true;

nixos/deploy.nix

@@ -8,6 +8,7 @@ with config.k3s-paas;
   sops.defaultSopsFile = "/home/${user.name}/secrets.yaml";
   sops.age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
 
+  networking.firewall.allowedTCPPorts = [ 80 443 ];
   services.tailscale.authKeyFile = config.sops.secrets.tailscaleNodeKey.path;
   services.tailscale.extraUpFlags = ["--ssh" "--hostname=${config.networking.hostName}" ];
 

nixos/install-k3s-manifest.nix

@@ -9,7 +9,8 @@ in {
   script = ''
     mkdir -p /var/lib/rancher/k3s/server/manifests;
     cp -fp ${file} /var/lib/rancher/k3s/server/manifests;
-    sleep 30;
-    ${pkgs.k3s}/bin/k3s kubectl wait --for='${condition}' ${toWait} ${namespaceExpr} --timeout=2m;
+    sleep 15;
+    export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+    ${pkgs.kubectl}/bin/kubectl wait --for='${condition}' ${toWait} ${namespaceExpr} --timeout=2m;
   '';
 }

tf-modules-cloud/tailscale/main.tf

@@ -103,20 +103,30 @@ resource "terraform_data" "destroy_node" {
   }
 }
 
+data "tailscale_devices" "already_present" {
+  name_prefix = var.node_hostname
+}
+
+locals  {
+  already_present = length(data.tailscale_devices.already_present.devices) > 0
+  node_fqdn = "${var.node_hostname}.${var.tailscale_tailnet}"
+}
+
 output "node_id" {
   value = var.node_id
 }
 
-output "node_ip" {
-  value = var.node_ip
+output "node_address" {
+  value = local.already_present ? local.node_fqdn : var.node_ip
 }
 
 output "config" {
   depends_on = [tailscale_tailnet_key.k3s_paas_node]
   value = {
-    node_hostname = var.node_hostname
-    node_fqdn     = "${var.node_hostname}.${var.tailscale_tailnet}"
-    node_key      = tailscale_tailnet_key.k3s_paas_node.key
-    k8s_operator_hostname  = "k8s-operator-${var.node_hostname}"
+    node_ip               = var.node_ip
+    node_hostname         = var.node_hostname
+    node_fqdn             = local.node_fqdn
+    node_key              = tailscale_tailnet_key.k3s_paas_node.key
+    k8s_operator_hostname = "k8s-operator-${var.node_hostname}"
   }
 }

tf-modules-k8s/cert-manager/main.tf

@@ -11,7 +11,7 @@ resource "helm_release" "cert_manager" {
   chart         = "cert-manager"
   version       = "1.14.4"
   wait_for_jobs = true
-  wait          = true
+  atomic = true
 
   set {
     name  = "installCRDs"

tf-modules-k8s/dex/main.tf

@@ -20,7 +20,7 @@ resource "helm_release" "dex" {
   chart         = "dex"
   timeout       = 600
   wait_for_jobs = true
-  wait          = true
+  atomic        = true
 
   values = [
     templatefile("${path.module}/values.yaml.tmpl", {

tf-modules-k8s/github/main.tf

@@ -2,17 +2,11 @@ data "github_organization" "org" {
   name = var.github_organization
 }
 
-data "github_membership" "all" {
-  for_each = toset(data.github_organization.org.members)
-  username = each.value
-}
-
-data "github_membership" "all_admin" {
-  for_each = {
-    for _, member in data.github_membership.all :
-    _ => member if member.role == "admin"
+locals {
+  admins = {
+    for _, member in data.github_organization.org.users :
+    _ => member.login if lower(member.role) == "admin"
   }
-  username = each.value.username
 }
 
 resource "github_team" "opsteam" {
@@ -22,9 +16,9 @@ resource "github_team" "opsteam" {
 }
 
 resource "github_team_membership" "opsteam_members" {
-  for_each = data.github_membership.all_admin
+  for_each = local.admins
   team_id  = github_team.opsteam.id
-  username = each.value.username
+  username = each.value
   role     = "maintainer"
 }
 

tf-modules-k8s/waypoint/main.tf

@@ -17,19 +17,16 @@ locals {
 }
 
 resource "helm_release" "waypoint" {
-  name          = "waypoint"
-  repository    = "https://helm.releases.hashicorp.com"
-  chart         = "waypoint"
-  version       = "0.1.21"
-  namespace     = local.namespace
-  values        = [local.waypoint_manifest_values]
-  wait_for_jobs = true
-  wait          = true
-
-  set {
-    name  = "targetNamespace"
-    value = local.namespace
-  }
+  name              = "waypoint"
+  repository        = "https://helm.releases.hashicorp.com"
+  chart             = "waypoint"
+  version           = "0.1.21"
+  namespace         = local.namespace
+  values            = [local.waypoint_manifest_values]
+  atomic            = true
+  wait_for_jobs     = true
+  dependency_update = var.dependency_update
+  timeout           = 120
 }
 
 data "kubernetes_secret" "waypoint_token" {

tf-modules-k8s/waypoint/values.yaml.tmpl

@@ -1,6 +1,10 @@
+
+targetNamespace: ${waypoint_namespace}
+
 server:
   image:
     tag: 0.11.4
+  # tokenSecret: TODO
   cert:
     secretName: ${paas_hostname}-tls
   extraVolumeMounts: ${jsonencode(waypoint_extra_volume_mounts)}

tf-modules-k8s/waypoint/variables.tf

@@ -27,3 +27,7 @@ variable "waypoint_extra_volumes" {
 variable "cert_manager_cluster_issuer" {
   description = "value of the cert-manager cluster issuer"
 }
+
+variable "dependency_update" {
+  default = true
+}

tf-modules-nix/deploy/main.tf

@@ -12,7 +12,7 @@ resource "terraform_data" "check_ssh" {
     type        = "ssh"
     user        = var.ssh_connection.user
     private_key = file(pathexpand(var.ssh_connection.private_key))
-    host        = var.node_ip
+    host        = var.node_address
     timeout     = "1m"
   }
 
@@ -26,13 +26,13 @@ data "external" "machine_key_pub" {
   program    = ["bash", "${path.module}/retrieve-vm-age-key.sh"]
 
   query = {
-    machine_ip = var.node_ip
+    machine_ip = var.node_address
   }
 }
 
 resource "local_sensitive_file" "non_encrypted_secrets" {
   content  = yamlencode(var.nixos_transient_secrets)
-  filename = "${path.cwd}/${var.node_ip}.yaml"
+  filename = "${path.cwd}/${var.node_address}.yaml"
 }
 
 resource "terraform_data" "create_transient_secrets" {
@@ -63,7 +63,7 @@ resource "terraform_data" "upload_secrets" {
     type        = "ssh"
     user        = var.ssh_connection.user
     private_key = file(pathexpand(var.ssh_connection.private_key))
-    host        = var.node_ip
+    host        = var.node_address
   }
 
   provisioner "file" {
@@ -81,8 +81,8 @@ locals {
     [
       "nixos-rebuild",
       "--fast",
-      "--build-host", "${var.ssh_connection.user}@${var.node_ip}",
-      "--target-host", "${var.ssh_connection.user}@${var.node_ip}"
+      "--build-host", "${var.ssh_connection.user}@${var.node_address}",
+      "--target-host", "${var.ssh_connection.user}@${var.node_address}"
     ],
     var.nix_rebuild_arguments
   )
@@ -108,7 +108,7 @@ resource "terraform_data" "deploy" {
 output "config" {
   depends_on = [terraform_data.deploy]
   value = merge(var.config, {
-    node_ip = var.node_ip
+    node_address = var.node_address
     node_id = var.node_id
   })
 }

tf-modules-nix/deploy/variables.tf

@@ -4,7 +4,7 @@ variable "dex_client_id" {
   default   = "dex-k3s-paas"
 }
 
-variable "node_ip" {
+variable "node_address" {
   type = string
 }