diff --git a/.github/workflows/release-nixos.yml b/.github/workflows/release-nixos.yml index 36a18e8b..3f272312 100644 --- a/.github/workflows/release-nixos.yml +++ b/.github/workflows/release-nixos.yml @@ -39,7 +39,7 @@ jobs: - name: Build id: build - run: nix build .#nixosConfigurations.contabo + run: nix build .#nixosConfigurations.contabo-qcow - name: Release uses: softprops/action-gh-release@v1 diff --git a/Makefile b/Makefile index 313567e5..069d1815 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ SHELL:=/usr/bin/env bash MAKEFLAGS += --no-builtin-rules --no-builtin-variables TF_CMD:=apply -auto-approve -VARIANT:=builder +VARIANT=builder #### Nix @@ -9,14 +9,14 @@ BUILDER_EXEC:= NIXOS_CONFIG:=qcow ifeq ($(shell uname -s),Darwin) - BUILDER_EXEC:=NIX_CONF_DIR=$(PWD)/bootstrap nix develop .\#$(BUILDER) --command + BUILDER_EXEC:=NIX_CONF_DIR=$(PWD)/bootstrap nix develop .\#builder --command endif bootstrap: @$(BUILDER_EXEC) echo "Started default build environment" bootstrap-x86: - @VARIANT=$(VARIANT)=builder-x86 $(BUILDER_EXEC) echo "Started x86 environment" + @VARIANT=builder-x86 $(BUILDER_EXEC) echo "Started x86 environment" nixos-local: @$(BUILDER_EXEC) nix build .#nixosConfigurations.default --system aarch64-linux @@ -24,8 +24,7 @@ nixos-local: TERRAGRUNT_FILES:=$(shell find terragrunt -type d -name '.*' -prune -o -name 'terragrunt.hcl' -exec dirname {} \;) $(TERRAGRUNT_FILES): - @echo "Running apply in $@ directory" - @chmod -f -R 777 result/ || true + @chmod -fR 755 $@/.terragrunt-cache/ && chmod -fR 755 result || true @cd $@ && terragrunt $(TF_CMD) release-stable: diff --git a/README.md b/README.md index 8e019310..deae25e1 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,13 @@ make terragrunt/network/contabo make terragrunt/paas/contabo ``` +**Note** : Retrieve any config with : + +```bash +make terragrunt/network/contabo TF_CMD='output -json k3s_config | yq -p json -o yaml' + +``` + ### Install in local Here is the command to set up the paas on a libvirt vm. @@ -233,14 +240,6 @@ See any assets : kubectl get all -A ``` -### Tailscale - -Retrieve kubeconfig : - -```bash -tailscale configure kubeconfig -``` - ### Release ```bash diff --git a/flake.nix b/flake.nix index 61aa8a29..db9ec384 100644 --- a/flake.nix +++ b/flake.nix @@ -12,7 +12,7 @@ # Environment/system management darwin.url = "github:LnL7/nix-darwin"; - darwin.inputs.nixpkgs.follows = "srvos/nixpkgs"; + darwin.inputs.nixpkgs.follows = "nixpkgs-stable-darwin"; home-manager = { url = "github:nix-community/home-manager/master"; @@ -169,7 +169,7 @@ format = "qcow"; }; - contabo = self.packages.${system}.nixosConfigurations.qcow.override { + contabo-qcow = self.packages.${system}.nixosConfigurations.qcow.override { modules = self.nixosAllModules.contabo ++ [ ./nixos/qcow-compressed.nix ]; @@ -227,7 +227,7 @@ }; shellHook = '' set -e - nix build .#darwinConfigurations.builder''${VARIANT:-builder}.system + nix build .#darwinConfigurations.''${VARIANT:-builder}.system ./result/sw/bin/darwin-rebuild switch --flake .#''${VARIANT:-builder} ''; }; diff --git a/nixos-darwin/configuration.nix b/nixos-darwin/configuration.nix index 789e1ca5..10b46689 100644 --- a/nixos-darwin/configuration.nix +++ b/nixos-darwin/configuration.nix @@ -17,7 +17,7 @@ }; services.tailscale.enable = true; - launchd.daemons."libvirt" = { + launchd.daemons.libvirt = { path = [ pkgs.gcc pkgs.qemu pkgs.dnsmasq pkgs.libvirt ]; serviceConfig = { KeepAlive = true; @@ -25,26 +25,29 @@ ProgramArguments = [ "${pkgs.libvirt}/bin/libvirtd" "-f" "/etc/libvirt/libvirtd.conf" "-v" ]; + WorkingDirectory = "/var/lib/libvirt"; StandardOutPath = "/var/log/libvirt/libvirt.log"; StandardErrorPath = "/var/log/libvirt/libvirt-error.log"; }; }; - launchd.daemons."virtlogd" = { + launchd.daemons.virtlogd = { path = [ pkgs.libvirt ]; serviceConfig = { KeepAlive = true; RunAtLoad = true; + WorkingDirectory = "/var/lib/libvirt"; ProgramArguments = [ "${pkgs.libvirt}/bin/virtlogd" "-d" ]; StandardOutPath = "/var/log/libvirt/virtlogd.log"; StandardErrorPath = "/var/log/libvirt/virtlogd-error.log"; }; }; - launchd.daemons."pebble" = { + launchd.daemons.pebble = { path = [ pkgs.pebble ]; serviceConfig = { KeepAlive = true; RunAtLoad = true; ProgramArguments = [ "${pkgs.pebble}/bin/pebble" "-config" "/etc/pebble/config.json" ]; + WorkingDirectory = "/var/lib/pebble"; StandardOutPath = "/var/log/pebble.log"; StandardErrorPath = "/var/log/pebble-error.log"; }; diff --git a/nixos/configuration.nix b/nixos/configuration.nix index 8cc2e684..13407688 100644 --- a/nixos/configuration.nix +++ b/nixos/configuration.nix @@ -72,6 +72,7 @@ in { role = "server"; extraFlags = lib.strings.concatStringsSep " " ( map (service: "--disable=${service}") k3s.disableServices + ++ ["--write-kubeconfig-mode=400" "--write-kubeconfig-user=${user.name}"] ++ k3s.serverExtraArgs ); }; @@ -105,6 +106,8 @@ in { shells = [ pkgs.bashInteractive ]; variables = { PAGER = "less -FirSwX"; + KUBECONFIG = "/etc/rancher/k3s/k3s.yaml"; + SYSTEMD_EDITOR = "vim"; }; systemPackages = with pkgs; [ glibcLocales @@ -192,7 +195,7 @@ in { firewall = { trustedInterfaces = [ "tailscale0" "cni0" ]; enable = true; - allowedTCPPorts = lib.mkDefault [80 443 22 6443]; + allowedTCPPorts = lib.mkDefault [ 80 443 22 ]; allowedUDPPorts = [ config.services.tailscale.port ]; }; nftables.enable = true; diff --git a/nixos/deploy.nix b/nixos/deploy.nix index fe43d900..9d6fe057 100644 --- a/nixos/deploy.nix +++ b/nixos/deploy.nix @@ -8,6 +8,7 @@ with config.k3s-paas; sops.defaultSopsFile = "/home/${user.name}/secrets.yaml"; sops.age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ]; + networking.firewall.allowedTCPPorts = [ 80 443 ]; services.tailscale.authKeyFile = config.sops.secrets.tailscaleNodeKey.path; services.tailscale.extraUpFlags = ["--ssh" "--hostname=${config.networking.hostName}" ]; diff --git a/nixos/install-k3s-manifest.nix b/nixos/install-k3s-manifest.nix index 48ea4f5c..509ccb21 100644 --- a/nixos/install-k3s-manifest.nix +++ b/nixos/install-k3s-manifest.nix @@ -9,7 +9,8 @@ in { script = '' mkdir -p /var/lib/rancher/k3s/server/manifests; cp -fp ${file} /var/lib/rancher/k3s/server/manifests; - sleep 30; - ${pkgs.k3s}/bin/k3s kubectl wait --for='${condition}' ${toWait} ${namespaceExpr} --timeout=2m; + sleep 15; + export KUBECONFIG=/etc/rancher/k3s/k3s.yaml + ${pkgs.kubectl}/bin/kubectl wait --for='${condition}' ${toWait} ${namespaceExpr} --timeout=2m; ''; } diff --git a/tf-modules-cloud/tailscale/main.tf b/tf-modules-cloud/tailscale/main.tf index ee452bdb..95c830e3 100644 --- a/tf-modules-cloud/tailscale/main.tf +++ b/tf-modules-cloud/tailscale/main.tf @@ -103,20 +103,30 @@ resource "terraform_data" "destroy_node" { } } +data "tailscale_devices" "already_present" { + name_prefix = var.node_hostname +} + +locals { + already_present = length(data.tailscale_devices.already_present.devices) > 0 + node_fqdn = "${var.node_hostname}.${var.tailscale_tailnet}" +} + output "node_id" { value = var.node_id } -output "node_ip" { - value = var.node_ip +output "node_address" { + value = local.already_present ? local.node_fqdn : var.node_ip } output "config" { depends_on = [tailscale_tailnet_key.k3s_paas_node] value = { - node_hostname = var.node_hostname - node_fqdn = "${var.node_hostname}.${var.tailscale_tailnet}" - node_key = tailscale_tailnet_key.k3s_paas_node.key - k8s_operator_hostname = "k8s-operator-${var.node_hostname}" + node_ip = var.node_ip + node_hostname = var.node_hostname + node_fqdn = local.node_fqdn + node_key = tailscale_tailnet_key.k3s_paas_node.key + k8s_operator_hostname = "k8s-operator-${var.node_hostname}" } } diff --git a/tf-modules-k8s/cert-manager/main.tf b/tf-modules-k8s/cert-manager/main.tf index 452a210e..1479cb3c 100644 --- a/tf-modules-k8s/cert-manager/main.tf +++ b/tf-modules-k8s/cert-manager/main.tf @@ -11,7 +11,7 @@ resource "helm_release" "cert_manager" { chart = "cert-manager" version = "1.14.4" wait_for_jobs = true - wait = true + atomic = true set { name = "installCRDs" diff --git a/tf-modules-k8s/dex/main.tf b/tf-modules-k8s/dex/main.tf index e8617269..066a3ba3 100644 --- a/tf-modules-k8s/dex/main.tf +++ b/tf-modules-k8s/dex/main.tf @@ -20,7 +20,7 @@ resource "helm_release" "dex" { chart = "dex" timeout = 600 wait_for_jobs = true - wait = true + atomic = true values = [ templatefile("${path.module}/values.yaml.tmpl", { diff --git a/tf-modules-k8s/github/main.tf b/tf-modules-k8s/github/main.tf index 05695f39..3ed17b3b 100644 --- a/tf-modules-k8s/github/main.tf +++ b/tf-modules-k8s/github/main.tf @@ -2,17 +2,11 @@ data "github_organization" "org" { name = var.github_organization } -data "github_membership" "all" { - for_each = toset(data.github_organization.org.members) - username = each.value -} - -data "github_membership" "all_admin" { - for_each = { - for _, member in data.github_membership.all : - _ => member if member.role == "admin" +locals { + admins = { + for _, member in data.github_organization.org.users : + _ => member.login if lower(member.role) == "admin" } - username = each.value.username } resource "github_team" "opsteam" { @@ -22,9 +16,9 @@ resource "github_team" "opsteam" { } resource "github_team_membership" "opsteam_members" { - for_each = data.github_membership.all_admin + for_each = local.admins team_id = github_team.opsteam.id - username = each.value.username + username = each.value role = "maintainer" } diff --git a/tf-modules-k8s/waypoint/main.tf b/tf-modules-k8s/waypoint/main.tf index d59c7642..5925ae00 100644 --- a/tf-modules-k8s/waypoint/main.tf +++ b/tf-modules-k8s/waypoint/main.tf @@ -17,19 +17,16 @@ locals { } resource "helm_release" "waypoint" { - name = "waypoint" - repository = "https://helm.releases.hashicorp.com" - chart = "waypoint" - version = "0.1.21" - namespace = local.namespace - values = [local.waypoint_manifest_values] - wait_for_jobs = true - wait = true - - set { - name = "targetNamespace" - value = local.namespace - } + name = "waypoint" + repository = "https://helm.releases.hashicorp.com" + chart = "waypoint" + version = "0.1.21" + namespace = local.namespace + values = [local.waypoint_manifest_values] + atomic = true + wait_for_jobs = true + dependency_update = var.dependency_update + timeout = 120 } data "kubernetes_secret" "waypoint_token" { diff --git a/tf-modules-k8s/waypoint/values.yaml.tmpl b/tf-modules-k8s/waypoint/values.yaml.tmpl index c7c5ada1..af64bce2 100644 --- a/tf-modules-k8s/waypoint/values.yaml.tmpl +++ b/tf-modules-k8s/waypoint/values.yaml.tmpl @@ -1,6 +1,10 @@ + +targetNamespace: ${waypoint_namespace} + server: image: tag: 0.11.4 + # tokenSecret: TODO cert: secretName: ${paas_hostname}-tls extraVolumeMounts: ${jsonencode(waypoint_extra_volume_mounts)} diff --git a/tf-modules-k8s/waypoint/variables.tf b/tf-modules-k8s/waypoint/variables.tf index fc6383e1..003e3f4f 100644 --- a/tf-modules-k8s/waypoint/variables.tf +++ b/tf-modules-k8s/waypoint/variables.tf @@ -27,3 +27,7 @@ variable "waypoint_extra_volumes" { variable "cert_manager_cluster_issuer" { description = "value of the cert-manager cluster issuer" } + +variable "dependency_update" { + default = true +} diff --git a/tf-modules-nix/deploy/main.tf b/tf-modules-nix/deploy/main.tf index def91e34..1c8ecc70 100644 --- a/tf-modules-nix/deploy/main.tf +++ b/tf-modules-nix/deploy/main.tf @@ -12,7 +12,7 @@ resource "terraform_data" "check_ssh" { type = "ssh" user = var.ssh_connection.user private_key = file(pathexpand(var.ssh_connection.private_key)) - host = var.node_ip + host = var.node_address timeout = "1m" } @@ -26,13 +26,13 @@ data "external" "machine_key_pub" { program = ["bash", "${path.module}/retrieve-vm-age-key.sh"] query = { - machine_ip = var.node_ip + machine_ip = var.node_address } } resource "local_sensitive_file" "non_encrypted_secrets" { content = yamlencode(var.nixos_transient_secrets) - filename = "${path.cwd}/${var.node_ip}.yaml" + filename = "${path.cwd}/${var.node_address}.yaml" } resource "terraform_data" "create_transient_secrets" { @@ -63,7 +63,7 @@ resource "terraform_data" "upload_secrets" { type = "ssh" user = var.ssh_connection.user private_key = file(pathexpand(var.ssh_connection.private_key)) - host = var.node_ip + host = var.node_address } provisioner "file" { @@ -81,8 +81,8 @@ locals { [ "nixos-rebuild", "--fast", - "--build-host", "${var.ssh_connection.user}@${var.node_ip}", - "--target-host", "${var.ssh_connection.user}@${var.node_ip}" + "--build-host", "${var.ssh_connection.user}@${var.node_address}", + "--target-host", "${var.ssh_connection.user}@${var.node_address}" ], var.nix_rebuild_arguments ) @@ -108,7 +108,7 @@ resource "terraform_data" "deploy" { output "config" { depends_on = [terraform_data.deploy] value = merge(var.config, { - node_ip = var.node_ip + node_address = var.node_address node_id = var.node_id }) } diff --git a/tf-modules-nix/deploy/variables.tf b/tf-modules-nix/deploy/variables.tf index b6de1a83..7c01545a 100644 --- a/tf-modules-nix/deploy/variables.tf +++ b/tf-modules-nix/deploy/variables.tf @@ -4,7 +4,7 @@ variable "dex_client_id" { default = "dex-k3s-paas" } -variable "node_ip" { +variable "node_address" { type = string } diff --git a/tf-root-network/main.tf b/tf-root-network/main.tf index c280393f..d639a748 100644 --- a/tf-root-network/main.tf +++ b/tf-root-network/main.tf @@ -26,7 +26,7 @@ resource "random_password" "admin_password" { module "deploy" { source = "../tf-modules-nix/deploy" node_id = module.tailscale.node_id - node_ip = module.tailscale.node_ip + node_address = module.tailscale.node_address config = module.tailscale.config nix_flake = var.nix_flake ssh_connection = var.ssh_connection @@ -39,7 +39,7 @@ module "deploy" { } } -module "retrieve_k3s_config" { +module "k3s_get_config" { source = "../tf-modules-cloud/k3s-get-config" ssh_connection = var.ssh_connection node_hostname = module.deploy.config.node_fqdn @@ -52,11 +52,11 @@ output "password" { } output "k3s_endpoint" { - value = "https://${module.retrieve_k3s_config.k3s_endpoint}:6443" + value = "https://${module.k3s_get_config.k3s_endpoint}:6443" sensitive = true } output "k3s_config" { sensitive = true - value = module.retrieve_k3s_config.k3s_config + value = module.k3s_get_config.k3s_config } diff --git a/tf-root-network/variables.tf b/tf-root-network/variables.tf index a9b9df76..3ef2c2f4 100644 --- a/tf-root-network/variables.tf +++ b/tf-root-network/variables.tf @@ -55,6 +55,10 @@ variable "tailscale_tailnet" { sensitive = true } +variable "tailscale_expected_device" { + default = "localhost" +} + variable "tailscale_trusted_device" { type = string }