From 1b90e30841671f442bc6f1739064919a3ef71823 Mon Sep 17 00:00:00 2001 From: loic-roux-404 Date: Sun, 1 Sep 2024 19:33:58 +0200 Subject: [PATCH] feat: move to rke2 --- .github/workflows/release-nixos.yml | 15 ++- Makefile | 15 ++- README.md | 2 +- flake.lock | 115 +++++++++++-------- flake.nix | 95 +++++++-------- nixos-darwin/configuration-x86.nix | 4 +- nixos-options/default.nix | 20 ++-- nixos/configuration.nix | 81 ++++++------- nixos/deploy.nix | 104 +++++++++++++++-- nixos/qcow-compressed.nix | 3 + tf-modules-cloud/k3s-get-config/main.tf | 2 +- tf-modules-cloud/k3s-get-config/variables.tf | 2 +- tf-modules-cloud/libvirt/get-ip.sh | 2 +- tf-modules-k8s/cilium-install/variables.tf | 2 +- tf-modules-nix/deploy/main.tf | 1 + tf-root-k3s-core/main.tf | 16 +-- tf-root-network/main.tf | 5 +- tf-root-network/variables.tf | 2 +- 18 files changed, 300 insertions(+), 186 deletions(-) diff --git a/.github/workflows/release-nixos.yml b/.github/workflows/release-nixos.yml index 3f272312..a6221731 100644 --- a/.github/workflows/release-nixos.yml +++ b/.github/workflows/release-nixos.yml @@ -3,6 +3,7 @@ on: push: tags: - nixos-stable + - nixos-testing paths: - 'nixos/**.nix' - 'nixos-options/**.nix' @@ -37,14 +38,20 @@ jobs: restore-prefixes-first-match: nix-${{ runner.os }}- gc-max-store-size-linux: 1073741824 - - name: Build - id: build - run: nix build .#nixosConfigurations.contabo-qcow + - name: Build production image + id: build-stable + if: github.ref == 'refs/tags/nixos-stable' + run: nix build .#nixosConfigurations.initial-contabo.config.formats.qcow + - name: Build testing image + id: build-testing + if: github.ref == 'refs/tags/nixos-testing' + run: nix build .#nixosConfigurations.initial.config.formats.qcow + - name: Release uses: softprops/action-gh-release@v1 with: - tag_name: nixos-stable + tag_name: ${{ github.ref_name }} token: "${{ secrets.GITHUB_TOKEN }}" generate_release_notes: true files: | diff --git a/Makefile b/Makefile index 443d5194..ccd9a387 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ SHELL:=/usr/bin/env bash MAKEFLAGS += --no-builtin-rules --no-builtin-variables TF_CMD:=apply -auto-approve VARIANT=builder +SYSTEM?=aarch64-linux #### Nix @@ -12,14 +13,20 @@ ifeq ($(shell uname -s),Darwin) BUILDER_EXEC:=NIX_CONF_DIR=$(PWD)/bootstrap nix develop .\#builder --command endif -bootstrap: +bootstrap-aarch64-linux: @$(BUILDER_EXEC) echo "Started default build environment" -bootstrap-x86: +bootstrap-x86_64-linux: @VARIANT=builder-x86 $(BUILDER_EXEC) echo "Started x86 environment" + @echo "Waiting builder to" + @sleep 15 -nixos-local: - @$(BUILDER_EXEC) nix build .#nixosConfigurations.default --system aarch64-linux +bootstrap: bootstrap-$(SYSTEM) + +nixos-local: bootstrap build + +build: + @nix build .#nixosConfigurations.default --system $(SYSTEM) TERRAGRUNT_FILES:=$(shell find terragrunt -type d -name '.*' -prune -o -name 'terragrunt.hcl' -exec dirname {} \;) diff --git a/README.md b/README.md index 61cce2cc..ed16e163 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Build an image between available nixos configuration `contabo` and `qcow2` : > Supported systems are `aarch64-linux`, `x86_64-linux`, `aarch64-darwin` and `x86_64-darwin`. ```bash -nix build .#nixosConfigurations.default --system aarch64-linux +nix build .#nixosConfigurations.default --system x86_64-linux ``` ### Uninstall on Darwin: diff --git a/flake.lock b/flake.lock index 04b71883..43fcade9 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1722082646, - "narHash": "sha256-od8dBWVP/ngg0cuoyEl/w9D+TCNDj6Kh4tr151Aax7w=", + "lastModified": 1724994893, + "narHash": "sha256-yutISDGg6HUaZqCaa54EcsfTwew3vhNtt/FNXBBo44g=", "owner": "LnL7", "repo": "nix-darwin", - "rev": "0413754b3cdb879ba14f6e96915e5fdf06c6aab6", + "rev": "c8d3157d1f768e382de5526bb38e74d2245cad04", "type": "github" }, "original": { @@ -62,11 +62,11 @@ ] }, "locked": { - "lastModified": 1722119539, - "narHash": "sha256-2kU90liMle0vKR8exJx1XM4hZh9CdNgZGHCTbeA9yzY=", + "lastModified": 1724435763, + "narHash": "sha256-UNky3lJNGQtUEXT2OY8gMxejakSWPTfWKvpFkpFlAfM=", "owner": "nix-community", "repo": "home-manager", - "rev": "d0240a064db3987eb4d5204cf2400bc4452d9922", + "rev": "c2cd2a52e02f1dfa1c88f95abeb89298d46023be", "type": "github" }, "original": { @@ -78,11 +78,11 @@ }, "nixlib": { "locked": { - "lastModified": 1721523216, - "narHash": "sha256-/NjnIKkBoqKdvOS8unooDg0HqMaRUwYLbyn0ntjEckQ=", + "lastModified": 1724547350, + "narHash": "sha256-WKkGeNpenNMKD1gOF0Xuqi3VsKX/QCAiwz9qe5PDvzA=", "owner": "nix-community", "repo": "nixpkgs.lib", - "rev": "31a99025ce3784c20dd11dafa5260e80e314f59e", + "rev": "b741d900fecd2f0c32d90f853b24be9f5f098b7d", "type": "github" }, "original": { @@ -100,11 +100,11 @@ ] }, "locked": { - "lastModified": 1721869487, - "narHash": "sha256-zcusn81g+0gO+tSMhfs4W+wAP9As/MWNTBCbS+Ggp7A=", + "lastModified": 1724893087, + "narHash": "sha256-M3+Z8SSpzKPQ+/vw9a99G9HfqKWbVGzhFz4p3KAX0NI=", "owner": "nix-community", "repo": "nixos-generators", - "rev": "c12f9a969c8cdf14618774515c7c6c96aef753c7", + "rev": "0dd0205bc3f6d602ddb62aaece5f62a8715a9e85", "type": "github" }, "original": { @@ -115,16 +115,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1721466660, - "narHash": "sha256-pFSxgSZqZ3h+5Du0KvEL1ccDZBwu4zvOil1zzrPNb3c=", + "lastModified": 1717179513, + "narHash": "sha256-vboIEwIQojofItm2xGCdZCzW96U85l9nDW3ifMuAIdM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "6e14bbce7bea6c4efd7adfa88a40dac750d80100", + "rev": "63dacb46bf939521bdc93981b4cbb7ecb58427a0", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixpkgs-unstable", + "ref": "24.05", "repo": "nixpkgs", "type": "github" } @@ -147,27 +147,27 @@ }, "nixpkgs-stable": { "locked": { - "lastModified": 1717179513, - "narHash": "sha256-vboIEwIQojofItm2xGCdZCzW96U85l9nDW3ifMuAIdM=", + "lastModified": 1721524707, + "narHash": "sha256-5NctRsoE54N86nWd0psae70YSLfrOek3Kv1e8KoXe/0=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "63dacb46bf939521bdc93981b4cbb7ecb58427a0", + "rev": "556533a23879fc7e5f98dd2e0b31a6911a213171", "type": "github" }, "original": { "owner": "NixOS", - "ref": "24.05", + "ref": "release-24.05", "repo": "nixpkgs", "type": "github" } }, "nixpkgs-stable-darwin": { "locked": { - "lastModified": 1722153173, - "narHash": "sha256-S46RmS9WkBOTLORpnWsNth4Ae6TlfJS9TKXqB55YQy4=", + "lastModified": 1725062077, + "narHash": "sha256-ARdb2SNoV+zAN80CXeweNm3FZ8NWLmVna7mGKWVONeE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "7bba2df68503b68634e0070483ddaf6d59dc6530", + "rev": "be55bcada114b8e0385544b15cc4bc2148046aee", "type": "github" }, "original": { @@ -177,45 +177,45 @@ "type": "github" } }, - "nixpkgs-stable_2": { + "nixpkgs-unstable": { "locked": { - "lastModified": 1721524707, - "narHash": "sha256-5NctRsoE54N86nWd0psae70YSLfrOek3Kv1e8KoXe/0=", + "lastModified": 1724819573, + "narHash": "sha256-GnR7/ibgIH1vhoy8cYdmXE6iyZqKqFxQSVkFgosBh6w=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "556533a23879fc7e5f98dd2e0b31a6911a213171", + "rev": "71e91c409d1e654808b2621f28a327acfdad8dc2", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-24.05", + "ref": "nixos-unstable", "repo": "nixpkgs", "type": "github" } }, - "nixpkgs-unstable": { + "nixpkgs_2": { "locked": { - "lastModified": 1722062969, - "narHash": "sha256-QOS0ykELUmPbrrUGmegAUlpmUFznDQeR4q7rFhl8eQg=", + "lastModified": 1721466660, + "narHash": "sha256-pFSxgSZqZ3h+5Du0KvEL1ccDZBwu4zvOil1zzrPNb3c=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b73c2221a46c13557b1b3be9c2070cc42cf01eb3", + "rev": "6e14bbce7bea6c4efd7adfa88a40dac750d80100", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-unstable", + "ref": "nixpkgs-unstable", "repo": "nixpkgs", "type": "github" } }, - "nixpkgs_2": { + "nixpkgs_3": { "locked": { - "lastModified": 1721838734, - "narHash": "sha256-o87oh2nLDzZ1E9+j1I6GaEvd9865OWGYvxaPSiH9DEU=", + "lastModified": 1724870369, + "narHash": "sha256-dGHUOi1tBiVOsVdT9QNEuk+FuSMtQxkyx+9CN/34kkk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "1855c9961e0bfa2e776fa4b58b7d43149eeed431", + "rev": "215ea7473ff80eb6cb157ee07223920cc53f4b09", "type": "github" }, "original": { @@ -225,6 +225,26 @@ "type": "github" } }, + "rke2": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1718609143, + "narHash": "sha256-HWDmtyLzohQb9kHI2AVKzb91EJTBi5YPnh+lKrjSOCY=", + "owner": "numtide", + "repo": "nixos-rke2", + "rev": "c28d68bac74a55e6dc5c32147b00e2c4620278a3", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "nixos-rke2", + "type": "github" + } + }, "root": { "inputs": { "darwin": "darwin", @@ -232,29 +252,30 @@ "flake-utils": "flake-utils", "home-manager": "home-manager", "nixos-generators": "nixos-generators", + "nixpkgs": "nixpkgs", "nixpkgs-legacy": "nixpkgs-legacy", "nixpkgs-srvos": [ "srvos", "nixpkgs" ], - "nixpkgs-stable": "nixpkgs-stable", "nixpkgs-stable-darwin": "nixpkgs-stable-darwin", "nixpkgs-unstable": "nixpkgs-unstable", + "rke2": "rke2", "sops-nix": "sops-nix", "srvos": "srvos" } }, "sops-nix": { "inputs": { - "nixpkgs": "nixpkgs", - "nixpkgs-stable": "nixpkgs-stable_2" + "nixpkgs": "nixpkgs_2", + "nixpkgs-stable": "nixpkgs-stable" }, "locked": { - "lastModified": 1722114803, - "narHash": "sha256-s6YhI8UHwQvO4cIFLwl1wZ1eS5Cuuw7ld2VzUchdFP0=", + "lastModified": 1723501126, + "narHash": "sha256-N9IcHgj/p1+2Pvk8P4Zc1bfrMwld5PcosVA0nL6IGdE=", "owner": "Mic92", "repo": "sops-nix", - "rev": "eb34eb588132d653e4c4925d862f1e5a227cc2ab", + "rev": "be0eec2d27563590194a9206f551a6f73d52fa34", "type": "github" }, "original": { @@ -265,14 +286,14 @@ }, "srvos": { "inputs": { - "nixpkgs": "nixpkgs_2" + "nixpkgs": "nixpkgs_3" }, "locked": { - "lastModified": 1721888498, - "narHash": "sha256-O5/s8e6CL99AQoKEn8k6F99UoJdAzQ8z9LZ7SxFJ3c4=", + "lastModified": 1725040185, + "narHash": "sha256-hOv19L8aRprqdm1Jz7T4kT8h/ckdj8BgLtLSNOOj+RE=", "owner": "numtide", "repo": "srvos", - "rev": "27b3a9b23847cb2e716334ee6ad58b82ddc3f7a7", + "rev": "b8e10788e84670049b30dc11d4c5893aedf7b32b", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 38a9f49a..e6328ab0 100644 --- a/flake.nix +++ b/flake.nix @@ -1,10 +1,10 @@ { - description = "Nix Darwin configuration for my systems (from https://github.com/malob/nixpkgs)"; + description = "Nix configurations for a k8s paas build"; inputs = { # Package sets + nixpkgs.url = "github:NixOS/nixpkgs/24.05"; nixpkgs-legacy.url = "github:NixOS/nixpkgs/23.11"; - nixpkgs-stable.url = "github:NixOS/nixpkgs/24.05"; nixpkgs-stable-darwin.url = "github:NixOS/nixpkgs/nixpkgs-24.05-darwin"; nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixos-unstable"; srvos.url = "github:numtide/srvos"; @@ -24,6 +24,11 @@ inputs.nixpkgs.follows = "srvos/nixpkgs"; }; + rke2 = { + url = "github:numtide/nixos-rke2"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + # Flake utilities flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; flake-utils.url = "github:numtide/flake-utils"; @@ -47,7 +52,7 @@ overlays = { pkgs-stable = _: prev: { - pkgs-stable = import inputs.nixpkgs-stable { + pkgs-stable = import inputs.nixpkgs { inherit (prev.stdenv) system; inherit (nixpkgsDefaults) config; }; @@ -85,6 +90,7 @@ }); nixosModules = { + rke2 = inputs.rke2.nixosModules.default; sops = inputs.sops-nix.nixosModules.sops; common = srvos.nixosModules.common; server = srvos.nixosModules.server; @@ -132,64 +138,58 @@ }; }; }; - } - // flake-utils.lib.eachDefaultSystem (system: - let - linux = builtins.replaceStrings ["darwin"] ["linux"] system; - oldLegacyPackages = import inputs.nixpkgs-legacy (nixpkgsDefaults // { system = linux; }); - specialArgs = { - inherit oldLegacyPackages; - }; - in { + } + // flake-utils.lib.eachDefaultSystem (baseSystem: + { + packages.nixosConfigurations = let + system = builtins.replaceStrings ["darwin"] ["linux"] baseSystem; + oldLegacyPackages = import inputs.nixpkgs-legacy (nixpkgsDefaults // { inherit system; }); + specialArgs = { + inherit oldLegacyPackages; + }; + qcowSystemFormat = [ + ({ ... }: { + imports = [ + nixos-generators.nixosModules.all-formats + ./nixos/qcow-compressed.nix + ]; + nixpkgs.hostPlatform = system; + }) + ]; + in { + ## Libvirt configurations - packages.nixosConfigurations = rec { - default = qcow; + initial = nixosSystem { + inherit system specialArgs; + modules = qcowSystemFormat ++ self.nixosAllModules.default; + }; deploy = nixosSystem { - system = linux; - inherit specialArgs; + inherit system specialArgs; modules = self.nixosAllModules.deploy; }; - deploy-contabo = nixosSystem { - system = "x86_64-linux"; - inherit specialArgs; - modules = self.nixosAllModules.deployContabo ++ [ - ./nixos/contabo-master-0.nix - ]; - }; + ## Contabo-specific configurations - initial = nixosSystem { - system = linux; - inherit specialArgs; - modules = self.nixosAllModules.default; + initial-contabo = nixosSystem { + inherit system specialArgs; + modules = qcowSystemFormat ++ self.nixosAllModules.contabo; }; - qcow = makeOverridable nixos-generators.nixosGenerate { + deploy-contabo = nixosSystem { inherit system specialArgs; - modules = self.nixosAllModules.default ++ [ - ./nixos/qcow-compressed.nix + modules = self.nixosAllModules.deployContabo ++ [ + ./nixos/contabo-master-0.nix ]; - format = "qcow"; - }; - - intial-contabo = nixosSystem { - system = "x86_64-linux"; - inherit specialArgs; - modules = self.nixosAllModules.contabo; }; - contabo-qcow = self.packages.${system}.nixosConfigurations.qcow.override { - modules = self.nixosAllModules.contabo ++ [ - ./nixos/qcow-compressed.nix - ]; - }; + ## Docker configurations - container = self.packages.${system}.nixosConfigurations.qcow.override { + container = nixosSystem { modules = self.nixosAllModules.default ++ [ + nixos-generators.nixosModules.docker ./nixos/docker.nix ]; - format = "docker"; }; }; @@ -198,8 +198,10 @@ # With `nix.registry.my.flake = inputs.self`, development shells can be created by running, # e.g., `nix develop my#python`. devShells = let + system = baseSystem; + oldLegacyPackages = import inputs.nixpkgs-legacy (nixpkgsDefaults // { inherit system; }); pkgs = import inputs.nixpkgs-srvos (nixpkgsDefaults // { inherit system; }); - stablePkgs = import inputs.nixpkgs-stable (nixpkgsDefaults // { inherit system; }); + stablePkgs = import inputs.nixpkgs (nixpkgsDefaults // { inherit system; }); in { default = pkgs.mkShell { @@ -207,8 +209,7 @@ packages = attrValues { inherit (pkgs) bashInteractive grpcurl jq coreutils e2fsprogs docker-client docker-credential-helpers libvirt qemu - tailscale pebble cntb kubernetes-helm - nil nix-tree; + tailscale pebble cntb kubernetes-helm nil nix-tree; inherit (stablePkgs) nix terragrunt terraform sops ssh-to-age nixos-rebuild; inherit (oldLegacyPackages) waypoint; }; diff --git a/nixos-darwin/configuration-x86.nix b/nixos-darwin/configuration-x86.nix index 11b7346b..a54bf478 100644 --- a/nixos-darwin/configuration-x86.nix +++ b/nixos-darwin/configuration-x86.nix @@ -2,11 +2,13 @@ pkgs, lib, ... }: { nix.linux-builder = { - package = lib.mkDefault pkgs.darwin.linux-builder-x86_64; + package = pkgs.darwin.linux-builder-x86_64; ephemeral = lib.mkDefault true; systems = ["x86_64-linux"]; config = lib.mkDefault ({ lib, ... }: { nixpkgs.hostPlatform = lib.mkForce "x86_64-linux"; + security.sudo.wheelNeedsPassword = false; + users.users.builder.extraGroups = [ "wheel" ]; }); }; } diff --git a/nixos-options/default.nix b/nixos-options/default.nix index f265258a..0433d003 100644 --- a/nixos-options/default.nix +++ b/nixos-options/default.nix @@ -4,11 +4,10 @@ options.k3s-paas = { certs = lib.mkOption { - default = [{ - url = "https://localhost:15000/intermediates/0"; - sha256 = "06fpbiljbzmcnfsxnr92p7mhm6i4yglbhj5q7csw2pcsklw68z8n"; - }]; - type = lib.types.listOf (lib.types.attrs); + default = [ + ../nixos-darwin/pebble/cert.crt + ]; + type = lib.types.listOf (lib.types.path); description = "Ca url to fetch and trust (need to be impure)"; }; @@ -19,8 +18,11 @@ }; dns.dest-ips = lib.mkOption { - default = ["127.0.0.1" "192.168.205.2" "192.168.205.3" "192.168.205.4" - "192.168.205.5" "192.168.205.6" "192.168.205.7" "192.168.205.8" "192.168.205.9"]; + default = [ + "127.0.0.1" "192.168.205.2" "192.168.205.3" + "192.168.205.4" "192.168.205.5" "192.168.205.6" + "192.168.205.7" "192.168.205.8" "192.168.205.9" + ]; type = lib.types.listOf lib.types.str; description = "Target IP address for dns.name (only in local dev)"; }; @@ -44,13 +46,13 @@ }; k3s.disableServices = lib.mkOption { - default = ["traefik" "metrics-server" "servicelb" ]; + default = ["traefik" "rke2-ingress-nginx" "servicelb" ]; type = lib.types.listOf lib.types.str; description = "Disable k8s services eg: traefik,servicelb"; }; k3s.serverExtraArgs = lib.mkOption { - default = []; + default = ["--disable-kube-proxy" "--egress-selector-mode=disabled"]; type = lib.types.listOf lib.types.str; description = "Extra arguments for k8s server"; }; diff --git a/nixos/configuration.nix b/nixos/configuration.nix index 1abfbe82..a80d3a60 100644 --- a/nixos/configuration.nix +++ b/nixos/configuration.nix @@ -9,13 +9,11 @@ with config.k3s-paas; let - certs = [ ../nixos-darwin/pebble/cert.crt ]; userSshConfig = { authorizedKeys = { keys = [ user.key ]; }; }; - k3sPkg = oldLegacyPackages.k3s_1_27; in { fileSystems."/" = { @@ -31,20 +29,36 @@ in { boot.growPartition = lib.mkDefault true; boot.loader.grub.device = lib.mkForce "/dev/sda"; + boot.tmp.useTmpfs = true; boot.tmp.cleanOnBoot = true; boot.kernelPackages = pkgs.linuxPackages_latest; boot.loader.systemd-boot.consoleMode = "auto"; + swapDevices = [ ]; zramSwap.algorithm = "zstd"; system.stateVersion = "24.05"; time = { timeZone = lib.mkForce "Europe/Paris"; + hardwareClockInLocalTime = true; }; i18n.defaultLocale = "en_US.UTF-8"; + networking = { + enableIPv6 = true; + useDHCP = true; + useNetworkd = true; + nftables.enable = true; + nftables.flushRuleset = true; + firewall = { + trustedInterfaces = [ "tailscale0" ]; + allowedTCPPorts = lib.mkDefault [ 80 443 22 ]; + allowedUDPPorts = [ config.services.tailscale.port ]; + }; + }; + programs.ssh.package = pkgs.openssh_hpn; services.openssh = { enable = true; @@ -87,35 +101,30 @@ in { ''; }; - systemd.services.k3s.serviceConfig.Environment = "PATH=${pkgs.tailscale}/bin"; - services.k3s = { + systemd.services.numtide-rke2.serviceConfig.Environment = "PATH=${pkgs.tailscale}/bin:${pkgs.coreutils}/bin"; + services.numtide-rke2 = { enable = lib.mkDefault false; role = "server"; - package = k3sPkg; - extraFlags = lib.strings.concatStringsSep " " ( - map (service: "--disable=${service}") k3s.disableServices + extraFlags = ( + builtins.concatMap (service: ["--disable" service]) k3s.disableServices ++ k3s.serverExtraArgs - ++ [ - "--flannel-backend=none" - "--disable-kube-proxy" - "--disable-network-policy" - "--egress-selector-mode=disabled" - ] ); }; services.fail2ban.enable = true; + security.pki.certificateFiles = certs; + home-manager.useGlobalPkgs = true; home-manager.useUserPackages = true; home-manager.users.${config.k3s-paas.user.name} = { xdg.enable = true; home.stateVersion = "24.05"; home.sessionVariables = { - KUBECONFIG = "/etc/rancher/k3s/k3s.yaml"; + KUBECONFIG = "/etc/rancher/rke2/rke2.yaml"; }; home.shellAliases = { - kubectl = "sudo kubectl"; + kubectl = "sudo -E kubectl"; helm = "sudo -E helm"; k-ks = "sudo -E kubectl -n kube-system"; }; @@ -146,12 +155,13 @@ in { dnsutils jq wget - k3sPkg + k3s kubectl kubernetes-helm oldLegacyPackages.waypoint tailscale cilium-cli + iptables ]; }; @@ -173,15 +183,15 @@ in { "${pkgs.systemd}/bin/systemctl list-jobs" "${pkgs.systemd}/bin/systemctl is-system-running" "${pkgs.systemd}/bin/journalctl" - "${pkgs.k3s}/bin/kubectl get" - "${pkgs.k3s}/bin/kubectl describe" - "${pkgs.k3s}/bin/kubectl explain" - "${pkgs.k3s}/bin/kubectl logs" - "${pkgs.k3s}/bin/kubectl diff" - "${pkgs.k3s}/bin/kubectl events" - "${pkgs.k3s}/bin/kubectl wait" - "${pkgs.k3s}/bin/kubectl api-resources" - "${pkgs.k3s}/bin/kubectl version" + "${pkgs.kubectl}/bin/kubectl get" + "${pkgs.kubectl}/bin/kubectl describe" + "${pkgs.kubectl}/bin/kubectl explain" + "${pkgs.kubectl}/bin/kubectl logs" + "${pkgs.kubectl}/bin/kubectl diff" + "${pkgs.kubectl}/bin/kubectl events" + "${pkgs.kubectl}/bin/kubectl wait" + "${pkgs.kubectl}/bin/kubectl api-resources" + "${pkgs.kubectl}/bin/kubectl version" "${pkgs.nettools}/bin/ifconfig" "${pkgs.iproute2}/bin/ip" "${pkgs.iptables}/bin/iptables" @@ -213,27 +223,6 @@ in { }; }; - networking = { - useNetworkd = true; - useDHCP = true; - firewall = { - trustedInterfaces = [ "tailscale0" ]; - allowedTCPPorts = lib.mkDefault [ 80 443 22 4240 ]; - allowedUDPPorts = [ config.services.tailscale.port ]; - }; - # Looks its not ready to work along cilium and k3s - nftables.enable = false; - networkmanager.enable = false; - usePredictableInterfaceNames = true; - }; - - boot.kernel.sysctl = { - "net.ipv4.ip_forward" = 1; - "net.ipv6.conf.all.forwarding" = 1; - }; - - security.pki.certificateFiles = certs; - nixpkgs = { config = { allowUnfree = true; diff --git a/nixos/deploy.nix b/nixos/deploy.nix index 84aa7e61..e3e48d12 100644 --- a/nixos/deploy.nix +++ b/nixos/deploy.nix @@ -1,7 +1,6 @@ -{ config, ... } : +{ config, ... } : with config.k3s-paas; - { networking.hostName = "localhost-0"; sops.validateSopsFiles = false; @@ -10,6 +9,7 @@ with config.k3s-paas; sops.age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" "${config.sops.secrets.nodePrivateKey.path}" ]; networking.firewall.allowedTCPPorts = [ 80 443 ]; + services.tailscale.authKeyFile = config.sops.secrets.tailscaleNodeKey.path; services.tailscale.extraUpFlags = [ "--ssh" "--hostname=${config.networking.hostName}" @@ -27,32 +27,112 @@ with config.k3s-paas; ]; sops.secrets.nodeIp = {}; + sops.secrets.internalNodeIp = {}; sops.secrets.nodePrivateKey = {}; sops.secrets.tailscaleNodeKey = {}; sops.secrets.paasDomain = {}; sops.secrets.tailscaleDomain = {}; sops.secrets.password = { neededForUsers = true; }; - services.k3s.enable = true; - services.k3s.configPath = config.sops.templates."config.yaml".path; + services.numtide-rke2.enable = true; + services.numtide-rke2.configFile = config.sops.templates."config.yaml".path; + services.numtide-rke2.manifests = { + "cilium-config.yaml" = config.sops.templates."cilium-config.yaml".path; + }; + + sops.templates."cilium-config.yaml".content = '' + apiVersion: helm.cattle.io/v1 + kind: HelmChartConfig + metadata: + name: rke2-cilium + namespace: kube-system + spec: + valuesContent: |- + ipam: + operator: + clusterPoolIPv4PodCIDRList: ["10.100.0.0/16"] + k8sServiceHost: ${config.sops.placeholder.internalNodeIp} + k8sServicePort: 6443 + l2announcements: + enabled: true + kubeProxyReplacement: true + bpf: + masquerade: true + lbExternalClusterIP: false + gatewayAPI: + enabled: false + routingMode: "tunnel" + tunnelProtocol: "vxlan" + ingressController: + enabled: true + default: true + loadbalancerMode: "dedicated" + service: + name: "cilium-ingress-external" + labels: + "k3s-paas/internal": "true" + prometheus: + enabled: true + serviceMonitor: + enabled: true + operator: + replicas: 1 + prometheus: + enabled: true + hubble: + relay: + enabled: true + metrics: + enabled: + - "dns" + - "drop" + - "tcp" + - "flow" + - "port-distribution" + - "icmp" + - "httpV2:exemplars=true;labelsContext=source_ip,source_namespace,source_workload,destination_ip,destination_namespace,destination_workload,traffic_direction" + enableOpenMetrics: true + ''; sops.templates."config.yaml".content = '' + advertise-address: ${config.sops.placeholder.internalNodeIp} node-name: "${config.networking.hostName}" + cluster-domain: ${config.sops.placeholder.paasDomain} node-external-ip: "${config.sops.placeholder.nodeIp}" + cluster-cidr: 10.100.0.0/16 + service-cidr: 10.110.0.0/16 + cluster-dns: 10.110.0.10 vpn-auth: "name=tailscale,joinKey=${config.sops.placeholder.tailscaleNodeKey}" tls-san: - localhost + - 10.43.0.1 - ${config.networking.hostName} - "${config.sops.placeholder.tailscaleDomain}" - "${config.sops.placeholder.nodeIp}" - '' + (if dex.dexClientId != "" then - '' - kube-apiserver-arg=authorization-mode: Node,RBAC - kube-apiserver-arg=oidc-issuer-url: https://dex.${config.sops.placeholder.paasDomain} - kube-apiserver-arg=oidc-client-id: ${dex.dexClientId} - kube-apiserver-arg=oidc-username-claim: email - kube-apiserver-arg=oidc-groups-claim: groups - '' else []); + - "${config.sops.placeholder.internalNodeIp}" + + cni: cilium + protect-kernel-defaults: true + + kube-apiserver-arg: + - '--authorization-mode=Node,RBAC' + - '--oidc-issuer-url=https://dex.${config.sops.placeholder.paasDomain}' + - '--oidc-client-id=${dex.dexClientId}' + - '--oidc-username-claim=email' + - '--oidc-groups-claim=groups' + - '--default-not-ready-toleration-seconds=30' + - '--default-unreachable-toleration-seconds=30' + + kube-controller-manager-arg: + - '--node-monitor-period=4s' + kubelet-arg: + - '--node-status-update-frequency=4s' + - '--max-pods=100' + + etcd-arg: "--quota-backend-bytes 2048000000" + etcd-snapshot-schedule-cron: "0 3 * * *" + etcd-snapshot-retention: 10 + ''; users.users.reader.hashedPasswordFile = config.sops.secrets.password.path; users.users.${user.name}.hashedPasswordFile = config.sops.secrets.password.path; diff --git a/nixos/qcow-compressed.nix b/nixos/qcow-compressed.nix index e8a565cc..e1d2c8e3 100644 --- a/nixos/qcow-compressed.nix +++ b/nixos/qcow-compressed.nix @@ -1,4 +1,7 @@ { lib, pkgs, config, modulesPath, ...} : { + imports = [ + "${toString modulesPath}/profiles/qemu-guest.nix" + ]; system.build.qcow = lib.mkForce (import "${toString modulesPath}/../lib/make-disk-image.nix" { inherit lib config pkgs; diskSize = "auto"; diff --git a/tf-modules-cloud/k3s-get-config/main.tf b/tf-modules-cloud/k3s-get-config/main.tf index 4d0dd9bf..b037827d 100644 --- a/tf-modules-cloud/k3s-get-config/main.tf +++ b/tf-modules-cloud/k3s-get-config/main.tf @@ -4,7 +4,7 @@ resource "terraform_data" "wait_ssh" { user = var.ssh_connection.user host = var.node_hostname private_key = file(pathexpand(var.ssh_connection.private_key)) - timeout = "1m" + timeout = "3m" } provisioner "remote-exec" { diff --git a/tf-modules-cloud/k3s-get-config/variables.tf b/tf-modules-cloud/k3s-get-config/variables.tf index c359ff24..f09cd151 100644 --- a/tf-modules-cloud/k3s-get-config/variables.tf +++ b/tf-modules-cloud/k3s-get-config/variables.tf @@ -17,7 +17,7 @@ variable "ssh_connection" { } variable "remote_k3s_config_location" { - default = "/etc/rancher/k3s/k3s.yaml" + default = "/etc/rancher/rke2/rke2.yaml" } variable "context_cluster_name" { diff --git a/tf-modules-cloud/libvirt/get-ip.sh b/tf-modules-cloud/libvirt/get-ip.sh index fd31b36e..7a7db831 100755 --- a/tf-modules-cloud/libvirt/get-ip.sh +++ b/tf-modules-cloud/libvirt/get-ip.sh @@ -5,7 +5,7 @@ eval "$(jq -r '@sh "timeout=\(.timeout) mac=\(.mac)"')" elapsed=0 ip_address="" -while [ -z "$ip_address" ] && [ $elapsed -lt ${timeout:-60} ]; do +while [ -z "$ip_address" ] && [ $elapsed -lt ${timeout:-90} ]; do ip_address=$(arp -a | grep "$mac" | awk -F'[()]' '{print $2}') if [ -n "$ip_address" ]; then export ip_address diff --git a/tf-modules-k8s/cilium-install/variables.tf b/tf-modules-k8s/cilium-install/variables.tf index eae10e72..d5fb26d0 100644 --- a/tf-modules-k8s/cilium-install/variables.tf +++ b/tf-modules-k8s/cilium-install/variables.tf @@ -5,7 +5,7 @@ variable "cilium_namespace" { variable "cilium_version" { description = "The version of Cilium to deploy" type = string - default = "1.15.7" + default = "1.16.1" } variable "k3s_host" { diff --git a/tf-modules-nix/deploy/main.tf b/tf-modules-nix/deploy/main.tf index b1e790e4..975a8ebd 100644 --- a/tf-modules-nix/deploy/main.tf +++ b/tf-modules-nix/deploy/main.tf @@ -142,6 +142,7 @@ resource "terraform_data" "reset" { provisioner "local-exec" { when = destroy + on_failure = continue interpreter = concat( self.input.nix_rebuild_interpreter, ["--flake", self.input.nix_flake_reset] diff --git a/tf-root-k3s-core/main.tf b/tf-root-k3s-core/main.tf index c9e06887..da321c37 100644 --- a/tf-root-k3s-core/main.tf +++ b/tf-root-k3s-core/main.tf @@ -1,12 +1,12 @@ -module "cilium_install" { - source = "../tf-modules-k8s/cilium-install" - node_name = var.k3s_node_name - k3s_host = var.k3s_endpoint -} +# module "cilium_install" { +# source = "../tf-modules-k8s/cilium-install" +# node_name = var.k3s_node_name +# k3s_host = var.k3s_endpoint +# } -module "metrics_server_install" { - source = "../tf-modules-k8s/metrics-server" -} +# module "metrics_server_install" { +# source = "../tf-modules-k8s/metrics-server" +# } module "cert_manager_install" { source = "../tf-modules-k8s/cert-manager-install" diff --git a/tf-root-network/main.tf b/tf-root-network/main.tf index 906159e2..8c2fd322 100644 --- a/tf-root-network/main.tf +++ b/tf-root-network/main.tf @@ -45,6 +45,7 @@ module "deploy" { nix_flake_reset = var.nix_flake_reset ssh_connection = var.ssh_connection nixos_transient_secrets = { + internalNodeIp = module.tailscale.node_address nodeIp = var.machine.node_ip dexClientId = "dex-client-id" tailscaleNodeKey = "${module.tailscale.config.node_key}" @@ -58,14 +59,14 @@ module "tailscale_destroy" { source = "../tf-modules-cloud/tailscale/destroy" tailscale_tailnet = var.tailscale_tailnet tailscale_oauth_client = var.tailscale_oauth_client - node_hostname = module.deploy.config.node_address + node_hostname = module.deploy.config.node_hostname } module "k3s_get_config" { source = "../tf-modules-cloud/k3s-get-config" ssh_connection = var.ssh_connection node_hostname = module.deploy.config.node_address - remote_k3s_config_location = "/etc/rancher/k3s/k3s.yaml" + remote_k3s_config_location = "/etc/rancher/rke2/rke2.yaml" } output "password" { diff --git a/tf-root-network/variables.tf b/tf-root-network/variables.tf index 0c7c6bc7..a24b1d3a 100644 --- a/tf-root-network/variables.tf +++ b/tf-root-network/variables.tf @@ -72,5 +72,5 @@ variable "nix_flake_reset" { } variable "remote_k3s_config_location" { - default = "/etc/rancher/k3s/k3s.yaml" + default = "/etc/rancher/rke2/rke2.yaml" }