feat: builder not far to be stable

loic-roux-404 · Nov 5, 2023 · f16242e · f16242e
1 parent c5eef27
commit f16242e
Show file tree

Hide file tree

Showing 7 changed files with 92 additions and 130 deletions.
diff --git a/README.md b/README.md
@@ -3,55 +3,20 @@
 - [Documentation](https://loic-roux-404.github.io/k3s-paas/)
 - [Original tutorial (FR)](https://github.com/esgi-lyon/paas-tutorial/blob/main/docs/index.md)
 
-## Start
-```bash
-echo 'experimental-features = nix-command flakes' | sudo tee -a /etc/nix/nix.conf
-# Speed ub remote builders
-echo 'builders-use-substitutes = true' | sudo tee -a /etc/nix/nix.conf
-
-```
-
-Restart nix
-
-```bash
-sudo launchctl kickstart -k system/org.nixos.nix-daemon
-```
-
 ## New Nix system (beta)
 
-Build linux image with nixos flakes
-
-Qemu :
-
-## Start Linux Builder on Mac Os :
-
-1. Start builder :
+### Setup (Darwin)
 
 ```bash
-nix run nixpkgs#darwin.linux-builder
-
-```
-
-2. Add ssh entry
-
-```bash
-
-cat <<EOF | sudo tee /etc/ssh/ssh_config.d/100-linux-builder.conf
-Host linux-builder
-  Hostname localhost
-  HostKeyAlias linux-builder
-  User builder
-  Port 31022
-  IdentityFile /etc/nix/builder_ed25519
-  StrictHostKeyChecking no
-EOF
-
+nix build .#darwinConfigurations.builder.system
+result/sw/bin/darwin-rebuild switch --flake .#builder
+sudo chown $USER:staff /etc/nix/builder_ed25519
 ```
 
-3. Build image :
+### Build vm
 
 ```bash
-nix build --builders "ssh://linux-builder aarch64-linux" .#nixosConfigurations.qcow
+nix build .#nixosConfigurations.default --system 'aarch64-linux' --max-jobs 0
 ```
 
 ## Test nix Os vm

diff --git a/devshell/flake.lock b/devshell/flake.lock
diff --git a/devshell/flake.nix b/devshell/flake.nix
@@ -24,7 +24,7 @@
       };
       eachSystem = f:
         lib.genAttrs
-          srvos.lib.devShellSystems
+          srvos.lib.supportedSystems
           (system: f (pkgsForSystem system));
 
       treefmtCfg = eachSystem (pkgs: treefmt-nix.lib.evalModule pkgs ./treefmt.nix);

diff --git a/docker/nix/nix.conf b/docker/nix/nix.conf
diff --git a/flake.lock b/flake.lock
diff --git a/flake.nix b/flake.nix
@@ -17,79 +17,32 @@
       "aarch64-linux"
       "aarch64-darwin"
     ];
-    lib.devShellSystems = self.lib.supportedSystems ++ [
-      "aarch64-darwin"
-    ];
 
     packages = nixpkgs.lib.genAttrs self.lib.supportedSystems (system: 
     let
       inherit (darwin.lib) darwinSystem;
       linuxSystem = builtins.replaceStrings [ "darwin" ] [ "linux" ] system;
+      pkgs = nixpkgs.legacyPackages."${system}";
+      username = builtins.getEnv "USER";
+      userHome = builtins.getEnv "HOME";
     in
     {
-      darwinConfigurations =
-        let
-          pkgs = nixpkgs.legacyPackages."${system}";
-          linuxBuilderDataDir = "/var/lib/nixos-builder";
-          linuxBuilderLog = "/var/log/linux-builder.log";
-          darwin-builder = (nixpkgs.lib.nixosSystem {
-            system = linuxSystem;
-            modules = [
-              "${nixpkgs}/nixos/modules/profiles/macos-builder.nix"
-              { 
-                virtualisation.host.pkgs = pkgs; 
-                system.nixos.revision = nixpkgs.lib.mkForce null;
-              }
-            ];
-          }).config.system.build.macos-builder-installer;
-          runLinuxBuilderScript = pkgs.writeShellScriptBin "run-linux-builder" ''
-            set -uo pipefail
-            trap 's=$?; echo "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR
-            IFS=$'\n\t'
-            mkdir -p "${linuxBuilderDataDir}"
-            cd "${linuxBuilderDataDir}"
-            ${darwin-builder}/bin/create-builder
-          '';
-        in
-       {
-        linux-builder = darwinSystem {
+
+      darwinConfigurations = {
+        builder = darwinSystem {
           inherit system;
           modules = [
             {
-              environment = {
-                etc = {
-                  "nix/ssh_config".text = ''
-                    Host linux-builder
-                      User builder
-                      HostName 127.0.0.1
-                      HostKeyAlias linux-builder
-                      Port 31022
-                      StrictHostKeyChecking no
-                  '';
-                };
-              };
-
-              launchd.daemons.darwin-builder = {
-                command = "${runLinuxBuilderScript}/bin/run-linux-builder";
-                serviceConfig = {
-                  KeepAlive = true;
-                  RunAtLoad = true;
-                  StandardOutPath = linuxBuilderDataDir;
-                  StandardErrorPath = linuxBuilderLog;
-                };
-              };
-
+              nix.settings.trusted-users = [ username ];
+              nix.configureBuildUsers = true;
+              nix.linux-builder.enable = true;
               nix.distributedBuilds = true;
-              nix.buildMachines = [{
-                hostName = "ssh://linux-builder";
-                sshUser = "builder";
-                system = linuxSystem;
-                maxJobs = 4;
-                sshKey = "/etc/nix/builder_ed25519";
-                supportedFeatures = [ "kvm" "benchmark" "big-parallel" ];
-              }];
-              nix.envVars = { NIX_SSHOPTS = "-F /etc/nix/ssh_config"; };
-              services.nix-daemon.enable = true;
+              nix.extraOptions = ''
+                auto-optimise-store = true
+                keep-outputs = true
+                keep-derivations = true
+                experimental-features = nix-command flakes
+                '';
             }
           ];
         };
@@ -99,7 +52,7 @@
         default = qcow;
 
         qcow = nixos-generators.nixosGenerate {
-          system = linuxSystem;
+          inherit system;
           modules = [
             srvos.nixosModules.server
             ./nixos/qemu.nix
@@ -109,7 +62,7 @@
         };
 
         docker = nixos-generators.nixosGenerate {
-          system = linuxSystem;
+          inherit system;
           modules = [ 
             srvos.nixosModules.server
             ./nixos/docker.nix

diff --git a/nixos/builder.nix b/nixos/builder.nix
@@ -0,0 +1,49 @@
+{ 
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}: 
+let
+  system = "aarch64-darwin";
+  linuxSystem = builtins.replaceStrings ["darwin"] ["linux"] system;
+
+  darwin-builder = lib.nixosSystem {
+    system = linuxSystem;
+    modules = [
+      "${modulesPath}/profiles/macos-builder.nix"
+      {
+        virtualisation.host.pkgs = pkgs;
+        virtualisation.darwin-builder.diskSize = lib.mkForce (40 * 1024);
+        virtualisation.darwin-builder.memorySize = lib.mkForce (4 * 1024);
+        virtualisation.darwin-builder.workingDirectory = "/var/lib/darwin-builder";
+        system.nixos.revision = lib.mkForce null;
+      }
+    ];
+  };
+  enableDarwinBuilder = true;
+in {
+  nix.distributedBuilds = true;
+  nix.buildMachines = [
+    {
+      protocol = "ssh-ng";
+      hostName = "linux-builder";
+      sshUser = "builder";
+      system = linuxSystem;
+      maxJobs = 4;
+      supportedFeatures = ["kvm" "benchmark" "big-parallel"];
+      sshKey = "/etc/nix/builder_ed25519";
+      publicHostKey = "c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSUpCV2N4Yi9CbGFxdDFhdU90RStGOFFVV3JVb3RpQzVxQkorVXVFV2RWQ2Igcm9vdEBuaXhvcwo=";
+    }
+  ];
+
+  launchd.daemons.darwin-builder = lib.mkIf enableDarwinBuilder {
+    command = "${darwin-builder.config.system.build.macos-builder-installer}/bin/create-builder";
+    serviceConfig = {
+      KeepAlive = true;
+      RunAtLoad = true;
+      StandardOutPath = "/var/log/darwin-builder.log";
+      StandardErrorPath = "/var/log/darwin-builder.log";
+    };
+  };
+}