diff --git a/.envrc b/.envrc new file mode 100644 index 00000000..8392d159 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake \ No newline at end of file diff --git a/.gitignore b/.gitignore index 31b0a4b6..e1f7c788 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,10 @@ result +disko-config.nix + +.direnv + +__pycache__/ + # Created by the NixOS interactive test driver .nixos-test-history \ No newline at end of file diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 00000000..5853a6dd --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,11 @@ +{ + "recommendations": [ + "mkhl.direnv", + "jnoortheen.nix-ide", + "ms-python.python", + "ms-python.debugpy", + "ms-python.mypy-type-checker", + "charliermarsh.ruff", + "ms-toolsai.jupyter" + ] +} \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..559ae5da --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,65 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "disko2 mount disko_file", + "type": "debugpy", + "request": "launch", + "module": "disko", + "env": { + "PYTHONPATH": "${workspaceFolder}/src" + }, + "console": "integratedTerminal", + "args": [ + "mount", + "example/simple-efi.nix" + ] + }, + { + "name": "disko2 mount flake", + "type": "debugpy", + "request": "launch", + "module": "disko", + "env": { + "PYTHONPATH": "${workspaceFolder}/src" + }, + "console": "integratedTerminal", + "args": [ + "mount", + "--flake", + ".#testmachine" + ] + }, + { + "name": "disko2 generate", + "type": "debugpy", + "request": "launch", + "module": "disko", + "env": { + "PYTHONPATH": "${workspaceFolder}/src" + }, + "console": "integratedTerminal", + "args": [ + "generate" + ] + }, + { + "name": "disko2 destroy,format,mount dry-run", + "type": "debugpy", + "request": "launch", + "module": "disko", + "env": { + "PYTHONPATH": "${workspaceFolder}/src" + }, + "console": "integratedTerminal", + "args": [ + "destroy,format,mount", + "--dry-run", + "disko-config.nix" + ] + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..c1bf8bb3 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,27 @@ +{ + "nix.serverSettings": { + "nixd": { + "formatting": { + "command": [ + "nixpkgs-fmt", + "--" + ] + } + } + }, + "cSpell.enabled": true, + "cSpell.words": [ + "Disko", + "nixos", + "nixpkgs" + ], + "python.analysis.extraPaths": [ + "./src" + ], + "mypy-type-checker.importStrategy": "fromEnvironment", + "python.testing.pytestArgs": [ + "tests" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e8540b5e..b68b7154 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,6 +12,9 @@ way to help us fix issues quickly. Check out For more information on how to run and debug tests, check out [Running and debugging tests](./docs/testing.md). +Also refer to [Setting up your development environment](./dev-setup.md) to get +the best possible development experience. + ## How to find issues to work on If you're looking for a low-hanging fruit, check out diff --git a/default.nix b/default.nix index bf3570e2..19ff77b4 100644 --- a/default.nix +++ b/default.nix @@ -1,44 +1,8 @@ { lib ? import , rootMountPoint ? "/mnt" , checked ? false -, diskoLib ? import ./lib { inherit lib rootMountPoint; } +, diskoLib ? import ./src/disko_lib { inherit lib rootMountPoint; } }: -let - eval = cfg: lib.evalModules { - modules = lib.singleton { - # _file = toString input; - imports = lib.singleton { disko.devices = cfg.disko.devices; }; - options = { - disko.devices = lib.mkOption { - type = diskoLib.toplevel; - }; - }; - }; - }; -in -{ - lib = lib.warn "the .lib.lib output is deprecated" diskoLib; - - # legacy alias - create = cfg: builtins.trace "the create output is deprecated, use format instead" (eval cfg).config.disko.devices._create; - createScript = cfg: pkgs: builtins.trace "the create output is deprecated, use format instead" ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).formatScript; - createScriptNoDeps = cfg: pkgs: builtins.trace "the create output is deprecated, use format instead" ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).formatScriptNoDeps; - - format = cfg: (eval cfg).config.disko.devices._create; - formatScript = cfg: pkgs: ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).formatScript; - formatScriptNoDeps = cfg: pkgs: ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).formatScriptNoDeps; - - mount = cfg: (eval cfg).config.disko.devices._mount; - mountScript = cfg: pkgs: ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).mountScript; - mountScriptNoDeps = cfg: pkgs: ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).mountScriptNoDeps; - - disko = cfg: (eval cfg).config.disko.devices._disko; - diskoScript = cfg: pkgs: ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).diskoScript; - diskoScriptNoDeps = cfg: pkgs: ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).diskoScriptNoDeps; - - # we keep this old output for backwards compatibility - diskoNoDeps = cfg: pkgs: builtins.trace "the diskoNoDeps output is deprecated, please use disko instead" ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).diskoScriptNoDeps; - - config = cfg: (eval cfg).config.disko.devices._config; - packages = cfg: (eval cfg).config.disko.devices._packages; +diskoLib.outputs { + inherit lib checked rootMountPoint; } diff --git a/disko b/disko index 4ef771a9..af97de37 100755 --- a/disko +++ b/disko @@ -153,7 +153,7 @@ else fi # The "--impure" is still pure, as the path is within the nix store. -script=$(nixBuild "${libexec_dir}"/cli.nix \ +script=$(nixBuild "${libexec_dir}"/src/cli.nix \ --no-out-link \ --impure \ --argstr mode "$mode" \ diff --git a/disko-install b/disko-install index a10ff80d..9c3fc8a8 100755 --- a/disko-install +++ b/disko-install @@ -197,7 +197,7 @@ main() { # shellcheck disable=SC2064 trap "cleanupMountPoint ${escapeMountPoint}" EXIT - outputs=$(nixBuild "${libexec_dir}"/install-cli.nix \ + outputs=$(nixBuild "${libexec_dir}"/src/install-cli.nix \ "${nix_args[@]}" \ --no-out-link \ --impure \ diff --git a/disko2 b/disko2 new file mode 100755 index 00000000..167539fc --- /dev/null +++ b/disko2 @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# This script only exists so you can run `./disko2` directly +# It should not be be installed as part of the package! +# Check src/disko/cli.py for the actual entrypoint + +set -euo pipefail +PYTHONPATH="$(dirname "$(realpath "$0")")"/src python3 -m disko "$@" \ No newline at end of file diff --git a/doc.nix b/doc.nix index d07f8f32..fae176fe 100644 --- a/doc.nix +++ b/doc.nix @@ -1,7 +1,7 @@ { lib, nixosOptionsDoc, runCommand, fetchurl, pandoc }: let - diskoLib = import ./lib { + diskoLib = import ./src/disko_lib { inherit lib; rootMountPoint = "/mnt"; }; diff --git a/docs/INDEX.md b/docs/INDEX.md index 2c1b0270..1e78b9a2 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -19,4 +19,5 @@ ### For contributors +- [Setting up your development environment](./dev-setup.md) - [Running and debugging tests](./testing.md) diff --git a/docs/dev-setup.md b/docs/dev-setup.md new file mode 100644 index 00000000..a8c8dc9b --- /dev/null +++ b/docs/dev-setup.md @@ -0,0 +1,75 @@ +# Setting up your development environment + +**This guide assumes you have flakes enabled.** + +disko uses Nix flake's `devShells` output and [direnv](https://direnv.net/) to +set up the development environment in two ways. + +The quickest way to get started is to run: + +``` +nix develop +``` + +However, if you use a shell other than bash, working inside `nix develop` might +get annoying quickly. An alternative is to use direnv, which sets up the +environment in your current shell session: + +```console +# nix shell nixpkgs#direnv +direnv: error /home/felix/repos-new/temp/disko/.envrc is blocked. Run `direnv allow` to approve its content +# direnv allow +direnv: loading ~/repos-new/temp/disko/.envrc +direnv: using flake +direnv: export +AR +AS +CC +CONFIG_SHELL +CXX +HOST_PATH +IN_NIX_SHELL +LD +NIX_BINTOOLS +NIX_BINTOOLS_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu +NIX_BUILD_CORES +NIX_BUILD_TOP +NIX_CC +NIX_CC_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu +NIX_CFLAGS_COMPILE +NIX_ENFORCE_NO_NATIVE +NIX_HARDENING_ENABLE +NIX_LDFLAGS +NIX_STORE +NM +OBJCOPY +OBJDUMP +RANLIB +READELF +SIZE +SOURCE_DATE_EPOCH +STRINGS +STRIP +TEMP +TEMPDIR +TMP +TMPDIR +__structuredAttrs +buildInputs +buildPhase +builder +cmakeFlags +configureFlags +depsBuildBuild +depsBuildBuildPropagated +depsBuildTarget +depsBuildTargetPropagated +depsHostHost +depsHostHostPropagated +depsTargetTarget +depsTargetTargetPropagated +doCheck +doInstallCheck +dontAddDisableDepTrack +mesonFlags +name +nativeBuildInputs +out +outputs +patches +phases +preferLocalBuild +propagatedBuildInputs +propagatedNativeBuildInputs +shell +shellHook +stdenv +strictDeps +system ~PATH ~XDG_DATA_DIRS +``` + +You can now run `./disko2 dev --help` or `pytest` to confirm everything is working. + +If you're working exclusively in the terminal, you're all set. + +## IDE Integration + +### VSCode + +If you're using VSCode or any of its forks, you should install the recommended +extensions. You can find them in .vscode/extensions.json, searching for +`@recommended` in the Extensions Panel, or by opening the +command pallette and searching "Show Recommended Extensions". + +You can then install all extensions in one click: + +![VSCode button "Install workspace recommended extensions](./img/vscode-recommended-ext.png) + +When you do this (and every time you open the repository again), the +[direnv extension](https://marketplace.visualstudio.com/items?itemName=mkhl.direnv) +will prompt you in the bottom right corner to restart all extensions once it +has loaded the development environment: + +![Direnv extension asking "Environment updated. Restart extensions?"](./img/vscode-direnv-prompt.png) + +Click "Restart" to make all dependencies available to VSCode. + +Afterwards, open the command pallette, search "Python: Select Interpreter" and +press Enter. The "Select Interpreter" dialog will open: + +![VSCode Python "Select Interpreter" dialog](./img/vscode-select-python.png) + +Do not select the interpreters tagged "Recommended" or "Global"! These will be +the ones installed on your system or currently selected in the extension. +Instead, pick one of the ones below them, this will be the latest available +package that includes the python packages specified in the `devShell`! + +Now you're all set! You might want to also enable the "Format on Save" settings +and run the "Format Document" command once to select a formatter for each file +type. + +Remember to go through these steps again whenever updating the flake. + +### Other IDEs + +These are just notes. Feel free to submit a PR that adds support and +documentation for other IDEs! + +[Jetbrains Marketplace has two direnv extensions available](https://plugins.jetbrains.com/search?excludeTags=internal&search=direnv) + diff --git a/docs/img/vscode-direnv-prompt.png b/docs/img/vscode-direnv-prompt.png new file mode 100644 index 00000000..dfa1f83e Binary files /dev/null and b/docs/img/vscode-direnv-prompt.png differ diff --git a/docs/img/vscode-recommended-ext.png b/docs/img/vscode-recommended-ext.png new file mode 100644 index 00000000..efd25448 Binary files /dev/null and b/docs/img/vscode-recommended-ext.png differ diff --git a/docs/img/vscode-select-python.png b/docs/img/vscode-select-python.png new file mode 100644 index 00000000..4e2c1c15 Binary files /dev/null and b/docs/img/vscode-select-python.png differ diff --git a/docs/reference.md b/docs/reference.md index 104ba1da..2ae88429 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -4,7 +4,7 @@ We are currently having issues being able to generate proper module option documentation for our recursive disko types. However you can read the available -options [here](https://github.com/nix-community/disko/tree/master/lib/types). +options [here](https://github.com/nix-community/disko/tree/master/src/disko_lib/types). Combined with the [examples](https://github.com/nix-community/disko/tree/master/example) this hopefully gives you an overview. diff --git a/docs/testing.md b/docs/testing.md index 769b1a33..8d96ca1f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1,5 +1,20 @@ # Running and debugging tests +This assumes you've already set up your development environment! + +See [Setting up your development environment](./dev-setup.md) + +## Python tests + +Big parts of disko are written in Python. You can test all Python +functionality by simply running + +``` +pytest +``` + +## VM tests + Disko makes extensive use of VM tests. All examples you can find in [the example directory](../example) have a respective test suite that verifies the example is working in [the tests directory](../tests/). They utilize the @@ -7,14 +22,14 @@ the example is working in [the tests directory](../tests/). They utilize the We use a wrapper around this called `makeDiskoTest`. There is currently (as of 2024-10-16) no documentation for all its arguments, but you can have a look at -[its current code](https://github.com/nix-community/disko/blob/master/lib/tests.nix#L44C5-L58C10), +[its current code](https://github.com/nix-community/disko/blob/master/src/disko_lib/tests.nix#L44C5-L58C10), that should already be helpful. However, you don't need to know about all of the inner workings to interact with the tests effectively. For some of the most common operations, see the sections below. -## Run just one of the tests +### Run just one of the tests ```sh nix build --no-link .#checks.x86_64-linux.simple-efi @@ -27,7 +42,7 @@ virtual devices, run disko to format them, reboot, verify the VM boots properly, and then run the code specified in `extraTestScript` to validate that the partitions have been created and were mounted as expected. -### How `extraTestScript` works +#### How `extraTestScript` works This is written in Python. The most common lines you'll see look something like this: @@ -45,7 +60,7 @@ Disko currently (as of 2024-10-16) doesn't have any tests that utilize multiple VMs at once, so the only machine available in these scripts is always just the default `machine`. -## Debugging tests +### Debugging tests If you make changes to disko, you might break a test, or you may want to modify a test to prevent regressions. In these cases, running the full test with @@ -131,7 +146,7 @@ vdb 253:16 0 4G 0 disk You can find some additional details in [the NixOS manual's section on interactive testing](https://nixos.org/manual/nixos/stable/#sec-running-nixos-tests-interactively). -## Running all tests at once +### Running all tests at once If you have a bit of experience, you might be inclined to run `nix flake check` to run all tests at once. However, we instead recommend using diff --git a/example/gpt-name-with-whitespace.nix b/example/gpt-name-with-special-chars.nix similarity index 100% rename from example/gpt-name-with-whitespace.nix rename to example/gpt-name-with-special-chars.nix diff --git a/flake.nix b/flake.nix index ae51fc59..a45317c6 100644 --- a/flake.nix +++ b/flake.nix @@ -19,19 +19,22 @@ versionInfo = import ./version.nix; version = versionInfo.version + (lib.optionalString (!versionInfo.released) "-dirty"); + + diskoLib = import ./src/disko_lib { + inherit (nixpkgs) lib; + }; in { + lib = diskoLib; nixosModules.default = self.nixosModules.disko; # convention nixosModules.disko.imports = [ ./module.nix ]; - lib = import ./lib { - inherit (nixpkgs) lib; - }; packages = forAllSystems (system: let pkgs = nixpkgs.legacyPackages.${system}; in { disko = pkgs.callPackage ./package.nix { diskoVersion = version; }; + disko2 = pkgs.callPackage ./package-disko2.nix { diskoVersion = version; }; # alias to make `nix run` more convenient disko-install = self.packages.${system}.disko.overrideAttrs (_old: { name = "disko-install"; @@ -59,19 +62,52 @@ diskoVersion = version; }; + # TODO: Add a CI pipeline instead that runs nix run .#pytest inside nix develop + pytest-ci-only = pkgs.runCommand "pytest" { nativeBuildInputs = [ pkgs.python3Packages.pytest ]; } '' + cd ${./.} + # eval_config runs nix, which is forbidden inside of nix derivations by default + pytest -vv --doctest-modules -p no:cacheprovider --ignore=tests/disko_lib/eval_config + touch $out + ''; + shellcheck = pkgs.runCommand "shellcheck" { nativeBuildInputs = [ pkgs.shellcheck ]; } '' cd ${./.} - shellcheck disk-deactivate/disk-deactivate disko + shellcheck src/disk-deactivate/disk-deactivate disko disko2 touch $out ''; + + jsonTypes = pkgs.writeTextFile { name = "jsonTypes"; text = (builtins.toJSON diskoLib.jsonTypes); }; in # FIXME: aarch64-linux seems to hang on boot lib.optionalAttrs pkgs.hostPlatform.isx86_64 (nixosTests // { inherit disko-install; }) // pkgs.lib.optionalAttrs (!pkgs.buildPlatform.isRiscV64 && !pkgs.hostPlatform.isx86_32) { - inherit shellcheck; + inherit pytest-ci-only shellcheck jsonTypes; inherit (self.packages.${system}) disko-doc; }); + devShells = forAllSystems (system: + let + pkgs = nixpkgs.legacyPackages.${system}; + in + { + default = pkgs.mkShell { + name = "disko-dev"; + packages = (with pkgs; [ + nixpkgs-fmt # Formatter for Nix code + shellcheck # Linter for shell scripts + ruff # Formatter and linter for Python + (python3.withPackages (ps: [ + ps.mypy # Static type checker + ps.pytest # Test runner + ps.ipykernel # Jupyter kernel for experimenting + + # Actual runtime depedencies + ps.pydantic # Validation of nixos configuration + ])) + ]); + }; + }); + nixosConfigurations.testmachine = lib.nixosSystem { system = "x86_64-linux"; modules = [ @@ -90,6 +126,7 @@ nixpkgs-fmt deno deadnix + ruff ]; text = '' showUsage() { @@ -134,12 +171,16 @@ nixpkgs-fmt -- "''${files[@]}" deno fmt -- "''${files[@]}" deadnix --edit -- "''${files[@]}" + ruff check --fix + ruff format else set -o xtrace nixpkgs-fmt --check -- "''${files[@]}" deno fmt --check -- "''${files[@]}" deadnix -- "''${files[@]}" + ruff check + ruff format --check fi } diff --git a/module.nix b/module.nix index 3f444de4..a7a9e568 100644 --- a/module.nix +++ b/module.nix @@ -4,13 +4,13 @@ let vmVariantWithDisko = extendModules { modules = [ - ./lib/interactive-vm.nix + ./src/disko_lib/interactive-vm.nix config.disko.tests.extraConfig ]; }; in { - imports = [ ./lib/make-disk-image.nix ]; + imports = [ ./src/disko_lib/make-disk-image.nix ]; options.disko = { imageBuilder = { @@ -205,7 +205,7 @@ in } ]; - _module.args.diskoLib = import ./lib { + _module.args.diskoLib = import ./src/disko_lib { inherit lib; rootMountPoint = config.disko.rootMountPoint; makeTest = import (pkgs.path + "/nixos/tests/make-test-python.nix"); diff --git a/package-disko2.nix b/package-disko2.nix new file mode 100644 index 00000000..3c9dc920 --- /dev/null +++ b/package-disko2.nix @@ -0,0 +1,42 @@ +{ python3Packages, lib, lix, coreutils, nixos-install-tools, binlore, diskoVersion }: + +let + self = python3Packages.buildPythonApplication { + pname = "disko2"; + version = diskoVersion; + src = ./.; + pyproject = true; + + build-system = [ python3Packages.setuptools ]; + dependencies = [ + lix # lix instead of nix because it produces way better eval errors + coreutils + nixos-install-tools + ]; + + # Otherwise resholve thinks that disko and disko-install might be able to execute their arguments + passthru.binlore.out = binlore.synthesize self '' + execer cannot bin/.disko2-wrapped + ''; + postInstall = '' + mkdir -p $out/share/disko/ + cp example/simple-efi.nix $out/share/disko/ + ''; + + makeWrapperArgs = [ "--set DISKO_VERSION ${diskoVersion}" ]; + + doCheck = true; + # installCheckPhase = '' + # $out/bin/disko2 mount $out/share/disko/simple-efi.nix + # ''; + meta = with lib; { + description = "Format disks with nix-config"; + homepage = "https://github.com/nix-community/disko"; + license = licenses.mit; + maintainers = with maintainers; [ lassulus ]; + platforms = platforms.linux; + mainProgram = "disko2"; + }; + }; +in +self diff --git a/package.nix b/package.nix index 3464854d..f1dd6e3f 100644 --- a/package.nix +++ b/package.nix @@ -9,7 +9,7 @@ let ]; installPhase = '' mkdir -p $out/bin $out/share/disko - cp -r install-cli.nix cli.nix default.nix disk-deactivate lib $out/share/disko + cp -r default.nix src $out/share/disko for i in disko disko-install; do sed -e "s|libexec_dir=\".*\"|libexec_dir=\"$out/share/disko\"|" "$i" > "$out/bin/$i" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..9d74b8f3 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,59 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "disko2" +version = "2.0.0-preview" + +[project.scripts] +disko2 = "disko:main" + +[tool.setuptools] +package-dir = { "" = "src" } + +[tool.setuptools.package-data] +"*" = ["*.nix"] + +[tool.mypy] +mypy_path = "src" +warn_unused_configs = true +# We don't use `strict = true` but write out all individual flags because +# strict is not strict enough and its meaning may change in the future. +# See https://mypy.readthedocs.io/en/stable/config_file.html#confval-strict +# Disallow dynamic typing +disallow_any_unimported = true +disallow_any_expr = true +disallow_any_decorated = true +disallow_any_explicit = false # We need to be able to use `Any` as an input parameter to some functions +disallow_any_generics = true +disallow_subclassing_any = true +# Untyped definitions and calls +disallow_untyped_calls = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true +disallow_untyped_decorators = true +# Warnings +warn_redundant_casts = true +warn_unused_ignores = true +warn_no_return = true +warn_return_any = true +warn_unreachable = true +# Miscellaneous strictness flags +no_implicit_reexport = true +strict_equality = true +extra_checks = true +enable_error_code = "ignore-without-code" +# Error message control +show_error_context = true +show_error_code_links = true +pretty = true + +[tool.pytest.ini_options] +pythonpath = ["src"] +addopts = ["--doctest-modules"] + +[tool.autoflake] +remove_all_unused_imports = true +in_place = true diff --git a/scripts/generate_python_types.py b/scripts/generate_python_types.py new file mode 100755 index 00000000..1580aeff --- /dev/null +++ b/scripts/generate_python_types.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 + +import io +import json +import sys +from typing import Any, Callable, Mapping, TypeGuard, TypeVar, TypedDict, cast + +JsonDict = dict[str, "JsonValue"] +JsonValue = str | int | float | bool | None | list["JsonValue"] | JsonDict + + +class TypeDefinition(TypedDict): + type: str | JsonDict + default: JsonValue + description: str + + +def is_type(type_field: JsonValue) -> TypeGuard[TypeDefinition]: + if not isinstance(type_field, dict): + return False + return set(type_field.keys()) == {"default", "description", "type"} + + +def parse_type( + containing_class: str, field_name: str, type_field: str | dict[str, Any] +) -> tuple[str, io.StringIO | None]: + """Parse a type field into a Python type annotation. + + If the type is a class itself, the second element of the tuple + will be a buffer containing the class definition. + """ + + if isinstance(type_field, str): + return _parse_simple_type(type_field), None + elif isinstance(type_field, dict): + if type_field.get("__isCompositeType"): + return _parse_composite_type(containing_class, field_name, type_field) + else: + class_name = f"{containing_class}_{field_name}" + class_code, inner_types_code = generate_class(class_name, type_field) + + if not inner_types_code: + inner_types_code = io.StringIO() + + inner_types_code.write("\n\n") + inner_types_code.write(class_code) + + return class_name, inner_types_code + + else: + raise ValueError(f"Invalid type field: {type_field}") + + +def _parse_composite_type( + containing_class: str, field_name: str, type_dict: dict[str, Any] +) -> tuple[str, io.StringIO | None]: + assert isinstance(type_dict["type"], str) + + type_name, type_code = None, None + if "subType" in type_dict: + try: + type_name, type_code = parse_type( + containing_class, field_name, type_dict["subType"] + ) + except Exception as e: + e.add_note(f"Error in subType {type_dict["subType"]}") + raise e + + match type_dict["type"]: + case "attrsOf": + return f"dict[str, {type_name}]", type_code + case "listOf": + return f"list[{type_name}]", type_code + case "nullOr": + return f"None | {type_name}", type_code + case "oneOf": + type_code = io.StringIO() + type_names = [] + for sub_type in type_dict["types"]: + try: + sub_type_name, sub_type_code = parse_type( + containing_class, field_name, sub_type + ) + except Exception as e: + e.add_note(f"Error in subType {sub_type}") + raise e + + type_names.append(sub_type_name) + if sub_type_code: + type_code.write(sub_type_code.getvalue()) + + # Can't use | syntax in all cases, Union always works + return f'Union[{", ".join(type_names)}]', type_code + case "enum": + return ( + f'Literal[{", ".join(f"{repr(value)}" for value in type_dict["choices"])}]', + None, + ) + case _: + return _parse_simple_type(type_dict["type"]), None + + +def _parse_simple_type(type_str: str) -> str: + match type_str: + case "str": + return "str" + case "absolute-pathname": + return "str" + case "bool": + return "bool" + case "int": + return "int" + case "anything": + return "Any" + # Set up discriminated unions to reduce error messages when validation fails + case "deviceType": + return '"deviceType" = Field(..., discriminator="type")' + case "partitionType": + return '"partitionType" = Field(..., discriminator="type")' + case _: + # Probably a type alias, needs to be quoted in case the type is defined later + return f'"{type_str}"' + + +def parse_field( + containing_class: str, field_name: str, field: str | JsonDict +) -> tuple[str, io.StringIO | None]: + if isinstance(field, str): + return _parse_simple_type(field), None + + if is_type(field): + return parse_type(containing_class, field_name, field["type"]) + + class_name = f"{containing_class}_{field_name}" + class_code, inner_types_code = generate_class(class_name, field) + + if not inner_types_code: + inner_types_code = io.StringIO() + + inner_types_code.write("\n\n") + inner_types_code.write(class_code) + + return class_name, inner_types_code + + +def generate_type_alias( + name: str, type_spec: str | dict[str, Any] +) -> io.StringIO | None: + buffer = io.StringIO() + + try: + type_code, sub_type_code = parse_type(name, "", type_spec) + except ValueError: + return None + + if sub_type_code: + buffer.write(sub_type_code.getvalue()) + buffer.write("\n\n") + + buffer.write(f"{name} = {type_code}") + buffer.write("\n\n") + + return buffer + + +def generate_class(name: str, fields: dict[str, Any]) -> tuple[str, io.StringIO | None]: + assert isinstance(fields, dict) + + contained_classes_buffer = io.StringIO() + + buffer = io.StringIO() + buffer.write(f"class {name}(BaseModel):\n") + + for field_name, field in fields.items(): + try: + type_name, type_code = parse_field(name, field_name, field) + except Exception as e: + e.add_note(f"Error in field {field_name}: {field}") + raise e + + if type_code: + contained_classes_buffer.write(type_code.getvalue()) + + # Fields starting with _ are swallowed, see https://github.com/pydantic/pydantic/issues/2105 + if field_name.startswith("_"): + field_definition = ( + f'{field_name.lstrip("_")}: {type_name} = Field(alias="{field_name}")' + ) + else: + field_definition = f"{field_name}: {type_name}" + + buffer.write(f" {field_definition}\n") + + if contained_classes_buffer.tell() == 0: + return buffer.getvalue(), None + + return buffer.getvalue(), contained_classes_buffer + + +T = TypeVar("T", bound=JsonValue) + + +def transform_dict_keys(d: T, transform_fn: Callable[[str], str]) -> T: + if not isinstance(d, Mapping): + return d + + return cast( + T, + {transform_fn(k): transform_dict_keys(v, transform_fn) for k, v in d.items()}, + ) + + +def generate_python_code(schema: JsonDict) -> io.StringIO: + assert isinstance(schema, dict) + + # Convert disallowed characters in Python identifiers + schema = transform_dict_keys(schema, lambda k: k.replace("-", "_")) + + buffer = io.StringIO() + + buffer.write( + """# File generated by scripts/generate_python_types.py +# Ignore warnings that decorators contain Any +# mypy: disable-error-code="misc" +# Disable auto-formatting for this file +# fmt: off +from typing import Any, Literal, Union +from pydantic import BaseModel, Field + + +""" + ) + + for type_name, fields in schema.items(): + assert isinstance(fields, dict) + if "__isCompositeType" in fields: + try: + alias_content, type_code = _parse_composite_type(type_name, "", fields) + except Exception as e: + e.add_note(f"Error in composite type {type_name}") + raise e + + if type_code: + buffer.write(type_code.getvalue()) + buffer.write("\n\n") + + buffer.write(f"{type_name} = {alias_content}") + buffer.write("\n\n") + continue + + try: + class_code, inner_types_code = generate_class(type_name, fields) + except Exception as e: + e.add_note(f"Error in class {type_name}") + raise e + + if inner_types_code: + buffer.write(inner_types_code.getvalue()) + buffer.write("\n\n") + + buffer.write(class_code) + buffer.write("\n\n") + + buffer.write( + """ +class DiskoConfig(BaseModel): + disk: dict[str, disk] + lvm_vg: dict[str, lvm_vg] + mdadm: dict[str, mdadm] + nodev: dict[str, nodev] + zpool: dict[str, zpool] +""" + ) + + return buffer + + +def main(in_file: str, out_file: str) -> None: + with open(in_file) as f: + schema = json.load(f) + + code_buffer = generate_python_code(schema) + + with open(out_file, "w") as f: + f.write(code_buffer.getvalue()) + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print( + """Usage: generate_python_types.py +Recommendation: Go to the root of this repository and run + + nix build .#checks.x86_64-linux.jsonTypes + +to generate the JSON schema file first, then run + + ./scripts/generate_python_types.py result src/disko_lib/config_types.py +""" + ) + sys.exit(1) + + main(sys.argv[1], sys.argv[2]) diff --git a/cli.nix b/src/cli.nix similarity index 98% rename from cli.nix rename to src/cli.nix index 3203cedf..cf8d9aa8 100644 --- a/cli.nix +++ b/src/cli.nix @@ -9,7 +9,7 @@ , ... }@args: let - disko = import ./. { + disko = import ../. { inherit rootMountPoint; inherit lib; }; diff --git a/disk-deactivate/disk-deactivate b/src/disk-deactivate/disk-deactivate similarity index 100% rename from disk-deactivate/disk-deactivate rename to src/disk-deactivate/disk-deactivate diff --git a/disk-deactivate/disk-deactivate.jq b/src/disk-deactivate/disk-deactivate.jq similarity index 100% rename from disk-deactivate/disk-deactivate.jq rename to src/disk-deactivate/disk-deactivate.jq diff --git a/src/disko/__init__.py b/src/disko/__init__.py new file mode 100644 index 00000000..9595be4a --- /dev/null +++ b/src/disko/__init__.py @@ -0,0 +1,5 @@ +from . import cli + + +def main() -> None: + cli.main() diff --git a/src/disko/__main__.py b/src/disko/__main__.py new file mode 100644 index 00000000..868d99ef --- /dev/null +++ b/src/disko/__main__.py @@ -0,0 +1,4 @@ +from . import main + +if __name__ == "__main__": + main() diff --git a/src/disko/cli.py b/src/disko/cli.py new file mode 100644 index 00000000..320c9101 --- /dev/null +++ b/src/disko/cli.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 + +import argparse +import dataclasses +import json +from typing import Any, Literal, cast + +from disko.mode_dev import run_dev +from disko.mode_generate import run_generate +from disko_lib.action import Action +from disko_lib.config_type import DiskoConfig +from disko_lib.eval_config import ( + eval_config_dict_as_json, + eval_config_file_as_json, + validate_config, +) +from disko_lib.generate_config import generate_config +from disko_lib.generate_plan import generate_plan +from disko_lib.logging import LOGGER, debug, info +from disko_lib.messages.msgs import err_missing_mode +from disko_lib.result import ( + DiskoError, + DiskoPartialSuccess, + DiskoResult, + DiskoSuccess, + exit_on_error, +) +from disko_lib.json_types import JsonDict + +Mode = ( + Action + | Literal[ + "destroy,format,mount", + "format,mount", + "generate", + "dev", + ] +) + +MODE_TO_ACTIONS: dict[Mode, set[Action]] = { + "destroy": {"destroy"}, + "format": {"format"}, + "mount": {"mount"}, + "destroy,format,mount": {"destroy", "format", "mount"}, + "format,mount": {"format", "mount"}, +} + + +# Modes to apply an existing configuration +APPLY_MODES: list[Mode] = [ + "destroy", + "format", + "mount", + "destroy,format,mount", + "format,mount", +] +ALL_MODES: list[Mode] = APPLY_MODES + ["generate", "dev"] + +MODE_DESCRIPTION: dict[Mode, str] = { + "destroy": "Destroy the partition tables on the specified disks", + "format": "Change formatting and filesystems on the specified disks", + "mount": "Mount the specified disks", + "destroy,format,mount": "Run destroy, format and mount in sequence", + "format,mount": "Run format and mount in sequence", + "generate": "Generate a disko configuration file from the system's current state", + "dev": "Print information useful for developers", +} + + +def run_apply( + *, + mode: Mode, + disko_file: str | None, + flake: str | None, + dry_run: bool, + **_kwargs: dict[str, Any], +) -> DiskoResult[JsonDict]: + assert mode in APPLY_MODES + + target_config_json = eval_config_file_as_json(disko_file=disko_file, flake=flake) + if isinstance(target_config_json, DiskoError): + return target_config_json + + target_config = validate_config(target_config_json.value) + if isinstance(target_config, DiskoError): + return target_config.with_context("validate evaluated config") + + current_status_dict = generate_config() + if isinstance(current_status_dict, DiskoError) and not isinstance( + current_status_dict, DiskoPartialSuccess + ): + return current_status_dict.with_context("generate current status") + + current_status_evaluated = eval_config_dict_as_json(current_status_dict.value) + if isinstance(current_status_evaluated, DiskoError): + return current_status_evaluated.with_context("eval current status") + + current_status = validate_config(current_status_evaluated.value) + if isinstance(current_status, DiskoError): + return current_status.with_context("validate current status") + + actions = MODE_TO_ACTIONS[mode] + + plan = generate_plan(actions, current_status.value, target_config.value) + if isinstance(plan, DiskoError): + return plan + + plan_as_dict: JsonDict = dataclasses.asdict(plan.value) + steps = {"steps": plan_as_dict.get("steps", [])} + + if dry_run: + return DiskoSuccess(steps, "generate plan") + + info("Plan execution is not implemented yet!") + + return DiskoSuccess(steps, "generate plan") + + +def run( + args: argparse.Namespace, +) -> DiskoResult[None | JsonDict | DiskoConfig]: + if cast(bool, args.verbose): + LOGGER.setLevel("DEBUG") + debug("Enabled debug logging.") + + match cast(Mode | None, args.mode): + case None: + return DiskoError.single_message( + err_missing_mode, "select mode", valid_modes=[str(m) for m in ALL_MODES] + ) + case "generate": + return run_generate() + case "dev": + return run_dev(args) + case _: + return run_apply(**vars(args)) # type: ignore[misc] + + +def parse_args() -> argparse.Namespace: + root_parser = argparse.ArgumentParser( + prog="disko2", + description="Automated disk partitioning and formatting tool for NixOS", + ) + + root_parser.add_argument( + "--verbose", + "-v", + action="store_true", + default=False, + help="Print more detailed output, helpful for debugging", + ) + + mode_parsers = root_parser.add_subparsers(dest="mode") + + def create_apply_parser(mode: Mode) -> argparse.ArgumentParser: + parser = mode_parsers.add_parser( + mode, + help=MODE_DESCRIPTION[mode], + ) + return parser + + def add_common_apply_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "disko_file", + nargs="?", + default=None, + help="Path to the disko configuration file", + ) + parser.add_argument( + "--flake", + "-f", + help="Flake to fetch the disko configuration from", + ) + parser.add_argument( + "--dry-run", + "-n", + action="store_true", + default=False, + help="Print the plan without executing it", + ) + + # Commands to apply an existing configuration + apply_parsers = [create_apply_parser(mode) for mode in APPLY_MODES] + for parser in apply_parsers: + add_common_apply_args(parser) + + # Other commands + _generate_parser = mode_parsers.add_parser( + "generate", + help=MODE_DESCRIPTION["generate"], + ) + + # Commands for developers + dev_parsers = mode_parsers.add_parser( + "dev", + help=MODE_DESCRIPTION["dev"], + ).add_subparsers(dest="dev_command") + dev_parsers.add_parser("lsblk", help="List block devices the way disko sees them") + dev_parsers.add_parser("ansi", help="Print defined ansi color codes") + + dev_eval_parser = dev_parsers.add_parser( + "eval", help="Evaluate a disko configuration and print the result as JSON" + ) + add_common_apply_args(dev_eval_parser) + dev_validate_parser = dev_parsers.add_parser( + "validate", + help="Validate a disko configuration file or flake", + ) + add_common_apply_args(dev_validate_parser) + + return root_parser.parse_args() + + +def main() -> None: + args = parse_args() + result = run(args) + output = exit_on_error(result) + if output: + info("Output:\n" + json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/src/disko/mode_dev.py b/src/disko/mode_dev.py new file mode 100644 index 00000000..a7c91ee4 --- /dev/null +++ b/src/disko/mode_dev.py @@ -0,0 +1,75 @@ +import argparse +import json +from typing import Any, cast + +from disko_lib.ansi import Colors +from disko_lib.eval_config import eval_config_file_as_json, validate_config +from disko_lib.messages.msgs import err_missing_mode +from disko_lib.result import DiskoError, DiskoSuccess, DiskoResult +from disko_lib.types.device import run_lsblk + + +def run_dev_lsblk() -> DiskoResult[None]: + output = run_lsblk() + if isinstance(output, DiskoError): + return output + + print(output.value) + return DiskoSuccess(None, "run disko dev lsblk") + + +def run_dev_ansi() -> DiskoResult[None]: + import inspect + + for name, value in inspect.getmembers(Colors): + if value != "_" and not name.startswith("_") and name != "RESET": # type: ignore[misc] + print("{:>30} {}".format(name, value + name + Colors.RESET)) # type: ignore[misc] + + return DiskoSuccess(None, "run disko dev ansi") + + +def run_dev_eval( + *, disko_file: str | None, flake: str | None, **_: Any +) -> DiskoResult[None]: + result = eval_config_file_as_json(disko_file=disko_file, flake=flake) + + if isinstance(result, DiskoError): + return result + + print(json.dumps(result.value, indent=2)) + return DiskoSuccess(None, "run disko dev eval") + + +def run_dev_validate( + *, disko_file: str | None, flake: str | None, **_: Any +) -> DiskoResult[None]: + eval_result = eval_config_file_as_json(disko_file=disko_file, flake=flake) + if isinstance(eval_result, DiskoError): + return eval_result + + validate_result = validate_config(eval_result.value) + if isinstance(validate_result, DiskoError): + return validate_result + + print( + validate_result.value.model_dump_json(indent=2, by_alias=True, warnings="error") + ) + return DiskoSuccess(None, "run disko dev validate") + + +def run_dev(args: argparse.Namespace) -> DiskoResult[None]: + match cast(str | None, args.dev_command): + case "lsblk": + return run_dev_lsblk() + case "ansi": + return run_dev_ansi() + case "eval": + return run_dev_eval(**vars(args)) # type: ignore[misc] + case "validate": + return run_dev_validate(**vars(args)) # type: ignore[misc] + case _: + return DiskoError.single_message( + err_missing_mode, + "select mode", + valid_modes=["lsblk", "ansi", "eval", "validate"], + ) diff --git a/src/disko/mode_generate.py b/src/disko/mode_generate.py new file mode 100644 index 00000000..f785acd1 --- /dev/null +++ b/src/disko/mode_generate.py @@ -0,0 +1,75 @@ +import json +import re +from disko_lib.eval_config import eval_config_dict_as_json +from disko_lib.logging import info +from disko_lib.result import DiskoPartialSuccess, DiskoResult, DiskoError +from disko_lib.generate_config import generate_config +from disko_lib.run_cmd import run +from disko_lib.json_types import JsonDict + +DEFAULT_CONFIG_FILE = "disko-config.nix" + +HEADER_COMMENT = """ +# This file was generated by disko generate +# Some disk and partition names were auto-generated from device attributes +# to be unique, but might not be very descriptive. Feel free to change them +# to something more meaningful. + +""" + +PARTIAL_FAILURE_COMMENT = """ +############################################################################### +# WARNING: This file is incomplete! Some devices failed to generate a config. # +# Check the logs of the 'disko generate' command for more information. # +############################################################################### +""" + + +def filter_internal_keys(d: JsonDict) -> JsonDict: + return { + k: (v if not isinstance(v, dict) else filter_internal_keys(v)) + for k, v in d.items() + if not k.startswith("_") + } + + +def run_generate() -> DiskoResult[JsonDict]: + generate_result = generate_config() + + if isinstance(generate_result, DiskoError) and not isinstance( + generate_result, DiskoPartialSuccess + ): + return generate_result + + config_to_write = filter_internal_keys(generate_result.value) + + evaluate_result = eval_config_dict_as_json(config_to_write) + if isinstance(evaluate_result, DiskoError): + # TODO: Add --no-validate flag and explanatory text + return evaluate_result + + config_as_nix = run( + [ + "nix", + "eval", + "--expr", + f"builtins.fromJSON(''{json.dumps(config_to_write)}'')", + ] + ) + if isinstance(config_as_nix, DiskoError): + return config_as_nix + + # Contract the main attribute path to a single line to match all the examples + nix_code = re.sub(r"^\{ disko = \{ devices", "{ disko.devices", config_as_nix.value) + nix_code = re.sub(r"\}; \}$", "}", nix_code) + + with open(DEFAULT_CONFIG_FILE, "w") as f: + f.write(HEADER_COMMENT) + if isinstance(generate_result, DiskoError): + f.write(PARTIAL_FAILURE_COMMENT) + f.write(nix_code) + info(f"Wrote generated config to {DEFAULT_CONFIG_FILE}") + + run(["nixfmt", DEFAULT_CONFIG_FILE]) + + return generate_result diff --git a/src/disko_lib/__init__.py b/src/disko_lib/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/disko_lib/action.py b/src/disko_lib/action.py new file mode 100644 index 00000000..2bf2d2eb --- /dev/null +++ b/src/disko_lib/action.py @@ -0,0 +1,44 @@ +from dataclasses import dataclass, field +from typing import Literal + + +Action = Literal["destroy", "format", "mount"] + +EMPTY_DESCRIPTION = "__empty__" + + +@dataclass +class Step: + action: Action # Which action the step belongs to + commands: list[list[str]] # List of commands to execute + description: str # Explanatory message to display to the user + + @classmethod + def empty(cls, action: Action) -> "Step": + return cls(action, [], EMPTY_DESCRIPTION) + + def is_empty(self) -> bool: + return self.commands == [] and self.description == EMPTY_DESCRIPTION + + +@dataclass +class Plan: + actions: set[Action] + steps: list[Step] = field(default_factory=list) + skipped_steps: list[Step] = field(default_factory=list) + + def extend(self, other: "Plan") -> None: + # For now I don't see a usecase for merging plans with action sets. + assert self.actions == other.actions + + self.steps.extend(other.steps) + self.skipped_steps.extend(other.skipped_steps) + + def append(self, step: Step) -> None: + if step.is_empty(): + return + + if step.action in self.actions: + self.steps.append(step) + else: + self.skipped_steps.append(step) diff --git a/src/disko_lib/ansi.py b/src/disko_lib/ansi.py new file mode 100644 index 00000000..7d0c5774 --- /dev/null +++ b/src/disko_lib/ansi.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +# ANSI escape sequences for coloring and formatting text +# Inspired by rene-d's colors.py, published in 2018 +# See https://gist.github.com/rene-d/9e584a7dd2935d0f461904b9f2950007 + +import sys + + +class Colors: + """ + ANSI escape sequences + These constants were generated using nushell with the following command: + + nix run nixpkgs#nushell -- -c ' + ansi -l + | where name !~ "^xterm" + | each { |line| + $'"'"'($line.name | str upcase) = "($line.code | str replace "\\e" "\\033")"'"'"' + } + | print -r' + + I then removed many useless codes that are related to cursor movement, clearing the screen, etc. + """ + + GREEN = "\033[32m" + GREEN_BOLD = "\033[1;32m" + GREEN_ITALIC = "\033[3;32m" + GREEN_DIMMED = "\033[2;32m" + GREEN_REVERSE = "\033[7;32m" + BG_GREEN = "\033[42m" + LIGHT_GREEN = "\033[92m" + LIGHT_GREEN_BOLD = "\033[1;92m" + LIGHT_GREEN_UNDERLINE = "\033[4;92m" + LIGHT_GREEN_ITALIC = "\033[3;92m" + LIGHT_GREEN_DIMMED = "\033[2;92m" + LIGHT_GREEN_REVERSE = "\033[7;92m" + BG_LIGHT_GREEN = "\033[102m" + RED = "\033[31m" + RED_BOLD = "\033[1;31m" + RED_UNDERLINE = "\033[4;31m" + RED_ITALIC = "\033[3;31m" + RED_DIMMED = "\033[2;31m" + RED_REVERSE = "\033[7;31m" + BG_RED = "\033[41m" + LIGHT_RED = "\033[91m" + LIGHT_RED_BOLD = "\033[1;91m" + LIGHT_RED_UNDERLINE = "\033[4;91m" + LIGHT_RED_ITALIC = "\033[3;91m" + LIGHT_RED_DIMMED = "\033[2;91m" + LIGHT_RED_REVERSE = "\033[7;91m" + BG_LIGHT_RED = "\033[101m" + BLUE = "\033[34m" + BLUE_BOLD = "\033[1;34m" + BLUE_UNDERLINE = "\033[4;34m" + BLUE_ITALIC = "\033[3;34m" + BLUE_DIMMED = "\033[2;34m" + BLUE_REVERSE = "\033[7;34m" + BG_BLUE = "\033[44m" + LIGHT_BLUE = "\033[94m" + LIGHT_BLUE_BOLD = "\033[1;94m" + LIGHT_BLUE_UNDERLINE = "\033[4;94m" + LIGHT_BLUE_ITALIC = "\033[3;94m" + LIGHT_BLUE_DIMMED = "\033[2;94m" + LIGHT_BLUE_REVERSE = "\033[7;94m" + BG_LIGHT_BLUE = "\033[104m" + BLACK = "\033[30m" + BLACK_BOLD = "\033[1;30m" + BLACK_UNDERLINE = "\033[4;30m" + BLACK_ITALIC = "\033[3;30m" + BLACK_DIMMED = "\033[2;30m" + BLACK_REVERSE = "\033[7;30m" + BG_BLACK = "\033[40m" + LIGHT_GRAY = "\033[97m" + LIGHT_GRAY_BOLD = "\033[1;97m" + LIGHT_GRAY_UNDERLINE = "\033[4;97m" + LIGHT_GRAY_ITALIC = "\033[3;97m" + LIGHT_GRAY_DIMMED = "\033[2;97m" + LIGHT_GRAY_REVERSE = "\033[7;97m" + BG_LIGHT_GRAY = "\033[107m" + YELLOW = "\033[33m" + YELLOW_BOLD = "\033[1;33m" + YELLOW_UNDERLINE = "\033[4;33m" + YELLOW_ITALIC = "\033[3;33m" + YELLOW_DIMMED = "\033[2;33m" + YELLOW_REVERSE = "\033[7;33m" + BG_YELLOW = "\033[43m" + LIGHT_YELLOW = "\033[93m" + LIGHT_YELLOW_BOLD = "\033[1;93m" + LIGHT_YELLOW_UNDERLINE = "\033[4;93m" + LIGHT_YELLOW_ITALIC = "\033[3;93m" + LIGHT_YELLOW_DIMMED = "\033[2;93m" + LIGHT_YELLOW_REVERSE = "\033[7;93m" + BG_LIGHT_YELLOW = "\033[103m" + PURPLE = "\033[35m" + PURPLE_BOLD = "\033[1;35m" + PURPLE_UNDERLINE = "\033[4;35m" + PURPLE_ITALIC = "\033[3;35m" + PURPLE_DIMMED = "\033[2;35m" + PURPLE_REVERSE = "\033[7;35m" + BG_PURPLE = "\033[45m" + LIGHT_PURPLE = "\033[95m" + LIGHT_PURPLE_BOLD = "\033[1;95m" + LIGHT_PURPLE_UNDERLINE = "\033[4;95m" + LIGHT_PURPLE_ITALIC = "\033[3;95m" + LIGHT_PURPLE_DIMMED = "\033[2;95m" + LIGHT_PURPLE_REVERSE = "\033[7;95m" + BG_LIGHT_PURPLE = "\033[105m" + MAGENTA = "\033[35m" + MAGENTA_BOLD = "\033[1;35m" + MAGENTA_UNDERLINE = "\033[4;35m" + MAGENTA_ITALIC = "\033[3;35m" + MAGENTA_DIMMED = "\033[2;35m" + MAGENTA_REVERSE = "\033[7;35m" + BG_MAGENTA = "\033[45m" + LIGHT_MAGENTA = "\033[95m" + LIGHT_MAGENTA_BOLD = "\033[1;95m" + LIGHT_MAGENTA_UNDERLINE = "\033[4;95m" + LIGHT_MAGENTA_ITALIC = "\033[3;95m" + LIGHT_MAGENTA_DIMMED = "\033[2;95m" + LIGHT_MAGENTA_REVERSE = "\033[7;95m" + BG_LIGHT_MAGENTA = "\033[105m" + CYAN = "\033[36m" + CYAN_BOLD = "\033[1;36m" + CYAN_UNDERLINE = "\033[4;36m" + CYAN_ITALIC = "\033[3;36m" + CYAN_DIMMED = "\033[2;36m" + CYAN_REVERSE = "\033[7;36m" + BG_CYAN = "\033[46m" + LIGHT_CYAN = "\033[96m" + LIGHT_CYAN_BOLD = "\033[1;96m" + LIGHT_CYAN_UNDERLINE = "\033[4;96m" + LIGHT_CYAN_ITALIC = "\033[3;96m" + LIGHT_CYAN_DIMMED = "\033[2;96m" + LIGHT_CYAN_REVERSE = "\033[7;96m" + BG_LIGHT_CYAN = "\033[106m" + WHITE = "\033[37m" + WHITE_BOLD = "\033[1;37m" + WHITE_UNDERLINE = "\033[4;37m" + WHITE_ITALIC = "\033[3;37m" + WHITE_DIMMED = "\033[2;37m" + WHITE_REVERSE = "\033[7;37m" + BG_WHITE = "\033[47m" + DARK_GRAY = "\033[90m" + DARK_GRAY_BOLD = "\033[1;90m" + DARK_GRAY_UNDERLINE = "\033[4;90m" + DARK_GRAY_ITALIC = "\033[3;90m" + DARK_GRAY_DIMMED = "\033[2;90m" + DARK_GRAY_REVERSE = "\033[7;90m" + BG_DARK_GRAY = "\033[100m" + DEFAULT = "\033[39m" + DEFAULT_BOLD = "\033[1;39m" + DEFAULT_UNDERLINE = "\033[4;39m" + DEFAULT_ITALIC = "\033[3;39m" + DEFAULT_DIMMED = "\033[2;39m" + DEFAULT_REVERSE = "\033[7;39m" + BG_DEFAULT = "\033[49m" + RESET = "\033[0m" + ATTR_NORMAL = "\033[0m" + ATTR_BOLD = "\033[1m" + ATTR_DIMMED = "\033[2m" + ATTR_ITALIC = "\033[3m" + ATTR_UNDERLINE = "\033[4m" + ATTR_BLINK = "\033[5m" + ATTR_HIDDEN = "\033[8m" + ATTR_STRIKE = "\033[9m" + + # cancel SGR codes if we don't write to a terminal + if not sys.stdout.isatty(): + for _ in dir(): + if isinstance(_, str) and _[0] != "_": + locals()[_] = "" # type: ignore[misc] + else: + import platform + + # set Windows console in VT mode + if platform.system() == "Windows": + import ctypes + + kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined, misc] + kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7) # type: ignore[misc] + del kernel32 diff --git a/src/disko_lib/config_type.py b/src/disko_lib/config_type.py new file mode 100644 index 00000000..11461bc1 --- /dev/null +++ b/src/disko_lib/config_type.py @@ -0,0 +1,233 @@ +# File generated by scripts/generate_python_types.py +# Ignore warnings that decorators contain Any +# mypy: disable-error-code="misc" +# Disable auto-formatting for this file +# fmt: off +from typing import Any, Literal, Union +from pydantic import BaseModel, Field + + + + +class btrfs_subvolumes_swap(BaseModel): + options: list[str] + path: str + priority: None | int + size: str + + +class btrfs_subvolumes(BaseModel): + extraArgs: list[str] + mountOptions: list[str] + mountpoint: None | str + name: str + swap: dict[str, btrfs_subvolumes_swap] + type: Literal['btrfs_subvol'] + + +class btrfs_swap(BaseModel): + options: list[str] + path: str + priority: None | int + size: str + + +class btrfs(BaseModel): + device: str + extraArgs: list[str] + mountOptions: list[str] + mountpoint: None | str + subvolumes: dict[str, btrfs_subvolumes] + swap: dict[str, btrfs_swap] + type: Literal['btrfs'] + + + + +deviceType = None | Union["btrfs", "filesystem", "gpt", "luks", "lvm_pv", "mdraid", "swap", "table", "zfs"] + +class disk(BaseModel): + content: "deviceType" = Field(..., discriminator="type") + device: str + imageName: str + imageSize: str + name: str + type: Literal['disk'] + + +class filesystem(BaseModel): + device: str + extraArgs: list[str] + format: str + mountOptions: list[str] + mountpoint: None | str + type: Literal['filesystem'] + + + + +class gpt_partitions_hybrid(BaseModel): + mbrBootableFlag: bool + mbrPartitionType: None | str + + +class gpt_partitions(BaseModel): + index: int = Field(alias="_index") + alignment: int + content: "partitionType" = Field(..., discriminator="type") + device: str + end: str + hybrid: None | gpt_partitions_hybrid + label: str + name: str + priority: int + size: Union[Literal['100%'], str] + start: str + type: Union[str, str] + + +class gpt(BaseModel): + device: str + efiGptPartitionFirst: bool + partitions: dict[str, gpt_partitions] + type: Literal['gpt'] + + +class luks(BaseModel): + additionalKeyFiles: list[str] + askPassword: bool + content: "deviceType" = Field(..., discriminator="type") + device: str + extraFormatArgs: list[str] + extraOpenArgs: list[str] + initrdUnlock: bool + keyFile: None | str + name: str + passwordFile: None | str + settings: dict[str, Any] + type: Literal['luks'] + + +class lvm_pv(BaseModel): + device: str + type: Literal['lvm_pv'] + vg: str + + + + +class lvm_vg_lvs(BaseModel): + content: "partitionType" = Field(..., discriminator="type") + extraArgs: list[str] + lvm_type: None | Literal['mirror', 'raid0', 'raid1', 'raid4', 'raid5', 'raid6', 'thin-pool', 'thinlv'] + name: str + pool: None | str + priority: int + size: str + + +class lvm_vg(BaseModel): + lvs: dict[str, lvm_vg_lvs] + name: str + type: Literal['lvm_vg'] + + +class mdadm(BaseModel): + content: "deviceType" = Field(..., discriminator="type") + level: int + metadata: Literal['1', '1.0', '1.1', '1.2', 'default', 'ddf', 'imsm'] + name: str + type: Literal['mdadm'] + + +class mdraid(BaseModel): + device: str + name: str + type: Literal['mdraid'] + + +class nodev(BaseModel): + device: str + fsType: str + mountOptions: list[str] + mountpoint: None | str + type: Literal['nodev'] + + + + +partitionType = None | Union["btrfs", "filesystem", "luks", "lvm_pv", "mdraid", "swap", "zfs"] + +class swap(BaseModel): + device: str + discardPolicy: None | Literal['once', 'pages', 'both'] + extraArgs: list[str] + mountOptions: list[str] + priority: None | int + randomEncryption: bool + resumeDevice: bool + type: Literal['swap'] + + + + +class table_partitions(BaseModel): + index: int = Field(alias="_index") + bootable: bool + content: "partitionType" = Field(..., discriminator="type") + end: str + flags: list[str] + fs_type: None | Literal['btrfs', 'ext2', 'ext3', 'ext4', 'fat16', 'fat32', 'hfs', 'hfs+', 'linux-swap', 'ntfs', 'reiserfs', 'udf', 'xfs'] + name: None | str + part_type: Literal['primary', 'logical', 'extended'] + start: str + + +class table(BaseModel): + device: str + format: Literal['gpt', 'msdos'] + partitions: list[table_partitions] + type: Literal['table'] + + +class zfs(BaseModel): + device: str + pool: str + type: Literal['zfs'] + + +class zfs_fs(BaseModel): + mountOptions: list[str] + mountpoint: None | str + name: str + options: dict[str, str] + type: Literal['zfs_fs'] + + +class zfs_volume(BaseModel): + content: "partitionType" = Field(..., discriminator="type") + mountOptions: list[str] + name: str + options: dict[str, str] + size: None | str + type: Literal['zfs_volume'] + + +class zpool(BaseModel): + datasets: dict[str, Union["zfs_fs", "zfs_volume"]] + mode: Union[Literal['', 'mirror', 'raidz', 'raidz1', 'raidz2', 'raidz3'], dict[str, Any]] + mountOptions: list[str] + mountpoint: None | str + name: str + options: dict[str, str] + rootFsOptions: dict[str, str] + type: Literal['zpool'] + + + +class DiskoConfig(BaseModel): + disk: dict[str, disk] + lvm_vg: dict[str, lvm_vg] + mdadm: dict[str, mdadm] + nodev: dict[str, nodev] + zpool: dict[str, zpool] diff --git a/lib/default.nix b/src/disko_lib/default.nix similarity index 80% rename from lib/default.nix rename to src/disko_lib/default.nix index d19acac6..87fcba43 100644 --- a/lib/default.nix +++ b/src/disko_lib/default.nix @@ -4,7 +4,6 @@ , eval-config ? import }: let - outputs = import ../default.nix { inherit lib diskoLib; }; diskoLib = { testLib = import ./tests.nix { inherit lib makeTest eval-config; }; # like lib.types.oneOf but instead of a list takes an attrset @@ -23,9 +22,10 @@ let }; # option for valid contents of partitions (basically like devices, but without tables) + _partitionTypes = { inherit (diskoLib.types) btrfs filesystem zfs mdraid luks lvm_pv swap; }; partitionType = extraArgs: lib.mkOption { type = lib.types.nullOr (diskoLib.subType { - types = { inherit (diskoLib.types) btrfs filesystem zfs mdraid luks lvm_pv swap; }; + types = diskoLib._partitionTypes; inherit extraArgs; }); default = null; @@ -33,9 +33,10 @@ let }; # option for valid contents of devices + _deviceTypes = { inherit (diskoLib.types) table gpt btrfs filesystem zfs mdraid luks lvm_pv swap; }; deviceType = extraArgs: lib.mkOption { type = lib.types.nullOr (diskoLib.subType { - types = { inherit (diskoLib.types) table gpt btrfs filesystem zfs mdraid luks lvm_pv swap; }; + types = diskoLib._deviceTypes; inherit extraArgs; }); default = null; @@ -79,7 +80,7 @@ let then "${dev}p${toString index}" # /dev/mapper/vg-lv1 style else abort '' - ${dev} seems not to be a supported disk format. Please add this to disko in https://github.com/nix-community/disko/blob/master/lib/default.nix + ${dev} seems not to be a supported disk format. Please add this to disko in https://github.com/nix-community/disko/blob/master/src/disko_lib/default.nix ''; /* Escape a string as required to be used in udev symlinks @@ -213,11 +214,11 @@ let isAttrsOfSubmodule = o: o.type.name == "attrsOf" && o.type.nestedTypes.elemType.name == "submodule"; isSerializable = n: o: !( lib.hasPrefix "_" n - || lib.hasSuffix "Hook" n - || isAttrsOfSubmodule o - # TODO don't hardcode diskoLib.subType options. - || n == "content" || n == "partitions" || n == "datasets" || n == "swap" - || n == "mode" + || lib.hasSuffix "Hook" n + || isAttrsOfSubmodule o + # TODO don't hardcode diskoLib.subType options. + || n == "content" || n == "partitions" || n == "datasets" || n == "swap" + || n == "mode" ); in lib.toShellVars @@ -297,6 +298,23 @@ let }; + /* Evaluate a disko configuration + + eval :: lib.types.devices -> AttrSet + */ + eval-disko = cfg: lib.evalModules { + modules = lib.singleton { + # _file = toString input; + imports = lib.singleton { disko.devices = cfg.disko.devices; }; + options = { + disko.devices = lib.mkOption { + type = diskoLib.toplevel; + }; + }; + }; + }; + + /* Takes a disko device specification, returns an attrset with metadata meta :: lib.types.devices -> AttrSet @@ -592,52 +610,132 @@ let typesSerializerLib = { rootMountPoint = ""; options = null; - config._module.args.name = "self.name"; - lib = { + config = { + _module = { + args.name = ""; + args._parent.name = ""; + args._parent.type = ""; + }; + name = ""; + }; + parent = { }; + device = "/dev/"; + # Spoof part of nixpkgs/lib to analyze the types + lib = lib // { mkOption = option: { - inherit (option) type description; - default = option.default or null; + inherit (option) type; + description = option.description or null; + default = option.defaultText or option.default or null; }; types = { attrsOf = subType: { type = "attrsOf"; inherit subType; + "__isCompositeType" = true; }; listOf = subType: { type = "listOf"; inherit subType; + "__isCompositeType" = true; }; nullOr = subType: { type = "nullOr"; inherit subType; + "__isCompositeType" = true; + }; + oneOf = types: { + type = "oneOf"; + inherit types; + "__isCompositeType" = true; + }; + either = t1: t2: { + type = "oneOf"; + types = [ t1 t2 ]; + "__isCompositeType" = true; }; enum = choices: { type = "enum"; inherit choices; + "__isCompositeType" = true; }; + anything = "anything"; + nonEmptyStr = "str"; + strMatching = _: "str"; str = "str"; bool = "bool"; int = "int"; - submodule = x: x { inherit (diskoLib.typesSerializerLib) lib config options; }; + submodule = x: (x { + inherit (diskoLib.typesSerializerLib) lib config options; + name = ""; + }).options; }; }; diskoLib = { optionTypes.absolute-pathname = "absolute-pathname"; - deviceType = "devicetype"; - partitionType = "partitiontype"; - subType = types: "onOf ${toString (lib.attrNames types)}"; + # Spoof these types to avoid infinite recursion + deviceType = _: "deviceType"; + partitionType = _: "partitionType"; + subType = { types, ... }: { + type = "oneOf"; + types = lib.attrNames types; + "__isCompositeType" = true; + }; + mkCreateOption = option: "_create"; }; }; - jsonTypes = lib.listToAttrs ( - map - (file: lib.nameValuePair - (lib.removeSuffix ".nix" file) - (diskoLib.serializeType (import ./types/${file} diskoLib.typesSerializerLib)) - ) - (lib.attrNames (builtins.readDir ./types)) + jsonTypes = lib.listToAttrs + ( + map + (file: lib.nameValuePair + (lib.removeSuffix ".nix" file) + (diskoLib.serializeType (import ./types/${file} diskoLib.typesSerializerLib)) + ) + (lib.filter (name: lib.hasSuffix ".nix" name) (lib.attrNames (builtins.readDir ./types))) + ) // ( + let types = diskoLib.typesSerializerLib.lib.types; in { + partitionType = types.nullOr (types.oneOf (lib.attrNames diskoLib._partitionTypes)); + deviceType = types.nullOr (types.oneOf (lib.attrNames diskoLib._deviceTypes)); + } ); - - } // outputs; + }; + + outputs = + { lib ? import + , rootMountPoint ? "/mnt" + , checked ? false + , diskoLib ? import ./. { inherit lib rootMountPoint; } + }: + let + eval = diskoLib.eval-disko; + in + { + # legacy alias + create = cfg: builtins.trace "the create output is deprecated, use format instead" (eval cfg).config.disko.devices._create; + createScript = cfg: pkgs: builtins.trace "the create output is deprecated, use format instead" ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).formatScript; + createScriptNoDeps = cfg: pkgs: builtins.trace "the create output is deprecated, use format instead" ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).formatScriptNoDeps; + + format = cfg: (eval cfg).config.disko.devices._create; + formatScript = cfg: pkgs: ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).formatScript; + formatScriptNoDeps = cfg: pkgs: ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).formatScriptNoDeps; + + mount = cfg: (eval cfg).config.disko.devices._mount; + mountScript = cfg: pkgs: ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).mountScript; + mountScriptNoDeps = cfg: pkgs: ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).mountScriptNoDeps; + + disko = cfg: (eval cfg).config.disko.devices._disko; + diskoScript = cfg: pkgs: ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).diskoScript; + diskoScriptNoDeps = cfg: pkgs: ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).diskoScriptNoDeps; + + # we keep this old output for backwards compatibility + diskoNoDeps = cfg: pkgs: builtins.trace "the diskoNoDeps output is deprecated, please use disko instead" ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).diskoScriptNoDeps; + + config = cfg: (eval cfg).config.disko.devices._config; + packages = cfg: (eval cfg).config.disko.devices._packages; + }; in diskoLib +// (outputs { + inherit lib rootMountPoint; +}) # Compatibility alias + // { inherit outputs; } diff --git a/src/disko_lib/eval-config.nix b/src/disko_lib/eval-config.nix new file mode 100644 index 00000000..9f691f79 --- /dev/null +++ b/src/disko_lib/eval-config.nix @@ -0,0 +1,55 @@ +{ pkgs ? import { } +, lib ? pkgs.lib +, flake ? null +, flakeAttr ? null +, diskoFile ? null +, configJsonStr ? null +, rootMountPoint ? "/mnt" +, ... +}@args: +let + disko = import ./. { + inherit rootMountPoint; + inherit lib; + }; + + flake' = (builtins.getFlake flake); + + hasDiskoFile = diskoFile != null; + + hasFlakeDiskoConfig = lib.hasAttrByPath [ "diskoConfigurations" flakeAttr ] flake'; + + hasConfigStr = configJsonStr != null; + + hasFlakeDiskoModule = + lib.hasAttrByPath [ "nixosConfigurations" flakeAttr "config" "disko" "devices" ] flake'; + + diskFormat = + let + diskoConfig = + if hasDiskoFile then + import diskoFile + else if hasConfigStr then + (builtins.fromJSON configJsonStr) + else + flake'.diskoConfigurations.${flakeAttr}; + in + if builtins.isFunction diskoConfig then + diskoConfig ({ inherit lib; } // args) + else + diskoConfig; + + evaluatedConfig = + if hasDiskoFile || hasConfigStr || hasFlakeDiskoConfig then + disko.eval-disko diskFormat + else if (lib.traceValSeq hasFlakeDiskoModule) then + flake'.nixosConfigurations.${flakeAttr} + else + (builtins.abort "couldn't find `diskoConfigurations.${flakeAttr}` or `nixosConfigurations.${flakeAttr}.config.disko.devices`"); + + diskoConfig = evaluatedConfig.config.disko.devices; + + shouldBeEvaluated = name: (!lib.hasPrefix "_" name) || (name == "_index"); + finalConfig = lib.filterAttrsRecursive (name: value: shouldBeEvaluated name) diskoConfig; +in +finalConfig diff --git a/src/disko_lib/eval_config.py b/src/disko_lib/eval_config.py new file mode 100644 index 00000000..67d91060 --- /dev/null +++ b/src/disko_lib/eval_config.py @@ -0,0 +1,127 @@ +import json +from pathlib import Path +import re +from typing import Any, cast + +from pydantic import ValidationError + +from disko_lib.config_type import DiskoConfig +from disko_lib.messages.bugs import bug_validate_config_failed + +from .json_types import JsonDict + +from disko_lib.messages.msgs import ( + err_eval_config_failed, + err_file_not_found, + err_flake_uri_no_attr, + err_missing_arguments, + err_too_many_arguments, +) + +from .run_cmd import run +from .result import DiskoError, DiskoResult, DiskoSuccess + +NIX_BASE_CMD = [ + "nix", + "--extra-experimental-features", + "nix-command", + "--extra-experimental-features", + "flakes", +] + +NIX_EVAL_EXPR_CMD = NIX_BASE_CMD + ["eval", "--impure", "--json", "--expr"] + +EVAL_CONFIG_NIX = Path(__file__).absolute().parent / "eval-config.nix" +assert ( + EVAL_CONFIG_NIX.exists() +), f"Can't find `eval-config.nix`, expected it next to {__file__}" + + +def _eval_config(args: dict[str, str]) -> DiskoResult[JsonDict]: + args_as_json = json.dumps(args) + + result = run( + NIX_EVAL_EXPR_CMD + + [f"import {EVAL_CONFIG_NIX} (builtins.fromJSON ''{args_as_json}'')"] + ) + + if isinstance(result, DiskoError): + return DiskoError.single_message( + err_eval_config_failed, + "evaluate disko configuration", + args=args, + stderr=cast(str, result.messages[0].details["stderr"]), + ) + + # We trust the output of `nix eval` to be valid JSON + return DiskoSuccess( + cast(JsonDict, json.loads(result.value)), "evaluate disko config" + ) + + +def _eval_disko_file(config_file: Path) -> DiskoResult[JsonDict]: + abs_path = config_file.absolute() + + if not abs_path.exists(): + return DiskoError.single_message( + err_file_not_found, + "evaluate disko_file", + path=abs_path, + ) + + return _eval_config({"diskoFile": str(abs_path)}) + + +def _eval_flake(flake_uri: str) -> DiskoResult[JsonDict]: + # arg parser should not allow empty strings + assert len(flake_uri) > 0 + + flake_match = re.match(r"^([^#]+)(?:#(.*))?$", flake_uri) + + # Match can't be none if we receive at least one character + assert flake_match is not None + flake = cast(str, flake_match.group(1)) + flake_attr = cast(str, flake_match.group(2)) + + if not flake_attr: + return DiskoError.single_message( + err_flake_uri_no_attr, "evaluate flake", flake_uri=flake_uri + ) + + flake_path = Path(flake) + if flake_path.exists(): + flake = str(flake_path.absolute()) + + return _eval_config({"flake": flake, "flakeAttr": flake_attr}) + + +def eval_config_file_as_json( + *, disko_file: str | None, flake: str | None +) -> DiskoResult[JsonDict]: + # match would be nicer, but mypy doesn't understand type narrowing in tuples + if not disko_file and not flake: + return DiskoError.single_message(err_missing_arguments, "validate args") + if not disko_file and flake: + return _eval_flake(flake) + if disko_file and not flake: + return _eval_disko_file(Path(disko_file)) + + return DiskoError.single_message(err_too_many_arguments, "validate args") + + +def eval_config_dict_as_json(config: JsonDict) -> DiskoResult[JsonDict]: + return _eval_config({"configJsonStr": json.dumps(config)}) + + +def validate_config(json_config: JsonDict) -> DiskoResult[DiskoConfig]: + try: + result = DiskoConfig(**cast(dict[str, Any], json_config)) # type: ignore[misc] + except ValidationError as e: + return DiskoError.single_message( + bug_validate_config_failed, + "validate disko config", + error=e, + config=json_config, + ) + + return DiskoSuccess(result, "validate disko config") diff --git a/src/disko_lib/generate_config.py b/src/disko_lib/generate_config.py new file mode 100644 index 00000000..9f7de79d --- /dev/null +++ b/src/disko_lib/generate_config.py @@ -0,0 +1,58 @@ +from disko_lib.logging import DiskoMessage +from disko_lib.messages.msgs import ( + help_generate_partial_failure, +) +from disko_lib.result import DiskoError, DiskoPartialSuccess, DiskoResult, DiskoSuccess +from disko_lib.types.device import BlockDevice +from disko_lib.json_types import JsonDict +import disko_lib.types.disk as disk + + +def generate_config(devices: list[BlockDevice] = []) -> DiskoResult[JsonDict]: + error = DiskoError([], "generate disko config") + + config: dict[str, JsonDict] = { + "disk": {}, + "lvm_vg": {}, + "mdadm": {}, + "nodev": {}, + "zpool": {}, + } + successful_sections = [] + failed_sections = [] + + disk_config = disk.generate_config(devices) + if isinstance(disk_config, DiskoSuccess): + config["disk"] = disk_config.value + successful_sections.append("disk") + else: + error.extend(disk_config) + failed_sections.append("disk") + if isinstance(disk_config, DiskoPartialSuccess): + config["disk"] = disk_config.value + successful_sections.append("disk") + + # TODO: Add generation for ZFS, MDADM, LVM, etc. + successful_sections.append("lvm_vg") + successful_sections.append("mdadm") + successful_sections.append("nodev") + successful_sections.append("zpool") + + final_config: JsonDict = {"disko": {"devices": config}} # type: ignore[dict-item] + + if not failed_sections: + return DiskoSuccess(final_config, "generate disko config") + + if not successful_sections: + return error + + error.append( + DiskoMessage( + help_generate_partial_failure, + partial_config=final_config, + successful=successful_sections, + failed=failed_sections, + ) + ) + + return error.to_partial_success(final_config) diff --git a/src/disko_lib/generate_plan.py b/src/disko_lib/generate_plan.py new file mode 100644 index 00000000..bb75cc6d --- /dev/null +++ b/src/disko_lib/generate_plan.py @@ -0,0 +1,11 @@ +from disko_lib.action import Action, Plan +from disko_lib.config_type import DiskoConfig +from disko_lib.result import DiskoResult +import disko_lib.types.disk as disk + + +def generate_plan( + actions: set[Action], current_status: DiskoConfig, target_config: DiskoConfig +) -> DiskoResult[Plan]: + # TODO: Add generation for ZFS, MDADM, LVM, etc. + return disk.generate_plan(actions, current_status, target_config) diff --git a/lib/interactive-vm.nix b/src/disko_lib/interactive-vm.nix similarity index 100% rename from lib/interactive-vm.nix rename to src/disko_lib/interactive-vm.nix diff --git a/src/disko_lib/json_types.py b/src/disko_lib/json_types.py new file mode 100644 index 00000000..0e62f532 --- /dev/null +++ b/src/disko_lib/json_types.py @@ -0,0 +1,2 @@ +JsonDict = dict[str, "JsonValue"] +JsonValue = str | int | float | bool | None | list["JsonValue"] | JsonDict diff --git a/src/disko_lib/logging.py b/src/disko_lib/logging.py new file mode 100644 index 00000000..c2479a8f --- /dev/null +++ b/src/disko_lib/logging.py @@ -0,0 +1,158 @@ +# Logging functionality and global logging configuration +from dataclasses import dataclass +import logging +import re +from typing import ( + Any, + Callable, + Generic, + Literal, + ParamSpec, + TypeAlias, +) + +from .ansi import Colors + +logging.basicConfig(format="%(message)s", level=logging.INFO) +LOGGER = logging.getLogger("disko_logger") + +MessageTypes = Literal["bug", "error", "warning", "info", "help", "debug"] + +RESET = Colors.RESET +BG_COLOR_MAP = { + "bug": Colors.BG_RED, + "error": Colors.BG_RED, + "warning": Colors.BG_YELLOW, + "info": Colors.BG_GREEN, + "help": Colors.BG_LIGHT_MAGENTA, + "debug": Colors.BG_LIGHT_CYAN, +} +DECOR_COLOR_MAP = { + "bug": Colors.RED, + "error": Colors.RED, + "warning": Colors.YELLOW, + "info": Colors.GREEN, + "help": Colors.LIGHT_MAGENTA, + "debug": Colors.LIGHT_CYAN, +} +TITLE_RAW_MAP = { + "bug": "BUG", + "error": "ERROR", + "warning": "WARNING", + "info": "INFO", + "help": "HELP", + "debug": "DEBUG", +} +LOG_MSG_FUNCTION_MAP = { + "bug": LOGGER.error, + "error": LOGGER.error, + "warning": LOGGER.warning, + "info": LOGGER.info, + "help": LOGGER.info, + "debug": LOGGER.debug, +} + + +@dataclass +class ReadableMessage: + type: MessageTypes + msg: str + + +P = ParamSpec("P") + +MessageFactory: TypeAlias = Callable[P, ReadableMessage | list[ReadableMessage]] + + +@dataclass +class DiskoMessage(Generic[P]): + factory: MessageFactory[P] + # Can't infer a TypedDict from a ParamSpec yet (mypy 1.10.1, python 3.12.5) + details: dict[str, object] + + def __init__(self, factory: MessageFactory[P], **details: P.kwargs) -> None: + self.factory = factory + self.details = details + + def to_readable(self) -> list[ReadableMessage]: + # This is only safe because the type of __init__ ensures that the + # keys in details are the same as the keys in the factory kwargs + result = self.factory(**self.details) # type: ignore[arg-type] + if isinstance(result, list): + return result + return [result] + + def print(self) -> None: + for msg in self.to_readable(): + render_message(msg) + + def is_message(self, factory: MessageFactory[Any]) -> bool: + return self.factory == factory # type: ignore[misc] + + +# Dedent lines based on the indent of the first line until a non-indented line is hit. +# This will dedent the lines written in multiline f-strigns without breaking +# indentation for verbatim output that is inserted at the end +def dedent_start_lines(lines: list[str]) -> list[str]: + spaces_prefix_match = re.match(r"^( *)", lines[0]) + # Regex will even match an empty string, match can't be none + dedent_width = len(spaces_prefix_match.group(1)) # type: ignore[union-attr, misc] + + if dedent_width == 0: + return lines + + indent_string = " " * dedent_width + + dedented_lines = [] + stop_dedenting = False + for line in lines: + if stop_dedenting: + dedented_lines.append(line) + continue + + if not line.startswith(indent_string): + stop_dedenting = True + + dedented_line = re.sub(indent_string, "", line) + dedented_lines.append(dedented_line) + + return dedented_lines + + +def render_message(message: ReadableMessage) -> None: + bg_color = BG_COLOR_MAP[message.type] + decor_color = DECOR_COLOR_MAP[message.type] + title_raw = TITLE_RAW_MAP[message.type] + log_msg = LOG_MSG_FUNCTION_MAP[message.type] + + msg_lines = message.msg.strip("\n").rstrip(" \n").splitlines() + + # "WARNING:" is 8 characters long, center in 10 for space on each side + title = f"{bg_color}{title_raw + ":":^10}{RESET}" + + if len(msg_lines) == 1: + log_msg(f" {title} {msg_lines[0]}") + return + + msg_lines = dedent_start_lines(msg_lines) + + log_msg(f"{decor_color}╭─{title} {msg_lines[0]}") + + for line in msg_lines[1:]: + log_msg(f"{decor_color}│ {RESET} {line}") + + log_msg(f"{decor_color}╰───────────{RESET}") # Exactly as long as the heading + + +def debug(msg: str) -> None: + # Check debug level immediately to avoid unnecessary formatting + if LOGGER.isEnabledFor(logging.DEBUG): + render_message(ReadableMessage("debug", str(msg))) + + +# In general, only debug messages should be logged directly, all other +# messages should be wrapped in a DiskoResult for easier testing +# Info is exposed only for testing during initial development of disko2 +# TODO: Remove this function and use DiskoResult instead +def info(msg: str) -> None: + render_message(ReadableMessage("info", str(msg))) diff --git a/lib/make-disk-image.nix b/src/disko_lib/make-disk-image.nix similarity index 100% rename from lib/make-disk-image.nix rename to src/disko_lib/make-disk-image.nix diff --git a/src/disko_lib/messages/__init__.py b/src/disko_lib/messages/__init__.py new file mode 100644 index 00000000..f62e4017 --- /dev/null +++ b/src/disko_lib/messages/__init__.py @@ -0,0 +1,3 @@ +# Re-export all messages +from .msgs import * # noqa +from .bugs import * # noqa diff --git a/src/disko_lib/messages/bugs.py b/src/disko_lib/messages/bugs.py new file mode 100644 index 00000000..1d1d9bb8 --- /dev/null +++ b/src/disko_lib/messages/bugs.py @@ -0,0 +1,94 @@ +import json +import pydantic +from disko_lib.logging import ReadableMessage +from disko_lib.messages.colors import FILE, INVALID, RESET, VALUE +from ..json_types import JsonDict + + +def __bug_help_message(error_code: str) -> ReadableMessage: + return ReadableMessage( + "help", + f""" + Please report this bug! + First, check if has already been reported at + https://github.com/nix-community/disko/issues?q=is%3Aissue+{error_code} + If not, open a new issue at + https://github.com/nix-community/disko/issues/new?title={error_code} + and include the full logs printed above! + """, + ) + + +def bug_success_without_context(*, value: object) -> list[ReadableMessage]: + return [ + ReadableMessage( + "bug", + f""" + Success message without context! + Returned value: + {value} + """, + ), + __bug_help_message("bug_success_without_context"), + ] + + +def bug_validate_config_failed( + *, config: JsonDict, error: pydantic.ValidationError +) -> list[ReadableMessage]: + errors_printed = json.dumps(error.errors(), indent=2) # type: ignore[misc] + return [ + ReadableMessage( + "info", + f""" + Evaluated configuration: + {json.dumps(config, indent=2)} + """, + ), + ReadableMessage( + "error", + f""" + Validation errors: + {errors_printed} + """, + ), + ReadableMessage( + "bug", + f""" + Configuration validation failed! + Most likely, the types in python are out-of-sync with those in nix. + The {INVALID}validation errors{RESET} and the {VALUE}evaluated configuration{RESET} are printed above. + """, + ), + __bug_help_message("bug_validate_config_failed"), + ] + + +def bug_unsupported_device_content_type( + *, name: str, device: str, type: str +) -> list[ReadableMessage]: + return [ + ReadableMessage( + "bug", + f""" + Configuration for device {FILE}{device}{RESET} (name={VALUE}{name}{RESET}) specifies unsupported + device content type {INVALID}{type}{RESET}, which was not implemented yet! + """, + ), + __bug_help_message("err_unsupported_device_content_type"), + ] + + +def bug_unsupported_partition_content_type( + *, name: str, device: str, type: str +) -> list[ReadableMessage]: + return [ + ReadableMessage( + "bug", + f""" + Configuration for partition {FILE}{device}{RESET} (name={VALUE}{name}{RESET}) specifies unsupported + partition content type {INVALID}{type}{RESET}, which was not implemented yet! + """, + ), + __bug_help_message("bug_unsupported_partition_content_type"), + ] diff --git a/src/disko_lib/messages/colors.py b/src/disko_lib/messages/colors.py new file mode 100644 index 00000000..e8ea8b74 --- /dev/null +++ b/src/disko_lib/messages/colors.py @@ -0,0 +1,14 @@ +from disko_lib.ansi import Colors + + +# Color definitions. Note: Sort them alphabetically when adding new ones! +COMMAND = Colors.CYAN_ITALIC # Commands that were run or can be run +EM = Colors.WHITE_ITALIC # Emphasized text +EM_WARN = Colors.YELLOW_ITALIC # Emphasized text that is a warning +FILE = Colors.BLUE # File paths +FLAG = Colors.GREEN # Command line flags (like --version or -f) +INVALID = Colors.RED # Invalid values +PLACEHOLDER = Colors.MAGENTA_ITALIC # Values that need to be replaced +VALUE = Colors.GREEN # Values that are allowed + +RESET = Colors.RESET # Shortcut to reset the color diff --git a/src/disko_lib/messages/msgs.py b/src/disko_lib/messages/msgs.py new file mode 100644 index 00000000..77cbca97 --- /dev/null +++ b/src/disko_lib/messages/msgs.py @@ -0,0 +1,202 @@ +import json +from pathlib import Path +from disko_lib.logging import ReadableMessage +from .colors import PLACEHOLDER, RESET, FLAG, COMMAND, INVALID, FILE, VALUE, EM, EM_WARN +from ..json_types import JsonDict + +ERR_ARGUMENTS_HELP_TXT = f"Provide either {PLACEHOLDER}disko_file{RESET} as the second argument or \ +{FLAG}--flake{RESET}/{FLAG}-f{RESET} {PLACEHOLDER}flake-uri{RESET}." + + +def err_command_failed(*, command: str, exit_code: int, stderr: str) -> ReadableMessage: + return ReadableMessage( + "error", + f""" + Command failed: {COMMAND}{command}{RESET} + Exit code: {INVALID}{exit_code}{RESET} + stderr: {stderr} + """, + ) + + +def err_disk_type_changed_no_destroy( + *, disk: str, device: str, old_type: str, new_type: str +) -> list[ReadableMessage]: + return [ + ReadableMessage( + "error", + f""" + Disk {VALUE}{disk}{RESET} ({FILE}{device}{RESET}) changed type from {INVALID}{old_type}{RESET} to {INVALID}{new_type}{RESET}. + Need to destroy and recreate the disk, but the current mode does not allow it! + """, + ), + ReadableMessage( + "help", + f""" + Run `{COMMAND}disko{RESET} {VALUE}destroy,format,mount{RESET}` to allow destructive changes, + or change {VALUE}{disk}{RESET}'s type back to {INVALID}{old_type}{RESET} to keep the data. + """, + ), + ] + + +def err_disk_not_found(*, disk: str, device: str) -> ReadableMessage: + return ReadableMessage( + "error", + f"Device path {FILE}{device}{RESET} (for disk {VALUE}{disk}{RESET}) was not found!", + ) + + +def err_duplicated_disk_devices( + *, devices: list[str], duplicates: set[str] +) -> list[ReadableMessage]: + return [ + ReadableMessage( + "error", + f""" + Your config sets the same device path for multiple disks! + Devices: {", ".join(f"{VALUE}{d}{RESET}" for d in sorted(devices))} + """, + ), + ReadableMessage( + "help", + f"""The duplicates are: + {", ".join(f"{INVALID}{d}{RESET}" for d in duplicates)} + """, + ), + ] + + +def err_eval_config_failed(*, args: dict[str, str], stderr: str) -> ReadableMessage: + return ReadableMessage( + "error", + f""" + Failed to evaluate disko config with args {INVALID}{args}{RESET}! + Stderr from {COMMAND}nix eval{RESET}:\n{stderr} + """, + ) + + +def err_file_not_found(*, path: Path) -> ReadableMessage: + return ReadableMessage("error", f"File not found: {FILE}{path}{RESET}") + + +def err_filesystem_changed_no_destroy( + *, device: str, old_format: str, new_format: str +) -> list[ReadableMessage]: + return [ + ReadableMessage( + "error", + f""" + Filesystem on device {FILE}{device}{RESET} changed from {INVALID}{old_format}{RESET} to {INVALID}{new_format}{RESET}. + Need to destroy and recreate the filesystem, but the current mode does not allow it! + """, + ), + ReadableMessage( + "help", + f""" + Run `{COMMAND}disko{RESET} {VALUE}destroy,format,mount{RESET}` to allow destructive changes, + or change the filesystem back to {INVALID}{old_format}{RESET} to keep the data. + """, + ), + ] + + +def err_flake_uri_no_attr(*, flake_uri: str) -> list[ReadableMessage]: + return [ + ReadableMessage( + "error", + f"Flake URI {INVALID}{flake_uri}{RESET} has no attribute.", + ), + ReadableMessage( + "help", + f"Append an attribute like {VALUE}#{PLACEHOLDER}foo{RESET} to the flake URI.", + ), + ] + + +def err_missing_arguments() -> list[ReadableMessage]: + return [ + ReadableMessage( + "error", + "Missing arguments!", + ), + ReadableMessage("help", ERR_ARGUMENTS_HELP_TXT), + ] + + +def err_too_many_arguments() -> list[ReadableMessage]: + return [ + ReadableMessage( + "error", + "Too many arguments!", + ), + ReadableMessage("help", ERR_ARGUMENTS_HELP_TXT), + ] + + +def err_missing_mode(*, valid_modes: list[str]) -> list[ReadableMessage]: + modes_list = "\n".join([f" - {VALUE}{m}{RESET}" for m in valid_modes]) + return [ + ReadableMessage("error", "Missing mode!"), + ReadableMessage("help", "Allowed modes are:\n" + modes_list), + ] + + +def err_unsupported_pttype(*, device: Path, pttype: str) -> ReadableMessage: + return ReadableMessage( + "error", + f"Device {FILE}{device}{RESET} has unsupported partition type {INVALID}{pttype}{RESET}!", + ) + + +def warn_generate_partial_failure( + *, + kind: str, + failed: list[str], + successful: list[str], +) -> ReadableMessage: + partially_successful = [x for x in successful if x in failed] + failed = [x for x in failed if x not in partially_successful] + successful = [x for x in successful if x not in partially_successful] + return ReadableMessage( + "warning", + f""" + Successfully generated config for {EM}some{RESET} {kind}s of your setup, {EM_WARN}but not all{RESET}! + Failed {kind}s: {", ".join(f"{INVALID}{d}{RESET}" for d in failed)} + Successful {kind}s: {", ".join(f"{VALUE}{d}{RESET}" for d in successful)} + Partially successful {kind}s: {", ".join(f"{EM_WARN}{d}{RESET}" for d in partially_successful)} + """, + ) + + +def help_generate_partial_failure( + *, + partial_config: JsonDict, + failed: list[str], + successful: list[str], +) -> list[ReadableMessage]: + return [ + ReadableMessage( + "info", + f""" + Successfully generated config for {EM}some{RESET} devices. + Errors are printed above. The generated partial config is: + {json.dumps(partial_config, indent=2)} + """, + ), + warn_generate_partial_failure( + kind="section", + failed=failed, + successful=successful, + ), + ReadableMessage( + "help", + f""" + The {INVALID}ERROR{RESET} messages are printed {EM}above the generated config{RESET}. + Take a look at them and see if you can fix or safely ignore them. + If you can't, but you need a solution now, you can try to use the generated config, + but there is {EM_WARN}no guarantee{RESET} that it will work! + """, + ), + ] diff --git a/src/disko_lib/result.py b/src/disko_lib/result.py new file mode 100644 index 00000000..16577547 --- /dev/null +++ b/src/disko_lib/result.py @@ -0,0 +1,83 @@ +from dataclasses import dataclass +from typing import Any, Generic, ParamSpec, TypeVar, cast + +from disko_lib.messages.bugs import bug_success_without_context + +from .logging import ( + DiskoMessage, + ReadableMessage, + debug, + MessageFactory, + render_message, +) + +T = TypeVar("T", covariant=True) +S = TypeVar("S") +P = ParamSpec("P") + + +@dataclass +class DiskoSuccess(Generic[T]): + value: T + context: None | str = None + + +@dataclass +class DiskoError: + messages: list[DiskoMessage[object]] + context: str + + def __len__(self) -> int: + return len(self.messages) + + @classmethod + def single_message( + cls, factory: MessageFactory[P], context: str, *_: P.args, **details: P.kwargs + ) -> "DiskoError": + _factory = cast(MessageFactory[object], factory) + return cls([DiskoMessage(_factory, **details)], context) + + def find_message( + self, message_factory: MessageFactory[P] + ) -> None | DiskoMessage[P]: + for message in self.messages: + if message.factory == message_factory: + return cast(DiskoMessage[P], message) + return None + + def append(self, message: DiskoMessage[Any]) -> None: + self.messages.append(message) # type: ignore[misc] + + def extend(self, other_error: "DiskoError") -> None: + self.messages.extend(other_error.messages) + + def with_context(self, context: str) -> "DiskoError": + return DiskoError(self.messages, context) + + def to_partial_success(self, value: S) -> "DiskoPartialSuccess[S]": + return DiskoPartialSuccess(self.messages, self.context, value) + + +@dataclass +class DiskoPartialSuccess(Generic[T], DiskoError): + value: T + + +DiskoResult = DiskoSuccess[T] | DiskoPartialSuccess[T] | DiskoError # type: ignore[misc, unused-ignore] + + +def exit_on_error(result: DiskoResult[T]) -> T: + if isinstance(result, DiskoSuccess): + if result.context is None: + DiskoMessage(bug_success_without_context, value=result.value).print() + else: + debug(f"Success in '{result.context}'") + debug(f"Returned value: {result.value}") + return result.value + + render_message(ReadableMessage("error", f"Failed to {result.context}!")) + + for message in result.messages: + message.print() + + exit(1) diff --git a/src/disko_lib/run_cmd.py b/src/disko_lib/run_cmd.py new file mode 100644 index 00000000..c9eb90f5 --- /dev/null +++ b/src/disko_lib/run_cmd.py @@ -0,0 +1,33 @@ +import subprocess + +from disko_lib.messages.msgs import err_command_failed + +from .logging import debug +from .result import DiskoError, DiskoResult, DiskoSuccess + + +def run(args: list[str]) -> DiskoResult[str]: + command = " ".join(args) + debug(f"Running: {command}") + + result = subprocess.run(args, capture_output=True, text=True) + + debug( + f""" + Ran: {command} + Exit code: {result.returncode} + Stdout: {result.stdout} + Stderr: {result.stderr} + """ + ) + + if result.returncode == 0: + return DiskoSuccess(result.stdout, "run command") + + return DiskoError.single_message( + err_command_failed, + "run command", + command=command, + stderr=result.stderr, + exit_code=result.returncode, + ) diff --git a/lib/tests.nix b/src/disko_lib/tests.nix similarity index 98% rename from lib/tests.nix rename to src/disko_lib/tests.nix index 0aff4207..58501d41 100644 --- a/lib/tests.nix +++ b/src/disko_lib/tests.nix @@ -79,7 +79,7 @@ let # so /dev/vdb becomes /dev/vda etc. testConfigBooted = testLib.prepareDiskoConfig diskoConfigWithArgs testLib.devices; - tsp-generator = pkgs.callPackage ../. { checked = true; }; + tsp-generator = pkgs.callPackage ../../. { checked = true; }; tsp-format = (tsp-generator.formatScript testConfigInstall) pkgs; tsp-mount = (tsp-generator.mountScript testConfigInstall) pkgs; tsp-disko = (tsp-generator.diskoScript testConfigInstall) pkgs; @@ -92,7 +92,7 @@ let (lib.optionalAttrs (testMode == "module") { disko.enableConfig = true; imports = [ - ../module.nix + ../../module.nix testConfigBooted ]; }) @@ -168,7 +168,7 @@ let imports = [ (lib.optionalAttrs (testMode == "module") { imports = [ - ../module.nix + ../../module.nix ]; disko = { enableConfig = false; diff --git a/src/disko_lib/types/__init__.py b/src/disko_lib/types/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/lib/types/btrfs.nix b/src/disko_lib/types/btrfs.nix similarity index 100% rename from lib/types/btrfs.nix rename to src/disko_lib/types/btrfs.nix diff --git a/src/disko_lib/types/device.py b/src/disko_lib/types/device.py new file mode 100644 index 00000000..3e2ff76f --- /dev/null +++ b/src/disko_lib/types/device.py @@ -0,0 +1,146 @@ +from dataclasses import dataclass +import json +from pathlib import Path +from typing import cast + +from ..result import DiskoError, DiskoResult, DiskoSuccess +from ..run_cmd import run +from ..json_types import JsonDict + +# To see what other fields are available in the lsblk output and what +# sort of values you can expect from them, run: +# lsblk -O | less -S +LSBLK_OUTPUT_FIELDS = [ + "ID-LINK", + "FSTYPE", + "FSSIZE", + "FSUSE%", + "KNAME", + "LABEL", + "MODEL", + "PARTFLAGS", + "PARTLABEL", + "PARTN", + "PARTTYPE", + "PARTTYPENAME", + "PARTUUID", # The UUID used for /dev/disk/by-partuuid + "PATH", # The canonical path of the block device + "PHY-SEC", + "PTTYPE", + "REV", + "SERIAL", + "SIZE", + "START", + "MOUNTPOINT", # Canonical mountpoint + "MOUNTPOINTS", # All mountpoints, including e.g. bind mounts + "TYPE", + "UUID", # The UUID used for /dev/disk/by-uuid, if available +] + + +# Could think about splitting this into multiple classes based on the type field +# Would make access to the fields more type safe +@dataclass +class BlockDevice: + id_link: str + fstype: str + fssize: str + fsuse_pct: str + kname: str + label: str + model: str + partflags: str + partlabel: str + partn: int | None + parttype: str + parttypename: str + partuuid: str + path: Path + phy_sec: int + pttype: str + rev: str + serial: str + size: str + start: str + mountpoint: str | None + mountpoints: list[str] + type: str + uuid: str + children: list["BlockDevice"] + + @classmethod + def from_json_dict(cls, json_dict: JsonDict) -> "BlockDevice": + children_list = json_dict.get("children", []) + assert isinstance(children_list, list) + children = [] + for child_dict in children_list: + assert isinstance(child_dict, dict) + children.append(cls.from_json_dict(child_dict)) + + # The mountpoints field will be a list containing a single null if there are no mountpoints + mountpoints = cast(list[str], json_dict["mountpoints"]) or [] + if not any(mountpoints): + mountpoints = [] + + # When we request the output fields from lsblk, the keys are guaranteed to exists, + # but some might be null. Set a default value for the fields we have observed to be optional. + return cls( + children=children, + id_link=cast(str, json_dict["id-link"]), + fstype=cast(str, json_dict["fstype"]) or "", + fssize=cast(str, json_dict["fssize"]) or "", + fsuse_pct=cast(str, json_dict["fsuse%"]) or "", + kname=cast(str, json_dict["kname"]), + label=cast(str, json_dict["label"]) or "", + model=cast(str, json_dict["model"]) or "", + partflags=cast(str, json_dict["partflags"]) or "", + partlabel=cast(str, json_dict["partlabel"]) or "", + partn=cast(int, json_dict["partn"]), + parttype=cast(str, json_dict["parttype"]) or "", + parttypename=cast(str, json_dict["parttypename"]) or "", + partuuid=cast(str, json_dict["partuuid"]) or "", + path=Path(cast(str, json_dict["path"])), + phy_sec=cast(int, json_dict["phy-sec"]), + pttype=cast(str, json_dict["pttype"]), + rev=cast(str, json_dict["rev"]) or "", + serial=cast(str, json_dict["serial"]) or "", + size=str(cast(int, json_dict["size"])), + start=cast(str, json_dict["start"]) or "", + mountpoint=cast(str, json_dict["mountpoint"]) or None, + mountpoints=mountpoints, + type=cast(str, json_dict["type"]), + uuid=cast(str, json_dict["uuid"]) or "", + ) + + +def run_lsblk() -> DiskoResult[str]: + return run( + [ + "lsblk", + "--json", # JSON output + "--tree", # Tree structure with `children` field + # Show sizes in bytes. The human readable abbreviation can be less precise and harder to parse + "--bytes", + # Determine and output only the fields we are interested in + "--output", + ",".join(LSBLK_OUTPUT_FIELDS), + ] + ) + + +def list_block_devices(lsblk_output: str = "") -> DiskoResult[list[BlockDevice]]: + if not lsblk_output: + lsblk_result = run_lsblk() + + if isinstance(lsblk_result, DiskoError): + return lsblk_result + + lsblk_output = lsblk_result.value + + # We trust the output of `lsblk` to be valid JSON + output: JsonDict = json.loads(lsblk_output) + lsblk_json: list[JsonDict] = output["blockdevices"] # type: ignore[assignment] + + blockdevices = [BlockDevice.from_json_dict(dev) for dev in lsblk_json] + + return DiskoSuccess(blockdevices, "list block devices") diff --git a/lib/types/disk.nix b/src/disko_lib/types/disk.nix similarity index 100% rename from lib/types/disk.nix rename to src/disko_lib/types/disk.nix diff --git a/src/disko_lib/types/disk.py b/src/disko_lib/types/disk.py new file mode 100644 index 00000000..ea04d2fd --- /dev/null +++ b/src/disko_lib/types/disk.py @@ -0,0 +1,204 @@ +from typing import cast + +from disko_lib.action import Action, Plan, Step +from disko_lib.config_type import DiskoConfig, disk, gpt, deviceType +from disko_lib.messages.msgs import ( + err_disk_not_found, + err_duplicated_disk_devices, + err_unsupported_pttype, + warn_generate_partial_failure, +) +from disko_lib.messages.bugs import bug_unsupported_device_content_type +from disko_lib.utils import find_by_predicate, find_duplicates +import disko_lib.types.gpt + +from ..logging import DiskoMessage, debug +from ..result import DiskoError, DiskoResult, DiskoSuccess +from ..types.device import BlockDevice, list_block_devices +from ..json_types import JsonDict + + +def _generate_config_content(device: BlockDevice) -> DiskoResult[JsonDict]: + match device.pttype: + case "gpt": + return disko_lib.types.gpt.generate_config(device) + case _: + return DiskoError.single_message( + err_unsupported_pttype, + "generate disk config", + device=device.path, + pttype=device.pttype, + ) + + +def generate_config(devices: list[BlockDevice] = []) -> DiskoResult[JsonDict]: + block_devices = devices + if not block_devices: + lsblk_result = list_block_devices() + + if isinstance(lsblk_result, DiskoError): + return lsblk_result + + block_devices = lsblk_result.value + + debug(f"Generating config for devices {[d.path for d in block_devices]}") + + disks: JsonDict = {} + error = DiskoError([], "generate disk config") + failed_devices = [] + successful_devices = [] + + for device in block_devices: + content = _generate_config_content(device) + + if isinstance(content, DiskoError): + error.extend(content) + failed_devices.append(device.path) + continue + + disks[f"MODEL:{device.model},SN:{device.serial}"] = { + "device": f"/dev/{device.kname}", + "type": device.type, + "content": content.value, + } + successful_devices.append(device.path) + + if not failed_devices: + return DiskoSuccess(disks, "generate disk config") + + if not successful_devices: + return error + + error.append( + DiskoMessage( + warn_generate_partial_failure, + kind="disk", + failed=failed_devices, + successful=successful_devices, + ) + ) + return error.to_partial_success(disks) + + +def _generate_plan_content( + actions: set[Action], + name: str, + device: str, + current_content: deviceType, + target_content: deviceType, +) -> DiskoResult[Plan]: + if target_content is None: + debug(f"Element '{name}': No target content") + return DiskoSuccess(Plan(actions, []), f"generate '{name}' content plan") + + target_type = target_content.type + + if current_content is not None: + assert ( + current_content.type == target_type + ), "BUG! Device content type mismatch, should've been resolved earlier!" + + match target_type: + case "gpt": + return disko_lib.types.gpt.generate_plan( + actions, cast(gpt | None, current_content), cast(gpt, target_content) + ) + case _: + return DiskoError.single_message( + bug_unsupported_device_content_type, + "generate disk plan", + name=name, + device=device, + type=target_type, + ) + + +def generate_plan( + actions: set[Action], current_status: DiskoConfig, target_config: DiskoConfig +) -> DiskoResult[Plan]: + debug("Generating plan for disko config") + + error = DiskoError([], "generate disk plan") + plan = Plan(actions) + + current_disks = current_status.disk + target_disks = target_config.disk + + target_devices = [d.device for d in target_disks.values()] + + if duplicate_devices := find_duplicates(target_devices): + error.append( + DiskoMessage( + err_duplicated_disk_devices, + devices=target_devices, + duplicates=duplicate_devices, + ) + ) + + current_disks_by_target_name: dict[str, disk] = {} + + # Create plan for this disk + for name, target_disk_config in target_disks.items(): + device = target_disk_config.device + _, current_disk_config = find_by_predicate( + current_disks, lambda k, v: v.device == device + ) + disk_exists = current_disk_config is not None + current_type = current_disk_config.type if current_disk_config else None + target_type = target_disk_config.type + disk_has_same_type = current_type == target_type + + debug( + f"Disk '{name}': {device=}, {disk_exists=}, {disk_has_same_type=}, {current_type=}, {target_type=}" + ) + + # Can't use disk_exists here, mypy doesn't understand that it + # narrows the type of current_disk_config + if current_disk_config is None: + error.append( + DiskoMessage( + err_disk_not_found, + disk=name, + device=device, + ) + ) + continue + + current_disks_by_target_name[name] = current_disk_config + + if disk_has_same_type: + continue + + plan.append( + Step( + "destroy", + [[f"disk-deactivate {device}"]], + f"destroy partition table on `{name}`, at {device}", + ) + ) + + # Create content plan + for name, target_disk_config in target_disks.items(): + current_content = None + current_disk_config = current_disks_by_target_name.get(name) + if current_disk_config is not None: + current_content = current_disk_config.content + + result = _generate_plan_content( + actions, + name, + target_disk_config.device, + current_content, + target_disk_config.content, + ) + + if isinstance(result, DiskoError): + error.extend(result) + continue + + plan.extend(result.value) + + if error: + return error + + return DiskoSuccess(plan, "generate disk plan") diff --git a/lib/types/filesystem.nix b/src/disko_lib/types/filesystem.nix similarity index 100% rename from lib/types/filesystem.nix rename to src/disko_lib/types/filesystem.nix diff --git a/src/disko_lib/types/filesystem.py b/src/disko_lib/types/filesystem.py new file mode 100644 index 00000000..eea1be25 --- /dev/null +++ b/src/disko_lib/types/filesystem.py @@ -0,0 +1,76 @@ +from disko_lib.action import Action, Plan, Step +from disko_lib.config_type import filesystem +from disko_lib.logging import debug +from .device import BlockDevice +from ..result import DiskoResult, DiskoSuccess +from ..json_types import JsonDict + + +def generate_config(device: BlockDevice) -> DiskoResult[JsonDict]: + assert ( + device.type == "part" + ), f"BUG! filesystem.generate_config called with non-partition device {device.path}" + + return DiskoSuccess( + { + "type": "filesystem", + "format": device.fstype, + "mountpoint": device.mountpoint, + } + ) + + +def _generate_mount_step(target_config: filesystem) -> Step: + if target_config.mountpoint is None: + return Step.empty("mount") + + return Step( + "mount", + [ + # TODO: Only try to mount if the device is not already mounted + # This will probably require us to change the way we specify steps, + # as they currently don't allow for conditional execution + [ + "mount", + target_config.device, + target_config.mountpoint, + "-t", + target_config.format, + ] + + target_config.mountOptions + + ["-o", "X-mount.mkdir"] + ], + "mount filesystem", + ) + + +def generate_plan( + actions: set[Action], current_config: filesystem | None, target_config: filesystem +) -> DiskoResult[Plan]: + debug("Generating plan for filesystem") + + plan = Plan(actions, []) + + current_format = current_config.format if current_config is not None else None + target_format = target_config.format + device = target_config.device + need_to_destroy_current = ( + current_config is not None and current_format != target_format + ) + + debug( + f"Filesystem {device}: {current_format=}, {target_format=}, {need_to_destroy_current=}" + ) + + if need_to_destroy_current: + plan.append( + Step( + "destroy", + [[f"mkfs.{target_format}"] + target_config.extraArgs + [device]], + "destroy current filesystem and create new one", + ) + ) + + plan.append(_generate_mount_step(target_config)) + + return DiskoSuccess(plan, "generate filesystem plan") diff --git a/lib/types/gpt.nix b/src/disko_lib/types/gpt.nix similarity index 92% rename from lib/types/gpt.nix rename to src/disko_lib/types/gpt.nix index c52ef8c8..b547b4dd 100644 --- a/lib/types/gpt.nix +++ b/src/disko_lib/types/gpt.nix @@ -42,6 +42,13 @@ in "/dev/disk/by-id/md-name-any:${config._parent.name}-part${toString partition.config._index}" else "/dev/disk/by-partlabel/${diskoLib.hexEscapeUdevSymlink partition.config.label}"; + defaultText = '' + if the parent is an mdadm device: + /dev/disk/by-id/md-name-any:''${config._parent.name}-part''${toString partition.config._index} + + otherwise: + /dev/disk/by-partlabel/''${diskoLib.hexEscapeUdevSymlink partition.config.label} + ''; description = "Device to use for the partition"; }; priority = lib.mkOption { @@ -80,6 +87,11 @@ in builtins.substring 0 limit (builtins.hashString "sha256" label) else label; + defaultText = '' + ''${config._parent.type}-''${config._parent.name}-''${partition.config.name} + + or a truncated hash of the above if it is longer than 36 characters + ''; }; size = lib.mkOption { type = lib.types.either (lib.types.enum [ "100%" ]) (lib.types.strMatching "[0-9]+[KMGTP]?"); @@ -93,6 +105,7 @@ in alignment = lib.mkOption { type = lib.types.int; default = if (builtins.substring (builtins.stringLength partition.config.start - 1) 1 partition.config.start == "s" || (builtins.substring (builtins.stringLength partition.config.end - 1) 1 partition.config.end == "s")) then 1 else 0; + defaultText = "1 if the unit of start or end is sectors, 0 otherwise"; description = "Alignment of the partition, if sectors are used as start or end it can be aligned to 1"; }; start = lib.mkOption { @@ -103,6 +116,9 @@ in end = lib.mkOption { type = lib.types.str; default = if partition.config.size == "100%" then "-0" else "+${partition.config.size}"; + defaultText = '' + if partition.config.size == "100%" then "-0" else "+''${partition.config.size}"; + ''; description = '' End of the partition, in sgdisk format. Use + for relative sizes from the partitions start @@ -142,8 +158,10 @@ in description = "Entry to add to the Hybrid MBR table"; }; _index = lib.mkOption { + type = lib.types.int; internal = true; default = diskoLib.indexOf (x: x.name == partition.config.name) sortedPartitions 0; + defaultText = null; }; }; })); diff --git a/src/disko_lib/types/gpt.py b/src/disko_lib/types/gpt.py new file mode 100644 index 00000000..ce5a648d --- /dev/null +++ b/src/disko_lib/types/gpt.py @@ -0,0 +1,247 @@ +from typing import cast +from disko_lib.action import Action, Plan, Step +from disko_lib.config_type import gpt, gpt_partitions, partitionType, filesystem +from disko_lib.messages.bugs import bug_unsupported_partition_content_type +from disko_lib.utils import find_by_predicate +import disko_lib.types.filesystem +from ..logging import debug +from .device import BlockDevice +from ..result import DiskoError, DiskoResult, DiskoSuccess +from ..json_types import JsonDict + + +def _add_type_if_required(device: BlockDevice, part_config: JsonDict) -> JsonDict: + type = { + "c12a7328-f81f-11d2-ba4b-00a0c93ec93b": "EF00", # EFI System + "21686148-6449-6e6f-744e-656564454649": "EF02", # BIOS boot + }.get(device.parttype) + + if type: + part_config["type"] = type + + return part_config + + +def _generate_name(device: BlockDevice) -> str: + if device.uuid: + return f"UUID:{device.uuid}" + + return f"PARTUUID:{device.partuuid}" + + +def _generate_config_content(device: BlockDevice) -> DiskoResult[JsonDict]: + match device.fstype: + # TODO: Add filesystems that are not supported by `mkfs` here + case _: + return disko_lib.types.filesystem.generate_config(device) + + +def generate_config(device: BlockDevice) -> DiskoResult[JsonDict]: + assert ( + device.pttype == "gpt" + ), f"BUG! gpt.generate_config called with non-gpt device {device.path}" + + debug(f"Generating GPT config for device {device.path}") + + partitions: JsonDict = {} + error = DiskoError([], "generate gpt config") + failed_partitions = [] + successful_partitions = [] + + for index, partition in enumerate(device.children): + content = _generate_config_content(partition) + + if isinstance(content, DiskoError): + error.extend(content) + failed_partitions.append(partition.path) + continue + + partitions[_generate_name(partition)] = _add_type_if_required( + partition, + {"_index": index + 1, "size": partition.size, "content": content.value}, + ) + successful_partitions.append(partition.path) + + if not failed_partitions: + return DiskoSuccess( + {"type": "gpt", "partitions": partitions}, "generate gpt config" + ) + + return error + + +def _generate_plan_content( + actions: set[Action], + name: str, + device: str, + current_content: partitionType, + target_content: partitionType, +) -> DiskoResult[Plan]: + if target_content is None: + return DiskoSuccess(Plan(actions, []), f"generate '{name}' content plan") + + target_type = target_content.type + + if current_content is not None: + assert ( + current_content.type == target_type + ), "BUG! Partition content type mismatch, should've been resolved earlier!" + + match target_type: + case "filesystem": + return disko_lib.types.filesystem.generate_plan( + actions, + cast(filesystem | None, current_content), + cast(filesystem, target_content), + ) + case _: + return DiskoError.single_message( + bug_unsupported_partition_content_type, + "generate partition plan", + name=name, + device=device, + type=target_type, + ) + + +def _step_clear_partition_table(device: str) -> Step: + return Step( + "format", [["sgdisk", "--clear", device]], f"Clear partition table on {device}" + ) + + +def _partprobe_settle(device: str) -> list[list[str]]: + # ensure /dev/disk/by-path/..-partN exists before continuing + return [ + ["partprobe", device], + ["udevadm", "trigger", "--subsystem-match=block"], + ["udevadm", "settle"], + ] + + +def _sgdisk_create_args(partition_config: gpt_partitions) -> list[str]: + alignment = partition_config.alignment + index = partition_config.index + start = partition_config.start + end = partition_config.end + + alignment_args = [] if alignment == 0 else [f"--set-alignment={alignment}"] + + return [ + "--align-end", + *alignment_args, + f"--new={index}:{start}:{end}", + ] + + +def _sgdisk_modify_args(partition_config: gpt_partitions) -> list[str]: + index = partition_config.index + label = partition_config.label + type = partition_config.type + + return [ + f'--change-name="{index}:{label}"', + f"--typecode=${index}:{type}", + ] + + +def _step_modify_partition(device: str, partition_config: gpt_partitions) -> Step: + return Step( + "format", + [ + [ + "sgdisk", + *_sgdisk_modify_args(partition_config), + device, + ] + ] + + _partprobe_settle(device), + "Create partition {}", + ) + + +def _step_create_partition(device: str, partition_config: gpt_partitions) -> Step: + return Step( + "format", + [ + [ + "sgdisk", + *_sgdisk_create_args(partition_config), + *_sgdisk_modify_args(partition_config), + device, + ] + ] + + _partprobe_settle(device), + "Create partition {}", + ) + + +def generate_plan( + actions: set[Action], current_gpt_config: gpt | None, target_gpt_config: gpt +) -> DiskoResult[Plan]: + device = target_gpt_config.device + debug(f"Generating GPT plan for disk {device}") + + if current_gpt_config is None: + current_partitions = {} + else: + current_partitions = current_gpt_config.partitions + target_partitions = target_gpt_config.partitions + + error_messages = [] + plan = Plan(actions) + + if current_gpt_config is None: + plan.append(_step_clear_partition_table(device)) + + current_partitions_by_target_name: dict[str, gpt_partitions] = {} + + # Create or modify all partitions first + for name, target_partition in target_partitions.items(): + _, current_partition = find_by_predicate( + current_partitions, lambda k, v: v.index == target_partition.index + ) + + if not current_partition: + plan.append(_step_create_partition(device, target_partition)) + continue + + current_partitions_by_target_name[name] = current_partition + + if ( + current_partition.type == target_partition.type + and current_partition.label == target_partition.label + ): + debug(f"Partition {name} has no changes we could apply") + continue + + # TODO: Determine if something else about the disk changed. Add a warning message if that change + # can't be applied by disko automatically, (plus a help message that explains how to target just + # a single disk in case the user wants to make the change destructively) or add the + # necessary steps to apply the changes + if "format" not in actions: + continue + + plan.append(_step_modify_partition(device, target_partition)) + + # Then dispatch to all the filesystems + for name, target_partition in target_partitions.items(): + current_content = None + current_partition_config = current_partitions_by_target_name.get(name) + if current_partition_config is not None: + current_content = current_partition_config.content + + content_plan_result = _generate_plan_content( + actions, + name, + target_partition.device, + current_content, + target_partition.content, + ) + if isinstance(content_plan_result, DiskoError): + error_messages.append(content_plan_result.messages) + continue + + plan.extend(content_plan_result.value) + + return DiskoSuccess(plan, "generate gpt plan") diff --git a/lib/types/luks.nix b/src/disko_lib/types/luks.nix similarity index 98% rename from lib/types/luks.nix rename to src/disko_lib/types/luks.nix index 8056b35d..637dc3ef 100644 --- a/lib/types/luks.nix +++ b/src/disko_lib/types/luks.nix @@ -59,9 +59,11 @@ in askPassword = lib.mkOption { type = lib.types.bool; default = config.keyFile == null && config.passwordFile == null && (! config.settings ? "keyFile"); + defaultText = "true if neither keyFile nor passwordFile are set"; description = "Whether to ask for a password for initial encryption"; }; settings = lib.mkOption { + type = lib.types.attrsOf lib.types.anything; default = { }; description = "LUKS settings (as defined in configuration.nix in boot.initrd.luks.devices.)"; example = ''{ diff --git a/lib/types/lvm_pv.nix b/src/disko_lib/types/lvm_pv.nix similarity index 100% rename from lib/types/lvm_pv.nix rename to src/disko_lib/types/lvm_pv.nix diff --git a/lib/types/lvm_vg.nix b/src/disko_lib/types/lvm_vg.nix similarity index 100% rename from lib/types/lvm_vg.nix rename to src/disko_lib/types/lvm_vg.nix diff --git a/lib/types/mdadm.nix b/src/disko_lib/types/mdadm.nix similarity index 100% rename from lib/types/mdadm.nix rename to src/disko_lib/types/mdadm.nix diff --git a/lib/types/mdraid.nix b/src/disko_lib/types/mdraid.nix similarity index 100% rename from lib/types/mdraid.nix rename to src/disko_lib/types/mdraid.nix diff --git a/lib/types/nodev.nix b/src/disko_lib/types/nodev.nix similarity index 100% rename from lib/types/nodev.nix rename to src/disko_lib/types/nodev.nix diff --git a/lib/types/swap.nix b/src/disko_lib/types/swap.nix similarity index 100% rename from lib/types/swap.nix rename to src/disko_lib/types/swap.nix diff --git a/lib/types/table.nix b/src/disko_lib/types/table.nix similarity index 98% rename from lib/types/table.nix rename to src/disko_lib/types/table.nix index 8aec03d8..249a5265 100644 --- a/lib/types/table.nix +++ b/src/disko_lib/types/table.nix @@ -63,8 +63,10 @@ }; content = diskoLib.partitionType { parent = config; device = diskoLib.deviceNumbering config.device partition.config._index; }; _index = lib.mkOption { + type = lib.types.int; internal = true; default = lib.toInt (lib.head (builtins.match ".*entry ([[:digit:]]+)]" name)); + defaultText = null; }; }; })); diff --git a/lib/types/zfs.nix b/src/disko_lib/types/zfs.nix similarity index 100% rename from lib/types/zfs.nix rename to src/disko_lib/types/zfs.nix diff --git a/lib/types/zfs_fs.nix b/src/disko_lib/types/zfs_fs.nix similarity index 100% rename from lib/types/zfs_fs.nix rename to src/disko_lib/types/zfs_fs.nix diff --git a/lib/types/zfs_volume.nix b/src/disko_lib/types/zfs_volume.nix similarity index 100% rename from lib/types/zfs_volume.nix rename to src/disko_lib/types/zfs_volume.nix diff --git a/lib/types/zpool.nix b/src/disko_lib/types/zpool.nix similarity index 100% rename from lib/types/zpool.nix rename to src/disko_lib/types/zpool.nix diff --git a/src/disko_lib/utils.py b/src/disko_lib/utils.py new file mode 100644 index 00000000..11a20f36 --- /dev/null +++ b/src/disko_lib/utils.py @@ -0,0 +1,23 @@ +from typing import Iterable, TypeVar, Callable + +T = TypeVar("T") + + +def find_by_predicate( + dct: dict[str, T], predicate: Callable[[str, T], bool] +) -> tuple[str, T] | tuple[None, None]: + for k, v in dct.items(): + if predicate(k, v): + return k, v + return None, None + + +def find_duplicates(it: Iterable[T]) -> set[T]: + seen = set() + duplicates = set() + for item in it: + if item in seen: + duplicates.add(item) + else: + seen.add(item) + return duplicates diff --git a/src/experiments.ipynb b/src/experiments.ipynb new file mode 100644 index 00000000..0a938307 --- /dev/null +++ b/src/experiments.ipynb @@ -0,0 +1,91 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Experiments\n", + "\n", + "This notebook is very useful for debugging parts of disko without having to run\n", + "the entire CLI. You can import anything from `disko` or `disko_lib` and call\n", + "functions directly. With VSCode, you can also start a debugging session from\n", + "within a cell!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Run this cell to automatically reload modules on every execution.\n", + "# This avoids having to restart the kernel every time you make a change to a module.\n", + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Examples\n", + "\n", + "These are some examples how you can use this notebook. Feel free to copy it and\n", + "create your own experiments. If you used a notebook to fix some bug, please\n", + "copy the code into a regression test and run it with pytest!" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Errors are printed above. The generated partial config is:\n", + "{\n", + " \"disko\": {\n", + " \"devices\": {\n", + " \"disk\": {\n" + ] + } + ], + "source": [ + "from disko_lib.logging import dedent_start_lines\n", + "\n", + "lines = \"\"\"\n", + " Errors are printed above. The generated partial config is:\n", + " {\n", + " \"disko\": {\n", + " \"devices\": {\n", + " \"disk\": {\n", + "\"\"\"\n", + "\n", + "print(\"\\n\".join(dedent_start_lines(lines.splitlines()[1:])))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/install-cli.nix b/src/install-cli.nix similarity index 100% rename from install-cli.nix rename to src/install-cli.nix diff --git a/tests/cli.nix b/tests/cli.nix index 76cc2b1c..3f81ec64 100644 --- a/tests/cli.nix +++ b/tests/cli.nix @@ -1,5 +1,5 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; diff --git a/tests/default.nix b/tests/default.nix index cb9d47d3..61857c8c 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -4,17 +4,28 @@ }: let lib = pkgs.lib; - diskoLib = import ../lib { inherit lib makeTest eval-config; }; + fs = lib.fileset; + diskoLib = import ../src/disko_lib { inherit lib makeTest eval-config; }; + + incompatibleTests = lib.optionals pkgs.buildPlatform.isRiscV64 [ "zfs" "zfs-over-legacy" "cli" "module" "complex" ]; allTestFilenames = - builtins.map (lib.removeSuffix ".nix") ( - builtins.filter - (x: lib.hasSuffix ".nix" x && x != "default.nix") - (lib.attrNames (builtins.readDir ./.)) + (fs.toList + (fs.difference + (fs.fileFilter + ({ name, hasExt, ... }: + hasExt "nix" + && name != "default.nix" + && !(lib.elem (lib.removeSuffix ".nix" name) incompatibleTests)) + ./.) + (fs.fileFilter ({ ... }: true) ./disko-install)) ); - incompatibleTests = lib.optionals pkgs.buildPlatform.isRiscV64 [ "zfs" "zfs-over-legacy" "cli" "module" "complex" ]; - allCompatibleFilenames = lib.subtractLists incompatibleTests allTestFilenames; - allTests = lib.genAttrs allCompatibleFilenames (test: import (./. + "/${test}.nix") { inherit diskoLib pkgs; }); + allTests = lib.listToAttrs (lib.map + (test: { + name = lib.removeSuffix ".nix" (builtins.baseNameOf test); + value = import test { inherit diskoLib pkgs; }; + }) + allTestFilenames); in allTests diff --git a/tests/disko_lib/eval_config/README.md b/tests/disko_lib/eval_config/README.md new file mode 100644 index 00000000..02aa0306 --- /dev/null +++ b/tests/disko_lib/eval_config/README.md @@ -0,0 +1,17 @@ +# eval_config.py tests + +If you change something about the evaluation and need to update one of the +result files here, you can run a command like this: + + ./disko2 dev eval example/simple-efi.nix > tests/disko_lib/eval_config/file-simple-efi-result.json + +Change the paths depending on the example whose evaluation result changed. + +If you're thinking "this sounds like snapshots to me" and "isn't there a pytest plugin for this?", +then you'd be correct, but [pytest-insta](https://github.com/vberlier/pytest-insta) is not packaged +in nixpkgs at the time of writing (2024-11-08). + +If you're reading this, and you +[search nixpkgs for "pytest-insta"](https://search.nixos.org/packages?channel=unstable&query=pytest-insta) +AND this returns the `pytest-insta` package (or there is a new, better snapshotting plugin for pytest), +please open an issue so we can replace this manual process with it! \ No newline at end of file diff --git a/tests/disko_lib/eval_config/file-simple-efi-eval-result.json b/tests/disko_lib/eval_config/file-simple-efi-eval-result.json new file mode 100644 index 00000000..40ae4614 --- /dev/null +++ b/tests/disko_lib/eval_config/file-simple-efi-eval-result.json @@ -0,0 +1,84 @@ +{ + "disk": { + "main": { + "content": { + "device": "/dev/disk/by-id/some-disk-id", + "efiGptPartitionFirst": true, + "partitions": { + "ESP": { + "_index": 1, + "alignment": 0, + "content": { + "device": "/dev/disk/by-partlabel/disk-main-ESP", + "extraArgs": [], + "format": "vfat", + "mountOptions": [ + "umask=0077" + ], + "mountpoint": "/boot", + "postCreateHook": "", + "postMountHook": "", + "preCreateHook": "", + "preMountHook": "", + "type": "filesystem" + }, + "device": "/dev/disk/by-partlabel/disk-main-ESP", + "end": "+500M", + "hybrid": null, + "label": "disk-main-ESP", + "name": "ESP", + "priority": 1000, + "size": "500M", + "start": "0", + "type": "EF00" + }, + "root": { + "_index": 2, + "alignment": 0, + "content": { + "device": "/dev/disk/by-partlabel/disk-main-root", + "extraArgs": [], + "format": "ext4", + "mountOptions": [ + "defaults" + ], + "mountpoint": "/", + "postCreateHook": "", + "postMountHook": "", + "preCreateHook": "", + "preMountHook": "", + "type": "filesystem" + }, + "device": "/dev/disk/by-partlabel/disk-main-root", + "end": "-0", + "hybrid": null, + "label": "disk-main-root", + "name": "root", + "priority": 9001, + "size": "100%", + "start": "0", + "type": "8300" + } + }, + "postCreateHook": "", + "postMountHook": "", + "preCreateHook": "", + "preMountHook": "", + "type": "gpt" + }, + "device": "/dev/disk/by-id/some-disk-id", + "imageName": "main", + "imageSize": "2G", + "name": "main", + "postCreateHook": "", + "postMountHook": "", + "preCreateHook": "", + "preMountHook": "", + "type": "disk" + } + }, + "lvm_vg": {}, + "mdadm": {}, + "nodev": {}, + "zpool": {} +} diff --git a/tests/disko_lib/eval_config/file-simple-efi-validate-result.json b/tests/disko_lib/eval_config/file-simple-efi-validate-result.json new file mode 100644 index 00000000..d3e50160 --- /dev/null +++ b/tests/disko_lib/eval_config/file-simple-efi-validate-result.json @@ -0,0 +1,68 @@ +{ + "disk": { + "main": { + "content": { + "device": "/dev/disk/by-id/some-disk-id", + "efiGptPartitionFirst": true, + "partitions": { + "ESP": { + "_index": 1, + "alignment": 0, + "content": { + "device": "/dev/disk/by-partlabel/disk-main-ESP", + "extraArgs": [], + "format": "vfat", + "mountOptions": [ + "umask=0077" + ], + "mountpoint": "/boot", + "type": "filesystem" + }, + "device": "/dev/disk/by-partlabel/disk-main-ESP", + "end": "+500M", + "hybrid": null, + "label": "disk-main-ESP", + "name": "ESP", + "priority": 1000, + "size": "500M", + "start": "0", + "type": "EF00" + }, + "root": { + "_index": 2, + "alignment": 0, + "content": { + "device": "/dev/disk/by-partlabel/disk-main-root", + "extraArgs": [], + "format": "ext4", + "mountOptions": [ + "defaults" + ], + "mountpoint": "/", + "type": "filesystem" + }, + "device": "/dev/disk/by-partlabel/disk-main-root", + "end": "-0", + "hybrid": null, + "label": "disk-main-root", + "name": "root", + "priority": 9001, + "size": "100%", + "start": "0", + "type": "8300" + } + }, + "type": "gpt" + }, + "device": "/dev/disk/by-id/some-disk-id", + "imageName": "main", + "imageSize": "2G", + "name": "main", + "type": "disk" + } + }, + "lvm_vg": {}, + "mdadm": {}, + "nodev": {}, + "zpool": {} +} \ No newline at end of file diff --git a/tests/disko_lib/eval_config/flake-testmachine-eval-result.json b/tests/disko_lib/eval_config/flake-testmachine-eval-result.json new file mode 100644 index 00000000..426f4e52 --- /dev/null +++ b/tests/disko_lib/eval_config/flake-testmachine-eval-result.json @@ -0,0 +1,98 @@ +{ + "disk": { + "main": { + "content": { + "device": "/dev/disk/by-id/ata-Samsung_SSD_850_EVO_250GB_S21PNXAGB12345", + "efiGptPartitionFirst": true, + "partitions": { + "ESP": { + "_index": 2, + "alignment": 0, + "content": { + "device": "/dev/disk/by-partlabel/disk-main-ESP", + "extraArgs": [], + "format": "vfat", + "mountOptions": [ + "umask=0077" + ], + "mountpoint": "/boot", + "postCreateHook": "", + "postMountHook": "", + "preCreateHook": "", + "preMountHook": "", + "type": "filesystem" + }, + "device": "/dev/disk/by-partlabel/disk-main-ESP", + "end": "+512M", + "hybrid": null, + "label": "disk-main-ESP", + "name": "ESP", + "priority": 1000, + "size": "512M", + "start": "0", + "type": "EF00" + }, + "boot": { + "_index": 1, + "alignment": 0, + "content": null, + "device": "/dev/disk/by-partlabel/disk-main-boot", + "end": "+1M", + "hybrid": null, + "label": "disk-main-boot", + "name": "boot", + "priority": 100, + "size": "1M", + "start": "0", + "type": "EF02" + }, + "root": { + "_index": 3, + "alignment": 0, + "content": { + "device": "/dev/disk/by-partlabel/disk-main-root", + "extraArgs": [], + "format": "ext4", + "mountOptions": [ + "defaults" + ], + "mountpoint": "/", + "postCreateHook": "", + "postMountHook": "", + "preCreateHook": "", + "preMountHook": "", + "type": "filesystem" + }, + "device": "/dev/disk/by-partlabel/disk-main-root", + "end": "-0", + "hybrid": null, + "label": "disk-main-root", + "name": "root", + "priority": 9001, + "size": "100%", + "start": "0", + "type": "8300" + } + }, + "postCreateHook": "", + "postMountHook": "", + "preCreateHook": "", + "preMountHook": "", + "type": "gpt" + }, + "device": "/dev/disk/by-id/ata-Samsung_SSD_850_EVO_250GB_S21PNXAGB12345", + "imageName": "main", + "imageSize": "2G", + "name": "main", + "postCreateHook": "", + "postMountHook": "", + "preCreateHook": "", + "preMountHook": "", + "type": "disk" + } + }, + "lvm_vg": {}, + "mdadm": {}, + "nodev": {}, + "zpool": {} +} diff --git a/tests/disko_lib/eval_config/flake-testmachine-validate-result.json b/tests/disko_lib/eval_config/flake-testmachine-validate-result.json new file mode 100644 index 00000000..1ddd6436 --- /dev/null +++ b/tests/disko_lib/eval_config/flake-testmachine-validate-result.json @@ -0,0 +1,82 @@ +{ + "disk": { + "main": { + "content": { + "device": "/dev/disk/by-id/ata-Samsung_SSD_850_EVO_250GB_S21PNXAGB12345", + "efiGptPartitionFirst": true, + "partitions": { + "ESP": { + "_index": 2, + "alignment": 0, + "content": { + "device": "/dev/disk/by-partlabel/disk-main-ESP", + "extraArgs": [], + "format": "vfat", + "mountOptions": [ + "umask=0077" + ], + "mountpoint": "/boot", + "type": "filesystem" + }, + "device": "/dev/disk/by-partlabel/disk-main-ESP", + "end": "+512M", + "hybrid": null, + "label": "disk-main-ESP", + "name": "ESP", + "priority": 1000, + "size": "512M", + "start": "0", + "type": "EF00" + }, + "boot": { + "_index": 1, + "alignment": 0, + "content": null, + "device": "/dev/disk/by-partlabel/disk-main-boot", + "end": "+1M", + "hybrid": null, + "label": "disk-main-boot", + "name": "boot", + "priority": 100, + "size": "1M", + "start": "0", + "type": "EF02" + }, + "root": { + "_index": 3, + "alignment": 0, + "content": { + "device": "/dev/disk/by-partlabel/disk-main-root", + "extraArgs": [], + "format": "ext4", + "mountOptions": [ + "defaults" + ], + "mountpoint": "/", + "type": "filesystem" + }, + "device": "/dev/disk/by-partlabel/disk-main-root", + "end": "-0", + "hybrid": null, + "label": "disk-main-root", + "name": "root", + "priority": 9001, + "size": "100%", + "start": "0", + "type": "8300" + } + }, + "type": "gpt" + }, + "device": "/dev/disk/by-id/ata-Samsung_SSD_850_EVO_250GB_S21PNXAGB12345", + "imageName": "main", + "imageSize": "2G", + "name": "main", + "type": "disk" + } + }, + "lvm_vg": {}, + "mdadm": {}, + "nodev": {}, + "zpool": {} +} \ No newline at end of file diff --git a/tests/disko_lib/eval_config/test_eval_config.py b/tests/disko_lib/eval_config/test_eval_config.py new file mode 100644 index 00000000..7249aaea --- /dev/null +++ b/tests/disko_lib/eval_config/test_eval_config.py @@ -0,0 +1,66 @@ +import json +from pathlib import Path +from typing import cast + +from disko_lib.eval_config import eval_config_file_as_json, validate_config +from disko_lib.messages.msgs import err_missing_arguments, err_too_many_arguments +from disko_lib.result import DiskoError, DiskoSuccess +from disko_lib.json_types import JsonDict + +CURRENT_DIR = Path(__file__).parent +ROOT_DIR = CURRENT_DIR.parent.parent.parent +assert (ROOT_DIR / "flake.nix").exists() + + +def test_eval_config_missing_arguments() -> None: + result = eval_config_file_as_json(disko_file=None, flake=None) + assert isinstance(result, DiskoError) + assert result.messages[0].is_message(err_missing_arguments) + assert result.context == "validate args" + + +def test_eval_config_too_many_arguments() -> None: + result = eval_config_file_as_json(disko_file="foo", flake="bar") + assert isinstance(result, DiskoError) + assert result.messages[0].is_message(err_too_many_arguments) + assert result.context == "validate args" + + +def test_eval_config_disko_file() -> None: + disko_file_path = ROOT_DIR / "example" / "simple-efi.nix" + result = eval_config_file_as_json(disko_file=str(disko_file_path), flake=None) + assert isinstance(result, DiskoSuccess) + with open(CURRENT_DIR / "file-simple-efi-eval-result.json") as f: + expected_result = cast(JsonDict, json.load(f)) + assert result.value == expected_result + + +def test_eval_config_flake_testmachine() -> None: + result = eval_config_file_as_json(disko_file=None, flake=f"{ROOT_DIR}#testmachine") + assert isinstance(result, DiskoSuccess) + with open(CURRENT_DIR / "flake-testmachine-eval-result.json") as f: + expected_result = cast(JsonDict, json.load(f)) + assert result.value == expected_result + + +def test_eval_and_validate_config_disko_file() -> None: + disko_file_path = ROOT_DIR / "example" / "simple-efi.nix" + eval_result = eval_config_file_as_json(disko_file=str(disko_file_path), flake=None) + assert isinstance(eval_result, DiskoSuccess) + validate_result = validate_config(eval_result.value) + assert isinstance(validate_result, DiskoSuccess) + with open(CURRENT_DIR / "file-simple-efi-validate-result.json") as f: + expected_result = f.read() + assert json.loads(validate_result.value.model_dump_json(by_alias=True)) == json.loads(expected_result) # type: ignore[misc] + + +def test_eval_and_validate_flake_testmachine() -> None: + eval_result = eval_config_file_as_json( + disko_file=None, flake=f"{ROOT_DIR}#testmachine" + ) + assert isinstance(eval_result, DiskoSuccess) + validate_result = validate_config(eval_result.value) + assert isinstance(validate_result, DiskoSuccess) + with open(CURRENT_DIR / "flake-testmachine-validate-result.json") as f: + expected_result = f.read() + assert json.loads(validate_result.value.model_dump_json(by_alias=True)) == json.loads(expected_result) # type: ignore[misc] diff --git a/tests/disko_lib/test_logging.py b/tests/disko_lib/test_logging.py new file mode 100644 index 00000000..afc0bf8d --- /dev/null +++ b/tests/disko_lib/test_logging.py @@ -0,0 +1,55 @@ +from disko_lib.logging import dedent_start_lines + + +def test_dedent_start_lines() -> None: + # Some later lines are indented as much or more than the first line, + # but they should NOT be dedented! + raw_lines = """ + Successfully generated config for some devices. + Errors are printed above. The generated partial config is: + { + "disko": { + "devices": { + "disk": { + "MODEL:SanDisk SD8TB8U256G1001,SN:171887425854": { + "device": "/dev/sdb", + "type": "disk", + "content": { + "type": "gpt", + "partitions": { + "UUID:01D60069CEED69C0": { + "_index": 1, + "size": "523730944", + "content": { + "type": "filesystem", + "format": "ntfs", + "mountpoint": null + } + """ + + expected_output = """Successfully generated config for some devices. +Errors are printed above. The generated partial config is: +{ + "disko": { + "devices": { + "disk": { + "MODEL:SanDisk SD8TB8U256G1001,SN:171887425854": { + "device": "/dev/sdb", + "type": "disk", + "content": { + "type": "gpt", + "partitions": { + "UUID:01D60069CEED69C0": { + "_index": 1, + "size": "523730944", + "content": { + "type": "filesystem", + "format": "ntfs", + "mountpoint": null + } + """ + + lines = raw_lines.splitlines()[1:] + result = "\n".join(dedent_start_lines(lines)) + + assert result == expected_output diff --git a/tests/disko_lib/types_disk/partial_failure_dos_table-generate-result.json b/tests/disko_lib/types_disk/partial_failure_dos_table-generate-result.json new file mode 100644 index 00000000..5ac6aee9 --- /dev/null +++ b/tests/disko_lib/types_disk/partial_failure_dos_table-generate-result.json @@ -0,0 +1,131 @@ +{ + "disko": { + "devices": { + "disk": { + "MODEL:SanDisk SD8TB8U256G1001,SN:171887425854": { + "device": "/dev/sdb", + "type": "disk", + "content": { + "type": "gpt", + "partitions": { + "UUID:01D60069CEED69C0": { + "_index": 1, + "size": "523730944", + "content": { + "type": "filesystem", + "format": "ntfs", + "mountpoint": null + } + }, + "UUID:01D6006B90875FE0": { + "_index": 2, + "size": "254301093376", + "content": { + "type": "filesystem", + "format": "ntfs", + "mountpoint": null + } + }, + "UUID:708B-C192": { + "_index": 3, + "size": "104857600", + "content": { + "type": "filesystem", + "format": "vfat", + "mountpoint": null + }, + "type": "EF00" + }, + "UUID:90B6FB81B6FB65DE": { + "_index": 4, + "size": "570425344", + "content": { + "type": "filesystem", + "format": "ntfs", + "mountpoint": null + } + } + } + } + }, + "MODEL:CT2000MX500SSD1,SN:2105E4F0DE85": { + "device": "/dev/sdc", + "type": "disk", + "content": { + "type": "gpt", + "partitions": { + "UUID:2740-1628": { + "_index": 1, + "size": "1048576000", + "content": { + "type": "filesystem", + "format": "vfat", + "mountpoint": "/boot" + }, + "type": "EF00" + }, + "UUID:ca548f68-4e51-4364-b366-690ecc27590f": { + "_index": 2, + "size": "631794302976", + "content": { + "type": "filesystem", + "format": "ext4", + "mountpoint": "/" + } + }, + "UUID:879299db-4147-4fac-9f34-5e8e92073efc": { + "_index": 3, + "size": "631810031616", + "content": { + "type": "filesystem", + "format": "crypto_LUKS", + "mountpoint": null + } + }, + "UUID:9A48E8C248E89E6F": { + "_index": 4, + "size": "735743836160", + "content": { + "type": "filesystem", + "format": "ntfs", + "mountpoint": "/mnt/s" + } + } + } + } + }, + "MODEL:ST2000LM003 HN-M201RAD,SN:S321J9GFC01497": { + "device": "/dev/sdd", + "type": "disk", + "content": { + "type": "gpt", + "partitions": { + "PARTUUID:c090741b-68e2-4867-96df-2ec00765f2c0": { + "_index": 1, + "size": "134217728", + "content": { + "type": "filesystem", + "format": "", + "mountpoint": null + } + }, + "UUID:7CA41E5EA41E1B6A": { + "_index": 2, + "size": "2000263577600", + "content": { + "type": "filesystem", + "format": "ntfs", + "mountpoint": "/mnt/g" + } + } + } + } + } + }, + "lvm_vg": {}, + "mdadm": {}, + "nodev": {}, + "zpool": {} + } + } +} \ No newline at end of file diff --git a/tests/disko_lib/types_disk/partial_failure_dos_table-lsblk-output.json b/tests/disko_lib/types_disk/partial_failure_dos_table-lsblk-output.json new file mode 100644 index 00000000..2b48ab76 --- /dev/null +++ b/tests/disko_lib/types_disk/partial_failure_dos_table-lsblk-output.json @@ -0,0 +1,423 @@ +{ + "blockdevices": [ + { + "id-link": "wwn-0x5002538043584d30", + "fstype": null, + "fssize": null, + "fsuse%": null, + "kname": "sda", + "label": null, + "model": "SAMSUNG SSD PM830 mSATA 256GB", + "partflags": null, + "partlabel": null, + "partn": null, + "parttype": null, + "parttypename": null, + "partuuid": null, + "path": "/dev/sda", + "phy-sec": 512, + "pttype": "dos", + "rev": "CXM13D1Q", + "serial": "S0XPNYAD407619", + "size": 256060514304, + "start": null, + "mountpoint": null, + "mountpoints": [ + null + ], + "type": "disk", + "uuid": null + },{ + "id-link": "wwn-0x5001b444a63292c3", + "fstype": null, + "fssize": null, + "fsuse%": null, + "kname": "sdb", + "label": null, + "model": "SanDisk SD8TB8U256G1001", + "partflags": null, + "partlabel": null, + "partn": null, + "parttype": null, + "parttypename": null, + "partuuid": null, + "path": "/dev/sdb", + "phy-sec": 512, + "pttype": "gpt", + "rev": "X4133101", + "serial": "171887425854", + "size": 256060514304, + "start": null, + "mountpoint": null, + "mountpoints": [ + null + ], + "type": "disk", + "uuid": null, + "children": [ + { + "id-link": "wwn-0x5001b444a63292c3-part1", + "fstype": "ntfs", + "fssize": null, + "fsuse%": null, + "kname": "sdb1", + "label": "System Reserved", + "model": null, + "partflags": null, + "partlabel": null, + "partn": 1, + "parttype": "ebd0a0a2-b9e5-4433-87c0-68b6b72699c7", + "parttypename": "Microsoft basic data", + "partuuid": "5cbaf771-8ffa-11eb-952d-fcaa14203853", + "path": "/dev/sdb1", + "phy-sec": 512, + "pttype": "gpt", + "rev": null, + "serial": null, + "size": 523730944, + "start": 64, + "mountpoint": null, + "mountpoints": [ + null + ], + "type": "part", + "uuid": "01D60069CEED69C0" + },{ + "id-link": "wwn-0x5001b444a63292c3-part2", + "fstype": "ntfs", + "fssize": null, + "fsuse%": null, + "kname": "sdb2", + "label": null, + "model": null, + "partflags": null, + "partlabel": null, + "partn": 2, + "parttype": "ebd0a0a2-b9e5-4433-87c0-68b6b72699c7", + "parttypename": "Microsoft basic data", + "partuuid": "5cbaf772-8ffa-11eb-952d-fcaa14203853", + "path": "/dev/sdb2", + "phy-sec": 512, + "pttype": "gpt", + "rev": null, + "serial": null, + "size": 254301093376, + "start": 1022976, + "mountpoint": null, + "mountpoints": [ + null + ], + "type": "part", + "uuid": "01D6006B90875FE0" + },{ + "id-link": "wwn-0x5001b444a63292c3-part3", + "fstype": "vfat", + "fssize": null, + "fsuse%": null, + "kname": "sdb3", + "label": null, + "model": null, + "partflags": "0x8000000000000000", + "partlabel": null, + "partn": 3, + "parttype": "c12a7328-f81f-11d2-ba4b-00a0c93ec93b", + "parttypename": "EFI System", + "partuuid": "5cbaf773-8ffa-11eb-952d-fcaa14203853", + "path": "/dev/sdb3", + "phy-sec": 512, + "pttype": "gpt", + "rev": null, + "serial": null, + "size": 104857600, + "start": 497704960, + "mountpoint": null, + "mountpoints": [ + null + ], + "type": "part", + "uuid": "708B-C192" + },{ + "id-link": "wwn-0x5001b444a63292c3-part4", + "fstype": "ntfs", + "fssize": null, + "fsuse%": null, + "kname": "sdb4", + "label": null, + "model": null, + "partflags": "0x8000000000000001", + "partlabel": null, + "partn": 4, + "parttype": "de94bba4-06d1-4d40-a16a-bfd50179d6ac", + "parttypename": "Windows recovery environment", + "partuuid": "5cbaf774-8ffa-11eb-952d-fcaa14203853", + "path": "/dev/sdb4", + "phy-sec": 512, + "pttype": "gpt", + "rev": null, + "serial": null, + "size": 570425344, + "start": 497909760, + "mountpoint": null, + "mountpoints": [ + null + ], + "type": "part", + "uuid": "90B6FB81B6FB65DE" + } + ] + },{ + "id-link": "wwn-0x500a0751e4f0de85", + "fstype": null, + "fssize": null, + "fsuse%": null, + "kname": "sdc", + "label": null, + "model": "CT2000MX500SSD1", + "partflags": null, + "partlabel": null, + "partn": null, + "parttype": null, + "parttypename": null, + "partuuid": null, + "path": "/dev/sdc", + "phy-sec": 4096, + "pttype": "gpt", + "rev": "M3CR033", + "serial": "2105E4F0DE85", + "size": 2000398934016, + "start": null, + "mountpoint": null, + "mountpoints": [ + null + ], + "type": "disk", + "uuid": null, + "children": [ + { + "id-link": "wwn-0x500a0751e4f0de85-part1", + "fstype": "vfat", + "fssize": 1046478848, + "fsuse%": "9%", + "kname": "sdc1", + "label": "boot", + "model": null, + "partflags": null, + "partlabel": "boot", + "partn": 1, + "parttype": "c12a7328-f81f-11d2-ba4b-00a0c93ec93b", + "parttypename": "EFI System", + "partuuid": "7f623bea-5891-49ee-9980-6534716f0f50", + "path": "/dev/sdc1", + "phy-sec": 4096, + "pttype": "gpt", + "rev": null, + "serial": null, + "size": 1048576000, + "start": 2048, + "mountpoint": "/boot", + "mountpoints": [ + "/boot" + ], + "type": "part", + "uuid": "2740-1628" + },{ + "id-link": "wwn-0x500a0751e4f0de85-part2", + "fstype": "ext4", + "fssize": 620727574528, + "fsuse%": "13%", + "kname": "sdc2", + "label": "root", + "model": null, + "partflags": null, + "partlabel": "root", + "partn": 2, + "parttype": "0fc63daf-8483-4772-8e79-3d69d8477de4", + "parttypename": "Linux filesystem", + "partuuid": "d562416c-1632-40c6-88ed-5095fb921698", + "path": "/dev/sdc2", + "phy-sec": 4096, + "pttype": "gpt", + "rev": null, + "serial": null, + "size": 631794302976, + "start": 2050048, + "mountpoint": "/", + "mountpoints": [ + "/nix/store", "/" + ], + "type": "part", + "uuid": "ca548f68-4e51-4364-b366-690ecc27590f" + },{ + "id-link": "wwn-0x500a0751e4f0de85-part3", + "fstype": "crypto_LUKS", + "fssize": null, + "fsuse%": null, + "kname": "sdc3", + "label": "home", + "model": null, + "partflags": null, + "partlabel": "home", + "partn": 3, + "parttype": "0fc63daf-8483-4772-8e79-3d69d8477de4", + "parttypename": "Linux filesystem", + "partuuid": "0ca34f17-5c80-4f2d-97b5-2a5f559b55b5", + "path": "/dev/sdc3", + "phy-sec": 4096, + "pttype": "gpt", + "rev": null, + "serial": null, + "size": 631810031616, + "start": 1236023296, + "mountpoint": null, + "mountpoints": [ + null + ], + "type": "part", + "uuid": "879299db-4147-4fac-9f34-5e8e92073efc", + "children": [ + { + "id-link": "dm-name-crypt-home", + "fstype": "btrfs", + "fssize": 631793254400, + "fsuse%": "47%", + "kname": "dm-0", + "label": null, + "model": null, + "partflags": null, + "partlabel": null, + "partn": null, + "parttype": null, + "parttypename": null, + "partuuid": null, + "path": "/dev/mapper/crypt-home", + "phy-sec": 4096, + "pttype": null, + "rev": null, + "serial": null, + "size": 631793254400, + "start": null, + "mountpoint": "/home", + "mountpoints": [ + "/home" + ], + "type": "crypt", + "uuid": "1900f25d-5b93-41f1-a3d4-18fdbd70fe8b" + } + ] + },{ + "id-link": "wwn-0x500a0751e4f0de85-part4", + "fstype": "ntfs", + "fssize": 735743832064, + "fsuse%": "92%", + "kname": "sdc4", + "label": "Schnell", + "model": null, + "partflags": null, + "partlabel": "Basic data partition", + "partn": 4, + "parttype": "ebd0a0a2-b9e5-4433-87c0-68b6b72699c7", + "parttypename": "Microsoft basic data", + "partuuid": "1d55f1cf-5a42-4fa1-a0ba-573fbd0d152c", + "path": "/dev/sdc4", + "phy-sec": 4096, + "pttype": "gpt", + "rev": null, + "serial": null, + "size": 735743836160, + "start": 2470027264, + "mountpoint": "/mnt/s", + "mountpoints": [ + "/mnt/s" + ], + "type": "part", + "uuid": "9A48E8C248E89E6F" + } + ] + },{ + "id-link": "wwn-0x50004cf20ecb1679", + "fstype": null, + "fssize": null, + "fsuse%": null, + "kname": "sdd", + "label": null, + "model": "ST2000LM003 HN-M201RAD", + "partflags": null, + "partlabel": null, + "partn": null, + "parttype": null, + "parttypename": null, + "partuuid": null, + "path": "/dev/sdd", + "phy-sec": 4096, + "pttype": "gpt", + "rev": "2BC10001", + "serial": "S321J9GFC01497", + "size": 2000398934016, + "start": null, + "mountpoint": null, + "mountpoints": [ + null + ], + "type": "disk", + "uuid": null, + "children": [ + { + "id-link": "wwn-0x50004cf20ecb1679-part1", + "fstype": null, + "fssize": null, + "fsuse%": null, + "kname": "sdd1", + "label": null, + "model": null, + "partflags": null, + "partlabel": "Microsoft reserved partition", + "partn": 1, + "parttype": "e3c9e316-0b5c-4db8-817d-f92df00215ae", + "parttypename": "Microsoft reserved", + "partuuid": "c090741b-68e2-4867-96df-2ec00765f2c0", + "path": "/dev/sdd1", + "phy-sec": 4096, + "pttype": "gpt", + "rev": null, + "serial": null, + "size": 134217728, + "start": 34, + "mountpoint": null, + "mountpoints": [ + null + ], + "type": "part", + "uuid": null + },{ + "id-link": "wwn-0x50004cf20ecb1679-part2", + "fstype": "ntfs", + "fssize": 2000263573504, + "fsuse%": "51%", + "kname": "sdd2", + "label": "Groß", + "model": null, + "partflags": null, + "partlabel": "Basic data partition", + "partn": 2, + "parttype": "ebd0a0a2-b9e5-4433-87c0-68b6b72699c7", + "parttypename": "Microsoft basic data", + "partuuid": "425b6415-db2b-4684-b619-994bdd0f9b71", + "path": "/dev/sdd2", + "phy-sec": 4096, + "pttype": "gpt", + "rev": null, + "serial": null, + "size": 2000263577600, + "start": 264192, + "mountpoint": "/mnt/g", + "mountpoints": [ + "/mnt/g" + ], + "type": "part", + "uuid": "7CA41E5EA41E1B6A" + } + ] + } + ] +} + diff --git a/tests/disko_lib/types_disk/test_types_disk.py b/tests/disko_lib/types_disk/test_types_disk.py new file mode 100644 index 00000000..ea3b898e --- /dev/null +++ b/tests/disko_lib/types_disk/test_types_disk.py @@ -0,0 +1,36 @@ +import json +from pathlib import Path, PosixPath + +from disko_lib.messages import err_unsupported_pttype, warn_generate_partial_failure +from disko_lib.result import DiskoPartialSuccess, DiskoSuccess +from disko_lib.generate_config import generate_config +from disko_lib.types import device + +CURRENT_DIR = Path(__file__).parent + + +def test_generate_config_partial_failure_dos_table() -> None: + with open(CURRENT_DIR / "partial_failure_dos_table-lsblk-output.json") as f: + lsblk_result = device.list_block_devices(f.read()) + + assert isinstance(lsblk_result, DiskoSuccess) + + result = generate_config(lsblk_result.value) + + assert isinstance(result, DiskoPartialSuccess) + + assert result.messages[0].is_message(err_unsupported_pttype) + assert result.messages[0].details == { + "pttype": "dos", + "device": PosixPath("/dev/sda"), + } + + assert result.messages[1].is_message(warn_generate_partial_failure) + with open(CURRENT_DIR / "partial_failure_dos_table-generate-result.json") as f: + assert result.value == json.load(f) # type: ignore[misc] + assert result.messages[1].details["failed"] == [PosixPath("/dev/sda")] + assert result.messages[1].details["successful"] == [ + PosixPath("/dev/sdb"), + PosixPath("/dev/sdc"), + PosixPath("/dev/sdd"), + ] diff --git a/tests/example/README.md b/tests/example/README.md new file mode 100644 index 00000000..0412d6b3 --- /dev/null +++ b/tests/example/README.md @@ -0,0 +1,3 @@ +# Tests for all example files + +You can check out the examples tested here in [../../example](../../example/). \ No newline at end of file diff --git a/tests/bcachefs.nix b/tests/example/bcachefs.nix similarity index 78% rename from tests/bcachefs.nix rename to tests/example/bcachefs.nix index f8806d55..4739cf3e 100644 --- a/tests/bcachefs.nix +++ b/tests/example/bcachefs.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "bcachefs"; - disko-config = ../example/bcachefs.nix; + disko-config = ../../example/bcachefs.nix; extraTestScript = '' machine.succeed("mountpoint /"); machine.succeed("lsblk >&2"); diff --git a/tests/boot-raid1.nix b/tests/example/boot-raid1.nix similarity index 78% rename from tests/boot-raid1.nix rename to tests/example/boot-raid1.nix index e020fdd8..b06801dd 100644 --- a/tests/boot-raid1.nix +++ b/tests/example/boot-raid1.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "boot-raid1"; - disko-config = ../example/boot-raid1.nix; + disko-config = ../../example/boot-raid1.nix; extraTestScript = '' machine.succeed("test -b /dev/md/boot"); machine.succeed("mountpoint /boot"); diff --git a/tests/btrfs-only-root-subvolume.nix b/tests/example/btrfs-only-root-subvolume.nix similarity index 63% rename from tests/btrfs-only-root-subvolume.nix rename to tests/example/btrfs-only-root-subvolume.nix index 818aec70..e360715e 100644 --- a/tests/btrfs-only-root-subvolume.nix +++ b/tests/example/btrfs-only-root-subvolume.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "btrfs-only-root-subvolume"; - disko-config = ../example/btrfs-only-root-subvolume.nix; + disko-config = ../../example/btrfs-only-root-subvolume.nix; extraTestScript = '' machine.succeed("btrfs subvolume list /"); ''; diff --git a/tests/btrfs-subvolumes.nix b/tests/example/btrfs-subvolumes.nix similarity index 86% rename from tests/btrfs-subvolumes.nix rename to tests/example/btrfs-subvolumes.nix index 7059c1a1..d22d271e 100644 --- a/tests/btrfs-subvolumes.nix +++ b/tests/example/btrfs-subvolumes.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "btrfs-subvolumes"; - disko-config = ../example/btrfs-subvolumes.nix; + disko-config = ../../example/btrfs-subvolumes.nix; extraTestScript = '' machine.succeed("test ! -e /test"); machine.succeed("test -e /home/user"); diff --git a/tests/complex.nix b/tests/example/complex.nix similarity index 89% rename from tests/complex.nix rename to tests/example/complex.nix index 26cc5e9d..94580628 100644 --- a/tests/complex.nix +++ b/tests/example/complex.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "complex"; - disko-config = ../example/complex.nix; + disko-config = ../../example/complex.nix; extraInstallerConfig.networking.hostId = "8425e349"; extraSystemConfig = { networking.hostId = "8425e349"; diff --git a/tests/f2fs.nix b/tests/example/f2fs.nix similarity index 78% rename from tests/f2fs.nix rename to tests/example/f2fs.nix index ebcf8d8d..fd901be8 100644 --- a/tests/f2fs.nix +++ b/tests/example/f2fs.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "f2fs"; - disko-config = ../example/f2fs.nix; + disko-config = ../../example/f2fs.nix; extraTestScript = '' machine.succeed("mountpoint /"); machine.succeed("lsblk --fs >&2"); diff --git a/tests/gpt-bios-compat.nix b/tests/example/gpt-bios-compat.nix similarity index 65% rename from tests/gpt-bios-compat.nix rename to tests/example/gpt-bios-compat.nix index de45d3b6..45a01634 100644 --- a/tests/gpt-bios-compat.nix +++ b/tests/example/gpt-bios-compat.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "gpt-bios-compat"; - disko-config = ../example/gpt-bios-compat.nix; + disko-config = ../../example/gpt-bios-compat.nix; extraTestScript = '' machine.succeed("mountpoint /"); ''; diff --git a/tests/gpt-name-with-special-chars.nix b/tests/example/gpt-name-with-special-chars.nix similarity index 73% rename from tests/gpt-name-with-special-chars.nix rename to tests/example/gpt-name-with-special-chars.nix index 48b4304d..77c1a070 100644 --- a/tests/gpt-name-with-special-chars.nix +++ b/tests/example/gpt-name-with-special-chars.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "gpt-name-with-whitespace"; - disko-config = ../example/gpt-name-with-whitespace.nix; + disko-config = ../../example/gpt-name-with-whitespace.nix; extraTestScript = '' machine.succeed("mountpoint /"); machine.succeed("mountpoint '/name with spaces'"); diff --git a/tests/hybrid-mbr.nix b/tests/example/hybrid-mbr.nix similarity index 63% rename from tests/hybrid-mbr.nix rename to tests/example/hybrid-mbr.nix index de68b264..b86c536c 100644 --- a/tests/hybrid-mbr.nix +++ b/tests/example/hybrid-mbr.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "hybrid-mbr"; - disko-config = ../example/hybrid-mbr.nix; + disko-config = ../../example/hybrid-mbr.nix; extraTestScript = '' machine.succeed("mountpoint /"); ''; diff --git a/tests/hybrid-tmpfs-on-root.nix b/tests/example/hybrid-tmpfs-on-root.nix similarity index 68% rename from tests/hybrid-tmpfs-on-root.nix rename to tests/example/hybrid-tmpfs-on-root.nix index 09d15d6a..c6e37903 100644 --- a/tests/hybrid-tmpfs-on-root.nix +++ b/tests/example/hybrid-tmpfs-on-root.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "hybrid-tmpfs-on-root"; - disko-config = ../example/hybrid-tmpfs-on-root.nix; + disko-config = ../../example/hybrid-tmpfs-on-root.nix; extraTestScript = '' machine.succeed("mountpoint /"); machine.succeed("findmnt / --types tmpfs"); diff --git a/tests/hybrid.nix b/tests/example/hybrid.nix similarity index 64% rename from tests/hybrid.nix rename to tests/example/hybrid.nix index adf7ccd1..40a98c81 100644 --- a/tests/hybrid.nix +++ b/tests/example/hybrid.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "hybrid"; - disko-config = ../example/hybrid.nix; + disko-config = ../../example/hybrid.nix; extraTestScript = '' machine.succeed("mountpoint /"); ''; diff --git a/tests/legacy-table-with-whitespace.nix b/tests/example/legacy-table-with-whitespace.nix similarity index 67% rename from tests/legacy-table-with-whitespace.nix rename to tests/example/legacy-table-with-whitespace.nix index 300c641f..352cfc4d 100644 --- a/tests/legacy-table-with-whitespace.nix +++ b/tests/example/legacy-table-with-whitespace.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "legacy-table-with-whitespace"; - disko-config = ../example/legacy-table-with-whitespace.nix; + disko-config = ../../example/legacy-table-with-whitespace.nix; extraTestScript = '' machine.succeed("mountpoint /"); machine.succeed("mountpoint /name_with_spaces"); diff --git a/tests/legacy-table.nix b/tests/example/legacy-table.nix similarity index 63% rename from tests/legacy-table.nix rename to tests/example/legacy-table.nix index 2c7b6461..d369f93d 100644 --- a/tests/legacy-table.nix +++ b/tests/example/legacy-table.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "legacy-table"; - disko-config = ../example/legacy-table.nix; + disko-config = ../../example/legacy-table.nix; extraTestScript = '' machine.succeed("mountpoint /"); ''; diff --git a/tests/long-device-name.nix b/tests/example/long-device-name.nix similarity index 63% rename from tests/long-device-name.nix rename to tests/example/long-device-name.nix index e328a9fe..25c3a132 100644 --- a/tests/long-device-name.nix +++ b/tests/example/long-device-name.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "long-device-name"; - disko-config = ../example/long-device-name.nix; + disko-config = ../../example/long-device-name.nix; extraTestScript = '' machine.succeed("mountpoint /"); ''; diff --git a/tests/luks-btrfs-raid.nix b/tests/example/luks-btrfs-raid.nix similarity index 65% rename from tests/luks-btrfs-raid.nix rename to tests/example/luks-btrfs-raid.nix index 6e127498..54c2710d 100644 --- a/tests/luks-btrfs-raid.nix +++ b/tests/example/luks-btrfs-raid.nix @@ -1,11 +1,11 @@ -{ - pkgs ? import { }, - diskoLib ? pkgs.callPackage ../lib { }, +{ pkgs ? import { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } +, }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "luks-btrfs-raid"; - disko-config = ../example/luks-btrfs-raid.nix; + disko-config = ../../example/luks-btrfs-raid.nix; extraTestScript = '' machine.succeed("cryptsetup isLuks /dev/vda2"); machine.succeed("cryptsetup isLuks /dev/vdb1"); diff --git a/tests/luks-btrfs-subvolumes.nix b/tests/example/luks-btrfs-subvolumes.nix similarity index 78% rename from tests/luks-btrfs-subvolumes.nix rename to tests/example/luks-btrfs-subvolumes.nix index 6a4e64b3..dfb0795f 100644 --- a/tests/luks-btrfs-subvolumes.nix +++ b/tests/example/luks-btrfs-subvolumes.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "luks-btrfs-subvolumes"; - disko-config = ../example/luks-btrfs-subvolumes.nix; + disko-config = ../../example/luks-btrfs-subvolumes.nix; extraTestScript = '' machine.succeed("cryptsetup isLuks /dev/vda2"); machine.succeed("btrfs subvolume list / | grep -qs 'path nix$'"); diff --git a/tests/luks-interactive-login.nix b/tests/example/luks-interactive-login.nix similarity index 73% rename from tests/luks-interactive-login.nix rename to tests/example/luks-interactive-login.nix index 6fae2e11..863bc9b1 100644 --- a/tests/luks-interactive-login.nix +++ b/tests/example/luks-interactive-login.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "luks-interactive-login"; - disko-config = ../example/luks-interactive-login.nix; + disko-config = ../../example/luks-interactive-login.nix; extraTestScript = '' machine.succeed("cryptsetup isLuks /dev/vda2"); ''; diff --git a/tests/luks-lvm.nix b/tests/example/luks-lvm.nix similarity index 70% rename from tests/luks-lvm.nix rename to tests/example/luks-lvm.nix index 848a5b7e..809d9c1f 100644 --- a/tests/luks-lvm.nix +++ b/tests/example/luks-lvm.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "luks-lvm"; - disko-config = ../example/luks-lvm.nix; + disko-config = ../../example/luks-lvm.nix; extraTestScript = '' machine.succeed("cryptsetup isLuks /dev/vda2"); machine.succeed("mountpoint /home"); diff --git a/tests/luks-on-mdadm.nix b/tests/example/luks-on-mdadm.nix similarity index 78% rename from tests/luks-on-mdadm.nix rename to tests/example/luks-on-mdadm.nix index bd49762e..184a0397 100644 --- a/tests/luks-on-mdadm.nix +++ b/tests/example/luks-on-mdadm.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "luks-on-mdadm"; - disko-config = ../example/luks-on-mdadm.nix; + disko-config = ../../example/luks-on-mdadm.nix; extraTestScript = '' machine.succeed("test -b /dev/md/raid1"); machine.succeed("mountpoint /"); diff --git a/tests/lvm-raid.nix b/tests/example/lvm-raid.nix similarity index 80% rename from tests/lvm-raid.nix rename to tests/example/lvm-raid.nix index b30332a8..17854a39 100644 --- a/tests/lvm-raid.nix +++ b/tests/example/lvm-raid.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "lvm-raid"; - disko-config = ../example/lvm-raid.nix; + disko-config = ../../example/lvm-raid.nix; extraTestScript = '' machine.succeed("mountpoint /home"); ''; diff --git a/tests/lvm-sizes-sort.nix b/tests/example/lvm-sizes-sort.nix similarity index 63% rename from tests/lvm-sizes-sort.nix rename to tests/example/lvm-sizes-sort.nix index 83ed472a..fb9207e8 100644 --- a/tests/lvm-sizes-sort.nix +++ b/tests/example/lvm-sizes-sort.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "lvm-sizes-sort"; - disko-config = ../example/lvm-sizes-sort.nix; + disko-config = ../../example/lvm-sizes-sort.nix; extraTestScript = '' machine.succeed("mountpoint /home"); ''; diff --git a/tests/lvm-thin.nix b/tests/example/lvm-thin.nix similarity index 64% rename from tests/lvm-thin.nix rename to tests/example/lvm-thin.nix index bfbcfc19..2584132e 100644 --- a/tests/lvm-thin.nix +++ b/tests/example/lvm-thin.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "lvm-thin"; - disko-config = ../example/lvm-thin.nix; + disko-config = ../../example/lvm-thin.nix; extraTestScript = '' machine.succeed("mountpoint /home"); ''; diff --git a/tests/mdadm-raid0.nix b/tests/example/mdadm-raid0.nix similarity index 70% rename from tests/mdadm-raid0.nix rename to tests/example/mdadm-raid0.nix index 7d40109d..69192261 100644 --- a/tests/mdadm-raid0.nix +++ b/tests/example/mdadm-raid0.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "mdadm-raid0"; - disko-config = ../example/mdadm-raid0.nix; + disko-config = ../../example/mdadm-raid0.nix; extraTestScript = '' machine.succeed("test -b /dev/md/raid0"); machine.succeed("mountpoint /"); diff --git a/tests/mdadm.nix b/tests/example/mdadm.nix similarity index 71% rename from tests/mdadm.nix rename to tests/example/mdadm.nix index 3bd90377..68627d3e 100644 --- a/tests/mdadm.nix +++ b/tests/example/mdadm.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "mdadm"; - disko-config = ../example/mdadm.nix; + disko-config = ../../example/mdadm.nix; extraTestScript = '' machine.succeed("test -b /dev/md/raid1"); machine.succeed("mountpoint /"); diff --git a/tests/multi-device-no-deps.nix b/tests/example/multi-device-no-deps.nix similarity index 75% rename from tests/multi-device-no-deps.nix rename to tests/example/multi-device-no-deps.nix index 77e34775..7a7b9cc8 100644 --- a/tests/multi-device-no-deps.nix +++ b/tests/example/multi-device-no-deps.nix @@ -1,11 +1,11 @@ # this is a regression test for https://github.com/nix-community/disko/issues/52 { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "multi-device-no-deps"; - disko-config = ../example/multi-device-no-deps.nix; + disko-config = ../../example/multi-device-no-deps.nix; testBoot = false; extraTestScript = '' machine.succeed("mountpoint /mnt/a"); diff --git a/tests/negative-size.nix b/tests/example/negative-size.nix similarity index 73% rename from tests/negative-size.nix rename to tests/example/negative-size.nix index b891b081..5b5ed84e 100644 --- a/tests/negative-size.nix +++ b/tests/example/negative-size.nix @@ -1,11 +1,11 @@ # this is a regression test for https://github.com/nix-community/disko/issues/52 { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "negative-size"; - disko-config = ../example/negative-size.nix; + disko-config = ../../example/negative-size.nix; testBoot = false; extraTestScript = '' machine.succeed("mountpoint /mnt"); diff --git a/tests/non-root-zfs.nix b/tests/example/non-root-zfs.nix similarity index 94% rename from tests/non-root-zfs.nix rename to tests/example/non-root-zfs.nix index 2338483a..caf71f72 100644 --- a/tests/non-root-zfs.nix +++ b/tests/example/non-root-zfs.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "non-root-zfs"; - disko-config = ../example/non-root-zfs.nix; + disko-config = ../../example/non-root-zfs.nix; extraInstallerConfig.networking.hostId = "8425e349"; extraSystemConfig.networking.hostId = "8425e349"; postDisko = '' diff --git a/tests/simple-efi.nix b/tests/example/simple-efi.nix similarity index 63% rename from tests/simple-efi.nix rename to tests/example/simple-efi.nix index f3b90719..43d8f428 100644 --- a/tests/simple-efi.nix +++ b/tests/example/simple-efi.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "simple-efi"; - disko-config = ../example/simple-efi.nix; + disko-config = ../../example/simple-efi.nix; extraTestScript = '' machine.succeed("mountpoint /"); ''; diff --git a/tests/standalone.nix b/tests/example/stand-alone.nix similarity index 72% rename from tests/standalone.nix rename to tests/example/stand-alone.nix index 5da5cc0f..e80d444b 100644 --- a/tests/standalone.nix +++ b/tests/example/stand-alone.nix @@ -1,5 +1,5 @@ { pkgs ? import { }, ... }: (pkgs.nixos [ - ../example/stand-alone/configuration.nix + ../../example/stand-alone/configuration.nix { documentation.enable = false; } ]).config.system.build.toplevel diff --git a/tests/swap.nix b/tests/example/swap.nix similarity index 93% rename from tests/swap.nix rename to tests/example/swap.nix index 7d1678b2..f5998c0c 100644 --- a/tests/swap.nix +++ b/tests/example/swap.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "swap"; - disko-config = ../example/swap.nix; + disko-config = ../../example/swap.nix; extraTestScript = '' import json machine.succeed("mountpoint /"); diff --git a/tests/tmpfs.nix b/tests/example/tmpfs.nix similarity index 69% rename from tests/tmpfs.nix rename to tests/example/tmpfs.nix index 21cb2174..a580c26d 100644 --- a/tests/tmpfs.nix +++ b/tests/example/tmpfs.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "tmpfs"; - disko-config = ../example/tmpfs.nix; + disko-config = ../../example/tmpfs.nix; extraTestScript = '' machine.succeed("mountpoint /"); machine.succeed("mountpoint /tmp"); diff --git a/tests/with-lib.nix b/tests/example/with-lib.nix similarity index 65% rename from tests/with-lib.nix rename to tests/example/with-lib.nix index d8274b7e..dd4fd208 100644 --- a/tests/with-lib.nix +++ b/tests/example/with-lib.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "with-lib"; - disko-config = ../example/with-lib.nix; + disko-config = ../../example/with-lib.nix; extraTestScript = '' machine.succeed("mountpoint /"); ''; diff --git a/tests/zfs-over-legacy.nix b/tests/example/zfs-over-legacy.nix similarity index 76% rename from tests/zfs-over-legacy.nix rename to tests/example/zfs-over-legacy.nix index af780606..3ac17a24 100644 --- a/tests/zfs-over-legacy.nix +++ b/tests/example/zfs-over-legacy.nix @@ -1,12 +1,12 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "zfs-over-legacy"; extraInstallerConfig.networking.hostId = "8425e349"; extraSystemConfig.networking.hostId = "8425e349"; - disko-config = ../example/zfs-over-legacy.nix; + disko-config = ../../example/zfs-over-legacy.nix; extraTestScript = '' machine.succeed("test -e /zfs_fs"); machine.succeed("mountpoint /zfs_fs"); diff --git a/tests/zfs-with-vdevs.nix b/tests/example/zfs-with-vdevs.nix similarity index 92% rename from tests/zfs-with-vdevs.nix rename to tests/example/zfs-with-vdevs.nix index da301a01..29a3c477 100644 --- a/tests/zfs-with-vdevs.nix +++ b/tests/example/zfs-with-vdevs.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "zfs-with-vdevs"; - disko-config = ../example/zfs-with-vdevs.nix; + disko-config = ../../example/zfs-with-vdevs.nix; extraInstallerConfig.networking.hostId = "8425e349"; extraSystemConfig = { networking.hostId = "8425e349"; diff --git a/tests/zfs.nix b/tests/example/zfs.nix similarity index 93% rename from tests/zfs.nix rename to tests/example/zfs.nix index da2e49cd..417a7863 100644 --- a/tests/zfs.nix +++ b/tests/example/zfs.nix @@ -1,10 +1,10 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs; name = "zfs"; - disko-config = ../example/zfs.nix; + disko-config = ../../example/zfs.nix; extraInstallerConfig.networking.hostId = "8425e349"; extraSystemConfig = { networking.hostId = "8425e349"; diff --git a/tests/module.nix b/tests/module.nix index c6eb0927..419f4df6 100644 --- a/tests/module.nix +++ b/tests/module.nix @@ -1,5 +1,5 @@ { pkgs ? import { } -, diskoLib ? pkgs.callPackage ../lib { } +, diskoLib ? pkgs.callPackage ../src/disko_lib { } }: diskoLib.testLib.makeDiskoTest { inherit pkgs;