Skip to content

Commit

Permalink
Merge pull request #29 from obsidiansystems/aliao/hotfix-v6-spec-tarball
Browse files Browse the repository at this point in the history
Fix v6 thunk nixpkgs fetching
madeline-os authored Aug 4, 2022
2 parents e2c3ee9 + 7a78fc7 commit d32f4c1
Showing 14 changed files with 267 additions and 34 deletions.
8 changes: 5 additions & 3 deletions .github/workflows/haskell.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
name: github-action

on: [push, pull_request]
on: [push, pull_request]

jobs:
build:
@@ -33,7 +33,9 @@ jobs:
run: |
cabal update
cabal build --only-dependencies --enable-tests --enable-benchmarks
echo -e "#!/bin/sh\necho \"CI build\"" >> print-nixpkgs-path
chmod 755 print-nixpkgs-path
- name: Build
run: cabal build --enable-tests --enable-benchmarks all
run: env PATH=.:$PATH cabal build --enable-tests --enable-benchmarks all
- name: Run tests
run: cabal test all
run: env PATH=.:$PATH cabal test all
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
# Revision history for nix-thunk

## 0.5.0.0

* Fix a critical bug where v6 thunks can not be used to fetch non-GitHub repositories. Please update all your thunks to use the new v7 thunk spec.
Updating your thunk can be done by running `nix-thunk unpack $path; nix-thunk pack $path`.
* Building a functional `nix-thunk` _must_ be done using the included Nix derivation.

## 0.4.0.0

* The default thunk specification ("v6") now uses a pinned version of nixpkgs, rather than the magic `<nixpkgs>`, for fetching thunks. This ensures that thunks can be fetched even in an environment where `NIX_PATH` is unset.
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -21,6 +21,8 @@ nix-thunk does this by creating and managing "thunks" - directories that stand i
nix-env -f https://github.com/obsidiansystems/nix-thunk/archive/master.tar.gz -iA command
```

**WARNING**: It is _not_ possible to compile `nix-thunk` without Nix. To ensure that packed thunks are buildable even in environments where diamond paths are unavailable (specifically `<nixpkgs>`), `nix-thunk` _must_ be built with knowledge of a known-good nixpkgs, _and_ for `nix-thunk` to be able to manipulate these thunks, it must _always_ be the same version of nixpkgs.

## Command Usage

### Create a dependency
38 changes: 36 additions & 2 deletions default.nix
Original file line number Diff line number Diff line change
@@ -6,8 +6,42 @@ with pkgs.haskell.lib;

let
inherit (pkgs) lib;

pinnedNixpkgs = builtins.fetchTarball {
url = "https://github.com/NixOS/nixpkgs/archive/3aad50c30c826430b0270fcf8264c8c41b005403.tar.gz";
sha256 = "0xwqsf08sywd23x0xvw4c4ghq0l28w2ki22h0bdn766i16z9q2gr";
};
in rec {
# Override a nix-thunk Cabal package so it knows where nixpkgs is.
# This function is exported so that it can be used by downstream
# consumers of nix-thunk as a library (e.g. Obelisk).
makeRunnableNixThunk = pkg: pkgs.haskell.lib.overrideCabal pkg {
librarySystemDepends = with pkgs; [
# The correct reaction to this code is "what", followed by
# some rather choice words about its author. The answer to
# "what":
# We need a known-good nixpkgs to be included in nix-thunk's
# closure *and* we need to know its path at compile time. The
# solution is to generate a tiny, tiny shell script that just
# prints the path to that nixpkgs; Nix takes care of making
# sure it's a dependency.
(writeTextFile {
name = "print-nixpkgs-path";
text = ''
#!/bin/sh
echo "${pinnedNixpkgs}"
'';
executable = true;
destination = "/bin/print-nixpkgs-path";
})
# You can verify that the nixpkgs was included by doing
#
# $ nix-build default.nix -A command
# $ nix path-info -rsSh ./result | grep -source
#
# and seeing that the closure includes a ~100MiB nixpkgs path.
];
};

haskellPackages = pkgs.haskell.packages."${ghc}".override {
overrides = self: super: {
which = self.callCabal2nix "which" (thunkSource ./dep/which) {};
@@ -64,7 +98,7 @@ in rec {
ver = "0.9.1";
sha256 = "152lnv339fg8nacvyhxjfy2ylppc33ckb6qrgy0vzanisi8pgcvd";
} {};
nix-thunk = self.callCabal2nix "nix-thunk" (gitignoreSource ./.) {};
nix-thunk = makeRunnableNixThunk (self.callCabal2nix "nix-thunk" (gitignoreSource ./.) {});
};
};

5 changes: 1 addition & 4 deletions dep/cli-extras/thunk.nix
Original file line number Diff line number Diff line change
@@ -2,10 +2,7 @@
let fetch = { private ? false, fetchSubmodules ? false, owner, repo, rev, sha256, ... }:
if !fetchSubmodules && !private then builtins.fetchTarball {
url = "https://github.com/${owner}/${repo}/archive/${rev}.tar.gz"; inherit sha256;
} else (builtins.fetchTarball {
url = "https://github.com/NixOS/nixpkgs/archive/3aad50c30c826430b0270fcf8264c8c41b005403.tar.gz";
sha256 = "0xwqsf08sywd23x0xvw4c4ghq0l28w2ki22h0bdn766i16z9q2gr";
}).fetchFromGitHub {
} else (import "/nix/store/qjg458n31xk1l6lj26c3b871d4i4is98-source" {}).fetchFromGitHub {
inherit owner repo rev sha256 fetchSubmodules private;
};
json = builtins.fromJSON (builtins.readFile ./github.json);
5 changes: 1 addition & 4 deletions dep/cli-git/thunk.nix
Original file line number Diff line number Diff line change
@@ -2,10 +2,7 @@
let fetch = { private ? false, fetchSubmodules ? false, owner, repo, rev, sha256, ... }:
if !fetchSubmodules && !private then builtins.fetchTarball {
url = "https://github.com/${owner}/${repo}/archive/${rev}.tar.gz"; inherit sha256;
} else (builtins.fetchTarball {
url = "https://github.com/NixOS/nixpkgs/archive/3aad50c30c826430b0270fcf8264c8c41b005403.tar.gz";
sha256 = "0xwqsf08sywd23x0xvw4c4ghq0l28w2ki22h0bdn766i16z9q2gr";
}).fetchFromGitHub {
} else (import "/nix/store/qjg458n31xk1l6lj26c3b871d4i4is98-source" {}).fetchFromGitHub {
inherit owner repo rev sha256 fetchSubmodules private;
};
json = builtins.fromJSON (builtins.readFile ./github.json);
5 changes: 1 addition & 4 deletions dep/cli-nix/thunk.nix
Original file line number Diff line number Diff line change
@@ -2,10 +2,7 @@
let fetch = { private ? false, fetchSubmodules ? false, owner, repo, rev, sha256, ... }:
if !fetchSubmodules && !private then builtins.fetchTarball {
url = "https://github.com/${owner}/${repo}/archive/${rev}.tar.gz"; inherit sha256;
} else (builtins.fetchTarball {
url = "https://github.com/NixOS/nixpkgs/archive/3aad50c30c826430b0270fcf8264c8c41b005403.tar.gz";
sha256 = "0xwqsf08sywd23x0xvw4c4ghq0l28w2ki22h0bdn766i16z9q2gr";
}).fetchFromGitHub {
} else (import "/nix/store/qjg458n31xk1l6lj26c3b871d4i4is98-source" {}).fetchFromGitHub {
inherit owner repo rev sha256 fetchSubmodules private;
};
json = builtins.fromJSON (builtins.readFile ./github.json);
2 changes: 1 addition & 1 deletion dep/github/thunk.nix
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@
let fetch = { private ? false, fetchSubmodules ? false, owner, repo, rev, sha256, ... }:
if !fetchSubmodules && !private then builtins.fetchTarball {
url = "https://github.com/${owner}/${repo}/archive/${rev}.tar.gz"; inherit sha256;
} else (import <nixpkgs> {}).fetchFromGitHub {
} else (import "/nix/store/qjg458n31xk1l6lj26c3b871d4i4is98-source" {}).fetchFromGitHub {
inherit owner repo rev sha256 fetchSubmodules private;
};
json = builtins.fromJSON (builtins.readFile ./github.json);
2 changes: 1 addition & 1 deletion dep/gitignore.nix/thunk.nix
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@
let fetch = { private ? false, fetchSubmodules ? false, owner, repo, rev, sha256, ... }:
if !fetchSubmodules && !private then builtins.fetchTarball {
url = "https://github.com/${owner}/${repo}/archive/${rev}.tar.gz"; inherit sha256;
} else (import <nixpkgs> {}).fetchFromGitHub {
} else (import "/nix/store/qjg458n31xk1l6lj26c3b871d4i4is98-source" {}).fetchFromGitHub {
inherit owner repo rev sha256 fetchSubmodules private;
};
json = builtins.fromJSON (builtins.readFile ./github.json);
2 changes: 1 addition & 1 deletion dep/which/thunk.nix
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@
let fetch = { private ? false, fetchSubmodules ? false, owner, repo, rev, sha256, ... }:
if !fetchSubmodules && !private then builtins.fetchTarball {
url = "https://github.com/${owner}/${repo}/archive/${rev}.tar.gz"; inherit sha256;
} else (import <nixpkgs> {}).fetchFromGitHub {
} else (import "/nix/store/qjg458n31xk1l6lj26c3b871d4i4is98-source" {}).fetchFromGitHub {
inherit owner repo rev sha256 fetchSubmodules private;
};
json = builtins.fromJSON (builtins.readFile ./github.json);
4 changes: 3 additions & 1 deletion nix-thunk.cabal
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
cabal-version: >=1.10
name: nix-thunk
version: 0.4.0.0
version: 0.5.0.0
license: BSD3
license-file: LICENSE
copyright: Obsidian Systems LLC 2020-2022
@@ -61,6 +61,8 @@ library
, unix >=2.7.2.2 && <2.8
, which >=0.2 && <0.3
, yaml >=0.11.1.2 && <0.12
, process >=1.6.0.0 && <1.7
, template-haskell >=2.14.0.0 && <2.19

executable nix-thunk
main-is: src-bin/nix-thunk.hs
2 changes: 1 addition & 1 deletion release.nix
Original file line number Diff line number Diff line change
@@ -14,4 +14,4 @@ in
builtins.listToAttrs (map (v: lib.nameValuePair (mkName v) (import ./. {
ghc = v.compiler;
pkgs = import (./dep/ci + "/${v.nixpkgs}") {};
}).command) versions)
}).command) versions) // { tests = import ./tests.nix {}; }
75 changes: 63 additions & 12 deletions src/Nix/Thunk.hs
Original file line number Diff line number Diff line change
@@ -103,6 +103,8 @@ import System.IO.Error (isDoesNotExistError)
import System.IO.Temp
import System.Posix.Files
import qualified Text.URI as URI
import Language.Haskell.TH (Exp(LitE), Lit(StringL), runIO)
import qualified System.Process as P

--------------------------------------------------------------------------------
-- Hacks
@@ -297,6 +299,17 @@ unpackedDirName = "."
attrCacheFileName :: FilePath
attrCacheFileName = ".attr-cache"

-- | A path from which our known-good nixpkgs can be fetched.
-- @print-nixpkgs-path@ is a shell script whose only purpose is to print
-- that path. It is generated and included in the build dependencies of
-- nix-thunk by our default.nix.
pinnedNixpkgsPath :: FilePath
pinnedNixpkgsPath =
$(do
p <- fmap init . runIO $ P.readCreateProcess (P.shell "print-nixpkgs-path") ""
pure $ LitE $ StringL $ p
)

-- | Specification for how a file in a thunk version works.
data ThunkFileSpec
= ThunkFileSpec_Ptr (LBS.ByteString -> Either String ThunkPtr) -- ^ This file specifies 'ThunkPtr' data
@@ -551,8 +564,9 @@ setThunk thunkConfig target gs branch = do
-- This tool will only ever produce the newest one when it writes a thunk.
gitHubThunkSpecs :: NonEmpty ThunkSpec
gitHubThunkSpecs =
gitHubThunkSpecV6 :|
[ gitHubThunkSpecV5
gitHubThunkSpecV7 :|
[ gitHubThunkSpecV6
, gitHubThunkSpecV5
, gitHubThunkSpecV4
, gitHubThunkSpecV3
, gitHubThunkSpecV2
@@ -627,10 +641,11 @@ let fetch = { private ? false, fetchSubmodules ? false, owner, repo, rev, sha256
in fetch json
|]

-- | Specification for GitHub thunks which use a specific, pinned
-- version of nixpkgs for fetching, rather than using @<nixpkgs>@ from
-- @NIX_PATH@. The "v6" specs ensure that thunks can be fetched even
-- when @NIX_PATH@ is unset.
-- | See 'gitHubThunkSpecV7'.
--
-- __NOTE__: v6 spec thunks are broken! They import the pinned nixpkgs
-- in an incorrect way. GitHub thunks for public repositories with no
-- submodules will still work, but update as soon as possible.
gitHubThunkSpecV6 :: ThunkSpec
gitHubThunkSpecV6 = mkThunkSpec "github-v6" "github.json" parseGitHubJsonBytes [here|
# DO NOT HAND-EDIT THIS FILE
@@ -647,14 +662,30 @@ let fetch = { private ? false, fetchSubmodules ? false, owner, repo, rev, sha256
in fetch json
|]

-- | Specification for GitHub thunks which use a specific, pinned
-- version of nixpkgs for fetching, rather than using @<nixpkgs>@ from
-- @NIX_PATH@. The "v7" specs ensure that thunks can be fetched even
-- when @NIX_PATH@ is unset.
gitHubThunkSpecV7 :: ThunkSpec
gitHubThunkSpecV7 = mkThunkSpec "github-v7" "github.json" parseGitHubJsonBytes [i|# DO NOT HAND-EDIT THIS FILE
let fetch = { private ? false, fetchSubmodules ? false, owner, repo, rev, sha256, ... }:
if !fetchSubmodules && !private then builtins.fetchTarball {
url = "https://github.com/\${owner}/\${repo}/archive/\${rev}.tar.gz"; inherit sha256;
} else (import ${pinnedNixpkgsPath} {}).fetchFromGitHub {
inherit owner repo rev sha256 fetchSubmodules private;
};
json = builtins.fromJSON (builtins.readFile ./github.json);
in fetch json|]

parseGitHubJsonBytes :: LBS.ByteString -> Either String ThunkPtr
parseGitHubJsonBytes = parseJsonObject $ parseThunkPtr $ \v ->
ThunkSource_GitHub <$> parseGitHubSource v <|> ThunkSource_Git <$> parseGitSource v

gitThunkSpecs :: NonEmpty ThunkSpec
gitThunkSpecs =
gitThunkSpecV6 :|
[ gitThunkSpecV5
gitThunkSpecV7 :|
[ gitThunkSpecV6
, gitThunkSpecV5
, gitThunkSpecV4
, gitThunkSpecV3
, gitThunkSpecV2
@@ -740,10 +771,10 @@ let fetch = {url, rev, branch ? null, sha256 ? null, fetchSubmodules ? false, pr
in fetch json
|]

-- | Specification for Git thunks which use a specific, pinned version
-- of nixpkgs for fetching, rather than using @<nixpkgs>@ from
-- @NIX_PATH@. The "v6" specs ensure that thunks can be fetched even
-- when @NIX_PATH@ is unset.
-- | See 'gitThunkSpecV7'.
-- __NOTE__: v6 spec thunks are broken! They import the pinned nixpkgs
-- in an incorrect way. GitHub thunks for public repositories with no
-- submodules will still work, but update as soon as possible.
gitThunkSpecV6 :: ThunkSpec
gitThunkSpecV6 = mkThunkSpec "git-v6" "git.json" parseGitJsonBytes [here|
# DO NOT HAND-EDIT THIS FILE
@@ -765,6 +796,26 @@ let fetch = {url, rev, branch ? null, sha256 ? null, fetchSubmodules ? false, pr
in fetch json
|]

-- | Specification for Git thunks which use a specific, pinned version
-- of nixpkgs for fetching, rather than using @<nixpkgs>@ from
-- @NIX_PATH@. The "v7" specs ensure that thunks can be fetched even
-- when @NIX_PATH@ is unset.
gitThunkSpecV7 :: ThunkSpec
gitThunkSpecV7 = mkThunkSpec "git-v7" "git.json" parseGitJsonBytes [i|# DO NOT HAND-EDIT THIS FILE
let fetch = {url, rev, branch ? null, sha256 ? null, fetchSubmodules ? false, private ? false, ...}:
let realUrl = let firstChar = builtins.substring 0 1 url; in
if firstChar == "/" then /. + url
else if firstChar == "." then ./. + url
else url;
in if !fetchSubmodules && private then builtins.fetchGit {
url = realUrl; inherit rev;
\${if branch == null then null else "ref"} = branch;
} else (import ${pinnedNixpkgsPath} {}).fetchgit {
url = realUrl; inherit rev sha256;
};
json = builtins.fromJSON (builtins.readFile ./git.json);
in fetch json|]

parseGitJsonBytes :: LBS.ByteString -> Either String ThunkPtr
parseGitJsonBytes = parseJsonObject $ parseThunkPtr $ fmap ThunkSource_Git . parseGitSource

145 changes: 145 additions & 0 deletions tests.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
{
supportedSystems ? [ builtins.currentSystem ]
}:
let
nix-thunk = import ./default.nix {};
# Get a version of nixpkgs corresponding to release-22.05, which
# contains the python based tests and recursive nix.
pkgs = import (builtins.fetchTarball https://github.com/nixos/nixpkgs/archive/478f3cbc8448b5852539d785fbfe9a53304133be.tar.gz) {};
sshKeys = import (pkgs.path + /nixos/tests/ssh-keys.nix) pkgs;
make-test = import (pkgs.path + /nixos/tests/make-test-python.nix);
snakeOilPrivateKey = sshKeys.snakeOilPrivateKey.text;
snakeOilPublicKey = sshKeys.snakeOilPublicKey;
in
make-test ({...}: {
name = "nix-thunk";
nodes = {
githost = {
networking.firewall.allowedTCPPorts = [ 22 80 443 ];
services.openssh = {
enable = true;
};
environment.systemPackages = [
pkgs.git
];
users.users.root.openssh.authorizedKeys.keys = [
snakeOilPublicKey
];
};

client = {
imports = [
(pkgs.path + /nixos/modules/installer/cd-dvd/channel.nix)
];
nix.useSandbox = false;
nix.binaryCaches = [];
environment.systemPackages = [
pkgs.nix-prefetch-git
nix-thunk.command
pkgs.git
];
};
};

testScript =
let
privateKeyFile = pkgs.writeText "id_rsa" ''${snakeOilPrivateKey}'';
thunkableSample = pkgs.writeText "default.nix" ''
let pkgs = import <nixpkgs> {}; in pkgs.git
'';
invalidThunkableSample = pkgs.writeText "default.nix" ''
let pkgs = import <nixpkgs> {}; in pkgtypo.git
'';
sshConfigFile = pkgs.writeText "ssh_config" ''
Host *
StrictHostKeyChecking no
UserKnownHostsFile=/dev/null
ConnectionAttempts=1
ConnectTimeout=1
IdentityFile=~/.ssh/id_rsa
User=root
'';
in ''
start_all()
with subtest("nix-thunk is installed and git can be configured"):
client.succeed("nix-thunk --help")
client.succeed('git config --global user.email "you@example.com"')
client.succeed('git config --global user.name "Your Name"')
githost.wait_for_open_port("22")
with subtest("the client can access the server via ssh"):
client.succeed("mkdir -p ~/.ssh/")
client.succeed("cp ${privateKeyFile} ~/.ssh/id_rsa")
client.succeed("chmod 600 ~/.ssh/id_rsa")
client.wait_until_succeeds(
"ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa githost true"
)
client.succeed("cp ${sshConfigFile} ~/.ssh/config")
client.wait_until_succeeds("ssh githost true")
with subtest("a remote bare repo can be started"):
githost.succeed("mkdir -p ~/myorg/myapp.git")
githost.succeed("cd ~/myorg/myapp.git && git init --bare")
with subtest("a git project can be configured with a remote using ssh"):
client.succeed("mkdir -p ~/code/myapp")
client.succeed("cd ~/code/myapp && git init")
client.succeed("cp ${thunkableSample} ~/code/myapp/default.nix")
client.succeed("cd ~/code/myapp && git add .")
client.succeed('cd ~/code/myapp && git commit -m "Initial"')
client.succeed("cd ~/code/myapp && git remote add origin root@githost:/root/myorg/myapp.git")
with subtest("pushing code to the remote"):
client.succeed("cd ~/code/myapp && git push -u origin master")
client.succeed("cd ~/code/myapp && git status")
with subtest("nix-thunk can pack and unpack"):
client.succeed("nix-thunk pack ~/code/myapp")
client.succeed("grep -qF 'git.json' ~/code/myapp/thunk.nix")
client.succeed("grep -qF 'myorg' ~/code/myapp/git.json")
client.succeed("nix-thunk unpack ~/code/myapp")
with subtest("nix-thunk can create from ssh remote"):
client.succeed("nix-thunk pack ~/code/myapp")
client.succeed("nix-thunk create -b master root@githost:/root/myorg/myapp.git ~/code/myapp-remote")
client.succeed("diff -u ~/code/myapp/git.json ~/code/myapp-remote/git.json")
client.succeed("cmp ~/code/myapp/git.json ~/code/myapp-remote/git.json")
client.succeed("nix-thunk unpack ~/code/myapp; nix-thunk unpack ~/code/myapp-remote")
with subtest("nix-thunk can create from local directory"):
client.succeed("nix-thunk create ~/code/myapp ~/code/myapp-local")
client.succeed("nix-thunk unpack ~/code/myapp-local")
with subtest("unpacked thunks can be built"):
client.succeed("nix-build ~/code/myapp")
client.succeed("nix-build ~/code/myapp-remote")
client.succeed("nix-build ~/code/myapp-local")
with subtest("packed thunks can be built"):
client.succeed("nix-thunk -v pack ~/code/myapp-remote; nix-thunk -v pack ~/code/myapp-local")
client.succeed("nix-build ~/code/myapp-remote")
client.succeed("nix-build ~/code/myapp-local")
client.succeed("nix-thunk unpack ~/code/myapp-remote")
with subtest("nix-thunk can update from ssh remote"):
client.succeed("""
cd ~/code/myapp;
touch test-file;
git add test-file;
git commit test-file -m "add test file";
git push;
""")
client.succeed("nix-thunk pack ~/code/myapp-remote")
client.succeed("nix-thunk update ~/code/myapp-remote")
client.succeed("nix-thunk unpack ~/code/myapp-remote")
client.succeed("test -f ~/code/myapp-remote/test-file")
with subtest("nix-thunk can update from local directory"):
client.succeed("nix-thunk update ~/code/myapp-local")
client.succeed("nix-thunk unpack ~/code/myapp-local")
client.succeed("test -f ~/code/myapp-local/test-file")
'';
}) {}

0 comments on commit d32f4c1

Please sign in to comment.