Rewrite tests in haskell

This commit is contained in:
Alex David 2025-08-19 00:08:44 -07:00
commit d8b14ca5ee
9 changed files with 530 additions and 255 deletions

1
tests/.gitignore vendored
View file

@ -1 +1,2 @@
/result /result
/dist-newstyle

View file

@ -5,78 +5,36 @@
outputs = { nixpkgs, ... }: let outputs = { nixpkgs, ... }: let
pkgs = import nixpkgs { system = "x86_64-linux"; }; pkgs = import nixpkgs { system = "x86_64-linux"; };
lib = pkgs.lib; lib = pkgs.lib;
jail-nix.lib = import ../lib; haskellPackages = pkgs.haskellPackages.callCabal2nix "jail-nix-tests" ./. { };
testDependencies = [
ansi = clrCode: "[${toString clrCode}m"; (pkgs.haskellPackages.ghc.withPackages (p: haskellPackages.buildInputs))
green = ansi 32; pkgs.cabal-install
red = ansi 31; ];
reset = ansi 0;
test = {
assertStdout = actual: expected: ''
ACTUAL=$(${lib.getExe actual})
if test "$ACTUAL" != ${lib.escapeShellArg expected}; then
printf '${red}Expected "%s" to equal "%s".${reset}' "$ACTUAL" ${lib.escapeShellArg expected}
exit 1
fi
'';
assertFileDoesNotExist = fileName: ''
if test -e ${lib.escapeShellArg fileName}; then
printf '${red}Expected "%s" to not exist.${reset}' ${lib.escapeShellArg fileName}
exit 1
fi
'';
};
normalizeTestBody = testBody:
let type = builtins.typeOf testBody; in
if type == "string" then testBody else
if type == "list" then lib.concatStringsSep "\n" (map normalizeTestBody testBody) else
if lib.isDerivation testBody then lib.getExe testBody else
throw "Unknown test body type ${type}";
testCases = import ./test-cases.nix { inherit pkgs lib test jail-nix; };
in { in {
apps.x86_64-linux.runChecks = { apps.x86_64-linux.runChecks = {
type = "app"; type = "app";
program = lib.getExe (pkgs.writeShellApplication { program = lib.getExe (pkgs.writeShellApplication {
name = "run-all-jail-nix-tests"; name = "run-all-jail-nix-tests";
runtimeInputs = testDependencies;
text = '' text = ''
printf 'Running tests...\n' ${lib.pipe {
TESTS=(${lib.pipe testCases [ jail-nix = ../.;
lib.attrNames nixpkgs = nixpkgs;
(map lib.escapeShellArg) } [
(lib.mapAttrsToList (name: path: "${name}=${path}"))
(lib.concatStringsSep " ") (lib.concatStringsSep " ")
]}) (lib.toShellVar "NIX_PATH")
TEST_FAIL_LOG=$(mktemp) ]} cabal test
trap 'rm $TEST_FAIL_LOG' EXIT
for TEST_NAME in "''${TESTS[@]}"; do
(
if TEST_OUTPUT=$(nix build -L .#checks.x86_64-linux."$TEST_NAME" 2>&1); then
printf ' %s [${green}✔${reset}]\n' "$TEST_NAME"
else
printf ' %s [${red}✖${reset}]\n' "$TEST_NAME"
printf '${red}✖ %s${reset}\n%s\n\n\n' "$TEST_NAME" "$TEST_OUTPUT" >> "$TEST_FAIL_LOG"
fi
)
done
FAILURES=$(cat "$TEST_FAIL_LOG")
if [ "$FAILURES" != "" ]; then
printf '\n${red}Failing tests:${reset}\n\n%s' "$FAILURES"
exit 1
fi
''; '';
}); });
}; };
checks.x86_64-linux = lib.mapAttrs (name: testBody: devShells.x86_64-linux.default = pkgs.mkShell {
pkgs.runCommand (lib.strings.sanitizeDerivationName name) {} '' buildInputs = testDependencies ++ [
export HOME=$(pwd) (pkgs.haskell-language-server.override { dynamic = true; })
export LANG= pkgs.ghcid
${normalizeTestBody testBody} pkgs.hpack
touch $out ];
'' };
) testCases;
}; };
} }

View file

@ -0,0 +1,46 @@
cabal-version: 1.12
-- This file has been generated from package.yaml by hpack version 0.37.0.
--
-- see: https://github.com/sol/hpack
name: jail-nix-tests
version: 0.0.0
build-type: Simple
test-suite spec
type: exitcode-stdio-1.0
main-is: Spec.hs
other-modules:
CombinatorsSpec
JailSpec
TestPrelude
Paths_jail_nix_tests
hs-source-dirs:
spec
default-extensions:
DerivingStrategies
FlexibleInstances
GeneralizedNewtypeDeriving
ImportQualifiedPost
NamedFieldPuns
QuasiQuotes
ScopedTypeVariables
TupleSections
TypeFamilies
ghc-options: -Wall -Werror -Wno-name-shadowing -threaded -rtsopts "-with-rtsopts=-T -N" -j
build-depends:
aeson
, base
, bytestring
, containers
, extra
, filepath
, hspec
, hspec-core
, interpolate
, mtl
, process
, string-conversions
, temporary
default-language: Haskell2010

39
tests/package.yaml Normal file
View file

@ -0,0 +1,39 @@
name: jail-nix-tests
tests:
spec:
source-dirs: [spec]
main: Spec.hs
dependencies:
- aeson
- base
- bytestring
- containers
- extra
- filepath
- hspec
- hspec-core
- interpolate
- mtl
- process
- string-conversions
- temporary
default-extensions:
- DerivingStrategies
- FlexibleInstances
- GeneralizedNewtypeDeriving
- ImportQualifiedPost
- NamedFieldPuns
- QuasiQuotes
- ScopedTypeVariables
- TupleSections
- TypeFamilies
ghc-options:
- -Wall
- -Werror
- -Wno-name-shadowing
- -threaded
- -rtsopts
- '"-with-rtsopts=-T -N"'
- -j

View file

@ -0,0 +1,129 @@
module CombinatorsSpec (spec) where
import Data.List.Extra
import Data.String.Interpolate.Util
import System.Directory.Extra (createDirectoryIfMissing, doesFileExist)
import Test.Hspec
import TestPrelude
spec :: Spec
spec = parallel $ inTestM $ do
describe "add-cleanup" $ do
it "runs arbitrary logic after the jail exits" $ do
tmpDir <- getTestDir
[i|
jail "test" (sh "cat /foo-in-jail") (c: [
(c.ro-bind "foo" "/foo-in-jail")
(c.add-runtime "echo hello > foo")
(c.add-cleanup "rm foo")
])
|]
`shouldOutput` "hello\n"
liftIO $ doesFileExist (tmpDir </> "foo") `shouldReturn` False
describe "add-path" $ do
it "prepends the passed path to $PATH" $ do
output <-
runNixDrv
[i|
jail "test" (sh ''echo "$PATH"'') (c: [
(c.add-path "/some/new/path")
])
|]
liftIO $ output `shouldStartWith` "/some/new/path:"
describe "add-pkg-deps" $ do
it "sets the path" $ do
helloPath <- evalNixExpr "toString pkgs.hello"
output <-
runNixDrv
[i|
jail "test" (sh ''echo "$PATH"'') (c: [
(c.add-pkg-deps [ pkgs.hello ])
])
|]
liftIO $ output `shouldStartWith` helloPath <> "/bin:"
describe "bind-pkg" $ do
it "binds the passed nix package at the specified location" $ do
[i|
jail "test" (sh "cat /foo") (c: [
(c.bind-pkg "/foo" (pkgs.writeText "foo" "bar\n"))
])
|]
`shouldOutput` "bar\n"
describe "fwd-env" $ do
it "forwards the passed environment variable" $ do
withEnv "MY_ENV_VAR" "secret" $ do
[i|
jail "test" (sh ''echo "''${MY_ENV_VAR-unset}"'') (c: [
(c.fwd-env "MY_ENV_VAR")
])
|]
`shouldOutput` "secret\n"
describe "readonly-from-path-var" $ do
forM_ [":", " "] $ \separator ->
describe ("with separator \"" <> separator <> "\"") $ do
it "binds all paths in the specified environment variable as read only" $ do
tmpDir <- getTestDir
let separator = " "
let mkTestPath path = liftIO $ createDirectoryIfMissing True (tmpDir </> path) >> writeFile (tmpDir </> path </> "file") ""
mkTestPath "some/path"
mkTestPath "some/other/path"
mkTestPath "some/not/in/var"
let testPaths =
intercalate
separator
$ map
(tmpDir </>)
[ "some/path",
"some/non/existant/path",
"some/other/path",
"some/other/non/existant/path"
]
withEnv "TEST_PATHS" testPaths $ do
[i|
jail "test" (sh "${lib.getExe pkgs.tree} ${#{toNixString (tmpDir </> "some")}}") (c: [
(c.readonly-paths-from-var "TEST_PATHS" #{toNixString separator})
])
|]
`shouldOutput` unindent
[i|
#{tmpDir </> "some"}
|-- other
| `-- path
| `-- file
`-- path
`-- file
4 directories, 2 files
|]
it "doesn't require the var to be set" $ do
void $
runNixDrv
[i|
jail "test" (sh "true") (c: [
(c.readonly-paths-from-var "THIS_VAR_IS_NOT_SET" #{toNixString separator})
])
|]
describe "set-argv" $ do
it "overrides argv passed to the jailed program" $ do
[i|
jail "test" (sh "printf '1=%s,2=%s,3=%s' \\"$@\\"") (c: [
(c.set-argv [ "foo" "bar baz" "foo>'bar'" ])
])
|]
`shouldOutput` "1=foo,2=bar baz,3=foo>'bar'"
describe "set-env" $ do
it "allows setting arbitrary environment variables" $ do
[i|
jail "test" (sh ''echo -n "''${MY_ENV_VAR-unset}"'') (c: [
(c.set-env "MY_ENV_VAR" "some-value")
])
|]
`shouldOutput` "some-value"

153
tests/spec/JailSpec.hs Normal file
View file

@ -0,0 +1,153 @@
module JailSpec (spec) where
import Test.Hspec
import TestPrelude
spec :: Spec
spec = parallel $ inTestM $ do
it "runs the wrapped program" $ do
"jail \"hello\" pkgs.hello []"
`shouldOutput` "Hello, world!\n"
it "can be imported using callPackage" $ do
[i|
let
jail-nix-without-flake = pkgs.callPackage <jail-nix> {};
jail-without-flake = jail-nix-without-flake.init pkgs;
in
jail-without-flake "hello" pkgs.hello []
|]
`shouldOutput` "Hello, world!\n"
it "sets a sane default path" $ do
defaultPath <- evalNixExpr [i| "${pkgs.coreutils}/bin" |]
[i| jail "test" (sh ''echo -n "$PATH"'') [] |]
`shouldOutput` defaultPath
it "it does not forward environment variables by default" $ do
withEnv "MY_ENV_VAR" "secret" $ do
[i| jail "test" (sh ''echo -n "''${MY_ENV_VAR-unset}"'') [] |]
`shouldOutput` "unset"
describe "advanced configuration" $ do
it "allows overriding base permissions shared across all jails" $ do
tmpDir <- getTestDir
let secretPath = tmpDir </> "secret"
liftIO $ writeFile secretPath "hunter2"
[i|
let
jail' = jail-nix.extend {
inherit pkgs;
basePermissions = c: [
(c.readonly #{toNixString secretPath})
(c.add-pkg-deps [ pkgs.coreutils ])
(c.readonly "/nix/store")
];
};
in
jail' "hello" (sh #{toNixString ("cat " <> secretPath)}) []
|]
`shouldOutput` "hunter2"
it "allows overriding the bubblewrap package" $ do
[i|
let
jail' = jail-nix.extend {
inherit pkgs;
bubblewrapPackage = pkgs.writeShellApplication {
name = "my-bubblewrap";
text = ''
# A version of bwrap that just prints its arguments instead of
# actually running a jail.
echo -n "$@"
'';
};
basePermissions = null;
};
in
jail' "test" "some-entrypoint" []
|]
`shouldOutput` "--unshare-user --unshare-ipc --unshare-pid --unshare-net --unshare-uts --unshare-cgroup --new-session -- some-entrypoint"
describe "sandboxed nix store" $ do
let checkRuntimeDepIsExposed :: Bool -> String -> TestM ()
checkRuntimeDepIsExposed shouldBeExposed jailExpr = do
output <-
runNixDrv
[i|
let
buildtime = pkgs.runCommand "buildtime" {} "touch $out";
runtime = pkgs.runCommand "runtime" {} "touch $out";
dep = pkgs.runCommand "dep" {} ''
echo ${buildtime}
echo ${runtime} > $out
'';
entryScript = sh ''
if [ -e "$1" ]; then
echo "file exists"
else
echo "file doesn't exist"
fi
'';
jailed = (#{jailExpr}) entryScript dep;
in
sh ''
echo "buildtime: $(${lib.getExe jailed} ${buildtime})"
echo "runtime: $(${lib.getExe jailed} ${runtime})"
''
|]
liftIO $
output
`shouldBe` unlines
[ "buildtime: file doesn't exist",
if shouldBeExposed
then "runtime: file exists"
else "runtime: file doesn't exist"
]
it "does not expose unrelated paths in the nix store" $ do
checkRuntimeDepIsExposed
False
[i| entryScript: _dep:
jail "test" entryScript []
|]
it "exposes runtime closures of deps in the entry" $ do
checkRuntimeDepIsExposed
True
[i| entryScript: dep:
jail
"test"
(sh ''
# ${dep}
${lib.getExe entryScript} "$@"
'')
[]
|]
it "exposes runtime closures of deps in add-pkg-deps" $ do
checkRuntimeDepIsExposed
True
[i| entryScript: dep:
jail "test" entryScript (c: [
(c.add-pkg-deps [ dep ])
])
|]
it "exposes runtime closures of deps in argv" $ do
checkRuntimeDepIsExposed
True
[i| entryScript: dep:
jail "test" entryScript (c: [
(c.set-argv [ (c.noescape ''"$1"'') dep ])
])
|]
it "exposes runtime closures of deps in write-text string contexts" $ do
checkRuntimeDepIsExposed
True
[i| entryScript: dep:
jail "test" entryScript (c: [
(c.write-text "/write/text/with/runtime/dep" "${dep}")
])
|]

1
tests/spec/Spec.hs Normal file
View file

@ -0,0 +1 @@
{-# OPTIONS_GHC -F -pgmF hspec-discover #-}

141
tests/spec/TestPrelude.hs Normal file
View file

@ -0,0 +1,141 @@
module TestPrelude
( (</>),
TestM,
buildNixDrvPath,
evalNixExpr,
forM,
forM_,
getTestDir,
i,
inTestM,
liftIO,
runNixDrv,
shouldOutput,
toNixString,
void,
withEnv,
)
where
import Control.Monad
import Control.Monad.Extra (mapMaybeM)
import Control.Monad.IO.Class (liftIO)
import Control.Monad.Reader (MonadIO, MonadReader, ReaderT, asks, local, runReaderT)
import Data.Aeson qualified as Aeson
import Data.Function ((&))
import Data.List.Extra
import Data.Map qualified as Map
import Data.Maybe (fromJust)
import Data.String.Conversions
import Data.String.Interpolate (i)
import System.Environment (lookupEnv)
import System.FilePath ((</>))
import System.IO.Temp (withSystemTempDirectory)
import System.Process qualified as Process
import Test.Hspec
import Test.Hspec.Core.Spec qualified
data Context = Context
{ tmpDir :: FilePath,
env :: Map.Map String String
}
newtype TestM a = TestM {testMReaderT :: ReaderT Context IO a}
deriving newtype
( Functor,
Applicative,
Monad,
MonadIO,
MonadReader Context
)
instance Example (TestM ()) where
type Arg (TestM ()) = Context
evaluateExample test _ hook _ = do
hook $ runReaderT $ testMReaderT test
pure $ Test.Hspec.Core.Spec.Result "" Test.Hspec.Core.Spec.Success
getTestDir :: TestM FilePath
getTestDir = asks tmpDir
inTestM :: SpecWith Context -> SpecWith ()
inTestM = aroundWith $ \action () -> do
withSystemTempDirectory "jail-nix-test" $ \tmpDir -> do
baseEnv <-
Map.fromList
<$> mapMaybeM
(\(name, getVal) -> ((name,) <$>) <$> getVal)
[ ("HOME", lookupEnv "HOME"),
("LANG", pure $ Just "en_US.UTF-8"),
("PATH", lookupEnv "PATH")
]
action $
Context
{ tmpDir,
env = baseEnv
}
toNixString :: String -> String
toNixString s =
s
& replace "\\" "\\\\"
& replace "\"" "\\\""
& replace "$" "\\$"
& ("\"" <>)
& (<> "\"")
withEnv :: String -> String -> TestM () -> TestM ()
withEnv name value = local $ \ctx -> ctx {env = Map.insert name value (env ctx)}
runNixDrv :: String -> TestM String
runNixDrv nixDrvExpr = do
(exe :: String, drvPath :: FilePath) <-
evalNixExpr
[i| let d = #{nixDrvExpr}; in [(lib.getExe d) d.drvPath] |]
void $ buildNixDrvPath drvPath
env <- asks env
let p = (Process.proc exe []) {Process.env = Just $ Map.toList env}
liftIO $ Process.readCreateProcess p ""
buildNixDrvPath :: FilePath -> TestM FilePath
buildNixDrvPath drvPath = do
liftIO $
Process.readProcess
"nix"
[ "build",
"--no-link",
"--print-out-paths",
drvPath <> "^*"
]
""
evalNixExpr :: (Aeson.FromJSON a) => String -> TestM a
evalNixExpr nixExpr = do
tmpDir <- getTestDir
liftIO $
writeFile
(tmpDir </> "test.nix")
[i|
let
pkgs = import <nixpkgs> {};
lib = pkgs.lib;
sh = script: (pkgs.writeShellApplication { name = "test-script"; text = script; });
jail-nix = import <jail-nix> {};
jail = jail-nix.extend { inherit pkgs; };
in #{nixExpr}
|]
liftIO $
fromJust . Aeson.decode' . cs
<$> Process.readProcess
"nix"
[ "eval",
"--json",
"--file",
tmpDir </> "test.nix"
]
""
shouldOutput :: String -> String -> TestM ()
shouldOutput nixDrvExpr expectedOutput = do
out <- runNixDrv nixDrvExpr
liftIO $ out `shouldBe` expectedOutput

View file

@ -1,193 +0,0 @@
{ pkgs, lib, test, jail-nix }: with test; let
sh = script: (pkgs.writeShellApplication { name = "test-script"; text = script; });
defaultPath = "${pkgs.coreutils}/bin";
jail = jail-nix.lib.extend { inherit pkgs; };
in {
"simple jail works" = assertStdout
(jail "hello" pkgs.hello [])
"Hello, world!";
"can be imported using callPackage" = let
jail-nix-without-flake = pkgs.callPackage ../. {};
jail-without-flake = jail-nix-without-flake.init pkgs;
in assertStdout (jail-without-flake "hello" pkgs.hello []) "Hello, world!";
"it only exposes the nix store paths that are part of the runtime closure" = let
mkDep = name: let
buildtime = pkgs.runCommand "${name}-buildtime" {} "touch $out";
runtime = pkgs.runCommand "${name}-runtime" {} "touch $out";
parent = pkgs.runCommand "entry-dep" {} ''
echo ${buildtime}
echo ${runtime} > $out
'';
in { inherit buildtime runtime parent; };
deps = {
add-pkg-deps = mkDep "add-pkg-deps";
argv = mkDep "argv";
entry = mkDep "entry";
unrelated = sh "echo this dep is unrelated";
write-text = mkDep "write-text";
};
checkExists = jail
"check-exists"
(sh ''
# ${deps.entry.parent}
if [ -e "$1" ]; then
echo "$1 file exists"
else
echo "$1 file doesn't exist"
fi
'')
(c: [
(c.add-pkg-deps [ deps.add-pkg-deps.parent ])
(c.set-argv [ (c.noescape "\"$1\"") deps.argv.parent ])
(c.write-text "/write/text/with/runtime/dep" "${deps.write-text.parent}")
]);
in
# Dependencies that should be in the jail
map (dep: assertStdout (sh "${lib.getExe checkExists} ${dep}") "${dep} file exists") [
deps.add-pkg-deps.runtime
deps.argv.runtime
deps.entry.runtime
deps.write-text.runtime
]
++
# Dependencies that shouldn't be in the jail
map (dep: assertStdout (sh "${lib.getExe checkExists} ${dep}") "${dep} file doesn't exist") [
deps.add-pkg-deps.buildtime
deps.argv.buildtime
deps.entry.buildtime
deps.unrelated
deps.write-text.buildtime
];
"it allows overriding base permissions shared across all jails" = let
jail' = jail-nix.lib.extend {
inherit pkgs;
basePermissions = c: [
(c.readonly "/build/secret")
(c.add-pkg-deps [ pkgs.coreutils ])
(c.readonly "/nix/store")
];
};
in [
"echo hunter2 > /build/secret"
(assertStdout (jail' "hello" (sh "cat /build/secret") []) "hunter2")
];
"it allows overriding the bubblewrap package" = let
jail' = jail-nix.lib.extend {
inherit pkgs;
bubblewrapPackage = pkgs.writeShellApplication {
name = "my-bubblewrap";
text = ''
# A version of bwrap that just prints its arguments instead of
# actually running a jail.
echo "$@"
'';
};
basePermissions = null;
};
in assertStdout
(jail' "test" "some-entrypoint" [])
"--unshare-user --unshare-ipc --unshare-pid --unshare-net --unshare-uts --unshare-cgroup --new-session -- some-entrypoint";
"it sets a sane default path" = assertStdout
(jail "test" (sh ''echo "$PATH"'') [])
defaultPath;
"it does not forward environment variables by default" = [
"export MY_ENV_VAR=secret"
(assertStdout
(jail "test" (sh ''echo "''${MY_ENV_VAR-unset}"'') [])
"unset"
)
];
"combinators/add-cleanup" = [
(assertStdout
(jail "test" (sh "cat /foo-in-jail") (c: [
(c.ro-bind "foo" "/foo-in-jail")
(c.add-runtime "echo hello > foo")
(c.add-cleanup "rm foo")
]))
"hello"
)
(assertFileDoesNotExist "foo")
];
"combinators/add-path" = assertStdout
(jail "test" (sh ''echo "$PATH"'') (c: [
(c.add-path "/some/new/path")
]))
"/some/new/path:${defaultPath}";
"combinators/add-pkg-deps" = assertStdout
(jail "test" (sh ''echo "$PATH"'') (c: [
(c.add-pkg-deps [ pkgs.hello ])
]))
"${pkgs.hello}/bin:${defaultPath}";
"combinators/bind-pkg" = assertStdout
(jail "test" (sh "cat /foo") (c: [
(c.bind-pkg "/foo" (pkgs.writeText "foo" "bar"))
]))
"bar";
"combinators/fwd-env" = [
"export MY_ENV_VAR=secret"
(assertStdout
(jail "test" (sh ''echo "''${MY_ENV_VAR-unset}"'') (c: [
(c.fwd-env "MY_ENV_VAR")
]))
"secret"
)
];
"combinators/readonly-from-path-var" = let
separator = " ";
mkTestPath = path: "mkdir -p /build/${path} && touch /build/${path}/file";
in [
(mkTestPath "some/path")
(mkTestPath "some/other/path")
(mkTestPath "some/not/in/var")
"export TEST_PATHS=${lib.escapeShellArg (lib.concatStringsSep separator [
"some/path"
"some/non/existant/path"
"some/other/path"
"some/other/non/existant/path"
])}"
(assertStdout
(jail "test" (sh "${lib.getExe pkgs.tree} /build/some") (c: [
(c.readonly-paths-from-var "TEST_PATHS" separator)
]))
(lib.trim ''
/build/some
|-- other
| `-- path
| `-- file
`-- path
`-- file
4 directories, 2 files
'')
)
];
"combinators/readonly-from-path-var doesn't require the var to be set" =
jail "test" (sh "true") (c: [
(c.readonly-paths-from-var "THIS_VAR_IS_NOT_SET" ":")
]);
"combinators/set-argv" = assertStdout
(jail "test" (sh "printf '1=%s,2=%s,3=%s' \"$@\"") (c: [
(c.set-argv [ "foo" "bar baz" "foo>'bar'" ])
]))
"1=foo,2=bar baz,3=foo>'bar'";
"combinators/set-env" = assertStdout
(jail "test" (sh ''echo "''${MY_ENV_VAR-unset}"'') (c: [
(c.set-env "MY_ENV_VAR" "some-value")
]))
"some-value";
}