test: unify the test harness, discover suites, fix a real hermeticity leak #134

Merged
rootiest merged 10 commits from test/test-harness-overhaul into main 2026-09-08 05:26:30 +00:00
3 changed files with 212 additions and 209 deletions
Showing only changes of commit 0a9cfebcad - Show all commits
-114
View File
@@ -1,114 +0,0 @@
# Copyright (C) 2026 Rootiest
# SPDX-License-Identifier: AGPL-3.0-or-later
#
# Functional checks for foundational config behavior. Sourced inside a
# fully-loaded, sandboxed interactive fish session by tests/run-tests.fish
# -- see that file for the sandbox setup. Each test_* function returns 0
# on pass, non-zero on fail; functional_test_main collects and runs them.
function test_xdg_defaults
test -n "$XDG_CONFIG_HOME" -a -n "$XDG_CACHE_HOME" \
-a -n "$XDG_DATA_HOME" -a -n "$XDG_STATE_HOME"
end
function test_path_additions
contains -- "$HOME/.local/bin" $PATH
end
function test_cdpath
contains -- "$HOME/projects" $CDPATH
end
function test_vi_key_bindings
test "$fish_key_bindings" = fish_vi_key_bindings
end
function test_abbreviations_loaded
abbr -q n
end
function test_core_functions_defined
for f in cat logs config-help fish-deps check_fish_deps config-settings
if not functions -q $f
echo " missing function: $f"
return 1
end
end
end
function test_exit_rewired
functions -q exit
and functions exit | string match -q '*smart_exit*'
end
function test_op_registry_lookup
functions -q __fish_config_op_registry_lookup
or return 1
set -l tags (__fish_config_op_registry_lookup config cdpath)
test $status -eq 0 -a (count $tags) -gt 0
end
function test_privacy_variables
test "$DO_NOT_TRACK" = "1" -a "$DISABLE_TELEMETRY" = "1"
end
function test_privacy_op_registry_lookup
functions -q __fish_config_op_registry_lookup
or return 1
set -l tags (__fish_config_op_registry_lookup config privacy)
test $status -eq 0 -a "$tags" = "overrides/privacy"
end
function test_op_enabled_fail_open
# An identity/site pair with no registry entry must resolve to
# enabled -- the documented fail-open default.
__fish_config_op_enabled __fish_config_test_never_registered somesite
end
function test_greeting_function_defined
functions -q fish_greeting
end
function test_agents_vault_defined
for f in agents-vault _agents_vault_dir _agents_repo_slug \
_agents_repo_ensure_symlink _agents_repo_sync \
_agents_repo_install_tools
if not functions -q $f
echo " missing function: $f"
return 1
end
end
end
function test_wrappers_call_agents_vault
functions -q claude; or return 1
functions claude | string match -q '*agents-vault*'; or return 1
functions -q agy; or return 1
functions agy | string match -q '*agents-vault*'
end
function test_vault_dir_honors_override
set -l saved
set -q __fish_agent_vault_dir; and set saved $__fish_agent_vault_dir
set -g __fish_agent_vault_dir /tmp/vault-override-check
set -l got (_agents_vault_dir)
set -e __fish_agent_vault_dir
test (count $saved) -gt 0; and set -g __fish_agent_vault_dir $saved
test "$got" = /tmp/vault-override-check
end
function functional_test_main
set -l names (functions -a | string match 'test_*' | sort)
set -l failed 0
for name in $names
if $name
echo " PASS $name"
else
echo " FAIL $name"
set failed (math $failed + 1)
end
end
echo ""
echo (math (count $names) - $failed)"/"(count $names)" passed"
return $failed
end
+141 -86
View File
@@ -4,15 +4,19 @@
# #
# CI test runner for this fish configuration. # CI test runner for this fish configuration.
# 1. Syntax-lints every tracked .fish file (fish -n). # 1. Syntax-lints every tracked .fish file (fish -n).
# 2. Copies the config-relevant files into a throwaway sandbox (never # 2. Discovers tests/test-*.fish and reads the mode each suite declares
# the live checkout -- this repo doubles as a real ~/.config/fish, # in its own header (`# MODE: isolated` or `# MODE: in-session`).
# so a symlinked sandbox would let universal-variable writes like # 3. Runs each isolated suite as its own --no-config fish process with
# first-run's escape into the real, gitignored fish_variables file) # throwaway XDG dirs.
# and loads it as an isolated interactive session. # 4. Sources every in-session suite into ONE sandboxed interactive fish
# 3. Runs the functional checks in tests/functional.fish inside that # session built from a copy of this config (never the live checkout
# loaded session. # -- this repo doubles as a real ~/.config/fish, so a symlinked
# 4. Runs tests/test-agents-vault.fish as its own process; that suite # sandbox would let universal-variable writes like first-run's escape
# builds its own throwaway repos and needs no loaded config. # into the real, gitignored fish_variables file).
# 5. Sums the per-suite assertion counts and reports a total.
#
# Assertions come from tests/lib.fish (section/check/report); suites are
# never special-cased here by name.
# #
# Usage: fish tests/run-tests.fish # Usage: fish tests/run-tests.fish
@@ -42,104 +46,155 @@ if test $lint_failed -ne 0
set overall_failed 1 set overall_failed 1
end end
# ---- Phase 2: isolated load + functional checks -------------------------- # ---- Phase 2: discover suites --------------------------------------------
echo "" # Mode is declared by the suite, not by this driver. Detection is
echo "== Sandboxed load + functional checks ==" # case-insensitive so a near-miss like "# Mode: in-session" is caught rather
# than silently read as "no marker"; the comparison is exact so only the two
set -l sandbox (mktemp -d) # real spellings are accepted. Absence means isolated, the safe default -- a
set -l sandbox_cfg $sandbox/xdgcfg/fish # suite that forgets the marker gets its own clean process instead of being
mkdir -p $sandbox_cfg # injected into a loaded session, and no typo can ever promote a suite into
# path-setup only adds directories that already exist (fish_add_path is a # in-session. Duplicate markers resolve first-match-wins.
# no-op on missing paths), so give it $HOME/.local/bin to find. set -l isolated_suites
mkdir -p $sandbox/home/.local/bin set -l session_suites
for f in (find $script_dir -name 'test-*.fish' | sort)
# Every utility below goes through `command`. This driver runs under the set -l decl (grep -im1 '^# *mode:' $f)
# very config it tests, which shadows these: `cp` is an alias for `cp -i`, if test -z "$decl"
# `rm` is a trash wrapper, `cat` resolves to bat. Only `cp` is an actual set -a isolated_suites $f
# hazard today -- `-i` on a non-empty destination reads EOF in a else if test "$decl" = "# MODE: in-session"
# non-interactive runner and SILENTLY SKIPS the copy while exiting 0, set -a session_suites $f
# which would leave the sandbox missing config files and report success. else if test "$decl" = "# MODE: isolated"
# `rm -rf` and `cat` were measured and behave correctly as-is (the rm set -a isolated_suites $f
# wrapper bails to `command rm` on any non-recursive flag, so -rf really else
# deletes and does not trash). Prefixed anyway: a test runner must not echo " FAIL "(basename $f)": unrecognized mode declaration: $decl" >&2
# depend on the configuration under test. set overall_failed 1
command cp $repo_root/config.fish $sandbox_cfg/ end
test -f $repo_root/fish_plugins
and command cp $repo_root/fish_plugins $sandbox_cfg/
for d in functions conf.d completions integrations themes data
test -d $repo_root/$d
and command cp -r $repo_root/$d $sandbox_cfg/
end end
set -l err_file (mktemp) set -l counts (mktemp)
env -i \
# ---- Phase 3: isolated suites --------------------------------------------
# HOME is deliberately NOT overridden here. Read this before "improving" it.
#
# Overriding XDG_CONFIG_HOME/XDG_DATA_HOME plus --no-config is what makes these
# runs isolated: the universal-variable file fish can reach is a fresh empty
# one, and no config.fish/conf.d is loaded. Without that, an "isolated" suite
# runs against the user's LIVE config and real universal variables -- this repo
# doubles as a real ~/.config/fish -- so a guard test doing
# `set -e __fish_config_op_logging` would erase a real universal variable out of
# the running shell. Measured: $__fish_config_op_registry_keys has 65 entries
# under a plain `fish`, 0 under `fish --no-config`.
#
# `env -i HOME=$sandbox` was tried and REJECTED. It looks strictly more
# hermetic, but test-agents-vault.fish's hermeticity floor snapshots the real
# $HOME/.claude/memory and $HOME/.gemini/antigravity-cli and asserts them
# unchanged at the end. Point HOME at a sandbox and both snapshots read "absent"
# before and after: the assertions still pass while asserting nothing. A change
# that turns a real assertion into a tautology without turning anything red is
# the worst failure mode a test harness has. Keeping HOME real is what keeps
# those two assertions biting.
#
# Overriding XDG_DATA_HOME is a hermeticity gain on top of the isolation:
# _agents_vault_dir falls back to
# ${XDG_DATA_HOME:-$HOME/.local/share}/agent-vault, so a vault path that no test
# overrode lands in a temp dir instead of the user's real ~/.local/share.
for suite in $isolated_suites
echo ""
echo "== "(string replace $repo_root/ '' $suite)" =="
set -l xdg (mktemp -d)
env XDG_CONFIG_HOME=$xdg/cfg XDG_DATA_HOME=$xdg/data \
FISH_CONFIG_TEST_ROOT=$repo_root FISH_CONFIG_TEST_COUNTS=$counts \
fish --no-config $suite
if test $status -ne 0
set overall_failed 1
end
command rm -rf $xdg
end
# ---- Phase 4: in-session suites ------------------------------------------
# All in-session suites share ONE sandboxed interactive session: building it
# (copying the config, starting fish -i) is the expensive part.
#
# The config is COPIED, never symlinked. This repo doubles as a real
# ~/.config/fish, so a symlinked sandbox would let universal-variable writes
# like first-run's escape into the real, gitignored fish_variables file.
if test (count $session_suites) -gt 0
echo ""
echo "== Sandboxed load + session checks =="
set -l sandbox (mktemp -d)
set -l sandbox_cfg $sandbox/xdgcfg/fish
mkdir -p $sandbox_cfg
# path-setup only adds directories that already exist (fish_add_path is a
# no-op on missing paths), so give it $HOME/.local/bin to find.
mkdir -p $sandbox/home/.local/bin
# Every utility below goes through `command`. This driver runs under the
# very config it tests, which shadows these: `cp` is an alias for `cp -i`,
# `rm` is a trash wrapper, `cat` resolves to bat. Only `cp` is an actual
# hazard today -- `-i` on a non-empty destination reads EOF in a
# non-interactive runner and SILENTLY SKIPS the copy while exiting 0,
# which would leave the sandbox missing config files and report success.
# `rm -rf` and `cat` were measured and behave correctly as-is (the rm
# wrapper bails to `command rm` on any non-recursive flag, so -rf really
# deletes and does not trash). Prefixed anyway: a test runner must not
# depend on the configuration under test.
command cp $repo_root/config.fish $sandbox_cfg/
test -f $repo_root/fish_plugins
and command cp $repo_root/fish_plugins $sandbox_cfg/
for d in functions conf.d completions integrations themes data
test -d $repo_root/$d
and command cp -r $repo_root/$d $sandbox_cfg/
end
set -l srcs
for s in $session_suites
set -a srcs "source $s;"
end
set -l err_file (mktemp)
env -i \
HOME=$sandbox/home \ HOME=$sandbox/home \
XDG_CONFIG_HOME=$sandbox/xdgcfg \ XDG_CONFIG_HOME=$sandbox/xdgcfg \
PATH="$PATH" \ PATH="$PATH" \
TERM=xterm \ TERM=xterm \
__fish_config_op_autoexec=off \ __fish_config_op_autoexec=off \
fish -i -c "source $repo_root/tests/functional.fish; functional_test_main" \ FISH_CONFIG_TEST_ROOT=$repo_root \
FISH_CONFIG_TEST_COUNTS=$counts \
fish -i -c "source $repo_root/tests/lib.fish; $srcs report" \
2>$err_file 2>$err_file
set -l functional_status $status set -l session_status $status
set -l stderr_out (command cat $err_file) set -l stderr_out (command cat $err_file)
command rm -rf $sandbox $err_file command rm -rf $sandbox $err_file
if test -n "$stderr_out" if test -n "$stderr_out"
# Diagnostic only, not a gate: on machines with vendor fish configs # Diagnostic only, not a gate: on machines with vendor fish configs
# (e.g. CachyOS's cachyos-fish-config, which this repo's config.fish # (e.g. CachyOS's cachyos-fish-config, which this repo's config.fish
# sources when present) unrelated vendor warnings can land here. Real # sources when present) unrelated vendor warnings can land here. Real
# breakage in this repo's own code is caught by the assertions below. # breakage in this repo's own code is caught by the assertions.
echo " Session stderr output (informational):" echo " Session stderr output (informational):"
printf '%s\n' $stderr_out printf '%s\n' $stderr_out
end end
if test $functional_status -ne 0 if test $session_status -ne 0
set overall_failed 1 set overall_failed 1
end
end end
# ---- Phase 3: hermetic vault helper tests -------------------------------- # ---- Phase 5: totals -----------------------------------------------------
# Run as its own fish process rather than inside the sandboxed session: set -l total_run 0
# the suite builds its own throwaway git repos and binds the vault, claude set -l total_failed 0
# and agy roots to them, so it needs no loaded config and must never see for line in (command cat $counts)
# the real ~/.claude. set -l parts (string split ' ' -- $line)
# set total_run (math $total_run + $parts[1])
# HOME is deliberately NOT overridden here. Read this before "improving" it. set total_failed (math $total_failed + $parts[2])
# end
# Overriding XDG_CONFIG_HOME/XDG_DATA_HOME plus --no-config is what makes command rm -f $counts
# this run isolated: the universal-variable file fish can reach is a fresh
# empty one, and no config.fish/conf.d is loaded. Without that, an
# "isolated" suite runs against the user's LIVE config and real universal
# variables -- this repo doubles as a real ~/.config/fish -- so a test
# doing `set -e __fish_config_op_logging` would erase a real universal
# variable out of the running shell. Measured:
# $__fish_config_op_registry_keys has 65 entries under a plain `fish`, 0
# under `fish --no-config`.
#
# `env -i HOME=$sandbox` was tried and REJECTED. It looks strictly more
# hermetic, but test-agents-vault.fish's hermeticity floor snapshots the
# real $HOME/.claude/memory and $HOME/.gemini/antigravity-cli and asserts
# them unchanged at the end. Point HOME at a sandbox and both snapshots
# read "absent" before and after: the assertions still pass while
# asserting nothing. A change that turns a real assertion into a tautology
# without turning anything red is the worst failure mode a test harness
# has. Keeping HOME real is what keeps those two assertions biting.
#
# Overriding XDG_DATA_HOME is a hermeticity gain on top of the isolation:
# _agents_vault_dir falls back to
# ${XDG_DATA_HOME:-$HOME/.local/share}/agent-vault, so a vault path that
# no test overrode lands in a temp dir instead of the user's real
# ~/.local/share/agent-vault.
echo "" echo ""
echo "== Vault helper tests ==" echo "TOTAL: "(math $total_run - $total_failed)"/$total_run assertions passed"
set -l vault_xdg (mktemp -d) if test $total_failed -ne 0
env XDG_CONFIG_HOME=$vault_xdg/cfg XDG_DATA_HOME=$vault_xdg/data \
fish --no-config $repo_root/tests/test-agents-vault.fish
if test $status -ne 0
set overall_failed 1 set overall_failed 1
end end
# `command rm`, not bare `rm`: this driver runs under the config it tests,
# which shadows rm/cp/cat. See the Phase 2 note for the full reasoning.
command rm -rf $vault_xdg
exit $overall_failed exit $overall_failed
+62
View File
@@ -0,0 +1,62 @@
# Copyright (C) 2026 Rootiest
# SPDX-License-Identifier: AGPL-3.0-or-later
#
# MODE: in-session
#
# Functional checks for foundational config behavior. Sourced by
# tests/run-tests.fish into a fully-loaded, sandboxed interactive fish
# session -- see that file for the sandbox setup. Assertions here are the
# ones that genuinely need a loaded config; anything testable against a
# single function belongs in an isolated suite instead.
section "session: environment"
check "XDG vars are all set" true (test -n "$XDG_CONFIG_HOME" -a -n "$XDG_CACHE_HOME" -a -n "$XDG_DATA_HOME" -a -n "$XDG_STATE_HOME"; and echo true; or echo false)
check "PATH includes ~/.local/bin" true (contains -- "$HOME/.local/bin" $PATH; and echo true; or echo false)
check "CDPATH includes ~/projects" true (contains -- "$HOME/projects" $CDPATH; and echo true; or echo false)
check "vi key bindings active" fish_vi_key_bindings "$fish_key_bindings"
check "abbreviations loaded" true (abbr -q n; and echo true; or echo false)
check "privacy variables set" true (test "$DO_NOT_TRACK" = 1 -a "$DISABLE_TELEMETRY" = 1; and echo true; or echo false)
section "session: functions"
set -l missing
for f in cat logs config-help fish-deps check_fish_deps config-settings
functions -q $f; or set -a missing $f
end
check "core functions defined" "" "$missing"
check "exit is rewired to smart_exit" true (functions -q exit; and functions exit | string match -q '*smart_exit*'; and echo true; or echo false)
check "fish_greeting defined" true (functions -q fish_greeting; and echo true; or echo false)
set -l missing_vault
for f in agents-vault _agents_vault_dir _agents_repo_slug \
_agents_repo_ensure_symlink _agents_repo_sync _agents_repo_install_tools
functions -q $f; or set -a missing_vault $f
end
check "agents-vault functions defined" "" "$missing_vault"
# One assertion, not two, so this file maps 1:1 onto the fifteen test_*
# predicates it replaces and the suite's 15/15 baseline is preserved.
check "claude and agy wrappers call agents-vault" true (functions -q claude; and functions claude | string match -q '*agents-vault*'; and functions -q agy; and functions agy | string match -q '*agents-vault*'; and echo true; or echo false)
section "session: guards in a loaded session"
set -l tags (__fish_config_op_registry_lookup config cdpath)
check "registry lookup finds config:cdpath" overrides/environment "$tags"
set -l ptags (__fish_config_op_registry_lookup config privacy)
check "registry lookup finds config:privacy" overrides/privacy "$ptags"
__fish_config_op_enabled __fish_config_test_never_registered somesite
check "unregistered component fails open" 0 $status
section "session: vault dir override"
set -l saved
set -q __fish_agent_vault_dir; and set saved $__fish_agent_vault_dir
set -g __fish_agent_vault_dir /tmp/vault-override-check
set -l got (_agents_vault_dir)
set -e __fish_agent_vault_dir
test (count $saved) -gt 0; and set -g __fish_agent_vault_dir $saved
check "vault dir honors the override" /tmp/vault-override-check "$got"