ack
5c29da52dc
Assembled from development commit 632297b7657a. Since 1b00738: when something
goes wrong while a klammer is being applied, the report begins with the
line of your document that applied it, then lists the klammers being
applied and where inside the klammer set the fault was raised; the
Python binding's line and column are the application's too. The default
image search path is the document's directory, imgsrc beside it and
../imgsrc, and a not-found report shows exactly those.
281 lines
11 KiB
Bash
Executable File
281 lines
11 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# eval_test.sh — the @eval primitive's contract with the outside world.
|
|
#
|
|
# @eval is the one primitive that reaches OUT of Klammertext, and until
|
|
# 2026-08-15 nothing tested what it did with what came back. ":shell" is
|
|
# covered here; the other modes (Python, :cpp, :haskell) are exercised
|
|
# incidentally by other suites and can grow into this one.
|
|
#
|
|
# The two defects this suite exists to hold shut, both of them the same shape
|
|
# as a msg() on the wrong stream -- output nobody chose to see, and a failure
|
|
# nobody was told about:
|
|
#
|
|
# * The command's STDERR went straight to the user's terminal, unattributed
|
|
# and unsuppressable. It is not the command's output in any of the three
|
|
# policy categories (CLAUDE.md, "Command output policy"): it belongs to a
|
|
# subprocess a klammer invoked, at a location the Locator can name. It is
|
|
# now captured and reported at "-v 1".
|
|
# * The EXIT STATUS was discarded, so a command that failed contributed its
|
|
# partial output (or nothing) to the document and said nothing at all.
|
|
#
|
|
# NOT to be confused with "eval_test", the C++ diagnostic program built from
|
|
# eval_test.cpp in this directory: that one constructs engine objects and prints
|
|
# what it gets, for a person to read, and asserts nothing (see the `smoke`
|
|
# target in tst/Makefile). This is the regression suite. The ".sh" is the only
|
|
# thing distinguishing them, and it is the first such collision in tst/.
|
|
#
|
|
# Usage: ./eval_test.sh (needs KLAMMERTEXT_HOME set; ktext on PATH)
|
|
# Exit code: 0 if all tests pass, 1 otherwise.
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
KTEXT=ktext
|
|
K=${KLAMMERTEXT_HOME:?KLAMMERTEXT_HOME must be set}
|
|
|
|
red=$'\033[31m'
|
|
green=$'\033[32m'
|
|
bold=$'\033[1m'
|
|
reset=$'\033[0m'
|
|
|
|
OUTF=$(mktemp "${TMPDIR:-/tmp}"/eval_out.XXXXXX)
|
|
ERRF=$(mktemp "${TMPDIR:-/tmp}"/eval_err.XXXXXX)
|
|
trap 'rm -f "$OUTF" "$ERRF"' EXIT
|
|
|
|
plain() { sed 's/\x1b\[[0-9;]*m//g'; }
|
|
|
|
# run SOURCE [EXTRA...] — sets STATUS, and fills OUTF/ERRF.
|
|
STATUS=0
|
|
run() {
|
|
local src="$1"; shift
|
|
"$KTEXT" --klammersets none -s "$src" -d "$@" >"$OUTF" 2>"$ERRF"
|
|
STATUS=$?
|
|
}
|
|
|
|
pass() { echo "${green}PASS${reset} $1"; PASS=$((PASS + 1)); }
|
|
fail() { echo "${red}FAIL${reset} $1"; [ -n "$2" ] && echo " $2"; FAIL=$((FAIL + 1)); }
|
|
|
|
echo "${bold}@eval tests${reset}"
|
|
echo "==========="
|
|
echo
|
|
|
|
echo "-- :shell, the ordinary case --"
|
|
run '@eval :shell echo hello @'
|
|
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "hello" ]; then
|
|
pass " 1. the command's stdout becomes document text"
|
|
else
|
|
fail " 1. exit $STATUS, stdout [$(cat "$OUTF")]"
|
|
fi
|
|
|
|
echo
|
|
echo "-- stderr belongs to the command, not to the terminal --"
|
|
# A signal death is not an error exit, and a crash must never read as a pass:
|
|
# every case here checks the status numerically.
|
|
run '@eval :shell echo OUT; echo NOISE >&2 @'
|
|
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "OUT" ]; then
|
|
pass " 2. stdout is the document; stderr is not in it"
|
|
else
|
|
fail " 2. exit $STATUS, stdout [$(cat "$OUTF")]"
|
|
fi
|
|
if [ ! -s "$ERRF" ]; then
|
|
pass " 3. ... and nothing leaks to the terminal at the default verbosity"
|
|
else
|
|
fail " 3. stderr leaked: $(head -1 "$ERRF")"
|
|
fi
|
|
run '@eval :shell echo OUT; echo NOISE >&2 @' -v 1
|
|
if grep -q "NOISE" "$ERRF"; then
|
|
pass " 4. ... while -v 1 reports what the command said"
|
|
else
|
|
fail " 4. -v 1 did not report the command's stderr" "$(head -2 "$ERRF")"
|
|
fi
|
|
if grep -qi "stderr" "$ERRF"; then
|
|
pass " 5. ... and says that is what it is"
|
|
else
|
|
fail " 5. the -v 1 report does not identify the stream"
|
|
fi
|
|
|
|
echo
|
|
echo "-- a failing command is an error, not silence --"
|
|
run '@eval :shell exit 3 @'
|
|
if [ $STATUS -ne 0 ] && [ $STATUS -lt 128 ]; then
|
|
pass " 6. a nonzero exit status fails the run"
|
|
else
|
|
fail " 6. exit $STATUS (128+ would be a signal death, 0 a silent pass)"
|
|
fi
|
|
for want in "exit status 3" "The shell command failed"; do
|
|
if grep -qF "$want" "$ERRF"; then
|
|
pass " 7. the error says [$want]"
|
|
else
|
|
fail " 7. the error does not say [$want]" "$(plain < "$ERRF" | head -2)"
|
|
fi
|
|
done
|
|
# What the command itself reported is the useful half of the diagnosis.
|
|
run '@eval :shell echo WHY-IT-FAILED >&2; exit 1 @'
|
|
if grep -qF "WHY-IT-FAILED" "$ERRF"; then
|
|
pass " 8. ... and includes what the command wrote to stderr"
|
|
else
|
|
fail " 8. the command's own message was dropped" "$(plain < "$ERRF" | head -3)"
|
|
fi
|
|
# An error is category 2: stderr, and nothing on stdout to confuse a pipe.
|
|
if [ ! -s "$OUTF" ]; then
|
|
pass " 9. ... and leaves stdout empty"
|
|
else
|
|
fail " 9. stdout carried [$(head -c 60 "$OUTF")]"
|
|
fi
|
|
|
|
echo
|
|
echo "-- the escape hatch, because some commands exit nonzero on purpose --"
|
|
# "grep" finding no match is the usual one. Strictness with an explicit way to
|
|
# say "I meant that" is the same shape as the @cond predicate rule.
|
|
run '@eval :shell exit 3 @'
|
|
if grep -qF "|| true" "$ERRF"; then
|
|
pass "10. the error names the way to say a nonzero status was intended"
|
|
else
|
|
fail "10. the error does not offer the remedy" "$(plain < "$ERRF" | head -3)"
|
|
fi
|
|
# NOT "exit N || true": exit terminates the shell before "||" is reached, so
|
|
# that spelling cannot work and is not what the message suggests. A command
|
|
# that merely RETURNS nonzero is the case the remedy is for.
|
|
run '@eval :shell echo kept; grep -q zzz /dev/null || true @'
|
|
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "kept" ]; then
|
|
pass "11. ... and it works"
|
|
else
|
|
fail "11. exit $STATUS, stdout [$(cat "$OUTF")]"
|
|
fi
|
|
# The same command without the remedy is an error, or case 11 proves nothing.
|
|
run '@eval :shell echo kept; grep -q zzz /dev/null @'
|
|
if [ $STATUS -ne 0 ] && [ $STATUS -lt 128 ]; then
|
|
pass "11a. ... and without it the same command fails"
|
|
else
|
|
fail "11a. exit $STATUS — expected a nonzero, non-signal exit"
|
|
fi
|
|
|
|
echo
|
|
echo "-- the command may contain its own pipeline --"
|
|
# The redirection that captures stderr must not disturb the writer's command.
|
|
run '@eval :shell echo one two three | tr " " "-" @'
|
|
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "one-two-three" ]; then
|
|
pass "12. a pipeline inside the command still works"
|
|
else
|
|
fail "12. exit $STATUS, stdout [$(cat "$OUTF")]"
|
|
fi
|
|
run '@eval :shell echo a > /dev/null; echo b @'
|
|
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "b" ]; then
|
|
pass "13. ... and so does a redirection of its own"
|
|
else
|
|
fail "13. exit $STATUS, stdout [$(cat "$OUTF")]"
|
|
fi
|
|
|
|
echo
|
|
echo "-- the other modes still work --"
|
|
run '@eval 6*7 @'
|
|
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "42" ]; then
|
|
pass "14. a Python expression"
|
|
else
|
|
fail "14. exit $STATUS, stdout [$(cat "$OUTF")]"
|
|
fi
|
|
|
|
echo
|
|
echo "-- :haskell, the same contract --"
|
|
# Skipped where GHC is absent: runghc is an optional dependency (the
|
|
# akopra/klammertext:haskell image, or a local GHCup install), and a suite that
|
|
# fails for its absence would be reporting the machine, not the code.
|
|
if ! command -v runghc >/dev/null 2>&1; then
|
|
echo "SKIP 15-18. :haskell (runghc not installed)"
|
|
else
|
|
# The defect: runghc ran with "2>&1", so on a SUCCESSFUL run everything the
|
|
# program or GHC wrote to stderr was merged into the result and became part
|
|
# of the document.
|
|
HS='@eval :haskell import System.IO
|
|
main = hPutStrLn stderr "HS-NOISE" >> putStrLn "HS-OUT" @'
|
|
run "$HS"
|
|
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "HS-OUT" ]; then
|
|
pass "15. the program's stdout is the document; its stderr is not"
|
|
else
|
|
fail "15. exit $STATUS, stdout [$(cat "$OUTF")]"
|
|
fi
|
|
run "$HS" -v 1
|
|
if grep -q "HS-NOISE" "$ERRF"; then
|
|
pass "16. ... and -v 1 reports what it wrote to stderr"
|
|
else
|
|
fail "16. -v 1 did not report it" "$(plain < "$ERRF" | head -2)"
|
|
fi
|
|
# A compile error was ALREADY reported rather than swallowed -- the exit
|
|
# status was checked -- so this pins behaviour that was right, and that the
|
|
# detail now comes from the captured stderr rather than a merged stream.
|
|
run '@eval :haskell main = putStrLn (1 + "x") @'
|
|
if [ $STATUS -ne 0 ] && [ $STATUS -lt 128 ] && grep -qi "error" "$ERRF"; then
|
|
pass "17. a compile error fails the run and shows what runghc said"
|
|
else
|
|
fail "17. exit $STATUS" "$(plain < "$ERRF" | head -3)"
|
|
fi
|
|
# A program that compiles, runs, and then exits nonzero is the other half.
|
|
run '@eval :haskell import System.Exit
|
|
main = putStrLn "partial" >> exitWith (ExitFailure 3) @'
|
|
if [ $STATUS -ne 0 ] && grep -qF "exit status 3" "$ERRF"; then
|
|
pass "18. a nonzero exit from the program itself is reported too"
|
|
else
|
|
fail "18. exit $STATUS" "$(plain < "$ERRF" | head -3)"
|
|
fi
|
|
fi
|
|
|
|
echo
|
|
echo "-- the environment can be missing: the errors name what and how --"
|
|
# Added 2026-08-22 by the error-gallery coverage review: neither message had
|
|
# ever been printed by a test.
|
|
run '@eval :cpp nonexistent_gallery_lib @'
|
|
if [ $STATUS -ne 0 ] && grep -qF "Cannot open library" "$ERRF"; then
|
|
pass "19. :cpp names the library it could not open"
|
|
else
|
|
fail "19. exit $STATUS" "$(plain < "$ERRF" | head -2)"
|
|
fi
|
|
# :haskell without runghc: strip PATH so the case is deterministic whether or
|
|
# not GHC is installed; ktext itself is invoked by absolute path (which works
|
|
# since the 2026-08-21 construct_command_pathname fix).
|
|
# A PATH without runghc. On Windows a program finds its DLLs through PATH,
|
|
# so there it keeps the directories of libklammertext.so and python312.dll
|
|
# -- neither holds a runghc, which is all the case needs.
|
|
NORUNGHC_PATH=/nonexistent
|
|
if command -v cygpath >/dev/null 2>&1; then
|
|
NORUNGHC_PATH="/nonexistent:$(cygpath -u "$K")/bin:$(dirname "$(command -v python)")"
|
|
fi
|
|
env PATH="$NORUNGHC_PATH" "$K/bin/ktext" --klammersets none \
|
|
-s '@eval :haskell main = putStrLn "x" @' -d >"$OUTF" 2>"$ERRF"
|
|
hstatus=$?
|
|
if [ $hstatus -ne 0 ] && tr '\n' ' ' < "$ERRF" | grep -qF "requires runghc"; then
|
|
pass "20. :haskell without runghc says what to install"
|
|
else
|
|
fail "20. exit $hstatus" "$(plain < "$ERRF" | head -2)"
|
|
fi
|
|
|
|
# An error raised INSIDE a klammer's body (here an @eval that divides by
|
|
# zero) is reported at the application in the user's document -- the
|
|
# headline -- with the chain of applications and the raising place below
|
|
# (2026-09-25; before, the headline was the @eval's own line, which for an
|
|
# SKS klammer is a line of a .k file the user never wrote). Wrap-insensitive
|
|
# greps: the report is justified to 80 columns.
|
|
CTX=$(mktemp -d "${TMPDIR:-/tmp}/eval_ctx.XXXXXX")
|
|
printf '@@bad x : @eval 1/*x* @ @@\n@@outer y : @bad *y* @ @@\ntext\n@bad 0 @\n' > "$CTX/ctx.kt"
|
|
printf '@@bad x : @eval 1/*x* @ @@\n@@outer y : @bad *y* @ @@\ntext\n@outer 0 @\n' > "$CTX/nested.kt"
|
|
"$K/bin/ktext" "$CTX/ctx.kt" --klammersets none -d >"$OUTF" 2>"$ERRF"; cstatus=$?
|
|
ctext=$(tr '\n' ' ' < "$ERRF" | tr -s ' ')
|
|
if [ $cstatus -ne 0 ] && case "$ctext" in *"ctx.kt, line 4, character 1:"*"applying @bad"*"at "*"ctx.kt, line 1, character "*"ZeroDivisionError"*) true;; *) false;; esac; then
|
|
pass "21. an error inside a klammer body is headlined at the application, with the chain"
|
|
else
|
|
fail "21. exit $cstatus" "$(plain < "$ERRF" | head -6 | tr '\n' '|')"
|
|
fi
|
|
"$K/bin/ktext" "$CTX/nested.kt" --klammersets none -d >"$OUTF" 2>"$ERRF"; cstatus=$?
|
|
ctext=$(tr '\n' ' ' < "$ERRF" | tr -s ' ')
|
|
if [ $cstatus -ne 0 ] && case "$ctext" in *"nested.kt, line 4, character 1:"*"applying @outer"*"applying @bad at "*"nested.kt, line 2, character "*"at "*"nested.kt, line 1, character "*) true;; *) false;; esac; then
|
|
pass "22. ... nested applications are listed outermost first, the raising place last"
|
|
else
|
|
fail "22. exit $cstatus" "$(plain < "$ERRF" | head -7 | tr '\n' '|')"
|
|
fi
|
|
rm -rf "$CTX"
|
|
|
|
echo
|
|
echo "==========="
|
|
echo "Results: ${PASS} passed, ${FAIL} failed"
|
|
[ "$FAIL" -eq 0 ] || exit 1
|
|
exit 0
|