klammertext/tst/eval_test.sh

#!/bin/bash
#
# eval_test.sh — the @eval primitive's contract with the outside world.
#
# @eval is the one primitive that reaches OUT of Klammertext, and until
# 2026-08-15 nothing tested what it did with what came back.  ":shell" is
# covered here; the other modes (Python, :cpp, :haskell) are exercised
# incidentally by other suites and can grow into this one.
#
# The two defects this suite exists to hold shut, both of them the same shape
# as a msg() on the wrong stream -- output nobody chose to see, and a failure
# nobody was told about:
#
#   * The command's STDERR went straight to the user's terminal, unattributed
#     and unsuppressable.  It is not the command's output in any of the three
#     policy categories (CLAUDE.md, "Command output policy"): it belongs to a
#     subprocess a klammer invoked, at a location the Locator can name.  It is
#     now captured and reported at "-v 1".
#   * The EXIT STATUS was discarded, so a command that failed contributed its
#     partial output (or nothing) to the document and said nothing at all.
#
# NOT to be confused with "eval_test", the C++ diagnostic program built from
# eval_test.cpp in this directory: that one constructs engine objects and prints
# what it gets, for a person to read, and asserts nothing (see the `smoke`
# target in tst/Makefile).  This is the regression suite.  The ".sh" is the only
# thing distinguishing them, and it is the first such collision in tst/.
#
# Usage: ./eval_test.sh   (needs KLAMMERTEXT_HOME set; ktext on PATH)
# Exit code: 0 if all tests pass, 1 otherwise.

PASS=0
FAIL=0
KTEXT=ktext
K=${KLAMMERTEXT_HOME:?KLAMMERTEXT_HOME must be set}

red=$'\033[31m'
green=$'\033[32m'
bold=$'\033[1m'
reset=$'\033[0m'

OUTF=$(mktemp "${TMPDIR:-/tmp}"/eval_out.XXXXXX)
ERRF=$(mktemp "${TMPDIR:-/tmp}"/eval_err.XXXXXX)
trap 'rm -f "$OUTF" "$ERRF"' EXIT

plain() { sed 's/\x1b\[[0-9;]*m//g'; }

# run SOURCE [EXTRA...] — sets STATUS, and fills OUTF/ERRF.
STATUS=0
run() {
    local src="$1"; shift
    "$KTEXT" --klammersets none -s "$src" -d "$@" >"$OUTF" 2>"$ERRF"
    STATUS=$?
}

pass() { echo "${green}PASS${reset} $1"; PASS=$((PASS + 1)); }
fail() { echo "${red}FAIL${reset} $1"; [ -n "$2" ] && echo "       $2"; FAIL=$((FAIL + 1)); }

echo "${bold}@eval tests${reset}"
echo "==========="
echo

echo "-- :shell, the ordinary case --"
run '@eval :shell echo hello @'
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "hello" ]; then
    pass " 1. the command's stdout becomes document text"
else
    fail " 1. exit $STATUS, stdout [$(cat "$OUTF")]"
fi

echo
echo "-- stderr belongs to the command, not to the terminal --"
# A signal death is not an error exit, and a crash must never read as a pass:
# every case here checks the status numerically.
run '@eval :shell echo OUT; echo NOISE >&2 @'
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "OUT" ]; then
    pass " 2. stdout is the document; stderr is not in it"
else
    fail " 2. exit $STATUS, stdout [$(cat "$OUTF")]"
fi
if [ ! -s "$ERRF" ]; then
    pass " 3. ... and nothing leaks to the terminal at the default verbosity"
else
    fail " 3. stderr leaked: $(head -1 "$ERRF")"
fi
run '@eval :shell echo OUT; echo NOISE >&2 @' -v 1
if grep -q "NOISE" "$ERRF"; then
    pass " 4. ... while -v 1 reports what the command said"
else
    fail " 4. -v 1 did not report the command's stderr" "$(head -2 "$ERRF")"
fi
if grep -qi "stderr" "$ERRF"; then
    pass " 5. ... and says that is what it is"
else
    fail " 5. the -v 1 report does not identify the stream"
fi

echo
echo "-- a failing command is an error, not silence --"
run '@eval :shell exit 3 @'
if [ $STATUS -ne 0 ] && [ $STATUS -lt 128 ]; then
    pass " 6. a nonzero exit status fails the run"
else
    fail " 6. exit $STATUS (128+ would be a signal death, 0 a silent pass)"
fi
for want in "exit status 3" "The shell command failed"; do
    if grep -qF "$want" "$ERRF"; then
        pass " 7. the error says [$want]"
    else
        fail " 7. the error does not say [$want]" "$(plain < "$ERRF" | head -2)"
    fi
done
# What the command itself reported is the useful half of the diagnosis.
run '@eval :shell echo WHY-IT-FAILED >&2; exit 1 @'
if grep -qF "WHY-IT-FAILED" "$ERRF"; then
    pass " 8. ... and includes what the command wrote to stderr"
else
    fail " 8. the command's own message was dropped" "$(plain < "$ERRF" | head -3)"
fi
# An error is category 2: stderr, and nothing on stdout to confuse a pipe.
if [ ! -s "$OUTF" ]; then
    pass " 9. ... and leaves stdout empty"
else
    fail " 9. stdout carried [$(head -c 60 "$OUTF")]"
fi

echo
echo "-- the escape hatch, because some commands exit nonzero on purpose --"
# "grep" finding no match is the usual one.  Strictness with an explicit way to
# say "I meant that" is the same shape as the @cond predicate rule.
run '@eval :shell exit 3 @'
if grep -qF "|| true" "$ERRF"; then
    pass "10. the error names the way to say a nonzero status was intended"
else
    fail "10. the error does not offer the remedy" "$(plain < "$ERRF" | head -3)"
fi
# NOT "exit N || true": exit terminates the shell before "||" is reached, so
# that spelling cannot work and is not what the message suggests.  A command
# that merely RETURNS nonzero is the case the remedy is for.
run '@eval :shell echo kept; grep -q zzz /dev/null || true @'
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "kept" ]; then
    pass "11. ... and it works"
else
    fail "11. exit $STATUS, stdout [$(cat "$OUTF")]"
fi
# The same command without the remedy is an error, or case 11 proves nothing.
run '@eval :shell echo kept; grep -q zzz /dev/null @'
if [ $STATUS -ne 0 ] && [ $STATUS -lt 128 ]; then
    pass "11a. ... and without it the same command fails"
else
    fail "11a. exit $STATUS — expected a nonzero, non-signal exit"
fi

echo
echo "-- the command may contain its own pipeline --"
# The redirection that captures stderr must not disturb the writer's command.
run '@eval :shell echo one two three | tr " " "-" @'
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "one-two-three" ]; then
    pass "12. a pipeline inside the command still works"
else
    fail "12. exit $STATUS, stdout [$(cat "$OUTF")]"
fi
run '@eval :shell echo a > /dev/null; echo b @'
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "b" ]; then
    pass "13. ... and so does a redirection of its own"
else
    fail "13. exit $STATUS, stdout [$(cat "$OUTF")]"
fi

echo
echo "-- the other modes still work --"
run '@eval 6*7 @'
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "42" ]; then
    pass "14. a Python expression"
else
    fail "14. exit $STATUS, stdout [$(cat "$OUTF")]"
fi

echo
echo "-- :haskell, the same contract --"
# Skipped where GHC is absent: runghc is an optional dependency (the
# akopra/klammertext:haskell image, or a local GHCup install), and a suite that
# fails for its absence would be reporting the machine, not the code.
if ! command -v runghc >/dev/null 2>&1; then
    echo "SKIP  15-18. :haskell (runghc not installed)"
else
    # The defect: runghc ran with "2>&1", so on a SUCCESSFUL run everything the
    # program or GHC wrote to stderr was merged into the result and became part
    # of the document.
    HS='@eval :haskell import System.IO
main = hPutStrLn stderr "HS-NOISE" >> putStrLn "HS-OUT" @'
    run "$HS"
    if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "HS-OUT" ]; then
        pass "15. the program's stdout is the document; its stderr is not"
    else
        fail "15. exit $STATUS, stdout [$(cat "$OUTF")]"
    fi
    run "$HS" -v 1
    if grep -q "HS-NOISE" "$ERRF"; then
        pass "16. ... and -v 1 reports what it wrote to stderr"
    else
        fail "16. -v 1 did not report it" "$(plain < "$ERRF" | head -2)"
    fi
    # A compile error was ALREADY reported rather than swallowed -- the exit
    # status was checked -- so this pins behaviour that was right, and that the
    # detail now comes from the captured stderr rather than a merged stream.
    run '@eval :haskell main = putStrLn (1 + "x") @'
    if [ $STATUS -ne 0 ] && [ $STATUS -lt 128 ] && grep -qi "error" "$ERRF"; then
        pass "17. a compile error fails the run and shows what runghc said"
    else
        fail "17. exit $STATUS" "$(plain < "$ERRF" | head -3)"
    fi
    # A program that compiles, runs, and then exits nonzero is the other half.
    run '@eval :haskell import System.Exit
main = putStrLn "partial" >> exitWith (ExitFailure 3) @'
    if [ $STATUS -ne 0 ] && grep -qF "exit status 3" "$ERRF"; then
        pass "18. a nonzero exit from the program itself is reported too"
    else
        fail "18. exit $STATUS" "$(plain < "$ERRF" | head -3)"
    fi
fi

echo
echo "-- the environment can be missing: the errors name what and how --"
# Added 2026-08-22 by the error-gallery coverage review: neither message had
# ever been printed by a test.
run '@eval :cpp nonexistent_gallery_lib @'
if [ $STATUS -ne 0 ] && grep -qF "Cannot open library" "$ERRF"; then
    pass "19. :cpp names the library it could not open"
else
    fail "19. exit $STATUS" "$(plain < "$ERRF" | head -2)"
fi
# :haskell without runghc: strip PATH so the case is deterministic whether or
# not GHC is installed; ktext itself is invoked by absolute path (which works
# since the 2026-08-21 construct_command_pathname fix).
# A PATH without runghc. On Windows a program finds its DLLs through PATH,
# so there it keeps the directories of libklammertext.so and python312.dll
# -- neither holds a runghc, which is all the case needs.
NORUNGHC_PATH=/nonexistent
if command -v cygpath >/dev/null 2>&1; then
    NORUNGHC_PATH="/nonexistent:$(cygpath -u "$K")/bin:$(dirname "$(command -v python)")"
fi
env PATH="$NORUNGHC_PATH" "$K/bin/ktext" --klammersets none \
    -s '@eval :haskell main = putStrLn "x" @' -d >"$OUTF" 2>"$ERRF"
hstatus=$?
if [ $hstatus -ne 0 ] && tr '\n' ' ' < "$ERRF" | grep -qF "requires runghc"; then
    pass "20. :haskell without runghc says what to install"
else
    fail "20. exit $hstatus" "$(plain < "$ERRF" | head -2)"
fi

# An error raised INSIDE a klammer's body (here an @eval that divides by
# zero) is reported at the application in the user's document -- the
# headline -- with the chain of applications and the raising place below
# (2026-09-25; before, the headline was the @eval's own line, which for an
# SKS klammer is a line of a .k file the user never wrote).  Wrap-insensitive
# greps: the report is justified to 80 columns.
CTX=$(mktemp -d "${TMPDIR:-/tmp}/eval_ctx.XXXXXX")
printf '@@bad x : @eval 1/*x* @ @@\n@@outer y : @bad *y* @ @@\ntext\n@bad 0 @\n' > "$CTX/ctx.kt"
printf '@@bad x : @eval 1/*x* @ @@\n@@outer y : @bad *y* @ @@\ntext\n@outer 0 @\n' > "$CTX/nested.kt"
"$K/bin/ktext" "$CTX/ctx.kt" --klammersets none -d >"$OUTF" 2>"$ERRF"; cstatus=$?
ctext=$(tr '\n' ' ' < "$ERRF" | tr -s ' ')
if [ $cstatus -ne 0 ] && case "$ctext" in *"ctx.kt, line 4, character 1:"*"applying @bad"*"at "*"ctx.kt, line 1, character "*"ZeroDivisionError"*) true;; *) false;; esac; then
    pass "21. an error inside a klammer body is headlined at the application, with the chain"
else
    fail "21. exit $cstatus" "$(plain < "$ERRF" | head -6 | tr '\n' '|')"
fi
"$K/bin/ktext" "$CTX/nested.kt" --klammersets none -d >"$OUTF" 2>"$ERRF"; cstatus=$?
ctext=$(tr '\n' ' ' < "$ERRF" | tr -s ' ')
if [ $cstatus -ne 0 ] && case "$ctext" in *"nested.kt, line 4, character 1:"*"applying @outer"*"applying @bad at "*"nested.kt, line 2, character "*"at "*"nested.kt, line 1, character "*) true;; *) false;; esac; then
    pass "22. ... nested applications are listed outermost first, the raising place last"
else
    fail "22. exit $cstatus" "$(plain < "$ERRF" | head -7 | tr '\n' '|')"
fi
rm -rf "$CTX"

echo
echo "==========="
echo "Results: ${PASS} passed, ${FAIL} failed"
[ "$FAIL" -eq 0 ] || exit 1
exit 0