klammertext/tst/escape_test.sh

#!/bin/bash
#
# escape_test.sh — Regression tests for the Klammermachine's target character
# escaping MECHANISM.
#
# Kept SKS-INDEPENDENT on purpose: tst/ tests the engine only (mac/target.cpp,
# mac/machine.cpp), not the SKS. A target is a MACHINE construct — declared
# with the @@@target system command, not owned by any klammer set — so the
# idiomatic engine-level test defines its own fixture target inline and loads
# no klammer set (`--klammersets none`); it does not "avoid" the SKS so much as have no
# need of it. Target `t` here escapes `& -> AMP`, `_ -> UND`, `\ -> BSL`
# (arbitrary tokens, easy to assert). The SKS's own targets (tex, html) and
# the specific characters they declare are exercised by the SKS suite.
#
# The `:escape` parameter on `@@@target` declares characters special in a
# target's output and their replacements. Escaping must reach writer content
# wherever it appears — including the body of a GENERAL klammer (no target
# suffix), which is target-agnostic writer text — while leaving target-native
# content alone:
#
#   - general klammer body           -> writer content, ESCAPE it
#   - target-specific body (.t)       -> already in target form, LEAVE it
#   - ^'...'^ literal span            -> raw target markup, LEAVE it
#   - nested target-native klammer    -> e.g. @n@ -> \newline, LEAVE it
#   - @eval/@read/@cond arg span      -> code/path/predicate, LEAVE it
#   - klammer-producing @eval result  -> klammer output, LEAVE it
#
# Cases 8-9 pin a regression: escaping a general klammer body once corrupted
# the @eval CODE inside it (an underscore in "offer.Price_list(K)" became a
# KTESC marker -> Python AttributeError). Only the general body's own literal
# writer text is escaped; @eval/@read/@cond argument spans (code, filenames,
# predicates) are skipped, and the Klammertext a nested @eval produces is
# klammer output, not writer text, so it is never escaped. Mechanism:
# Klammer::m_body_generic + the hook in Machine::apply_klammer; see the
# "Target character escaping" section of CLAUDE.md.
#
# Usage: ./escape_test.sh   (needs KLAMMERTEXT_HOME set; ktext on PATH)
# Exit code: 0 if all tests pass, 1 otherwise.

PASS=0
FAIL=0
KTEXT=ktext
K=${KLAMMERTEXT_HOME:?KLAMMERTEXT_HOME must be set}
ERR=/tmp/escape_test_err.$$

red=$'\033[31m'
green=$'\033[32m'
bold=$'\033[1m'
reset=$'\033[0m'

# strip leading/trailing blank lines and trailing whitespace
trim() { awk '{ sub(/[ \t\r]+$/, "") } { line[NR]=$0 } END { f=1; while (f<=NR && line[f]=="") f++; l=NR; while (l>=1 && line[l]=="") l--; for (i=f;i<=l;i++) print line[i] }'; }

# check_eq NAME EXPECTED KTEXT_ARGS... — exit 0 and stdout==EXPECTED.
check_eq() {
    local name="$1" expected="$2"; shift 2
    local out status err
    out=$("$KTEXT" "$@" 2>"$ERR"); status=$?
    err=$(cat "$ERR")
    out=$(printf '%s' "$out" | trim)
    if [ $status -ne 0 ]; then
        echo "${red}FAIL${reset} $name — ktext exited $status"
        echo "       stderr: $(echo "$err" | head -2)"; FAIL=$((FAIL+1)); return
    fi
    if [ "$out" = "$expected" ]; then
        echo "${green}PASS${reset} $name"; PASS=$((PASS+1))
    else
        echo "${red}FAIL${reset} $name"
        echo "       expected: [$expected]"; echo "       got:      [$out]"; FAIL=$((FAIL+1))
    fi
}

echo "${bold}Target character escaping tests (Machine mechanism, fixture target 't')${reset}"
echo "======================================================================="
echo

# A self-contained fixture target, defined inline via the @@@target system
# command, escaping & _ \ to distinct tokens.  --klammersets none loads no klammer set,
# so nothing below depends on the SKS.
T='@@@target t | test target :escape & AMP _ UND \ BSL @@@'

check_eq " 1. top-level text: & escaped"                    'A AMP B'            --klammersets none -t t -s "$T A & B"
check_eq " 2. general klammer body: & escaped"              'A AMP B'            --klammersets none -t t -s "$T @@g : A & B @@ @g@"
check_eq " 3. general klammer body: _ escaped"              'AUNDB'              --klammersets none -t t -s "$T @@g : A_B @@ @g@"
check_eq " 4. general klammer body: backslash escaped"      'aBSLb'              --klammersets none -t t -s "$T @@g : a\\b @@ @g@"
check_eq " 5. target-specific body: NOT escaped"            'A & B'              --klammersets none -t t -s "$T @@g.k : d @@ @@g.t :: A & B @@ @g@"
check_eq " 6. general body ^'...'^ literal: NOT escaped"     'a&b'                --klammersets none -t t -s "$T @@g : ^'a&b'^ @@ @g@"
check_eq " 7. nested target-native klammer survives escape" 'X AMP Y \newline Z' --klammersets none -t t -s "$T @@n.t : \\newline @@ @@g : X & Y @n@ Z @@ @g@"
# 8-9: @eval inside a general body. Code (with underscores) must not be
# escaped or Python breaks; the Klammertext it returns is klammer output and
# must not be escaped either. chr(64) builds a literal '@' so the returned
# klammer call is not parsed as one in this source line.
check_eq " 8. general body @eval code NOT escaped"          '3'                  --klammersets none -t t -s "$T @@g : @eval (1).__add__(2) @ @@ @g@"
check_eq " 9. general body @eval klammer result NOT escaped" '\textbf{hi}'       --klammersets none -t t -s "$T @@b.k s : d @@ @@b.t :: \\textbf{*s*} @@ @@g : @eval chr(64)+'b hi '+chr(64) @ @@ @g@"
# 10-11: an @eval result that still holds klammers is a GENERATOR (Klammertext
# with data) -- its writer text is escaped for the target before the klammers
# are applied; a result with no klammers is a RENDERER (final markup) -- left
# untouched.  The signal is "does the read-back result contain a klammer".
check_eq "10. @eval generator: klammer result's data escaped" '[a AMP b]'        --klammersets none -t t -s "$T @@wrap z : [*z*] @@ @@g : @eval chr(64)+'wrap a & b '+chr(64) @ @@ @g@"
check_eq "11. @eval renderer: final markup NOT escaped"      'raw & markup'       --klammersets none -t t -s "$T @@g : @eval 'raw & markup' @ @@ @g@"

# 12-18: quoted KLAMMERTEXT specials (^@ ^| ^# ^^ ^: ^*) and ^'...'^ literal
# regions. The katomizer strips the "^"; hide_special_katoms() and
# mark_literal_katoms() then hold the character as a KTESC marker so it stays
# inert through sub-Machine re-katomization and the @eval result read-back,
# resolving to the literal character at final processing. Regression: after
# the marker mechanism replaced the old hide/restore pass, a quoted "@"
# leaked as a bare apply-end katom into re-read text ("A klammer ends
# without a beginning"). Exact-match expectations also guard against KTESC
# markers leaking into output.
check_eq "12. quoted @ | # resolve to the characters"       'x @ | # y'          --klammersets none -t t -s "$T x ^@ ^| ^# y"
check_eq "13. quoted ^ : * resolve to the characters"       'x ^ : * y'          --klammersets none -t t -s "$T x ^^ ^: ^* y"
check_eq "14. ^'...'^ region: specials stay literal"         'a  @ |  b'          --klammersets none -t t -s "$T a ^' @ | '^ b"
check_eq "15. general body: quoted @ resolves"              'x @ y'              --klammersets none -t t -s "$T @@g : x ^@ y @@ @g@"
# 16: inside an @eval argument span a quoted special reaches the CODE as the
# character (the span is skipped by hide_special_katoms, like the escape pass).
check_eq "16. @eval code: quoted : reaches shell as ':'"     'x:y'                --klammersets none -t t -s "$T @@g : @eval :shell echo x^:y @ @@ @g@"
# 17: an @eval result emitting the two characters ^ @ is re-read as a quoted
# special and survives to the output as a literal @ (the generator idiom for
# a literal @; a bare @ in a result is a parse error by design).
check_eq "17. @eval result ^@ survives read-back as @"      '@'                  --klammersets none -t t -s "$T @@g : @eval chr(94)+chr(64) @ @@ @g@"
# 18: a bare-Python :after_apply phase receives the RESOLVED result text
# (K_result) and its return is taken as raw target text, not re-read as
# Klammertext -- a resolved @ in the result must not be re-parsed.
check_eq "18. :after_apply phase: raw result, @ intact"     'A @ B'              --klammersets none -t u -s '@@@target u | up :after_apply string.capwords @@@ a ^@ b'

# --- Literal klammers and REMOVED text (TODO #40, fixed 2026-08-16) ---
#
# mark_literal_klammer_content() runs FIRST in process_katoms(), before
# mark_ignored_katoms() takes out the "#" forms.  That ordering is deliberate --
# a literal klammer's content must be marked before anything can interpret what
# is inside it, or removal would take a "#" belonging to the literal body -- but
# it meant the scan saw text the writer had removed, so a literal klammer merely
# NAMED in a comment was read as an opening delimiter, went unclosed, and failed
# the whole file.  Andy's minimal case was `ktext -s "# @code" -d`.
#
# Both directions need a case, and the second is the one a careless fix breaks.

# A literal klammer with a literal parameter, defined inline: the engine tier
# loads no klammer set, so @code is not available here.
LIT='@@lit.k t.literal : A literal klammer @@ @@lit.t :: [*t*] @@'

echo
echo "-- literal klammers named inside removed text --"

check_eq "19. named in a # comment: removed, not an opening delimiter" \
    'kept' --klammersets none -t t -s "$T $LIT # @lit
kept"
check_eq "20. ... including the realistic case that raised it" \
    'kept' --klammersets none -t t -s "$T $LIT # @image and @lit share the same arguments.
kept"
check_eq "21. named inside #[ ... ]#" \
    'kept' --klammersets none -t t -s "$T $LIT #[ @lit ]# kept"
check_eq "22. ... and inside NESTED #[ #[ ]# ]#" \
    'kept' --klammersets none -t t -s "$T $LIT #[ a #[ @lit ]# b ]# kept"
check_eq "23. after ## the rest of the file is gone" \
    'kept' --klammersets none -t t -s "$T $LIT kept ## @lit"

echo
echo "-- and the converse: a # INSIDE literal content is CONTENT --"
# The direction a careless fix breaks.  Skipping removed text must happen only
# while looking for an OPENING delimiter; once one is found the scan jumps past
# the whole span, so a "#" in the content is never examined.
# A BARE "#" -- not a quoted "^#".  The quoted form would pass whether or not
# the content was treated as literal, and so would prove nothing.
#
# The use must be in a FILE with the definition in "-s", not both in one "-s":
# mark_literal_klammer_content() only knows the klammers REGISTERED WHEN IT
# RUNS, and "-s" is processed as one unit, so a literal klammer defined and used
# in the same string is not yet literal while that string is scanned.  "-s" is
# processed before the input files, so this ordering is what a real document
# has -- the klammer set is loaded first.
LITUSE=$(mktemp /tmp/escape_lit.XXXXXX).kt
printf '@lit a # b lit@\n' > "$LITUSE"
# "-d" is required here and nowhere else in this suite: with a FILE input ktext
# writes a file instead of displaying, so stdout would be empty.
check_eq "24. a bare # inside literal content survives as content" \
    '[a # b]' "$LITUSE" --klammersets none -t t -d -s "$T $LIT"
rm -f "$LITUSE"
check_eq "25. a real literal klammer still works after a comment naming it" \
    '[x]' --klammersets none -t t -s "$T $LIT # mentions @lit here
@lit x lit@"

echo
echo "-- 25a-e: only a DECLARED option name opens an option region --"
# The content of a literal klammer begins after the bar that follows its
# options.  ANY ":word" before a bar counted as an option until 2026-09-20,
# declared or not, so code holding one ("ls :w | wc", a regex's "(?:a|b)")
# lost everything up to the bar to an option region and the file failed with
# 'Optional argument ":w" not defined' -- an error about an option the author
# never wrote, in content promised to be read as nothing but text.
# OPT declares one option, :flag, so both directions can be asserted.
OPT='@@opt.k :flag.bool | t.literal : A literal klammer with an option @@ @@opt.t :: [*flag*/*t*] @@'
OPTUSE=$(mktemp /tmp/escape_opt.XXXXXX).kt
opt_case() {   # opt_case NAME EXPECTED USE
    printf '%s\n' "$3" > "$OPTUSE"
    check_eq "$1" "$2" "$OPTUSE" --klammersets none -t t -d -s "$T $OPT"
}
opt_case "25a. an UNDECLARED :name before a bar is content" \
    '[/x :y | z]'     '@opt x :y | z opt@'
opt_case "25b. ... as is a regex's (?:a|b)" \
    '[/(?:a|b)]'      '@opt (?:a|b) opt@'
opt_case "25c. a DECLARED option still opens the option region" \
    '[true/x]'        '@opt :flag | x opt@'
opt_case "25d. ... and after its bar an undeclared :name is content again" \
    '[true/x :y | z]' '@opt :flag | x :y | z opt@'
# The one ambiguity left, and the author's way out of it: a name the klammer
# DOES declare, written inside the code before a bar, is an option -- unless
# an option region has already been written, which ends at ITS bar.
opt_case "25e. a declared name in the code is content once the options are written" \
    '[false/x :flag | z]' '@opt :flag false | x :flag | z opt@'
rm -f "$OPTUSE"

rm -f "$ERR"

echo
echo "-- 26-29: a general body escapes only its OWN literal text, never a --"
echo "-- substituted argument value (ordering defect fixed 2026-08-19)    --"
# An argument value is writer text already escaped at the top level (KTESC
# markers, idempotent) PLUS final target markup from klammers the writer
# nested in the argument.  apply_klammer used to substitute values first and
# escape the general body after, so that markup was swept as if it were the
# body's literal text: "@sig @b Andy @ @" emitted BSLtextbf{Andy}.  The escape
# now runs before substitution.  BOLD is a target-specific nested klammer
# (case 7/9 pattern) whose raw output must survive untouched.
BOLD='@@b.k s : d @@ @@b.t :: \textbf{*s*} @@'
check_eq "26. substituted klammer output NOT re-escaped" \
    'Sincerely, \textbf{Andy}' --klammersets none -t t \
    -s "$T $BOLD @@sig name : Sincerely, *name* @@ @sig @b Andy @ @"
check_eq "27. ... while the body's own literal & still IS escaped" \
    'Fish AMP \textbf{Chips}' --klammersets none -t t \
    -s "$T $BOLD @@sig2 name : Fish & *name* @@ @sig2 @b Chips @ @"
check_eq "28. writer specials in the argument still escaped (markers)" \
    'Sincerely, a AMP b' --klammersets none -t t \
    -s "$T @@sig name : Sincerely, *name* @@ @sig a & b @"
# The variable shares one text katom with literal specials on both sides:
# the katom's literal "_"s must be escaped and the value spliced in raw
# afterwards -- the ordering this section exists to pin.
check_eq "29. mixed katom: literal specials escaped, spliced value raw" \
    'AUND\textbf{Q}UNDB' --klammersets none -t t \
    -s "$T $BOLD @@w s : A_*s*_B @@ @w @b Q @ @"

echo
echo "============================================"
echo "Results: ${green}$PASS passed${reset}, ${red}$FAIL failed${reset}"
[ $FAIL -eq 0 ]