klammertext/tst/character_test.sh

#!/bin/bash
#
# character_test.sh — Regression tests for the ^-character forms the engine
# resolves at read time: the ^UUUU^ Unicode code point (4-6 hex digits,
# since 2026-08-23 covering the full range to U+10FFFF), the
# ^UUUU UUUU ...^ SEQUENCE form (added 2026-08-25: one caret pair around
# whitespace-delimited tokens, assembled at katom level by
# process_unicode_sequences — so, unlike the string-level single form, it
# is skipped inside @eval/@read/@cond spans and for literal content), and
# the character tables (mnemonics, diacritics) it must not disturb.
#
# SKS-independent: character handling is katomizer/engine machinery
# (mac/ktype.h nonascii katom, mac/character.cpp), so everything runs with
# --klammersets none under the default (general) target.
#
# The 2026-08-23 changes these cases pin:
#   - a 5-digit code point converts WHOLE: ^13000^ (EGYPTIAN HIEROGLYPH
#     A001) rendered as U+1300 followed by a literal "0" before, because
#     process_unicode_codepoint() re-scanned its capture at a fixed 4-digit
#     width;
#   - 6 digits are accepted (^10FFFD^ is the top of the range; the katom
#     regex capped at {1,5});
#   - a value above U+10FFFF renders as the "Invalid Unicode" text, the
#     same treatment as a surrogate.
#
# Note: expected strings use $'\xHH' UTF-8 BYTE escapes, not $'\U...' code
# points -- macOS ships bash 3.2, which does not expand \U (it failed there
# exactly as a naive check would).
#
# Usage: ./character_test.sh   (needs KLAMMERTEXT_HOME set; ktext on PATH)
# Exit code: 0 if all tests pass, 1 otherwise.

PASS=0
FAIL=0
KTEXT=ktext
K=${KLAMMERTEXT_HOME:?KLAMMERTEXT_HOME must be set}
ERR=$(mktemp)

red=$'\033[31m'
green=$'\033[32m'
bold=$'\033[1m'
reset=$'\033[0m'

trim() { awk '{ sub(/[ \t\r]+$/, "") } { line[NR]=$0 } END { f=1; while (f<=NR && line[f]=="") f++; l=NR; while (l>=1 && line[l]=="") l--; for (i=f;i<=l;i++) print line[i] }'; }

# check_err NAME PHRASE INPUT — the render must FAIL (nonzero exit) with
# PHRASE on stderr.  Matched wrap-insensitively: error messages are
# justified to 80 columns and may wrap anywhere.
check_err() {
    local name="$1" phrase="$2" input="$3"
    local err status
    err=$("$KTEXT" --klammersets none -s "$input" -d 2>&1 >/dev/null); status=$?
    err=$(printf '%s' "$err" | tr '\n' ' ' | tr -s ' ')
    if [ $status -eq 0 ]; then
        echo "${red}FAIL${reset} $name — expected an error, got exit 0"
        FAIL=$((FAIL+1)); return
    fi
    case "$err" in
        *"$phrase"*) echo "${green}PASS${reset} $name"; PASS=$((PASS+1));;
        *) echo "${red}FAIL${reset} $name"
           echo "       wanted phrase: [$phrase]"
           echo "       stderr: $(echo "$err" | head -c 200)"; FAIL=$((FAIL+1));;
    esac
}

# check NAME EXPECTED INPUT — render INPUT (no klammer set, default
# target); exit 0 and stdout == EXPECTED.
check() {
    local name="$1" expected="$2" input="$3"
    local out status
    out=$("$KTEXT" --klammersets none -s "$input" -d 2>"$ERR"); status=$?
    out=$(printf '%s' "$out" | trim)
    if [ $status -ne 0 ]; then
        echo "${red}FAIL${reset} $name — ktext exited $status"
        head -3 "$ERR" | sed 's/^/       /'; FAIL=$((FAIL+1)); return
    fi
    if [ "$out" = "$expected" ]; then
        echo "${green}PASS${reset} $name"; PASS=$((PASS+1))
    else
        echo "${red}FAIL${reset} $name"
        echo "       expected: [$expected]"; echo "       got:      [$out]"; FAIL=$((FAIL+1))
    fi
}

echo "${bold}Character form tests (engine: ^UUUU^ code points and the tables)${reset}"
echo "================================================================="
echo

check " 1. 4-digit BMP code point"        $'A ☺ B'      'A ^263A^ B'
check " 2. 5-digit code point, whole"     $'A \xf0\x93\x80\x80 B'  'A ^13000^ B'
check " 3. 5-digit emoji"                 $'\xf0\x9f\x98\x80'  '^1F600^'
check " 4. 6-digit top of the range"      $'\xf4\x8f\xbf\xbd'  '^10FFFD^'
# An invalid code point is a located ERROR (2026-08-25, the
# no-warn-and-continue policy) -- it was in-band "Invalid Unicode" text in
# the document before, which is the wrong channel for a generator's typo.
check_err " 5. above U+10FFFF is an error"  'beyond U+10FFFF'   '^FFFFFF^'
check_err " 6. surrogate is an error"       'a surrogate'       '^D800^'
check_err " 6a. noncharacter is an error"   'a noncharacter'    '^FDD0^'
check " 7. mnemonic undisturbed"          'ß'                '^s^'
# The tables emit the PRECOMPOSED (NFC) character where one exists (since
# 2026-08-25; they formerly emitted base + combining mark) -- the same
# bytes an editor's direct entry produces, so ^a" and a typed ä compare
# equal in search and copy-paste.  The literal ä below is precomposed.
check " 8. diacritic is the precomposed character" 'Mädchen'  'M^a"dchen'
check " 9. code point beside quoted punctuation" $'☺ --' '^263A^ ^-^-'

echo
echo "-- sequences: ^UUUU UUUU ...^ (one caret pair, tokens 4-6 hex digits,"
echo "   no space at the elbows, one line; process_unicode_sequences)"
check "10. CJK sequence"  $'X \xe9\x81\x93\xe5\x8f\xaf\xe9\x81\x93\xe9\x9d\x9e\xe5\xb8\xb8\xe9\x81\x93 Y' 'X ^9053 53EF 9053 975E 5E38 9053^ Y'
check "11. mixed widths"  $'\xe2\x98\xba\xf0\x93\x80\x80\xf0\x9f\x98\x80' '^263A 13000 1F600^'
check "12. space after opening caret: not a sequence"  'A ^ 263A 263A^ B' 'A ^ 263A 263A^ B'
check "13. space before closing caret: not a sequence" 'A ^263A 263A ^ B' 'A ^263A 263A ^ B'
check "14. newline aborts (one line only)"  $'A ^263A\n263A^ B' $'A ^263A\n263A^ B'
check "15. short token aborts whole sequence" 'A ^263A 263^ B' 'A ^263A 263^ B'
check "16. @eval code keeps its carets"    '11' '@eval len("^263A 263A^") @'
check "17. literal span: sequence stays verbatim" 'x ^263A 263A^ y' "^'x ^263A 263A^ y'^"
check "17a. single-character token stands for itself" $'\xe9\x81\x93x\xe5\x8f\xaf' '^9053 x 53EF^'
check "17b. non-alnum single token behaves as quoted" '☺-☺' '^263A - 263A^'
check "17c. structural code points are re-read-safe"  '@@' '^0040 0040^'
check_err "17d. invalid code point in a CLOSED sequence" 'in a ^...^ sequence' '^263A FFFFFF^'
check "17e. unclosed hex-like prose is untouched" 'about ^263A FFFFFF prose' 'about ^263A FFFFFF prose'

echo
echo "-- the delimited mnemonic form ^x^ (multi-letter entries, and a form"
echo "   that names no character at all)"
# The two-letter mnemonics had NO lexical rule: the nonascii katom pattern
# offered "^Xy" and "^<hex>{1,6}^", and "oe" is neither.  So "^oe^" matched
# no katom type, fell through to the "non-diacritic non-ascii" rewrite rule
# -- which pads the word with whitespace and hands back the SAME single
# word -- and make_katoms_from_word() recursed on it until the stack died:
# SIGSEGV, no message, exit 139.  ^ae^ and ^AE^ appeared to work only
# because "ae" happens to be hex digits, so they were lexed as CODE POINTS
# and rendered correctly only because encode() runs process_extended_latin()
# before process_unicode_codepoint(); reversing those two passes would have
# turned ^ae^ into U+00AE.  Cases 20-22 pin the mnemonics, 23-25 the forms
# that resolve to nothing.  A crash must never read as a pass, so 20a
# distinguishes a signal death from an error exit explicitly.
check "20. two-letter mnemonic"          'œ'  '^oe^'
check "21. two-letter mnemonic, capital" 'Œ'  '^OE^'
check "22. hex-spelled mnemonic is the MNEMONIC, not U+00AE" 'æ' '^ae^'
check "22a. mnemonic in running text"    'cœur'  'c^oe^ur'

# 20a: exit 139 (128+SIGSEGV) rather than a clean status is the failure
# this suite exists to catch, and check_err alone cannot tell them apart.
"$KTEXT" --klammersets none -s '^oe^' -d >/dev/null 2>&1; st=$?
if [ $st -ge 128 ]; then
    echo "${red}FAIL${reset} 20a. ^oe^ dies on a signal (exit $st)"
    FAIL=$((FAIL+1))
else
    echo "${green}PASS${reset} 20a. ^oe^ does not die on a signal"
    PASS=$((PASS+1))
fi

# 25a: the katomizer's rewrite loop must terminate on a word no rule can
# decompose.  "^\s" is such a word -- rule "diacritic non-ascii" matches it
# whole (its [a-zA-z] range spans ASCII 65-122 and so includes the
# backslash), pads it with spaces, and hands the same word back.  It killed
# kdiag until 2026-08-30.  Not a character-form case at heart, but this is
# where the class is pinned.
"$KTEXT" --klammersets none -s '^\s' -d >/dev/null 2>&1; st=$?
if [ $st -ge 128 ]; then
    echo "${red}FAIL${reset} 25a. a barren rewrite rule terminates (exit $st)"
    FAIL=$((FAIL+1))
else
    echo "${green}PASS${reset} 25a. a barren rewrite rule terminates"
    PASS=$((PASS+1))
fi

check_err "23. a form naming no character is an error"  'is not a character code'  '^zz^'
check_err "24. ... at any length the lexer accepts"     'is not a character code'  '^hello^'
check_err "25. ... letters and digits mixed"            'is not a character code'  '^g1^'

echo
echo "-- direct entry and NFC convergence"
check "18. pinyin tone is precomposed"  'ā'  '^a1'
check "19. typed UTF-8 passes through byte-identical" 'Mädchen naïveté ü — done' 'Mädchen naïveté ü — done'

echo
echo "================================================================="
echo "${bold}Results: $PASS passed, $FAIL failed${reset}"
rm -f "$ERR"
[ $FAIL -eq 0 ] && exit 0 || exit 1