259 lines
13 KiB
Bash
259 lines
13 KiB
Bash
|
#!/bin/bash
|
||
|
|
#
|
||
|
|
# coverage_test.sh — Target coverage: which targets a klammer can render to.
|
||
|
|
#
|
||
|
|
# "kdesc --coverage" computes the coverage FACT and reports it. Three rules
|
||
|
|
# (mac/coverage.{h,cpp}, notes/target_coverage.md):
|
||
|
|
#
|
||
|
|
# DERIVED a general body of plain text covers every target; a general body
|
||
|
|
# of klammer calls covers the INTERSECTION of what those klammers
|
||
|
|
# cover, computed as a greatest fixpoint after loading.
|
||
|
|
# DECLARED a general body holding @eval, @read or a ^'...'^ literal span
|
||
|
|
# cannot be interpreted, so its targets must be written down.
|
||
|
|
# UNKNOWN no definition at all. Absence never means "deliberately
|
||
|
|
# unavailable" -- the SKS is incomplete on schedule, not by design.
|
||
|
|
#
|
||
|
|
# The analysis modifies nothing. These tests therefore assert only what is
|
||
|
|
# REPORTED, and a companion case checks that rendering is unaffected.
|
||
|
|
#
|
||
|
|
# Engine tier: the fixtures in tst/coverage/ declare their own targets with
|
||
|
|
# @@@target, so no klammer set is involved.
|
||
|
|
#
|
||
|
|
# Usage: ./coverage_test.sh (needs KLAMMERTEXT_HOME set; kdesc on PATH)
|
||
|
|
# Exit code: 0 if all tests pass, 1 otherwise.
|
||
|
|
|
||
|
|
PASS=0
|
||
|
|
FAIL=0
|
||
|
|
KDESC=kdesc
|
||
|
|
KTEXT=ktext
|
||
|
|
K=${KLAMMERTEXT_HOME:?KLAMMERTEXT_HOME must be set}
|
||
|
|
DIR="$(cd "$(dirname "$0")" && pwd)/coverage"
|
||
|
|
|
||
|
|
red=$'\033[31m'
|
||
|
|
green=$'\033[32m'
|
||
|
|
bold=$'\033[1m'
|
||
|
|
reset=$'\033[0m'
|
||
|
|
|
||
|
|
# Each fixture is analysed once; the tests match against the saved report.
|
||
|
|
declare -A REPORT VREPORT
|
||
|
|
for f in basic undecidable cycle split clean; do
|
||
|
|
REPORT[$f]=$(timeout 30 "$KDESC" -i "$DIR/$f.k" --coverage 2>&1 |
|
||
|
|
sed 's/\x1b\[[0-9;]*m//g')
|
||
|
|
# "all" adds the source-file column AND the empty problem categories.
|
||
|
|
VREPORT[$f]=$(timeout 30 "$KDESC" -i "$DIR/$f.k" --coverage all 2>&1 |
|
||
|
|
sed 's/\x1b\[[0-9;]*m//g')
|
||
|
|
if [ -z "${REPORT[$f]}" ] || [ -z "${VREPORT[$f]}" ]; then
|
||
|
|
echo "${red}FAIL${reset} $f.k produced no report"; FAIL=$((FAIL+1))
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
|
||
|
|
# vlacks NAME FIXTURE REGEX — the VERBOSE report does NOT match REGEX.
|
||
|
|
vlacks() {
|
||
|
|
local name="$1" fixture="$2" rgx="$3"
|
||
|
|
if printf '%s\n' "${VREPORT[$fixture]}" | grep -Eq "$rgx"; then
|
||
|
|
echo "${red}FAIL${reset} $name — unexpected match: $rgx"; FAIL=$((FAIL+1))
|
||
|
|
else
|
||
|
|
echo "${green}PASS${reset} $name"; PASS=$((PASS+1))
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
# vhas NAME FIXTURE REGEX — the VERBOSE report matches REGEX.
|
||
|
|
vhas() {
|
||
|
|
local name="$1" fixture="$2" rgx="$3"
|
||
|
|
if printf '%s\n' "${VREPORT[$fixture]}" | grep -Eq "$rgx"; then
|
||
|
|
echo "${green}PASS${reset} $name"; PASS=$((PASS+1))
|
||
|
|
else
|
||
|
|
echo "${red}FAIL${reset} $name"
|
||
|
|
echo " no line matching: $rgx"; FAIL=$((FAIL+1))
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
# has NAME FIXTURE REGEX — the report matches REGEX.
|
||
|
|
has() {
|
||
|
|
local name="$1" fixture="$2" rgx="$3"
|
||
|
|
if printf '%s\n' "${REPORT[$fixture]}" | grep -Eq "$rgx"; then
|
||
|
|
echo "${green}PASS${reset} $name"; PASS=$((PASS+1))
|
||
|
|
else
|
||
|
|
echo "${red}FAIL${reset} $name"
|
||
|
|
echo " no line matching: $rgx"; FAIL=$((FAIL+1))
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
# section_lacks NAME FIXTURE HEADER REGEX — REGEX does not appear within the
|
||
|
|
# named section. Needed wherever the same klammer legitimately appears in a
|
||
|
|
# LATER section: a whole-report "lacks" would match there and report a failure
|
||
|
|
# that is not one. A section runs from its header to the next blank line.
|
||
|
|
section_lacks() {
|
||
|
|
local name="$1" fixture="$2" header="$3" rgx="$4"
|
||
|
|
local body
|
||
|
|
body=$(printf '%s\n' "${REPORT[$fixture]}" |
|
||
|
|
awk -v h="$header" 'index($0, h) == 1 { f = 1; next } f && /^$/ { exit } f')
|
||
|
|
if [ -z "$body" ]; then
|
||
|
|
echo "${red}FAIL${reset} $name — section [$header] not found"; FAIL=$((FAIL+1)); return
|
||
|
|
fi
|
||
|
|
if printf '%s\n' "$body" | grep -Eq "$rgx"; then
|
||
|
|
echo "${red}FAIL${reset} $name — unexpected match in [$header]: $rgx"; FAIL=$((FAIL+1))
|
||
|
|
else
|
||
|
|
echo "${green}PASS${reset} $name"; PASS=$((PASS+1))
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
# lacks NAME FIXTURE REGEX — the report does NOT match REGEX.
|
||
|
|
lacks() {
|
||
|
|
local name="$1" fixture="$2" rgx="$3"
|
||
|
|
if printf '%s\n' "${REPORT[$fixture]}" | grep -Eq "$rgx"; then
|
||
|
|
echo "${red}FAIL${reset} $name — unexpected match: $rgx"; FAIL=$((FAIL+1))
|
||
|
|
else
|
||
|
|
echo "${green}PASS${reset} $name"; PASS=$((PASS+1))
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
echo "${bold}Target coverage tests${reset}"
|
||
|
|
echo "====================="
|
||
|
|
echo
|
||
|
|
|
||
|
|
echo "-- derived coverage --"
|
||
|
|
has " 1. a general text body covers every target" basic '^All targets, derived \(1\)'
|
||
|
|
has " 2. ... and that body is @plain" basic '@plain'
|
||
|
|
has " 3. a body calling one klammer takes its targets" basic '@calls_ab +ta tb +from @ab'
|
||
|
|
has " 4. two klammers intersect" basic '@calls_both +ta +from @ab @cd'
|
||
|
|
lacks " 5. the intersection drops the target only one has" basic '@calls_both +ta tb'
|
||
|
|
|
||
|
|
echo
|
||
|
|
echo "-- what the definitions themselves say --"
|
||
|
|
has " 6. per-target definitions are not derived" basic 'Defined per target \(2\)'
|
||
|
|
has " 7. a comma list covers each target it names" basic '^ ta +6 covered'
|
||
|
|
has " 8. a target named by no definition is undecided" basic '^ tc +3 covered +3 undecided'
|
||
|
|
|
||
|
|
echo
|
||
|
|
echo "-- coverage that cannot be derived --"
|
||
|
|
has " 9. an @eval general body must be declared" undecidable '@evaluated +@eval body'
|
||
|
|
has "10. a @read general body must be declared" undecidable '@included +@read body'
|
||
|
|
has "11. a literal span must be declared" undecidable "@literally +\\^'\\.\\.\\.'\\^ literal span"
|
||
|
|
has "12. all three are counted together" undecidable '^Must be declared \(3\)'
|
||
|
|
has "13. the report says they are offered everywhere" undecidable 'currently offered by an underivable general body'
|
||
|
|
|
||
|
|
echo
|
||
|
|
echo "-- declared and never defined --"
|
||
|
|
has "14. a .k with no definitions is reported" undecidable '^Declared but never defined \(1\)'
|
||
|
|
has "15. ... and named" undecidable '^ @orphan$'
|
||
|
|
|
||
|
|
echo
|
||
|
|
echo "-- \".*\": every target, asserted --"
|
||
|
|
# The distinction the report exists to keep: a bare definition asserts
|
||
|
|
# nothing (the writer's macro form), while ".*" states that the klammer works
|
||
|
|
# for any target, including targets that do not exist yet. A list of the
|
||
|
|
# targets defined today cannot say that.
|
||
|
|
has "37. an explicit .* is its own category" basic '^All targets, declared \(1\)'
|
||
|
|
has "38. ... and covers every target" basic '^ @universal +ta tb tc'
|
||
|
|
lacks "39. it is not counted as derived" basic '^ @universal +ta tb tc$(.*)from'
|
||
|
|
section_lacks "40. nor as a bare general body" basic 'All targets, derived' '@universal'
|
||
|
|
# ".*" outranks the body scan: the point of writing it is to assert what an
|
||
|
|
# @eval body cannot be read to mean.
|
||
|
|
has "41. an @eval body under .* is not flagged" basic '^All targets, declared'
|
||
|
|
|
||
|
|
echo
|
||
|
|
echo "-- klammers with no \".k\" declaration --"
|
||
|
|
# The mirror image of the section above: that one has a declaration and no
|
||
|
|
# definitions, this one definitions and no declaration. Such a klammer works
|
||
|
|
# but has no DESCRIPTION, so kdesc can say nothing about it and "-k <text>"
|
||
|
|
# can only find it by name. Orthogonal to the coverage kinds -- a klammer can
|
||
|
|
# be defined for every target and still have none.
|
||
|
|
has "32. the section counts them" basic '^No "\.k" declaration \(3\)'
|
||
|
|
has "33. a general klammer has none" basic '^ @plain +ta tb tc$'
|
||
|
|
has "34. ... nor one defined without .k" basic '^ @calls_ab +ta tb$'
|
||
|
|
section_lacks "35. a declared klammer is not listed" basic 'No ".k" declaration' '^ @ab '
|
||
|
|
# @orphan has a .k and no definitions, so it belongs to the OTHER section.
|
||
|
|
lacks "36. the two sections do not overlap" undecidable '^No "\.k" declaration \(1\)\n @orphan'
|
||
|
|
|
||
|
|
echo
|
||
|
|
echo "-- the fixpoint --"
|
||
|
|
# A recursive analysis would not terminate on these; the report existing at
|
||
|
|
# all is most of the assertion.
|
||
|
|
has "16. mutual reference terminates" cycle '@ping +ta tb +from @pong'
|
||
|
|
has "17. ... in both directions" cycle '@pong +ta tb +from @ping'
|
||
|
|
has "18. a constraint propagates around a cycle" cycle '@loop_a +ta +from @loop_b @only_ta'
|
||
|
|
has "19. ... to the klammer that does not name it" cycle '@loop_b +ta +from @loop_a'
|
||
|
|
has "20. disjoint coverage is its own category" cycle '^Covers no target \(1\)'
|
||
|
|
has "21. ... naming the klammers to look at" cycle '^ @impossible +from @only_ta @only_tb'
|
||
|
|
section_lacks "42. and it is not listed as derived" cycle 'Derived from the klammers' '@impossible'
|
||
|
|
|
||
|
|
echo
|
||
|
|
echo "-- the source-file column (-v only) --"
|
||
|
|
# The file is the LAST column -- anchored, so a change of position is caught.
|
||
|
|
vhas "23. a klammer's file is shown, last" undecidable '@evaluated +@eval body +tst/coverage/undecidable\.k$'
|
||
|
|
vhas "24. ... in the derived section too" basic '@calls_ab +ta tb +from @ab +tst/coverage/basic\.k$'
|
||
|
|
vhas "25. ... and in the per-target listing" basic '@ab +ta tb +tst/coverage/basic\.k$'
|
||
|
|
vhas "26. a declaration with no definition" undecidable '^ @orphan +tst/coverage/undecidable\.k$'
|
||
|
|
lacks "27. no file column without -v" basic '@calls_ab +tst/coverage'
|
||
|
|
# Andy's policy keeps a klammer's targets together, so several files is the
|
||
|
|
# exception -- which is the case the column exists to make visible.
|
||
|
|
vhas "28. definitions in two files, comma-separated" split \
|
||
|
|
'@x +ta tb +tst/coverage/split\.k, tst/coverage/split_more\.k$'
|
||
|
|
# The "All targets" section lists many names on one line, so there is nothing
|
||
|
|
# for a file to attach to; it must not sprout a column.
|
||
|
|
vhas "29. the many-names row keeps its shape" basic '^ @plain$'
|
||
|
|
# A continuation line belongs to the row above it and takes no file.
|
||
|
|
lacks "30. the old continuation line is gone" cycle '^ +\^ covers no target'
|
||
|
|
|
||
|
|
echo
|
||
|
|
echo "-- the problems come last, and are grouped --"
|
||
|
|
# A terminal is read from the bottom: an 80-klammer listing scrolls a
|
||
|
|
# three-line warning off the screen, so the actionable part must be last.
|
||
|
|
has "43. a banner counts the distinct klammers" basic '^Needs attention: [0-9]+ klammers?$'
|
||
|
|
# An empty category is hidden, but the two halves hide for different reasons.
|
||
|
|
#
|
||
|
|
# A REPORTING category describes the shape of the klammer set, so an empty one
|
||
|
|
# still says something and "all" shows it as a designer's checklist.
|
||
|
|
lacks "44. an empty reporting category is hidden" cycle '^All targets, declared \(0\)'
|
||
|
|
vhas "45. ... and shown by \"all\"" cycle '^All targets, declared \(0\)'
|
||
|
|
# A PROBLEM category sits under a banner reading "Needs attention", and an
|
||
|
|
# empty one does not. Printing it there would state the opposite of the
|
||
|
|
# heading above it, so it stays hidden even under "all".
|
||
|
|
lacks "47. an empty problem category is hidden" basic '^Covers no target \(0\)'
|
||
|
|
vlacks "48. ... and stays hidden under \"all\"" basic '^Covers no target \(0\)'
|
||
|
|
# A category WITH entries is always shown, with or without "all".
|
||
|
|
has "49. a non-empty category needs no \"all\"" cycle '^Derived from the klammers the body calls \(4\)'
|
||
|
|
# The banner still counts, so "all" is not silent about the problems.
|
||
|
|
vhas "50. the banner survives under \"all\"" basic '^Needs attention: 3 klammers$'
|
||
|
|
# "No .k" is orthogonal to the others, so summing the counts would overstate.
|
||
|
|
has "46. the banner counts distinct klammers" basic '^Needs attention: 3 klammers$'
|
||
|
|
|
||
|
|
echo
|
||
|
|
echo "-- the category explanations --"
|
||
|
|
# The prose under a heading teaches the categories; a reader who knows them
|
||
|
|
# wants headings and rows. So it appears only under "all", with the empty
|
||
|
|
# categories and the file column.
|
||
|
|
lacks "51. no explanation by default" basic 'the author states that these work'
|
||
|
|
vhas "52. ... and one under \"all\"" basic 'the author states that these work'
|
||
|
|
has "53. the heading is always there" basic '^All targets, declared \(1\)'
|
||
|
|
|
||
|
|
echo
|
||
|
|
echo "-- a klammer set with nothing outstanding --"
|
||
|
|
# Nothing to attend to, nothing said: no banner and no problem categories,
|
||
|
|
# with or without "all". A banner reading "0" would contradict itself in the
|
||
|
|
# same way an empty category under it would.
|
||
|
|
lacks "54. no banner when nothing needs attention" clean '^Needs attention'
|
||
|
|
vlacks "55. ... not even under \"all\"" clean '^Needs attention'
|
||
|
|
vlacks "56. ... and no problem categories" clean '^No "\.k" declaration'
|
||
|
|
# The reporting categories still describe the set.
|
||
|
|
vhas "57. the reporting categories remain" clean '^Defined per target \(1\)'
|
||
|
|
|
||
|
|
echo
|
||
|
|
echo "-- the analysis changes nothing --"
|
||
|
|
# Coverage is a report, not a policy: a klammer whose general body cannot be
|
||
|
|
# interpreted is still offered to every target, exactly as before.
|
||
|
|
out=$("$KTEXT" -k none -s '@@@target ta | Target A @@@
|
||
|
|
@@e : @eval 6 * 7 @ @@ x @e@' -t ta -d 2>&1 | tr -d '\n ')
|
||
|
|
if [ "$out" = "x42" ]; then
|
||
|
|
echo "${green}PASS${reset} 22. an underivable klammer still renders"; PASS=$((PASS+1))
|
||
|
|
else
|
||
|
|
echo "${red}FAIL${reset} 22. an underivable klammer still renders — got [$out]"; FAIL=$((FAIL+1))
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo
|
||
|
|
echo "====================="
|
||
|
|
echo "Results: ${PASS} passed, ${FAIL} failed"
|
||
|
|
[ "$FAIL" -eq 0 ] || exit 1
|
||
|
|
exit 0
|