klammertext/mac/katom_rewrite_rules.cpp

// katom_rewrite_rules.cpp -- the categorized rule table.  See katom_rewrite_rules.h for
// what this experiment is and what was measured to justify it.
//
// The patterns and replacements are copied VERBATIM from ktype.h's
// katom_rewrite_rules -- mechanically, not retyped -- and the trailing
// index comment on each is its position there.  Nothing about a rule
// changed; the only additions are the category, the never_fires mark, and
// the abbreviated-application rule, which ktype.h does not carry because it
// lives inside expand_compound_katom() instead.

#include <algorithm>
#include <cctype>
#include <iomanip>
#include <iostream>
#include <map>
#include <regex>

#include "katom_rewrite_rules.h"
#include "file.h"
#include "log.h"
#include "show.h"
#include "util.h"

namespace {

// The abbreviated application: @i-a-b becomes "@i a | b @".  Copied from
// expand_compound_katom() in katom.cpp, where it sits ABOVE the rule loop
// as a special case.  Note the regex_match: it is a WHOLE-word test, which
// is why a trailing period defeats it and why a separate rule ("trailing
// punctuation, shortcut") exists to strip one first.
std::string abbreviated_application(const std::string& word)
{
    static const std::regex shortcut_rgx(R"((@\w+)(-\w+)+)");
    std::smatch match {};
    if (!std::regex_match(word, match, shortcut_rgx))
        return word;
    std::string modified = string_replace(word, "-", " | ") + " @";
    return std::regex_replace(modified, std::regex(R"((@\w+) \|)"), "$1");
}

} // namespace

const std::vector<Rewrite_rule> rewrite_rules {
    { rule_category::caret, "bar precedence",
      Rgx(R"((.*?)(\^\|)(.*))"), "$1 $2 $3", nullptr },   // 0
    { rule_category::caret, "literal without space",
      Rgx(R"((\^')(.*?)('\^))"), "$1 $2 $3", nullptr },   // 1
    { rule_category::caret, "literal start to the left",
      Rgx(R"((.+)(\^'))"), "$1 $2", nullptr },   // 2
    { rule_category::caret, "literal start to the right",
      Rgx(R"((\^')(.+))"), "$1 $2", nullptr },   // 3
    { rule_category::caret, "literal end to the left",
      Rgx(R"(('\^)(.+))"), "$1 $2", nullptr },   // 4
    { rule_category::caret, "literal end to the right",
      Rgx(R"((.+)('\^))"), "$1 $2", nullptr },   // 5
    { rule_category::caret, "special character",
      Rgx(R"((.*?)(\^[@|^#*:])(.*))"), "$1 $2 $3", nullptr },   // 6
    { rule_category::caret, "non-diacritic non-ascii",
      Rgx(R"((.*?)(\^[a-zA-Z0-9]{5}\^)(.*?))"), "$1 $2 $3", nullptr },   // 7
    { rule_category::caret, "non-diacritic non-ascii 2",
      Rgx(R"((.*?)(\^[a-zA-Z0-9]{1,4}\^)(.*?))"), "$1 $2 $3", nullptr },   // 8
    { rule_category::caret, "diacritic non-ascii",
      Rgx(R"((.*?)(\^[a-zA-z][^^])(.*?))"), "$1 $2 $3", nullptr },   // 9
    { rule_category::bar, "double bar separator",
      Rgx(R"(([^\s\^]+?)(\|\|)(.*))"), "$1 $2 $3", nullptr },   // 10
    { rule_category::bar, "bar separator",
      Rgx(R"(([^\s\^]+?)(\|)(.*))"), "$1 $2 $3", nullptr },   // 11
    { rule_category::removal_span, "embedded ignore begin",
      Rgx(R"((.*?)#\[(.*))"), "$1 #[ $2", nullptr },   // 12
    { rule_category::removal_span, "embedded ignore end",
      Rgx(R"((.*?)\]#(.*))"), "$1 ]# $2", nullptr },   // 13
    { rule_category::removal_span, "embedded remove whitespace",
      Rgx(R"((.*?)#-(.*))"), "$1 #- $2", nullptr },   // 14
    // RETIRED 2026-09-02, kept as a record rather than deleted.  UNREACHABLE:
    // its pattern requires "^" followed by one of @ | : * ^, which is exactly
    // what the caret rule "special character" matches -- and caret is the
    // first category tried, so it always fires first.  Demonstrated:
    //     kdiag --rewrite '@x-^@@'   ->  Rewrite [caret] (special character)
    //     kdiag --rewrite '@x-^|@'   ->  Rewrite [caret] (bar precedence)
    // { rule_category::desugar, "single argument for special character",
    //   Rgx(R"((@\w+)-(\^[@\|:\*\^])@?)"), "$1 $2 @" },   // was 15
    { rule_category::karg, "klammer body arguments",
      Rgx(R"((.*?)(\*\w+\*)(.*))"), "$1 $2 $3", nullptr },   // 16
    { rule_category::punctuation, "trailing punctuation",
      Rgx(R"((\w?)@([\]\}().,:;?'!]))"), "$1@ $2", nullptr },   // 17
    // RETIRED 2026-09-02, kept as a record rather than deleted.  UNREACHABLE
    // BY PROOF, the only one of the eight that is: the rule above has an
    // OPTIONAL (\w?), so it matches everything this one does and stands
    // first.  Demonstrated:
    //     kdiag --rewrite '@.'       ->  Rewrite [punctuation] (trailing punctuation)
    //     kdiag --rewrite '@,rest'   ->  Rewrite [punctuation] (trailing punctuation)
    // { rule_category::punctuation, "trailing punctuation",
    //   Rgx(R"(@([().,:;?'!].*))"), "@ $1" },   // was 18
    { rule_category::desugar, "trailing punctuation, shortcut",
      Rgx(R"((@\w+)-(\w+)([\]\}().,:;?'!][^\s]*))"), "$1 $2 @ $3", nullptr },   // 19
    { rule_category::klammer, "no argument klammer",
      Rgx(R"((@\w+)@)"), "$1 @", nullptr },   // 20
    // Renamed from "left parenthesis" and widened to [ and { on 2026-09-03
    // (Andy).  An author writes a klammer inside brackets and expects the
    // bracket not to be swallowed; only "(" was honoured, which is the
    // asymmetry notes/Klammertext_improvements.md 4.3 records.  The closing
    // side is the punctuation classes above, which gained "]" and "}" in
    // the same change.
    //
    // Why this is safe to whitelist: none of ( [ { ) ] } can appear in a
    // definition_name ([a-zA-Z][a-zA-Z0-9_.]*) or in an abbreviated
    // argument, so a delimiter meeting one has no second reading.  That is
    // also why LETTERS can never join the list -- "x@" is a legitimate
    // token, the named closing delimiter, so "x@i" is ambiguous in the
    // grammar rather than merely unhandled.
    { rule_category::klammer, "opening bracket",
      Rgx(R"(([(\[{])@(\w+))"), "$1 @$2", nullptr },   // 21
    { rule_category::klammer, "embedded klammer",
      Rgx(R"(([^@]+)(@[^@]+@)([^@]+))"), "$1 $2 $3", nullptr },   // 22
    { rule_category::klammer, "embedded klammer to the right",
      Rgx(R"(([^@]+)(@[^@]+@))"), "$1 $2", nullptr },   // 23
    { rule_category::klammer, "embedded klammer to the left",
      Rgx(R"((@[^@]+@)([^@]+))"), "$1 $2", nullptr },   // 24
    { rule_category::spacing, "embedded add whitespace",
      Rgx(R"(([^\s])(#\+\d*)([^\s]))"), "$1 $2 $3", nullptr },   // 25
    { rule_category::spacing, "embedded add whitespace to the left",
      Rgx(R"(([^\s])(#\+\d*))"), "$1 $2", nullptr },   // 26
    { rule_category::spacing, "embedded add whitespace to the right",
      Rgx(R"((#\+\d*)([^\s]))"), "$1 $2", nullptr },   // 27
    { rule_category::removal_line, "embedded ignore on the left",
      Rgx(R"(#([^-+/\s]))"), "# $1", nullptr },   // 28
    { rule_category::removal_line, "embedded ignore on the right",
      Rgx(R"(([^\s])#)"), "$1 #", nullptr },   // 29
    // Not in ktype.h's table: today this is the special case at the top of
    // expand_compound_katom(), tried before every rule rather than as one.
    { rule_category::desugar, "abbreviated application",
      Rgx(R"((@\w+)(-\w+)+)"), "", abbreviated_application },   // (not in ktype.h)
};

std::string category_name(rule_category category)
{
    static const std::map<rule_category, std::string> names {
        { rule_category::caret,        "caret" },
        { rule_category::bar,          "bar" },
        { rule_category::removal_span, "removal_span" },
        { rule_category::karg,         "karg" },
        { rule_category::punctuation,  "punctuation" },
        { rule_category::desugar,      "desugar" },
        { rule_category::klammer,      "klammer" },
        { rule_category::spacing,      "spacing" },
        { rule_category::removal_line, "removal_line" },
    };
    auto found = names.find(category);
    return found == names.end() ? "?" : found->second;
}

const std::vector<rule_category>& category_order()
{
    // The cross-category ordering decision, entire.  Every adjacency here
    // is either forced by a measured preemption (see katom_rewrite_rules.h) or
    // free -- and where it is free, this keeps the position the rule
    // already had in ktype.h, so the two orders differ as little as
    // possible.
    static const std::vector<rule_category> order {
        rule_category::caret,
        rule_category::bar,
        rule_category::removal_span,
        rule_category::karg,
        rule_category::punctuation,
        rule_category::desugar,
        rule_category::klammer,
        rule_category::spacing,
        rule_category::removal_line,
    };
    return order;
}

const std::vector<size_t>& rule_order(rewrite_order order)
{
    static const std::vector<size_t> declared = [] {
        std::vector<size_t> v(rewrite_rules.size());
        for (size_t i = 0; i < v.size(); ++i) v[i] = i;
        return v;
    }();

    static const std::vector<size_t> grouped = [] {
        std::vector<size_t> v {};
        for (rule_category category : category_order())
            for (size_t i = 0; i < rewrite_rules.size(); ++i)
                if (rewrite_rules[i].category == category)
                    v.push_back(i);   // stable: declaration order within a category
        return v;
    }();

    return order == rewrite_order::as_declared ? declared : grouped;
}

namespace {

// Apply one rule.  A rule with a special function uses it; every other is a
// plain regex_replace, exactly as the current loop does.
std::string apply_rule(const Rewrite_rule& rule, const std::string& word)
{
    return rule.special ? rule.special(word)
                        : std::regex_replace(word, rule.pattern.m_regex, rule.replacement);
}

// Whether a rule's rewrite makes PROGRESS.  Two conditions, and the second
// is the termination guard: the word must change, and no part of the result
// may be the whole input.  A rule that only puts whitespace around a match
// spanning the entire word satisfies the first and fails the second, and
// recursing on what it returns is what killed ktext on "^\s" until
// 2026-08-30.  Here the guard is part of what "matched" MEANS, rather than
// a check the caller has to remember to make.
bool rewrite_is_useful(const std::string& word, const std::string& rewritten)
{
    if (rewritten == word)
        return false;
    strings_t parts = word_split(rewritten);
    return std::ranges::none_of(parts, [&](const std::string& p) {
        return !p.empty() && p == word; });
}

// THE RULES ARE AUTHOR-MODE, and this is where that becomes structural
// rather than a convention (Andy, 2026-09-04).
//
// A katom rewrite rule exists to spare an AUTHOR typing -- to let "@i-x."
// and "[@i-x]" mean what they look like.  A designer writing "@@" or a
// programmer writing "@@@" is already working in a language system
// deliberately, so there is no convenience to provide; and in fact those
// two tiers need none, because "@@name", "name@@" and "@@@target" are whole
// katom TYPES with nothing left for a rule to repair.
//
// Measured before the rule went in: of eleven definition- and system-mode
// forms, ten fired no rule at all.  The exception was "@@w@@", where "no
// argument klammer" ((@\w+)@) found "@w@" inside it and split it -- into the
// RIGHT katoms, but by coincidence of where the boundary fell, so any change
// to that rule would have moved definition parsing with it.  "@@w@@" is
// malformed anyway (a definition needs :, ::, ::: or ::::), and it is now
// reported rather than silently split.
//
// Three things this buys: no rule can reach definition text, now or in
// future; a proposed rule has a test for whether it is legitimate at all
// (does it fire on text only a designer writes?); and 1,657 of the corpus's
// 44,771 words stop walking the whole rule list to fall off the end.
//
// A WORD CAN MIX TIERS, which is why this is not simply "starts with @@".
// The first attempt was, and it broke tst/modulepath_test.sh, whose fixture
// ends a definition flush against a whitespace operator:
//
//     @@desc : @eval pmod.hello() @ @@#-
//                                   ^^^^ one word: a definition close,
//                                        then author-mode text removal
//
// The rules must see that word -- "embedded remove whitespace" is what
// splits it.  So the test is whether the word is definition-mode THROUGHOUT:
// nothing in it but delimiters and a name.  Anything else -- a #, a bar, a
// caret, punctuation -- means an author-mode construct is glued on and the
// rules are needed after all.
// Named for the question it asks -- could an AUTHOR have written this? --
// rather than for the mode, because "author_mode" would sit a few files
// from defmode_t/def_create/def_override, which is exactly where the two
// senses of "mode" collide.  In prose the three are author mode, designer
// mode and programmer mode (Andy, 2026-09-04).
bool author_written(const std::string& text)
{
    if (text.size() < 2 || text[0] != '@' || text[1] != '@')
        return true;
    // definition_begin_name characters, plus the "@" of the delimiters
    // themselves.  The comma is here because a definition may name several
    // targets: "@@table.html,tex".
    for (char c : text) {
        if (c != '@' && !std::isalnum(static_cast<unsigned char>(c))
            && c != '_' && c != '.' && c != ',')
            return true;
    }
    return false;
}

} // namespace

Rewrite_result find_rewrite(const std::string& word, rewrite_order order)
{
    if (!author_written(word))
        return {};
    for (size_t i : rule_order(order)) {
        std::string rewritten = apply_rule(rewrite_rules[i], word);
        if (rewrite_is_useful(word, rewritten))
            return { true, i, rewritten };
    }
    return {};
}

std::vector<size_t> matching_rules(const std::string& word, rewrite_order order)
{
    std::vector<size_t> result {};
    if (!author_written(word))
        return result;
    for (size_t i : rule_order(order)) {
        std::string rewritten = apply_rule(rewrite_rules[i], word);
        if (rewrite_is_useful(word, rewritten))
            result.push_back(i);
    }
    return result;
}

bool category_order_holds(const std::string& word, std::string& why, rewrite_order order)
{
    std::vector<size_t> matches = matching_rules(word, order);
    if (matches.size() < 2)
        return true;
    const Rewrite_rule& winner = rewrite_rules[matches.front()];

    const std::vector<rule_category>& sequence = category_order();
    auto position = [&](rule_category c) {
        return std::find(sequence.begin(), sequence.end(), c) - sequence.begin(); };

    for (size_t i = 1; i < matches.size(); ++i) {
        const Rewrite_rule& loser = rewrite_rules[matches[i]];
        if (position(loser.category) < position(winner.category)) {
            why = "\"" + word + "\": " + category_name(winner.category)
                + " rule \"" + winner.description + "\" preempted the earlier category "
                + category_name(loser.category) + " rule \"" + loser.description + "\"";
            return false;
        }
    }
    return true;
}

void describe_rewrite_patterns()
{
    if (verbose_level > 0) {
        std::string desc_text = "doc/katom_patterns.desc";
        std::cout << "\n"
                  << justify(string_from_file(klammertext_filename(desc_text)))
                  << "\n\n";
    }
    size_t width0 = 0;
    size_t width1 = 0;
    size_t width2 = 0;
    for (const Rewrite_rule& rule : rewrite_rules) {
        width0 = std::max(width0, category_name(rule.category).size());
        width1 = std::max(width1, rule.description.size());
        width2 = std::max(width2, rule.pattern.m_pattern.size());
    }
    width0 += 2;
    width1 += 2;
    width2 += 2;

    std::cout << boldblack << std::setfill(' ')
              << std::setw(width0) << std::left << "  Category"
              << std::setw(width1) << std::left << "Description"
              << std::setw(width2) << std::left << "Pattern"
              << "Replacement" << black << "\n";

    // Dispatch order, not declaration order: what a reader needs is the
    // sequence the rules are actually TRIED in, since that is what decides
    // which one fires.
    rule_category shown {};
    bool first = true;
    for (size_t i : rule_order(rewrite_order::by_category)) {
        const Rewrite_rule& rule = rewrite_rules[i];
        std::string category = category_name(rule.category);
        if (first || rule.category != shown) {
            shown = rule.category;
            first = false;
        } else {
            category = "";              // name a category once, at its head
        }
        std::string desc = rule.description;
        desc[0] = static_cast<char>(toupper(desc[0]));
        std::cout << "  "
                  << std::left
                  << std::setw(width0) << category
                  << std::setw(width1) << desc
                  << std::setw(width2) << rule.pattern.m_pattern
                  << (rule.special ? "(not a pattern replacement)" : rule.replacement)
                  << "\n";
    }
    if (verbose_level > 0) {
        std::string notes_text = "doc/katom_patterns.notes";
        std::cout << "\n\n"
                  << justify(string_from_file(klammertext_filename(notes_text)))
                  << "\n\n";
    }
}