klammertext/mac/katom_matching.cpp

// katom_matching.cpp -- finding katoms in text by matching.  See
// katom_matching.h for what this is and is not.  The whole of it is the
// three functions below.
//
// THE THREE RULES.  Everything the direct katomizer does follows from these; there
// is nothing else to know, and no rule ordering to reconstruct.
//
//   1. LONGEST MATCH.  At each position, every source katom type is tried
//      anchored at that position, and the one matching the most characters
//      wins.
//   2. DECLARATION ORDER BREAKS TIES.  Two types matching the same length
//      at the same position: the one written earlier in katom_types wins.
//      This is the same precedence katomization by rewriting already uses, where
//      std::find_if takes the first type that matches the whole text -- so
//      the table's order keeps exactly the meaning it has today.
//   3. ALWAYS ADVANCE.  Every step consumes at least one character.  A
//      position where nothing matches emits one character as an unparsed
//      katom rather than stalling.
//
// Rule 3 is why the direct katomizer cannot have the failure that killed ktext on
// "^oe^" (2026-08-29): a rewrite fixed point can fire a rule, change the
// string, and still hand back the identical text.  Here, non-termination is
// not something to guard against -- it is unavailable.
//
// WHAT IS NOT HERE, and deliberately.  There is no whitespace splitting, so
// there are no rules to repair it.  Roughly thirty of the thirty-four
// entries in katom_rewrite_rules say "these two katoms may abut without a
// space between them" -- "bar separator", "embedded klammer to the left",
// "trailing punctuation", "left parenthesis", "literal start to the right".
// A direct katomizer never joins two katoms into one piece of text,
// so it never needs to be told how to take one apart.  Those thirty rules have no counterpart in
// this file; the four that remain are the abbreviated form, which is a
// separate pass at the bottom.

#include <map>
#include <regex>

#include "katom_matching.h"
#include "katomize.h"
#include "ktype.h"
#include "util.h"

namespace {

// The abbreviated application @name-arg-arg.  The one katom the katom_types
// table does not carry: today it lives as a special case at the top of
// expand_compound_katom() rather than as a type, which is why a trailing
// period defeats it (that regex_match is a WHOLE-text match, so "@i-arg."
// fails it) and why katom_rewrite_rules needs a rule to strip the period
// first.  Here it is one pattern among the rest, and longest match keeps
// the period out of it with no rule at all.  Its parts are \w+ rather than
// definition_name, matching what expand_compound_katom() accepts today.
const std::regex abbreviated_apply_rgx { R"(@\w+(?:-\w+)+)" };

// Processing states, not source categories -- see appears_in_source() in
// katom_matching.h.  Listed by name rather than detected by an empty pattern,
// because ws_added's "pattern" is the DESCRIPTION "<space> or \n" and would
// pass an emptiness test while matching nothing a document contains.
const std::set<katom_t> state_only_types {
    katom_t::ws_added,
    katom_t::literal,
    katom_t::eval_result,
    katom_t::replaced,
    katom_t::ignored,
};

// PATTERNS THAT ASSUME A WHOLE-WORD MATCHER.  This is the experiment's
// first real finding, and the only entry so far.
//
// katom_types is asked one question today: "does this WHOLE piece of text match?"
// A pattern written for that question may be sloppy about characters a
// piece of text cannot contain, because the whitespace split removed them.  The
// "text" type is written [^^@#|]+ -- everything except the four structural
// characters -- and whitespace is not excluded because the whitespace
// split had already removed any.  Under longest match that pattern eats spaces and newlines whole,
// and "@i arg @." is katomized as one text katom from the space to the period.
//
// So a direct katomizer needs [^^@#|\s]+ there.  That is not a workaround: it is the
// pattern saying what it always meant, now that something asks it at every
// position instead of once per piece.  Adopting the direct katomizer means making
// this edit in ktype.h; the override lives here so the experiment changes
// no shipped file.  If more of these turn up, this table is the honest
// measure of what adoption costs.
const std::map<katom_t, std::regex> matching_patterns {
    // THIRD FINDING -- recorded, not fixed, because at this level it cannot
    // be.  The literal span "^'...'^" written in PROSE is katomized wrong: after
    // the opening delimiter, text runs greedily to "...'", eating the
    // apostrophe that literal-end ('\^) needed, and the lone caret then
    // matches nothing.  A lookahead stopping text before "'\^" repairs that
    // one line and BREAKS SIX -- every argtype :pattern written
    // "('style_code'^|\s)+", where the same two characters are an
    // apostrophe ending a type reference followed by a quoted bar.
    // Measured: sks/ went from 16 divergences to 22, so the lookahead is
    // not here.
    //
    // "'^" is genuinely ambiguous and no amount of longest match decides
    // it: what it means depends on whether a "^'" is OPEN, which is state,
    // not lookahead.  The standard answer is a katomizer MODE (flex calls them
    // start conditions) -- one bool, set by literal-begin, cleared by
    // literal-end, consulted here.  That is a real fourth rule on top of
    // the three at the top of this file, and it is left undone
    // deliberately: whether the direct katomizer may carry state is a decision about
    // the experiment, not a detail of it.  The rewrite katomizer never meets
    // the question, because word_split has already cut "^'...'^" out as its
    // own piece before anything is asked.
    { katom_t::text, std::regex(R"([^^@#|\s]+)") },

    // Second finding, and a subtler one: within ONE pattern, ECMAScript
    // alternation is ORDERED, not longest.  The nonascii type is written
    // (^Xy) | (^<alnum>{1,6}^), and regex_search anchored at a caret takes
    // the first alternative that succeeds -- so "^4E2D^" matched as "^4E",
    // three characters, and the rest of the code point fell out as text.
    // regex_match never sees this, because there both alternatives must
    // consume the whole text and only the right one can.  Longest match
    // BETWEEN types does not rescue a pattern that is not longest-match
    // WITHIN itself.  The repair is to order the alternatives longest
    // first; the pattern is otherwise identical.
    { katom_t::nonascii, std::regex(R"((\^[A-Za-z0-9]{1,6}\^)|(\^(\w(?:.|\^))))") },
};

// "@sup-2-3" -> { "@sup", "2", "3" }.  Local rather than util.h's
// regex_split(), which trims its parts -- an argument's own whitespace is
// not this pass's to remove.
strings_t hyphen_split(const std::string& s)
{
    strings_t parts {};
    size_t start = 0;
    for (size_t i = s.find('-'); i != std::string::npos; i = s.find('-', start)) {
        parts.push_back(s.substr(start, i - start));
        start = i + 1;
    }
    parts.push_back(s.substr(start));
    return parts;
}

// How many characters of s, starting at pos, this katom type matches; 0 if
// it does not match there.  match_continuous is what anchors the match to
// pos -- without it regex_search would find the pattern anywhere later in
// the string, which is the search the rewrite rules do and the reason they
// need those (.*?) context groups to say where the match landed.
size_t match_length(const Ktype& ktype, const std::string& s, size_t pos)
{
    if (ktype.m_use_equal) {
        return s.compare(pos, ktype.m_pattern.size(), ktype.m_pattern) == 0
            ? ktype.m_pattern.size() : 0;
    }
    auto override_pattern = matching_patterns.find(ktype.m_type);
    const std::regex& rgx = override_pattern == matching_patterns.end()
        ? ktype.m_rgx : override_pattern->second;

    std::smatch hit {};
    auto from = s.cbegin() + static_cast<long>(pos);
    if (std::regex_search(from, s.cend(), hit, rgx,
                          std::regex_constants::match_continuous)) {
        return static_cast<size_t>(hit.length(0));
    }
    return 0;
}

// Rules 1 and 2: the longest match at pos, earliest declaration winning a
// tie.  Returns a length of 0 when no type matches.
std::pair<katom_t, size_t> longest_match(const std::string& s, size_t pos)
{
    katom_t best_type { katom_t::word };
    size_t best_length { 0 };

    size_t abbreviated = 0;
    std::smatch hit {};
    auto from = s.cbegin() + static_cast<long>(pos);
    if (std::regex_search(from, s.cend(), hit, abbreviated_apply_rgx,
                          std::regex_constants::match_continuous)) {
        abbreviated = static_cast<size_t>(hit.length(0));
        best_type = katom_t::apply_begin;
        best_length = abbreviated;
    }

    for (const Ktype& ktype : katom_types) {
        if (!appears_in_source(ktype.m_type))
            continue;
        size_t length = match_length(ktype, s, pos);
        // Strictly greater: a tie leaves the earlier winner in place, which
        // is rule 2.  The abbreviated form is seeded above so that it wins
        // a tie against apply_begin -- they can only tie when there is no
        // hyphen, and then they are the same answer anyway.
        if (length > best_length) {
            best_length = length;
            best_type = ktype.m_type;
        }
    }
    return { best_type, best_length };
}

} // namespace

bool appears_in_source(katom_t type)
{
    return !state_only_types.contains(type);
}

katom_list katomize_by_matching(const std::string& text, const std::string& source, int source_line)
{
    // The same normalization split_into_katoms() does, so the two are
    // compared on identical input rather than on a difference in what a tab
    // means.
    std::string s = string_replace(text, "\r", "");
    s = string_replace(s, "\t", "        ");

    katom_list result {};
    size_t pos = 0;
    while (pos < s.size()) {
        auto [type, length] = longest_match(s, pos);
        if (length == 0) {
            // Rule 3.  Nothing matched here, so take one character and say
            // so.  The rewrite katomizer reaches the same outcome by falling
            // off the end of its rule list; the difference is that it can
            // only do so for a whole whitespace-delimited piece, so "x@i" arrives as one
            // unparsed katom where this reports the one character that is
            // actually unaccounted for.
            Katom k(s.substr(pos, 1), katom_t::word,
                    Locator(source, source_line, static_cast<int>(pos)));
            k.m_unparsed = true;
            result.push_back(k);
            pos += 1;
            continue;
        }
        result.push_back(source_katom(s.substr(pos, length), type,
                                      Locator(source, source_line, static_cast<int>(pos))));
        pos += length;
    }
    return result;
}

katom_list expand_abbreviations(const katom_list& katoms)
{
    katom_list result {};
    for (const Katom& k : katoms) {
        if (k.m_type != katom_t::apply_begin
            || k.m_src.find('-') == std::string::npos) {
            result.push_back(k);
            continue;
        }
        // "@sup-2-3" -> "@sup" "2" "|" "3" "@".  The arguments are re-run
        // through katomize_by_matching() rather than typed here, so there is one answer to
        // "what is this text" and it lives in one place: an argument of
        // letters becomes a word katom and an argument of digits a text
        // katom, exactly as it does when the same characters are written
        // out longhand.  No whitespace is emitted between them, matching
        // what katomization by rewriting produces today (word_split() discards the
        // spaces its own rewrite inserted).
        strings_t parts = hyphen_split(k.m_src);
        result.push_back(Katom(parts.front(), katom_t::apply_begin, k.m_loc));
        for (size_t i = 1; i < parts.size(); ++i) {
            if (i > 1)
                result.push_back(Katom("|", katom_t::bar, k.m_loc));
            katom_list arg = katomize_by_matching(parts[i], k.m_loc.m_filename, k.m_loc.m_line);
            result.insert(result.end(), arg.begin(), arg.end());
        }
        result.push_back(Katom("@", katom_t::apply_end, k.m_loc));
    }
    return result;
}