klammertext/mac/katomize.cpp

// katomize.cpp -- the katomization pipeline over the categorized rules.
// See katomize.h for what this experiment is, and katom_rewrite_rules.h
// for what
// was measured to justify the categories.
//
// A faithful re-expression of the former make_katoms_from_word() and its
// two callers, not a redesign -- proved identical over every tracked .k and
// .kt before it replaced them.  What changed is where the decisions live:
//
//   * The termination guard is inside find_rewrite(), as part of what it
//     means for a rule to match, instead of a condition the loop has to
//     remember.  A caller cannot forget it.
//   * The abbreviated application is a RULE in the desugar category rather
//     than a special case above the loop, so it can be named when it fires
//     and counted when it does not.
//   * The rule loop is gone from here entirely: this file asks
//     find_rewrite() a question and acts on the answer.  Which rule, and
//     why that one, is katom_rewrite_rules.cpp's business.
//
// What deliberately did NOT change: the katom types are consulted first and
// unchanged, the recursion structure is the same, the character position
// handed to each katom is that of the text it came from (as in the
// original), and text no rule can divide is still an unparsed katom rather
// than an error.

#include <algorithm>
#include <iostream>

#include "katomize.h"
#include "katom_matching.h"
#include "log.h"
#include "show.h"
#include "util.h"

namespace {

// The katom type this text IS, or nullptr.  Whole-text match, exactly as
// the current katomizer does it -- the first type in declaration order
// whose pattern matches the entire text.
const Ktype* whole_text_type(const std::string& text)
{
    auto found = std::find_if(katom_types.begin(), katom_types.end(),
                              [&](const Ktype& k) { return k.match(text); });
    return found == katom_types.end() ? nullptr : &*found;
}

} // namespace

Katom source_katom(const std::string& text, katom_t type, const Locator& loc)
{
    Katom k(text, type, loc);
    if (type == katom_t::special && !k.m_text.empty() && k.m_text[0] == '^')
        k.m_text.erase(0, 1);
    return k;
}

katom_list katomize_by_rewriting(const std::string& text, const std::string& source, int line, int chr,
                         rewrite_order order)
{
    Locator loc(source, line, chr);

    if (const Ktype* type = whole_text_type(text))
        return { source_katom(text, type->m_type, loc) };

    Rewrite_result rewrite = find_rewrite(text, order);
    if (rewrite.matched) {
        const Rewrite_rule& rule = rewrite_rules[rewrite.rule];
        if (show_rewrite_rules) {
            std::cout << loc << " Rewrite [" << category_name(rule.category) << "] ("
                      << rule.description << "): " << rule.pattern.m_pattern
                      << " " << right_arrow << " " << rewrite.text << "\n";
        }
        katom_list result {};
        for (const std::string& piece : word_split(rewrite.text)) {
            if (piece.empty())
                continue;
            katom_list ks = katomize_by_rewriting(piece, source, line, chr, order);
            result.insert(result.end(), ks.begin(), ks.end());
        }
        return result;
    }

    // No katom type, no katom rewrite rule that makes progress.  The
    // warning is deferred to warn_unparsed_katoms(), after removal and
    // literal marking, so text inside a comment never warns.
    Katom unparsed(text, katom_t::word, loc);
    unparsed.m_unparsed = true;
    return { unparsed };
}

katom_list katomize_text(const std::string& text, const std::string& source, int line,
                         rewrite_order order)
{
    std::string s = string_replace(text, "\r", "");
    s = string_replace(s, "\t", "        ");

    // Whitespace is preserved, so the split keeps each single space and
    // newline as a piece in its own right -- they become katoms like any
    // other text.  Same regex the current katomizer splits with.
    static const std::regex split_rgx(R"([ ]|[\n]|[^\s]+|.+)");
    katom_list result {};
    int position = 0;
    for (auto it = std::sregex_iterator(s.begin(), s.end(), split_rgx);
         it != std::sregex_iterator(); ++it) {
        for (Katom& k : katomize_by_rewriting(it->str(), source, line, position, order)) {
            k.m_loc = Locator(source, line, position);
            result.push_back(k);
            position += static_cast<int>(k.m_src.size());
        }
    }
    return result;
}

katom_list katomize_text_by_matching(const std::string& text, const std::string& source, int line)
{
    return expand_abbreviations(katomize_by_matching(text, source, line));
}

katom_list katomize_lines(const strings_t& lines, const std::string& source, rewrite_order order)
{
    katom_list result {};
    int number = 0;
    for (const std::string& line : lines) {
        katom_list ks = katomize_text(line + "\n", source, ++number, order);
        result.insert(result.end(), ks.begin(), ks.end());
    }
    if (!result.empty())
        result.pop_back();          // the newline added to the last line
    return result;
}

void warn_unparsed_katoms(katom_list& katoms, bool warn)
{
    for (Katom& k : katoms) {
        if (!k.m_unparsed)
            continue;
        if (!warn) {
            k.m_unparsed = false;
            continue;
        }
        if (k.m_type != katom_t::ignored
            && k.m_type != katom_t::replaced
            && k.m_type != katom_t::literal) {
            // One of the two surviving WARNINGS (the commands otherwise treat
            // an anomaly as an error and stop -- see the output policy in
            // CLAUDE.md).  It stays a warning because the judgment is a
            // HEURISTIC, not exact: the @eval read-back legitimately fails to
            // katomize raw target markup, which is why m_warn_unparsed exists.
            // A heuristic that halts turns every false positive into a blocker
            // on a document that would otherwise render.  Promote it to an
            // error when the judgment becomes exact.
            //
            // Through warning(), not a raw std::cerr, so every warning the
            // commands issue has one format.
            warning("Word not parsed:\n      " + k.m_text
                    + "\nTo include a punctuation character literally, put "
                      "\"^\" before it.",
                    k.m_loc);
            k.m_unparsed = false;
        }
    }
}