klammertext/mac/katom_process.cpp

// katom_process.cpp -- the read-time passes.  See katom_process.h.

// #include <algorithm>
#include <numeric>

#include "katom_process.h"
#include "util.h"
#include "ktype.h"
#include "file.h"
#include "log.h"
#include "show.h"
#include "character.h"
#include "target.h"

std::string to_string(std::vector<Katom>::const_iterator begin, std::vector<Katom>::const_iterator end, bool trim_result)
{
    (void)K::log(3);
    std::string result =
        std::accumulate(
            begin, end, 
            std::string(""),
            [](const std::string& s, const Katom& k)
                { return k.is_active() ? s + k.m_text : s; });
    if (trim_result) {
        result = trim(result);
    }
    return result;
}

std::string to_string(const katom_list& katoms, bool trim_result)
{
    return to_string(katoms.cbegin(), katoms.cend(), trim_result);
}

bool level_increase(const Katom& k)
{
    return k.m_type == katom_t::define_begin
        || k.m_type == katom_t::literal_begin
        || k.m_type == katom_t::ignore_begin
        || k.m_type == katom_t::read_begin
        || k.m_type == katom_t::eval_begin
        || k.m_type == katom_t::cond_begin
        || k.m_type == katom_t::machine_begin
        || k.m_type == katom_t::apply_begin;
}

bool level_decrease(const Katom& k)
{
    return k.m_type == katom_t::apply_end
        || k.m_type == katom_t::define_end
        || k.m_type == katom_t::literal_end
        || k.m_type == katom_t::ignore_end
        || k.m_type == katom_t::machine_end;
}

const katom_list::iterator
find_katom_named(const katom_list::iterator begin, const katom_list::iterator end, std::string name)
{
    auto result = find_if(
        begin, end, [&](const Katom& ki) {
            return ki.m_text == ("@" + name); });
    return result;
}

// Spans

//katom_iter get_katom_iterator(katom_list& katoms, size_t index)
const katom_list::iterator
find_katom(const katom_list::iterator begin, const katom_list::iterator end, size_t index)
{
    // msg() << "find_katom: " << std::pair(begin, end) << "\n";
    auto result = find_if(
        begin, end, [&](const Katom& ki) {
            return ki.m_index == index && ki.m_type != katom_t::ignored && ki.m_type != katom_t::replaced; });
    if (result == end) {
        std::stringstream ss {};
        ss << "Katom with index " << index << " not found in katom list";
        throw Internal_error(ss.str(), begin->m_loc);
    } else {
        return result;
    }
}

std::pair<katom_iter, katom_iter>
find_span_katoms(katom_list& katoms, const Katom& begin, const Katom& end) // Lint error
{
    //(void)K::log(3, begin, end);
    //static std::mutex katoms_mutex;
    //std::lock_guard<std::mutex> lock(katoms_mutex);
    katom_iter kbegin = find_katom(katoms.begin(), katoms.end(), begin.m_index);
    katom_iter kend = find_katom(kbegin, katoms.end(), end.m_index) + 1;
    //(void)K::log(3, kbegin->m_text, (kbegin+1)->m_text, (kbegin+2)->m_text, "...",  kend->m_text);

    //(void)K::log(3, "resolved:", kbegin, (kbegin+2), "..."); //,  kend);
    (void)K::log(3, "resolved:", kbegin, kend - 1);
    return { kbegin, kend };
}

std::pair<katom_iter, katom_iter>
find_span_katoms(katom_iter katoms_begin, katom_iter katoms_end, const Katom& begin, const Katom& end) // Lint error
{
    katom_iter kbegin = find_katom(katoms_begin, katoms_end, begin.m_index);
    katom_iter kend = find_katom(katoms_begin, katoms_end, end.m_index) + 1;
    //(void)K::log(3, kbegin->m_text, (kbegin+1)->m_text, (kbegin+2)->m_text, "...",  kend->m_text);

    //(void)K::log(3, "resolved:", kbegin, (kbegin+2), "..."); //,  kend);
    (void)K::log(3, "resolved:", kbegin, kend - 1);
    return { kbegin, kend };
}



void missing_open(const Katom& k, bool error_exit)
{
    (void)K::log(2);
    std::stringstream ss {};
    ss << "A klammer ends without a beginning: " << k;
    if (error_exit) {
        throw Parsing_error(ss.str(), k.m_loc);
    } else {
        std::cout << "  " << ss.str() << "\n";
    }
}

//void missing_close(std::vector<katom_ptr>& bounds, bool error_exit)
void missing_close(const katom_list& bounds, bool error_exit)
{
    (void)K::log(2);
    std::stringstream ss {};
    if (bounds.size() == 1) {
        ss << "  Beginning of a span that does not end:\n";
    } else {
        ss << "  A span ends that does not have a beginning:\n";
    }
    for (const auto& k : bounds) {
        ss << "  " << k.m_loc << " " << k.m_src << "\n";
    }
    if (error_exit) {
        throw Parsing_error(ss.str(), bounds[0].m_loc);
    } else {
        std::cout << ss.str() << "\n";
    }
}

void bad_close(const Katom& open, const Katom& close, bool error_exit)
{
    (void)K::log(2);
    std::stringstream ss {};
    ss << "  Klammer begins with " << open << " but ends with " << close << ".\n"
       << "    " << open.m_loc << " " << open << "\n"
       << "    " << close.m_loc << " " << close;
    if (error_exit) {
        throw Parsing_error(ss.str(), open.m_loc);
    } else {
        std::cout << ss.str() << "\n";
    }
}

std::string trim_span_markers(const Katom& katom)
{
    // Trim @ as well as ^ (for literal span)
    return trim_char(trim_char(katom.m_text, '@'), '^');
}
    

void check_named_katom_span(const Katom& begin, const Katom& end)
{
    (void)K::log(4);
    std::string end_text = trim_span_markers(end);
    if (!end_text.empty() &&
        //!(begin.m_type == katom_t::ignore_begin && end.m_type == katom_t::ignore_end)) {
        !(begin_ignore(begin) && end_ignore(end))) {
        std::string begin_text = trim_span_markers(begin);
        if (begin_text != end_text) {
            std::stringstream ss;
            ss << "  A named end katom does not match:\n"
               << "    " << begin.m_loc << " " << begin << "\n"
               << "    " << end.m_loc << " " << end;
            throw Parsing_error(ss.str(), end.m_loc);
        }
    }
}

spans_t find_spans(
    katom_iter begin, katom_iter end,
    const std::function<bool(const Katom&)>& level_inc,
    const std::function<bool(const Katom&)>& level_dec,
    bool error_exit,
    const std::string& name)
{
    (void)K::log(4, name);
    bool _dbg = false;
    spans_t spans {};
    katom_list bounds {};
    for (auto k = begin; k < end; k++) {
        if (_dbg) msg() << ktype << kall << kignored << kreplaced << "     " << k << "  type: " << type_to_name(k->m_type) << "\n";
        if (level_inc(*k)) {
            if (_dbg) msg() << boldblack << "  level_inc: " << k << black << "\n";
            bounds.push_back(*k);
        } else if (level_dec(*k)) {
            if (_dbg) msg() << boldblack << "  level_dec: " << k << "  bounds: " << bounds << black << "\n";
            if (!bounds.empty()) {
                auto span_start = bounds.back();
                if (_dbg) msg() << "  span_start of this level: " << span_start << "\n";
                if (katom_spans[span_start.m_type] == k->m_type) {
                    check_named_katom_span(span_start, *k);
                    spans.push_back({std::move(span_start), std::move(*k)});
                    bounds.pop_back();
                } else {
                    bad_close(span_start, *k, error_exit);
                }
            } else {
                missing_open(*k, error_exit);
            }
        }
    }
    if (!bounds.empty()) {
        missing_close(bounds, error_exit);
    }
    if (_dbg) msg() << black;

    // Post-order invariant: if span A is nested inside span B, then A
    // appears before B in the list. This is a necessary consequence of
    // stack-based bracket matching and is required for correct inside-out
    // evaluation of nested structures. See doc/taxonomy.md, "Spans".
    if (verbose_level >= 2) {
        for (size_t i = 0; i < spans.size(); i++) {
            for (size_t j = i + 1; j < spans.size(); j++) {
                bool j_inside_i =
                    spans[j].first.m_index > spans[i].first.m_index &&
                    spans[j].second.m_index < spans[i].second.m_index;
                if (j_inside_i) {
                    std::stringstream ss;
                    ss << "Post-order span invariant violated: span "
                       << j << " (" << spans[j].first << ")"
                       << " is nested inside span "
                       << i << " (" << spans[i].first << ")"
                       << " but appears after it";
                    throw Internal_error(ss.str(), spans[j].first.m_loc);
                }
            }
        }
    }

    return spans;
}

spans_t find_spans(
    katom_list& katoms,
    const std::function<bool(const Katom&)>& level_inc,
    const std::function<bool(const Katom&)>& level_dec,
    bool error_exit,
    const std::string& name)
{
    return find_spans(katoms.begin(), katoms.end(), level_inc, level_dec, error_exit, name);
}

void describe_spans(const katom_list& katoms)
{
    katom_list non_const_katoms = katoms;
    auto spans = find_spans(non_const_katoms, level_increase, level_decrease);
    std::vector<std::tuple<std::string, Katom, Katom>> span_specs {};
    size_t width = 0;
    std::string margin = "  ";
    string_map relpath {};
    for (auto [op,cl] : spans) {
        std::string loc = locator_range(op.m_loc, cl.m_loc, relpath);
        width = std::max(width, loc.size());
        span_specs.push_back(std::make_tuple(margin + loc, op, cl));
    }
    width += margin.size();
    for (auto [span,op,cl] : span_specs) {
        std::cout << std::setw(width) << std::left << span << "  "
                  << ktype << kreplaced
                  << op << right_arrow << cl << "\n";
    }
}


// Nonascii

void encode_nonascii_characters(katom_list& katoms)
{
    for (Katom& k : katoms) {
        if (is_nonascii(k)) {
            k.m_text = encode(k.m_text, k.m_loc);
        }
    }
}

// Unicode sequences

// ^UUUU UUUU ...^ -- a whitespace-delimited sequence of Unicode code
// points between ONE pair of carets: ^9053 53EF 9053^ renders the three
// characters adjacently.  The single-token form ^UUUU^ is the katomizer's
// nonascii katom, resolved by encode(); a sequence cannot be a katom at
// all, because whitespace is preserved as separate katoms and no katom
// spans it.  So the tokens arrive as word katoms -- the opener "^"+hex,
// the closer hex+"^", bare hex between, separated by SPACE katoms only (a
// newline aborts: a sequence is one line; so does any other katom, which
// is also what enforces "no space inside the delimiters' elbows": a bare
// "^" is not an opener) -- and are assembled here.  Each token is 4-6 hex
// digits, the same rule as the single form.  The rendered characters
// replace the opening katom's text; the other members are marked
// replaced.  Skipped inside @@/@@@ definition spans and
// @eval/@read/@cond argument spans (code keeps its carets -- the rule the
// string-level single-form pass cannot honor) and for literal katoms
// (verbatim content; mark_literal_* run before this).
namespace {

bool hex_token(const std::string& s)
{
    if (s.size() < 4 || s.size() > 6) return false;
    for (char c : s) {
        bool hex = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')
                || (c >= 'A' && c <= 'F');
        if (!hex) return false;
    }
    return true;
}

// A sequence token is a 4-6-digit hex code point or a SINGLE character
// standing for itself ("^" excluded -- a lone caret cannot say which
// caret closes).  The single-character reading exists only inside a
// sequence; ^s^ alone stays the mnemonic.
bool shape_token(const std::string& s)
{
    return hex_token(s) || (s.size() == 1 && s[0] != '^');
}

// Render one scanned token.  A hex token is validated -- an invalid code
// point is a located error, not text in the document -- and a structural
// character in the result is hidden as its KTESC marker.  A single
// character behaves exactly as if quoted with "^": alphanumerics stand
// bare, anything else becomes its marker.
std::string render_token(const std::string& s, const Locator& loc)
{
    if (hex_token(s)) {
        int cp = std::stoi(s, nullptr, 16);
        if (!valid_codepoint(cp)) {
            throw Parsing_error(
                "Invalid Unicode code point \"" + s + "\" in a ^...^ sequence: "
                + invalid_codepoint_reason(cp), loc);
        }
        return hide_structural_characters(utf8char(cp));
    }
    char c = s[0];
    bool alnum = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z')
              || (c >= 'A' && c <= 'Z');
    return alnum ? s : Target::escape_marker(s);
}

} // namespace

void process_unicode_sequences(katom_list& katoms)
{
    (void)K::log(4);
    int definition_depth = 0;
    int code_depth = 0;              // inside an @eval/@read/@cond span
    std::vector<bool> apply_is_code; // one entry per open application
    for (size_t i = 0; i < katoms.size(); ++i) {
        Katom& k = katoms[i];
        switch (k.m_type) {
        case katom_t::define_begin:
        case katom_t::machine_begin:
            ++definition_depth;
            continue;
        case katom_t::define_end:
        case katom_t::machine_end:
            if (definition_depth > 0) --definition_depth;
            continue;
        case katom_t::eval_begin:
        case katom_t::read_begin:
        case katom_t::cond_begin:
            apply_is_code.push_back(true);
            ++code_depth;
            continue;
        case katom_t::apply_begin:
            apply_is_code.push_back(false);
            continue;
        case katom_t::apply_end:
            if (!apply_is_code.empty()) {
                if (apply_is_code.back()) --code_depth;
                apply_is_code.pop_back();
            }
            continue;
        default:
            break;
        }
        if (definition_depth > 0 || code_depth > 0) continue;
        if (k.m_type != katom_t::word && k.m_type != katom_t::text) continue;
        // The opener: "^" + one token, nothing else.
        if (k.m_text.size() < 2 || k.m_text[0] != '^') continue;
        if (!shape_token(k.m_text.substr(1))) continue;
        // Phase 1 -- SCAN by shape only, no validation: an unclosed run
        // must fall through untouched (it may be innocent prose about hex),
        // so an invalid code point may only error once the sequence has
        // actually closed.  Tokens are word katoms separated by SPACE
        // katoms; any other katom (a newline included: one line only)
        // aborts.
        std::vector<std::pair<std::string, const Locator*>> tokens {
            { k.m_text.substr(1), &k.m_loc } };
        std::vector<size_t> members {};
        bool closed = false;
        size_t j = i + 1;
        for (; j < katoms.size(); ++j) {
            Katom& t = katoms[j];
            if (t.m_type == katom_t::space) {
                members.push_back(j);
                continue;
            }
            if (t.m_type != katom_t::word && t.m_type != katom_t::text) break;
            const std::string& s = t.m_text;
            if (s.size() >= 2 && s.back() == '^'
                && shape_token(s.substr(0, s.size() - 1))) {
                tokens.push_back({ s.substr(0, s.size() - 1), &t.m_loc });
                members.push_back(j);
                closed = true;
                ++j;
                break;
            }
            if (shape_token(s)) {
                tokens.push_back({ s, &t.m_loc });
                members.push_back(j);
                continue;
            }
            break;
        }
        if (!closed) continue;
        // Phase 2 -- render, validating each code point at its own katom.
        std::string rendered {};
        for (const auto& [text, loc] : tokens) {
            rendered += render_token(text, *loc);
        }
        k.m_text = rendered;
        k.m_unparsed = false;
        for (size_t m : members) {
            mark_as_replaced(katoms[m]);
        }
        i = j - 1;
    }
}

// Literal

void mark_literal_katoms(katom_list& katoms)
{
    (void)K::log(4);
    for (auto [op, cl] : find_spans(katoms, begin_literal, end_literal, true, "literal")) {
        auto [begin, end] = find_span_katoms(katoms, op, cl);
        //if (begin->m_type == katom_t::literal_begin) {
        if (begin_literal(*begin)) {
            //begin->m_type = katom_t::replaced;
            mark_as_replaced(*begin);
            //end->m_type = katom_t::replaced;
            mark_as_replaced(*(end - 1));
            //std::for_each(begin + 1, end, [](Katom& k) { k.m_type = katom_t::literal; });
            // Hide Klammertext structural characters in the content as KTESC
            // markers so the literal text survives re-katomization (the
            // @document :text sub-Machine, the @eval result read-back).
            // Resolved back to the characters at final processing.  Literal
            // KLAMMER content (@code) is NOT treated this way -- it is marked
            // by mark_literal_klammer_content() and reaches the @eval code raw.
            std::for_each(begin + 1, end - 1, [](Katom& k) {
                mark_as_literal(k);
                k.m_text = hide_structural_characters(k.m_text);
            });
        }
    }
}

void hide_special_katoms(katom_list& katoms)
{
    // Replace the text of ^-quoted special-character katoms (^@, ^|, ^#, ^^,
    // ^:, ^*) with KTESC markers.  The katomizer strips the "^" when the
    // katom is constructed, so without this the bare character leaks into
    // assembled strings (state values, @eval results) and is re-interpreted
    // as Klammertext syntax when those strings are re-katomized -- by
    // @document's :text sub-Machine or the @eval result read-back in
    // Eval::eval.  Markers are inert text at every level and are resolved to
    // the characters at final processing (Target::resolve_escapes).
    //
    // Skipped inside:
    //   * @@...@@ and @@@...@@@ definition spans -- parameter declarations,
    //     descriptions, and argtype patterns are extracted as plain strings
    //     (kdesc display, validation regexes); a klammer BODY is re-processed
    //     through process_katoms() at application time, outside any
    //     definition span, so its quoted specials are hidden then.
    //   * @eval/@read/@cond argument spans -- code, filenames, and
    //     predicates consumed by the primitive, not target text (the same
    //     rule as the general-body escape pass in Machine::apply_klammer).
    (void)K::log(4);
    int definition_depth = 0;
    int code_depth = 0;              // inside an @eval/@read/@cond span
    std::vector<bool> apply_is_code; // one entry per open application
    for (auto& k : katoms) {
        switch (k.m_type) {
        case katom_t::define_begin:
        case katom_t::machine_begin:
            ++definition_depth;
            continue;
        case katom_t::define_end:
        case katom_t::machine_end:
            if (definition_depth > 0) --definition_depth;
            continue;
        case katom_t::eval_begin:
        case katom_t::read_begin:
        case katom_t::cond_begin:
            apply_is_code.push_back(true);
            ++code_depth;
            continue;
        case katom_t::apply_begin:
            apply_is_code.push_back(false);
            continue;
        case katom_t::apply_end:
            if (!apply_is_code.empty()) {
                if (apply_is_code.back()) --code_depth;
                apply_is_code.pop_back();
            }
            continue;
        default:
            break;
        }
        if (k.m_type == katom_t::special &&
            definition_depth == 0 && code_depth == 0) {
            k.m_text = hide_structural_characters(k.m_text);
        }
        // "^" before ANY punctuation character quotes it, not only the six
        // Klammertext specials the katomizer knows: ^- is a literal hyphen
        // (no dash transform), ^~ a literal tilde.  The pair becomes the
        // KTESC marker of the character here -- text-level, on writer-text
        // katoms only, under the same span skips as the quoted specials
        // above -- so code inside an @eval keeps its carets (grep '^-'
        // reaches the shell intact) and ^'...'^ content (katom_t::literal,
        // marked before this pass runs) stays exactly as typed.  How the
        // quoted character renders is the target's decision: the :escape
        // and :resolve tables map its marker, and an unmapped marker
        // decodes to the character itself.  A katom this converts is no
        // longer unparsed (the ^- warning would otherwise misfire).
        if ((k.m_type == katom_t::word || k.m_type == katom_t::text) &&
            definition_depth == 0 && code_depth == 0) {
            if (hide_quoted_punctuation(k.m_text)) {
                k.m_unparsed = false;
            }
        }
    }
}

// Ignore

void mark_ignored_katoms(katom_list& katoms)
{
    (void)K::log(4);
    //auto is_ignore_begin = [](const Katom& k) { return k.m_type == katom_t::ignore_begin; };
    //auto is_ignore_end = [](const Katom& k) { return k.m_type == katom_t::ignore_end; };
    for (auto [op,cl] : find_spans(katoms, begin_ignore, end_ignore, true, "ignored")) {
        auto [begin, end] = find_span_katoms(katoms, op, cl);
        // std::for_each(begin, end + 1, [](Katom& k) { k.m_type = katom_t::ignored; });
        std::for_each(begin, end + 1, mark_as_ignored);
        //if (end + 1 != katoms.end() && (end+1)->m_type == katom_t::newline) {
        if (end + 1 != katoms.end() && is_newline(*(end + 1))) {
            //(end+1)->m_type = katom_t::ignored;
            mark_as_ignored(*(end + 1));
        }
    }
    bool ignore_line = false;
    bool ignore_rest = false;
    std::vector<katom_iter> after_line {};
    for (Katom& k : katoms) {
        //if (k.m_type == katom_t::ignore_rest) {
        if (is_ignore_rest(k)) { 
            ignore_rest = true;
        // } else if (k.m_type == katom_t::ignore_line) {
        } else if (is_ignore_line(k)) {
            ignore_line = true;
        }
        //if (!ignore_rest && ignore_line && k.m_type == katom_t::newline) {
        if (!ignore_rest && ignore_line && is_newline(k)) {
            //k.m_type = katom_t::ignored;
            mark_as_ignored(k);
            ignore_line = false;
        }
        if (ignore_line or ignore_rest) {
            //k.m_type = katom_t::ignored;
            mark_as_ignored(k);
        }
    }
    size_t i = 0;
    while (i < katoms.size()) {
        if (katoms[i].m_initial_type == katom_t::ignore_end) {
            // std::cout << "IGNORED: " << kall << ktype << kignored << katoms[i] << " " << katoms[i+1] << black << "\n";
            i++;
            while (i < katoms.size() && katoms[i].is_whitespace()) {
                katoms[i].m_type = katom_t::ignored;
                i++;
            }
        } else {
            i++;
        }
    }
}

// Klammers:

void process_klammer_katoms(katom_list& katoms)
{
    (void)K::log(4);
    for (auto [op, cl] : find_spans(katoms, begin_klammer_def, end_klammer_def , true, "define")) {
        auto [begin, end] = find_span_katoms(katoms, op, cl);
        for (auto k = begin + 1; k < end - 1; k++) {
            //k->m_type = katom_t::literal;
            mark_as_literal(*k);
        }
    }
}

// Application: read, cond, and defined klammers

// Cond

/*
void check_bar_count(katom_iter begin, katom_iter end)
{
    //int count = std::count_if(begin, end, [](const Katom& k) { return k.m_type == katom_t::bar; });
    int count = std::count_if(begin, end, is_bar); //[](const Katom& k) { return k.m_type == katom_t::bar; });

    if (count != 1 && count != 2) {
        std::stringstream ss {};
        ss << "Incorrectly formatted @cond klammer. There should only be one or two bar characters:\n"
           << "  @cond <predicate> | <result-if-true @\nor:\n"
           << "  @cond <predicate> | <result-if-true> | <result-if-false> @";
        throw Argument_error(ss.str(), begin->m_loc);
    }
}

bool is_true(const std::string& s)
{
    return s == "True" || s == "true" || s == "1";
}

void process_cond_katoms(katom_list& katoms)
{
    if (std::find_if(katoms.begin(), katoms.end(), begin_cond) != katoms.end()) {
        (void)K::log(3);
        //for (auto [op, cl] : find_spans(katoms, begin_cond, end_apply, true, "cond")) {
        for (auto [op, cl] : find_spans(katoms, level_increase, level_decrease, true, "cond")) {
            auto [begin, end] = find_span_katoms(katoms, op, cl);
            // msg() << "find_spans: " << std::pair(begin, end) << "\n";
            if (begin_cond(*begin)) {
                check_bar_count(begin, end);
                auto bar_1 = std::find_if(begin, end, is_bar);
                std::string predicate = to_string(begin + 1, bar_1, true);
                auto bar_2 = std::find_if(bar_1 + 1, end - 1, is_bar);
                katom_list true_clause {};
                katom_list false_clause {};
                if (bar_2->m_type == katom_t::bar) {
                    true_clause = katom_list(bar_1 + 1, bar_2);
                    false_clause = katom_list(bar_2 + 1, end - 1);
                } else {
                    true_clause = katom_list(bar_1 + 1, end - 1);
                }
                katom_list result = is_true(predicate)
                    ? trim_whitespace(true_clause) : trim_whitespace(false_clause);
                std::for_each(begin, end, mark_as_replaced);
                katoms.insert(end, result.begin(), result.end());
            }
        }
    }
}
*/