klammertext/mac/machine.cpp

#include <set>
#include <utility>
#include "machine.h"
#include "error.h"
#include "show.h"
#include "util.h"
#include "file.h"
#include "log.h"
#include "eval.h"

Machine::Machine()
    : m_argtypes(Argtype_registry())
    , m_state(State())
    , m_targets(Target_registry())
    , m_klammers(Klammer_registry())
{
    (void)K::log(3);
    m_state.add_environment_frame();
    /*
    if (sks) {
        fs::path sks_filename(klammertext_filename("sks/sks.k"));
        // msg() << "SKS filename: " << sks_filename << "\n";
        read(sks_filename);
    }
    */
}

// Klammer application recursion guard.
//
// Applying a klammer expands its body, which is processed and applied in
// turn (apply_klammer -> process_katoms -> apply -> apply_klammer), so a
// klammer that reaches itself -- directly (@@f : x @f@ @@) or through a
// cycle -- descends without bound.  Before this guard the descent simply
// exhausted the C++ stack: SIGSEGV, no message, no location.
//
// The counter is a translation-unit static rather than a Machine member for
// two reasons: recursion can cross Machine instances (Eval::eval builds a
// sub-Machine to re-read an @eval result, and that sub-Machine applies
// klammers on the same C++ stack), and keeping it out of Machine avoids
// changing the class layout shared with the dlopened sks/document.so.
//
// The limit bounds the C++ stack, not the language: it is far above any
// plausible nesting depth in a document, and reaching it means a klammer
// does not terminate.  NOTE: not thread-safe; if input files are ever
// processed in parallel this needs to become thread_local.
namespace {

constexpr int max_apply_depth = 200;
int apply_depth = 0;

// Rounds of the top-level fixed-point loop (see Machine::apply below).  The
// former limit of 5 was a silent truncation; it is now an error, so it is set
// well clear of any legitimate chain of klammers generating klammers.
constexpr int apply_round_limit = 100;

class Depth_guard
{
public:
    Depth_guard(const std::string& name, const Locator& loc)
    {
        if (apply_depth >= max_apply_depth) {
            std::stringstream ss {};
            ss << "Klammer application nested more than " << max_apply_depth
               << " levels deep while applying " << q_(name) << ".\n"
               << "A klammer that applies itself, directly or through a cycle "
               << "of klammers, does not terminate.";
            throw Recursion_error(ss.str(), loc, false);
        }
        ++apply_depth;
    }
    ~Depth_guard() { --apply_depth; }
    Depth_guard(const Depth_guard&) = delete;
    Depth_guard& operator=(const Depth_guard&) = delete;
};

} // namespace

void Machine::process_eval_katoms(katom_list& katoms)
{
    (void)K::log(3);
    if (std::find_if(katoms.begin(), katoms.end(), begin_eval) != katoms.end()) {
        for (const auto& [op, cl] : find_spans(katoms, level_increase, level_decrease, true, "eval")) {
            auto [begin, end] = find_span_katoms(katoms, op, cl);
            // Inert: inside a @cond branch that has not been selected.  An
            // @eval in a discarded branch must not run.
            if (begin->m_deferred) continue;
            if (begin_eval(*begin)) {
                Eval E(*this, begin->m_loc);
                katom_list eval_katoms = E.eval(begin, end);
                std::for_each(begin, end, mark_as_replaced);
                katoms.insert(end, eval_katoms.begin(), eval_katoms.end());
            }
        }
    }
}

// Collect the bars that are direct argument separators of a @cond span:
// the bar katoms at nesting depth 0 within the span.  A bar that lies
// inside a nested span — for example the "|" in an inner @frac a | b @, or
// in a nested @eval/@read/@cond — has positive depth and is excluded.
//
// This makes @cond's argument delimitation a property of the span tree
// (the operad's arity: each operator owns the bars at its own level) rather
// than of the flat katom range.  Counting every bar in the range, as the
// original check did, conflated the arities of nested operators and rejected
// well-formed input such as
//   @cond *bool* | @frac 1 | 2 @ | @frac 2 | 1 @ @
// because the inner @frac bars were miscounted as @cond separators.
//
// begin is the cond_begin katom; end is one past the closing apply_end, so
// *(end - 1) is the apply_end.  Bars are returned in source order.
std::vector<katom_iter> cond_separator_bars(katom_iter begin, katom_iter end)
{
    std::vector<katom_iter> bars {};
    int depth = 0;
    for (auto it = begin + 1; it != end - 1; ++it) {
        if (is_bar(*it) && depth == 0) {
            bars.push_back(it);
        } else if (level_increase(*it)) {
            ++depth;
        } else if (level_decrease(*it)) {
            --depth;
        }
    }
    return bars;
}

void check_bar_count(katom_iter begin, std::size_t count)
{
    if (count != 1 && count != 2) {
        std::stringstream ss {};
        ss << "Incorrectly formatted @cond klammer. There should only be one or two bar characters:\n"
           << "  @cond <predicate> | <result-if-true @\nor:\n"
           << "  @cond <predicate> | <result-if-true> | <result-if-false> @";
        throw Argument_error(ss.str(), begin->m_loc, false);
    }
}

bool is_true(const std::string& s)
{
    return s == "True" || s == "true" || s == "1";
}

// @cond's predicate relation is TOTAL AND STRICT (Andy, 2026-08-15, deciding
// notes/Klammertext_improvements.md §4.1): there is a defined true set, a
// defined false set, and anything else is an error at the @cond.
//
// It was partial in effect until then -- is_true() recognized three strings and
// treated EVERYTHING else as false, so a misspelled state variable, a "TRUE", a
// "yes", or a Python traceback all silently selected the false branch.  The
// 2026-08-01 work made that visible with a warning while the policy was
// undecided; the warning found nothing in the SKS, which is the evidence that
// the corpus uses well-formed predicates and that the blast radius is small.
//
// Empty stays in the FALSE set, and deliberately: an optional argument that
// was not written substitutes as empty, so "absent means false" is what
// carries the "@cond *opt* | ... @" idiom.  The entangled sub-question in
// §4.1 -- whether empty means false or means "not supplied" -- is answered
// "false" by that use, not left open.
bool is_recognized_predicate(const std::string& s)
{
    return s.empty()
        || s == "True"  || s == "true"  || s == "1"
        || s == "False" || s == "false" || s == "0";
}

void check_predicate(const std::string& predicate, const Locator& loc)
{
    if (is_recognized_predicate(predicate)) return;
    std::stringstream ss {};
    ss << "The @cond predicate " << q_(predicate) << " is not a truth value. "
       << "Recognized: true, True, 1 (true); false, False, 0, and empty (false). "
       << "A value outside these is an error rather than false, so a misspelled "
       << "variable or a failed @eval cannot silently select a branch.";
    throw Argument_error(ss.str(), loc);
}

// Mark the interior of every @cond span INERT.  Runs during process_katoms,
// before the passes with observable effects, so that @eval and @read inside a
// branch do nothing until a branch is selected -- which is the non-strictness
// doc/cond_evaluation_order.md already specifies ("with side-effecting
// @read/@eval, wrong ... must not read the missing file") and which @eval did
// not honour: an @eval in a discarded branch used to run, because eval swept
// the list before cond did.
//
// The delimiters themselves stay unmarked, so the span is still found later.
// Nested @cond spans are marked by the enclosing one and become live only when
// the branch holding them is selected and processed.
void Machine::mark_cond_content(katom_list& katoms)
{
    (void)K::log(4);
    if (std::find_if(katoms.begin(), katoms.end(), begin_cond) == katoms.end()) {
        return;
    }
    for (const auto& [op, cl] : find_spans(katoms, level_increase, level_decrease, true, "cond")) {
        auto [begin, end] = find_span_katoms(katoms, op, cl);
        if (!begin_cond(*begin)) continue;
        // Only the BRANCHES are inert.  The predicate is always evaluated --
        // that is what a conditional is -- so marking from "begin + 1" would
        // stop "@cond @eval 1==1 @ | yes | no @" from ever computing its own
        // predicate.  Mark from the first depth-0 bar onward.
        std::vector<katom_iter> bars = cond_separator_bars(begin, end);
        if (bars.empty()) continue;   // malformed; reported when it resolves
        for (auto k = bars[0] + 1; k < end - 1; ++k) {
            k->m_deferred = true;
        }
    }
}

// Resolve the @cond spans in `katoms`, at APPLICATION time.  Returns the number
// resolved, so the caller's fixed point accounts for them.
//
// The selected branch is spliced and then processed exactly as a klammer body
// is (process_katoms + apply, the two lines apply_klammer already uses): that
// is what makes the document behave like a function body whose state variables
// are its arguments -- they are bound by the substitution at the top of
// Machine::apply, BEFORE any conditional in the document is decided.  Resolving
// @cond at read time meant a top-level "@cond *Flag*" saw the literal "*Flag*".
int Machine::resolve_cond_katoms(katom_list& katoms, const std::string& target)
{
    int resolved = 0;
    if (std::find_if(katoms.begin(), katoms.end(), begin_cond) != katoms.end()) {
        (void)K::log(3);
        for (const auto& [op, cl] : find_spans(katoms, level_increase, level_decrease, true, "cond")) {
            auto [begin, end] = find_span_katoms(katoms, op, cl);
            if (begin_cond(*begin)) {
                // A @cond nested inside an unresolved outer @cond is still
                // inert; the outer one will process it when its branch is
                // selected.  Without this an inner branch would be decided
                // before it is known whether it is reached at all.
                if (begin->m_deferred) continue;
                // Delimit @cond's arguments by the bars at depth 0 within the
                // span, so that bars belonging to nested klammers are not
                // mistaken for @cond's own separators (see cond_separator_bars).
                std::vector<katom_iter> bars = cond_separator_bars(begin, end);
                check_bar_count(begin, bars.size());
                auto bar_1 = bars[0];
                std::string predicate = to_string(begin + 1, bar_1, true);
                check_predicate(predicate, begin->m_loc);
                katom_list true_clause {};
                katom_list false_clause {};
                if (bars.size() == 2) {
                    auto bar_2 = bars[1];
                    true_clause = katom_list(bar_1 + 1, bar_2);
                    false_clause = katom_list(bar_2 + 1, end - 1);
                } else {
                    true_clause = katom_list(bar_1 + 1, end - 1);
                }
                // Splice only the selected branch.  Its nested klammers remain
                // unreduced here and are reduced by the outer fixed-point apply
                // loop; the unselected branch is discarded without evaluation
                // (@cond is a non-strict special form).
                katom_list result = is_true(predicate)
                    ? trim_whitespace(true_clause) : trim_whitespace(false_clause);
                // The selected branch becomes live: clear the inert flag and
                // give it the same processing a klammer body gets.  The
                // unselected branch is discarded still inert, so nothing in it
                // ever ran.
                for (Katom& k : result) {
                    k.m_deferred = false;
                }
                process_katoms(result, command_name);
                apply(m_klammers, result, target);
                std::for_each(begin, end, mark_as_replaced);
                katoms.insert(end, result.begin(), result.end());
                ++resolved;
            }
        }
    }
    return resolved;
}


// Expand the constant klammers written in a definition's BODY.  A constant is
// expanded at definition time, which is what makes it a constant; the body is
// where that is meaningful.
//
// The parameter list is deliberately excluded.  A klammer application there
// is an error (see expand_option_sets): parameters shared between klammers
// are declared by an option set, whose ".o" declaration is resolved as the
// parameter list is read.  Expanding a constant into a parameter list used to
// be the way to share parameters, and it silently destroyed the parameter
// list of a ".k" declaration -- the spliced options AND the declared
// positionals -- surfacing only as an argument error at the first
// application, in the document rather than the declaration.
void Machine::expand_constant_klammers(katom_list& katoms, const Katom& op, const Katom& cl)
{
    auto [begin, end] = find_span_katoms(katoms, op, cl);
    restore_initial_type(begin + 1, end - 1);
    auto body_begin = std::find_if(
        begin + 1, end - 1, [](const Katom& k) { return is_deftype(k.m_type); });
    if (body_begin == end - 1) return;
    if (std::find_if(body_begin, end - 1, begin_klammer_apply) == end - 1) return;
    for (const auto& [app_op, app_cl] : find_spans(body_begin, end - 1, begin_apply, end_apply, false, "def-time")) {
        auto [app_begin, app_end] = find_span_katoms(katoms, app_op, app_cl);
        if (app_begin->m_type == katom_t::apply_begin) {
            std::string name = trim_char(app_begin->m_text, '@');
            const auto* body = m_klammers.constant_body(name);
            if (body) {
                // Set both m_type and m_initial_type so that
                // restore_initial_type() in add() won't resurrect them
                for (auto it = app_begin; it != app_end; ++it) {
                    it->m_type = katom_t::replaced;
                    it->m_initial_type = katom_t::replaced;
                }
                katoms.insert(app_end, body->begin(), body->end());
            }
        }
    }
}

//katom_list
void Machine::mark_literal_klammer_content(katom_list& katoms)
{
    (void)K::log(4);
    // Collect names of klammers that have a literal parameter
    std::set<std::string> literal_names {};
    for (const auto& [name, klammer] : m_klammers.m_klammers) {
        if (klammer.has_literal_param())
            literal_names.insert(name);
    }
    if (literal_names.empty()) return;

    // Scan for matching @name ... name@ spans.
    //
    // REMOVED TEXT IS SKIPPED.  This pass runs first in process_katoms(), before
    // mark_ignored_katoms() takes out the "#" forms, and that ordering is not
    // accidental: a literal klammer's content must be marked before anything
    // else can interpret what is inside it, or removal would take a "#" that
    // belongs to the literal body.  The cost is that this scan sees text the
    // writer has removed, so a literal klammer merely NAMED in a comment --
    //
    //     # @image and @code share the same arguments.
    //
    // -- was found as an opening delimiter, went unclosed, and failed the whole
    // file (TODO #40, diagnosed 2026-08-05).  "##" was already handled, which is
    // half the case; the fix is to teach the scan the other three removal forms
    // rather than to reorder the passes.
    //
    // The converse still holds, structurally: this skipping happens only while
    // looking for an OPENING delimiter, and finding one jumps k past the whole
    // span, so a "#" inside literal content is never examined here and stays
    // content.
    for (auto k = katoms.begin(); k != katoms.end(); ++k) {
        if (k->m_type == katom_t::ignore_rest) break;   // "##": rest of the file
        if (k->m_type == katom_t::ignore_line) {        // "#": rest of the line
            while (k + 1 != katoms.end() && (k + 1)->m_type != katom_t::newline) {
                ++k;
            }
            continue;
        }
        if (k->m_type == katom_t::ignore_begin) {       // "#[ ... ]#", nestable
            int removed = 1;
            while (++k != katoms.end() && removed > 0) {
                if (k->m_type == katom_t::ignore_begin) {
                    ++removed;
                } else if (k->m_type == katom_t::ignore_end) {
                    --removed;
                }
            }
            if (k == katoms.end()) break;   // unterminated: the rest is removed
            --k;                            // the loop's ++k steps past the "]#"
            continue;
        }
        if (k->m_type != katom_t::apply_begin) continue;
        std::string name = trim_char(k->m_text, '@');
        if (literal_names.count(name) == 0) continue;
        (void)K::log(2, "Literal klammer: " + name);

        // Find the matching named closing delimiter
        std::string close_text = name + "@";
        auto close = k + 1;
        int depth = 1;
        while (close != katoms.end()) {
            if (close->m_type == katom_t::apply_begin &&
                trim_char(close->m_text, '@') == name)
                depth++;
            else if (close->m_type == katom_t::apply_end &&
                     trim_char(close->m_text, '@') == name)
                depth--;
            if (depth == 0) break;
            ++close;
        }
        if (close == katoms.end()) {
            throw Parsing_error(
                "Klammer " + q_(name) + " has a literal parameter and must be closed with "
                + q_(close_text),
                k->m_loc);
        }

        // Count positional parameters before the literal one.
        // The literal parameter is always last. Bars separate
        // the preceding positional arguments and the literal content.
        const auto& klammer = m_klammers.m_klammers[name];
        int bars_before_literal = 0;
        for (const auto& p : klammer.m_parameters.m_positional) {
            if (p.m_argtype.m_name == "literal") break;
            bars_before_literal++;
        }

        // Find where literal content starts.
        // Skip bars_before_literal bars (separating preceding positional args).
        // If options are present, skip past the bar after them.
        // Options are identified by :name katoms before any bar.
        auto literal_start = k + 1;
        bool has_options = false;
        for (auto j = k + 1; j < close; ++j) {
            if (j->m_type == katom_t::option_name) {
                has_options = true;
            }
            if (j->m_type == katom_t::bar) {
                if (bars_before_literal > 0) {
                    bars_before_literal--;
                    literal_start = j + 1;
                } else if (has_options) {
                    // This bar separates options from literal content
                    literal_start = j + 1;
                    break;
                } else {
                    // No preceding args, no options: bar is part of literal
                    break;
                }
            }
        }
        std::for_each(literal_start, close, mark_as_literal);

        k = close;  // Skip past this span
    }
}

void Machine::process_katoms(
    katom_list& katoms, const std::string& source,
    bool nonascii, bool literal, bool ignore, bool whitespace, bool klammers,
    bool eval, bool cond, bool read)
{
    mark_literal_klammer_content(katoms);
    if (literal) mark_literal_katoms(katoms);
    hide_special_katoms(katoms);
    if (nonascii) encode_nonascii_characters(katoms);
    if (ignore) mark_ignored_katoms(katoms);
    if (whitespace) process_whitespace_modifiers(katoms);
    if (klammers) process_klammer_katoms(katoms);
    // @cond is no longer RESOLVED here -- it is resolved in the apply fold
    // (see mark_cond_content / resolve_cond_katoms).  What happens here is the
    // marking that makes its branches inert, and it must run BEFORE the eval
    // and read passes, which are the ones with observable effects.  Leaving it
    // where process_cond_katoms used to sit -- after eval -- kept the old
    // "eval in a discarded branch runs anyway" behaviour, since the marking
    // arrived too late to stop it.
    if (cond) mark_cond_content(katoms);
    if (eval) process_eval_katoms(katoms);
    if (read) expand_read_katoms(
        katoms, source,
        nonascii, literal, ignore, whitespace, klammers, eval, cond, read);
    warn_unparsed_katoms(katoms, m_warn_unparsed);
//    return katoms;
}

katom_list Machine::process(
    std::string text, const std::string& source,
    bool nonascii, bool literal, bool ignore, bool whitespace, bool klammers,
    bool eval, bool cond, bool read)
{
    katom_list katoms = katomize(line_split(text), source);
    //katoms =
    process_katoms(
        katoms, source,
        nonascii, literal, ignore, whitespace, klammers, eval, cond, read);    
    return katoms;
}

void Machine::read(const fs::path& pathname)
{
    (void)K::log(3, pathname.string());
    // A file's directory joins the @eval search path (Python modules and
    // :cpp libraries live next to the file that uses them).
    m_state.add_search_dir(fs::absolute(pathname).parent_path().string());
    m_state.open_frame("Machine state: " + pathname.string());
    std::string text = m_state.subst(trim_right(string_from_file(pathname)));
    katom_list katoms = process(text, pathname);
    m_katoms.insert(m_katoms.end(), katoms.begin(), katoms.end());
    extract_machine_definitions();
    extract_klammer_definitions();
    m_sources.push_back(pathname);
}

void Machine::read(const std::string& s)
{
    (void)K::log(3, s);
    m_state.open_frame("Machine state: " + s);
    std::string text = m_state.subst(trim_right(s));
    katom_list katoms = process(text, command_pathname);
    m_katoms.insert(m_katoms.end(), katoms.begin(), katoms.end());
    extract_machine_definitions();
    extract_klammer_definitions();
    m_sources.push_back(s);
}    

// Read

void Machine::expand_read_katoms(
    katom_list& katoms, std::string current_filename,
    bool nonascii, bool literal, bool ignore, bool whitespace,
    bool klammers, bool eval, bool cond, bool read)
{
    (void)K::log(3);
    current_filename = resolve_relative_to(current_filename);
    // msg() << "current_filename: " << current_filename << "\n";
    if (std::find_if(katoms.begin(), katoms.end(), begin_read) != katoms.end()) {
        (void)K::log(3);
        for (const auto& [op, cl] : find_spans(katoms, begin_apply, end_apply, true, "read")) {
            auto [begin, end] = find_span_katoms(katoms, op, cl);
            // Inert: see the same guard in process_eval_katoms.
            if (begin->m_deferred) continue;
            if (begin_read(*begin)) {
                std::string read_filename = to_string(begin + 1, end - 1, true);

                // msg() << "read: " << resolve_relative_to(read_filename, current_filename) << "\n";

                /*
                std::string current_directory =
                    fs::path(current_filename).parent_path().string();
                fs::path input_filename =
                    fs::path(current_directory + "/" + read_filename);
                */

                fs::path input_filename = resolve_relative_to(read_filename, current_filename);
                // msg() << "read: " << input_filename << "\n";

                (void)K::log(2, input_filename.string());
                if (!fs::exists(input_filename)) {
                    std::stringstream ss{};
                    ss <<"File " << input_filename << " does not exist";
                    throw File_error(ss.str(), begin->m_loc);
                }
                std::for_each(begin, end, mark_as_replaced);

                input_filename = fs::canonical(input_filename);
                m_state.add_search_dir(input_filename.parent_path().string());
                std::string text = trim_right(string_from_file(input_filename.string()));
                katom_list ks = katomize(line_split(text), input_filename);
                // ks = 
                process_katoms(
                    // ks, command_pathname,
                    ks, input_filename,
                    nonascii, literal, ignore, whitespace, klammers, eval, cond, read);
                katoms.insert(end, ks.begin(), ks.end());
            }
        }
    }
}


// One @@@ declaration.  Factored out so the tolerant and strict paths share
// it; `rescan` is set when a klammerset has spliced files into the stream,
// which invalidates the caller's span list.
void Machine::add_machine_definition(
    const std::string& name, katom_iter begin, katom_iter end, bool& rescan)
{
    if (name == "@@@target") {
        m_targets.add(begin, end, m_katoms);
    } else if (name == "@@@argtype") {
        m_argtypes.add(begin, end, m_katoms);
    } else if (name == "@@@state") {
        m_state.parse_state_katoms(begin, end, m_katoms);
    } else if (name == "@@@klammerset") {
        if (auto klammerset = m_klammersets.add(begin, end, m_katoms)) {
            load_klammerset_files(*klammerset, end);
            rescan = true;
        }
    }
}

void Machine::extract_machine_definitions(bool tolerant)
{
    (void)K::log(3);
    if (m_katoms.empty()) {
        return;
    }
    // A @@@klammerset declaration inserts its files' katoms into the stream
    // at the declaration point, invalidating the span list, so the scan
    // restarts.  Processed spans are marked replaced and are never found
    // again, which also bounds the restarts.
    bool rescan = true;
    while (rescan) {
        rescan = false;
        for (const auto& [op, cl] : find_spans(m_katoms, begin_machine_def, end_machine_def, true, command_name)) {
            auto [begin, end] = find_span_katoms(m_katoms, op, cl);
            std::string name = begin->m_text;
            if (tolerant) {
                // kdiag: a declaration that cannot be carried out -- a
                // @@@klammerset naming a file that is not there, say -- is
                // skipped rather than fatal.  Its span stays unreplaced, so
                // the katoms remain visible and report the failure themselves.
                try {
                    add_machine_definition(name, begin, end, rescan);
                } catch (Error& e) {
                    K::log(1, "Machine definition not registered: " + e.m_desc);
                    continue;
                }
            } else {
                add_machine_definition(name, begin, end, rescan);
            }
            if (rescan) {
                break;
            }
        }
    }
}

void Machine::load_klammerset_files(const Klammerset& klammerset, katom_iter insert_at)
{
    (void)K::log(2, "Loading klammerset \"" + klammerset.m_symbol + "\"");
    // Relative names resolve against the declaring file's directory, never
    // the cwd.  :requires files are read before the set's own files; each
    // holds its own @@@klammerset declaration, whose already-loaded guard
    // makes repeated requirements a no-op.
    fs::path declaring(klammerset.m_loc.m_filename);
    fs::path base = fs::exists(declaring) ? declaring : fs::current_path();
    std::string base_dir = (is_directory(base) ? base : base.parent_path()).string();

    // A :requires entry may be a bare symbol, resolved on the klammerset
    // search path with the declaring directory as the local stage; :files
    // entries are always filenames (this set's own definition files).
    std::vector<std::string> filenames {};
    for (const auto& required : klammerset.m_requires) {
        std::string path = is_klammerset_symbol(required)
            ? resolve_klammerset_symbol(required, base_dir, klammerset.m_loc).string()
            : required;
        // ALREADY LOADED?  Decided BEFORE the file is opened.  The guard used
        // to sit in Klammerset_registry::add, which only runs once the file has
        // been read and its declaration reached -- by which time the declaring
        // file's OWN definitions have been re-executed.  Two sets requiring a
        // third therefore died on "Target ... is already defined", pointing at
        // a line the author wrote once.
        //
        // Deciding it here is possible only because a klammerset symbol X is
        // declared in X/X.k, so the symbol is a function of the path: it is
        // known for a ":requires" written as a filename just as much as for one
        // written as a symbol.  That is what the X/X.k requirement bought.
        std::string symbol = is_klammerset_symbol(required)
            ? required : fs::path(path).stem().string();
        if (m_klammersets.has(symbol)) {
            (void)K::log(1, "Klammerset \"" + symbol + "\": already loaded, skipped");
            continue;
        }
        filenames.push_back(path);
    }
    filenames.insert(filenames.end(), klammerset.m_files.begin(), klammerset.m_files.end());

    // Collect all files into one list and insert once: insert_at is
    // invalidated by the first insertion into m_katoms.
    // Recorded on the set: what it actually opened, canonical, so a display
    // can tell its klammers from the input's.  The declaring file counts --
    // definitions may sit before or after the declaration in it.
    strings_t loaded_files {};
    if (fs::exists(declaring)) {
        loaded_files.push_back(fs::canonical(declaring).string());
    }
    katom_list loaded {};
    for (const auto& filename : filenames) {
        fs::path pathname = resolve_relative_to(filename, base);
        if (!fs::exists(pathname)) {
            throw Klammerset_error(
                "Klammerset \"" + klammerset.m_symbol + "\" lists the file \"" + filename
                + "\", which does not exist (resolved to \"" + pathname.string() + "\")",
                klammerset.m_loc);
        }
        pathname = fs::canonical(pathname);
        loaded_files.push_back(pathname.string());
        m_state.add_search_dir(pathname.parent_path().string());
        std::string text = trim_right(string_from_file(pathname.string()));
        katom_list ks = katomize(line_split(text), pathname);
        process_katoms(ks, pathname);
        loaded.insert(loaded.end(), ks.begin(), ks.end());
    }
    m_katoms.insert(insert_at, loaded.begin(), loaded.end());
    // The registry holds the registered copy; the parameter is a const
    // reference to it, so the record goes back through the registry.
    m_klammersets.set_loaded_files(klammerset.m_symbol, loaded_files);
}

// Register one "@@...@@" definition.  An ".o" target declares an option set
// -- parameters shared by klammers -- and goes to its own registry: it
// defines no klammer and produces no output for any target.
void Machine::add_definition(katom_list& katoms, const Katom& op, const Katom& cl)
{
    expand_constant_klammers(katoms, op, cl);
    auto [begin, end] = find_span_katoms(katoms, op, cl);
    auto [name, target_names, general_declared] = parse_name(m_targets, *begin);
    (void)general_declared;   // routing only cares whether this is a ".o"
    // parse_name rejects ".o" as a member of a target list, so an option set
    // declaration is always the single-target form.
    if (target_names.size() == 1 && target_names[0] == Target_registry::optionset_name) {
        m_option_sets.add(m_argtypes, name, begin, end, katoms);
    } else {
        m_klammers.add(m_argtypes, m_targets, m_option_sets, begin, end, katoms);
    }
}

void Machine::extract_klammer_definitions(katom_list katoms)
{
    // fmsg() << katoms << "\n";
    (void)K::log(3, katoms);
    for (const auto& [op, cl] : find_spans(katoms, begin_klammer_def, end_klammer_def, true, command_name)) {
        add_definition(katoms, op, cl);
    }
    m_klammers.rationalize(m_targets);
}

void Machine::extract_klammer_definitions(bool tolerant)
{
    (void)K::log(3);
    for (const auto& [op, cl] : find_spans(m_katoms, begin_klammer_def, end_klammer_def, true, command_name)) {
        if (tolerant) {
            // add_definition marks the span replaced only on success, so a
            // skipped definition keeps its katoms and stays visible.
            try {
                add_definition(m_katoms, op, cl);
            } catch (Error& e) {
                K::log(1, "Definition not registered: " + e.m_desc);
            }
        } else {
            add_definition(m_katoms, op, cl);
        }
    }
    m_klammers.rationalize(m_targets);
}

void Machine::update_state(const std::map<std::string, std::string>& arg_map)
{
    for (const auto& [k, v] : arg_map) {
        m_state.set(k, v);
    }
}

katom_list Machine::apply_klammer(
    Klammer& klammer, const std::string& target, katom_iter arguments_begin, katom_iter arguments_end)
{
    (void)K::log(3, "argument substitution", *arguments_begin, *(arguments_end - 1));
    Depth_guard depth_guard(klammer.m_name, arguments_begin->m_loc);
    m_state.replace("K_loc", arguments_begin->m_loc.str(), false);
    auto [positional, optional, rest] =
        argument_split(arguments_begin + 1, arguments_end - 1, klammer.m_parameters.m_positional.size());
    auto values = klammer.m_parameters.value_map(positional, optional, rest, arguments_begin->m_loc);

    // Resolve KTESC markers in argument values so that @eval code receives
    // the original characters (e.g., filenames with underscores).  The markers
    // remain in the klammer body substitution for final target-specific output.
    katom_list result(klammer.m_body[target].begin(), klammer.m_body[target].end());
    auto varmap = klammer.m_varmap[target];
    m_state.open_frame("Arguments for klammer " + q_(klammer.m_name));
    m_state.set(values, klammer.m_parameters);
    for (const auto& [name, indices] : varmap) {
        std::regex arg("\\*" + name + "\\*");
        for (auto i : indices) {
            result[i].m_text = std::regex_replace(result[i].m_text, arg, m_state.value(name));
            result[i].m_type = katom_t::text;
        }
    }
    // Escape target-specific characters (e.g. tex "&" -> "\&") in the writer
    // text of a GENERAL klammer's body.  Runs BEFORE process_katoms/apply()
    // below expand the body, so that target-native markup pulled in by nested
    // klammers (e.g. nl.tex -> "\newline") is left untouched -- only this
    // klammer's own literal writer text is escaped here; nested klammers escape
    // theirs when they are applied in turn.  Bodies from target-specific
    // definitions (m_body_generic[target] == false) are already in target form
    // and skipped.  KTESC markers are idempotent, so text already escaped at the
    // top level passes through unchanged.  Two kinds of body content are NOT
    // writer text and must be skipped:
    //   * ^'...'^ literal spans -- raw target markup the writer typed directly.
    //     At this point they are typed literal_begin/literal_end with plain-text
    //     content (the literal phase runs in process_katoms, below), so track
    //     span depth rather than testing katom type.
    //   * @eval / @read / @cond argument spans -- code, filenames, and
    //     predicates consumed by the primitive, NOT emitted as target text.
    //     (Escaping an underscore in "offer.Price_list(K)" broke @eval.)  The
    //     primitive's KLAMMERTEXT result, produced by process_katoms below, is
    //     klammer output and is likewise never escaped -- it is inserted after
    //     this pass and so is untouched, matching the top-level behavior where
    //     @eval is resolved before the escape pass runs.
    auto gen = klammer.m_body_generic.find(target);
    if (gen != klammer.m_body_generic.end() && gen->second) {
        Target tgt = m_targets.get(target, Locator());
        if (!tgt.m_escapes.empty()) {
            int literal_depth = 0;
            int code_depth = 0;              // inside an @eval/@read/@cond span
            std::vector<bool> apply_is_code; // one entry per open application
            for (auto& k : result) {
                if (k.m_type == katom_t::literal_begin) { ++literal_depth; continue; }
                if (k.m_type == katom_t::literal_end) {
                    if (literal_depth > 0) --literal_depth;
                    continue;
                }
                if (k.m_type == katom_t::eval_begin ||
                    k.m_type == katom_t::read_begin ||
                    k.m_type == katom_t::cond_begin) {
                    apply_is_code.push_back(true);
                    ++code_depth;
                    continue;
                }
                if (k.m_type == katom_t::apply_begin) {
                    apply_is_code.push_back(false);
                    continue;
                }
                if (k.m_type == katom_t::apply_end) {
                    if (!apply_is_code.empty()) {
                        if (apply_is_code.back()) --code_depth;
                        apply_is_code.pop_back();
                    }
                    continue;
                }
                if (literal_depth == 0 && code_depth == 0 &&
                    (k.m_type == katom_t::text ||
                     k.m_type == katom_t::word ||
                     k.m_type == katom_t::newline))
                    k.m_text = tgt.escape_text(k.m_text);
            }
        }
    }
    process_katoms(result, klammer.m_name);
    apply(m_klammers, result, target);
    m_state.close_frame();
    // msg() << boldblack << "APPLY: " << std::pair(arguments_begin, arguments_end) << "\n"
    //      << boldblack << "RESULT: " << ktype << result << black << "\n";

    modify_type(katom_t::replaced, arguments_begin, arguments_end);
    return result;
}

void Machine::apply_klammer_registry(
    Klammer_registry& klammer_registry, katom_list& katoms, const std::string& target, katom_iter begin, katom_iter end)
{
    (void)K::log(3, "Klammer");
    std::string name = trim_char(begin->m_text, '@');
    katom_list applied_katoms = apply_klammer(klammer_registry.m_klammers[name], target, begin, end);
    for (auto& k : applied_katoms) {
        if (k.m_type == katom_t::bar || k.m_type == katom_t::double_bar || k.m_type == katom_t::option_name) {
            k.m_type = katom_t::text;
        }
    }
    katoms.insert(end, applied_katoms.begin(), applied_katoms.end());
}

int Machine::apply(
    Klammer_registry& klammer_registry, katom_list& katoms, const std::string& target)
{
    (void)K::log(3, "Klammer_registry");
    // @cond is a special form handled here rather than by the klammer loop
    // below: its span head is a cond_begin, which begin_klammer_apply does not
    // match.  Resolving first means a klammer revealed by the selected branch
    // is applied in this same pass.  The count is returned with the klammer
    // applications, so the caller's fixed point iterates while either happens.
    int applied = resolve_cond_katoms(katoms, target);
    for (const auto& [op, cl] : find_spans(
             katoms, begin_klammer_apply, end_klammer_apply, true, command_name)) {
        auto [begin, end] = find_span_katoms(katoms, op, cl);
        klammer_registry.check_klammer(
            klammer_name_from_katom(begin->m_text, begin->m_loc),
            target, begin->m_loc);
        apply_klammer_registry(klammer_registry, katoms, target, begin, end);
        ++applied;
    }
    return applied;
}

std::string Machine::run_phase_functions()
{
    Target target = m_targets.get(m_state.value("K_target"), Locator());
    if (!target.m_after_apply.empty()) {
        (void)K::log(2, target);
        for (auto f : target.m_after_apply) {
            // A mode-tagged spec (":cpp ...") names a function that receives
            // the Machine itself; a bare Python function is called with the
            // result text. The Eval is constructed per phase so a chained
            // phase sees its predecessor's result in K_result.
            Eval E(*this, Locator());
            if (!f.empty() && f[0] != ':') {
                f += "(K_result)";
            }
            f = "@eval " + f + " @";
            auto katoms = katomize(line_split(f), "phase");
            // A phase function's input and output are final target text, not
            // Klammertext: take the raw result string.  Re-reading it as
            // Klammertext (Eval::eval) would misparse target characters --
            // e.g. a "@" from a quoted ^@ in justified txt output.
            m_result = E.eval_command(katoms.begin(), katoms.end() - 2);
        }
    }
    return m_result;
}


void Machine::escape_target_characters(const Target& target, katom_list& katoms)
{
    if (target.m_escapes.empty()) return;
    for (auto& k : katoms) {
        // Only escape writer content katoms — text, words, and newlines.
        // Skip structural katoms (option names, bars, klammer delimiters)
        // whose text is Klammertext syntax, not writer content.
        if (k.m_type == katom_t::text ||
            k.m_type == katom_t::word ||
            k.m_type == katom_t::newline) {
            k.m_text = target.escape_text(k.m_text);
        }
    }
}

std::string Machine::apply(const std::string& target_name, bool final_processing, bool escape_characters)
{
    (void)K::log(3, "top level");
    m_state.set("K_target", target_name);
    m_state.subst(m_katoms.begin(), m_katoms.end());

    // Escape target-specific characters in writer text before klammer application.
    // Characters produced later by klammer bodies will not be escaped.
    // Skipped for sub-Machine apply() calls (e.g., from @eval), where the
    // text is already in target-specific form.
    auto target = m_targets.get(target_name, Locator());
    if (escape_characters)
        escape_target_characters(target, m_katoms);

    // Reduce to a fixed point.  A pass reports how many klammers it applied;
    // the loop ends when a pass applies none.  (It formerly ended when the
    // katom list stopped GROWING, which is not the same thing: a klammer whose
    // body expands to nothing is a reduction that adds no katoms.)  Exceeding
    // the round limit is now an error rather than a message followed by
    // rendering the unreduced document -- silently emitting a document with
    // live klammers still in it is worse than not emitting one.  Runaway
    // recursion is caught earlier and more precisely by the depth guard in
    // apply_klammer(); this limit only bounds the number of ROUNDS, which is
    // the length of a chain of klammers that generate further klammers.
    int apply_count = 0;
    while (apply(m_klammers, m_katoms, target_name) > 0) {
        if (++apply_count > apply_round_limit) {
            std::stringstream ss {};
            ss << "Klammer application did not reach a fixed point after "
               << apply_round_limit << " rounds.\n"
               << "Each round applies every klammer present; a klammer whose "
               << "result contains further klammers starts another round.";
            throw Recursion_error(ss.str(), Locator(), false);
        }
    }
    m_result = to_string(m_katoms.begin(), m_katoms.end());

    if (final_processing) {
        for (const auto& [old_str, new_str] : target.m_transforms) {
            m_result = string_replace(m_result, old_str, new_str);
        }
        m_result = target.resolve_escapes(m_result);
        m_result = run_phase_functions();
    }
    m_result = trim_char(m_result, '\n');
    return m_result;
}