// #include #include #include "katom_list.h" #include "util.h" #include "ktype.h" #include "file.h" #include "log.h" #include "show.h" #include "character.h" #include "target.h" std::string to_string(std::vector::const_iterator begin, std::vector::const_iterator end, bool trim_result) { (void)K::log(3); std::string result = std::accumulate( begin, end, std::string(""), [](const std::string& s, const Katom& k) { return k.is_active() ? s + k.m_text : s; }); if (trim_result) { result = trim(result); } return result; } std::string to_string(const katom_list& katoms, bool trim_result) { return to_string(katoms.cbegin(), katoms.cend(), trim_result); } bool level_increase(const Katom& k) { return k.m_type == katom_t::define_begin || k.m_type == katom_t::literal_begin || k.m_type == katom_t::ignore_begin || k.m_type == katom_t::read_begin || k.m_type == katom_t::eval_begin || k.m_type == katom_t::cond_begin || k.m_type == katom_t::machine_begin || k.m_type == katom_t::apply_begin; } bool level_decrease(const Katom& k) { return k.m_type == katom_t::apply_end || k.m_type == katom_t::define_end || k.m_type == katom_t::literal_end || k.m_type == katom_t::ignore_end || k.m_type == katom_t::machine_end; } const katom_list::iterator find_katom_named(const katom_list::iterator begin, const katom_list::iterator end, std::string name) { auto result = find_if( begin, end, [&](const Katom& ki) { return ki.m_text == ("@" + name); }); return result; } // Spans //katom_iter get_katom_iterator(katom_list& katoms, size_t index) const katom_list::iterator find_katom(const katom_list::iterator begin, const katom_list::iterator end, size_t index) { // msg() << "find_katom: " << std::pair(begin, end) << "\n"; auto result = find_if( begin, end, [&](const Katom& ki) { return ki.m_index == index && ki.m_type != katom_t::ignored && ki.m_type != katom_t::replaced; }); if (result == end) { std::stringstream ss {}; ss << "Katom with index " << index << " not found in katom list"; throw Internal_error(ss.str(), begin->m_loc); } else { return result; } } std::pair find_span_katoms(katom_list& katoms, const Katom& begin, const Katom& end) // Lint error { //(void)K::log(3, begin, end); //static std::mutex katoms_mutex; //std::lock_guard lock(katoms_mutex); katom_iter kbegin = find_katom(katoms.begin(), katoms.end(), begin.m_index); katom_iter kend = find_katom(kbegin, katoms.end(), end.m_index) + 1; //(void)K::log(3, kbegin->m_text, (kbegin+1)->m_text, (kbegin+2)->m_text, "...", kend->m_text); //(void)K::log(3, "resolved:", kbegin, (kbegin+2), "..."); //, kend); (void)K::log(3, "resolved:", kbegin, kend - 1); return { kbegin, kend }; } std::pair find_span_katoms(katom_iter katoms_begin, katom_iter katoms_end, const Katom& begin, const Katom& end) // Lint error { katom_iter kbegin = find_katom(katoms_begin, katoms_end, begin.m_index); katom_iter kend = find_katom(katoms_begin, katoms_end, end.m_index) + 1; //(void)K::log(3, kbegin->m_text, (kbegin+1)->m_text, (kbegin+2)->m_text, "...", kend->m_text); //(void)K::log(3, "resolved:", kbegin, (kbegin+2), "..."); //, kend); (void)K::log(3, "resolved:", kbegin, kend - 1); return { kbegin, kend }; } void missing_open(const Katom& k, bool error_exit) { (void)K::log(2); std::stringstream ss {}; ss << "A klammer ends without a beginning: " << k; if (error_exit) { throw Parsing_error(ss.str(), k.m_loc, false); } else { std::cout << " " << ss.str() << "\n"; } } //void missing_close(std::vector& bounds, bool error_exit) void missing_close(const katom_list& bounds, bool error_exit) { (void)K::log(2); std::stringstream ss {}; if (bounds.size() == 1) { ss << " Beginning of a span that does not end:\n"; } else { ss << " A span ends that does not have a beginning:\n"; } for (const auto& k : bounds) { ss << " " << k.m_loc << " " << k.m_src << "\n"; } if (error_exit) { throw Parsing_error(ss.str(), bounds[0].m_loc, false); } else { std::cout << ss.str() << "\n"; } } void bad_close(const Katom& open, const Katom& close, bool error_exit) { (void)K::log(2); std::stringstream ss {}; ss << " Klammer begins with " << open << " but ends with " << close << ".\n" << " " << open.m_loc << " " << open << "\n" << " " << close.m_loc << " " << close; if (error_exit) { throw Parsing_error(ss.str(), open.m_loc, false); } else { std::cout << ss.str() << "\n"; } } std::string trim_span_markers(const Katom& katom) { // Trim @ as well as ^ (for literal span) return trim_char(trim_char(katom.m_text, '@'), '^'); } void check_named_katom_span(const Katom& begin, const Katom& end) { (void)K::log(4); std::string end_text = trim_span_markers(end); if (!end_text.empty() && //!(begin.m_type == katom_t::ignore_begin && end.m_type == katom_t::ignore_end)) { !(begin_ignore(begin) && end_ignore(end))) { std::string begin_text = trim_span_markers(begin); if (begin_text != end_text) { std::stringstream ss; ss << " A named end katom does not match:\n" << " " << begin.m_loc << " " << begin << "\n" << " " << end.m_loc << " " << end; throw Parsing_error(ss.str(), end.m_loc, false); } } } spans_t find_spans( katom_iter begin, katom_iter end, const std::function& level_inc, const std::function& level_dec, bool error_exit, const std::string& name) { (void)K::log(4, name); bool _dbg = false; spans_t spans {}; katom_list bounds {}; for (auto k = begin; k < end; k++) { if (_dbg) msg() << ktype << kall << kignored << kreplaced << " " << k << " type: " << type_to_name(k->m_type) << "\n"; if (level_inc(*k)) { if (_dbg) msg() << boldblack << " level_inc: " << k << black << "\n"; bounds.push_back(*k); } else if (level_dec(*k)) { if (_dbg) msg() << boldblack << " level_dec: " << k << " bounds: " << bounds << black << "\n"; if (!bounds.empty()) { auto span_start = bounds.back(); if (_dbg) msg() << " span_start of this level: " << span_start << "\n"; if (katom_spans[span_start.m_type] == k->m_type) { check_named_katom_span(span_start, *k); spans.push_back({std::move(span_start), std::move(*k)}); bounds.pop_back(); } else { bad_close(span_start, *k, error_exit); } } else { missing_open(*k, error_exit); } } } if (!bounds.empty()) { missing_close(bounds, error_exit); } if (_dbg) msg() << black; // Post-order invariant: if span A is nested inside span B, then A // appears before B in the list. This is a necessary consequence of // stack-based bracket matching and is required for correct inside-out // evaluation of nested structures. See doc/taxonomy.md, "Spans". if (verbose_level >= 2) { for (size_t i = 0; i < spans.size(); i++) { for (size_t j = i + 1; j < spans.size(); j++) { bool j_inside_i = spans[j].first.m_index > spans[i].first.m_index && spans[j].second.m_index < spans[i].second.m_index; if (j_inside_i) { std::stringstream ss; ss << "Post-order span invariant violated: span " << j << " (" << spans[j].first << ")" << " is nested inside span " << i << " (" << spans[i].first << ")" << " but appears after it"; throw Internal_error(ss.str(), spans[j].first.m_loc); } } } } return spans; } spans_t find_spans( katom_list& katoms, const std::function& level_inc, const std::function& level_dec, bool error_exit, const std::string& name) { return find_spans(katoms.begin(), katoms.end(), level_inc, level_dec, error_exit, name); } void describe_spans(const katom_list& katoms) { katom_list non_const_katoms = katoms; auto spans = find_spans(non_const_katoms, level_increase, level_decrease); std::vector> span_specs {}; size_t width = 0; std::string margin = " "; string_map relpath {}; for (auto [op,cl] : spans) { std::string loc = locator_range(op.m_loc, cl.m_loc, relpath); width = std::max(width, loc.size()); span_specs.push_back(std::make_tuple(margin + loc, op, cl)); } width += margin.size(); for (auto [span,op,cl] : span_specs) { std::cout << std::setw(width) << std::left << span << " " << ktype << kreplaced << op << right_arrow << cl << "\n"; } } // Nonascii void encode_nonascii_characters(katom_list& katoms) { for (Katom& k : katoms) { if (is_nonascii(k)) { k.m_text = encode(k.m_text); } } } // Literal void mark_literal_katoms(katom_list& katoms) { (void)K::log(4); for (auto [op, cl] : find_spans(katoms, begin_literal, end_literal, true, "literal")) { auto [begin, end] = find_span_katoms(katoms, op, cl); //if (begin->m_type == katom_t::literal_begin) { if (begin_literal(*begin)) { //begin->m_type = katom_t::replaced; mark_as_replaced(*begin); //end->m_type = katom_t::replaced; mark_as_replaced(*(end - 1)); //std::for_each(begin + 1, end, [](Katom& k) { k.m_type = katom_t::literal; }); // Hide Klammertext structural characters in the content as KTESC // markers so the literal text survives re-katomization (the // @document :text sub-Machine, the @eval result read-back). // Resolved back to the characters at final processing. Literal // KLAMMER content (@code) is NOT treated this way -- it is marked // by mark_literal_klammer_content() and reaches the @eval code raw. std::for_each(begin + 1, end - 1, [](Katom& k) { mark_as_literal(k); k.m_text = hide_structural_characters(k.m_text); }); } } } void hide_special_katoms(katom_list& katoms) { // Replace the text of ^-quoted special-character katoms (^@, ^|, ^#, ^^, // ^:, ^*) with KTESC markers. The katomizer strips the "^" when the // katom is constructed, so without this the bare character leaks into // assembled strings (state values, @eval results) and is re-interpreted // as Klammertext syntax when those strings are re-katomized -- by // @document's :text sub-Machine or the @eval result read-back in // Eval::eval. Markers are inert text at every level and are resolved to // the characters at final processing (Target::resolve_escapes). // // Skipped inside: // * @@...@@ and @@@...@@@ definition spans -- parameter declarations, // descriptions, and argtype patterns are extracted as plain strings // (kdesc display, validation regexes); a klammer BODY is re-processed // through process_katoms() at application time, outside any // definition span, so its quoted specials are hidden then. // * @eval/@read/@cond argument spans -- code, filenames, and // predicates consumed by the primitive, not target text (the same // rule as the general-body escape pass in Machine::apply_klammer). (void)K::log(4); int definition_depth = 0; int code_depth = 0; // inside an @eval/@read/@cond span std::vector apply_is_code; // one entry per open application for (auto& k : katoms) { switch (k.m_type) { case katom_t::define_begin: case katom_t::machine_begin: ++definition_depth; continue; case katom_t::define_end: case katom_t::machine_end: if (definition_depth > 0) --definition_depth; continue; case katom_t::eval_begin: case katom_t::read_begin: case katom_t::cond_begin: apply_is_code.push_back(true); ++code_depth; continue; case katom_t::apply_begin: apply_is_code.push_back(false); continue; case katom_t::apply_end: if (!apply_is_code.empty()) { if (apply_is_code.back()) --code_depth; apply_is_code.pop_back(); } continue; default: break; } if (k.m_type == katom_t::special && definition_depth == 0 && code_depth == 0) { k.m_text = hide_structural_characters(k.m_text); } } } // Ignore void mark_ignored_katoms(katom_list& katoms) { (void)K::log(4); //auto is_ignore_begin = [](const Katom& k) { return k.m_type == katom_t::ignore_begin; }; //auto is_ignore_end = [](const Katom& k) { return k.m_type == katom_t::ignore_end; }; for (auto [op,cl] : find_spans(katoms, begin_ignore, end_ignore, true, "ignored")) { auto [begin, end] = find_span_katoms(katoms, op, cl); // std::for_each(begin, end + 1, [](Katom& k) { k.m_type = katom_t::ignored; }); std::for_each(begin, end + 1, mark_as_ignored); //if (end + 1 != katoms.end() && (end+1)->m_type == katom_t::newline) { if (end + 1 != katoms.end() && is_newline(*(end + 1))) { //(end+1)->m_type = katom_t::ignored; mark_as_ignored(*(end + 1)); } } bool ignore_line = false; bool ignore_rest = false; std::vector after_line {}; for (Katom& k : katoms) { //if (k.m_type == katom_t::ignore_rest) { if (is_ignore_rest(k)) { ignore_rest = true; // } else if (k.m_type == katom_t::ignore_line) { } else if (is_ignore_line(k)) { ignore_line = true; } //if (!ignore_rest && ignore_line && k.m_type == katom_t::newline) { if (!ignore_rest && ignore_line && is_newline(k)) { //k.m_type = katom_t::ignored; mark_as_ignored(k); ignore_line = false; } if (ignore_line or ignore_rest) { //k.m_type = katom_t::ignored; mark_as_ignored(k); } } size_t i = 0; while (i < katoms.size()) { if (katoms[i].m_initial_type == katom_t::ignore_end) { // std::cout << "IGNORED: " << kall << ktype << kignored << katoms[i] << " " << katoms[i+1] << black << "\n"; i++; while (i < katoms.size() && katoms[i].is_whitespace()) { katoms[i].m_type = katom_t::ignored; i++; } } else { i++; } } } // Klammers: void process_klammer_katoms(katom_list& katoms) { (void)K::log(4); for (auto [op, cl] : find_spans(katoms, begin_klammer_def, end_klammer_def , true, "define")) { auto [begin, end] = find_span_katoms(katoms, op, cl); for (auto k = begin + 1; k < end - 1; k++) { //k->m_type = katom_t::literal; mark_as_literal(*k); } } } // Application: read, cond, and defined klammers // Cond /* void check_bar_count(katom_iter begin, katom_iter end) { //int count = std::count_if(begin, end, [](const Katom& k) { return k.m_type == katom_t::bar; }); int count = std::count_if(begin, end, is_bar); //[](const Katom& k) { return k.m_type == katom_t::bar; }); if (count != 1 && count != 2) { std::stringstream ss {}; ss << "Incorrectly formatted @cond klammer. There should only be one or two bar characters:\n" << " @cond | | | @"; throw Argument_error(ss.str(), begin->m_loc, false); } } bool is_true(const std::string& s) { return s == "True" || s == "true" || s == "1"; } void process_cond_katoms(katom_list& katoms) { if (std::find_if(katoms.begin(), katoms.end(), begin_cond) != katoms.end()) { (void)K::log(3); //for (auto [op, cl] : find_spans(katoms, begin_cond, end_apply, true, "cond")) { for (auto [op, cl] : find_spans(katoms, level_increase, level_decrease, true, "cond")) { auto [begin, end] = find_span_katoms(katoms, op, cl); // msg() << "find_spans: " << std::pair(begin, end) << "\n"; if (begin_cond(*begin)) { check_bar_count(begin, end); auto bar_1 = std::find_if(begin, end, is_bar); std::string predicate = to_string(begin + 1, bar_1, true); auto bar_2 = std::find_if(bar_1 + 1, end - 1, is_bar); katom_list true_clause {}; katom_list false_clause {}; if (bar_2->m_type == katom_t::bar) { true_clause = katom_list(bar_1 + 1, bar_2); false_clause = katom_list(bar_2 + 1, end - 1); } else { true_clause = katom_list(bar_1 + 1, end - 1); } katom_list result = is_true(predicate) ? trim_whitespace(true_clause) : trim_whitespace(false_clause); std::for_each(begin, end, mark_as_replaced); katoms.insert(end, result.begin(), result.end()); } } } } */