#pragma once #include #include #include #include #include #include inline const std::string at_s {"@"}; inline const std::string at2_s { "@@" }; inline const std::string at3_s { "@@@" }; inline const std::string bar_s { "|" }; inline const std::string open_s { R"(\()" }; inline const std::string close_s { R"(\))" }; inline const std::string karg_s { "*" }; inline const std::string read_s { "read" }; inline const std::string eval_s { "eval" }; inline const std::string cond_s { "cond" }; inline const std::string lit_s { "lit" }; inline const std::string hat_s { "^" }; inline const std::string ignore_line_s { "#" }; inline const std::string ignore_begin_s { R"(#\[)" }; inline const std::string ignore_end_s { R"(\]#)" }; inline const std::string ignore_rest_s { "##" }; inline const std::string ws_remove_s { "#-" }; inline const std::string ws_space_s { R"(#\+\d*)" }; inline const std::string ws_newline_s { R"(#/\d*)" }; // const std::string kname = "[a-zA-Z]+[a-zA-Z0-9_.]*"; // const std::string dname = "[a-zA-Z]+[a-zA-Z0-9_]*"; // const std::string k_name = R"([^^@#|\:]+)"; inline const std::string definition_name = R"([a-zA-Z][a-zA-Z0-9_.]*)"; // The name at the START of a definition may carry a comma-separated list of // targets ("@@table.html,tex"), so its katom runs through commas. This is // deliberately looser than the grammar: katomization's job is to delimit // the katom, and parse_name() in klammer.cpp validates the list, where a // malformed one can be reported with its location. The comma stays out of // definition_name itself -- that pattern also delimits klammer APPLICATIONS, // option names and "*arg*" variables, where a comma is ordinary writer text. // "*" is here for the same reason: "@@date.*" declares a klammer for every // target explicitly, and "*" is a Klammertext special everywhere else. inline const std::string definition_begin_name = R"([a-zA-Z][a-zA-Z0-9_.,*]*)"; enum class katom_t { space, word, newline, text, apply_begin, apply_end, bar, double_bar, option_name, read_begin, eval_begin, cond_begin, define_begin, define_end, klammer_definition, klammer_instance, klammer_override, klammer_default, karg, machine_begin, machine_end, ignore_begin, ignore_end, ignore_rest, ignore_line, ws_remove, ws_space, ws_newline, special, nonascii, literal_begin, literal_end, ws_added, literal, eval_result, replaced, ignored, }; inline const std::vector katom_type_display_order { katom_t::space, katom_t::word, katom_t::newline, katom_t::text, katom_t::apply_begin, katom_t::apply_end, katom_t::bar, katom_t::double_bar, katom_t::option_name, katom_t::read_begin, katom_t::eval_begin, katom_t::cond_begin, katom_t::define_begin, katom_t::define_end, katom_t::klammer_definition, katom_t::klammer_instance, katom_t::klammer_override, katom_t::klammer_default, katom_t::karg, katom_t::machine_begin, katom_t::machine_end, katom_t::ignore_begin, katom_t::ignore_end, katom_t::ignore_rest, katom_t::ignore_line, katom_t::ws_remove, katom_t::ws_space, katom_t::ws_newline, katom_t::special, katom_t::nonascii, katom_t::literal_begin, katom_t::literal_end, katom_t::ws_added, katom_t::literal, katom_t::eval_result, katom_t::replaced, katom_t::ignored }; class Ktype; inline std::map katom_type_names {}; inline std::map katom_type_descs {}; inline std::vector katom_type_list {}; class Ktype { public: // Ktype() = default; Ktype(katom_t type, std::string name, std::string pattern, std::string description, bool use_equal = false) : m_type(type), m_name(name), m_pattern(pattern), m_description(description), m_use_equal(use_equal), m_rgx(std::regex(pattern)) { katom_type_list.push_back(type); katom_type_names.insert({m_type, m_name}); katom_type_descs.insert({m_type, m_description}); } bool match(const std::string& s) const { return m_use_equal ? s == m_pattern : std::regex_match(s, m_rgx); } katom_t m_type; std::string m_name; std::string m_pattern; std::string m_description; bool m_use_equal; std::regex m_rgx; }; inline const std::vector katom_types { Ktype(katom_t::space, "space", " ", "One space character", false), Ktype(katom_t::word, "word", "[a-zA-Z]+", "Text only containing letters", false), Ktype(katom_t::newline, "newline", "\n", "One newline character", false), Ktype(katom_t::bar, "bar", "\\|", "Bar character used as positional argument separator", false), Ktype(katom_t::double_bar, "double-bar", "\\|\\|", "Separator for compound positional arguments", false), Ktype(katom_t::read_begin, "read-begin", at_s+read_s, "Beginning of the file input klammer", false), Ktype(katom_t::eval_begin, "eval-begin", at_s+eval_s, "Beginning of the evaluation klammer", false), Ktype(katom_t::cond_begin, "cond-begin", at_s+cond_s, "Beginning of the conditional (if/then/else) klammer", false), Ktype(katom_t::apply_begin, "apply-begin", at_s + definition_name, "Beginning of a klammer call"), Ktype(katom_t::apply_end, "apply-end", "(" + definition_name + ")?" + at_s, "End of a klammer call"), Ktype(katom_t::option_name, "option-name", ":" + definition_name, "Optional argument name"), Ktype(katom_t::define_begin, "define-begin", at2_s + definition_begin_name, "Beginning of a klammer definition"), Ktype(katom_t::define_end, "define-end", "(" + definition_name + ")?" + at2_s, "End of a klammer definition"), Ktype(katom_t::klammer_default, "klammer-default", "::::", "Define klammer default value for possible override"), Ktype(katom_t::klammer_override, "klammer-override", ":::", "Override existing klammer definition"), Ktype(katom_t::klammer_definition, "klammer-definition", ":", "Klammer definition, including parameters"), Ktype(katom_t::klammer_instance, "klammer-instance", "::", "Klammer definition using previously defined parameters"), Ktype(katom_t::karg, "klammer-arg", "\\*" + definition_name +"\\*", "Klammer argument in klammer body definition"), Ktype(katom_t::machine_begin, "machine-begin", at3_s + definition_name, "Beginning of a processor modification definition"), Ktype(katom_t::machine_end, "machine-end", "(" + definition_name + ")?" + at3_s, "End of a processor modification definition"), Ktype(katom_t::ignore_begin, "ignore-begin", ignore_begin_s, "Beginning of text to remove"), Ktype(katom_t::ignore_end, "ignore-end", ignore_end_s, "End of text to remove"), Ktype(katom_t::ignore_rest, "ignore-rest", ignore_rest_s, "Remove all text to the end of file or string"), Ktype(katom_t::ignore_line, "ignore-line", ignore_line_s, "Remove all text to the first newline, inclusive"), Ktype(katom_t::ws_remove, "ws-remove", ws_remove_s, "Remove all whitespace at this point"), Ktype(katom_t::ws_space, "ws-space", ws_space_s, "Remove all whitespace, leaving spaces (default: 1)"), Ktype(katom_t::ws_newline, "ws-newline", ws_newline_s, "Remove all whitespace, leaving newlines (default: 1)"), Ktype(katom_t::special, "special-char", R"(\^[@|#^:*])", "Klammertext special character treated as regular text"), //Ktype(katom_t::nonascii, "non-ascii-char", R"(\^\w(?:.|\^))", "Non-ASCII character"), // The second alternative is the "^...^" delimited form: a code point // (^263A^, 4-6 hex digits) OR a mnemonic (^s^, ^oe^). Its character // class is [A-Za-z0-9], not [A-Fa-f0-9], because a mnemonic is not // hex: "^oe^" matched NEITHER alternative and so was not a katom at // all, fell through to the "non-diacritic non-ascii" rewrite rule, // which pads it with whitespace and hands back the same single word -- // and make_katoms_from_word() then recursed on the identical string // until the stack died (SIGSEGV, no message). The two entries that // appeared to work, ^ae^ and ^AE^, did so only because "ae" happens to // be hex digits; they were being katomized as CODE POINTS and rendered // correctly only because encode() runs process_extended_latin() before // process_unicode_codepoint(). Which of the two a form means is // decided there, by the tables, not here -- katomization's job is only to // recognize the shape. A shape that resolves to no character at all is // a located error, raised at the end of encode(). Ktype(katom_t::nonascii, "non-ascii-char", R"((\^(\w(?:.|\^)))|(\^[A-Za-z0-9]{1,6}\^))", "Non-ASCII character"), // Ktype(katom_t::word, "word", R"([a-z][a-z0-9_]*)", "Lower-case letters, numbers, or underscore"), // Ktype(katom_t::text, "text", Ktype(katom_t::literal_begin, "literal-begin", R"(\^')", "Begin unprocessed text"), Ktype(katom_t::literal_end, "literal-end", R"('\^)", "End unprocessed text"), Ktype(katom_t::text, "text", R"([^^@#|]+)", "No special characters, whitespace, or \":\" at the beginning"), // spaces, @, #, |, or ^, Ktype(katom_t::ws_added, "ws-added", " or \\n", "Added whitespace characters from #+ and #/"), Ktype(katom_t::literal, "literal", "", "Literal katom (changed from its original type by ^'...'^)"), Ktype(katom_t::eval_result, "eval-result", "", "Katom produced by @eval klammer"), Ktype(katom_t::replaced, "replaced", "", "A katom replaced by definitions, applications, or file input"), Ktype(katom_t::ignored, "ignored", "", "A katom ignored by the action of a \"#\" katom"), // Ktype(katom_t::undefined, "undefined", "", "Undefined pattern") }; inline const std::set printable_katom_types { katom_t::word, katom_t::space, katom_t::newline, katom_t::literal, katom_t::ws_added }; inline const std::set open_level { katom_t::literal_begin, katom_t::read_begin, katom_t::eval_begin, katom_t::cond_begin, katom_t::apply_begin, katom_t::define_begin, katom_t::machine_begin, katom_t::ignore_begin }; inline const std::set close_level { katom_t::literal_end, katom_t::apply_end, katom_t::apply_end, katom_t::apply_end, katom_t::apply_end, katom_t::define_end, katom_t::machine_end, katom_t::ignore_end }; inline std::map katom_spans { { katom_t::literal_begin, katom_t::literal_end }, { katom_t::read_begin, katom_t::apply_end }, { katom_t::eval_begin, katom_t::apply_end }, { katom_t::cond_begin, katom_t::apply_end }, { katom_t::apply_begin, katom_t::apply_end }, { katom_t::define_begin, katom_t::define_end }, { katom_t::machine_begin, katom_t::machine_end }, { katom_t::ignore_begin, katom_t::ignore_end } }; bool is_active(katom_t t); bool is_printable(katom_t type); void describe_katoms(bool show_regex = false); bool is_deftype(katom_t type); std::string type_to_name(katom_t type);