2026-07-18 18:48:23 +02:00
|
|
|
#pragma once
|
|
|
|
|
|
|
|
|
|
// https://jakubmarian.com/special-characters-diacritics-used-in-european-languages/
|
|
|
|
|
|
|
|
|
|
#include <string>
|
|
|
|
|
#include <map>
|
|
|
|
|
#include <iomanip>
|
|
|
|
|
#include <tuple>
|
|
|
|
|
#include <vector>
|
|
|
|
|
#include <utility>
|
|
|
|
|
#include <regex>
|
|
|
|
|
#include <unistd.h>
|
|
|
|
|
|
2026-08-25 23:30:55 +02:00
|
|
|
#include "locator.h"
|
|
|
|
|
|
2026-09-07 22:37:05 +02:00
|
|
|
inline
|
|
|
|
|
std::string klammertext_special_characters { "@|*^#" };
|
|
|
|
|
|
|
|
|
|
inline
|
|
|
|
|
std::string encoding_marker { "UU" };
|
|
|
|
|
|
|
|
|
|
inline
|
|
|
|
|
std::string diacritic_symbols = "-'`h~\"cbrdwa";
|
|
|
|
|
|
|
|
|
|
inline
|
|
|
|
|
std::string diacritic_symbols_order = "'`h~\"c-brdwa";
|
|
|
|
|
|
2026-08-23 20:48:26 +02:00
|
|
|
// ^UUUU^ — a Unicode code point in hex, 4 to 6 digits (^263A^ is the BMP,
|
|
|
|
|
// ^13000^ EGYPTIAN HIEROGLYPH A001, ^10FFFD^ the top of the range), or a
|
|
|
|
|
// single digit. Lengths 2-3 are NOT accepted: two- and three-letter
|
|
|
|
|
// sequences of a-f collide with the ^s^-style mnemonic names.
|
|
|
|
|
const std::regex unicode_re(R"(\^(([0-9A-Fa-f]{4,6})|([0-9A-Fa-f]))\^)");
|
2026-07-18 18:48:23 +02:00
|
|
|
const std::regex unicode_hide_re(R"(=([0-9A-Fa-f]{2})=)");
|
|
|
|
|
const std::regex hex2_re(R"(([0-9A-Fa-f]{2}))");
|
|
|
|
|
const std::regex hex4_re(R"(([0-9A-Fa-f]{4}))");
|
|
|
|
|
const std::regex hex5_re(R"(([0-9A-Fa-f]{5}))");
|
|
|
|
|
|
|
|
|
|
// The hide_special_characters vector was removed. The ^X mechanism for
|
|
|
|
|
// Klammertext special characters is handled by the katomizer (type
|
|
|
|
|
// katom_t::special) and the general KTESC escape mechanism for
|
|
|
|
|
// target-specific characters.
|
|
|
|
|
|
|
|
|
|
inline
|
|
|
|
|
std::map<std::string, std::string> diacritic_example_letter {
|
|
|
|
|
{"'", "e"},
|
|
|
|
|
{"`", "a"},
|
|
|
|
|
{"h", "o"},
|
|
|
|
|
{"~", "n"},
|
|
|
|
|
{"\"", "u"},
|
|
|
|
|
{"c", "c"},
|
|
|
|
|
{"-", "o"},
|
|
|
|
|
{"b", "g"},
|
|
|
|
|
{"r", "a"},
|
|
|
|
|
{"d", "e"},
|
|
|
|
|
{"w", "s"},
|
|
|
|
|
{"a", "o"}};
|
|
|
|
|
|
|
|
|
|
inline
|
|
|
|
|
std::map<std::string, std::pair<std::string, std::string>> diacritics {
|
|
|
|
|
{"`", {"0300", "grave accent"}},
|
|
|
|
|
{"'", {"0301", "acute accent"}},
|
2026-09-12 18:11:48 +02:00
|
|
|
{"h", {"0302", "circumflex (hat)"}},
|
2026-07-18 18:48:23 +02:00
|
|
|
{"~", {"0303", "tilde"}},
|
|
|
|
|
{"-", {"0304", "macron"}},
|
|
|
|
|
{"\"", {"0308", "diaresis"}},
|
|
|
|
|
// {"v", {"0305", "vinculum"}},
|
|
|
|
|
{"b", {"0306", "breve"}},
|
|
|
|
|
{"d", {"0307", "dot above"}},
|
|
|
|
|
{"r", {"030A", "ring above"}},
|
|
|
|
|
{"w", {"030C", "wedge"}},
|
|
|
|
|
{"c", {"0327", "cedilla"}},
|
|
|
|
|
{"a", {"030B", "double acute accent"}}};
|
|
|
|
|
|
|
|
|
|
inline
|
|
|
|
|
std::vector<std::string> extended_latin_symbols = {
|
|
|
|
|
"s",
|
|
|
|
|
"i", "I", "t", "T", "e", "E", "o", "O", "d", "D",
|
|
|
|
|
"ae", "AE", "oe", "OE"
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
inline
|
|
|
|
|
std::map<std::string, std::pair<std::string, std::string>> extended_latin {
|
|
|
|
|
{"ae", {"00E6", "ae ligature"}},
|
|
|
|
|
{"AE", {"00C6", "ae ligature capital"}},
|
|
|
|
|
{"oe", {"0153", "oe ligature"}},
|
|
|
|
|
{"OE", {"0152", "oe ligature capital"}},
|
|
|
|
|
{"i", {"0131", "dotless i"}},
|
|
|
|
|
{"I", {"0130", "capital dotted i"}},
|
|
|
|
|
{"t", {"00FE", "thorn"}},
|
|
|
|
|
{"T", {"00DE", "thorn capital"}},
|
|
|
|
|
{"e", {"00F0", "eth"}},
|
|
|
|
|
{"E", {"00D0", "eth capital"}},
|
|
|
|
|
{"o", {"00F8", "o stroke"}},
|
|
|
|
|
{"O", {"00D8", "o stroke capital"}},
|
|
|
|
|
{"s", {"00DF", "Eszett"}},
|
|
|
|
|
{"d", {"0111", "d stroke"}},
|
|
|
|
|
{"D", {"0110", "d stroke capital"}}};
|
|
|
|
|
|
|
|
|
|
inline
|
|
|
|
|
std::map<std::string, std::pair<std::string, std::string>> pinyin_tones {
|
|
|
|
|
{"1", {"0304", "high"}},
|
|
|
|
|
{"2", {"0301", "rising"}},
|
|
|
|
|
{"3", {"030C", "falling-rising"}},
|
|
|
|
|
{"4", {"0300", "falling"}}};
|
|
|
|
|
|
2026-08-25 23:30:55 +02:00
|
|
|
inline
|
|
|
|
|
std::vector<std::vector<std::string>> ptolemy {
|
|
|
|
|
{"Q3", "P", "132AA"},
|
|
|
|
|
{"X1", "t", "133CF"},
|
|
|
|
|
{"V4", "o/wa", "13371"},
|
|
|
|
|
{"E23", "l/r", "130ED"},
|
|
|
|
|
{"Aa15", "m", "1340D"},
|
|
|
|
|
{"M17A", "e/y", "131CC"},
|
|
|
|
|
{"S29", "s", "132F4"}};
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
道 (dào): U+9053
|
|
|
|
|
可 (kě): U+53EF
|
|
|
|
|
道 (dào): U+9053
|
|
|
|
|
非 (fēi): U+975E
|
|
|
|
|
常 (cháng): U+5E38
|
|
|
|
|
道 (dào): U+9053
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
inline
|
|
|
|
|
std::vector<std::vector<std::string>> dao {
|
|
|
|
|
{"dào", "9053"},
|
|
|
|
|
{"kě", "53EF"},
|
|
|
|
|
{"dào", "9053"},
|
|
|
|
|
{"fēi", "975E"},
|
|
|
|
|
{"cháng", "5E38"},
|
|
|
|
|
{"dào", "9053"}};
|
2026-07-18 18:48:23 +02:00
|
|
|
|
|
|
|
|
inline
|
|
|
|
|
bool is_tty() { return isatty(fileno(stdout)); }
|
|
|
|
|
inline
|
|
|
|
|
const std::string italic_on() { return is_tty() ? "\033[3m" : ""; }
|
|
|
|
|
inline
|
|
|
|
|
const std::string italic_off() { return is_tty() ? "\033[0m" : ""; }
|
|
|
|
|
|
2026-09-12 18:11:48 +02:00
|
|
|
std::string process_unicode_codepoint(std::string s, const Locator& loc = Locator::none());
|
|
|
|
|
|
2026-08-25 23:30:55 +02:00
|
|
|
std::string encode(const std::string& s, const Locator& loc);
|
|
|
|
|
|
|
|
|
|
// UTF-8 encoding of one code point (to U+10FFFF; surrogates and values
|
|
|
|
|
// above the range yield an "Invalid Unicode: N" text). Used by the
|
|
|
|
|
// character tables here and by the ^UUUU UUUU ...^ sequence pass
|
2026-09-07 22:37:05 +02:00
|
|
|
// (process_unicode_sequences in katom_process.cpp).
|
2026-08-25 23:30:55 +02:00
|
|
|
std::string utf8char(int cp);
|
2026-07-18 18:48:23 +02:00
|
|
|
|
2026-08-25 23:30:55 +02:00
|
|
|
// Whether cp may be written in a ^...^ form: within the Unicode range,
|
|
|
|
|
// not a surrogate, and not a NONCHARACTER (U+FDD0-U+FDEF and the last two
|
|
|
|
|
// code points of every plane) -- the standard reserves those for internal
|
|
|
|
|
// use, never for interchange. An invalid code point in a document is a
|
|
|
|
|
// located Parsing_error (the no-warn-and-continue policy; the in-band
|
|
|
|
|
// "Invalid Unicode" text is no longer reachable from input).
|
|
|
|
|
bool valid_codepoint(int cp);
|
|
|
|
|
std::string invalid_codepoint_reason(int cp);
|