klammertext/kpy/klammertext.cpp

// kpy/klammertext.cpp -- the Python binding of the Klammermachine.
//
// A CPython extension module, written against the C API by hand: the
// project takes no third-party C++ library, which rules out pybind11 and
// nanobind, and the tree already links libpython for @eval, so this adds no
// dependency.  It is ktext running in-process:
//
//     import klammertext
//     M = klammertext.Machine(target="html", files=["doc.kt"])
//     M.read("@@version : 3.2.1 @@")     # what "ktext -s" does
//     M.read_files()                     # the constructor's files, in order
//     html = M.apply()
//
// The constructor does what the top of ktext's main() does -- the same
// K_* state, through the same set_command_state(), and the same klammerset
// loading through load_klammersets() -- so a document that renders under
// ktext renders under the binding, and the two cannot drift apart because
// neither restates the other.
//
// Decisions (Andy, 2026-09-08), each of which made the binding smaller:
//
//   * ONE Machine at a time.  A second constructor call while one exists
//     raises RuntimeError.  The engine keeps process-wide state -- the
//     klammerset search path is frozen on first use, the per-directory
//     Python-file answer is cached per process, the recursion counter is a
//     translation-unit static -- and with one Machine every one of them is
//     correct by construction.  A new Machine may be created once the old
//     one is garbage-collected.
//   * ONE Python version: the one the library was built against.  No stable
//     ABI, no wheels; the module is found on PYTHONPATH.
//   * Errors map ONE-TO-ONE.  The C++ Error hierarchy is a base with a kind
//     string and nine subclasses that add nothing but their kind, so the
//     Python side is klammertext.Error with nine subclasses under the C++
//     NAMES VERBATIM (Parsing_error, ...): an identity mapping is the
//     easiest kind to document, and two of the nine (Environment_error,
//     Recursion_error) would otherwise shadow Python builtins.  str(e) is
//     the report text ktext prints; e.kind, e.description, e.filename,
//     e.line, e.column carry the parts.  An @eval that raised a Python
//     exception arrives as a Parsing_error whose __cause__ IS that
//     exception, traceback attached (Eval_python::take_last_exception).
//   * Non-standard behaviour is documented, not removed: @eval appends to
//     the host's sys.path, logging goes to stderr under the verbose level,
//     and the constructor's verbose= sets the process-wide level.
//
// The Machine's @eval runs in the host interpreter (Eval_python checks
// Py_IsInitialized) in globals of its own, so a script's variables and a
// document's @eval code do not see each other; data crosses through the
// state (M.set), exactly as it crosses ktext's command line through -s.

#include <Python.h>

#include <map>
#include <string>
#include <vector>

#include "command.h"
#include "error.h"
#include "eval_python.h"
#include "file.h"
#include "log.h"
#include "machine.h"
#include "target_registry.h"
#include "util.h"

namespace {

// ---------------------------------------------------------------------------
// The exception classes
// ---------------------------------------------------------------------------

PyObject* error_base = nullptr;
std::map<std::string, PyObject*> error_classes {};   // kind -> class

// The nine C++ subclasses, by the kind string their constructors pass to
// Error(), in error.h's order.  A kind not listed here (there is none today)
// would arrive as the base class, so nothing is ever lost.
const std::vector<std::pair<std::string, std::string>> error_kinds = {
    {"parsing",     "Parsing_error"},
    {"file",        "File_error"},
    {"target",      "Target_error"},
    {"klammerset",  "Klammerset_error"},
    {"definition",  "Definition_error"},
    {"argument",    "Argument_error"},
    {"environment", "Environment_error"},
    {"recursion",   "Recursion_error"},
    {"internal",    "Internal_error"},
};

void set_attr(PyObject* obj, const char* name, PyObject* value)
{
    // value: a NEW reference, consumed here (nullptr -> None).
    if (!value) {
        Py_INCREF(Py_None);
        value = Py_None;
    }
    PyObject_SetAttrString(obj, name, value);
    Py_DECREF(value);
}

// Raise the Python exception for a C++ Error.  Always returns nullptr, so a
// method can write "return raise_error(e);".
PyObject* raise_error(const Error& e)
{
    auto found = error_classes.find(e.m_type);
    PyObject* cls = found == error_classes.end() ? error_base : found->second;
    PyObject* inst = PyObject_CallFunction(cls, "s", e.message().c_str());
    if (!inst) {
        return nullptr;
    }
    set_attr(inst, "kind", PyUnicode_FromString(e.m_type.c_str()));
    set_attr(inst, "description", PyUnicode_FromString(justify(e.m_desc, 80, 0).c_str()));
    set_attr(inst, "filename",
             e.location().m_filename.empty() ? nullptr : PyUnicode_FromString(e.location().m_filename.c_str()));
    set_attr(inst, "line", e.location().m_line > -1 ? PyLong_FromLong(e.location().m_line) : nullptr);
    // 1-based, as the report prints it (the Locator holds a 0-based index).
    set_attr(inst, "column", e.location().m_chr > -1 ? PyLong_FromLong(e.location().m_chr + 1) : nullptr);
    if (PyObject* cause = Eval_python::take_last_exception()) {
        PyException_SetCause(inst, cause);   // steals the reference
    }
    PyErr_SetObject(cls, inst);
    Py_DECREF(inst);
    return nullptr;
}

// Run f() and translate anything it throws.  Every entry point from Python
// goes through here: a C++ exception must never reach the interpreter.
template <typename F>
PyObject* guarded(F f)
{
    try {
        return f();
    }
    catch (const Error& e) {
        return raise_error(e);
    }
    catch (const std::exception& e) {
        PyErr_SetString(PyExc_RuntimeError, e.what());
        return nullptr;
    }
}

// ---------------------------------------------------------------------------
// Argument conversion
// ---------------------------------------------------------------------------

// None -> empty; str -> its words; a sequence of str -> the strings.  Used
// for klammersets= and files=, both of which ktext takes as word lists.
bool strings_from(PyObject* obj, const char* what, std::vector<std::string>& out)
{
    out.clear();
    if (!obj || obj == Py_None) {
        return true;
    }
    if (PyUnicode_Check(obj)) {
        out = word_split(PyUnicode_AsUTF8(obj));
        return true;
    }
    PyObject* seq = PySequence_Fast(obj, "expected a string or a sequence of strings");
    if (!seq) {
        return false;
    }
    Py_ssize_t n = PySequence_Fast_GET_SIZE(seq);
    for (Py_ssize_t i = 0; i < n; ++i) {
        PyObject* item = PySequence_Fast_GET_ITEM(seq, i);   // borrowed
        if (!PyUnicode_Check(item)) {
            Py_DECREF(seq);
            PyErr_Format(PyExc_TypeError, "%s: element %zd is not a string", what, i);
            return false;
        }
        out.push_back(PyUnicode_AsUTF8(item));
    }
    Py_DECREF(seq);
    return true;
}

// None -> "", str -> itself.
bool string_from(PyObject* obj, const char* what, std::string& out)
{
    out.clear();
    if (!obj || obj == Py_None) {
        return true;
    }
    if (!PyUnicode_Check(obj)) {
        PyErr_Format(PyExc_TypeError, "%s must be a string or None", what);
        return false;
    }
    out = PyUnicode_AsUTF8(obj);
    return true;
}

PyObject* string_list(const std::vector<std::string>& v)
{
    PyObject* list = PyList_New(Py_ssize_t(v.size()));
    for (size_t i = 0; i < v.size(); ++i) {
        PyList_SET_ITEM(list, Py_ssize_t(i), PyUnicode_FromString(v[i].c_str()));
    }
    return list;
}

// ---------------------------------------------------------------------------
// The Machine type
// ---------------------------------------------------------------------------

struct Machine_object {
    PyObject_HEAD
    Machine* machine;
    std::string* target;            // the constructor's target
    std::string* output_filename;   // as parse_args derived it
    std::vector<std::string>* files;
};

int instances = 0;

// The pseudo source file for string input: a file IN THE CWD named after the
// module, so a relative @read in read() text resolves against the directory
// the script runs in -- the same rule ktext applies to -s input, and set at
// each read() because a script may chdir between two of them.
// The command's identity, through the library (set_command_identity): on
// Windows an assignment to error.h's inline variables from this module
// would land in this module's own copies, not the library's.
void set_string_source()
{
    set_command_identity("klammertext", (fs::current_path() / "klammertext").generic_string());
}

PyObject* Machine_new(PyTypeObject* type, PyObject*, PyObject*)
{
    if (instances > 0) {
        PyErr_SetString(
            PyExc_RuntimeError,
            "Only one klammertext.Machine may exist at a time: the engine keeps "
            "process-wide state (the klammerset search path, the module-directory "
            "cache, the recursion counter). Delete the existing Machine first.");
        return nullptr;
    }
    auto* self = reinterpret_cast<Machine_object*>(type->tp_alloc(type, 0));
    if (!self) {
        return nullptr;
    }
    self->machine = nullptr;
    self->target = new std::string();
    self->output_filename = new std::string();
    self->files = new std::vector<std::string>();
    ++instances;
    return reinterpret_cast<PyObject*>(self);
}

void Machine_dealloc(PyObject* obj)
{
    auto* self = reinterpret_cast<Machine_object*>(obj);
    delete self->machine;
    delete self->target;
    delete self->output_filename;
    delete self->files;
    --instances;
    Py_TYPE(obj)->tp_free(obj);
}

int Machine_init(PyObject* obj, PyObject* args, PyObject* kwargs)
{
    auto* self = reinterpret_cast<Machine_object*>(obj);
    static const char* kwlist[] = {
        "target", "klammersets", "files", "output", "display", "verbose", nullptr};
    const char* target_arg = nullptr;
    PyObject* klammersets_arg = nullptr;
    PyObject* files_arg = nullptr;
    PyObject* output_arg = nullptr;
    int display = 0;
    int verbose = 0;
    if (!PyArg_ParseTupleAndKeywords(
            args, kwargs, "|sOOOpi", const_cast<char**>(kwlist),
            &target_arg, &klammersets_arg, &files_arg, &output_arg, &display, &verbose)) {
        return -1;
    }
    std::vector<std::string> klammersets {};
    std::vector<std::string> files {};
    std::string output {};
    if (!strings_from(klammersets_arg, "klammersets", klammersets)
        || !strings_from(files_arg, "files", files)
        || !string_from(output_arg, "output", output)) {
        return -1;
    }
    PyObject* result = guarded([&]() -> PyObject* {
        verbose_level = verbose;
        std::string target = target_arg ? target_arg : Target_registry::general_name;
        // argv boundaries do not exist here -- a list element is one
        // filename -- but the standalone-"/" list form and the existence
        // rescue for a single space-containing string still apply.
        files = group_filename_tokens(files);
        for (auto& f : files) {
            f = expand_tilde(f);
        }
        auto [output_target, output_dir, output_basename, output_filename,
              write_files, display_only] =
            parse_args(files, target, expand_tilde(output), display != 0);
        (void)write_files;
        delete self->machine;
        self->machine = new Machine();
        set_command_state(*self->machine, "klammertext", output_target, files,
                          output_dir, output_basename, display_only);
        load_klammersets(*self->machine, klammersets);
        *self->target = output_target;
        *self->output_filename = output_filename;
        *self->files = files;
        Py_RETURN_NONE;
    });
    if (!result) {
        return -1;
    }
    Py_DECREF(result);
    return 0;
}

Machine& machine_of(PyObject* obj)
{
    auto* self = reinterpret_cast<Machine_object*>(obj);
    if (!self->machine) {
        throw Internal_error("The Machine was not initialized", Locator::none());
    }
    return *self->machine;
}

PyObject* Machine_read(PyObject* obj, PyObject* arg)
{
    if (!PyUnicode_Check(arg)) {
        PyErr_SetString(PyExc_TypeError, "read() takes the text to read, a string");
        return nullptr;
    }
    std::string text = PyUnicode_AsUTF8(arg);
    return guarded([&]() -> PyObject* {
        set_string_source();
        machine_of(obj).read_string(text + "\n");
        Py_RETURN_NONE;
    });
}

PyObject* Machine_read_file(PyObject* obj, PyObject* arg)
{
    if (!PyUnicode_Check(arg)) {
        PyErr_SetString(PyExc_TypeError, "read_file() takes a filename, a string");
        return nullptr;
    }
    std::string filename = expand_tilde(PyUnicode_AsUTF8(arg));
    return guarded([&]() -> PyObject* {
        machine_of(obj).read(fs::path(filename));
        Py_RETURN_NONE;
    });
}

PyObject* Machine_read_files(PyObject* obj, PyObject*)
{
    auto* self = reinterpret_cast<Machine_object*>(obj);
    return guarded([&]() -> PyObject* {
        for (const auto& f : *self->files) {
            machine_of(obj).read(fs::path(f));
        }
        Py_RETURN_NONE;
    });
}

PyObject* Machine_apply(PyObject* obj, PyObject* args, PyObject* kwargs)
{
    auto* self = reinterpret_cast<Machine_object*>(obj);
    static const char* kwlist[] = {"target", "final_processing", "escape_characters", nullptr};
    const char* target_arg = nullptr;
    int final_processing = 1;
    int escape_characters = 1;
    if (!PyArg_ParseTupleAndKeywords(
            args, kwargs, "|spp", const_cast<char**>(kwlist),
            &target_arg, &final_processing, &escape_characters)) {
        return nullptr;
    }
    return guarded([&]() -> PyObject* {
        std::string target = target_arg ? target_arg : *self->target;
        // Trimmed, as ktext trims before it writes or displays.
        std::string result = trim(
            machine_of(obj).apply(target, final_processing != 0, escape_characters != 0));
        return PyUnicode_FromString(result.c_str());
    });
}

PyObject* Machine_write(PyObject* obj, PyObject* args, PyObject* kwargs)
{
    auto* self = reinterpret_cast<Machine_object*>(obj);
    static const char* kwlist[] = {"text", nullptr};
    PyObject* text_arg = nullptr;
    if (!PyArg_ParseTupleAndKeywords(
            args, kwargs, "|O", const_cast<char**>(kwlist), &text_arg)) {
        return nullptr;
    }
    std::string text {};
    if (!string_from(text_arg, "text", text)) {
        return nullptr;
    }
    return guarded([&]() -> PyObject* {
        if (!text_arg || text_arg == Py_None) {
            text = trim(machine_of(obj).m_result);
        }
        // ktext writes nothing when the result is empty: a klammer that
        // handles its own output (@document under html) leaves it so.
        if (text.empty()) {
            Py_RETURN_NONE;
        }
        string_to_file(*self->output_filename, text + "\n");
        (void)K::log(1, "Wrote file: " + *self->output_filename);
        return PyUnicode_FromString(self->output_filename->c_str());
    });
}

PyObject* Machine_get(PyObject* obj, PyObject* arg)
{
    if (!PyUnicode_Check(arg)) {
        PyErr_SetString(PyExc_TypeError, "get() takes a state variable name, a string");
        return nullptr;
    }
    std::string name = PyUnicode_AsUTF8(arg);
    return guarded([&]() -> PyObject* {
        return PyUnicode_FromString(machine_of(obj).m_state.value(name, true).c_str());
    });
}

PyObject* Machine_set(PyObject* obj, PyObject* args)
{
    const char* name = nullptr;
    const char* value = nullptr;
    if (!PyArg_ParseTuple(args, "ss", &name, &value)) {
        return nullptr;
    }
    return guarded([&]() -> PyObject* {
        machine_of(obj).m_state.set(name, value);
        Py_RETURN_NONE;
    });
}

PyObject* Machine_names(PyObject* obj, PyObject*)
{
    return guarded([&]() -> PyObject* {
        return string_list(machine_of(obj).m_state.all_names());
    });
}

PyObject* Machine_describe(PyObject* obj, PyObject*)
{
    return guarded([&]() -> PyObject* {
        return PyUnicode_FromString(machine_of(obj).m_state.describe().c_str());
    });
}

PyObject* Machine_get_target(PyObject* obj, void*)
{
    auto* self = reinterpret_cast<Machine_object*>(obj);
    return PyUnicode_FromString(self->target->c_str());
}

PyObject* Machine_get_output_filename(PyObject* obj, void*)
{
    auto* self = reinterpret_cast<Machine_object*>(obj);
    return PyUnicode_FromString(self->output_filename->c_str());
}

PyObject* Machine_get_files(PyObject* obj, void*)
{
    auto* self = reinterpret_cast<Machine_object*>(obj);
    return string_list(*self->files);
}

PyObject* Machine_get_result(PyObject* obj, void*)
{
    return guarded([&]() -> PyObject* {
        return PyUnicode_FromString(machine_of(obj).m_result.c_str());
    });
}

PyObject* Machine_repr(PyObject* obj)
{
    auto* self = reinterpret_cast<Machine_object*>(obj);
    return PyUnicode_FromFormat("<klammertext.Machine target='%s'>", self->target->c_str());
}

PyMethodDef Machine_methods[] = {
    {"read", Machine_read, METH_O,
     "read(text)\n\nRead Klammertext from a string, as \"ktext -s\" does: definitions\n"
     "it makes are available to what is read after it."},
    {"read_file", Machine_read_file, METH_O,
     "read_file(filename)\n\nRead a Klammertext file (\"~\" is expanded)."},
    {"read_files", Machine_read_files, METH_NOARGS,
     "read_files()\n\nRead the constructor's files, in order."},
    {"apply", reinterpret_cast<PyCFunction>(reinterpret_cast<void (*)()>(Machine_apply)),
     METH_VARARGS | METH_KEYWORDS,
     "apply(target=None, final_processing=True, escape_characters=True) -> str\n\n"
     "Apply every klammer under the target (the constructor's by default) and\n"
     "return the result, trimmed as ktext trims it.  Empty when a klammer wrote\n"
     "its own files (@document under html)."},
    {"write", reinterpret_cast<PyCFunction>(reinterpret_cast<void (*)()>(Machine_write)),
     METH_VARARGS | METH_KEYWORDS,
     "write(text=None) -> str | None\n\n"
     "Write text (by default the last apply() result) to output_filename and\n"
     "return that name; nothing is written for an empty text, and None returned."},
    {"get", Machine_get, METH_O,
     "get(name) -> str\n\nThe value of a state variable; an error if it is not defined."},
    {"set", Machine_set, METH_VARARGS,
     "set(name, value)\n\nSet a state variable in the current frame."},
    {"names", Machine_names, METH_NOARGS,
     "names() -> list[str]\n\nEvery defined state variable name."},
    {"describe", Machine_describe, METH_NOARGS,
     "describe() -> str\n\nThe state as ktext -v describes it."},
    {nullptr, nullptr, 0, nullptr}
};

PyGetSetDef Machine_getset[] = {
    {"target", Machine_get_target, nullptr, "The constructor's target.", nullptr},
    {"output_filename", Machine_get_output_filename, nullptr,
     "Where write() writes: derived from files= and output= as ktext derives it.", nullptr},
    {"files", Machine_get_files, nullptr, "The constructor's input files, resolved.", nullptr},
    {"result", Machine_get_result, nullptr, "The last apply() result, untrimmed.", nullptr},
    {nullptr, nullptr, nullptr, nullptr, nullptr}
};

// Every slot is assigned in PyInit_klammertext(); the aggregate here only
// supplies the object header, which is what the CPython docs show.  The
// remaining fields are deliberately left to the aggregate's zero-fill.
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmissing-field-initializers"
PyTypeObject Machine_type = {
    PyVarObject_HEAD_INIT(nullptr, 0)
};
#pragma GCC diagnostic pop

const char* Machine_doc =
    "Machine(target='*', klammersets=None, files=None, output=None, display=False, verbose=0)\n"
    "\n"
    "The Klammermachine, set up as ktext sets it up for\n"
    "    ktext <files> -t <target> --klammersets <klammersets> -o <output>\n"
    "and with the klammersets loaded (the Standard Klammer Set when klammersets\n"
    "is None; 'none' loads none).  Nothing is read: call read(), read_file()\n"
    "or read_files(), then apply().  Only one Machine may exist at a time.";

PyMethodDef module_methods[] = {
    {nullptr, nullptr, 0, nullptr}
};

PyModuleDef module_def = {
    PyModuleDef_HEAD_INIT,
    "klammertext",
    "The Python binding of the Klammermachine: ktext in-process.\n\n"
    "    import klammertext\n"
    "    M = klammertext.Machine(target='html', klammersets=['sks'])\n"
    "    M.read('@i Hello @')\n"
    "    print(M.apply())\n\n"
    "Errors are klammertext.Error and its subclasses, named as the C++ classes\n"
    "are (Parsing_error, File_error, ...).",
    -1,
    module_methods,
    nullptr, nullptr, nullptr, nullptr
};

bool make_error_classes(PyObject* module)
{
    error_base = PyErr_NewExceptionWithDoc(
        "klammertext.Error",
        "A Klammertext error: str(e) is the report ktext prints; e.kind,\n"
        "e.description, e.filename, e.line and e.column are its parts.  An\n"
        "@eval that raised a Python exception has it as e.__cause__.",
        nullptr, nullptr);
    if (!error_base) {
        return false;
    }
    Py_INCREF(error_base);
    PyModule_AddObject(module, "Error", error_base);
    for (const auto& [kind, name] : error_kinds) {
        std::string qualified = "klammertext." + name;
        std::string doc = "A Klammertext " + kind + " error (the C++ " + name + ").";
        PyObject* cls = PyErr_NewExceptionWithDoc(
            qualified.c_str(), doc.c_str(), error_base, nullptr);
        if (!cls) {
            return false;
        }
        error_classes[kind] = cls;
        Py_INCREF(cls);
        PyModule_AddObject(module, name.c_str(), cls);
    }
    return true;
}

}  // namespace

PyMODINIT_FUNC PyInit_klammertext(void)
{
    Machine_type.tp_name = "klammertext.Machine";
    Machine_type.tp_basicsize = sizeof(Machine_object);
    Machine_type.tp_itemsize = 0;
    Machine_type.tp_flags = Py_TPFLAGS_DEFAULT;
    Machine_type.tp_doc = Machine_doc;
    Machine_type.tp_new = Machine_new;
    Machine_type.tp_init = Machine_init;
    Machine_type.tp_dealloc = Machine_dealloc;
    Machine_type.tp_repr = Machine_repr;
    Machine_type.tp_methods = Machine_methods;
    Machine_type.tp_getset = Machine_getset;
    if (PyType_Ready(&Machine_type) < 0) {
        return nullptr;
    }
    PyObject* module = PyModule_Create(&module_def);
    if (!module) {
        return nullptr;
    }
    Py_INCREF(&Machine_type);
    PyModule_AddObject(module, "Machine", reinterpret_cast<PyObject*>(&Machine_type));
    if (!make_error_classes(module)) {
        return nullptr;
    }
    // What an error report names as the command.
    set_string_source();
    const char* home = std::getenv(klammertext_home_var.c_str());
    PyObject* home_obj = home ? PyUnicode_FromString(home) : Py_None;
    if (!home) {
        Py_INCREF(Py_None);
    }
    PyModule_AddObject(module, "home", home_obj);
    return module;
}