The Python binding (kpy/): klammertext.Machine from a script

Assembled from klammertext-dev ed13dfaa5b7a.  kpy/klammertext.cpp is a
CPython extension module giving a Python script an instance of the
Machine class -- ktext in-process, set up through the same functions the
command uses.  Errors map one-to-one to Python exceptions under the C++
names; an @eval's Python exception is chained as __cause__.  Built by
make -C mac, found through PYTHONPATH (env/runtime.env sets it), tested
by tst/kpy_test.sh.  Three engine changes came with it: Error::message(),
set_command_state(), and the @eval error report carrying its traceback.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uv7KWxafinDPraswEf4GsC

mac/Makefile

@@ -31,11 +31,12 @@ DEPFLAGS = -MMD -MP -MF $(@:.o=.d)
 	$(CXX) -c $(CPPFLAGS) $(CXXFLAGS) $(DEPFLAGS) $< -o $@
 
 # Default target
-.PHONY: all sks ref clean redo clang
+.PHONY: all sks ref kpy clean redo clang
 
 all : $(LIBRARY)
 	$(MAKE) sks
 	$(MAKE) ref
+	$(MAKE) kpy
 
 # Create lib directory
 $(LIBDIR):
@@ -60,6 +61,13 @@ sks :
 ref :
 	$(MAKE) -C $(K)/ref
 
+# kpy/ is the Python binding: a CPython extension module that links against
+# libklammertext.so and shares its class layouts, like the sks DSOs and
+# ref/.  A fourth consumer of the library, and neither a command nor a
+# klammer set.
+kpy :
+	$(MAKE) -C $(K)/kpy
+
 clean :
 	rm -f $(OBJECTS) $(DEPFILES) $(LIBRARY) *~
 

mac/command.cpp

@@ -121,6 +121,29 @@ parse_args(
 }
 
 
+void set_command_state(
+    Machine& machine, const std::string& frame_name, const std::string& target,
+    const std::vector<std::string>& input_filenames, const std::string& output_dir,
+    const std::string& output_basename, bool display_only)
+{
+    State& S = machine.m_state;
+    S.open_frame(frame_name);
+    S.set("K_target", target);
+    S.set("K_output_dir", output_dir);
+    S.set("K_output_basename", output_basename);
+    S.set("K_stdout_only", display_only ? "true" : "false");
+    // The list form uses the standalone-"/" separator (filenames may
+    // contain spaces); K_input_filename is the single root input file.
+    S.set("K_input_filenames", join(input_filenames, " / "));
+    S.set("K_input_filename", input_filenames.empty() ? "" : input_filenames[0]);
+    S.set("K_verbose_level", std::to_string(verbose_level));
+    if (!input_filenames.empty()) {
+        S.set("K_input_dir", absolute_pathname(file_directory(input_filenames[0])));
+    } else {
+        S.set("K_input_dir", fs::current_path().string());
+    }
+}
+
 void load_klammersets(Machine& machine, const strings_t& symbols)
 {
     // "none" is the absence of a klammerset, not the name of one, so it says

mac/command.h

@@ -20,3 +20,18 @@ parse_args(
     std::string output_basename, bool display_only);
 
 void load_klammersets(Machine& machine, const strings_t& symbols);
+
+// The state a command gives a Machine before anything is read: the frame
+// named for the command and the K_* variables that describe the run --
+// target, output directory and basename, whether output is displayed rather
+// than written, the input filenames, the verbosity, and the input directory
+// (the first input file's, or the cwd when there is none).  This was the top
+// of ktext's main(), and it is policy, not mechanism: a klammer set reads
+// these names (@document reads K_stdout_only and K_output_dir), so a second
+// consumer of the Machine -- the Python binding in kpy/ -- must set exactly
+// the same ones or a document that renders under ktext fails under it.  One
+// function, two callers, nothing restated.
+void set_command_state(
+    Machine& machine, const std::string& frame_name, const std::string& target,
+    const std::vector<std::string>& input_filenames, const std::string& output_dir,
+    const std::string& output_basename, bool display_only);

mac/error.cpp

@@ -1,3 +1,5 @@
+#include <sstream>
+
 #include "error.h"
 #include "show.h"
 #include "util.h"
@@ -13,26 +15,33 @@ bool display_source(const std::string& filename_arg)
     }
 }
 
-void Error::print_message(const std::string& epilog)
+std::string Error::message(const std::string& epilog) const
 {
     // The separator is not cosmetic: the description ends with a sentence, and
     // appending the advice straight onto it ran the two together into one word
     // ("...for an unspecified targetkdiag loads no klammerset...").  m_desc is
     // then justified to 80 columns, so the break has to be a blank line rather
     // than a single newline, which justify() would fold back into the flow.
+    std::string desc = m_desc;
     if (epilog != "") {
-        m_desc += "\n\n" + epilog + "\n";
+        desc += "\n\n" + epilog + "\n";
     }
-    m_desc = justify(m_desc, 80, 0);
-    std::cerr << "\n" << red << command_name << " (" << m_type << " error)";
+    std::stringstream ss {};
+    ss << command_name << " (" << m_type << " error)";
     if (display_source(m_loc.m_filename)) {
-        std::cerr << ":\n  " << m_loc.m_filename;
+        ss << ":\n  " << m_loc.m_filename;
     }
     if (m_loc.m_line > -1) {
-        std::cerr << ", line " << m_loc.m_line;
+        ss << ", line " << m_loc.m_line;
     }
     if (m_loc.m_chr > -1) {
-        std::cerr << ", character " << m_loc.m_chr + 1;
+        ss << ", character " << m_loc.m_chr + 1;
     }
-    std::cerr << ":\n\n" << m_desc << reset << "\n";
+    ss << ":\n\n" << justify(desc, 80, 0);
+    return ss.str();
+}
+
+void Error::print_message(const std::string& epilog)
+{
+    std::cerr << "\n" << red << message(epilog) << reset << "\n";
 }

mac/error.h

@@ -17,6 +17,14 @@ public:
         , m_loc(locator)
         {}
 
+    // The report text as the commands print it -- the header naming the
+    // command and the error kind, the location when there is one, and the
+    // description justified to 80 columns -- WITHOUT colour or the
+    // surrounding newlines.  print_message() writes exactly this to stderr;
+    // the Python binding (kpy/) makes it the exception's str(), which is
+    // why the two are one function: an error reads the same from a script
+    // as from ktext.  const: the former print_message mutated m_desc.
+    std::string message(const std::string& epilog="") const;
     void print_message(const std::string& epilog="");
 
     std::string m_type {};

mac/eval_python.cpp

@@ -4,6 +4,7 @@
 #include "log.h"
 
 std::regex Eval_python::statement_delimiter("\\s*;\\s*");
+PyObject* Eval_python::s_last_exception = nullptr;
 
 Eval_python::Eval_python(Machine& machine, const Locator& loc)
     : m_machine(machine)
@@ -119,52 +120,10 @@ void Eval_python::import_module(const std::string& module_name, bool verify)
     }
     PyObject* module = PyImport_ImportModule(module_name.c_str());
     if (module == nullptr) {
-        // Extract the Python traceback before clearing the error.
-        // This reveals the actual source of the failure (e.g., a syntax
-        // error in a transitively imported module), not just the top-level
-        // module name that failed to load.
-        std::string detail;
-        PyObject* ptype;
-        PyObject* pvalue;
-        PyObject* ptraceback;
-        PyErr_Fetch(&ptype, &pvalue, &ptraceback);
-        if (pvalue) {
-            PyErr_NormalizeException(&ptype, &pvalue, &ptraceback);
-            PyObject* str = PyObject_Str(pvalue);
-            if (str) {
-                detail = PyUnicode_AsUTF8(str);
-                Py_DECREF(str);
-            }
-            // Format the traceback if available
-            if (ptraceback) {
-                PyObject* tb_module = PyImport_ImportModule("traceback");
-                if (tb_module) {
-                    PyObject* format_tb = PyObject_GetAttrString(tb_module, "format_exception");
-                    if (format_tb) {
-                        PyObject* args = PyTuple_Pack(3, ptype, pvalue, ptraceback);
-                        PyObject* tb_list = PyObject_CallObject(format_tb, args);
-                        if (tb_list) {
-                            PyObject* separator = PyUnicode_FromString("");
-                            PyObject* joined = PyUnicode_Join(separator, tb_list);
-                            if (joined) {
-                                detail = PyUnicode_AsUTF8(joined);
-                                Py_DECREF(joined);
-                            }
-                            Py_DECREF(separator);
-                            Py_DECREF(tb_list);
-                        }
-                        Py_XDECREF(args);
-                        Py_DECREF(format_tb);
-                    }
-                    Py_DECREF(tb_module);
-                }
-            }
-        }
-        Py_XDECREF(ptype);
-        Py_XDECREF(pvalue);
-        Py_XDECREF(ptraceback);
-        PyErr_Clear();
-
+        // The traceback, not just the top-level module name that failed to
+        // load: it reveals the actual source of the failure (e.g. a syntax
+        // error in a transitively imported module).
+        std::string detail = fetch_python_exception();
         std::string message = "Cannot import module \"" + module_name + "\"";
         if (!detail.empty()) {
             message += ":\n\n" + detail;
@@ -176,6 +135,68 @@ void Eval_python::import_module(const std::string& module_name, bool verify)
     PyDict_SetItemString(m_globals, module_name.c_str(), module);
 }
 
+PyObject* Eval_python::take_last_exception()
+{
+    PyObject* e = s_last_exception;
+    s_last_exception = nullptr;
+    return e;
+}
+
+std::string Eval_python::fetch_python_exception()
+{
+    std::string detail {};
+    PyObject* ptype = nullptr;
+    PyObject* pvalue = nullptr;
+    PyObject* ptraceback = nullptr;
+    PyErr_Fetch(&ptype, &pvalue, &ptraceback);
+    if (!pvalue) {
+        Py_XDECREF(ptype);
+        Py_XDECREF(ptraceback);
+        PyErr_Clear();
+        return detail;
+    }
+    PyErr_NormalizeException(&ptype, &pvalue, &ptraceback);
+    if (ptraceback) {
+        // Attach the traceback to the instance, so whoever receives the
+        // object (the binding, as __cause__) can still print it.
+        PyException_SetTraceback(pvalue, ptraceback);
+    }
+    PyObject* str = PyObject_Str(pvalue);
+    if (str) {
+        detail = PyUnicode_AsUTF8(str);
+        Py_DECREF(str);
+    }
+    PyObject* tb_module = PyImport_ImportModule("traceback");
+    if (tb_module) {
+        PyObject* format = PyObject_GetAttrString(tb_module, "format_exception");
+        if (format) {
+            PyObject* args = PyTuple_Pack(
+                3, ptype, pvalue, ptraceback ? ptraceback : Py_None);
+            PyObject* tb_list = PyObject_CallObject(format, args);
+            if (tb_list) {
+                PyObject* separator = PyUnicode_FromString("");
+                PyObject* joined = PyUnicode_Join(separator, tb_list);
+                if (joined) {
+                    detail = PyUnicode_AsUTF8(joined);
+                    Py_DECREF(joined);
+                }
+                Py_DECREF(separator);
+                Py_DECREF(tb_list);
+            }
+            Py_XDECREF(args);
+            Py_DECREF(format);
+        }
+        Py_DECREF(tb_module);
+    }
+    // Keep the instance (it owns its traceback now); drop the rest.
+    Py_XDECREF(s_last_exception);
+    s_last_exception = pvalue;
+    Py_XDECREF(ptype);
+    Py_XDECREF(ptraceback);
+    PyErr_Clear();
+    return detail;
+}
+
 std::string Eval_python::get_result(PyObject* result_object)
 {
     std::string result {};
@@ -184,9 +205,24 @@ std::string Eval_python::get_result(PyObject* result_object)
        result = std::string(value);
        Py_DECREF(result_object);
     } else {
-        std::cout << red;
-        PyErr_Print();
-        throw Parsing_error("Python code error in @eval", m_loc);
+        // Until 2026-09-08 this was "std::cout << red; PyErr_Print();" --
+        // the colour escape on stdout, the traceback on stderr, and then an
+        // Error carrying neither.  The traceback now travels IN the error,
+        // indented so justify() leaves its lines verbatim (a traceback line
+        // is not prose), and the exception object is kept for the binding.
+        std::string detail = fetch_python_exception();
+        std::string message = "Python code error in @eval";
+        if (!detail.empty()) {
+            message += ":\n\n";
+            std::string::size_type from = 0;
+            while (from < detail.size()) {
+                auto nl = detail.find('\n', from);
+                if (nl == std::string::npos) nl = detail.size();
+                message += "  " + detail.substr(from, nl - from) + "\n";
+                from = nl + 1;
+            }
+        }
+        throw Parsing_error(message, m_loc);
     }
     return result;
 }

mac/eval_python.h

@@ -17,6 +17,17 @@ public:
     std::vector<std::string> parse_modules(std::string code);
     void import_module(const std::string& module_name, bool verify = true);
     std::string get_result(PyObject* result_object);
+    // The Python exception currently raised (there must be one): clear it,
+    // return the formatted traceback, and keep the exception object in the
+    // slot below.  ktext puts the text into the Error it throws; the Python
+    // binding (kpy/) additionally chains the object as the Error's __cause__,
+    // so a script sees the ZeroDivisionError its @eval raised, with its
+    // traceback, and not only a description of it.
+    std::string fetch_python_exception();
+    // Take ownership of the exception kept by the last fetch (nullptr if
+    // none); the slot is cleared.  ktext never calls this: it terminates on
+    // the Error, and the interpreter is torn down with the process.
+    static PyObject* take_last_exception();
     std::string eval_expression(const std::string& expression, bool import_modules = true);
     std::string eval_statements(const std::string& script);
     std::string eval(std::string code);
@@ -29,4 +40,5 @@ public:
     Locator m_loc;
     PyObject* m_globals;
     PyObject* m_locals;
+    static PyObject* s_last_exception;
 };

mac/machine.cpp

@@ -935,7 +935,12 @@ void Machine::escape_target_characters(const Target& target, katom_list& katoms)
 std::string Machine::apply(const std::string& target_name, bool final_processing, bool escape_characters)
 {
     (void)K::log(3, "top level");
-    m_state.set("K_target", target_name);
+    // update=true: K_target is THIS apply's target.  A command applies once,
+    // but the Python binding (kpy/) applies the same Machine under several
+    // targets -- apply("html") then apply("tex") -- and the second call
+    // found the first's value in the same frame and died with "Variable
+    // K_target is already defined" (found by tst/kpy_test.sh, 2026-09-08).
+    m_state.set("K_target", target_name, true);
     m_state.subst(m_katoms.begin(), m_katoms.end());
 
     // Escape target-specific characters in writer text before klammer application.

mac/show.h

@@ -84,6 +84,7 @@ std::set sks_commands = {
     "ktext"s,
     "kdesc"s,
     "kdiag"s,
+    "klammertext"s,   // the Python binding's string input (kpy/)
     "argument_test"s,
     "argtype_test"s
 };