ack
fc2e848e91
Assembled from development commit d5f9c20b94b3. Since 5c29da5: a print() in
@eval, or in any Python a klammer set runs, is a log line -- on the
error stream under the command's name at -v 1, silent otherwise, never
in the document -- and K_log(level, text) is the builtin behind it; a
Python script using the binding keeps its own stdout. Previously such
output was lost, or on a terminal landed in the document.
352 lines
12 KiB
C++
352 lines
12 KiB
C++
#include "eval_python.h"
|
|
#include "show.h"
|
|
#include "util.h"
|
|
#include "log.h"
|
|
|
|
std::regex Eval_python::statement_delimiter("\\s*;\\s*");
|
|
PyObject* Eval_python::s_last_exception = nullptr;
|
|
|
|
static bool s_python_output_to_log = false;
|
|
|
|
void set_python_output_to_log(bool on)
|
|
{
|
|
s_python_output_to_log = on;
|
|
}
|
|
|
|
// K_log(level, text): the engine's K::log, for Python. A builtin, so a
|
|
// klammer set's modules reach it without an import; absent outside a
|
|
// Machine (kutil.msg falls back to stderr then).
|
|
static PyObject* py_K_log(PyObject*, PyObject* args)
|
|
{
|
|
int level = 1;
|
|
const char* text = nullptr;
|
|
if (!PyArg_ParseTuple(args, "is", &level, &text)) {
|
|
return nullptr;
|
|
}
|
|
(void)K::log(level, std::string(text));
|
|
Py_RETURN_NONE;
|
|
}
|
|
|
|
static PyMethodDef K_log_method = {
|
|
"K_log", py_K_log, METH_VARARGS,
|
|
"K_log(level, text): write text to the Klammertext log, shown at -v level or higher."};
|
|
|
|
static void install_python_log(PyObject* globals)
|
|
{
|
|
PyObject* builtins = PyEval_GetBuiltins(); // borrowed
|
|
if (!PyDict_GetItemString(builtins, "K_log")) {
|
|
PyObject* f = PyCFunction_New(&K_log_method, nullptr);
|
|
if (f) {
|
|
PyDict_SetItemString(builtins, "K_log", f);
|
|
Py_DECREF(f);
|
|
}
|
|
}
|
|
if (!s_python_output_to_log) {
|
|
return;
|
|
}
|
|
// sys.stdout -> the log, line by line, once per process (the class
|
|
// attribute marks ours; a second Eval_python leaves it in place).
|
|
PyRun_String(
|
|
"import sys\n"
|
|
"if not getattr(sys.stdout, '_klammertext_log', False):\n"
|
|
" class _K_log_stream:\n"
|
|
" _klammertext_log = True\n"
|
|
" def __init__(self): self._buf = ''\n"
|
|
" def write(self, s):\n"
|
|
" self._buf += s\n"
|
|
" while '\\n' in self._buf:\n"
|
|
" line, self._buf = self._buf.split('\\n', 1)\n"
|
|
" K_log(1, line)\n"
|
|
" return len(s)\n"
|
|
" def flush(self):\n"
|
|
" if self._buf: K_log(1, self._buf); self._buf = ''\n"
|
|
" def isatty(self): return False\n"
|
|
" sys.stdout = _K_log_stream()\n",
|
|
Py_file_input, globals, globals);
|
|
}
|
|
|
|
Eval_python::Eval_python(Machine& machine, const Locator& loc)
|
|
: m_machine(machine)
|
|
, m_loc(loc)
|
|
, m_globals(nullptr)
|
|
, m_locals(nullptr)
|
|
{
|
|
(void)K::log(3);
|
|
// Only initialize if Python is not already initialized
|
|
if (!Py_IsInitialized()) {
|
|
#if PY_VERSION_HEX >= 0x030B0000
|
|
// Python 3.11+ uses PyConfig API
|
|
PyConfig config;
|
|
PyConfig_InitPythonConfig(&config);
|
|
Py_InitializeFromConfig(&config);
|
|
PyConfig_Clear(&config);
|
|
#else
|
|
Py_Initialize();
|
|
#endif
|
|
}
|
|
add_module_path("..");
|
|
add_module_path(".");
|
|
m_globals = PyDict_New();
|
|
m_locals = PyDict_New();
|
|
PyDict_SetItemString(m_globals, "__builtins__", PyEval_GetBuiltins());
|
|
install_python_log(m_globals);
|
|
// The machine's result text, so :after_apply phase functions can take it
|
|
// as an argument (the Python counterpart of a :cpp phase function reading
|
|
// machine.m_result). Set directly rather than through the state's
|
|
// python_code() because document text cannot be safely embedded in a
|
|
// quoted Python source string.
|
|
PyObject* result_text = PyUnicode_FromString(m_machine.m_result.c_str());
|
|
if (result_text) {
|
|
PyDict_SetItemString(m_globals, "K_result", result_text);
|
|
Py_DECREF(result_text);
|
|
}
|
|
import_module("inspect", false);
|
|
// The :after_apply phase dispatcher: a phase that declares a K
|
|
// parameter receives the state's class K; one that does not (a stdlib
|
|
// function like string.capwords, whose second parameter is a separator)
|
|
// is called with the text alone. Discrimination is by parameter NAME,
|
|
// not count -- capwords has two parameters and K is not one of them.
|
|
// Builtins whose signature inspect cannot read are treated as not
|
|
// wanting K.
|
|
PyRun_String(
|
|
"def K_phase_call(f, text, K):\n"
|
|
" try:\n"
|
|
" wants = 'K' in inspect.signature(f).parameters\n"
|
|
" except (ValueError, TypeError):\n"
|
|
" wants = False\n"
|
|
" return f(text, K=K) if wants else f(text)\n",
|
|
Py_file_input, m_globals, m_globals);
|
|
if (!m_machine.m_state.m_frames.empty()) {
|
|
PyRun_String(m_machine.m_state.python_code().c_str(), Py_file_input, m_globals, m_locals);
|
|
}
|
|
}
|
|
|
|
Eval_python::~Eval_python()
|
|
{
|
|
(void)K::log(3, "destructor");
|
|
// Clean up our objects BEFORE finalizing Python
|
|
if (m_globals) {
|
|
Py_DECREF(m_globals);
|
|
m_globals = nullptr;
|
|
}
|
|
if (m_locals) {
|
|
Py_DECREF(m_locals);
|
|
m_locals = nullptr;
|
|
}
|
|
// Don't call Py_Finalize() here - it can cause double-free if other
|
|
// Eval_python objects exist or if Python is used elsewhere.
|
|
// Python will clean up automatically at program exit.
|
|
}
|
|
|
|
std::string remove_string_values(std::string s)
|
|
{
|
|
return std::regex_replace(s, std::regex(R"(\".*?\")"), "\"\"");
|
|
}
|
|
|
|
void Eval_python::add_module_path(const std::string& path)
|
|
{
|
|
PyObject* sys_path = PySys_GetObject("path"); // Borrowed reference
|
|
if (sys_path) {
|
|
PyObject* py_path = PyUnicode_FromString(path.c_str());
|
|
if (py_path) {
|
|
PyList_Insert(sys_path, 0, py_path); // Insert at front for priority
|
|
Py_DECREF(py_path);
|
|
}
|
|
}
|
|
}
|
|
|
|
strings_t Eval_python::parse_modules(std::string code)
|
|
{
|
|
(void)K::log(3, code);
|
|
code = remove_string_values(code); // Hack! Don't look for module patterns in strings.
|
|
std::regex module_re(R"(([A-Za-z]\w*)\.[A-Za-z_]\w*)");
|
|
auto code_begin = std::sregex_iterator(code.begin(), code.end(), module_re);
|
|
auto code_end = std::sregex_iterator();
|
|
std::vector<std::string> modules;
|
|
for (std::sregex_iterator it = code_begin; it != code_end; ++it) {
|
|
modules.push_back((*it).str(1));
|
|
}
|
|
return modules;
|
|
}
|
|
|
|
void Eval_python::import_module(const std::string& module_name, bool verify)
|
|
{
|
|
(void)K::log(3, module_name);
|
|
|
|
std::string module_check =
|
|
"\"" + module_name + "\" in locals() and inspect.isclass(" + module_name + ")";
|
|
if (verify && eval_expression(module_check, false) == "True") {
|
|
return;
|
|
}
|
|
PyObject* module = PyImport_ImportModule(module_name.c_str());
|
|
if (module == nullptr) {
|
|
// The traceback, not just the top-level module name that failed to
|
|
// load: it reveals the actual source of the failure (e.g. a syntax
|
|
// error in a transitively imported module).
|
|
std::string detail = fetch_python_exception();
|
|
std::string message = "Cannot import module \"" + module_name + "\"";
|
|
if (!detail.empty()) {
|
|
message += ":\n\n" + detail;
|
|
}
|
|
throw Argument_error(message, m_loc);
|
|
}
|
|
// PyDict_SetItemString steals a reference, so we don't need to DECREF module
|
|
// The dictionary will own the reference
|
|
PyDict_SetItemString(m_globals, module_name.c_str(), module);
|
|
}
|
|
|
|
PyObject* Eval_python::take_last_exception()
|
|
{
|
|
PyObject* e = s_last_exception;
|
|
s_last_exception = nullptr;
|
|
return e;
|
|
}
|
|
|
|
std::string Eval_python::fetch_python_exception()
|
|
{
|
|
std::string detail {};
|
|
PyObject* ptype = nullptr;
|
|
PyObject* pvalue = nullptr;
|
|
PyObject* ptraceback = nullptr;
|
|
PyErr_Fetch(&ptype, &pvalue, &ptraceback);
|
|
if (!pvalue) {
|
|
Py_XDECREF(ptype);
|
|
Py_XDECREF(ptraceback);
|
|
PyErr_Clear();
|
|
return detail;
|
|
}
|
|
PyErr_NormalizeException(&ptype, &pvalue, &ptraceback);
|
|
if (ptraceback) {
|
|
// Attach the traceback to the instance, so whoever receives the
|
|
// object (the binding, as __cause__) can still print it.
|
|
PyException_SetTraceback(pvalue, ptraceback);
|
|
}
|
|
PyObject* str = PyObject_Str(pvalue);
|
|
if (str) {
|
|
detail = PyUnicode_AsUTF8(str);
|
|
Py_DECREF(str);
|
|
}
|
|
PyObject* tb_module = PyImport_ImportModule("traceback");
|
|
if (tb_module) {
|
|
PyObject* format = PyObject_GetAttrString(tb_module, "format_exception");
|
|
if (format) {
|
|
PyObject* args = PyTuple_Pack(
|
|
3, ptype, pvalue, ptraceback ? ptraceback : Py_None);
|
|
PyObject* tb_list = PyObject_CallObject(format, args);
|
|
if (tb_list) {
|
|
PyObject* separator = PyUnicode_FromString("");
|
|
PyObject* joined = PyUnicode_Join(separator, tb_list);
|
|
if (joined) {
|
|
detail = PyUnicode_AsUTF8(joined);
|
|
Py_DECREF(joined);
|
|
}
|
|
Py_DECREF(separator);
|
|
Py_DECREF(tb_list);
|
|
}
|
|
Py_XDECREF(args);
|
|
Py_DECREF(format);
|
|
}
|
|
Py_DECREF(tb_module);
|
|
}
|
|
// Keep the instance (it owns its traceback now); drop the rest.
|
|
Py_XDECREF(s_last_exception);
|
|
s_last_exception = pvalue;
|
|
Py_XDECREF(ptype);
|
|
Py_XDECREF(ptraceback);
|
|
PyErr_Clear();
|
|
return detail;
|
|
}
|
|
|
|
std::string Eval_python::get_result(PyObject* result_object)
|
|
{
|
|
std::string result {};
|
|
if (result_object) {
|
|
const char* value = PyUnicode_AsUTF8(result_object);
|
|
result = std::string(value);
|
|
Py_DECREF(result_object);
|
|
} else {
|
|
// Until 2026-09-08 this was "std::cout << red; PyErr_Print();" --
|
|
// the colour escape on stdout, the traceback on stderr, and then an
|
|
// Error carrying neither. The traceback now travels IN the error,
|
|
// indented so justify() leaves its lines verbatim (a traceback line
|
|
// is not prose), and the exception object is kept for the binding.
|
|
std::string detail = fetch_python_exception();
|
|
std::string message = "Python code error in @eval";
|
|
if (!detail.empty()) {
|
|
message += ":\n\n";
|
|
std::string::size_type from = 0;
|
|
while (from < detail.size()) {
|
|
auto nl = detail.find('\n', from);
|
|
if (nl == std::string::npos) nl = detail.size();
|
|
message += " " + detail.substr(from, nl - from) + "\n";
|
|
from = nl + 1;
|
|
}
|
|
}
|
|
throw Parsing_error(message, m_loc);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
std::string Eval_python::eval_expression(const std::string& expression, bool import_modules)
|
|
{
|
|
(void)K::log(3, expression);
|
|
// msg() << "expression: " << expression << "\n";
|
|
if (import_modules && expression.find('.') != std::string::npos) {
|
|
for (const auto& m : parse_modules(expression)) {
|
|
import_module(m);
|
|
}
|
|
}
|
|
return get_result(
|
|
PyRun_String(
|
|
std::string("str(" + expression +")").c_str(),
|
|
Py_eval_input, m_globals, m_locals));
|
|
}
|
|
|
|
std::string Eval_python::eval_statements(const std::string& script)
|
|
{
|
|
(void)K::log(3);
|
|
strings_t statements = regex_split(script, statement_delimiter);
|
|
for (auto iter = statements.begin(); iter < statements.end() - 1; iter++) {
|
|
(void)K::log(3, " Run: " + (*iter));
|
|
PyRun_String(iter->c_str(), Py_file_input, m_globals, m_locals);
|
|
}
|
|
(void)K::log(3, " Result from: " + statements.back());
|
|
return get_result(
|
|
PyRun_String(std::string("str("+statements.back()+")").c_str(),
|
|
Py_eval_input, m_globals, m_locals));
|
|
}
|
|
|
|
std::string Eval_python::eval(std::string code)
|
|
{
|
|
(void)K::log(3, code);
|
|
code = m_machine.m_state.subst(code, true);
|
|
// Kept as a record of a value worth watching; line comments rather than a
|
|
// block, so the msg() guard can see it is inert (dev/check_output_policy.sh
|
|
// is line-based and cannot tell it is inside a /* */).
|
|
// msg() << "\n"
|
|
// << std::string(80, '-') << "\n"
|
|
// << code << "\n"
|
|
// << std::string(80, '-') << "\n";
|
|
if (std::regex_search(code, statement_delimiter)) {
|
|
return eval_statements(code);
|
|
} else {
|
|
return eval_expression(code);
|
|
}
|
|
}
|
|
|
|
std::string Eval_python::eval_katom_list(
|
|
katom_list& katoms, const katom_iter& begin, const katom_iter& end)
|
|
{
|
|
(void)K::log(3, katoms);
|
|
katom_iter code_begin = begin + 1;
|
|
katom_iter code_end = end - 1;
|
|
std::string code_result = eval(as_string(code_begin, code_end, true));
|
|
(void)K::log(3, "code_result:", code_result);
|
|
katom_list code_katoms = m_machine.process(code_result, command_name);
|
|
for (auto kiter = code_begin; kiter < code_end; kiter++) {
|
|
kiter->m_type = katom_t::replaced;
|
|
}
|
|
katoms.insert(end, code_katoms.begin(), code_katoms.end());
|
|
return code_result;
|
|
}
|