Filenames with spaces; output beside the input file; warning fixes (from dev c08eb1bbfa4c)

- Filenames may contain spaces: quote on the command line; filename lists use a standalone "/" separator; unseparated names that do not exist are rejoined into names that do (announced); leading ~ expands. - Without -o, output is written to the input file's directory; -o fully specifies the output directory and basename. - "Word not parsed" warnings now appear without -v, only for text that survives removal, with correct line numbers. - New tst/filename_test.sh regression suite. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-26 23:22:59 +02:00
parent d9c98ac86d
commit fd9a370af7
21 changed files with 430 additions and 34 deletions
--- a/mac/argv.cpp
+++ b/mac/argv.cpp
@@ -175,6 +175,7 @@ void Argv::parse_vars(strings_t& words, string_map& named_args)
            last++;
        }
        named_args[name] = join(strings_t(first, last), " ");
+        m_vectors[name] = strings_t(first, last);
        words.erase(it, last);
    }
 }
@@ -369,6 +370,11 @@ Argv::classify_arguments(int argc, char* argv[], bool full_parse)
    parse_optional(words, named_args);
    parse_positional(argv_to_string(argc, argv), join(words, " "), named_args);
    words.erase(std::remove(words.begin(), words.end(), Argv::delimiter), words.end());
+    if (m_req_names.size() == 1) {
+        // A single positional argument owns all remaining words; keep the
+        // original argv boundaries alongside the joined named_args value.
+        m_vectors[m_req_names[0]] = words;
+    }
    // std::cout << "Named args:\n" << named_args << "\n";
    return named_args;
 }
@@ -556,7 +562,16 @@ std::string Argv::as_string(const std::string& name)
 strings_t Argv::as_vector(const std::string& name)
 {
    (void)K::log(2, name);
-    return regex_split(get(name), std::regex(R"(\s+)"));
+    if (m_vectors.count(name) > 0) {
+        return m_vectors.at(name);
+    }
+    // No stored boundaries (e.g. an opt, whose value is a single argv
+    // word): the value is one element, spaces and all -- never re-split.
+    std::string value = get(name);
+    if (value.empty()) {
+        return {};
+    }
+    return { value };
 }

 std::pair<std::string, strings_t> Argv::as_input(const std::string& name, bool allow_empty)
--- a/mac/argv.h
+++ b/mac/argv.h
@@ -113,5 +113,10 @@ public:
    std::vector<std::string> m_var_names {};
    std::set<std::string> m_given {};
    std::vector<std::string> m_hyphen_markers {};
+    // Original argv word boundaries for multi-word arguments (the single
+    // positional list and variadic --name options).  as_vector() returns
+    // these, so a shell-quoted filename containing spaces stays one
+    // element; the space-joined m_value remains only for get()/describe().
+    std::map<std::string, std::vector<std::string>> m_vectors {};
    long unsigned int m_syntax_size = 0;
 };
--- a/mac/command.cpp
+++ b/mac/command.cpp
@@ -73,7 +73,9 @@ parse_args(
        output_dir = file_directory(output_basename);
        output_basename = file_basename(output_basename);
    } else if (!input_filenames.empty()) {
-        output_dir = "";  // defaults to cwd via absolute_pathname below
+        // No -o: output goes next to the input file ("" -- an input with no
+        // directory component -- resolves to cwd below).
+        output_dir = file_directory(input_filenames[0]);
        output_basename = file_basename(input_filenames[0]);
    }
    output_dir = absolute_pathname(output_dir);
--- a/mac/eval.cpp
+++ b/mac/eval.cpp
@@ -158,6 +158,7 @@ katom_list Eval::eval(katom_iter begin, katom_iter end)
    std::string eval_result = eval_command(begin, end);
    katom_list result {};
    Machine M = m_machine;
+    M.m_warn_unparsed = false; // Result text is machine-generated (see machine.h)
    size_t before = M.m_katoms.size();
    M.read(eval_result);
    // If the @eval produced more Klammertext -- the read-back result still
--- a/mac/file.cpp
+++ b/mac/file.cpp
@@ -115,6 +115,120 @@ bool file_exists(const std::string& pathname, bool error_if_not, bool is_directo
 }


+// Filename lists (see file.h): standalone-"/" separation, existence-rescue
+// grouping, and tilde expansion.
+
+std::string expand_tilde(const std::string& path)
+{
+    if (path == "~" || (path.size() > 1 && path[0] == '~' && path[1] == '/')) {
+        std::string home = get_env_var("HOME");
+        if (!home.empty()) {
+            return home + path.substr(1);
+        }
+    }
+    return path;
+}
+
+static bool filename_exists(const std::string& name, const std::string& base_dir)
+{
+    std::string resolved = expand_tilde(name);
+    if (!base_dir.empty() && !fs::path(resolved).is_absolute()) {
+        resolved = base_dir + "/" + resolved;
+    }
+    return fs::is_regular_file(resolved);
+}
+
+strings_t group_filename_tokens(const strings_t& tokens, const std::string& base_dir)
+{
+    strings_t result {};
+    if (std::find(tokens.begin(), tokens.end(), "/") != tokens.end()) {
+        // Deterministic form: standalone "/" separates the filenames; the
+        // tokens between separators form one name.  No existence checks.
+        strings_t group {};
+        for (const std::string& t : tokens) {
+            if (t == "/") {
+                if (!group.empty()) result.push_back(join(group));
+                group.clear();
+            } else {
+                group.push_back(t);
+            }
+        }
+        if (!group.empty()) result.push_back(join(group));
+    } else {
+        // Rescue: a token naming an existing file stands alone; one that
+        // does not is joined with following tokens until the accumulated
+        // name exists.  A name that never resolves is kept as given, so the
+        // missing-file error downstream reports what the user wrote.
+        size_t i = 0;
+        while (i < tokens.size()) {
+            if (filename_exists(tokens[i], base_dir)) {
+                result.push_back(tokens[i]);
+                ++i;
+                continue;
+            }
+            std::string acc = tokens[i];
+            size_t j = i + 1;
+            bool found = false;
+            while (j < tokens.size()) {
+                acc += " " + tokens[j];
+                ++j;
+                if (filename_exists(acc, base_dir)) {
+                    std::cerr << command_name << ": interpreting \""
+                              << acc << "\" as one filename\n";
+                    result.push_back(acc);
+                    i = j;
+                    found = true;
+                    break;
+                }
+            }
+            if (!found) {
+                result.push_back(tokens[i]);
+                ++i;
+            }
+        }
+    }
+    for (std::string& name : result) {
+        name = expand_tilde(name);
+    }
+    return result;
+}
+
+strings_t resolve_filename_list(const std::string& text, const std::string& base_dir)
+{
+    // Split on standalone "/" at the string level first, so a name's inner
+    // spacing survives exactly; without a separator, fall back to
+    // whitespace tokens and the rescue in group_filename_tokens().
+    auto standalone_slash = [&](size_t i) {
+        return text[i] == '/'
+            && (i == 0 || std::isspace(static_cast<unsigned char>(text[i-1])))
+            && (i + 1 == text.size() || std::isspace(static_cast<unsigned char>(text[i+1])));
+    };
+    bool has_separator = false;
+    for (size_t i = 0; i < text.size(); ++i) {
+        if (standalone_slash(i)) {
+            has_separator = true;
+            break;
+        }
+    }
+    if (has_separator) {
+        strings_t result {};
+        std::string part {};
+        for (size_t i = 0; i < text.size(); ++i) {
+            if (standalone_slash(i)) {
+                part = trim(part);
+                if (!part.empty()) result.push_back(expand_tilde(part));
+                part.clear();
+            } else {
+                part += text[i];
+            }
+        }
+        part = trim(part);
+        if (!part.empty()) result.push_back(expand_tilde(part));
+        return result;
+    }
+    return group_filename_tokens(word_split(text), base_dir);
+}
+
 std::string string_from_file(const std::string& pathname, bool strip_surrounding_whitespace)
 {
    std::regex klammertext_filename_re { R"(.*\.kt?)" };
--- a/mac/file.h
+++ b/mac/file.h
@@ -17,6 +17,18 @@ std::string relative_pathname(const std::string& filename, const std::string& ba
 fs::path relative_to_cwd(const fs::path& input);

 bool file_exists(const std::string& pathname, bool error_if_not=false, bool is_directory=false);
+
+// Filenames may contain spaces.  A filename LIST in a flat string or an
+// argument vector is separated by a standalone "/" token (whitespace on both
+// sides; never a legal input filename -- "/" alone is the root directory).
+// Without a separator, whitespace-split tokens that do not name existing
+// files are greedily rejoined with their neighbors into names that do (the
+// rescue is announced).  A leading "~/" (or bare "~") expands to $HOME.
+std::string expand_tilde(const std::string& path);
+std::vector<std::string> group_filename_tokens(
+    const std::vector<std::string>& tokens, const std::string& base_dir="");
+std::vector<std::string> resolve_filename_list(
+    const std::string& text, const std::string& base_dir="");
 std::string string_from_file(const std::string& pathname, bool strip_surrounding_whitespace=false);
 void string_to_file(const std::string& pathname, std::string contents);
 std::vector<std::string> get_files_in_directory(const std::string& dir);
--- a/mac/katom.cpp
+++ b/mac/katom.cpp
@@ -92,16 +92,39 @@ katom_list make_katoms_from_word(std::string s, const std::string& source_desc,
                return klist;
            }
        }
-        if (verbose_level > 0) {
-        std::cerr << command_name
-                  << " [warning]: Word not parsed in "
-                  << source_desc << ", line " << line+1 << ":\n"
-                  << "      " << s << "\n"
-                  << "To include a special character (@, |, #, and ^), put \"^\" before it.\n";
-        //return std::vector{ std::make_shared<Katom>(s, Locator(), katom_t::word) };
-        //return std::vector{ std::make_shared<Katom>(s, katom_t::word, Locator(source_desc, line, chr)) };
+        Katom k(s, katom_t::word, Locator(source_desc, line, chr));
+        k.m_unparsed = true; // Warning deferred to warn_unparsed_katoms()
+        return std::vector{ k };
+    }
+}
+
+// Warn about words that matched no katom type -- but only those that
+// survive processing: text removed by #, ##, or #[...]#, replaced spans,
+// and literal content (definition interiors, @code bodies) never warn.
+// Called at the end of Machine::process_katoms(), after those passes have
+// marked the katoms.  Clears the flag after warning so repeated processing
+// of the same katom list does not warn twice.  With warn=false (the @eval
+// read-back sub-Machine, whose katoms hold machine-generated result text)
+// no warning is printed and every flag is cleared, so the katoms stay
+// silent after they are spliced into the calling Machine's list.
+void warn_unparsed_katoms(katom_list& katoms, bool warn)
+{
+    for (Katom& k : katoms) {
+        if (!k.m_unparsed) continue;
+        if (!warn) {
+            k.m_unparsed = false;
+            continue;
+        }
+        if (k.m_type != katom_t::ignored
+            && k.m_type != katom_t::replaced
+            && k.m_type != katom_t::literal) {
+            std::cerr << command_name
+                      << " [warning]: Word not parsed in "
+                      << k.m_loc.m_filename << ", line " << k.m_loc.m_line << ":\n"
+                      << "      " << k.m_text << "\n"
+                      << "To include a special character (@, |, #, and ^), put \"^\" before it.\n";
+            k.m_unparsed = false;
        }
-        return std::vector{ Katom(s, katom_t::word, Locator(source_desc, line, chr)) };
    }
 }

--- a/mac/katom.h
+++ b/mac/katom.h
@@ -16,7 +16,7 @@ public:
    Katom(const std::string& src, katom_t type, Locator loc);

    // Copy constructor
-    Katom(const Katom& other) 
+    Katom(const Katom& other)
        : m_index(other.m_index)
        , m_text(other.m_text)
        , m_src(other.m_src)
@@ -24,6 +24,7 @@ public:
        , m_type(other.m_type)
        , m_initial_type(other.m_initial_type)
        , m_display(other.m_display)
+        , m_unparsed(other.m_unparsed)
        {}

    // Copy assignment operator
@@ -36,6 +37,7 @@ public:
            m_type = other.m_type;
            m_initial_type = other.m_initial_type;
            m_display = other.m_display;
+            m_unparsed = other.m_unparsed;
        }
        return *this;
    }
@@ -68,6 +70,10 @@ public:
    katom_t m_type;
    katom_t m_initial_type;
    std::string m_display {};
+    // Set when the word matched no katom type and fell back to katom_t::word.
+    // The warning is deferred to warn_unparsed_katoms(), after removal and
+    // literal marking, so removed text (comments, #[...]# blocks) never warns.
+    bool m_unparsed {};
 };

 bool active(const std::vector<Katom>& katoms);
@@ -97,6 +103,7 @@ std::vector<std::string> line_split(std::string s);
 std::pair<std::string, std::vector<std::string>> line_split(fs::path pathname);
 std::vector<Katom> katomize(const std::vector<std::string>& lines, const std::string& source_desc);

+void warn_unparsed_katoms(std::vector<Katom>& katoms, bool warn = true);
 void process_whitespace_modifiers(std::vector<Katom>& katoms);
 std::vector<Katom> trim_whitespace(std::vector<Katom> katoms);

--- a/mac/machine.cpp
+++ b/mac/machine.cpp
@@ -249,6 +249,7 @@ void Machine::process_katoms(
    if (read) expand_read_katoms(
        katoms, source,
        nonascii, literal, ignore, whitespace, klammers, eval, cond, read);
+    warn_unparsed_katoms(katoms, m_warn_unparsed);
 //    return katoms;
 }

--- a/mac/machine.h
+++ b/mac/machine.h
@@ -87,6 +87,11 @@ public:
    input_sources_t m_sources {};
    std::string m_result {};
    std::vector<Katom> m_katoms {};
+    // False on the sub-Machine that re-reads an @eval result (Eval::eval):
+    // result text is machine-generated -- a renderer's raw target markup
+    // (e.g. a LaTeX column spec "@{}...") legitimately fails katom parsing
+    // and must not produce "Word not parsed" warnings.
+    bool m_warn_unparsed = true;
 };

 /*