klammertext/doc/edit/shared/klammertext_edit.py

#!/usr/bin/env python3
# klammertext_edit.py
#
# The SHARED implementation of Klammertext's editor-support algorithms:
# everything about the language's *structural* layer — @-run tiers, bar-run
# dimension, ^-escapes, # removal, literal spans, nesting depth — that more
# than one editor integration needs.  One algorithm, one file:
#
#   * scanner helpers        name_char_p, escaped_p, block_end, at_run_end
#   * delimiter matching     app_delim_info, app_match, ... (jump / highlight)
#   * indentation            reindent_lines, target_column, ...
#   * table alignment        enclosing_span, compute_edits, ...
#   * diagnostics            diagnostics(): unclosed / mismatched delimiters
#   * a CLI                  indent | align | match | check  over stdin/stdout
#
# Consumers:
#   * doc/edit/sublime/      the Sublime Text plugins import this module
#                            directly (Sublime's plugin host is Python)
#   * doc/edit/vim/          the Vim plugin shells out to the CLI
#   * doc/edit/shared/klammertext_ls.py
#                            the language server (used by VS Code, and by any
#                            LSP client: Neovim, Emacs eglot, Sublime LSP)
#   * tst/editor_test.sh     the regression suite drives the fixtures through
#                            this module and through the CLI
#   * doc/edit/emacs/        NOT a consumer: the Emacs mode is an independent
#                            elisp implementation of the same algorithms,
#                            held equal to this file by the byte-equality
#                            checks in tst/editor_test.sh
#
# This file is the SOURCE OF TRUTH for the policy lists (LITERAL_KLAMMERS,
# TRANSPARENT_KLAMMERS, CODE_KLAMMERS, ALIGN_KLAMMERS) and the numeric limits
# (INDENT_OFFSET, CELL_MAX, ROW_MAX).  It replaces the former per-editor
# copies in Klammertext.py / Klammertext_indent.py / Klammertext_align.py.
# SYNC: the following per-editor artifacts cannot import Python and must be
# kept in step by hand:
#   * the Emacs defcustoms (klammertext-literal-klammers, -transparent-,
#     -code-, -align-klammers, -indent-offset, -align-cell-max, -align-row-max)
#     in doc/edit/emacs/klammertext-mode.el / -indent.el / -align.el
#   * the '@code' rule + literal_code context in
#     doc/edit/sublime/Klammertext.sublime-syntax
#   * the '@code' verbatim region in doc/edit/vim/syntax/klammertext.vim
#   * the '@code' rule in doc/edit/vscode/syntaxes/klammertext.tmLanguage.json
#
# Installation note: editors locate this file either next to their own plugin
# files (a vendored copy, placed there by doc/make_editing_zip.sh), as
# ../shared/klammertext_edit.py relative to the plugin directory (the layout
# of this repository), or under $KLAMMERTEXT_HOME/doc/edit/shared/.
#
# Python floor: 3.8 (the Sublime Text 4 plugin host) — no 3.9+ syntax here.
#
# Known limitation (inherited by every consumer): a raw @ or # inside a
# ^'...'^ literal region confuses the scanners; use ^@ / ^# there instead.

import sys

# --- policy ----------------------------------------------------------------

# Klammer names whose content is a literal argument (verbatim interior,
# closed by a named NAME@ delimiter).
LITERAL_KLAMMERS = set(["code"])

# Klammers that contribute no indentation level (a @document's paragraphs
# stay at the left margin).
TRANSPARENT_KLAMMERS = set(["document"])

# Klammers whose argument span holds code (inline Python is
# indentation-sensitive): lines inside are never reindented.
CODE_KLAMMERS = set(["eval"])

# Klammers whose rows the table-alignment command aligns.
ALIGN_KLAMMERS = set(["table"])

# Spaces per nesting level.
INDENT_OFFSET = 2

# Alignment limits: a row with a cell longer than CELL_MAX characters (or
# spanning lines) is left untouched; if the aligned rows would exceed ROW_MAX
# columns, nothing is changed.
CELL_MAX = 30
ROW_MAX = 100


# --- scanner helpers -------------------------------------------------------

def name_char_p(ch):
    """True if CH can be part of a klammer name (letter, digit or _).
    A hyphen is NOT a name char: @name-arg1 ends the name at the first hyphen."""
    if ch is None:
        return False
    return (('a' <= ch <= 'z') or ('A' <= ch <= 'Z')
            or ('0' <= ch <= '9') or ch == '_')


def escaped_p(s, pos):
    """True if the char at POS is escaped by an odd run of ^ before it.
    In Klammertext ^# and ^@ are literal, so such a char is not a delimiter."""
    n = 0
    i = pos - 1
    while i >= 0 and s[i] == '^':
        n += 1
        i -= 1
    return (n % 2) == 1


def block_end(s, frm):
    """Index just after the ]# that closes a #[ block opened at FROM (the index
    just after the opening #[).  Counts nested #[ ... ]#; len(s) if unclosed."""
    depth = 1
    i = frm
    n = len(s)
    while depth > 0:
        a = s.find('#[', i)
        b = s.find(']#', i)
        if a == -1 and b == -1:
            return n
        if b == -1 or (a != -1 and a < b):
            depth += 1
            i = a + 2
        else:
            depth -= 1
            i = b + 2
    return i


def at_run_end(s, pos):
    """Index just after the run of @ that begins at POS."""
    p = pos
    n = len(s)
    while p < n and s[p] == '@':
        p += 1
    return p


def _name_forward(s, pos):
    """Index just past the run of name chars starting at POS."""
    n = len(s)
    k = pos
    while k < n and name_char_p(s[k]):
        k += 1
    return k


# --- delimiter matching (klammer APPLICATIONS, the single-@ tier) ----------
#
# Matching is context-dependent (the same @ is both open and close, decided
# by its neighbors), so no editor's built-in bracket matching can express it;
# every editor integration routes its jump-to-match and live match
# highlighting through these functions (Emacs excepted; see the header).
# A literal klammer's own delimiters are matched BY NAME (@code <-> code@,
# content opaque); all others match by depth.

def next_app_delim(s, i, limit):
    """From index I, find the next single-@ application delimiter before LIMIT.
    Step over @@/@@@ runs, removed text, literal spans, escaped ^@, and the
    abbreviated @name-arg form.  Return (pos, kind, next_i) with kind 'open' or
    'close' and next_i the index to resume from, or None when none is found."""
    n = len(s)
    if limit is None:
        limit = n
    while i < limit:
        j = i
        while j < limit and s[j] != '@' and s[j] != '#':
            j += 1
        if j >= limit:
            return None
        hit = j
        i = hit + 1                      # default: advance past the hit
        if escaped_p(s, hit):            # ^@ / ^# : keep going
            continue
        nxt = s[hit + 1] if hit + 1 < n else None
        if s[hit] == '#':                # removal: step over it
            if nxt == '#':
                i = n
            elif nxt == '[':
                i = block_end(s, hit + 2)
            elif nxt in ('+', '/', '-'):
                i = hit + 1
            else:                        # to end of line
                eol = s.find('\n', hit)
                i = n if eol == -1 else eol
            continue
        # s[hit] == '@'
        if nxt == '@':                   # @@ / @@@ : step over the run
            i = at_run_end(s, hit)
            continue
        if name_char_p(nxt):             # @name : opening?
            k = _name_forward(s, hit + 1)
            name = s[hit + 1:k]
            after = s[k] if k < n else None
            if name in LITERAL_KLAMMERS:  # literal span: skip to its close
                close = name + '@'
                idx = s.find(close, k)
                i = n if idx == -1 else idx + len(close)
                continue
            elif after == '-':           # @name-arg : opens no span
                i = k
                continue
            else:
                return (hit, 'open', k)
        else:                            # name@ / bare @ : closing
            return (hit, 'close', hit + 1)
    return None


def match_forward(s, open_pos):
    """OPEN_POS is the @ of an opening application.  Return the matching close @
    index, or None if unbalanced."""
    i = _name_forward(s, open_pos + 1)   # past the opening name
    depth = 1
    while depth > 0:
        d = next_app_delim(s, i, None)
        if d is None:
            return None
        pos, kind, nxt = d
        i = nxt
        if kind == 'open':
            depth += 1
        else:
            depth -= 1
            if depth == 0:
                return pos
    return None


def match_backward(s, close_pos):
    """CLOSE_POS is the @ of a closing application.  Return the matching open @
    index, or None if unbalanced.  Scans forward from 0 with a stack."""
    stack = []
    i = 0
    limit = close_pos + 1
    while True:
        d = next_app_delim(s, i, limit)
        if d is None:
            return None
        pos, kind, nxt = d
        i = nxt
        if kind == 'open':
            stack.append(pos)
        else:
            open_pos = stack.pop() if stack else None
            if pos == close_pos:
                return open_pos


def app_delim_info(s, pos):
    """If the char at POS is a single-@ application delimiter, return
    (pos, kind) with kind 'open' or 'close'; else None.  The abbreviated
    @name-arg form (which opens no span) returns None."""
    n = len(s)
    if not (0 <= pos < n):
        return None
    if s[pos] != '@':
        return None
    if pos > 0 and s[pos - 1] == '@':
        return None
    if pos + 1 < n and s[pos + 1] == '@':
        return None
    if escaped_p(s, pos):
        return None
    nxt = s[pos + 1] if pos + 1 < n else None
    if name_char_p(nxt):
        k = _name_forward(s, pos + 1)
        after = s[k] if k < n else None
        if after == '-':
            return None
        return (pos, 'open')
    return (pos, 'close')


def open_name(s, open_pos):
    """Name of the opening @name whose @ is at OPEN_POS."""
    return s[open_pos + 1:_name_forward(s, open_pos + 1)]


def close_name(s, close_pos):
    """Name of a named close NAME@ whose @ is at CLOSE_POS, or None for a bare @
    (including the compact @name@ form, whose name belongs to the opening)."""
    ns = close_pos
    while ns > 0 and name_char_p(s[ns - 1]):
        ns -= 1
    if ns < close_pos and (ns == 0 or s[ns - 1] != '@'):
        return s[ns:close_pos]
    return None


def paren_mismatch(s, open_pos, close_pos):
    """True if the pair is unbalanced (either side None) or the named close
    disagrees with the opening name."""
    if open_pos is None or close_pos is None:
        return True
    cname = close_name(s, close_pos)
    return cname is not None and cname != open_name(s, open_pos)


def token_region(s, pos, kind):
    """(start, end) of the whole delimiter token whose @ is at POS.
    Opening: @ plus its name.  Named close: the name plus @.  Bare @: just @."""
    if kind == 'open':
        return (pos, _name_forward(s, pos + 1))
    ns = pos
    while ns > 0 and name_char_p(s[ns - 1]):
        ns -= 1
    if ns < pos and (ns == 0 or s[ns - 1] != '@'):
        return (ns, pos + 1)          # named close NAME@
    return (pos, pos + 1)             # bare @ (or @name@)


def literal_delim_name(s, pos, kind):
    """If the application delimiter at POS (kind 'open'/'close') belongs to a
    literal klammer (name in LITERAL_KLAMMERS), return its name; else None."""
    name = open_name(s, pos) if kind == 'open' else close_name(s, pos)
    if name and name in LITERAL_KLAMMERS:
        return name
    return None


def literal_match_forward(s, open_pos, name):
    """Index of the @ of the NAME@ that closes the literal @NAME at OPEN_POS, or
    None.  The content is opaque, so search for the literal close string."""
    start = open_pos + 1 + len(name)
    idx = s.find(name + '@', start)
    return idx + len(name) if idx != -1 else None


def literal_match_backward(s, close_pos, name):
    """Index of the @ of the @NAME that opens the literal NAME@ whose @ is at
    CLOSE_POS, or None.  Literal spans do not nest, so the nearest preceding
    real @NAME is the opener (not @@NAME, and not escaped)."""
    open_str = '@' + name
    end = close_pos
    while True:
        idx = s.rfind(open_str, 0, end)
        if idx == -1:
            return None
        before = s[idx - 1] if idx > 0 else None
        if before != '@' and not escaped_p(s, idx):
            return idx
        end = idx


def app_match(s, pos, kind):
    """Matching application delimiter for the delimiter at POS of KIND
    ('open'/'close'), or None.  A literal klammer matches by name (@NAME <->
    NAME@) with content opaque; other klammers match by depth."""
    lit = literal_delim_name(s, pos, kind)
    if lit is not None:
        return (literal_match_forward(s, pos, lit) if kind == 'open'
                else literal_match_backward(s, pos, lit))
    return match_forward(s, pos) if kind == 'open' else match_backward(s, pos)


def match_at(s, pos):
    """The full matching story for a caret at POS (used by jump-to-match and
    the live highlighters).  The delimiter is taken at POS or, failing that,
    just before POS (the on-or-just-after rule).  Return None when POS is not
    at an application delimiter; else a dict:
      {'pos', 'kind', 'token': (start, end),
       'match': int or None, 'match_token': (start, end) or None,
       'mismatch': bool, 'message': str or None}"""
    info = app_delim_info(s, pos)
    if info is None and pos > 0:
        info = app_delim_info(s, pos - 1)
    if info is None:
        return None
    dpos, kind = info
    match = app_match(s, dpos, kind)
    open_pos = dpos if kind == 'open' else match
    close_pos = match if kind == 'open' else dpos
    mism = paren_mismatch(s, open_pos, close_pos)
    message = None
    if mism:
        if match is None:
            if kind == 'open':
                message = ("opening @%s has no matching close"
                           % open_name(s, open_pos))
            else:
                message = "closing delimiter has no matching open"
        else:
            message = ("closing %s@ does not match opening @%s"
                       % (close_name(s, close_pos) or '?',
                          open_name(s, open_pos)))
    other_kind = 'close' if kind == 'open' else 'open'
    return {'pos': dpos, 'kind': kind,
            'token': token_region(s, dpos, kind),
            'match': match,
            'match_token': (token_region(s, match, other_kind)
                            if match is not None else None),
            'mismatch': mism, 'message': message}


# --- indentation -----------------------------------------------------------
#
# The convention (2026-07-27): a line indents to  INDENT_OFFSET x (effective
# depth); a line that BEGINS with a bar run (|, ||, ...) or a closing
# delimiter sits at its opener's column.  Effective depth counts every
# enclosing span uniformly — applications (@), definitions (@@), system
# commands (@@@) — except that TRANSPARENT_KLAMMERS contribute no level, and
# lines inside literal/verbatim content, CODE_KLAMMERS argument spans, or
# removed regions are never touched.  Reindentation is explicit-only in every
# editor: whitespace is content in Klammertext, so nothing reformats as a
# side effect of typing.

def state_at(s, pos):
    """Scan s[0:POS] (POS a line beginning).  Return (stack, opaque): STACK is
    the list of names of the klammer applications, @@ definitions and @@@
    commands open at POS, outermost first; OPAQUE is True when POS lies inside
    content that indentation must not touch (removed text, a literal klammer's
    verbatim span, or a code klammer's argument span)."""
    stack = []
    n = len(s)
    i = 0
    while i < pos:
        j = i
        while j < pos and s[j] != '@' and s[j] != '#':
            j += 1
        if j >= pos:
            break
        hit = j
        i = hit + 1
        if escaped_p(s, hit):            # ^@ / ^# : plain text
            continue
        nxt = s[hit + 1] if hit + 1 < n else None
        if s[hit] == '#':
            if nxt == '#':               # ## removes to end of buffer
                return (stack, True)
            elif nxt == '[':             # #[ ... ]# (nestable)
                end = block_end(s, hit + 2)
                if pos < end:
                    return (stack, True)
                i = end
            elif nxt in ('+', '/', '-'):  # whitespace operators
                pass
            else:                        # # to end of line
                eol = s.find('\n', hit)
                i = n if eol == -1 else eol
            continue
        # an @-run
        run_end = at_run_end(s, hit)
        run_len = run_end - hit
        after = s[run_end] if run_end < n else None
        if name_char_p(after):
            # @name / @@name / @@@name : an opener (or, for a literal
            # klammer, a verbatim span to step over).
            k = _name_forward(s, run_end)
            name = s[run_end:k]
            i = k
            if run_len == 1 and name in LITERAL_KLAMMERS:
                # Verbatim interior: find the closing NAME@ by name.
                idx = s.find(name + '@', k)
                if idx == -1:            # never closed
                    return (stack, True)
                close_end = idx + len(name) + 1
                if pos < close_end:
                    return (stack, True)
                i = close_end
            elif run_len == 1 and k < n and s[k] == '-':
                pass                     # @name-arg : opens no span
            else:
                stack.append(name)
        else:
            # a bare @-run, or the run of a named close: a close.
            if stack:
                stack.pop()
            i = run_end
    opaque = any(name in CODE_KLAMMERS for name in stack)
    return (stack, opaque)


def depth(stack):
    """Number of indentation levels STACK contributes.
    Transparent klammers contribute none."""
    return sum(1 for name in stack if name not in TRANSPARENT_KLAMMERS)


def dedent_line_p(s, bol):
    """True when the line starting at BOL begins with a token that sits at its
    owner's opening column: a bar run (|, ||, ...), a bare close run (@, @@,
    @@@), or a named close (name@, name@@, name@@@).  A line beginning with an
    opener (@name, @@name, @@@name) is content-level."""
    n = len(s)
    i = bol
    while i < n and s[i] in ' \t':
        i += 1
    if i >= n:
        return False
    c = s[i]
    if c == '|':
        return True
    if c == '@':
        run_end = at_run_end(s, i)
        return not name_char_p(s[run_end] if run_end < n else None)
    if name_char_p(c):
        # A named close: name chars followed by an @-run (an unescaped @ can
        # only be a delimiter).
        k = _name_forward(s, i)
        return k < n and s[k] == '@'
    return False


def target_column(s, bol):
    """Column for the line starting at BOL, or None for lines that must not
    be touched (verbatim, code, or removed content)."""
    stack, opaque = state_at(s, bol)
    if opaque:
        return None
    if dedent_line_p(s, bol) and stack:
        stack = stack[:-1]
    return INDENT_OFFSET * depth(stack)


def line_beginnings(s):
    """Offsets of every line beginning in S."""
    return [0] + [i + 1 for i, ch in enumerate(s)
                  if ch == '\n' and i + 1 < len(s)]


def reindent_lines(s, bols):
    """Compute the edits that reindent the lines whose beginnings are BOLS.
    Return a list of (start, end, replacement) triples over S, in ascending
    order, replacing each line's leading whitespace; untouchable lines and
    already-correct lines produce no edit.  Pure function."""
    n = len(s)
    edits = []
    for bol in bols:
        tgt = target_column(s, bol)
        if tgt is None:
            continue
        i = bol
        while i < n and s[i] in ' \t':
            i += 1
        if s[bol:i] != ' ' * tgt:
            edits.append((bol, i, ' ' * tgt))
    return edits


# --- table alignment -------------------------------------------------------
#
# Pads the cells of the enclosing @table's rows so the depth-0 | separators
# line up.  Alignment is for SMALL data items: a row with a cell longer than
# CELL_MAX or spanning lines is untouched (and contributes no width); if the
# aligned rows would exceed ROW_MAX columns nothing changes.  No whitespace
# is ever inserted inside a bar run (|| is a row separator; | | is an empty
# cell — the load-bearing-whitespace trap).  Bars inside a nested klammer
# belong to that klammer: only depth-0 bars count, the same rule the
# Klammermachine applies to @cond.

def enclosing_span(s, pos, names):
    """Innermost span of a klammer named in NAMES that contains POS.
    Return (name, content_start, content_end) with content_start just after
    the opening @name token and content_end at the start of the closing
    delimiter token, or None.  Scans S from the start with a position stack,
    stepping over removed text, literal spans, escaped characters, and the
    abbreviated @name-arg form."""
    stack = []                       # (name, open_token_start, content_start)
    n = len(s)
    i = 0
    while i < n:
        j = i
        while j < n and s[j] != '@' and s[j] != '#':
            j += 1
        if j >= n:
            break
        hit = j
        i = hit + 1
        if escaped_p(s, hit):
            continue
        nxt = s[hit + 1] if hit + 1 < n else None
        if s[hit] == '#':
            if nxt == '#':
                break                # ## removes the rest of the buffer
            elif nxt == '[':
                i = block_end(s, hit + 2)
            elif nxt in ('+', '/', '-'):
                pass
            else:
                eol = s.find('\n', hit)
                i = n if eol == -1 else eol
            continue
        run_end = at_run_end(s, hit)
        run_len = run_end - hit
        after = s[run_end] if run_end < n else None
        if name_char_p(after):
            k = _name_forward(s, run_end)
            name = s[run_end:k]
            i = k
            if run_len == 1 and name in LITERAL_KLAMMERS:
                idx = s.find(name + '@', k)
                if idx == -1:
                    break
                i = idx + len(name) + 1
            elif run_len == 1 and k < n and s[k] == '-':
                pass                 # @name-arg : opens no span
            else:
                stack.append((name, hit, k))
        else:
            # a close: the token starts at the preceding name run, if any
            ns = hit
            while ns > 0 and name_char_p(s[ns - 1]):
                ns -= 1
            tok_start = ns if (ns < hit and (ns == 0 or s[ns - 1] != '@')) else hit
            if stack:
                name, open_start, content_start = stack.pop()
                if name in names and open_start <= pos <= run_end:
                    return (name, content_start, tok_start)
            i = run_end
    return None


def scan_lines(content):
    """Scan CONTENT (the text of a table span).  Return a list of line
    records, one per line: dicts with start, end (offsets into CONTENT, end
    excludes the newline), start_depth, end_depth (klammer nesting relative
    to the span), bars (list of (pos, runlen) for unescaped depth-0 bar
    runs), blocked (inside removed/verbatim content), comment (a # removes
    the rest of the line)."""
    import bisect
    n = len(content)
    line_starts = [0]
    for idx, ch in enumerate(content):
        if ch == '\n':
            line_starts.append(idx + 1)
    nlines = len(line_starts)

    def line_index(p):
        return bisect.bisect_right(line_starts, p) - 1

    lines = [{'start': line_starts[k],
              'end': (line_starts[k + 1] - 1 if k + 1 < nlines else n),
              'bars': [], 'blocked': False, 'comment': False,
              'start_depth': None, 'end_depth': None}
             for k in range(nlines)]
    lines[0]['start_depth'] = 0

    def block_range(a, b):
        """Mark every line touched by [a, b) as blocked."""
        last = max(a, b - 1)
        for k in range(line_index(a), line_index(min(last, n - 1)) + 1):
            lines[k]['blocked'] = True

    depth = 0
    i = 0
    while i < n:
        j = i
        while j < n and content[j] not in '@#|\n':
            j += 1
        if j >= n:
            break
        hit = j
        i = hit + 1
        c = content[hit]
        if c == '\n':
            k = line_index(hit)
            lines[k]['end_depth'] = depth
            if k + 1 < nlines:
                lines[k + 1]['start_depth'] = depth
            continue
        if escaped_p(content, hit):
            continue
        nxt = content[hit + 1] if hit + 1 < n else None
        if c == '#':
            if nxt == '#':
                block_range(hit, n)
                break
            elif nxt == '[':
                e = block_end(content, hit + 2)
                if line_index(max(hit, e - 1)) != line_index(hit):
                    block_range(hit, e)
                i = e
            elif nxt in ('+', '/', '-'):
                pass
            else:                    # # to end of line
                lines[line_index(hit)]['comment'] = True
                eol = content.find('\n', hit)
                i = n if eol == -1 else eol
            continue
        if c == '|':
            if hit > 0 and content[hit - 1] == '|':
                continue             # mid-run (after an escaped ^|)
            k = hit
            while k < n and content[k] == '|':
                k += 1
            if depth == 0:
                lines[line_index(hit)]['bars'].append((hit, k - hit))
            i = k
            continue
        # '@'
        run_end = at_run_end(content, hit)
        run_len = run_end - hit
        after = content[run_end] if run_end < n else None
        if name_char_p(after):
            k = _name_forward(content, run_end)
            name = content[run_end:k]
            i = k
            if run_len == 1 and name in LITERAL_KLAMMERS:
                idx = content.find(name + '@', k)
                e = n if idx == -1 else idx + len(name) + 1
                if line_index(max(hit, e - 1)) != line_index(hit):
                    block_range(hit, e)
                i = e
            elif run_len == 1 and k < n and content[k] == '-':
                pass
            else:
                depth += 1
        else:
            depth = max(0, depth - 1)
            i = run_end

    for ln in lines:
        if ln['start_depth'] is None:
            ln['blocked'] = True
        if ln['end_depth'] is None:
            ln['end_depth'] = depth
    return lines


def compute_edits(content):
    """Compute the alignment edits for CONTENT (a table span's text).
    Return (edits, message): edits is a list of (start, end, new_text)
    triples relative to CONTENT, ascending; message is a status string (a
    reason when edits is empty)."""
    lines = scan_lines(content)

    # The last line holding actual content (a row there may omit its ||).
    last_content = None
    for k in range(len(lines) - 1, 0, -1):
        if content[lines[k]['start']:lines[k]['end']].strip():
            last_content = k
            break

    rows = []                        # (line record, cells, trailing_p)
    chain_ok = True                  # a row must START a row: the previous
    for k in range(1, len(lines)):   # content line ended with || (or was the
        ln = lines[k]                # opener / an option line / blank)
        text = content[ln['start']:ln['end']]
        stripped = text.strip()
        if not stripped:
            continue                 # blank line: chain unchanged
        if stripped.startswith(':') and not ln['bars']:
            continue                 # option line: chain unchanged
        row = _parse_row(content, ln, text, chain_ok, k == last_content)
        trailing = _trailing_rowsep(content, ln)
        chain_ok = trailing
        if row is not None:
            rows.append(row)

    if not rows:
        return ([], "no alignable rows found")

    widths = []
    for _ln, cells, _tr in rows:
        for c_idx, cell in enumerate(cells):
            if c_idx >= len(widths):
                widths.append(0)
            widths[c_idx] = max(widths[c_idx], len(cell))

    indent = ' ' * _indent_width(content, rows[0][0])
    if ROW_MAX is not None:
        longest = 0
        for _ln, cells, trailing in rows:
            m = len(cells)
            w = (len(indent) + sum(widths[:m]) + 3 * (m - 1)
                 + (3 if trailing else 0))
            longest = max(longest, w)
        if longest > ROW_MAX:
            return ([], "aligned rows would be %d characters (limit %d); "
                        "not aligning" % (longest, ROW_MAX))

    edits = []
    for ln, cells, trailing in rows:
        parts = [cells[c].ljust(widths[c]) for c in range(len(cells) - 1)]
        last = cells[-1]
        if trailing:
            last = last.ljust(widths[len(cells) - 1])
        parts.append(last)
        new = indent + ' | '.join(parts) + (' ||' if trailing else '')
        if new != content[ln['start']:ln['end']]:
            edits.append((ln['start'], ln['end'], new))
    msg = ("aligned %d rows" % len(rows)) if edits else "already aligned"
    return (edits, msg)


def _indent_width(content, ln):
    i = ln['start']
    while i < ln['end'] and content[i] in ' \t':
        i += 1
    return i - ln['start']


def _trailing_rowsep(content, ln):
    """True when the line's LAST depth-0 bar run is a || sitting at the end of
    the line (only whitespace after it)."""
    if not ln['bars']:
        return False
    pos, runlen = ln['bars'][-1]
    return (runlen == 2
            and content[pos + 2:ln['end']].strip() == '')


def _parse_row(content, ln, text, chain_ok, is_last_content):
    """If the line is an alignable row, return (ln, cells, trailing_p);
    else None."""
    if ln['blocked'] or ln['comment'] or not chain_ok:
        return None
    if ln['start_depth'] != 0 or ln['end_depth'] != 0:
        return None
    if not ln['bars']:
        return None
    trailing = _trailing_rowsep(content, ln)
    singles = ln['bars'][:-1] if trailing else ln['bars']
    for _pos, runlen in singles:
        if runlen != 1:
            return None              # a mid-line || (or |||): not one row
    if not trailing and not is_last_content:
        return None                  # row continues onto the next line
    cell_start = ln['start'] + _indent_width(content, ln)
    cell_end = ln['bars'][-1][0] if trailing else ln['end']
    bounds = [cell_start] + [p for p, _r in singles] + [cell_end]
    cells = []
    for b_idx in range(len(bounds) - 1):
        a = bounds[b_idx] + (1 if b_idx > 0 else 0)   # skip the | itself
        cell = content[a:bounds[b_idx + 1]].strip()
        if len(cell) > CELL_MAX:
            return None
        cells.append(cell)
    return (ln, cells, trailing)


# --- diagnostics -----------------------------------------------------------
#
# A whole-buffer balance check over all three @-tiers, for editor problem
# panels (LSP publishDiagnostics, Vim quickfix).  The scan is the uniform
# one the indentation uses — every @-run is an opener when a name follows it
# and a close otherwise — extended with positions and names so problems can
# be reported where they are:
#
#   * a closing delimiter with no opening to match
#   * a named close whose name disagrees with its opening
#   * a close whose @-run length differs from its opening's (@name ... @@)
#   * an opening never closed (reported at the opening, at end of scan)
#   * a literal klammer never closed (@code without code@)
#   * an unclosed #[ removal block
#
# Content removed by ## is not scanned (it is not part of the document).

def diagnostics(s):
    """Scan S and return a list of problems, each a dict:
    {'start': int, 'end': int, 'message': str, 'severity': 'error'|'warning'}.
    Positions are character offsets into S (token start/end)."""
    probs = []
    stack = []                       # (name, run_len, tok_start, tok_end)
    n = len(s)
    i = 0
    while i < n:
        j = i
        while j < n and s[j] != '@' and s[j] != '#':
            j += 1
        if j >= n:
            break
        hit = j
        i = hit + 1
        if escaped_p(s, hit):
            continue
        nxt = s[hit + 1] if hit + 1 < n else None
        if s[hit] == '#':
            if nxt == '#':           # rest of file removed: stop scanning
                break
            elif nxt == '[':
                end = block_end(s, hit + 2)
                if end >= n and not s.endswith(']#'):
                    probs.append({'start': hit, 'end': hit + 2,
                                  'message': "#[ has no closing ]#",
                                  'severity': 'warning'})
                i = end
            elif nxt in ('+', '/', '-'):
                pass
            else:
                eol = s.find('\n', hit)
                i = n if eol == -1 else eol
            continue
        # an @-run
        run_end = at_run_end(s, hit)
        run_len = run_end - hit
        after = s[run_end] if run_end < n else None
        if name_char_p(after):
            k = _name_forward(s, run_end)
            name = s[run_end:k]
            i = k
            if run_len == 1 and name in LITERAL_KLAMMERS:
                idx = s.find(name + '@', k)
                if idx == -1:
                    probs.append({'start': hit, 'end': k,
                                  'message': ("literal klammer @%s has no "
                                              "closing %s@" % (name, name)),
                                  'severity': 'error'})
                    break            # everything after is verbatim
                i = idx + len(name) + 1
            elif run_len == 1 and k < n and s[k] == '-':
                pass                 # @name-arg : opens no span
            else:
                stack.append((name, run_len, hit, k))
        else:
            # a close; find the token (a preceding name run makes it named)
            ns = hit
            while ns > 0 and name_char_p(s[ns - 1]):
                ns -= 1
            named = ns < hit and (ns == 0 or s[ns - 1] != '@')
            cname = s[ns:hit] if named else None
            tok_start = ns if named else hit
            if not stack:
                probs.append({'start': tok_start, 'end': run_end,
                              'message': ("closing delimiter %s has no "
                                          "matching opening"
                                          % s[tok_start:run_end]),
                              'severity': 'error'})
            else:
                oname, orun, ostart, oend = stack.pop()
                if cname is not None and cname != oname:
                    probs.append({'start': tok_start, 'end': run_end,
                                  'message': ("closing %s does not match "
                                              "opening %s"
                                              % (s[tok_start:run_end],
                                                 s[ostart:oend])),
                                  'severity': 'error'})
                elif orun != run_len:
                    probs.append({'start': tok_start, 'end': run_end,
                                  'message': ("closing %s does not match "
                                              "opening %s (%d-@ close for "
                                              "a %d-@ opening)"
                                              % (s[tok_start:run_end],
                                                 s[ostart:oend],
                                                 run_len, orun)),
                                  'severity': 'error'})
            i = run_end
    for name, run_len, ostart, oend in stack:
        probs.append({'start': ostart, 'end': oend,
                      'message': "opening %s is never closed" % s[ostart:oend],
                      'severity': 'error'})
    probs.sort(key=lambda p: p['start'])
    return probs


# --- whole-buffer conveniences (used by the CLI, the LSP server, tests) ----

def apply_edits(s, edits):
    """Apply (start, end, new) EDITS (ascending, non-overlapping) to S."""
    out = []
    last = 0
    for a, b, new in edits:
        out.append(s[last:a])
        out.append(new)
        last = b
    out.append(s[last:])
    return ''.join(out)


def indent_text(s, first=None, last=None):
    """Reindent S (whole buffer, or 1-based inclusive line range FIRST..LAST).
    Return the new text."""
    bols = line_beginnings(s)
    if first is not None:
        lo = max(first, 1)
        hi = len(bols) if last is None else min(last, len(bols))
        bols = bols[lo - 1:hi]
    return apply_edits(s, reindent_lines(s, bols))


def align_text(s, pos):
    """Align the table klammer enclosing POS.  Return (new_text, message)."""
    span = enclosing_span(s, pos, ALIGN_KLAMMERS)
    if span is None:
        return (s, "not inside a table klammer (%s)"
                % ", ".join("@" + name for name in sorted(ALIGN_KLAMMERS)))
    _name, cs, ce = span
    edits, msg = compute_edits(s[cs:ce])
    shifted = [(cs + a, cs + b, new) for a, b, new in edits]
    return (apply_edits(s, shifted), msg)


def offset_of(s, line, col):
    """Character offset of 1-based LINE, 1-based character column COL."""
    bols = line_beginnings(s)
    line = max(1, min(line, len(bols)))
    bol = bols[line - 1]
    eol = s.find('\n', bol)
    if eol == -1:
        eol = len(s)
    return min(bol + max(col - 1, 0), eol)


def line_col(s, offset):
    """(1-based line, 1-based character column) of character OFFSET."""
    offset = max(0, min(offset, len(s)))
    line = s.count('\n', 0, offset) + 1
    bol = s.rfind('\n', 0, offset) + 1
    return (line, offset - bol + 1)


# --- CLI -------------------------------------------------------------------
#
# The shell-out interface for editors that are neither Python-hosted nor LSP
# clients (the Vim plugin).  Reads the buffer on stdin (UTF-8), writes the
# transformed buffer on stdout; status messages go to stderr.
#
#   klammertext_edit.py indent [FIRST[-LAST]]     reindent all / a line range
#   klammertext_edit.py align LINE COL            align the enclosing table
#   klammertext_edit.py match LINE COL            print "match L C" | "none MSG"
#                                                 (with "mismatch" when the
#                                                 pair disagrees); no buffer
#   klammertext_edit.py check                     print "L:C: message" lines;
#                                                 no buffer output
#
# LINE and COL are 1-based; COL counts characters.  Exit code: 0 (including
# "nothing to do"), 2 on a usage error.

def _cli(argv):
    def usage():
        sys.stderr.write(
            "usage: klammertext_edit.py indent [FIRST[-LAST]] |"
            " align LINE COL | match LINE COL | check\n")
        return 2

    if len(argv) < 1:
        return usage()
    mode = argv[0]
    s = sys.stdin.read()

    if mode == 'indent':
        first = last = None
        if len(argv) > 1:
            rng = argv[1]
            try:
                if '-' in rng:
                    a, b = rng.split('-', 1)
                    first, last = int(a), int(b)
                else:
                    first = last = int(rng)
            except ValueError:
                return usage()
        sys.stdout.write(indent_text(s, first, last))
        return 0

    if mode == 'align':
        if len(argv) != 3:
            return usage()
        try:
            pos = offset_of(s, int(argv[1]), int(argv[2]))
        except ValueError:
            return usage()
        out, msg = align_text(s, pos)
        sys.stdout.write(out)
        sys.stderr.write(msg + "\n")
        return 0

    if mode == 'match':
        if len(argv) != 3:
            return usage()
        try:
            pos = offset_of(s, int(argv[1]), int(argv[2]))
        except ValueError:
            return usage()
        m = match_at(s, pos)
        if m is None:
            print("none point is not on a klammer application delimiter (@)")
        elif m['match'] is None:
            print("none " + (m['message'] or "no matching delimiter"))
        else:
            line, col = line_col(s, m['match'])
            kind = "mismatch" if m['mismatch'] else "match"
            msg = (" " + m['message']) if m['mismatch'] and m['message'] else ""
            print("%s %d %d%s" % (kind, line, col, msg))
        return 0

    if mode == 'check':
        for p in diagnostics(s):
            line, col = line_col(s, p['start'])
            print("%d:%d: %s" % (line, col, p['message']))
        return 0

    return usage()


if __name__ == '__main__':
    sys.exit(_cli(sys.argv[1:]))