#!/usr/bin/env python3 """Markdown -> Standard Klammer Set converter (first implementation). SPECIFICATION: doc/markdown_to_klammertext.md. That document is authoritative and is written to be sufficient to rebuild this program; this program is its reference implementation, not the other way round. Its "The algorithm" section states the pass order, which is forced, and the traps that make it so. The conversion is to the SKS, not to Klammertext the language: the correspondence between a Markdown construct and a klammer is a property of the klammer set. That is why this lives under sks/ rather than in mac/. What cannot be converted exactly is recorded in the output as a #[MD : ... ]# marker -- lossy, gap, or judgment; with --chapter also section -- so a converted document carries its own worklist and "how far from done is this file" is a grep. A file with no markers is inside the convertible subset. The marker is the contract: a construct this program does not handle must leave one, never pass through as prose. It does not hold yet -- see "What the reference implementation does not do" in the spec for the constructs that still pass through unmarked. First run: the Rectify user guide (1,805 lines), 2026-08-07. It rendered to html and pdf without error or warning, with 99 markers -- 91 of them internal anchor links, the one gap that appears at volume. Second run: three German vocabulary lists (4,200 lines, 152 tables), 2026-09-30, which rendered without an error and with two silently wrong results -- an emphasis that wrapped and a table's empty header corner -- so "renders clean" is not "converted"; the fixes are in the spec's "What the second conversion taught". A review against a probe document (2026-10-11) found twelve more ways the output was silently wrong and eight constructs passed through with no marker; four were fixed first (a ragged nested list dropped items, a loose list became one @ol per item, the "@@" separator ran over @c literals, an escaped bar split a table cell), and the probe became the fixture sks/tst/md_to_sks_test.sh: the conversion, in both modes, must equal a golden file line for line and then render. The golden files pin only what is right; the remaining defects are listed in the spec and join the probe as they are fixed. Usage: python3 md_to_sks.py [--chapter] [sibling.md ...] The siblings are Markdown files converted alongside this one. Without --chapter each file is a document of its own, and a link to a sibling becomes a link to its html page (..//index.html, where ktext writes it); a link to any other .md file is a gap. With --chapter (Andy, 2026-10-12) the file is one CHAPTER of a book whose @document is written by hand and names the chapters through :files: no @document wrapper is emitted, no :title is lifted, and every heading maps one level deeper -- "#" is @s1, the chapter's own heading. A link to a sibling is then a link to another chapter of the same book, written "@link :section @" (:section finds a section by its heading; implemented 2026-10-12), the title being the sibling's first "#" heading read from the file. A Markdown link targets a FILE, so the title is a guess, and a "section" marker records it. The same mechanism converts a Markdown anchor, "#some-heading" or "other.md#some-heading": the slug is matched against the headings of the file it points into, and the link names that heading's text -- again marked, since the match is a guess. The motivating case was ~/projects/kokkoja/docs/, five topic files cross-linked as "[A host's Kokkoja](host.md)". """ import re, sys, os, datetime, argparse from collections import Counter SPECIALS = "^@#|*" # ^ first: the other quotings introduce ^ marks = Counter() SIBLINGS = {} # stem -> path of the .md files converted together CHAPTER = False # --chapter: a chapter of a hand-written book STEM = "" # the file being converted HEADINGS = {} # stem -> [(slug, level, title as Klammertext)], lazily def slug(text): """GitHub's anchor for a heading: lower case, punctuation other than hyphen and underscore removed, every whitespace a hyphen. A duplicate heading gets -1, -2 ... appended, which lookups retry without.""" return re.sub(r'\s', '-', re.sub(r'[^\w\s-]', '', text.lower()).strip()) def headings_of(stem): """The headings of a file converted alongside (or this one): the anchor each would have in Markdown, its level, and the title AS THE CONVERTER EMITS IT -- manual number stripped, inline Markdown converted -- which is the text a :section link must name. Headings inside fenced code are not headings.""" if stem in HEADINGS: return HEADINGS[stem] found = [] try: with open(SIBLINGS[stem], encoding="utf-8") as f: fenced = False for line in f: line = line.rstrip("\n") if FENCE.match(line): fenced = not fenced continue m = HEADING.match(line) if m and not fenced: text = m.group(2) title = MANUAL_NUMBER.sub("", text) if MANUAL_NUMBER.match(text) else text found.append((slug(text), len(m.group(1)), title)) except (OSError, KeyError): pass HEADINGS[stem] = found return found def find_heading(stem, anchor): """The (level, title) a Markdown anchor names in STEM's file, or None. An anchor made unique with -N is retried as its base.""" for s_, level, title in headings_of(stem): if s_ == anchor: return level, title m = re.match(r'^(.*)-\d+$', anchor) if m: for s_, level, title in headings_of(stem): if s_ == m.group(1): return level, title return None def sibling_title(stem, fallback): """A sibling chapter's title: its first "#" heading.""" for _, level, title in headings_of(stem): if level == 1: return title return fallback def mark(kind, text): """A #[MD ... ]# marker. Two things have to be neutralised in the text, both of which cost a session's debugging if they are not: * "]#" would close the block early and "#[" would open one that must then be balanced (the nesting hazard); * a klammer name written with its "@" is READ before the block is removed -- mark_literal_klammer_content() runs first in process_katoms() -- so "@code has no :language" makes the engine demand a "code@" close from inside removed text. Quoting the @ makes it a special katom, which the literal scan does not see. (TODO #39.) """ marks[kind] += 1 text = text.replace("]#", "] #").replace("#[", "# [") text = text.replace("@", "^@").replace("|", "^|") return f"#[MD {kind}: {text} ]#" def quote(s): """Quote every Klammertext special in writer text.""" for ch in SPECIALS: s = s.replace(ch, "^" + ch) return s # ---- inline --------------------------------------------------------------- CODE = re.compile(r'`([^`\n]+)`') LINK = re.compile(r'(? 40: widest = widths.index(max(widths)) spec = " ".join("fill" if c == widest else "fit" for c in range(len(widths))) opts += f" :column_width {spec}" out.append(mark("judgment", "column widths are not in the Markdown; the " f"widest column ({max(widths)} characters) was " "given fill so the table wraps instead of " "overflowing")) out.append(f"@table{opts} |") out.append(body) out.append("table@") out.append("") continue if BULLET.match(line) or ORDERED.match(line): block = [] while i < n: if (BULLET.match(src[i]) or ORDERED.match(src[i]) or (src[i].strip() and src[i].startswith((" ", "\t")))): block.append(src[i]); i += 1 elif not src[i].strip(): # A blank line ends the list UNLESS the next non-blank # line is another item of it: a loose list (items # separated by blank lines) is one list, and ending it # here made every item its own @ol, each numbered 1. # "Of it" is CommonMark's rule: an indented item (a # nested one) or one with the same marker; a different # marker starts a new list. An indented non-item after # the blank (a code block inside the item) still ends # the list, as it always did. j = i while j < n and not src[j].strip(): j += 1 if j < n and list_marker(src[j]) and ( src[j].startswith((" ", "\t")) or list_marker(src[j]) == list_marker(block[0])): i = j else: break else: break out.append(convert_list(block)) out.append("") continue if not line.strip(): out.append(""); i += 1 continue # A PARAGRAPH goes through inline() whole, its line breaks kept. # Converting line by line left a literal "*" on each line of an # emphasis that wrapped, and the leftovers then paired with EACH # OTHER, so the wrong words came out italic -- 33 paragraphs of the # vocabulary lists, visible only in a diff. para = [line]; i += 1 while i < n and src[i].strip() and not (HEADING.match(src[i]) or TABLEROW.match(src[i]) or FENCE.match(src[i]) or BULLET.match(src[i])): if ORDERED.match(src[i]): # A numbered line hard against the line above is a wrapped # line that happens to begin with a number ("1. and 3. # Sg."). CommonMark reads it as a list; the writer did not # mean one. Kept in the paragraph, and said. out.append(mark("judgment", "a line beginning with a number " "was kept in its paragraph, not read as a list")) para.append(src[i]); i += 1 out.append(inline("\n".join(para))) return out, title, stripped_numbers, dropped_toc def convert_list(block): """One list, possibly nested, as @ul/@ol with bar-separated items.""" def emit(items, ordered, depth): klammer = "@ol" if ordered else "@ul" parts = [] for text, children in items: body = inline(text) if children: body += " " + emit(children[0], children[1], depth + 1) parts.append(body) return klammer + " " + " | ".join(parts) + " @" def parse(lines): """The items at one level, each with its children. An item's children are the item lines that begin at or past its CONTENT column (where the text after the marker starts, as in CommonMark); an item line that begins before that is the next item at THIS level, however it is indented. The loop never stops on an indent: the caller has already bounded the lines, so there is nothing an indent could hand back to. The earlier rule took the first child's indent as the sublist's and broke off at anything shallower, so " - a1" followed by " - a2" DROPPED a2 -- and the document rendered without a word about it. """ items, i = [], 0 ordered = bool(ORDERED.match(lines[0])) if lines else False while i < len(lines): m = BULLET.match(lines[i]) or ORDERED.match(lines[i]) if not m: # a stray wrapped line: previous item's if items: text, children = items[-1] items[-1] = (text + " " + lines[i].strip(), children) i += 1 continue text = m.groups()[-1] content_col = m.start(m.lastindex) i += 1 child_lines = [] while i < len(lines): m2 = BULLET.match(lines[i]) or ORDERED.match(lines[i]) if m2 and len(m2.group(1)) >= content_col: child_lines.append(lines[i]); i += 1 elif m2: break elif child_lines: # a wrapped line of the last child child_lines.append(lines[i]); i += 1 else: # a wrapped line of this item text += " " + lines[i].strip(); i += 1 children = parse(child_lines) if child_lines else None items.append((text, children)) return items, ordered items, ordered = parse([l for l in block if l.strip()]) return emit(items, ordered, 0) def main(): global CHAPTER ap = argparse.ArgumentParser( description="Convert a Markdown file to the Standard Klammer Set.", epilog="Siblings are the .md files converted alongside; a link to one " "is converted as a link to its page (or, with --chapter, to " "its section), a link to any other .md file is a gap.") ap.add_argument("--chapter", action="store_true", help="the file is a chapter of a hand-written @document: " "no wrapper, '#' maps to @s1, sibling links are :section links") ap.add_argument("input", metavar="input.md") ap.add_argument("output", metavar="output.kt") ap.add_argument("siblings", metavar="sibling.md", nargs="*") args = ap.parse_args() global STEM CHAPTER = args.chapter src_path, dst_path = args.input, args.output STEM = os.path.splitext(os.path.basename(src_path))[0] SIBLINGS.update((os.path.splitext(os.path.basename(a))[0], a) for a in args.siblings) SIBLINGS[STEM] = src_path # its own headings resolve "#anchor" links body, title, stripped, dropped_toc = convert(src_path) today = datetime.date.today().isoformat() head = [f"#[MD source: {os.path.abspath(src_path)}", f" converted {today} ]#", ""] if stripped: head.append(mark("judgment", f"{stripped} headings carried a manual number " '("2.1 ..."); the numbers were removed because @s1/@s2 ' "number the headings themselves")) head.append("") if CHAPTER: tail = [""] else: head += ["@document", ":structure plain"] if title: head.append(f":title {title}") head += [":text", ""] tail = ["", "@", ""] text = "\n".join(head + body + tail) text = re.sub(r'\n{3,}', "\n\n", text) open(dst_path, "w", encoding="utf-8").write(text) print(f"{dst_path}: {len(text.splitlines())} lines") for kind in ("lossy", "gap", "judgment", "section"): if kind != "section" or CHAPTER: print(f" {kind:9} {marks[kind]}") main()