328 lines
14 KiB
Python
328 lines
14 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""Markdown -> Standard Klammer Set converter (first implementation).
|
||
|
|
|
||
|
|
SPECIFICATION: doc/markdown_to_klammertext.md. That document is authoritative
|
||
|
|
and is written to be sufficient to rebuild this program; this program is its
|
||
|
|
reference implementation, not the other way round. Its "The algorithm"
|
||
|
|
section states the pass order, which is forced, and the traps that make it so.
|
||
|
|
|
||
|
|
The conversion is to the SKS, not to Klammertext the language: the
|
||
|
|
correspondence between a Markdown construct and a klammer is a property of
|
||
|
|
the klammer set. That is why this lives under sks/ rather than in mac/.
|
||
|
|
|
||
|
|
What cannot be converted exactly is recorded in the output as a
|
||
|
|
#[MD <kind>: ... ]# marker -- lossy, gap, or judgment -- so a converted
|
||
|
|
document carries its own worklist and "how far from done is this file" is a
|
||
|
|
grep. A file with no markers is inside the convertible subset.
|
||
|
|
|
||
|
|
First run: the Rectify user guide (1,805 lines), 2026-08-07. It rendered to
|
||
|
|
html and pdf without error or warning, with 99 markers -- 91 of them internal
|
||
|
|
anchor links, the one gap that appears at volume.
|
||
|
|
|
||
|
|
Usage: python3 md_to_sks.py <input.md> <output.kt>
|
||
|
|
"""
|
||
|
|
import re, sys, os, datetime
|
||
|
|
from collections import Counter
|
||
|
|
|
||
|
|
SPECIALS = "^@#|*" # ^ first: the other quotings introduce ^
|
||
|
|
marks = Counter()
|
||
|
|
|
||
|
|
def mark(kind, text):
|
||
|
|
"""A #[MD ... ]# marker.
|
||
|
|
|
||
|
|
Two things have to be neutralised in the text, both of which cost a
|
||
|
|
session's debugging if they are not:
|
||
|
|
|
||
|
|
* "]#" would close the block early and "#[" would open one that must then
|
||
|
|
be balanced (the nesting hazard);
|
||
|
|
* a klammer name written with its "@" is READ before the block is
|
||
|
|
removed -- mark_literal_klammer_content() runs first in
|
||
|
|
process_katoms() -- so "@code has no :language" makes the engine demand
|
||
|
|
a "code@" close from inside removed text. Quoting the @ makes it a
|
||
|
|
special katom, which the literal scan does not see. (TODO #39.)
|
||
|
|
"""
|
||
|
|
marks[kind] += 1
|
||
|
|
text = text.replace("]#", "] #").replace("#[", "# [")
|
||
|
|
text = text.replace("@", "^@").replace("|", "^|")
|
||
|
|
return f"#[MD {kind}: {text} ]#"
|
||
|
|
|
||
|
|
def quote(s):
|
||
|
|
"""Quote every Klammertext special in writer text."""
|
||
|
|
for ch in SPECIALS:
|
||
|
|
s = s.replace(ch, "^" + ch)
|
||
|
|
return s
|
||
|
|
|
||
|
|
# ---- inline ---------------------------------------------------------------
|
||
|
|
|
||
|
|
CODE = re.compile(r'`([^`\n]+)`')
|
||
|
|
LINK = re.compile(r'(?<!\!)\[([^\]]*)\]\(([^)]+)\)')
|
||
|
|
BOLD = re.compile(r'\*\*([^*]+)\*\*|__([^_]+)__')
|
||
|
|
# Markdown's underscore emphasis does not open or close INSIDE a word, and
|
||
|
|
# technical prose is full of identifiers like *name*_rectified_1.*ext* where
|
||
|
|
# the underscores are part of the name. So the closing _ must not be
|
||
|
|
# followed by a word character -- without that, the converter invents
|
||
|
|
# emphasis in the middle of filenames.
|
||
|
|
ITAL = re.compile(r'(?<![\*\w])\*([^*\s][^*]*)\*(?!\*)|(?<![_\w])_([^_\s][^_]*)_(?![\w_])')
|
||
|
|
STRIKE = re.compile(r'~~([^~]+)~~')
|
||
|
|
|
||
|
|
def inline(text):
|
||
|
|
"""Convert one run of inline Markdown.
|
||
|
|
|
||
|
|
Order matters: code spans are protected first (their content must not be
|
||
|
|
read as emphasis), links and emphasis consume their markers next, and only
|
||
|
|
then are the remaining specials quoted -- quoting first would hide the
|
||
|
|
** and * that emphasis is recognised by.
|
||
|
|
"""
|
||
|
|
spans = []
|
||
|
|
def stash(m):
|
||
|
|
spans.append(m.group(1))
|
||
|
|
return f"\x00{len(spans)-1}\x00"
|
||
|
|
text = CODE.sub(stash, text)
|
||
|
|
|
||
|
|
# Markers are Klammertext, not writer text: they must survive the quoting
|
||
|
|
# pass below untouched, or the "#[" that makes them a removal block is
|
||
|
|
# itself quoted and the marker renders into the output instead of
|
||
|
|
# vanishing from it. Stash them like code spans.
|
||
|
|
verbatim = []
|
||
|
|
def keep(s):
|
||
|
|
verbatim.append(s)
|
||
|
|
return f"\x01{len(verbatim)-1}\x01"
|
||
|
|
|
||
|
|
# OPEN and CLOSE wrap the delimiters this function emits, so the
|
||
|
|
# whitespace fix below can find exactly its own delimiters. Matching
|
||
|
|
# them by regex on the finished text does not work: " @" is both the
|
||
|
|
# closing delimiter and the space before an opening "@i", and a rule
|
||
|
|
# keyed on it rewrites "@i" into "@#- i".
|
||
|
|
OPEN, CLOSE = "\x03", "\x02"
|
||
|
|
def opening(s): return keep(OPEN + s)
|
||
|
|
def closing(): return keep(" @" + CLOSE)
|
||
|
|
|
||
|
|
def link(m):
|
||
|
|
label, target = m.group(1), m.group(2)
|
||
|
|
if target.startswith("#"):
|
||
|
|
# An anchor into this document: no klammer gives an arbitrary
|
||
|
|
# heading an anchor, so keep the words and record the loss.
|
||
|
|
return emphasis(label) + " " + keep(mark("gap", f"link to {target}"))
|
||
|
|
return opening("@link " + quote(target) + " :text ") + emphasis(label) + closing()
|
||
|
|
|
||
|
|
def emphasis(t):
|
||
|
|
"""The marker-consuming transforms, sharing THIS call's stashes.
|
||
|
|
|
||
|
|
A link label or a struck-through run is itself inline Markdown and
|
||
|
|
must be converted -- but not by re-entering inline(), which would
|
||
|
|
start empty stashes while the text already holds this call's
|
||
|
|
placeholders. A code span inside a link label then restores against
|
||
|
|
the wrong list: IndexError, found by converting a second document.
|
||
|
|
"""
|
||
|
|
t = LINK.sub(link, t)
|
||
|
|
t = STRIKE.sub(lambda m: emphasis(m.group(1)) + " " +
|
||
|
|
keep(mark("gap", "strikethrough")), t)
|
||
|
|
t = BOLD.sub(lambda m: opening("@b ") + (m.group(1) or m.group(2)) + closing(), t)
|
||
|
|
return ITAL.sub(lambda m: opening("@i ") + (m.group(1) or m.group(2)) + closing(), t)
|
||
|
|
|
||
|
|
text = emphasis(text)
|
||
|
|
|
||
|
|
# Everything that is still writer text gets quoted; the stashed fragments
|
||
|
|
# are Klammertext already.
|
||
|
|
parts = re.split(r'(\x00\d+\x00|\x01\d+\x01)', text)
|
||
|
|
text = "".join(p if re.match(r'^[\x00\x01]', p) else quote(p) for p in parts)
|
||
|
|
|
||
|
|
text = re.sub(r'\x01(\d+)\x01', lambda m: verbatim[int(m.group(1))], text)
|
||
|
|
text = re.sub(r'\x00(\d+)\x00',
|
||
|
|
lambda m: OPEN + "@c " + quote(spans[int(m.group(1))]) + " @" + CLOSE, text)
|
||
|
|
# Klammertext delimiters need whitespace around them, but Markdown
|
||
|
|
# emphasis abuts its neighbours: "un**bold**ed" and "*name*_rectified"
|
||
|
|
# both put a word character hard against a delimiter, which the
|
||
|
|
# katomizer then reads as an unparsable word. A space makes it parse,
|
||
|
|
# and "#-" removes that space again from the OUTPUT, so intra-word
|
||
|
|
# emphasis converts exactly rather than approximately.
|
||
|
|
# Any non-space neighbour, not just a word character: "*name*_x" and
|
||
|
|
# "_1.*ext*" put an underscore or a period against the delimiter.
|
||
|
|
text = re.sub(r'(?<=\S)' + OPEN, ' #-', text).replace(OPEN, "")
|
||
|
|
text = re.sub(CLOSE + r'(?=[^\s])', '#- ', text).replace(CLOSE, "")
|
||
|
|
# A closing "@" abutting the next klammer's opening "@" spells "@@",
|
||
|
|
# which is a DEFINITION delimiter -- the engine then reports a span that
|
||
|
|
# ends without a beginning, pointing at text the writer never wrote.
|
||
|
|
# Adjacent klammers are ordinary in converted prose (*a*_b_ produces
|
||
|
|
# two), so separate them.
|
||
|
|
return re.sub(r'@(?=@)', '@ ', text)
|
||
|
|
|
||
|
|
# ---- blocks ---------------------------------------------------------------
|
||
|
|
|
||
|
|
HEADING = re.compile(r'^(#{1,6})\s+(.*?)\s*$')
|
||
|
|
MANUAL_NUMBER = re.compile(r'^\d+(\.\d+)*\.?\s+')
|
||
|
|
FENCE = re.compile(r'^\s*```(\w*)\s*$')
|
||
|
|
BULLET = re.compile(r'^(\s*)[-*+]\s+(.*)$')
|
||
|
|
ORDERED = re.compile(r'^(\s*)(\d+)[.)]\s+(.*)$')
|
||
|
|
TABLEROW = re.compile(r'^\s*\|(.*)\|\s*$')
|
||
|
|
ALIGNROW = re.compile(r'^\s*\|[\s:|-]+\|\s*$')
|
||
|
|
|
||
|
|
def cells(line):
|
||
|
|
return [c.strip() for c in TABLEROW.match(line).group(1).split("|")]
|
||
|
|
|
||
|
|
def convert(path):
|
||
|
|
src = open(path, encoding="utf-8").read().split("\n")
|
||
|
|
out, i, n = [], 0, len(src)
|
||
|
|
title = None
|
||
|
|
stripped_numbers = 0
|
||
|
|
dropped_toc = False
|
||
|
|
|
||
|
|
while i < n:
|
||
|
|
line = src[i]
|
||
|
|
|
||
|
|
m = FENCE.match(line)
|
||
|
|
if m: # fenced code
|
||
|
|
lang, body, i = m.group(1), [], i + 1
|
||
|
|
while i < n and not FENCE.match(src[i]):
|
||
|
|
body.append(src[i]); i += 1
|
||
|
|
i += 1
|
||
|
|
if lang:
|
||
|
|
out.append(mark("lossy", f'fenced language "{lang}" dropped '
|
||
|
|
"-- @code has no :language"))
|
||
|
|
out.append("@code |")
|
||
|
|
out.extend(body) # literal: nothing to quote
|
||
|
|
out.append("code@")
|
||
|
|
out.append("")
|
||
|
|
continue
|
||
|
|
|
||
|
|
m = HEADING.match(line)
|
||
|
|
if m:
|
||
|
|
level, text = len(m.group(1)), m.group(2)
|
||
|
|
if level == 1 and title is None:
|
||
|
|
title = inline(text); i += 1; continue
|
||
|
|
if MANUAL_NUMBER.match(text):
|
||
|
|
text = MANUAL_NUMBER.sub("", text); stripped_numbers += 1
|
||
|
|
if re.match(r'^contents$', text, re.I):
|
||
|
|
# Klammertext generates a table of contents; a hand-written
|
||
|
|
# one would duplicate it. Skip to the next heading.
|
||
|
|
j = i + 1
|
||
|
|
while j < n and not HEADING.match(src[j]):
|
||
|
|
j += 1
|
||
|
|
out.append(mark("judgment",
|
||
|
|
"a hand-written Contents section was dropped; "
|
||
|
|
":structure article generates one"))
|
||
|
|
out.append("")
|
||
|
|
dropped_toc = True
|
||
|
|
i = j
|
||
|
|
continue
|
||
|
|
out.append(f"@s{level-1} {inline(text)} @")
|
||
|
|
out.append("")
|
||
|
|
i += 1
|
||
|
|
continue
|
||
|
|
|
||
|
|
if TABLEROW.match(line): # table
|
||
|
|
rows, align = [], None
|
||
|
|
while i < n and TABLEROW.match(src[i]):
|
||
|
|
if ALIGNROW.match(src[i]):
|
||
|
|
align = [("r" if c.endswith(":") and c.startswith(":") is False
|
||
|
|
else "c" if c.startswith(":") and c.endswith(":")
|
||
|
|
else "l") for c in cells(src[i])]
|
||
|
|
else:
|
||
|
|
rows.append([inline(c) for c in cells(src[i])])
|
||
|
|
i += 1
|
||
|
|
body = " ||\n".join(" | ".join(r) for r in rows)
|
||
|
|
opts = f" :cell_hpos {' '.join(align)}" if align and set(align) != {"l"} else ""
|
||
|
|
# A Markdown table says nothing about column widths, and
|
||
|
|
# Klammertext's default "fit" never wraps -- a prose column then
|
||
|
|
# runs off the page (the pdf target warns). Give the widest
|
||
|
|
# column "fill", which takes the remaining width but no more than
|
||
|
|
# its own widest line.
|
||
|
|
widths = [max((len(r[c]) for r in rows if c < len(r)), default=0)
|
||
|
|
for c in range(max(len(r) for r in rows))]
|
||
|
|
if max(widths) > 40:
|
||
|
|
widest = widths.index(max(widths))
|
||
|
|
spec = " ".join("fill" if c == widest else "fit"
|
||
|
|
for c in range(len(widths)))
|
||
|
|
opts += f" :column_width {spec}"
|
||
|
|
out.append(mark("judgment",
|
||
|
|
"column widths are not in the Markdown; the "
|
||
|
|
f"widest column ({max(widths)} characters) was "
|
||
|
|
"given fill so the table wraps instead of "
|
||
|
|
"overflowing"))
|
||
|
|
out.append(f"@table{opts} |")
|
||
|
|
out.append(body)
|
||
|
|
out.append("table@")
|
||
|
|
out.append("")
|
||
|
|
continue
|
||
|
|
|
||
|
|
if BULLET.match(line) or ORDERED.match(line):
|
||
|
|
block, base = [], None
|
||
|
|
while i < n and (BULLET.match(src[i]) or ORDERED.match(src[i]) or
|
||
|
|
(src[i].strip() and src[i].startswith((" ", "\t")))):
|
||
|
|
block.append(src[i]); i += 1
|
||
|
|
out.append(convert_list(block))
|
||
|
|
out.append("")
|
||
|
|
continue
|
||
|
|
|
||
|
|
out.append(inline(line) if line.strip() else "")
|
||
|
|
i += 1
|
||
|
|
|
||
|
|
return out, title, stripped_numbers, dropped_toc
|
||
|
|
|
||
|
|
def convert_list(block):
|
||
|
|
"""One list, possibly nested, as @ul/@ol with bar-separated items."""
|
||
|
|
def emit(items, ordered, depth):
|
||
|
|
klammer = "@ol" if ordered else "@ul"
|
||
|
|
parts = []
|
||
|
|
for text, children in items:
|
||
|
|
body = inline(text)
|
||
|
|
if children:
|
||
|
|
body += " " + emit(children[0], children[1], depth + 1)
|
||
|
|
parts.append(body)
|
||
|
|
return klammer + " " + " | ".join(parts) + " @"
|
||
|
|
|
||
|
|
def parse(lines, indent):
|
||
|
|
items, i = [], 0
|
||
|
|
ordered = bool(ORDERED.match(lines[0])) if lines else False
|
||
|
|
while i < len(lines):
|
||
|
|
m = BULLET.match(lines[i]) or ORDERED.match(lines[i])
|
||
|
|
if not m or len(m.group(1)) < indent:
|
||
|
|
break
|
||
|
|
text = m.groups()[-1]
|
||
|
|
i += 1
|
||
|
|
child_lines = []
|
||
|
|
while i < len(lines):
|
||
|
|
m2 = BULLET.match(lines[i]) or ORDERED.match(lines[i])
|
||
|
|
if m2 and len(m2.group(1)) > indent:
|
||
|
|
child_lines.append(lines[i]); i += 1
|
||
|
|
elif not m2 and lines[i].strip():
|
||
|
|
text += " " + lines[i].strip(); i += 1
|
||
|
|
else:
|
||
|
|
break
|
||
|
|
children = None
|
||
|
|
if child_lines:
|
||
|
|
sub, sub_ordered = parse(child_lines, len(BULLET.match(child_lines[0]).group(1))
|
||
|
|
if BULLET.match(child_lines[0])
|
||
|
|
else len(ORDERED.match(child_lines[0]).group(1)))
|
||
|
|
children = (sub, sub_ordered)
|
||
|
|
items.append((text, children))
|
||
|
|
return items, ordered
|
||
|
|
|
||
|
|
items, ordered = parse([l for l in block if l.strip()], 0)
|
||
|
|
return emit(items, ordered, 0)
|
||
|
|
|
||
|
|
def main():
|
||
|
|
src_path, dst_path = sys.argv[1], sys.argv[2]
|
||
|
|
body, title, stripped, dropped_toc = convert(src_path)
|
||
|
|
today = datetime.date.today().isoformat()
|
||
|
|
head = [f"#[MD source: {os.path.abspath(src_path)}",
|
||
|
|
f" converted {today} ]#", ""]
|
||
|
|
if stripped:
|
||
|
|
head.append(mark("judgment",
|
||
|
|
f"{stripped} headings carried a manual number "
|
||
|
|
'("2.1 ..."); the numbers were removed because @s1/@s2 '
|
||
|
|
"number the headings themselves"))
|
||
|
|
head.append("")
|
||
|
|
head += ["@document", ":structure plain"]
|
||
|
|
if title:
|
||
|
|
head.append(f":title {title}")
|
||
|
|
head += [":text", ""]
|
||
|
|
text = "\n".join(head + body + ["", "@", ""])
|
||
|
|
text = re.sub(r'\n{3,}', "\n\n", text)
|
||
|
|
open(dst_path, "w", encoding="utf-8").write(text)
|
||
|
|
print(f"{dst_path}: {len(text.splitlines())} lines")
|
||
|
|
for kind in ("source", "lossy", "gap", "judgment"):
|
||
|
|
print(f" {kind:9} {marks[kind]}")
|
||
|
|
|
||
|
|
main()
|