diff --git a/README.md b/README.md index 335c00a..6c1d7bd 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ are regenerated on each release — patches cannot be merged directly. Report problems (or send patches) to the author; accepted changes are applied to the development tree and appear in a following snapshot. -This snapshot was assembled from development commit `d8ea0e973914`. +This snapshot was assembled from development commit `12929fdff53b`. ## License diff --git a/doc/install/linux_source_install.md b/doc/install/linux_source_install.md index 46c5ca4..2aaaad4 100644 --- a/doc/install/linux_source_install.md +++ b/doc/install/linux_source_install.md @@ -225,6 +225,15 @@ python3 $KLAMMERTEXT_HOME/sks/tns/md_to_pdf.py --setup sudo apt install chromium-browser ``` +A browser installed as a **flatpak** works too and is found without being +told: that is what the Pop!_Shop installs on Pop!_OS, and a flatpak puts +nothing on `PATH`, so the command looks for the application by name in the +flatpak export directories as well. The document is served to the browser +over the loopback interface rather than handed to it as a file path, so a +sandboxed browser needs no filesystem permission of any kind — including +none to reach the fonts, which is what silently cost a flatpak render its +typography before this was so. + Then: ```bash diff --git a/sks/tns/markdown.css b/sks/tns/markdown.css index 861ae81..2cb2853 100644 --- a/sks/tns/markdown.css +++ b/sks/tns/markdown.css @@ -60,6 +60,17 @@ h3, h4, h5, h6 { margin-right: var(--secnum-gap); } +/* A table-of-contents entry carries no bullet. Each entry is a section + title, and a marker in front of it reads as a list of things rather than + as a contents. md_to_pdf.py puts this class on a list item whose first + content is a link to a fragment, which is what a Markdown contents is, so + an ordinary bulleted list keeps its bullets. The entries stay on the + list's indent; add "padding-left: 0" to the surrounding ul to bring a + contents out to the margin instead. */ +li.toc { + list-style: none; +} + p, ol, ul { margin-top: 0.5rem; margin-bottom: 0.5rem; diff --git a/sks/tns/md_to_pdf.py b/sks/tns/md_to_pdf.py index 98bb74a..f4c4a10 100644 --- a/sks/tns/md_to_pdf.py +++ b/sks/tns/md_to_pdf.py @@ -34,8 +34,10 @@ Usage: """ import argparse import base64 +import http.server import re import json +import mimetypes import os import shutil import socket @@ -43,6 +45,7 @@ import struct import subprocess import sys import tempfile +import threading import time import urllib.request from pathlib import Path @@ -55,11 +58,41 @@ VENV = Path.home() / ".venvs" / "klammertext-tns" # Chromium will do: the protocol is the same. The .app paths are for macOS, # where nothing lands on PATH -- without them this finds no browser on a Mac # that has one installed. +# +# Flatpak-installed browsers come LAST, after every native one. Not because +# they work less well -- the protocol is identical -- but because a flatpak +# runs in a sandbox whose filesystem grants vary per application, and a native +# browser has no such variable. Preferring native keeps the simplest case the +# common one; the flatpaks are here so that a machine which has ONLY those +# still works, which is the ordinary state of a Pop!_OS box (the Pop!_Shop +# installs flatpaks) and so of a System76 user. +FLATPAK_BROWSERS = ["com.brave.Browser", "com.google.Chrome", + "org.chromium.Chromium"] + + +def flatpak_paths(): + """Where a flatpak's runnable wrapper lives, user installs before system. + + A flatpak puts NOTHING on PATH and nothing in /opt, so the names above + find it only through these export directories. What is exported is an + ordinary executable that passes its arguments through to `flatpak run`, + so it needs no special handling anywhere else in this program. + + Note the name: the wrapper is called `com.brave.Browser`, not + `brave-browser`. Putting the export directory on PATH therefore does NOT + make the earlier entries in BROWSERS resolve -- the application id has to + be looked for by name, which is what this does. + """ + roots = [Path.home() / ".local/share/flatpak/exports/bin", + Path("/var/lib/flatpak/exports/bin")] + return [str(root / app) for root in roots for app in FLATPAK_BROWSERS] + + BROWSERS = ["/opt/brave.com/brave/brave", "brave-browser", "google-chrome", "chromium", "chromium-browser", "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser", "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", - "/Applications/Chromium.app/Contents/MacOS/Chromium"] + "/Applications/Chromium.app/Contents/MacOS/Chromium"] + flatpak_paths() # --- the renderer ---------------------------------------------------------- @@ -165,6 +198,7 @@ def render_markdown(text): # see mark_section_numbers(). HEADING_NUMBER = re.compile(r"(]*>)\s*(\d+(?:\.\d+)*\.?)\s+") TOC_NUMBER = re.compile(r'(
  • \s*]*>)\s*(\d+(?:\.\d+)*\.?)\s+') +TOC_ITEM = re.compile(r'(
  • )(\s*(?:

    \s*)? a[href^="#"]) would also catch a paragraph that merely ends in a + cross-reference. This mirrors how .secnum is done, for the same reason. + + The optional

    is markdown-it's loose-list rendering: a contents with + blank lines between its entries wraps each in a paragraph. + """ + return TOC_ITEM.sub(r'

  • \2', html) + + def font_store_dirs(): """The Klammertext font store's search order: the KLAMMERTEXT_FONTS directories, then the distribution's own fnt/. Same order the engine @@ -213,20 +267,27 @@ def font_store_dirs(): return dirs -def font_face_css(name): +def font_face_css(name, publish): """The @font-face rules for one font in the store, with absolute URLs. A font in the store is a / directory of .ttf files beside a .css declaring its variants -- already the browser's own format, which is why this needs no conversion, only a path fix: the store writes url('eb-garamond/Regular.ttf') relative to itself, and the generated HTML - carries a pointing at the Markdown file's directory, so a relative - URL would resolve somewhere else entirely and the font would silently - fall back. + carries a that points elsewhere, so a relative URL would resolve + somewhere else entirely and the font would silently fall back. + + *publish* turns a font file into the URL the page should ask for. It is a + parameter rather than a fixed `as_uri()` because the two consumers need + different answers: the page being printed is SERVED (so a sandboxed + browser can fetch it without any filesystem grant), while a --keep-html + file is meant to be opened later by hand, when no server is running, and + needs a file:// URL to be worth keeping. Returns (css, family). A font the browser cannot find is not an error it reports -- it just uses something else -- so an unknown name must fail - here instead. + here instead. That guard covers an unknown font NAME; an unreadable font + FILE is caught after loading instead, by check_fonts(). """ import re for d in font_store_dirs(): @@ -235,7 +296,7 @@ def font_face_css(name): continue css = css_path.read_text(encoding="utf-8") css = re.sub(r"url\(\s*['\"]?([^'\")]+)['\"]?\s*\)", - lambda m: f"url('{(d / m.group(1)).resolve().as_uri()}')", css) + lambda m: f"url('{publish((d / m.group(1)).resolve())}')", css) family = re.search(r"font-family:\s*['\"]([^'\"]+)['\"]", css) return css, (family.group(1) if family else name) available = sorted({p.stem for d in font_store_dirs() if d.is_dir() @@ -327,7 +388,7 @@ def scale_factors(names, match="average"): return {role: serif / m for role, m in metrics.items() if m > 0} -def font_css(serif, sans, mono, match="average"): +def font_css(serif, sans, mono, match="average", publish=None): """@font-face blocks plus the SKS's own custom-property names. --serif, --sans and --mono are what sks/font/css/font.css calls them, so a @@ -335,18 +396,24 @@ def font_css(serif, sans, mono, match="average"): names are QUOTED: an unquoted digit-initial name ("Source Sans 3") is invalid CSS, and a font-family using it via var() computes to inherit -- the font is lost with no error anywhere. + + Returns (css, families): the family names are what check_fonts() later + asserts the browser actually loaded. """ - faces, variables = [], [] + if publish is None: + publish = lambda p: p.as_uri() + faces, variables, families = [], [], [] for role, name, fallback in (("serif", serif, "serif"), ("sans", sans, "sans-serif"), ("mono", mono, "monospace")): if not name: continue - css, family = font_face_css(name) + css, family = font_face_css(name, publish) faces.append(css) + families.append(family) variables.append(f' --{role}: "{family}", {fallback};') if not variables: - return "" + return "", [] # Scale factors, computed rather than guessed. Both spellings are # emitted: the short ones this script has always used, and the ones # sks/font/css/font.css defines, so a stylesheet written for either @@ -357,7 +424,8 @@ def font_css(serif, sans, mono, match="average"): variables.append(f" --{role}-scale: {factor:.4f};") if role in alias: variables.append(f" --{alias[role]}-scale: {factor:.4f};") - return ("\n".join(faces) + "\n:root {\n" + "\n".join(variables) + "\n}\n") + return ("\n".join(faces) + "\n:root {\n" + "\n".join(variables) + "\n}\n", + families) # A language's line-continuation character, where continuing a line is @@ -464,10 +532,15 @@ def wrap_fenced_code(text, width): return "\n".join(out), wrapped, skipped -def wrap_and_report(text, wrap): - """Wrap the fenced blocks and say what happened, or did not.""" +def wrap_and_report(text, wrap, report=True): + """Wrap the fenced blocks and say what happened, or did not. + + *report* is off when the same wrap is applied a second time to build the + --keep-html copy: the wrapping is identical, so saying so twice would only + suggest it had happened twice. + """ text, wrapped, skipped = wrap_fenced_code(text, wrap) - if wrapped or skipped: + if report and (wrapped or skipped): note = f"wrapped {wrapped} code lines at {wrap} columns" if skipped: note += (f"; {skipped} left long -- no continuation character " @@ -476,7 +549,7 @@ def wrap_and_report(text, wrap): return text -def build_html(md_path, css_paths, fonts_css="", wrap=0): +def build_html(md_path, css_paths, fonts_css="", wrap=0, base=None, report=True): """One self-contained HTML document. The stylesheets are INLINED rather than linked: a headless browser fetches @@ -484,16 +557,19 @@ def build_html(md_path, css_paths, fonts_css="", wrap=0): arrives -- an unstyled PDF that looks like a CSS bug. Inline text cannot lose that race. - A element points at the Markdown file's own directory so that - relative image paths resolve, which lets the generated HTML live in a - temporary directory instead of beside the source. + A element makes the document's relative image paths resolve. It is + a parameter because there are two answers: while printing, the base is the + local server's root, which serves the Markdown file's directory; for a + --keep-html file it is that directory's own file:// URL, so the kept file + still works when nothing is serving it. """ text = md_path.read_text(encoding="utf-8") if wrap: - text = wrap_and_report(text, wrap) - body = mark_section_numbers(render_markdown(text)) + text = wrap_and_report(text, wrap, report) + body = mark_contents_items(mark_section_numbers(render_markdown(text))) css = fonts_css + "\n".join(Path(p).read_text(encoding="utf-8") for p in css_paths) - base = md_path.resolve().parent.as_uri() + "/" + if base is None: + base = md_path.resolve().parent.as_uri() + "/" return (f'\n\n' f'\n' f'{md_path.stem}\n' @@ -675,6 +751,118 @@ class WebSocket: raise RuntimeError(f"timed out waiting for {event}") +# --- serving the document -------------------------------------------------- + +# The document is SERVED to the browser over the loopback interface rather +# than handed to it as a file:// path. The reason is sandboxing, and it is +# not hypothetical: measured on Kukka (Pop!_OS, browsers from the Pop!_Shop, +# 2026-08-09) the Brave flatpak grants itself /tmp while the Chrome flatpak +# does not, so the same file:// scheme works with one browser and fails with +# the other ON ONE MACHINE. Neither could read the font store under +# ~/projects at all, and a browser does not report a font it cannot fetch -- +# it silently substitutes, so the PDF came out in Liberation Serif and the +# command reported success. +# +# Every sandbox shares the network namespace, which is already relied on: the +# DevTools port is reached at 127.0.0.1. So HTTP needs no grant from anyone, +# from any sandbox technology, present or future -- and it delivers the +# stylesheet, the fonts AND the document's own images by the same route. It +# also retires the -points-at-the-source-directory arrangement, since +# the server root IS that directory. +# +# Bound to 127.0.0.1 only, on an ephemeral port, for the seconds the render +# takes. + +DOCUMENT_URL = "/__md_to_pdf__.html" + +# Stated here rather than trusted to the platform's table: what mimetypes +# knows about fonts differs by Python version and by /etc/mime.types, and a +# font served as application/octet-stream is at the mercy of the browser's +# sniffing. +for _suffix, _type in ((".ttf", "font/ttf"), (".otf", "font/otf"), + (".woff", "font/woff"), (".woff2", "font/woff2")): + mimetypes.add_type(_type, _suffix) + + +class Assets: + """Files the page may fetch, published under stable, opaque URL paths. + + A font lives outside the served directory (the font store is wherever + KLAMMERTEXT_FONTS or the distribution puts it), so it cannot be reached by + a relative URL. Rather than serve those directories wholesale, each file + is published individually and nothing else is reachable. + """ + + def __init__(self): + self.by_url = {} + + def publish(self, path): + path = Path(path).resolve() + for url, known in self.by_url.items(): + if known == path: + return url + url = f"/__asset__/{len(self.by_url)}/{path.name}" + self.by_url[url] = path + return url + + +def start_server(doc_root, assets, html): + """Serve *doc_root*, the published assets, and the document itself. + + *html* is a one-element list, not a string: with --wrap-code auto the + document is rebuilt after being measured, and the server must then hand + out the new text. Returns (server, port); the caller shuts it down. + """ + class Handler(http.server.SimpleHTTPRequestHandler): + def __init__(self, *a, **kw): + super().__init__(*a, directory=str(doc_root), **kw) + + def log_message(self, *a): + pass # a render is not a web server log + + def _send(self, body, content_type): + self.send_response(200) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + if self.command != "HEAD": + self.wfile.write(body) + + def _route(self): + path = self.path.split("?", 1)[0] + if path == DOCUMENT_URL: + self._send(html[0].encode("utf-8"), "text/html; charset=utf-8") + return True + asset = assets.by_url.get(path) + if asset: + # A published file that cannot be read is a 404, not a + # traceback: the font store can name a file that is not there, + # and the useful report is check_fonts's ("the font X failed + # to load"), not this thread's stack. + try: + body = asset.read_bytes() + except OSError: + self.send_error(404) + return True + kind = mimetypes.guess_type(asset.name)[0] or "application/octet-stream" + self._send(body, kind) + return True + return False + + def do_GET(self): + if not self._route(): + super().do_GET() + + def do_HEAD(self): + if not self._route(): + super().do_HEAD() + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + server.daemon_threads = True + threading.Thread(target=server.serve_forever, daemon=True).start() + return server, server.server_address[1] + + # --- the browser ----------------------------------------------------------- def find_browser(explicit): @@ -765,12 +953,59 @@ def page_socket(port): return pages[0]["webSocketDebuggerUrl"] -def print_to_pdf(port, url, pdf_path, paper, margin, page_numbers=False): +def check_fonts(ws, families): + """Fail if a font the document asked for did not actually load. + + This is the guard the rest of the program could not provide. An unknown + font NAME is caught in font_face_css, when the store is searched. But a + font whose file the browser cannot fetch -- wrong path, no permission, a + sandbox that cannot see the store -- is not an error the browser reports: + the CSS is valid, the family is simply unavailable, and it renders in + something else. The output looks finished and is wrong, which is the + worst failure a document tool can have. It cost a full diagnosis to + notice on Kukka, and only because pdffonts was run on the result. + + Asked of the page rather than inferred: document.fonts holds one FontFace + per @font-face rule, each with a status. A face the browser tried and + failed to fetch reports "error". A face it never needed reports + "unloaded" -- an italic in a document with no italics -- and that is not a + failure, so only "error" counts. document.fonts.ready settles the + in-flight loads first, or the answer would be whatever had arrived. + """ + if not families: + return + status = ws.call("Runtime.evaluate", returnByValue=True, awaitPromise=True, + expression=""" + (async () => { + try { await document.fonts.ready; } catch (e) {} + const worst = {}; + for (const face of document.fonts) { + const name = face.family.replace(/^['"]|['"]$/g, ''); + // "error" is sticky: one failed variant condemns the family. + if (worst[name] !== 'error') worst[name] = face.status; + } + return worst; + })()""")["result"]["value"] + bad = [f for f in families if status.get(f) == "error"] + missing = [f for f in families if f not in status] + if bad or missing: + for family in bad: + print(f"error: the font {family!r} failed to load; the document " + f"would have been rendered in a substitute.", file=sys.stderr) + for family in missing: + print(f"error: no @font-face for {family!r} reached the page.", + file=sys.stderr) + sys.exit("refusing to write a PDF in the wrong fonts.") + + +def print_to_pdf(port, url, pdf_path, paper, margin, page_numbers=False, + families=()): """Drive one page through load and print.""" ws = WebSocket(page_socket(port)) ws.call("Page.enable") ws.call("Page.navigate", url=url) ws.wait_for("Page.loadEventFired") + check_fonts(ws, list(families)) width, height = (8.27, 11.69) if paper == "A4" else (8.5, 11.0) # Warn if the document is wider than the page. This is the failure that # cost the most to find: Chromium does not clip or paginate overflow when @@ -885,17 +1120,27 @@ def main(): if not Path(c).exists(): sys.exit(f"no such stylesheet: {c}") - fonts_css = font_css(args.serif, args.sans, args.mono, args.match) + assets = Assets() + fonts_css, families = font_css(args.serif, args.sans, args.mono, + args.match, assets.publish) auto_wrap = str(args.wrap_code).lower() == "auto" if not auto_wrap and not str(args.wrap_code).isdigit(): ap.error(f"--wrap-code takes a column count or \"auto\", " f"not {args.wrap_code!r}") wrap = 0 if auto_wrap else int(args.wrap_code) - html = build_html(md_path, css_paths, fonts_css, wrap) + + # The handler reads this list, so replacing element 0 republishes the + # document without restarting anything. + document = [""] + doc_root = md_path.resolve().parent + server, http_port = start_server(doc_root, assets, document) + base = f"http://127.0.0.1:{http_port}/" + url = base.rstrip("/") + DOCUMENT_URL + columns = 0 work = Path(tempfile.mkdtemp(prefix="md_to_pdf.")) try: - html_path = work / (md_path.stem + ".html") - html_path.write_text(html, encoding="utf-8") + html = build_html(md_path, css_paths, fonts_css, wrap, base) + document[0] = html browser = find_browser(args.browser) port = free_port() proc = start_browser(browser, work / "profile", port) @@ -905,13 +1150,12 @@ def main(): # Two loads of the same page cost about a second and remove the # only number in this pipeline that had to be guessed. if auto_wrap: - columns = measure_code_columns(port, html_path.as_uri(), - args.paper, args.margin) + columns = measure_code_columns(port, url, args.paper, args.margin) if columns: - html = build_html(md_path, css_paths, fonts_css, columns) - html_path.write_text(html, encoding="utf-8") - print_to_pdf(port, html_path.as_uri(), pdf_path, args.paper, - args.margin, args.page_numbers) + html = build_html(md_path, css_paths, fonts_css, columns, base) + document[0] = html + print_to_pdf(port, url, pdf_path, args.paper, + args.margin, args.page_numbers, families) finally: proc.terminate() try: @@ -919,13 +1163,22 @@ def main(): except subprocess.TimeoutExpired: proc.kill() if args.keep_html: + # Rebuilt for standing on its own: a kept file outlives the server, + # so its base and its fonts must be file:// URLs, not dead links to + # a port that closed when this program exited. + standalone_css, _ = font_css(args.serif, args.sans, args.mono, + args.match) kept = pdf_path.with_suffix(".html") - kept.write_text(html, encoding="utf-8") + kept.write_text(build_html(md_path, css_paths, standalone_css, + columns or wrap, report=False), + encoding="utf-8") print(f"{kept}") print(f"{pdf_path} ({pdf_path.stat().st_size} bytes, " f"{Path(browser).name}, {len(css_paths)} stylesheet" f"{'s' if len(css_paths) != 1 else ''})") finally: + server.shutdown() + server.server_close() shutil.rmtree(work, ignore_errors=True)