commit 5f21d9099c22684f6d30d1afcf6c2f95e3938d52 Author: Andy Kopra Date: Sat Jun 27 16:34:28 2026 +0200 Initial commit: Chatter — assistive-writing app for reMarkable Paper Pro Move Direct-framebuffer ink pipeline (stock-quality strokes), finger-wipe erase, growable scrolling canvas with color-ghost cleanup, bidirectional toggle with a persistent 4-finger return launcher, instant button feedback. Includes prebuilt aarch64 binaries (dist/), build/deploy/install scripts, a user guide, and a complete technical reference. Co-Authored-By: Claude Opus 4.8 (1M context) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..03dd0c0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,19 @@ +# Build output +/build/ +build*/ + +# Secrets — device credentials & cloud tokens must never be committed +doc/tablet_access.md +*.token + +# Editor/OS cruft +*~ +.DS_Store + +# Local Claude Code data +.claude/ + +# Compiled tool binaries (built by build.sh / manually) +/tools/chatter-launcher +/tools/grabtest +/tools/*.so diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..984573f --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,42 @@ +cmake_minimum_required(VERSION 3.16) +project(chatter VERSION 0.1 LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_AUTOMOC ON) + +# Per reMarkable's qt_epaper guide: pure Qt Quick (no Qt Widgets). +find_package(Qt6 REQUIRED COMPONENTS Quick Gui) + +qt_standard_project_setup(REQUIRES 6.5) + +qt_add_executable(chatter + src/main.cpp + src/PenDevice.h + src/PenDevice.cpp + src/AppControl.h + src/AppControl.cpp + src/InkEngine.h + src/InkEngine.cpp + src/FbCapture.h + src/FbCapture.cpp + src/epfb.h +) + +qt_add_qml_module(chatter + URI Chatter + VERSION 1.0 + QML_FILES + qml/Main.qml +) + +target_link_libraries(chatter PRIVATE + Qt6::Quick + Qt6::Gui + # EPFramebuffer::instance/swapBuffers live in the epaper scenegraph plugin. + ${CMAKE_SYSROOT}/usr/lib/plugins/scenegraph/libqsgepaper.so +) + +# Export the executable's dynamic symbols so our setBuffers interposer (in +# FbCapture.cpp) wins the cross-DSO call from the epaper platform plugin. +target_link_options(chatter PRIVATE -Wl,--export-dynamic) diff --git a/README.md b/README.md new file mode 100644 index 0000000..c0a51b5 --- /dev/null +++ b/README.md @@ -0,0 +1,134 @@ +# Chatter + +A minimal, purpose-built writing interface for the **reMarkable Paper Pro Move** +(model RM03A), used as an **assistive-speech device**: the user writes, her +conversation partner reads. Chatter *augments* the stock reMarkable software; it +does not replace it. + +- **End-user instructions:** [`doc/Chatter_user_guide.md`](doc/Chatter_user_guide.md) +- **Full technical reference (developers start here):** + [`doc/Chatter_technical_reference.md`](doc/Chatter_technical_reference.md) +- **Plan & history:** [`doc/Chatter_implementation.md`](doc/Chatter_implementation.md) + +## Repository layout + +``` +dist/ Prebuilt tablet binaries (ready to install — see Quick install) +src/ C++ sources (ink engine, framebuffer capture, pen reader, …) +qml/ Qt Quick UI (the static Back/Clear bar + touch handling) +tools/ Standalone helpers (the return-to-Chatter launcher, probes) +scripts/ Build, deploy, and install scripts +doc/ Documentation (the technical reference has a full document map) +``` + +--- + +## Quick install (prebuilt binary — no build needed) + +For installing/testing on a tablet, you do **not** need to build anything. The +repo ships ready-to-run aarch64 binaries in [`dist/`](dist/) (`dist/chatter`, +`dist/chatter-launcher`). You just need SSH access to the tablet. + +### 1. One-time device setup + +On the tablet (this **factory-resets** it — back up / sync first): + +1. **Enable developer mode:** Hamburger menu → Settings → Software → Advanced → + Developer mode → Accept. (Details: [`doc/developer_mode_screen.md`](doc/developer_mode_screen.md).) +2. Reconnect wifi (the reset wipes it). +3. Get the root credentials: Settings → Help → **Copyright and licenses** → + under **GPLv3 Compliance** (root username, password, and the device IPs). +4. **Enable SSH over wifi** (off by default): connect over USB (`10.11.99.1`), + SSH in, run `rm-ssh-over-wlan on`. +5. Add your workstation's SSH public key to the tablet's + `/home/root/.ssh/authorized_keys`. + +### 2. Point the `chatter` SSH alias at the tablet + +In your `~/.ssh/config`: + +``` +Host chatter + HostName 192.168.x.x # the tablet's wifi IP (or 10.11.99.1 over USB) + User root + IdentityFile ~/.ssh/id_ed25519 +``` + +> The tablet's wifi IP can change (DHCP), and it drops wifi when asleep. If +> `ssh chatter` stops connecting, wake the tablet and re-check the IP on the same +> **Copyright and licenses → GPLv3 Compliance** screen (a DHCP reservation on your +> router avoids this). + +### 3. Install + +``` +scripts/deploy-and-run.sh # copies dist/chatter and launches it on the panel +scripts/install-launcher.sh # installs the persistent 4-finger return launcher +``` + +`scripts/restore-xochitl.sh` returns to the standard reMarkable GUI at any time. + +> The tablet's actual root password / IPs are **not** in this repo (they live in +> the gitignored `doc/tablet_access.md`) — get them from Andreas. + +--- + +## Building from source + +Only needed if you're **changing the code**. Chatter is a Qt 6 app +**cross-compiled** to the tablet's aarch64 CPU; you build on a host and copy the +binary over. + +### Prerequisites + +1. **A Linux host** (the reMarkable SDK is a Yocto toolchain — Linux only). +2. **The reMarkable "Chiappa" SDK** (cross-compiler + Qt 6.8.2 sysroot, ~486 MB), + from developer.remarkable.com. Install it, e.g.: + ``` + ./remarkable-…-chiappa-…-toolchain.sh -d ~/external/remarkable-sdk/chiappa-3.27.0.97 + ``` + The confirmed **x86_64-host** build is: + `https://storage.googleapis.com/remarkable-codex-toolchain/3.27.0.97/chiappa/remarkable-production-image-5.7.119-chiappa-public-x86_64-toolchain.sh` + An **aarch64-host** variant exists at the same path with `x86_64`→`aarch64` + (confirm the exact link on developer.remarkable.com). + +### Build & deploy + +``` +CHATTER_SDK=/path/to/chiappa-sdk scripts/build.sh # -> build/chatter + tools/chatter-launcher +scripts/deploy-and-run.sh # prefers build/chatter when present +scripts/install-launcher.sh +``` + +### Building from macOS (Linux container) + +There is **no macOS build of the reMarkable SDK**, so build inside a **Linux +container**. The cross-compiled output targets the tablet either way; only the +*host* running the SDK must be Linux. + +1. Install a container runtime — **OrbStack** or **Docker Desktop**. +2. Start a Linux container matching your Mac's CPU and mount the repo: + - **Apple Silicon Mac:** an **arm64** image (runs natively, no emulation): + ``` + docker run -it --platform linux/arm64 -v "$PWD":/work ubuntu:24.04 + ``` + Inside it, install the **aarch64-host** Chiappa SDK. + - **Intel Mac:** an **x86_64** image + the **x86_64** SDK. +3. In the container, install what the SDK/build need + (`apt-get update && apt-get install -y build-essential cmake file which`), + run the SDK self-extractor, then `cd /work && CHATTER_SDK=… scripts/build.sh`. +4. Deploy from the container (or from the Mac, if SSH to the tablet is set up + there): `scripts/deploy-and-run.sh build/chatter`. + +> This container route is the standard reMarkable cross-build approach but has +> not been verified end-to-end here — if you hit a missing-package error in the +> container, install that package and continue. + +--- + +## Status + +Working on hardware: stock-quality ink matching the selected pen, finger-wipe +erase, whole-page Clear, stylus eraser, bidirectional toggle with a persistent +return launcher, and vertical scrolling on a growable canvas. Open items (Save, +horizontal scroll/zoom, field tuning) are in the technical reference §10. diff --git a/dist/chatter b/dist/chatter new file mode 100755 index 0000000..e4a619c Binary files /dev/null and b/dist/chatter differ diff --git a/dist/chatter-launcher b/dist/chatter-launcher new file mode 100755 index 0000000..d9a2f77 Binary files /dev/null and b/dist/chatter-launcher differ diff --git a/doc/.gitignore b/doc/.gitignore new file mode 100644 index 0000000..b028300 --- /dev/null +++ b/doc/.gitignore @@ -0,0 +1,2 @@ +tablet_access.md + diff --git a/doc/Chatter_application_proposal.md b/doc/Chatter_application_proposal.md new file mode 100644 index 0000000..c9cf35c --- /dev/null +++ b/doc/Chatter_application_proposal.md @@ -0,0 +1,96 @@ +# Chatter — A custom application for the reMarkable tablet + +## Background + +I am helping someone who has a hard time speaking and who is using a +[reMarkable tablet](https://image.email.remarkable.com/lib/fe3511737364047c771479/m/1/e39f7595-fc99-4f8c-b7e3-dde042451668.pdf) +in conversations to write what she cannot say. I believe that the +reMarkable table is the best choice for a digital handwriting interface. +However, the tablet is designed for taking notes that can be preserved or even +transformed into digital text. It is not intentionally designed to be used as an +*assistive-speech device*. There are also a wide variety of customizable options +that are typically unnecessary if using the table in that way. + +## Project goal + +I would like to create a replacement for the reMarkable graphical user interface +(GUI) that would be tailored for using the table as an assistive-speech device. +This interface would not replace the standard GUI; choosing tablet settings for +the writing style or other interface parameters would still be done in the +standard GUI. This assumes, however, that the user can change settings as needed +in the normal way (or that technical help is available when necessary). +The goal here is to make the interaction with the tablet during conversation as +simple as possible. + +The custom application's name is "Chatter", with the implication that +conversational interaction can be accelerated. The word also can refer to the +speaker as well as to the result of speaking. + +## User-interface elements + +In watching the user with the tablet's standard GUI, I have noticed a confusion +about whether writing or erasing is the active mode. A simple action like +clearing the page requires several user-interface gestures. The user's +inability to easily use the table is often not wholly due to a lack of technical +understanding but a failure to remember technical details when they are not +intuitive. + +Chatter implements four actions, ordered here by their potential frequency of +use: + +- *Erase part of the text* — This should be simple and intuitive gesture. For + example, I have seen the user wiping her fingers back and forth across some + text to erase it. + +- *Erase the page* — Erasing the entire page should be a single gesture. The + blank page that results continues to use the current pen styling and other + parameters set in the standard GUI. + +- *Save current text to a file named by time* — The current page is saved + with an automatically generated name that contains a human-readable text of the year, + month, date, hour, minutes and seconds. The seconds may not be necessary, but + I would rather have an elaborate name than risk losing a previously saved + file. These Chatter transcripts should be saved to a separate directory to + avoid cluttering the main screen. Deleting and renaming these files are + possible through the standard GUI; a tutorial about that should be provided if + the user expresses interesting in managing the transcripts in that way. + +- *Toggle between the standard GUI and Chatter* — The standard GUI should still + be available through a single gesture; Chatter augments the tablet's interface + and does not replace it. However, returning to Chatter will require an + addition to the standard GUI that will be memorable for the user. + +### Implementation considerations + +If the table can differentiate between a fingertip and the stylus, then erasing +part of the text as well as the entire page could be implemented by a finger +gesture. Rubbing the finger back and forth of an area of text could remove any +drawing close to that area (where "close" might be determined experimentally +working with the user). A diagonal stroke with a finger from upper-right to +lower-left would be interpreted as a request to erase the entire page. + +If differentiating between fingertip and stylus is not possible, then Chatter +should define some shortcut for the stylus that easily creates the standard +region-based erase gesture. For example, if the user circles an area twice, +that could signify erasure, not drawing. A stylus stroke from upper-right to +lower-left could specify whole-page erasure, even if the finger stroke +method is also possible. + +The toggling the standard GUI and saving a file are not actions that the user +would frequently perform. I think that small icons at the bottom of the page, +activated by stylus or finger, would provide an ongoing reminder of how to do +those actions. The standard GUI will need to provide some way of switching to +Chatter; the user should suggest a method that she would find memorable. + + +## Deployment and refinement + +A developer working on this project will install the custom GUI on the user's +tablet. After verifying that the installation is successful, the developer will +watch the user with the tablet to evaluate the current interface efficiency of +Chatter, talking with the user about what is missing and what could be improved. +Paying attention to the user's questions that begin, "Why does this..." and "Why +doesn't this..." will be very important in refining the design. I believe that +using a custom interface for the tablet might also inspire the user to suggest +other features that would not be obvious to the developers. + diff --git a/doc/Chatter_implementation.md b/doc/Chatter_implementation.md new file mode 100644 index 0000000..9811d47 --- /dev/null +++ b/doc/Chatter_implementation.md @@ -0,0 +1,381 @@ +# Chatter — Implementation Plan & Method + +**Status:** Living document — revised as we learn. Read the Changelog at the bottom for what changed. +**Version:** 0.7 (2026-06-26) +**Audience:** The Chatter development team (Andreas, Matt, and Claude as development assistant). + +> This document is the shared source of truth for *how* we build Chatter. It is +> deliberately incremental: early phases will answer questions that later phases +> depend on, and this document will be updated as those answers come in. If +> something here is marked **(unknown / to confirm on-device)**, treat it as an +> open question, not a decision. + +--- + +## 1. What Chatter is + +Chatter is a minimal, purpose-built interface for a reMarkable tablet used as an +**assistive-speech device** by a user who has difficulty speaking — she writes, +and her conversation partner reads. Chatter **augments** the tablet; it does not +replace the standard reMarkable GUI. + +The design priority is **intuitiveness under intermittent recall**: the user +should not have to remember technical steps. See the full motivation and the +user-experience reasoning in [`Chatter_application_proposal.md`](Chatter_application_proposal.md). + +Chatter's core actions, by expected frequency: + +1. **Erase part of the text** — a simple, intuitive gesture (e.g. wiping a + fingertip back and forth over an area). +2. **Erase the whole page** — a single gesture (e.g. a diagonal finger stroke, + upper-right → lower-left). +3. **Toggle to/from the standard GUI** — *necessary*; Chatter must be able to + hand off to the stock interface and be returned to. +4. **Save the current page** — *importance to the user is not yet confirmed*; a + single button writes a timestamped transcript into a "Chatter" folder. + +--- + +## 2. Target device and key constraints + +**Device: reMarkable Paper Pro Move — model RM03A.** (Identified from the +official user-guide PDF linked in the proposal.) + +| Property | Value | Why it matters | +|---|---|---| +| Display | E Ink Gallery 3, **color**, 7.3", 954×1696 (264 PPI), backlit | Color refresh pipeline differs from older grayscale models | +| Touch | Multi-point capacitive | Finger gestures (erase) are viable | +| Pen | Active Marker / Marker Plus (flip-to-erase on Plus) | Pen and touch are **separate input streams** → we can tell finger from stylus natively | +| SoC / RAM | ARM 1.7 GHz dual A55, 2 GB RAM, 64 GB | Standard ARM-Linux cross-compile target | +| OS | "Codex" (custom Linux) | Stock note app is Xochitl (Qt/C++, proprietary) | +| Dev access | Enable **developer mode**, which requires a **factory reset** | **Back up the device first** | + +**Key risk — this is the newest model.** The mature community frameworks +(rmkit, libreMarkable, Plato, KOReader ports) were built against the older +RM1/RM2 grayscale framebuffer and refresh ioctls. They likely **do not support +the Paper Pro Move yet**, or only partially. We should expect to do +**first-principles work** — discovering this model's framebuffer format, color +refresh path, and input device nodes by probing the hardware over SSH — rather +than relying on an existing framework. This raises the value of hands-on device +access early and argues for a small, well-proven first milestone. + +--- + +## 3. Roles + +- **Andreas** — development, with Claude's assistance, on his own Paper Pro Move + over SSH. +- **Matt** — installation and QA, including on-site testing with the end user on + her tablet. Matt has the same model (Paper Pro Move). Matt may also take on + development; the split will be worked out as the project progresses. +- **Claude** — development assistant: writes/reviews code, drives the + build-deploy loop over SSH where possible, and maintains this document. + +This document is how Matt stays in sync. When the design changes, the change +lands here first. + +--- + +## 4. Development phases + +The phases are ordered so each de-risks the next. **Toggle (necessary) precedes +Save (provisional).** + +### Phase 0 — Safety & access +- **Back up first** (critical on the user's device; skippable on a new tablet + with nothing saved). Enabling developer mode **performs a factory reset** — on + the Paper Pro family, SSH is unavailable until developer mode is on, and + turning it on wipes the device. Sync to the reMarkable cloud account before + enabling. Document this backup procedure so Matt can repeat it. +- **Enable developer mode:** Hamburger menu (upper left) → Settings → Software → + Advanced → Developer mode → Accept. This factory-resets the device (deletes + all local files; user data cleared on reboot). Re-pair with the reMarkable + account afterward to access cloud content. (Disabling later requires + reMarkable's recovery application.) +- **Reconnect wifi** after the reset (the wifi config is wiped). +- **Get SSH credentials:** Menu → Settings → Help → Copyright and licenses; + the root username (`root`), password, and device IP addresses are under the + "GPLv3 Compliance" header. USB exposes the tablet at `10.11.99.1`. +- **Enable SSH over wifi (it is OFF by default):** connect over USB, SSH in, + and run `rm-ssh-over-wlan on`. Only then is the listed `192.168.x.x` address + reachable; after that the USB cable is optional. +- Install your workstation's SSH public key into the tablet's + `~/.ssh/authorized_keys` (user `root`) for passwordless access. +- Confirm SSH login, `scp` of a binary, and remote execution. +- Stand up the x86_64-hosted cross-toolchain for Codex OS; verify a trivial + ARM binary runs on-device. + +> Dev environment note: development host is **Jatke** (on the wired LAN); we +> connect to the tablet **over wifi** (USB not in use). + +### Phase 1 — On-device investigation (no app code; produces a findings writeup) +Answer the questions that can only be answered with the tablet over SSH. See the +checklist in §5. Output: a short findings document added to `doc/`. +**Status: substantially complete (2026-06-25)** — see +[`Chatter_phase1_findings.md`](Chatter_phase1_findings.md). Key results: pen +(`event2`) and finger (`event3`) are separate evdev devices; the display is +DRM/KMS only (no fbdev) driven by `imx-drm`; and the stock app is a Qt 6.8.2 app +rendering through a reusable **`epaper` QPA plugin** — so Chatter should be a +Qt 6 app run with `-platform epaper`. Remaining sub-items (verifying `-platform +epaper` from a third-party binary, input delivery, toggle hand-off) roll into +the Phase 2 spike. + +### Phase 2 — Minimal canvas spike +Read the pen, render strokes to the (color) display, and clear the page — +nothing more. This proves the two hardest pieces (input + color refresh) end to +end and de-risks everything after it. + +**Status (2026-06-25): display/toolchain half PROVEN on hardware.** A +third-party Qt Quick app (`build/chatter`) cross-built with the Chiappa SDK +renders crisp black-on-white on the e-paper via `-platform epaper` +(`QT_QUICK_BACKEND=epaper`). + +**Status (2026-06-26): inking WORKS, with one known quality issue.** Custom +evdev pen reader (`src/PenDevice.cpp`, reads `event2`) drives a +`QQuickPaintedItem` canvas (`src/InkCanvas.cpp`): ink tracks the pen, orientation +correct, eraser (BTN_TOOL_RUBBER) erases, finger-tap "Clear" works, latency good. +- **Known issue:** strokes render as a "dashed line that fills in" (two-pass), + unlike the solid stock pen. **Root cause:** we use the panel's default + *grayscale* mode. **Fix path:** xochitl wraps its writing area in a QML + `ScreenModeItem` set to **Mono/FAST** with a dedicated **pen waveform** + (`/usr/share/remarkable/ct33_pen.bin`). The control is the private + `EPScreenModeItem` class in `libqsgepaper.so` (`setMode()`, modes + Mono/FAST/FastGrayscale) — not a public QML type, no SDK header — so Chatter + must integrate it directly (instantiate, link the plugin, drive `mode` via the + meta-object, wrap the canvas). Antialiasing on/off does not affect it. +- **Pen-style decision:** Chatter uses the pen style set in the standard GUI + (xochitl.conf `LastPen`/`LastPenSize`/`LastPenColor`, readable via QSettings), + re-read so it tracks changes the user makes after toggling — no style menu in + Chatter. + +**Lessons learned (encoded in `scripts/`):** +- The e-paper framebuffer is a **singleton guarded by an flock** + (`/tmp/epframebuffer.lock`, from `libqsgepaper`'s `EPFramebufferAcep2`). Only + ONE process may hold it; a stranded instance shows `Failed to lock + epframebuffer` / `Failed to initialize SWTCON` and the panel stays blank. + Always stop the previous instance first. Journal success marker: + `SWTCON initialized \o/`. +- **Run as a transient systemd service** (`systemd-run --unit=chatter …`), not an + ssh background job (which holds the SSH channel open and can strand the + process). Manage with `systemctl stop chatter` / `journalctl -u chatter`. +- A full-screen `Window` needs an **explicit size** (`width: Screen.width; + height: Screen.height`); `visibility: FullScreen` alone left it unsized/blank. +- xochitl runs with **no special Qt env** — `-platform epaper` + + `QT_QUICK_BACKEND=epaper` suffices. + +**Run recipe:** `scripts/build.sh && scripts/deploy-and-run.sh`; +`scripts/restore-xochitl.sh` to return to the standard GUI. + +### Phase 3 — Erase gestures +- Region erase: finger wipe removes ink near the wiped area ("near" tuned + experimentally with the user). +- Whole-page erase: single diagonal finger stroke (upper-right → lower-left). +- Provide stylus fallbacks if finger/stylus separation proves unreliable. + +### Phase 4 — Toggle to/from the standard GUI *(necessary)* + +**Status (2026-06-26): BIDIRECTIONAL toggle works on-device (verified).** +- **Chatter → standard:** top-left **"Back"** button → `AppControl::returnToStandard()` + → transient unit stops chatter, starts xochitl. +- **standard → Chatter:** a persistent **launcher daemon** (`tools/chatter_launcher.c`, + installed as a rootfs systemd service via `scripts/install-launcher.sh`) watches + the touch device and, on a **4-finger hold (~700 ms)** while xochitl is in front, + runs `to-chatter.sh` to switch to Chatter. Gesture is tunable; finalize the + memorable one with the user. +- **Deployment note:** `/etc` and `/run` are volatile overlays (wiped on reboot); + persistent installs go to `/home` (app/binaries/scripts) or the rootfs + (`mount -o remount,rw /`, lost on OS update). Disable OS auto-updates on a + delivered device. + +- A gesture/button in Chatter hands off to the standard reMarkable GUI. +- A **memorable** mechanism returns from the standard GUI to Chatter. The + proposal asks that the user help choose this; we will prototype options and + let her pick. +- **Phase 1 insight:** because the display is DRM/KMS (single master) and + xochitl owns it while running, the toggle is a **hand-off**, not an overlay — + stop/pause `xochitl` to give Chatter the display, and reverse to return. + Mechanism (`systemctl stop/start` vs. `SIGSTOP/SIGCONT`) to be validated. + +### Phase 5 — Save the page *(provisional — pending confirmation it matters to the user)* +- A single visible button saves the current page. +- Filename fully derived from date/time (year, month, day, hour, minute, + second). +- Written into a folder named **"Chatter"**, using the same on-disk document + format the stock app uses, so transcripts appear as normal notebooks and can + be renamed/deleted/synced via the standard GUI (see §6). + +### Phase 6 — Field test & refine +Matt installs and observes the user. Capture especially the "Why does this…" and +"Why doesn't this…" questions; feed them back into this document and the phase +plan. + +--- + +## 5. On-device investigation checklist (Phase 1) + +These are the open questions the hands-on phase exists to answer. + +- **Input devices:** which `/dev/input/event*` is the pen vs. the capacitive + touch; event format, coordinate range and orientation, pressure; the + flip-to-erase signal on Marker Plus. +- **Display:** how the color Gallery 3 panel is driven — framebuffer device and + pixel format, and the refresh mechanism (ioctls or this model's equivalent). + *(Highest-risk unknown.)* +- **Document store:** where Xochitl stores notebooks; the + `.metadata` / `.content` / per-page `.rm` layout for this model; the `.rm` + format version (newer, color-capable); how a *collection* (folder) is + represented; and whether a hand-built document appears in the library after a + restart/sync. +- **Coexistence & toggle:** how to launch our binary alongside / over Xochitl, + and what a reliable toggle and return-to-Chatter mechanism looks like. + +--- + +## 6. The "save" approach (clarification) + +reMarkable exposes **no public callable on-device API** for creating notebooks. +The stock app simply **writes files to an on-disk document store** in a specific +format (roughly: a UUID per document, a `.metadata` JSON, a `.content` JSON, and +one binary `.rm` line-file per page, plus the color/folder data and a sync +layer). A "folder" such as the **Chatter** folder is itself a document of type +*collection*. + +So "save using the same method the environment uses" means: **write our canvas +into the exact on-disk format Xochitl reads, for this model's format version.** +The open question for Phase 5 is confirming/replicating the Paper Pro Move's +current `.rm` format. This is investigated in Phase 1. + +--- + +## 6a. Confirmed device facts (from SSH, 2026-06-25) + +- **Internal codename:** "Chiappa" (`/proc/device-tree/model` = "reMarkable + Chiappa"; hostname `imx93-chiappa`). This is the Paper Pro Move (RM03A). +- **SoC:** NXP **i.MX93** (Arm Cortex-A55). Relevant to the toolchain target and + to how the display is driven. +- **OS:** Codex Linux **5.7.121** (Yocto **scarthgap**), image version + `3.27.1.0`. +- **SSH server:** **dropbear** (not OpenSSH). Root login; `~/.ssh/authorized_keys` + honored. Wifi SSH enabled via `rm-ssh-over-wlan on`. +- **Access from Jatke:** passwordless via ed25519 key as `root` over wifi (the + tablet's IP) or `root@10.11.99.1` (USB, fixed on every reMarkable). The tablet's + actual IP/MAC and credentials are in `tablet_access.md` (SENSITIVE, gitignored). + +**Phase 0 status:** access established (dev mode, SSH over USB + wifi, key-based +login, scp verified). Remaining for Phase 0: stand up the cross-toolchain and +run a trivial ARM binary on-device. + +## 6b. Toolchain / build & deploy (researched 2026-06-25) + +**Official SDK (cross-toolchain + Qt 6.8.2 sysroot), self-extracting `.sh`, +x86_64 host only.** For this device (codename **chiappa**): + +``` +https://storage.googleapis.com/remarkable-codex-toolchain/3.27.0.97/chiappa/remarkable-production-image-5.7.119-chiappa-public-x86_64-toolchain.sh +``` + +- ~486 MB. Device image is `3.27.1.0` (Codex 5.7.121); newest published Chiappa + SDK is `3.27.0.97` (5.7.119) — same `3.27.x` family, so binaries built against + it run on the device. (An aarch64-host variant also exists; we use x86_64 for + Jatke.) Download links live on developer.remarkable.com/links; the GCS bucket + itself is not directly listable (403). +- Build host **Jatke** (x86_64, ample disk). SDKs expect a Yocto-supported Linux + host. + +**Install & activate:** +``` +chmod u+x remarkable-production-image-5.7.119-chiappa-public-x86_64-toolchain.sh +./remarkable-production-image-5.7.119-chiappa-public-x86_64-toolchain.sh -d +source /environment-setup-cortexa55-remarkable-linux # exact filename confirmed post-install (i.MX93 = Cortex-A55) +``` + +**Build a Qt Quick app** (per developer.remarkable.com/documentation/qt_epaper): +CMake with `find_package(Qt6 REQUIRED COMPONENTS Quick)` + +`qt_add_executable` / `qt_add_qml_module`; `cmake . -B build && cmake --build +build` after sourcing the SDK env. + +**Deploy & run on device:** +``` +scp build/ chatter:/home/root/chatter/ # deploy under /home, not rootfs +ssh chatter 'systemctl stop xochitl' # release the display (pre-authorized) +ssh chatter 'cd /home/root/chatter && QT_QUICK_BACKEND=epaper ./ -platform epaper' +``` + +Constraints (official + Phase 1): **Qt Quick/QML only, no Qt Widgets**; **xochitl +must be stopped** (DRM single-master); **touch is automatic, pen is custom** +(read `event2`). Some SDK/OS combos require copying `libqsgepaper.so` to the +device — verify during the spike. + +## 7. Decisions & open questions + +**Decided (from Phase 1):** +- **Implementation stack: C++ / Qt 6.8.2**, rendered via the device's **`epaper` + QPA plugin** (`-platform epaper`). Chosen because the stock app proves this + path works and gives us e-ink display + refresh for free; the older + fbdev-mmap approach is not available (DRM/KMS only). Cross-build with a Yocto + scarthgap aarch64 SDK matching Qt 6.8.2. *(Matt — flag any objection.)* +- **Deploy location: under `/home`** (45 GB free), never the rootfs (~88 MB + free, overlay, reset by updates). +- **Toggle is a display hand-off** with xochitl (see Phase 4), not an overlay. + +**Still open:** +- **Save mechanism** — leaning **PDF-backed document** over authoring native v6 + `.rm`; decide in Phase 5 (and gated by whether save matters to the user). +- **Whether "save" matters to the user** — learned from observation. +- **Toggle gesture + return-to-Chatter mechanism** — prototype and choose with + the user (Phase 4). +- **Erase "closeness" thresholds** — tuned experimentally with the user + (Phase 3). + +--- + +## Changelog + +- **0.6 (2026-06-26)** — Inking engine working on hardware: custom evdev pen + reader (`event2`) + `QQuickPaintedItem` canvas — pen draws, eraser erases, + finger-tap **Clear**, good latency. Added top control bar (**Back** left, + **Clear** right); Back hands off to the standard GUI (verified). Bound the + private `EPScreenModeItem` (key-function trick) to control panel refresh mode — + but the **dashed two-pass stroke refresh is the render loop's behavior, + independent of mode**, so it's deferred to a framebuffer-direct effort + (needs disassembly of `swapBuffers` for `EPContentType`/`UpdateFlag`). + Implemented **pen-style matching**: Chatter reads the standard GUI's selected + pen (type/size/color) from `xochitl.conf` `LastWritingTool` and applies width + + color — verified (Calligraphy, size 3, black). +- **0.7 (2026-06-26)** — ⭐ **Solved stroke quality with a direct-framebuffer ink + pipeline.** After proving Qt Quick can't render stock-quality e-ink strokes + (geometry nodes unsupported by the software backend; painted-textures always + get the dashed two-pass; screen-mode and antialiasing irrelevant), switched + architecture: `FbCapture` interposes `EPFramebuffer::setBuffers` in-process + (`-Wl,--export-dynamic`) to capture the real framebuffer image; `InkEngine` + draws strokes straight into it and refreshes each segment with an explicit + `swapBuffers` — **solid, crisp, low-latency, correctly-aligned strokes on + hardware (verified), plus working Clear.** Qt Quick now only renders the static + Back/Clear UI. Key RE findings: `swapBuffers` renders solid (not dashed) on a + single explicit call; intra-DSO calls aren't LD_PRELOAD-interposable but the + cross-DSO `setBuffers` is. Remaining: calligraphic nib (uniform round width vs + the angled direction-dependent Calligraphy nib), return-to-Chatter trigger, + Save, field testing. +- **0.5 (2026-06-25)** — Phase 2 spike: cross-built a Qt Quick hello-app with the + Chiappa SDK and rendered it on the e-paper via `-platform epaper` (proven on + hardware). Recorded lessons (panel flock singleton, run-as-systemd-service, + explicit Window size) and added `scripts/build.sh`, `deploy-and-run.sh`, + `restore-xochitl.sh`. Seeded the source tree (`CMakeLists.txt`, `src/main.cpp`, + `qml/Main.qml`). +- **0.4 (2026-06-25)** — Researched the toolchain (§6b): identified the official + Chiappa SDK (`3.27.0.97` / Codex 5.7.119, x86_64 host, ~486 MB), install/ + activate steps, the Qt Quick build + `-platform epaper` run commands, and the + deploy-and-stop-xochitl flow. Confirmed Qt-Quick-only + custom-pen constraints. +- **0.3 (2026-06-25)** — Phase 1 on-device investigation substantially complete + (see `Chatter_phase1_findings.md`). Resolved the stack decision to C++/Qt 6 + + `epaper` QPA plugin; recorded display = DRM/KMS only, pen/finger as separate + evdev devices, toggle = display hand-off, deploy under `/home`, and the + document-store format for the Save phase. +- **0.2 (2026-06-25)** — Phase 0 access established. Added confirmed device + facts (§6a): codename "Chiappa", NXP i.MX93 SoC, Codex Linux 5.7.121 + (scarthgap), dropbear SSH, passwordless key access from Jatke. Corrected the + dev-mode menu path and documented that wifi SSH is off by default + (`rm-ssh-over-wlan on`). +- **0.1 (2026-06-25)** — Initial version: scope, target device (RM03A) and its + constraints, roles, phase plan (with Toggle before Save), Phase 1 investigation + checklist, save-format clarification, and open decisions. diff --git a/doc/Chatter_phase1_findings.md b/doc/Chatter_phase1_findings.md new file mode 100644 index 0000000..9fecfcf --- /dev/null +++ b/doc/Chatter_phase1_findings.md @@ -0,0 +1,140 @@ +# Chatter — Phase 1 On-Device Findings + +**Date:** 2026-06-25 +**Device:** reMarkable Paper Pro Move (RM03A, codename "Chiappa") +**Method:** Read-only SSH probing from Jatke (`ssh chatter`). No changes made to +the device except enabling developer mode / SSH (Phase 0) and adding our SSH key. + +This document records what we learned and the implementation decisions those +findings support. It feeds the phase plan in +[`Chatter_implementation.md`](Chatter_implementation.md). + +--- + +## 1. Hardware & OS + +- **SoC / arch:** NXP **i.MX93**, **aarch64** (Arm Cortex-A55). Kernel + `6.12.49+git-imx93-chiappa`. +- **OS:** Codex Linux 5.7.121 (Yocto **scarthgap**), image `3.27.1.0`. glibc + (`/lib/ld-linux-aarch64.so.1`, `libc.so.6`). +- **RAM:** 2 GB (~1.4 GB free) — ample for our app. +- **Storage (matters for deployment):** + - `/` (rootfs): ~435 MB, **only ~88 MB free (78% used)** — also an + OS-managed overlay, likely reset by firmware updates. **Do not install here.** + - `/home`: **46 GB encrypted volume, ~45.8 GB free.** Install Chatter under + `/home` (e.g. `/home/root/chatter`). + +## 2. Input devices + +Pen and finger are **separate evdev devices** — finger-vs-stylus discrimination +is free, exactly what Chatter's gesture design needs. + +| Node | Name | Role | Notes | +|---|---|---|---| +| `event0` | `bbnsm:pwrkey` | Power button | | +| `event1` | Hall effect sensors | Folio open/close | switch events | +| **`event2`** | **Elan marker input** | **Stylus / Marker** | on SPI | +| **`event3`** | **Elan touch input** | **Capacitive multitouch (finger)** | `INPUT_PROP_DIRECT` | + +**Pen (`event2`)** — digitizer space **6760 × 11960**: +- Buttons: `BTN_TOOL_PEN`, **`BTN_TOOL_RUBBER`** (Marker Plus eraser end — + flip-to-erase is detectable), `BTN_TOUCH`, `BTN_STYLUS`, `BTN_STYLUS2`. +- `ABS_PRESSURE` 0–4096, `ABS_DISTANCE` (hover) 0–65535, `ABS_TILT_X/Y` ±9000. + +**Touch (`event3`)** — grid **1248 × 2208**: +- Up to **10 contacts** (`ABS_MT_SLOT` 0–9) with `ABS_MT_POSITION_X/Y`, + `ABS_MT_PRESSURE` (0–255), `ABS_MT_TRACKING_ID`, `ABS_MT_TOOL_TYPE`. + +All three coordinate spaces share the screen's portrait aspect (~0.5625): +panel 954×1696, pen 6760×11960, touch 1248×2208. + +## 3. Display pipeline + +- **DRM/KMS only — there is NO `/dev/fb*`** and `/sys/class/graphics` is empty. + Driver is **`imx-drm`**; device `/dev/dri/card0`; connector + **`card0-LVDS-1`** ("connected", "disabled" at rest — normal for e-ink, which + holds its image without continuous scanout). +- The connector advertises a **packed `365×1700` mode** (hardware buffer for the + E Ink Gallery 3 color subpixel layout); the logical screen is 954×1696. The + mapping is handled by the epaper plugin (below) — we don't touch it directly. +- **Implication:** the `mmap /dev/fb0` approach used by rmkit / libreMarkable on + older grayscale models **does not apply** to this device. + +## 4. Graphics / UI stack — the key finding + +- The stock app **`xochitl`** is a **Qt 6.8.2 / Qt Quick (QML)** application. +- It renders through a **custom Qt platform (QPA) plugin: `epaper`** + (`/usr/lib/plugins/platforms/libepaper.so`, links `libdrm`). Available QPA + plugins on-device: **`epaper`**, `minimal`, `offscreen`, `vnc`. +- Qt 6.8.2 runtime libraries and QML modules (`QtQuick`, `QtQuickControls2`, + etc.) are all present under `/usr/lib` and `/usr/lib/qml`. No dev headers + on-device (expected — we cross-build). + +**This means the native path is wide open:** build Chatter as a **Qt 6 app** and +run it with **`-platform epaper`** to inherit working e-ink display + refresh, +instead of reverse-engineering DRM/KMS and waveform handling. This resolves the +project's highest-risk unknown in our favor. + +## 5. Stock app & services + +- `xochitl.service` — the main UI app (`/usr/bin/xochitl`, pid varies). +- `marker-manager.service` — "Remarkable CSL Marker Manager" (pen support). +- `rm-sync.service` — document sync to the reMarkable cloud. +- Plus metrics / MDM / crash-uploader services. + +**Coexistence / toggle implication:** on a DRM/KMS device only one process owns +the display (DRM master) at a time, and xochitl holds it (and likely grabs the +input devices) while running. So the Chatter ↔ standard-GUI **toggle is a +hand-off** (stop or pause `xochitl` ↔ run Chatter, and a return path), **not an +overlay**. The exact mechanism (systemctl stop/start vs. SIGSTOP/SIGCONT) is to +be validated in the Phase 2 spike. + +## 6. Document store (for the Save phase) + +Location: **`/home/root/.local/share/remarkable/xochitl/`**. Per document: + +- `UUID.metadata` — JSON: `visibleName`, `parent` (empty = top level; a folder + is a `CollectionType` document whose UUID is used as children's `parent`), + `type` (`DocumentType`), timestamps, `pinned`, etc. +- `UUID.content` — JSON: page list under `cPages.pages[]` (each page has an `id`, + ordering `idx`, `template`, scroll position, CRDT-style `timestamp` fields). +- `UUID/` — directory of per-page **`.rm`** files. +- `UUID.thumbnails/` — page thumbnails. +- Store-root extras (Codex additions): `.tree` (binary "rM sync tree" index) and + `rm-search-index.db` (SQLite **search index**, not the canonical store). + +**`.rm` page format:** header confirmed as **`reMarkable .lines file, +version=6`** — the v6 binary scene-tree format (documented by the community, +e.g. `rmscene`). Authoring valid v6 is possible but non-trivial. + +**Save options (decide in Phase 5):** +1. **PDF/PNG-backed document** — render Chatter's canvas to PDF/PNG and create a + document that references it (xochitl already supports PDF documents). Far + simpler and robust; still a real, renamable/syncable library item. +2. **Native v6 `.rm`** — author the binary page format so the transcript is an + editable notebook. More work; revisit only if option 1 proves insufficient. + +Either way, transcripts go under a **`Chatter`** `CollectionType` folder, with +filenames derived from date/time. + +--- + +## 7. Decisions supported by Phase 1 + +1. **Stack: C++ / Qt 6.8.2**, rendered via the on-device **`epaper` QPA plugin**. + Cross-build with a Yocto **scarthgap** aarch64 SDK matching Qt 6.8.2 / the + device glibc. +2. **Input:** pen from `event2` (including `BTN_TOOL_RUBBER` for flip-to-erase), + finger from `event3` (10-pt). Confirm in Phase 2 whether Qt+epaper already + delivers these or we read evdev directly for custom gestures. +3. **Toggle = display/input hand-off** with xochitl, not an overlay. +4. **Save = PDF-backed document** (leaning), in a `Chatter` folder. +5. **Deploy under `/home`,** never the rootfs. + +## 8. Open items for the Phase 2 spike + +- Stand up the cross-SDK and build a trivial Qt app; run it with + `-platform epaper` **after stopping xochitl**; confirm it draws and refreshes. +- Verify how input arrives in a Qt app under epaper (Qt event stream vs. raw + evdev) and how reliably finger and pen separate at the Qt layer. +- Validate the toggle hand-off (stop/start vs. stop/cont) and a return path. diff --git a/doc/Chatter_stylus_research.md b/doc/Chatter_stylus_research.md new file mode 100644 index 0000000..089a652 --- /dev/null +++ b/doc/Chatter_stylus_research.md @@ -0,0 +1,79 @@ +# Chatter — Stylus & Pen-Rendering Research + +A record of what we learned trying to reproduce the reMarkable stock pen +behavior in Chatter, so we can resume this later. (Status 2026-06-26: the solid +ink pipeline is done and shipping; calligraphy is "close enough for now" per the +user and is **not essential** for the conversational-assistance use case.) + +## 1. The render pipeline (solved) + +The reMarkable epaper Qt backend is a **software scene-graph renderer**: +- Custom `QSGGeometryNode`s are silently dropped — they don't render. +- `QQuickPaintedItem` textures render but always via the e-ink **dashed + two-pass** refresh, with latency. Screen mode (Pen/Mono) and antialiasing do + not change this. +- **Solution:** bypass Qt for ink. Capture the framebuffer `QImage` by + interposing `EPFramebuffer::setBuffers` (cross-DSO, interposable; the + executable is linked `-Wl,--export-dynamic` so its symbol wins). Draw strokes + straight into that buffer (`InkEngine`) and refresh each segment with an + explicit `EPFramebuffer::swapBuffers(rect, …)` — which renders **solid, + single-pass**. Qt Quick is used only for the static UI. +- Buffer A is `960×1696 RGB32` (logical screen 954×1696, padded to 960); + `bytesPerLine 3840`. There is **no PNG image-format plugin** on the device + (only gif/ico/jpeg/svg) — save snapshots as **BMP**. + +## 2. The Calligraphy pen model + +reMarkable's Calligraphy is **not a fixed geometric/flat nib**. Confirmed by +experiment + reMarkable docs: it's a **dynamic** model combining: +- **Stroke direction** — downstrokes thick, upstrokes thin. (User's stock-pen + circles: thick on the descending side — right for clockwise, left for CCW. A + 4-direction asterisk showed *no* variation, because quick uniform straight + strokes don't trigger it.) +- **Pressure** — heavy pressure ≈ doubles the width. +- **Speed** — faster = thinner. +- **Tilt/orientation** — simulates an angled nib. + +The exact angle/algorithm is compiled into xochitl's proprietary brush engine +(assets `LS_Calligraphy_*` / `P_Calligraphy_*`, `rm-brushgfx`) — **not** in a +readable config, so exact duplication would need deep reverse-engineering. + +## 3. Width calibration + +- `LastPenSize` (from `xochitl.conf` `LastWritingTool`) is only a **category**: + 1/2/3 = thin/thicker/thickest. Not a pixel width. +- Measured on the physical tablet: **thicker(2) ≈ 1 mm, thickest(3) ≈ 2 mm, + thinnest ≈ 2 px**. Pressure can **almost double** the width. +- Display is **264 PPI → 264/25.4 ≈ 10.4 px/mm.** +- Chatter mapping (`main.cpp`): `maxMm = max(0.2, (size-1)·1.0)` → + size3 = 2 mm, size2 = 1 mm; `setWidthRange(2 px, maxMm·pxPerMm)`. + +## 4. Chatter's current approximation (`InkEngine`, pen type 21) + +``` +f = 0.25·dirF + 0.50·pressure + 0.12·speedF + 0.13·tiltMag // clamp 0..1 +width = minWidth + (maxWidth − minWidth)·f + dirF = 0.5 + 0.5·(Δy/len) // downstroke → 1, upstroke → 0 + speedF = 1 − clamp(speed/3, 0, 1) // speed = len/dt (px/ms), slow → 1 + tiltMag = clamp(hypot(tiltX, tiltY), 0, 1) // from event2 ABS_TILT_X/Y (±9000) +``` +Uniform (non-calligraphy) pens: `f = 0.5 + 0.5·pressure`. +`PenDevice` emits `strokeMove(pos, pressure, tiltX, tiltY, eraser)`. + +## 5. Tooling (in `tools/`) + +- `setbufshim.cpp` — LD_PRELOAD shim; proved `setBuffers` capture (feasibility). +- `fbdump.cpp` — LD_PRELOAD into xochitl; snapshots the framebuffer to + `/home/root/fbdump.{png,bmp}` when `/tmp/fbdump` is touched (use BMP — no PNG + plugin). Note: `setBuffers` interposition into *xochitl* via LD_PRELOAD was + flaky/unconfirmed in one attempt; the in-process `--export-dynamic` capture in + Chatter itself is reliable. + +## 6. Open questions / ways to go deeper later + +- Recover exact reMarkable widths-per-size and the nib/pressure curves by + snapshotting stock strokes (fix the xochitl fbdump path) and measuring, or by + parsing the v6 `.rm` per-point width/direction data. +- Add a proper speed estimate (smoothed) and tilt-direction (not just magnitude). +- Confirm `LastPenSize` values for the thin/medium categories (only size 3 = 3.0 + is confirmed). diff --git a/doc/Chatter_technical_reference.md b/doc/Chatter_technical_reference.md new file mode 100644 index 0000000..e886f45 --- /dev/null +++ b/doc/Chatter_technical_reference.md @@ -0,0 +1,334 @@ +# Chatter — Technical Reference (master document) + +**Audience:** a programmer who needs to understand, build, modify, or maintain +Chatter — including how the reMarkable standard software works and how Chatter +fits alongside it. + +This is the **index + complete architecture**. Several topics have their own +deep-dive documents; this file summarizes each and links to it, then fills in +everything not covered elsewhere. Read this first; follow the links for detail. + +## Document map + +| Document | What it covers | +|---|---| +| [`Chatter_application_proposal.md`](Chatter_application_proposal.md) | The why: the assistive-speech use case, user-experience goals. | +| [`Chatter_implementation.md`](Chatter_implementation.md) | The phased plan and its running changelog; device facts, toolchain, decisions. | +| [`Chatter_phase1_findings.md`](Chatter_phase1_findings.md) | On-device investigation results (input devices, display, doc store). | +| [`Chatter_stylus_research.md`](Chatter_stylus_research.md) | Pen rendering, the Calligraphy model, width calibration, tooling. | +| [`Chatter_user_guide.md`](Chatter_user_guide.md) | The end-user-facing instructions. | +| [`developer_mode_screen.md`](developer_mode_screen.md) | Enabling developer mode on the device. | +| `tablet_access.md` | **SENSITIVE** (root password + IPs) — gitignored, not in the repo. | + +--- + +## 1. The device + +**reMarkable Paper Pro Move — model RM03A, codename "Chiappa".** + +- **SoC:** NXP i.MX93 (Arm Cortex-A55, aarch64), 2 GB RAM, 64 GB storage. +- **OS:** "Codex" Linux 5.7.121 (Yocto *scarthgap*), image `3.27.1.0`. SSH server + is **dropbear** (not OpenSSH). +- **Display:** E Ink **Gallery 3 — color** (ACeP, "acep2"), 7.3", **954×1696 + logical / 960×1696 framebuffer, 264 PPI** (≈10.4 px/mm). Driven by **DRM/KMS + only** (`imx-drm`); there is **no `/dev/fb*`**. +- **Input:** pen = `/dev/input/event2` (Elan marker: `ABS_X` 0–6760, `ABS_Y` + 0–11960, `ABS_PRESSURE` 0–4096, `ABS_TILT_X/Y` ±9000, `BTN_TOOL_PEN`/ + `BTN_TOOL_RUBBER` for flip-to-erase). Finger = `/dev/input/event3` (10-point + capacitive, `ABS_MT_*`, grid 1248×2208). Power = `event0`, hall/folio = `event1`. + Pen and finger are **separate evdev devices** — Chatter tells them apart natively. + +Full investigation: [`Chatter_phase1_findings.md`](Chatter_phase1_findings.md). + +### 1.1 Filesystem / persistence model (critical) + +- **`/` (rootfs, `/dev/mmcblk0p3`)** — ext4, mounted **read-only**, but + **persistent**. Remount rw (`mount -o remount,rw /`) to write; survives reboots, + **lost on an OS update** (A/B partition swap). +- **`/etc`, `/run`, `/var/volatile`** — **VOLATILE overlays** (upperdir on tmpfs). + Anything written here is **lost on reboot**. (This is why a systemd unit dropped + in `/etc/systemd/system` vanished after a reboot.) +- **`/home`** — encrypted, **fully persistent**, ~45 GB free. Everything Chatter + installs lives here (`/home/root/chatter/`). SSH keys persist because they are + under `/home/root/.ssh`. + +**Consequence:** the launcher's systemd unit is installed onto the **rootfs** +(`/usr/lib/systemd/system`) so it survives reboots; the binary and scripts live +under `/home`. **An OS update wipes rootfs changes — disable OS auto-updates on a +delivered device, and keep `scripts/install-launcher.sh` to reinstall.** + +--- + +## 2. The standard reMarkable software stack + +Understanding the stock stack is necessary because Chatter reuses its display +plumbing and hands off to it. + +- **`xochitl`** — the stock note app. Qt **6.8.2** / Qt Quick, proprietary. Owns + the display while running. Started/stopped as the `xochitl` systemd service. +- **The `epaper` QPA platform plugin** (`libqsgepaper.so`, in + `/usr/lib/plugins/`). A Qt platform + **software** scene-graph renderer for the + e-paper. A Qt app runs on it with `-platform epaper` (+ `QT_QUICK_BACKEND=epaper`). + Key facts learned by reverse engineering: + - It is a **software** renderer: custom `QSGGeometryNode`s are silently dropped; + only textures/rects/glyphs render. (This is why Chatter cannot draw ink via + the scene graph — see §3.) + - **`EPFramebuffer`** is the panel singleton (actually `EPFramebufferAcep2` on + this color device), guarded by an **flock** at `/tmp/epframebuffer.lock` — + only one process may hold the panel. Success marker in the journal: + `SWTCON initialized \o/`. A stranded holder → `Failed to lock epframebuffer` / + `Failed to initialize SWTCON` and a blank panel. **Always stop the previous + holder first.** + - Relevant `EPFramebuffer` methods (mangled symbols bound in `src/epfb.h`): + - `instance()` → the singleton. + - `setBuffers(std::tuple, QImage*)` — sets the front/back + buffers; **cross-DSO and interposable** (see §3). + - `swapBuffers(QRect, EPContentType, EPScreenMode, QFlags)` — + pushes a region to the panel. A single explicit call renders **solid** + (no dashed two-pass). Screen modes (from `EPScreenModeItem::Mode`): + `Pen=0, Mono=1, Animation=2, UI=3, Content=4, Sleep=5`. Chatter uses + `Pen` (0) for fast ink and `Content` (4, "full update, STD") for full + refreshes. + - `ghostControl(GhostControlMode)` — the panel's anti-ghosting API. Modes 0/3 + do an immediate full-screen de-ghost via the *region* `swapBuffers`; mode 1 + schedules one. **Chatter does NOT call it** — its region-swap path uses + internal `EPContentMap`/`EPScreenModeMap` members the stock app maintains and + we do not, so calling it corrupts state and hangs after a few calls. Chatter + de-ghosts manually instead (§6.3). + - The panel uses **ACeP color waveforms** (`acep2_lut`, `get_waveform_data`, + software TCON "SWTCON"). Color **ghosting** is real and only cleared by a + full-update waveform driven to the **dark** extreme (§6.3). +- **Pen styles** live in `~/.config/remarkable/xochitl.conf`, key + **`LastWritingTool`** (a `@Variant` `QVariantMap`, readable via `QSettings`): + `LastPen` (tool id, e.g. 21 = Calligraphy, 16 = a fineliner), `LastPenSize` + (category 1/2/3 = thin/thicker/thickest), `LastPenColorCode` (`0xAARRGGBB`). + **xochitl writes this to disk only when you leave a document** (return to the + document list) — not on toolbar taps, and not reliably on an abrupt stop. + +--- + +## 3. Chatter architecture (the core idea) + +**Problem:** the make-or-break requirement is that ink look like the stock pen — +solid, crisp, low-latency. The `epaper` software scene graph **cannot** do this: +geometry nodes don't render, and `QQuickPaintedItem` textures always come out as +the e-ink **dashed two-pass** refresh, regardless of screen mode or antialiasing. + +**Solution — a direct-framebuffer ink pipeline that bypasses the Qt scene:** + +1. **`FbCapture`** (`src/FbCapture.{h,cpp}`) interposes + `EPFramebuffer::setBuffers` **in-process**. The executable is linked + `-Wl,--export-dynamic`, so its definition of that symbol wins the cross-DSO + call from the plugin; we capture the real framebuffer's pixel memory and wrap + it as a `QImage` (`FbCapture::framebuffer()`) with **no copy** (`constBits`). + (Intra-DSO calls like `swapBuffers` are *not* LD_PRELOAD-interposable due to + direct binding — but in-process `--export-dynamic` on `setBuffers` works.) +2. **`InkEngine`** (`src/InkEngine.{h,cpp}`) draws strokes with `QPainter` + **straight into that framebuffer image**, then calls + `EPFramebuffer::swapBuffers` (via `src/epfb.h`) on just the dirty rectangle. + One explicit swap → **solid, single-pass** ink. +3. **Qt Quick renders only the static UI** (the two buttons). The scene never + recomposites over the ink because nothing in it animates. + +This is what gives stock-quality strokes. The reverse-engineering trail and the +dead-ends (geometry nodes, screen modes, antialiasing) are in +[`Chatter_stylus_research.md`](Chatter_stylus_research.md) and the +`Chatter_implementation.md` changelog (v0.7). + +### 3.1 Process / run model + +- Chatter is a single Qt 6 executable run as a **transient systemd service** + (`systemd-run --unit=chatter …`), with `QT_QUICK_BACKEND=epaper`, + `LD_LIBRARY_PATH=/usr/lib/plugins/scenegraph`, `-platform epaper`. +- It **requires xochitl to be stopped** (single DRM master + the panel flock). +- Managed via `systemctl {stop,status} chatter` and `journalctl -u chatter`. + +--- + +## 4. Source components + +``` +src/ + main.cpp Entry point: reads pen style, wires PenDevice → InkEngine, + exposes `ink`/`appControl` to QML, loads the QML UI. + FbCapture.{h,cpp} Interposes EPFramebuffer::setBuffers; exposes the live + framebuffer QImage + FbCapture::ready(). + InkEngine.{h,cpp} THE core. Virtual canvas, ink drawing, finger-erase, buttons + (paint/hit-test/actions), two-finger scroll, de-ghosting. + PenDevice.{h,cpp} evdev reader for the pen (event2); maps to screen coords; + emits strokeStart / strokeMove(pos,pressure,tiltX,tiltY,eraser) + / strokeEnd. + AppControl.{h,cpp} returnToStandard(): transient unit stops chatter, starts xochitl. + epfb.h asm-label bindings to the private EPFramebuffer symbols + (instance, swapBuffers, ghostControl). + EPScreenModeItem.{h,cpp} Legacy/unused: binding to the private screen-mode item + from the scene-graph era. Kept for reference. + InkCanvas.*, Experiment.* Legacy from the Qt-Quick-canvas spike; NOT built. + +qml/Main.qml White Window; the Back/Clear buttons (TopButton); a + MultiPointTouchArea for finger erase (1) and scroll (2). + +tools/ + chatter_launcher.c The return-to-Chatter daemon (4-finger watcher on event3). + grabtest.c Probe whether an input device is EVIOCGRAB-exclusive. + fbdump.cpp LD_PRELOAD framebuffer snapshot (BMP; device has no PNG plugin). + swapshim.cpp, setbufshim.cpp Feasibility shims used during RE. + +scripts/ + build.sh Source the SDK env, cmake build. + deploy-and-run.sh Stop xochitl+chatter, scp, relaunch as a transient unit. + to-chatter.sh Switch standard → Chatter (run by the launcher). + install-launcher.sh Persistently install the launcher unit on the rootfs. + chatter-launcher.service The systemd unit (installed to rootfs). + restore-xochitl.sh Return to the standard GUI. +``` + +--- + +## 5. Input & gesture model + +Two independent input streams, never confused: + +- **Pen (`event2`)** — read by `PenDevice` and delivered to `InkEngine`. The + `epaper` QPA does **not** deliver the pen to Qt, so the pen never triggers QML. +- **Finger (`event3`)** — delivered by the `epaper` QPA to Qt as touch, handled in + QML (`MultiPointTouchArea`, and the buttons' `MouseArea`s). + +Gesture map: + +| Input | Action | Where handled | +|---|---|---| +| Stylus draw | Ink (flip = erase via `BTN_TOOL_RUBBER`) | PenDevice → InkEngine | +| Stylus tap on a button | Button action | InkEngine hit-tests (`m_buttons`) | +| **1 finger** drag | **Erase wipe** (~12 mm; a tap erases nothing) | QML MultiPointTouchArea → `InkEngine::erase*` | +| **2 fingers** drag | **Vertical scroll** | QML → `InkEngine::panBy/panEnd` | +| Finger tap on a button | Button action (gray feedback) | QML MouseArea → `InkEngine::flashButton/activateButton` | +| **4 fingers** hold (~700 ms) | **Return to Chatter** (only while xochitl is front) | `tools/chatter_launcher.c` | + +The MultiPointTouchArea **latches** the gesture type until all fingers lift, so a +two-finger scroll never degrades into an erase when one finger is raised. Erase +requires movement past a ~1.5 mm threshold (tap-safe). + +Buttons (`Back`, `Clear`) are a **single source of truth** in `InkEngine`: QML +registers their geometry (`registerButton`) so the engine can exclude ink, redraw +them after a blit, and hit-test stylus taps. Press feedback (`flashButton`) and +actions (`activateButton`) are drawn directly to the framebuffer (instant, no +scene flashing); both finger and stylus route through them. + +--- + +## 6. The virtual canvas, scrolling, and de-ghosting + +### 6.1 Growable raster canvas +`InkEngine` holds a `QImage` **canvas** larger than the screen (starts 2× tall, +**grows downward** as you write near the bottom). The screen is a **viewport** +into it at vertical offset `m_panY`. Drawing maps screen→canvas (`+m_panY`); a +dirty canvas rect is blitted back to the framebuffer (`blitRegion`) and the +buttons are repainted on top. (Storage model chosen: raster, to preserve the +exact ink quality; trade-off is no crisp zoom-in. Horizontal/zoom are future work.) + +### 6.2 Scrolling +Two-finger drag → `panBy(dy)` (natural: content follows fingers), clamped to the +canvas, fast-blitted per step. `panEnd()` debounces a de-ghost (§6.3). + +### 6.3 De-ghosting (color ACeP ghosting) +Fast `Pen`-waveform swaps leave **color residue** ("faint red duplicate") that +accumulates while scrolling and is **not** cleared by a white redraw — it is +panel retention, cleared only by a full-update waveform driven to **black** +(white/gray do not clear it; this was tested). Chatter's `fullRefresh()`: +fills the screen **black** + full-update swap (`screenMode=Content`), waits +~220 ms, then full-updates the real content. `ghostControl()` would be the +"proper" API but corrupts state (§2), so this manual flash is used. + +To keep the flash from being intrusive: +- It runs **only after scrolling**, **debounced** ~1 s after the last scroll + (not on every finger-lift). +- **Clear has two methods:** if there was **no scrolling** since the last clear + (the common fill-one-screen case) it uses the **gentle fast clear**; if there + **was** scrolling, it does the **black de-ghost** clear. (`m_scrolled` flag.) +- Tunables via env: `CHATTER_FULL_SM` (full-update screen mode, default 4), + `CHATTER_FLASH_GRAY` (flash level 0=black..255; black is what actually clears). + +A black flash is intrinsic to clearing color ghosting (the stock UI flashes on +its full refreshes too); we minimized *when* it happens rather than eliminating it. + +--- + +## 7. Pen-style matching, width calibration, calligraphy + +- Chatter reads `xochitl.conf` `LastWritingTool` at startup (`readPenStyle` in + `main.cpp`) and applies tool type, size, and color — **no style menu in + Chatter**. Because each switch-to-Chatter restarts the process, it re-reads the + current pen. **The flush sequence matters** (§2): set the pen, *use it*, **leave + the document**, then switch to Chatter. +- **Width** is calibrated to measured widths on the 264-PPI panel: + size 3 ≈ 2 mm, size 2 ≈ 1 mm, thinnest ≈ 2 px; pressure ≈ doubles width. +- **Calligraphy** (tool 21) is approximated with a dynamic + direction/pressure/speed/tilt width model. Full detail, measurements, and the + formula: [`Chatter_stylus_research.md`](Chatter_stylus_research.md). + +--- + +## 8. Toggle & the return launcher + +- **Chatter → standard:** `Back` → `InkEngine::backRequested` → + `AppControl::returnToStandard()` → a transient unit stops chatter and starts + xochitl (sequenced so the panel lock is released first). +- **standard → Chatter:** `tools/chatter_launcher.c` runs always as a rootfs + systemd service. It reads `event3` **without grabbing it** (verified possible + via `tools/grabtest.c` — xochitl does not hold an exclusive grab), counts + multitouch slots, and on a **4-finger hold ≥700 ms** while xochitl is the front + app runs `to-chatter.sh`. It reopens the device on any read interruption (sleep/ + wake) so it never dies. +- **Install persistently:** `scripts/install-launcher.sh` (remounts the rootfs rw, + places the unit + its `multi-user.target.wants` symlink under + `/usr/lib/systemd/system`). See the persistence model in §1.1. + +--- + +## 9. Build & deploy + +**Toolchain:** official reMarkable Chiappa SDK `3.27.0.97` (Qt 6.8.2 sysroot, +x86_64 host). Install, then `source environment-setup-cortexa55-remarkable-linux`. + +``` +scripts/build.sh # source SDK env + cmake build -> build/chatter +scripts/deploy-and-run.sh # stop xochitl+chatter, scp, relaunch as transient unit +scripts/restore-xochitl.sh # back to the standard GUI +scripts/install-launcher.sh # one-time (and after each OS update): persist the launcher +``` + +`CMakeLists.txt`: Qt6 Quick app (`qt_add_executable` + `qt_add_qml_module`), links +`libqsgepaper.so`, and crucially `target_link_options(... -Wl,--export-dynamic)` +so the `setBuffers` interposition wins. + +**Device housekeeping:** only one process may hold the panel — always stop the +previous holder. Deploy under `/home/root/chatter`, never the rootfs (except the +launcher unit). The device sleeps and DHCP may reassign its IP on wake; if you +script a reconnect, rescan for the tablet's MAC address (shown on the device's +GPLv3-compliance screen, alongside its IPs). + +--- + +## 10. Status & open items + +**Working on hardware:** solid stock-quality ink matching the selected pen; +finger-wipe erase; whole-page Clear (two methods); stylus eraser; bidirectional +toggle (Back + 4-finger launcher, persistent); vertical scroll on a growable +canvas with debounced de-ghosting; instant button feedback (finger + stylus). + +**Open / future:** +- **Save** (Phase 5) — write a timestamped transcript into a "Chatter" folder in + xochitl's on-disk document format; importance to the user still unconfirmed. + See `Chatter_implementation.md` §6 / §6a. +- **Horizontal scroll & zoom-out overview** — deferred extensions of the canvas. +- **Calligraphy fidelity** — "close enough" approximation; exact nib unknown. +- **Erase / gesture thresholds** — to be tuned with the end user (Matt's field test). +- **OS auto-update** would wipe the rootfs launcher unit (and could break paths) — + disable it on a delivered device. + +--- + +*This document is the technical entry point. When the design changes, update this +file and the relevant component doc; keep `Chatter_implementation.md`'s changelog +as the chronological record.* diff --git a/doc/Chatter_user_guide.md b/doc/Chatter_user_guide.md new file mode 100644 index 0000000..83a06d0 --- /dev/null +++ b/doc/Chatter_user_guide.md @@ -0,0 +1,58 @@ +# Chatter — How to Use It + +Chatter is a simple writing screen. You write with the pen; the person you are +talking with reads what you wrote. That's it. + +## Writing + +- **Write with the pen**, just like on paper. +- The pen's color and thickness are whatever is set in the normal reMarkable + screen. (To change them, see "Changing the pen" below.) + +## The two buttons (top of the screen) + +``` + ┌──────┐ ┌───────┐ + │ Back │ │ Clear │ + └──────┘ └───────┘ +``` + +- **Clear** (top right) — erases everything and gives you a fresh page. +- **Back** (top left) — leaves Chatter and goes to the normal reMarkable screen. + +A button turns **gray** the moment you touch it, so you know it heard you. You +can press a button with **either the pen or your finger**. + +## Erasing + +- **Erase a little:** rub a **finger** back and forth over what you want to + remove, like erasing pencil with your fingertip. +- **Erase everything:** tap **Clear**. +- You can also flip the pen over and use the **eraser end**, like a pencil. + +## More room to write + +- **Two fingers, drag up or down** to scroll, when you want more space than one + screen. The page follows your fingers. +- (After scrolling, the screen may blink once to clean itself up. That's normal.) + +## Going back and forth + +- **Leave Chatter:** tap **Back**. +- **Return to Chatter** from the normal reMarkable screen: place **four fingers + on the screen and hold** for about a second. + +## Changing the pen (color / thickness) + +Chatter copies the pen you last used in the normal reMarkable screen. To change +it: + +1. Tap **Back** to go to the normal screen. +2. Open a page, pick the pen, color, and thickness you want, and **write a little + with it**. +3. **Go back to the document list** (this is what saves your choice). +4. Return to Chatter (four-finger hold). Your new pen is now used. + +--- + +*Questions or problems during setup go to Matt.* diff --git a/doc/developer_mode_screen.md b/doc/developer_mode_screen.md new file mode 100644 index 0000000..c445f05 --- /dev/null +++ b/doc/developer_mode_screen.md @@ -0,0 +1,17 @@ +Developer mode + +This mode is designed for experienced developers to make local software changes via SH access to the system. + +Please note that any changes to software puts the device security at increased risk. + +- Access to root shell is offered in compliance with GPL V3 license. +- Running custom or modified software may cause the device to stop working or adversely affect performance. +- A factory reset is required to enter. +- This will delete all files stored on your paper tablet. +- User data will be cleared once the device is rebooted. +- The device will no longer verify the authenticity of the software, putting your system stability and data security at risk. +- Access any content stored in the cloud by re-pairing with your reMarkable account. + +For instructions on how to exit developer mode, visit support.remarkable.com + + (Accept) diff --git a/doc/initial_Claude_discussion.pdf b/doc/initial_Claude_discussion.pdf new file mode 100644 index 0000000..2387e40 Binary files /dev/null and b/doc/initial_Claude_discussion.pdf differ diff --git a/qml/Main.qml b/qml/Main.qml new file mode 100644 index 0000000..baa65dc --- /dev/null +++ b/qml/Main.qml @@ -0,0 +1,106 @@ +// Chatter — UI only. The white "page" plus the top control bar. Ink is drawn +// directly into the framebuffer by the C++ InkEngine, not by the scene graph, +// so there is no canvas item here. Keep the scene STATIC (only the buttons ever +// repaint, on tap) so it never recomposites over the direct-drawn ink. + +import QtQuick +import QtQuick.Window + +Window { + id: root + width: Screen.width + height: Screen.height + visible: true + color: "white" + + // Finger gestures (the stylus is handled in C++ and is NOT delivered here): + // 1 finger = erase wipe (a tap erases nothing — only movement does) + // 2 fingers = vertical scroll (content follows the fingers) + // The gesture type is latched until all fingers lift, so a 2-finger scroll + // never turns into an erase when one finger is raised. Buttons (z:10) are above. + MultiPointTouchArea { + id: touch + anchors.fill: parent + z: 0 + minimumTouchPoints: 1 + maximumTouchPoints: 2 + touchPoints: [ + TouchPoint { id: tpA }, + TouchPoint { id: tpB } + ] + property int mode: 0 // 0 idle, 1 erase, 2 scroll + property real lastPanY: 0 + + function activeCount() { return (tpA.pressed ? 1 : 0) + (tpB.pressed ? 1 : 0); } + function soloPoint() { return tpA.pressed ? tpA : tpB; } + + function handle() { + var n = activeCount(); + if (n >= 2) { + var cy = (tpA.y + tpB.y) / 2; + if (mode !== 2) { + if (mode === 1) ink.eraseEnd(); + mode = 2; + lastPanY = cy; + } else { + ink.panBy(cy - lastPanY); + lastPanY = cy; + } + } else if (n === 1) { + if (mode === 0) { mode = 1; ink.eraseStart(soloPoint().x, soloPoint().y); } + else if (mode === 1) { ink.eraseMove(soloPoint().x, soloPoint().y); } + // mode === 2 (one finger left after a scroll): stay idle until all up + } else { + if (mode === 1) ink.eraseEnd(); + else if (mode === 2) ink.panEnd(); // full refresh to clear scroll ghosts + mode = 0; + } + } + onPressed: handle() + onUpdated: handle() + onReleased: handle() + } + + // Buttons keep a FIXED appearance in the Qt scene (rendered once); their + // pressed/normal feedback and actions are handled by the ink engine + // (ink.flashButton / ink.activateButton) so both finger AND stylus work and + // updates are instant single-pass (no scene flashing). + component TopButton: Rectangle { + id: btn + height: 90 + // Size to the label so horizontal padding ≈ vertical padding. + width: label.implicitWidth + (height - label.implicitHeight) + color: "white" + border.color: "black" + border.width: 3 + radius: 8 + z: 10 + property alias text: label.text + Text { id: label; anchors.centerIn: parent; anchors.verticalCenterOffset: -2; font.pixelSize: 38 } + // Register geometry once laid out (ink-exclusion, clear-redraw, stylus hit-test). + Component.onCompleted: Qt.callLater(function() { + ink.registerButton(btn.x, btn.y, btn.width, btn.height, btn.text); + }) + MouseArea { + anchors.fill: parent + onPressed: ink.flashButton(btn.x, btn.y, btn.width, btn.height, btn.text, true) + onClicked: ink.activateButton(btn.text) + } + } + + TopButton { + id: backBtn + text: "Back" + anchors.left: parent.left + anchors.top: parent.top + anchors.margins: 24 + } + + TopButton { + id: clearBtn + text: "Clear" + anchors.right: parent.right + anchors.top: parent.top + anchors.margins: 24 + } +} diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 0000000..cafd8ac --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Cross-build Chatter with the reMarkable Paper Pro Move (Chiappa) SDK. +# +# Usage: scripts/build.sh [extra cmake args] +# Override SDK location with CHATTER_SDK=... +set -euo pipefail + +SDK="${CHATTER_SDK:-/home/ack/external/remarkable-sdk/chiappa-3.27.0.97}" +ENV="$SDK/environment-setup-cortexa55-remarkable-linux" +[ -f "$ENV" ] || { echo "SDK env-setup not found: $ENV"; exit 1; } + +cd "$(dirname "$0")/.." + +# Yocto SDKs refuse to operate with LD_LIBRARY_PATH set (Jatke sets it globally). +unset LD_LIBRARY_PATH +# shellcheck disable=SC1090 +. "$ENV" + +cmake -S . -B build "$@" +cmake --build build -j"$(nproc)" +echo "built: build/chatter" + +# The return-to-Chatter launcher is a tiny standalone C daemon (not part of the +# CMake target). Build it here too so install-launcher.sh has its binary. +# $CC carries flags (mcpu, sysroot, …) so it must stay UNquoted. +# shellcheck disable=SC2086 +$CC -O2 -o tools/chatter-launcher tools/chatter_launcher.c +echo "built: tools/chatter-launcher" diff --git a/scripts/chatter-launcher.service b/scripts/chatter-launcher.service new file mode 100644 index 0000000..bf38911 --- /dev/null +++ b/scripts/chatter-launcher.service @@ -0,0 +1,13 @@ +[Unit] +Description=Chatter launcher (multi-finger gesture watcher) +After=home.mount data.mount multi-user.target +# Binary + scripts live on the persistent /home volume. +RequiresMountsFor=/home/root/chatter + +[Service] +ExecStart=/home/root/chatter/chatter-launcher +Restart=always +RestartSec=2 + +[Install] +WantedBy=multi-user.target diff --git a/scripts/deploy-and-run.sh b/scripts/deploy-and-run.sh new file mode 100755 index 0000000..c79d877 --- /dev/null +++ b/scripts/deploy-and-run.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Deploy the chatter binary to the tablet and run it on the e-paper display. +# +# Usage: scripts/deploy-and-run.sh [path-to-binary] +# default binary: build/chatter +# +# Requires: `ssh chatter` working (see doc/tablet_access.md). +# +# IMPORTANT lessons baked in here: +# * The e-paper framebuffer is a singleton guarded by an flock +# (/tmp/epframebuffer.lock). Only ONE process may hold it. A stranded +# instance blanks the panel for everyone, so we always stop the previous +# chatter unit first. +# * Launch as a transient systemd service (systemd-run), NOT an ssh background +# job: that detaches cleanly (no held SSH channel) and is stoppable via +# `systemctl stop chatter`. +# * xochitl owns the display while running, so stop it first (pre-authorized +# during development). Run scripts/restore-xochitl.sh to bring it back. +set -euo pipefail +cd "$(dirname "$0")/.." + +HOST="chatter" +DEST="/home/root/chatter" # under /home, NOT the nearly-full rootfs + +# Use an explicit binary if given; else a fresh local build; else the prebuilt. +BIN="${1:-}" +if [ -z "$BIN" ]; then + if [ -f build/chatter ]; then BIN=build/chatter; else BIN=dist/chatter; fi +fi +[ -f "$BIN" ] || { echo "binary not found: $BIN (build it, or use the prebuilt dist/chatter)"; exit 1; } + +echo ">> stopping any previous chatter + xochitl (release the panel lock)" +ssh "$HOST" 'systemctl stop chatter 2>/dev/null; systemctl reset-failed chatter 2>/dev/null; systemctl stop xochitl 2>/dev/null; sleep 1' + +echo ">> deploying $(basename "$BIN") to $HOST:$DEST" +ssh "$HOST" "mkdir -p $DEST" +scp "$BIN" "$HOST:$DEST/" + +echo ">> launching chatter as a transient systemd service" +# LD_LIBRARY_PATH lets the loader resolve libqsgepaper.so (linked for the private +# EPScreenModeItem type), which lives in the scenegraph plugin dir. +ssh "$HOST" "systemd-run --unit=chatter --collect \ + --setenv=QT_QUICK_BACKEND=epaper \ + --setenv=LD_LIBRARY_PATH=/usr/lib/plugins/scenegraph \ + --working-directory=$DEST \ + $DEST/$(basename "$BIN") -platform epaper" + +sleep 4 +ssh "$HOST" 'echo "chatter: $(systemctl is-active chatter)"' +echo ">> logs: ssh chatter 'journalctl -u chatter -f'" +echo ">> stop: ssh chatter 'systemctl stop chatter' (then scripts/restore-xochitl.sh)" diff --git a/scripts/install-launcher.sh b/scripts/install-launcher.sh new file mode 100644 index 0000000..31b9d54 --- /dev/null +++ b/scripts/install-launcher.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Persistently install the chatter-launcher systemd service ON THE DEVICE. +# +# Why this is needed: on the reMarkable Paper Pro Move, /etc and /run are +# VOLATILE overlays (upperdir on tmpfs) — units placed there are lost on reboot. +# The root partition is read-only but persistent, so we remount it rw and place +# the unit (and its enable symlink) under /usr/lib/systemd/system, which survives +# reboots. (An OS update replaces the rootfs and would require re-running this.) +# +# Uses the prebuilt dist/chatter-launcher if present, else a locally built one. +set -euo pipefail +cd "$(dirname "$0")/.." +HOST="${1:-chatter}" + +LAUNCHER=dist/chatter-launcher +[ -f "$LAUNCHER" ] || LAUNCHER=tools/chatter-launcher +[ -f "$LAUNCHER" ] || { echo "launcher binary not found (dist/ or tools/) — run scripts/build.sh"; exit 1; } + +scp -q "$LAUNCHER" "$HOST:/home/root/chatter/chatter-launcher" +scp -q scripts/to-chatter.sh scripts/chatter-launcher.service "$HOST:/home/root/chatter/" + +ssh "$HOST" ' + set -e + chmod +x /home/root/chatter/chatter-launcher /home/root/chatter/to-chatter.sh + mount -o remount,rw / + cp /home/root/chatter/chatter-launcher.service /usr/lib/systemd/system/chatter-launcher.service + mkdir -p /usr/lib/systemd/system/multi-user.target.wants + ln -sf ../chatter-launcher.service /usr/lib/systemd/system/multi-user.target.wants/chatter-launcher.service + sync + mount -o remount,ro / + systemctl daemon-reload + systemctl restart chatter-launcher.service + echo "launcher: $(systemctl is-active chatter-launcher)" +' diff --git a/scripts/restore-xochitl.sh b/scripts/restore-xochitl.sh new file mode 100755 index 0000000..6924f86 --- /dev/null +++ b/scripts/restore-xochitl.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Bring the standard reMarkable GUI back after running Chatter. +set -euo pipefail +ssh chatter 'systemctl stop chatter 2>/dev/null; systemctl reset-failed chatter 2>/dev/null; systemctl start xochitl' +echo "chatter stopped; xochitl restarted — standard GUI restored." diff --git a/scripts/to-chatter.sh b/scripts/to-chatter.sh new file mode 100644 index 0000000..50d1793 --- /dev/null +++ b/scripts/to-chatter.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# Switch from the standard interface to Chatter (invoked by the launcher daemon +# on the multi-finger gesture). Lives on the device at /home/root/chatter/. +systemctl stop xochitl 2>/dev/null +systemctl stop chatter 2>/dev/null +systemctl reset-failed chatter 2>/dev/null +sleep 1 +systemd-run --unit=chatter --collect \ + --setenv=QT_QUICK_BACKEND=epaper \ + --setenv=LD_LIBRARY_PATH=/usr/lib/plugins/scenegraph \ + --working-directory=/home/root/chatter \ + /home/root/chatter/chatter -platform epaper diff --git a/src/AppControl.cpp b/src/AppControl.cpp new file mode 100644 index 0000000..ad431a7 --- /dev/null +++ b/src/AppControl.cpp @@ -0,0 +1,14 @@ +#include "AppControl.h" + +#include + +void AppControl::returnToStandard() +{ + // Launch a transient unit so the work isn't killed when chatter stops. + QProcess::startDetached( + QStringLiteral("systemd-run"), + {QStringLiteral("--collect"), + QStringLiteral("--unit=chatter-to-standard"), + QStringLiteral("/bin/sh"), QStringLiteral("-c"), + QStringLiteral("systemctl stop chatter; systemctl start xochitl")}); +} diff --git a/src/AppControl.h b/src/AppControl.h new file mode 100644 index 0000000..4fa59e7 --- /dev/null +++ b/src/AppControl.h @@ -0,0 +1,15 @@ +#pragma once +#include + +// Small controller exposed to QML for system actions (e.g. the "Back" button +// returning to the standard reMarkable GUI). +class AppControl : public QObject { + Q_OBJECT +public: + explicit AppControl(QObject *parent = nullptr) : QObject(parent) {} + + // Hand the display back to xochitl: detached so it survives this process + // being stopped, and sequenced (stop chatter -> release panel -> start + // xochitl) to avoid framebuffer-lock contention. + Q_INVOKABLE void returnToStandard(); +}; diff --git a/src/EPScreenModeItem.h b/src/EPScreenModeItem.h new file mode 100644 index 0000000..332b8cc --- /dev/null +++ b/src/EPScreenModeItem.h @@ -0,0 +1,34 @@ +#pragma once +#include + +// EPScreenModeItem is a PRIVATE reMarkable type that lives in libqsgepaper.so +// (the epaper scenegraph backend). It is a QQuickItem that tags the screen region +// it covers with a panel refresh "mode" (Mono / FAST / FastGrayscale ...), which +// selects the e-ink waveform. xochitl wraps its writing area in one of these to +// get crisp, solid pen strokes instead of the default grayscale two-pass refresh. +// +// The SDK ships no header for it, so we declare a minimal interface matching the +// exported symbols (the constructor and the Q_OBJECT staticMetaObject) and link +// against the plugin. We drive its `mode` property via the runtime meta-object, +// so we never hardcode the Mode enum's integer values. +class EPScreenModeItem : public QQuickItem { +public: + // Values confirmed at runtime from the real meta-object. + enum Mode { Pen = 0, Mono = 1, Animation = 2, UI = 3, Content = 4, Sleep = 5 }; + + explicit EPScreenModeItem(QQuickItem *parent = nullptr); + + // Exported by libqsgepaper (_ZN16EPScreenModeItem7setModeENS_4ModeE). The + // `mode` property is read-only, so this is how you actually change it. + void setMode(Mode m); + Mode mode() const; // _ZNK16EPScreenModeItem4modeEv + + // Declared (defined in libqsgepaper.so, symbol exported) so this override is + // the class "key function". That stops our TU from emitting a competing weak + // vtable, so the loader uses libqsgepaper's real vtable — giving the real + // metaObject (with the Mode enum) and real setMode(). + const QMetaObject *metaObject() const override; + + // Resolved at link time from libqsgepaper.so (_ZN16EPScreenModeItem16staticMetaObjectE). + static const QMetaObject staticMetaObject; +}; diff --git a/src/Experiment.cpp b/src/Experiment.cpp new file mode 100644 index 0000000..1eb4bd6 --- /dev/null +++ b/src/Experiment.cpp @@ -0,0 +1,45 @@ +#include "Experiment.h" +#include "InkCanvas.h" +#include "epfb.h" + +#include +#include + +Experiment::Experiment(QObject *parent) : QObject(parent) +{ + // Sweep screenMode Pen(0) and Mono(1) across contentType 0..5, flags 0. + for (int sm : {0, 1}) + for (int ct = 0; ct < 6; ++ct) + m_combos.append({sm, ct, 0}); +} + +void Experiment::start() +{ + auto *t = new QTimer(this); + connect(t, &QTimer::timeout, this, &Experiment::tick); + t->start(3000); + QTimer::singleShot(600, this, &Experiment::tick); +} + +void Experiment::tick() +{ + if (!m_canvas || m_combos.isEmpty()) + return; + m_idx = (m_idx + 1) % m_combos.size(); + const Combo c = m_combos[m_idx]; + + m_canvas->drawTestStroke(); + m_status = QStringLiteral("[%1/%2] screen=%3 content=%4 flags=%5") + .arg(m_idx + 1).arg(m_combos.size()).arg(c.sm).arg(c.ct).arg(c.flags); + emit statusChanged(); + qInfo("EXPERIMENT %s", qPrintable(m_status)); + + // After the scene composites the test pattern, force a refresh of just the + // test region with these parameters. + QTimer::singleShot(350, this, [this, c]() { + const QRect r = m_canvas->testRectQ(); + void *fb = epfb_instance(); + if (fb) + epfb_swapBuffers(fb, {r.left(), r.top(), r.right(), r.bottom()}, c.ct, c.sm, c.flags); + }); +} diff --git a/src/Experiment.h b/src/Experiment.h new file mode 100644 index 0000000..4ead2ff --- /dev/null +++ b/src/Experiment.h @@ -0,0 +1,34 @@ +#pragma once +#include +#include +#include + +class InkCanvas; + +// Screen-mode experiment harness. On a timer it cycles through (screenMode, +// contentType, flags) combinations: for each, it redraws a fixed test pattern +// and then calls EPFramebuffer::swapBuffers on the test region with those +// parameters. The current combination is shown on screen (status), so we can +// see which combination renders the strokes SOLID rather than dashed. +class Experiment : public QObject { + Q_OBJECT + Q_PROPERTY(QString status READ status NOTIFY statusChanged) +public: + explicit Experiment(QObject *parent = nullptr); + + QString status() const { return m_status; } + void setCanvas(InkCanvas *c) { m_canvas = c; } + void start(); + +signals: + void statusChanged(); + +private: + void tick(); + + InkCanvas *m_canvas = nullptr; + QString m_status; + int m_idx = -1; + struct Combo { int sm; int ct; int flags; }; + QVector m_combos; +}; diff --git a/src/FbCapture.cpp b/src/FbCapture.cpp new file mode 100644 index 0000000..9fb2e3d --- /dev/null +++ b/src/FbCapture.cpp @@ -0,0 +1,49 @@ +#define _GNU_SOURCE 1 +#include "FbCapture.h" + +#include +#include +#include + +namespace { +uchar *g_bits = nullptr; +int g_w = 0, g_h = 0, g_bpl = 0, g_fmt = 0; +} + +// Interposes EPFramebuffer::setBuffers(std::tuple, QImage*). +// The tuple is non-trivially-copyable, so passed by reference (a pointer); the +// QImage* is a pointer. We capture buffer A's pixel pointer/geometry, then chain +// to the real implementation. +extern "C" void _ZN13EPFramebuffer10setBuffersESt5tupleIJ6QImageS1_EEPS1_( + void *self, void *tuplePtr, void *imgPtr) +{ + static void (*real)(void *, void *, void *) = nullptr; + if (!real) + real = (void (*)(void *, void *, void *))dlsym( + RTLD_NEXT, "_ZN13EPFramebuffer10setBuffersESt5tupleIJ6QImageS1_EEPS1_"); + + auto *t = reinterpret_cast *>(tuplePtr); + const QImage &a = std::get<0>(*t); + g_bits = const_cast(a.constBits()); + g_w = a.width(); + g_h = a.height(); + g_bpl = a.bytesPerLine(); + g_fmt = int(a.format()); + qInfo("FbCapture: framebuffer %dx%d fmt=%d bpl=%d bits=%p", + g_w, g_h, g_fmt, g_bpl, (void *)g_bits); + + if (real) real(self, tuplePtr, imgPtr); +} + +namespace FbCapture { + +bool ready() { return g_bits != nullptr; } + +QImage framebuffer() +{ + if (!g_bits) + return QImage(); + return QImage(g_bits, g_w, g_h, g_bpl, QImage::Format(g_fmt)); +} + +} // namespace FbCapture diff --git a/src/FbCapture.h b/src/FbCapture.h new file mode 100644 index 0000000..bb76387 --- /dev/null +++ b/src/FbCapture.h @@ -0,0 +1,12 @@ +#pragma once +#include + +// Captures the e-paper framebuffer's backing image by interposing +// EPFramebuffer::setBuffers (defined in FbCapture.cpp). The executable is linked +// with -Wl,--export-dynamic so its symbol wins the cross-DSO call from the +// epaper platform plugin into libqsgepaper. The ink engine then draws strokes +// straight into the framebuffer memory and refreshes with an explicit swap. +namespace FbCapture { +bool ready(); +QImage framebuffer(); // a QImage wrapping the real buffer (no copy) — draw into it +} diff --git a/src/InkCanvas.cpp b/src/InkCanvas.cpp new file mode 100644 index 0000000..b9bf391 --- /dev/null +++ b/src/InkCanvas.cpp @@ -0,0 +1,157 @@ +#include "InkCanvas.h" +#include "epfb.h" + +#include +#include +#include + +InkCanvas::InkCanvas(QQuickItem *parent) : QQuickPaintedItem(parent) +{ + setRenderTarget(QQuickPaintedItem::Image); + // Crisp 1-bit black so the Mono waveform renders single-pass (no gray dither). + setAntialiasing(false); +} + +void InkCanvas::ensureBuffer(const QSize &size) +{ + if (size.isEmpty() || m_buffer.size() == size) + return; + QImage img(size, QImage::Format_RGB32); + img.fill(Qt::white); + if (!m_buffer.isNull()) { + QPainter p(&img); + p.drawImage(0, 0, m_buffer); + } + m_buffer = img; +} + +void InkCanvas::geometryChange(const QRectF &newGeometry, const QRectF &oldGeometry) +{ + QQuickPaintedItem::geometryChange(newGeometry, oldGeometry); + ensureBuffer(newGeometry.size().toSize()); + update(); +} + +void InkCanvas::paint(QPainter *painter) +{ + if (!m_buffer.isNull()) + painter->drawImage(0, 0, m_buffer); +} + +void InkCanvas::strokeStart(QPointF p) +{ + ensureBuffer(boundingRect().size().toSize()); + m_last = p; + m_drawing = true; +} + +void InkCanvas::strokeMove(QPointF p, qreal pressure, bool eraser) +{ + if (m_buffer.isNull()) + ensureBuffer(boundingRect().size().toSize()); + if (m_buffer.isNull()) + return; + if (!m_drawing) { + m_last = p; + m_drawing = true; + } + + qreal w; + QPainter painter(&m_buffer); + painter.setRenderHint(QPainter::Antialiasing, false); + if (eraser) { + w = 40.0; + painter.setPen(QPen(Qt::white, w, Qt::SolidLine, Qt::RoundCap, Qt::RoundJoin)); + } else { + w = m_baseWidth * (0.6 + 0.4 * pressure); + painter.setPen(QPen(m_inkColor, w, Qt::SolidLine, Qt::RoundCap, Qt::RoundJoin)); + } + painter.drawLine(m_last, p); + painter.end(); + + QRectF dirty = QRectF(m_last, p).normalized().adjusted(-w - 1, -w - 1, w + 1, w + 1); + m_last = p; + update(dirty.toRect()); + queueRefresh(dirty.toRect()); + scheduleFlush(); +} + +void InkCanvas::strokeEnd() +{ + m_drawing = false; + scheduleFlush(); +} + +void InkCanvas::queueRefresh(const QRect &r) +{ + m_pending = m_pending.isValid() ? m_pending.united(r) : r; +} + +// Coalesce: schedule one delayed flush (GUI thread). The delay lets the scene +// composite the new ink into the framebuffer before we re-push it solid; the +// throttle keeps us from flooding the SWTCON. +void InkCanvas::scheduleFlush() +{ + if (m_flushScheduled) + return; + m_flushScheduled = true; + QTimer::singleShot(40, this, [this]() { + m_flushScheduled = false; + flushRefresh(); + }); +} + +// GUI thread: re-push the accumulated region with a single explicit swapBuffers, +// which renders SOLID (replacing the scene's dashed two-pass). Params are not +// critical — screen=Pen(0), content=0, flags=0. +void InkCanvas::flushRefresh() +{ + const QRect r = m_pending; + m_pending = QRect(); + if (!r.isValid()) + return; + if (void *fb = epfb_instance()) + epfb_swapBuffers(fb, {r.left(), r.top(), r.right(), r.bottom()}, 0, 0, 0); + // If more ink arrived while we were flushing, keep cleaning up. + if (m_drawing) + scheduleFlush(); +} + +void InkCanvas::clearPage() +{ + if (m_buffer.isNull()) + return; + m_buffer.fill(Qt::white); + update(); + queueRefresh(boundingRect().toRect()); +} + +QRect InkCanvas::testRectQ() const +{ + return QRect(80, 300, 780, 280); +} + +void InkCanvas::drawTestStroke() +{ + ensureBuffer(boundingRect().size().toSize()); + if (m_buffer.isNull()) + return; + m_buffer.fill(Qt::white); + + QPainter p(&m_buffer); + p.setRenderHint(QPainter::Antialiasing, false); + p.setPen(QPen(Qt::black, m_baseWidth, Qt::SolidLine, Qt::RoundCap, Qt::RoundJoin)); + + const QRect r = testRectQ(); + // A zigzag plus the two diagonals — a recognizable, stroke-like pattern. + QPolygon zig; + int i = 0; + for (int x = r.left(); x <= r.right(); x += 70, ++i) + zig << QPoint(x, (i % 2 == 0) ? r.top() + 40 : r.bottom() - 40); + p.drawPolyline(zig); + p.drawLine(r.topLeft(), r.bottomRight()); + p.drawLine(r.bottomLeft(), r.topRight()); + p.end(); + + update(); +} diff --git a/src/InkCanvas.h b/src/InkCanvas.h new file mode 100644 index 0000000..df26861 --- /dev/null +++ b/src/InkCanvas.h @@ -0,0 +1,52 @@ +#pragma once +#include +#include +#include +#include +#include +#include + +// Drawing surface. Renders strokes via QPainter into an offscreen QImage that is +// blitted as a texture (QSGPainterNode) — the only stroke-rendering path the +// device's *software* epaper scene-graph backend actually draws (it silently +// drops custom QSGGeometryNodes). The trade-off is the e-ink's grayscale +// two-pass ("dashed→fill") refresh for texture updates; eliminating that needs a +// framebuffer-direct pen path (see doc). +class InkCanvas : public QQuickPaintedItem { + Q_OBJECT +public: + explicit InkCanvas(QQuickItem *parent = nullptr); + + void paint(QPainter *painter) override; + +public slots: + void strokeStart(QPointF p); + void strokeMove(QPointF p, qreal pressure, bool eraser); + void strokeEnd(); + void clearPage(); + + // Experiment harness: clear + draw a fixed recognizable test pattern. + void drawTestStroke(); + QRect testRectQ() const; + + void setInkColor(const QColor &c) { m_inkColor = c; } + void setBaseWidth(qreal w) { m_baseWidth = w; } + +protected: + void geometryChange(const QRectF &newGeometry, const QRectF &oldGeometry) override; + +private: + void ensureBuffer(const QSize &size); + void queueRefresh(const QRect &r); // accumulate a region needing a solid swap + void scheduleFlush(); // coalesce; fire flushRefresh once, soon + void flushRefresh(); // GUI thread: explicit solid swapBuffers + + QImage m_buffer; + QPointF m_last; + bool m_drawing = false; + QColor m_inkColor = Qt::black; + qreal m_baseWidth = 8.0; + + QRect m_pending; // region awaiting an explicit solid refresh + bool m_flushScheduled = false; +}; diff --git a/src/InkEngine.cpp b/src/InkEngine.cpp new file mode 100644 index 0000000..9fe5297 --- /dev/null +++ b/src/InkEngine.cpp @@ -0,0 +1,330 @@ +#include "InkEngine.h" +#include "FbCapture.h" +#include "epfb.h" + +#include +#include +#include +#include +#include +#include +#include + +InkEngine::InkEngine(QObject *parent) : QObject(parent) +{ + m_timer.start(); + bool ok = false; + const int v = qEnvironmentVariableIntValue("CHATTER_FULL_SM", &ok); + if (ok) m_fullSm = v; + const int g = qEnvironmentVariableIntValue("CHATTER_FLASH_GRAY", &ok); + if (ok) m_flashGray = qBound(0, g, 255); + + // De-ghost once the user settles, not on every scroll-lift. + m_deghostTimer.setSingleShot(true); + connect(&m_deghostTimer, &QTimer::timeout, this, [this] { renderAll(true); }); +} + +void InkEngine::pushRect(const QRect &r) +{ + if (!r.isValid()) + return; + if (void *fb = epfb_instance()) + epfb_swapBuffers(fb, {r.left(), r.top(), r.right(), r.bottom()}, 0, 0, 0); +} + +// Manual de-ghost: drive the whole panel to black with a full update, then a +// full update to the real content. The black full-update cycles every particle +// (incl. the color ones that leave the red residue). Uses only our reliable +// QRect swapBuffers — NOT ghostControl(), whose region-swap corrupts internal +// state we don't maintain. Only called on scroll-end / clear (rare), so the +// ~0.4 s flash is acceptable. +void InkEngine::fullRefresh() +{ + if (!FbCapture::ready() || m_canvas.isNull()) + return; + QImage fb = FbCapture::framebuffer(); + const QRect full(0, 0, m_screenW, m_screenH); + const EpRect er{full.left(), full.top(), full.right(), full.bottom()}; + void *inst = epfb_instance(); + + { QPainter p(&fb); p.fillRect(full, QColor(m_flashGray, m_flashGray, m_flashGray)); } + if (inst) epfb_swapBuffers(inst, er, 0, m_fullSm, 0); + QThread::msleep(220); + + { QPainter p(&fb); + p.drawImage(full.topLeft(), m_canvas, full.translated(0, m_panY)); + for (const ButtonDef &b : m_buttons) paintButton(p, b.rect, b.text, false); } + if (inst) epfb_swapBuffers(inst, er, 0, m_fullSm, 0); +} + +// ---- virtual canvas ------------------------------------------------------ + +void InkEngine::ensureCanvas() +{ + if (!m_canvas.isNull() || !FbCapture::ready()) + return; + const QImage fb = FbCapture::framebuffer(); + m_screenW = fb.width(); + m_screenH = fb.height(); + m_canvas = QImage(m_screenW, m_screenH * 2, QImage::Format_RGB32); + m_canvas.fill(Qt::white); +} + +void InkEngine::growIfNeeded(int canvasBottomY) +{ + if (m_canvas.isNull() || canvasBottomY < m_canvas.height() - m_screenH / 2) + return; + QImage bigger(m_screenW, m_canvas.height() + m_screenH, QImage::Format_RGB32); + bigger.fill(Qt::white); + QPainter p(&bigger); + p.drawImage(0, 0, m_canvas); + p.end(); + m_canvas = bigger; +} + +// Copy a screen-space rect from the canvas window into the framebuffer, keep the +// buttons on top, and refresh just that rect. +void InkEngine::blitRegion(const QRect &screenRect, bool full) +{ + if (!FbCapture::ready() || m_canvas.isNull()) + return; + if (full) { // full-screen de-ghosting refresh (draws content itself) + fullRefresh(); + return; + } + const QRect sr = screenRect.intersected(QRect(0, 0, m_screenW, m_screenH)); + if (sr.isEmpty()) + return; + QImage fb = FbCapture::framebuffer(); + QPainter p(&fb); + p.drawImage(sr.topLeft(), m_canvas, sr.translated(0, m_panY)); // canvas -> screen + for (const ButtonDef &b : m_buttons) + if (b.rect.intersects(sr)) + paintButton(p, b.rect, b.text, false); + p.end(); + pushRect(sr); +} + +void InkEngine::renderAll(bool full) { blitRegion(QRect(0, 0, m_screenW, m_screenH), full); } + +void InkEngine::panBy(qreal dyScreen) +{ + ensureCanvas(); + if (m_canvas.isNull()) + return; + const int maxPan = qMax(0, m_canvas.height() - m_screenH); + const int np = qBound(0, m_panY - qRound(dyScreen), maxPan); // natural scroll + if (np == m_panY) + return; + m_deghostTimer.stop(); // still moving — postpone the de-ghost + m_scrolled = true; + m_panY = np; + renderAll(false); // fast during the drag +} + +// Scrolling stopped: de-ghost once after a short settle (debounced), so rapid +// scrolling doesn't flash on every lift. +void InkEngine::panEnd() { m_deghostTimer.start(1000); } + +// ---- buttons ------------------------------------------------------------- + +void InkEngine::registerButton(int x, int y, int w, int h, const QString &text) +{ + const QRect r(x, y, w, h); + m_buttons.append({r, text}); + m_uiMask = m_uiMask.united(QRegion(r)); +} + +int InkEngine::buttonAt(const QPointF &p) const +{ + for (int i = 0; i < m_buttons.size(); ++i) + if (m_buttons[i].rect.contains(p.toPoint())) + return i; + return -1; +} + +void InkEngine::paintButton(QPainter &p, const QRect &rect, const QString &text, bool pressed) const +{ + p.setRenderHint(QPainter::Antialiasing, true); + const QRectF box(rect.x() + 1.5, rect.y() + 1.5, rect.width() - 3.0, rect.height() - 3.0); + p.setPen(QPen(Qt::black, 3)); + p.setBrush(pressed ? QColor(0x9e, 0x9e, 0x9e) : QColor(Qt::white)); + p.drawRoundedRect(box, 8, 8); + p.setPen(Qt::black); + QFont f = p.font(); + f.setPixelSize(38); + p.setFont(f); + p.drawText(rect.translated(0, -2), Qt::AlignCenter, text); // keep in sync with Main.qml offset +} + +void InkEngine::flashButton(int x, int y, int w, int h, const QString &text, bool pressed) +{ + if (!FbCapture::ready()) + return; + QImage fb = FbCapture::framebuffer(); + QPainter p(&fb); + paintButton(p, QRect(x, y, w, h), text, pressed); + p.end(); + pushRect(QRect(x - 2, y - 2, w + 4, h + 4)); +} + +void InkEngine::activateButton(const QString &text) +{ + if (text == QLatin1String("Clear")) + clearPage(); // also repaints the buttons (reverting any pressed state) + else if (text == QLatin1String("Back")) + emit backRequested(); // wired to AppControl::returnToStandard +} + +// ---- finger-wipe erase --------------------------------------------------- + +void InkEngine::eraseStart(qreal x, qreal y) +{ + ensureCanvas(); + m_erasePress = QPointF(x, y + m_panY); // canvas coords + m_eraseLast = m_erasePress; + m_erasing = false; // wait for real movement (tap-safe) +} + +void InkEngine::eraseMove(qreal x, qreal y) +{ + if (!FbCapture::ready()) + return; + ensureCanvas(); + if (m_canvas.isNull()) + return; + const QPointF cp(x, y + m_panY); // canvas coords + if (!m_erasing) { + // Ignore tap jitter; only a deliberate wipe (moved > ~1.5 mm) erases. + if (QLineF(m_erasePress, cp).length() < 16.0) + return; + m_erasing = true; + m_eraseLast = m_erasePress; // erase the whole wipe, from first contact + } + constexpr qreal w = 124.0; // finger-wipe width (~12 mm @ 264 PPI) + QPainter painter(&m_canvas); + painter.setPen(QPen(Qt::white, w, Qt::SolidLine, Qt::RoundCap, Qt::RoundJoin)); + painter.drawLine(m_eraseLast, cp); + painter.end(); + const QRect dirty = QRectF(m_eraseLast, cp).normalized() + .adjusted(-w - 2, -w - 2, w + 2, w + 2) + .toRect(); + m_eraseLast = cp; + blitRegion(dirty.translated(0, -m_panY)); +} + +void InkEngine::eraseEnd() { m_erasing = false; } + +// ---- ink ----------------------------------------------------------------- + +void InkEngine::strokeStart(QPointF p) +{ + m_btnDown = buttonAt(p); // screen coords + if (m_btnDown >= 0) { + const ButtonDef &b = m_buttons[m_btnDown]; + flashButton(b.rect.x(), b.rect.y(), b.rect.width(), b.rect.height(), b.text, true); + m_drawing = false; + return; + } + ensureCanvas(); + m_last = p + QPointF(0, m_panY); // canvas coords + m_drawing = true; + m_lastT = m_timer.elapsed(); +} + +void InkEngine::strokeMove(QPointF p, qreal pressure, qreal tiltX, qreal tiltY, bool eraser) +{ + if (!FbCapture::ready()) + return; + + // In a stylus button-tap: if the pen slides off the button, cancel it. + if (m_btnDown >= 0) { + if (buttonAt(p) != m_btnDown) { + const ButtonDef &b = m_buttons[m_btnDown]; + flashButton(b.rect.x(), b.rect.y(), b.rect.width(), b.rect.height(), b.text, false); + m_btnDown = -1; + } + return; // never ink during a button tap + } + + ensureCanvas(); + if (m_canvas.isNull()) + return; + const QPointF cp = p + QPointF(0, m_panY); // canvas coords + if (!m_drawing) { + m_last = cp; + m_drawing = true; + m_lastT = m_timer.elapsed(); + } + + QPainter painter(&m_canvas); + painter.setRenderHint(QPainter::Antialiasing, false); + + const qreal span = m_maxWidth - m_minWidth; + qreal extent; // half-width used to size the dirty rect + if (eraser) { + extent = 40.0; + painter.setPen(QPen(Qt::white, 40.0, Qt::SolidLine, Qt::RoundCap, Qt::RoundJoin)); + painter.drawLine(m_last, cp); + } else if (m_penType == 21) { + // reMarkable Calligraphy is a DYNAMIC model (per reMarkable docs + + // observed behavior): thicker on downstrokes, with more pressure, when + // slow, and when the stylus is tilted — not a fixed geometric nib. + const qreal len = QLineF(m_last, cp).length(); + const qreal dyN = len > 1e-3 ? (cp.y() - m_last.y()) / len : 0.0; // +1 down, -1 up + const qreal dirF = 0.5 + 0.5 * dyN; // 0 (up) .. 1 (down) + + const qint64 now = m_timer.elapsed(); + const qreal dt = qMax(1, now - m_lastT); + m_lastT = now; + const qreal speed = len / dt; // px / ms + const qreal speedF = 1.0 - qBound(0.0, speed / 3.0, 1.0); // slow -> 1 + + const qreal tiltMag = qBound(0.0, std::hypot(tiltX, tiltY), 1.0); + + const qreal f = 0.25 * dirF + 0.50 * pressure + 0.12 * speedF + 0.13 * tiltMag; + const qreal w = m_minWidth + span * qBound(0.0, f, 1.0); + painter.setPen(QPen(m_color, w, Qt::SolidLine, Qt::RoundCap, Qt::RoundJoin)); + painter.drawLine(m_last, cp); + extent = w; + } else { + const qreal w = m_minWidth + span * qBound(0.0, 0.5 + 0.5 * pressure, 1.0); + painter.setPen(QPen(m_color, w, Qt::SolidLine, Qt::RoundCap, Qt::RoundJoin)); + painter.drawLine(m_last, cp); + extent = w; + } + painter.end(); + + const QRect dirty = QRectF(m_last, cp).normalized() + .adjusted(-extent - 2, -extent - 2, extent + 2, extent + 2) + .toRect(); + growIfNeeded(qMax(m_last.y(), cp.y()) + extent + 2); // may reallocate the canvas + m_last = cp; + blitRegion(dirty.translated(0, -m_panY)); +} + +void InkEngine::strokeEnd() +{ + if (m_btnDown >= 0) { + const QString text = m_buttons[m_btnDown].text; + m_btnDown = -1; + activateButton(text); // Clear repaints (reverts); Back leaves it pressed + return; + } + m_drawing = false; +} + +void InkEngine::clearPage() +{ + if (!FbCapture::ready()) + return; + ensureCanvas(); + m_deghostTimer.stop(); + if (!m_canvas.isNull()) + m_canvas.fill(Qt::white); + m_panY = 0; + // Two Clear methods: if the user scrolled, de-ghost (black flash); otherwise + // (the common fill-one-screen-then-clear case) use the gentle fast clear. + renderAll(m_scrolled); + m_scrolled = false; +} diff --git a/src/InkEngine.h b/src/InkEngine.h new file mode 100644 index 0000000..a9cc5ed --- /dev/null +++ b/src/InkEngine.h @@ -0,0 +1,96 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +class QPainter; + +// Draws ink strokes DIRECTLY into the e-paper framebuffer (captured via +// FbCapture) and refreshes each segment with an explicit EPFramebuffer swap, +// which renders solid/single-pass — the reMarkable fast-pen path. No Qt scene +// graph in the ink loop. Driven by PenDevice signals. Also owns the top-bar +// button visuals/hit-testing so both finger and stylus can drive them. +class InkEngine : public QObject { + Q_OBJECT +public: + explicit InkEngine(QObject *parent = nullptr); + + void setInkColor(const QColor &c) { m_color = c; } + void setWidthRange(qreal minPx, qreal maxPx) { m_minWidth = minPx; m_maxWidth = maxPx; } + void setPenType(int t) { m_penType = t; } + + // QML registers each top-bar button's geometry + label (single source of + // truth): used to exclude ink, redraw on clear, and detect stylus taps. + Q_INVOKABLE void registerButton(int x, int y, int w, int h, const QString &text); + // Show a button's pressed/normal state directly in the framebuffer (single + // clean swap, no scene flashing). Called from QML on finger press. + Q_INVOKABLE void flashButton(int x, int y, int w, int h, const QString &text, bool pressed); + // Run a button's action: Clear here, Back via the backRequested() signal. + Q_INVOKABLE void activateButton(const QString &text); + + // Finger-wipe erase (finger touches arrive via QML; the stylus does not). + // Erases ink along the finger path; a tap (no movement) erases nothing. + Q_INVOKABLE void eraseStart(qreal x, qreal y); + Q_INVOKABLE void eraseMove(qreal x, qreal y); + Q_INVOKABLE void eraseEnd(); + + // Two-finger vertical scroll. dyScreen = centroid movement since last call + // (natural: content follows the fingers). Grows the canvas downward as needed. + Q_INVOKABLE void panBy(qreal dyScreen); + // Called when the scroll gesture ends: one full-refresh pass to clear the + // ghosting that the fast per-step swaps leave on the color panel. + Q_INVOKABLE void panEnd(); + +signals: + void backRequested(); + +public slots: + void strokeStart(QPointF p); + void strokeMove(QPointF p, qreal pressure, qreal tiltX, qreal tiltY, bool eraser); + void strokeEnd(); + void clearPage(); + +private: + struct ButtonDef { QRect rect; QString text; }; + + void pushRect(const QRect &r); // fast (Pen waveform) — ink strokes + void fullRefresh(); // ghostControl() de-ghost (scroll/clear) + void paintButton(QPainter &p, const QRect &rect, const QString &text, bool pressed) const; + int buttonAt(const QPointF &p) const; // index into m_buttons, or -1 + + void ensureCanvas(); // lazily create the virtual canvas + void growIfNeeded(int canvasBottomY); // extend the canvas downward + void blitRegion(const QRect &screenRect, bool full = false); // canvas -> framebuffer + void renderAll(bool full = false); // re-blit the whole viewport (on pan) + + QPointF m_last; + bool m_drawing = false; + QColor m_color = Qt::black; + qreal m_minWidth = 2.0; // hairline (px) + qreal m_maxWidth = 14.0; // size's max (px), from DPI + size category + int m_penType = 2; // reMarkable tool id (21 = Calligraphy) + QRegion m_uiMask; // button regions excluded from ink/clear + QVector m_buttons; + int m_btnDown = -1; // button under an in-progress stylus tap, or -1 + QPointF m_erasePress; // finger-wipe erase state + QPointF m_eraseLast; + bool m_erasing = false; // true once movement passed the tap threshold + + QImage m_canvas; // virtual document (>= screen; grows downward) + int m_panY = 0; // top of the viewport within the canvas + int m_screenW = 0, m_screenH = 0; + int m_fullSm = 4; // screen mode for full updates (env CHATTER_FULL_SM) + int m_flashGray = 0; // de-ghost flash level 0=black..255 (env CHATTER_FLASH_GRAY) + QTimer m_deghostTimer; // debounce: de-ghost once ~1s after scrolling stops + bool m_scrolled = false; // any scrolling since last clear? -> Clear de-ghosts + + QElapsedTimer m_timer; + qint64 m_lastT = 0; +}; diff --git a/src/PenDevice.cpp b/src/PenDevice.cpp new file mode 100644 index 0000000..5fbd26b --- /dev/null +++ b/src/PenDevice.cpp @@ -0,0 +1,80 @@ +#include "PenDevice.h" + +#include + +#include +#include +#include +#include + +PenDevice::PenDevice(int screenWidth, int screenHeight, + const QString &devicePath, QObject *parent) + : QObject(parent), m_screenW(screenWidth), m_screenH(screenHeight) +{ + m_fd = ::open(devicePath.toLocal8Bit().constData(), O_RDONLY | O_NONBLOCK); + if (m_fd < 0) { + qWarning("PenDevice: cannot open %s", qPrintable(devicePath)); + return; + } + + input_absinfo ai; + if (ioctl(m_fd, EVIOCGABS(ABS_X), &ai) == 0) { m_xMin = ai.minimum; m_xMax = ai.maximum; } + if (ioctl(m_fd, EVIOCGABS(ABS_Y), &ai) == 0) { m_yMin = ai.minimum; m_yMax = ai.maximum; } + if (ioctl(m_fd, EVIOCGABS(ABS_PRESSURE), &ai) == 0) { m_pMin = ai.minimum; m_pMax = ai.maximum; } + + m_notifier = new QSocketNotifier(m_fd, QSocketNotifier::Read, this); + connect(m_notifier, &QSocketNotifier::activated, this, &PenDevice::readEvents); +} + +PenDevice::~PenDevice() +{ + if (m_fd >= 0) + ::close(m_fd); +} + +QPointF PenDevice::mapToScreen() const +{ + double nx = (m_xMax > m_xMin) ? double(m_rawX - m_xMin) / double(m_xMax - m_xMin) : 0.0; + double ny = (m_yMax > m_yMin) ? double(m_rawY - m_yMin) / double(m_yMax - m_yMin) : 0.0; + if (m_invertX) nx = 1.0 - nx; + if (m_invertY) ny = 1.0 - ny; + if (m_swapXY) { double t = nx; nx = ny; ny = t; } + return QPointF(nx * m_screenW, ny * m_screenH); +} + +void PenDevice::readEvents() +{ + struct input_event ev; + ssize_t n; + while ((n = ::read(m_fd, &ev, sizeof(ev))) == sizeof(ev)) { + switch (ev.type) { + case EV_ABS: + if (ev.code == ABS_X) m_rawX = ev.value; + else if (ev.code == ABS_Y) m_rawY = ev.value; + else if (ev.code == ABS_PRESSURE) m_rawP = ev.value; + else if (ev.code == ABS_TILT_X) m_rawTiltX = ev.value; + else if (ev.code == ABS_TILT_Y) m_rawTiltY = ev.value; + break; + case EV_KEY: + if (ev.code == BTN_TOUCH) { + m_touching = (ev.value != 0); + if (m_touching) emit strokeStart(mapToScreen()); + else emit strokeEnd(); + } else if (ev.code == BTN_TOOL_RUBBER) { + m_eraser = (ev.value != 0); + } + break; + case EV_SYN: + if (ev.code == SYN_REPORT && m_touching) { + double pressure = (m_pMax > m_pMin) + ? double(m_rawP - m_pMin) / double(m_pMax - m_pMin) : 1.0; + const double tiltX = m_rawTiltX / 9000.0; // -1 .. 1 + const double tiltY = m_rawTiltY / 9000.0; + emit strokeMove(mapToScreen(), pressure, tiltX, tiltY, m_eraser); + } + break; + default: + break; + } + } +} diff --git a/src/PenDevice.h b/src/PenDevice.h new file mode 100644 index 0000000..d021b63 --- /dev/null +++ b/src/PenDevice.h @@ -0,0 +1,52 @@ +#pragma once +#include +#include +#include + +class QSocketNotifier; + +// Reads the reMarkable stylus (Elan marker, /dev/input/event2) directly from +// evdev. The epaper Qt platform delivers finger touch (event3) but NOT the pen, +// so Chatter handles the stylus itself. Maps raw digitizer coordinates to screen +// pixels and emits stroke signals for the ink canvas. The eraser end of the +// Marker Plus arrives as BTN_TOOL_RUBBER. +class PenDevice : public QObject { + Q_OBJECT +public: + explicit PenDevice(int screenWidth, int screenHeight, + const QString &devicePath = QStringLiteral("/dev/input/event2"), + QObject *parent = nullptr); + ~PenDevice() override; + + bool isOpen() const { return m_fd >= 0; } + +signals: + void strokeStart(QPointF pos); + void strokeMove(QPointF pos, qreal pressure, qreal tiltX, qreal tiltY, bool eraser); + void strokeEnd(); + +private: + void readEvents(); + QPointF mapToScreen() const; + + int m_fd = -1; + QSocketNotifier *m_notifier = nullptr; + int m_screenW; + int m_screenH; + + // Device axis ranges (filled from EVIOCGABS; defaults are the measured ones). + int m_xMin = 0, m_xMax = 6760; + int m_yMin = 0, m_yMax = 11960; + int m_pMin = 0, m_pMax = 4096; + + // Orientation — flip these if a test stroke comes out mirrored/rotated. + bool m_invertX = false; + bool m_invertY = false; + bool m_swapXY = false; + + // Current sample state. + int m_rawX = 0, m_rawY = 0, m_rawP = 0; + int m_rawTiltX = 0, m_rawTiltY = 0; // ABS_TILT_X/Y, range +/-9000 + bool m_touching = false; + bool m_eraser = false; +}; diff --git a/src/epfb.h b/src/epfb.h new file mode 100644 index 0000000..351442b --- /dev/null +++ b/src/epfb.h @@ -0,0 +1,19 @@ +#pragma once + +// Direct access to the private EPFramebuffer singleton in libqsgepaper.so, for +// the screen-mode experiment. We bind to the exact mangled symbols via asm +// labels (no class declaration, so no vtable issues). QRect is passed by value +// as its raw {left,top,right,bottom} layout; the enums/QFlags are int-sized. + +struct EpRect { int x1, y1, x2, y2; }; // == QRect internal layout + +void *epfb_instance() asm("_ZN13EPFramebuffer8instanceEv"); + +void epfb_swapBuffers(void *self, EpRect r, int contentType, int screenMode, int flags) + asm("_ZN13EPFramebuffer11swapBuffersE5QRect13EPContentType12EPScreenMode6QFlagsINS_10UpdateFlagEE"); + +// Anti-ghost full refresh. The color panel (EPFramebufferAcep2) overrides this; +// modes 0/3 do an immediate full-screen de-ghosting refresh of the current +// framebuffer, mode 1 schedules one. Call with the instance() pointer. +void epfb_ghostControl(void *self, int mode) + asm("_ZN18EPFramebufferAcep212ghostControlEN13EPFramebuffer16GhostControlModeE"); diff --git a/src/main.cpp b/src/main.cpp new file mode 100644 index 0000000..3429d1f --- /dev/null +++ b/src/main.cpp @@ -0,0 +1,89 @@ +// Chatter — Qt Quick app with a direct-framebuffer ink pipeline. +// Pen strokes are drawn straight into the e-paper framebuffer (captured by +// FbCapture) and refreshed with an explicit solid swap (InkEngine). Qt Quick is +// used only for the static UI (Back / Clear). This is what gives stock-quality +// solid, low-latency ink — the Qt scene graph can't. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "PenDevice.h" +#include "AppControl.h" +#include "InkEngine.h" +#include "FbCapture.h" // links the setBuffers interposer into the executable + +struct PenStyle { + int type = 2; + qreal size = 2.0; + QColor color = Qt::black; +}; + +// Read the pen the user last selected in the standard GUI (xochitl.conf, +// [General]/LastWritingTool — flushed when xochitl stops, i.e. on Back). +static PenStyle readPenStyle() +{ + PenStyle ps; + QSettings xs(QStringLiteral("/home/root/.config/remarkable/xochitl.conf"), + QSettings::IniFormat); + const QVariantMap t = xs.value(QStringLiteral("LastWritingTool")).toMap(); + if (t.isEmpty()) { + qWarning("readPenStyle: LastWritingTool unreadable; using defaults"); + return ps; + } + ps.type = t.value("LastPen").toInt(); + ps.size = t.value("LastPenSize").toDouble(); + const uint code = t.value("LastPenColorCode").toUInt(); + if (code != 0) + ps.color = QColor::fromRgba(QRgb(code)); + qInfo("readPenStyle: pen=%d size=%.2f color=%s", ps.type, ps.size, + qPrintable(ps.color.name(QColor::HexArgb))); + return ps; +} + +int main(int argc, char *argv[]) +{ + QGuiApplication app(argc, argv); + const PenStyle penStyle = readPenStyle(); + + const QSize screen = app.primaryScreen()->geometry().size(); + PenDevice pen(screen.width(), screen.height()); + AppControl appControl; + + InkEngine ink; + ink.setInkColor(penStyle.color); + // reMarkable exposes only the size CATEGORY (LastPenSize 1/2/3 = + // thin/thicker/thickest). Calibrate to measured widths on this 264-PPI panel: + // thicker ~1mm, thickest ~2mm, thinnest ~2px. px/mm = 264/25.4 ~= 10.4. + constexpr double pxPerMm = 264.0 / 25.4; + const double maxMm = qMax(0.2, (penStyle.size - 1.0) * 1.0); // sz3->2mm, sz2->1mm + ink.setWidthRange(2.0, maxMm * pxPerMm); + ink.setPenType(penStyle.type); + + // Back button (finger or stylus) → return to the standard GUI. + QObject::connect(&ink, &InkEngine::backRequested, &appControl, &AppControl::returnToStandard); + + // Pen drives the ink engine directly (no QML in the ink loop). + QObject::connect(&pen, &PenDevice::strokeStart, &ink, &InkEngine::strokeStart); + QObject::connect(&pen, &PenDevice::strokeMove, &ink, &InkEngine::strokeMove); + QObject::connect(&pen, &PenDevice::strokeEnd, &ink, &InkEngine::strokeEnd); + + QQmlApplicationEngine engine; + engine.rootContext()->setContextProperty("appControl", &appControl); + engine.rootContext()->setContextProperty("ink", &ink); + + QObject::connect( + &engine, &QQmlApplicationEngine::objectCreationFailed, + &app, []() { QCoreApplication::exit(-1); }, + Qt::QueuedConnection); + + engine.loadFromModule("Chatter", "Main"); + + return app.exec(); +} diff --git a/tools/chatter_launcher.c b/tools/chatter_launcher.c new file mode 100644 index 0000000..f1b5b49 --- /dev/null +++ b/tools/chatter_launcher.c @@ -0,0 +1,86 @@ +// Chatter launcher daemon. Runs always (systemd service), reads the touch device +// (event3) WITHOUT grabbing it, and when it sees a multi-finger hold gesture +// (>= FINGERS contacts held >= HOLD_MS) while xochitl is the active app, it +// switches to Chatter. This is the "return to Chatter from the standard +// interface" trigger (xochitl has no plugin API to add a real button). + +#include +#include +#include +#include +#include +#include +#include + +#define DEV "/dev/input/event3" +#define MAX_SLOTS 16 +#define FINGERS 4 // contacts required +#define HOLD_MS 700 // how long they must be held +#define COOLDOWN_MS 4000 // ignore re-triggers for this long + +static long now_ms(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ts.tv_sec * 1000L + ts.tv_nsec / 1000000L; +} + +int main(void) +{ + long lastTrigger = 0; + fprintf(stderr, "chatter-launcher: started (fingers=%d hold=%dms)\n", FINGERS, HOLD_MS); + + // Outer loop: (re)open the device forever, so a read interruption (e.g. the + // device sleeping/waking) never kills the daemon. + for (;;) { + int fd = open(DEV, O_RDONLY); + if (fd < 0) { sleep(2); continue; } + + int slot = 0; + int tid[MAX_SLOTS]; + for (int i = 0; i < MAX_SLOTS; i++) + tid[i] = -1; + long holdStart = 0; + int triggered = 0; + + struct input_event ev; + while (read(fd, &ev, sizeof(ev)) == (ssize_t)sizeof(ev)) { + if (ev.type == EV_ABS) { + if (ev.code == ABS_MT_SLOT) { + slot = ev.value; + if (slot < 0) slot = 0; + if (slot >= MAX_SLOTS) slot = MAX_SLOTS - 1; + } else if (ev.code == ABS_MT_TRACKING_ID) { + tid[slot] = ev.value; // >=0 active, -1 released + } + } else if (ev.type == EV_SYN && ev.code == SYN_REPORT) { + int active = 0; + for (int i = 0; i < MAX_SLOTS; i++) + if (tid[i] >= 0) active++; + + const long t = now_ms(); + if (active >= FINGERS) { + if (holdStart == 0) + holdStart = t; + else if (!triggered && (t - holdStart) >= HOLD_MS && + (t - lastTrigger) >= COOLDOWN_MS) { + triggered = 1; + lastTrigger = t; + if (system("systemctl is-active --quiet xochitl") == 0) { + fprintf(stderr, "chatter-launcher: gesture -> switching to Chatter\n"); + system("/home/root/chatter/to-chatter.sh >/dev/null 2>&1 &"); + } + } + } else { + holdStart = 0; + triggered = 0; + } + } + } + + close(fd); + fprintf(stderr, "chatter-launcher: input read ended, reopening\n"); + sleep(1); + } + return 0; +} diff --git a/tools/fbdump.cpp b/tools/fbdump.cpp new file mode 100644 index 0000000..eb95385 --- /dev/null +++ b/tools/fbdump.cpp @@ -0,0 +1,56 @@ +// Framebuffer snapshot shim: preloaded into xochitl, it captures the e-paper +// framebuffer (via setBuffers) and, when the trigger file /tmp/fbdump appears, +// saves the current framebuffer to /home/root/fbdump.png. Used to recover the +// real Calligraphy nib angle by measuring xochitl's actual rendered strokes. + +#define _GNU_SOURCE 1 +#include +#include +#include +#include +#include +#include +#include + +namespace { +uchar *g_bits = nullptr; +int g_w = 0, g_h = 0, g_bpl = 0, g_fmt = 0; +bool g_started = false; + +void *dumper(void *) +{ + fprintf(stderr, "FBSHIM dumper thread started\n"); + for (;;) { + usleep(400000); + struct stat st; + if (g_bits && stat("/tmp/fbdump", &st) == 0) { + QImage snap = QImage(g_bits, g_w, g_h, g_bpl, QImage::Format(g_fmt)).copy(); + bool ok = snap.save("/home/root/fbdump.png"); + if (!ok) ok = snap.save("/home/root/fbdump.bmp", "BMP"); + ::unlink("/tmp/fbdump"); + fprintf(stderr, "FBDUMP %dx%d saved ok=%d\n", g_w, g_h, int(ok)); + } + } + return nullptr; +} +} + +extern "C" void _ZN13EPFramebuffer10setBuffersESt5tupleIJ6QImageS1_EEPS1_( + void *self, void *tuplePtr, void *imgPtr) +{ + static void (*real)(void *, void *, void *) = nullptr; + if (!real) + real = (void (*)(void *, void *, void *))dlsym( + RTLD_NEXT, "_ZN13EPFramebuffer10setBuffersESt5tupleIJ6QImageS1_EEPS1_"); + auto *t = reinterpret_cast *>(tuplePtr); + const QImage &a = std::get<0>(*t); + g_bits = const_cast(a.constBits()); + g_w = a.width(); g_h = a.height(); g_bpl = a.bytesPerLine(); g_fmt = int(a.format()); + fprintf(stderr, "FBSHIM setBuffers %dx%d fmt=%d\n", g_w, g_h, g_fmt); + if (!g_started) { + g_started = true; + pthread_t th; + pthread_create(&th, nullptr, dumper, nullptr); + } + if (real) real(self, tuplePtr, imgPtr); +} diff --git a/tools/grabtest.c b/tools/grabtest.c new file mode 100644 index 0000000..518eaeb --- /dev/null +++ b/tools/grabtest.c @@ -0,0 +1,26 @@ +// Probe whether an input device is exclusively grabbed (EVIOCGRAB) by another +// process (e.g. xochitl). If we can grab it, it was free -> a background watcher +// can read it alongside xochitl. If EBUSY, xochitl holds it exclusively. +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char **argv) +{ + const char *dev = argc > 1 ? argv[1] : "/dev/input/event3"; + int fd = open(dev, O_RDONLY); + if (fd < 0) { printf("%s: open failed: %s\n", dev, strerror(errno)); return 1; } + int r = ioctl(fd, EVIOCGRAB, (void *)1); + if (r == 0) { + ioctl(fd, EVIOCGRAB, (void *)0); // release immediately + printf("%s: NOT exclusively grabbed -> a watcher can read it\n", dev); + } else { + printf("%s: GRABBED by another process (%s)\n", dev, strerror(errno)); + } + close(fd); + return 0; +} diff --git a/tools/setbufshim.cpp b/tools/setbufshim.cpp new file mode 100644 index 0000000..4c891cd --- /dev/null +++ b/tools/setbufshim.cpp @@ -0,0 +1,35 @@ +// Feasibility check: can we intercept EPFramebuffer::setBuffers to capture the +// framebuffer's backing QImages? setBuffers is called from the epaper platform +// plugin (libepaper) into libqsgepaper — a cross-DSO call, so unlike swapBuffers +// it should be interposable via LD_PRELOAD. If this fires and reports real image +// dimensions + a pixel pointer, the direct-framebuffer ink pipeline is viable. +// +// ABI: std::tuple is non-trivially-copyable, so it's passed by +// reference (a pointer); QImage* is a pointer. So three pointer args after this. + +#include +#include +#include +#include + +extern "C" void _ZN13EPFramebuffer10setBuffersESt5tupleIJ6QImageS1_EEPS1_( + void *self, void *tuplePtr, void *imgPtr) +{ + static void (*real)(void *, void *, void *) = nullptr; + if (!real) + real = (void (*)(void *, void *, void *))dlsym( + RTLD_NEXT, "_ZN13EPFramebuffer10setBuffersESt5tupleIJ6QImageS1_EEPS1_"); + + auto *t = reinterpret_cast *>(tuplePtr); + QImage *c = reinterpret_cast(imgPtr); + const QImage &a = std::get<0>(*t); + const QImage &b = std::get<1>(*t); + fprintf(stderr, + "SETBUFFERS self=%p | A=%dx%d fmt=%d bytesPerLine=%d cbits=%p | " + "B=%dx%d fmt=%d | C=%p %dx%d\n", + self, a.width(), a.height(), int(a.format()), a.bytesPerLine(), + (const void *)a.constBits(), b.width(), b.height(), int(b.format()), + (void *)c, c ? c->width() : -1, c ? c->height() : -1); + + if (real) real(self, tuplePtr, imgPtr); +} diff --git a/tools/setbufshim.so b/tools/setbufshim.so new file mode 100755 index 0000000..74bde17 Binary files /dev/null and b/tools/setbufshim.so differ diff --git a/tools/swapshim.cpp b/tools/swapshim.cpp new file mode 100644 index 0000000..1d04075 --- /dev/null +++ b/tools/swapshim.cpp @@ -0,0 +1,46 @@ +// LD_PRELOAD trace shim: intercept EPFramebuffer::swapBuffers in libqsgepaper to +// learn the exact (contentType, screenMode, flags) the stock app uses for pen +// strokes. We define functions with the real mangled names so the dynamic loader +// interposes them, log the args, then chain to the real implementation. +// +// QRect is passed by value; its memory layout is {int x1,y1,x2,y2} (l,t,r,b), so +// we model it as a 16-byte POD to match the ABI without linking Qt. The enums and +// QFlags are all 4-byte int-sized. + +#include +#include + +struct RawRect { int x1, y1, x2, y2; }; + +extern "C" { + +// swapBuffers(QRect, EPContentType, EPScreenMode, QFlags) +void _ZN13EPFramebuffer11swapBuffersE5QRect13EPContentType12EPScreenMode6QFlagsINS_10UpdateFlagEE( + void *self, RawRect r, int contentType, int screenMode, int flags) +{ + static void (*real)(void *, RawRect, int, int, int) = nullptr; + if (!real) + real = (void (*)(void *, RawRect, int, int, int))dlsym( + RTLD_NEXT, + "_ZN13EPFramebuffer11swapBuffersE5QRect13EPContentType12EPScreenMode6QFlagsINS_10UpdateFlagEE"); + fprintf(stderr, "SWAPTRACE1 rect=(%d,%d)-(%d,%d) %dx%d content=%d screen=%d flags=%d\n", + r.x1, r.y1, r.x2, r.y2, r.x2 - r.x1 + 1, r.y2 - r.y1 + 1, + contentType, screenMode, flags); + if (real) real(self, r, contentType, screenMode, flags); +} + +// swapBuffers(const QRegion&, const EPContentMap&, const EPScreenModeMap&, QFlags) +void _ZN13EPFramebuffer11swapBuffersERK7QRegionRK12EPContentMapRK15EPScreenModeMap6QFlagsINS_10UpdateFlagEE( + void *self, const void *region, const void *contentMap, const void *modeMap, int flags) +{ + static void (*real)(void *, const void *, const void *, const void *, int) = nullptr; + if (!real) + real = (void (*)(void *, const void *, const void *, const void *, int))dlsym( + RTLD_NEXT, + "_ZN13EPFramebuffer11swapBuffersERK7QRegionRK12EPContentMapRK15EPScreenModeMap6QFlagsINS_10UpdateFlagEE"); + fprintf(stderr, "SWAPTRACE2 region=%p contentMap=%p modeMap=%p flags=%d\n", + region, contentMap, modeMap, flags); + if (real) real(self, region, contentMap, modeMap, flags); +} + +} // extern "C" diff --git a/tools/swapshim.so b/tools/swapshim.so new file mode 100755 index 0000000..e462f9a Binary files /dev/null and b/tools/swapshim.so differ