From a4abb5d15d2969a2da1ed681a53a69eb4b408c15 Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Mon, 7 Sep 2026 20:56:28 -0700 Subject: [PATCH 1/9] feat(e2e): Meshtasticator radio-level validation harness for the adapter Adds a skip-gated end-to-end suite (plugin/tests/e2e) that drives the adapter against live Meshtasticator simulated nodes - real MeshtasticD instances speaking the actual TCP/protobuf API - covering the layer no contract test can reach: TCP handshake, inbound text over the simulated mesh, and chunked outbound sends arriving at a peer node. scripts/meshtasticator_e2e/run_e2e.sh boots N nodes (Docker or Linux native MeshtasticD), discovers their TCP API ports, and runs the suite. Radio-level reconnect and CI wiring are documented as open items pending a Linux/Docker validation run. --- plugin/tests/e2e/test_meshtasticator_e2e.py | 227 ++++++++++++++++++++ scripts/meshtasticator_e2e/README.md | 99 +++++++++ scripts/meshtasticator_e2e/run_e2e.sh | 125 +++++++++++ 3 files changed, 451 insertions(+) create mode 100644 plugin/tests/e2e/test_meshtasticator_e2e.py create mode 100644 scripts/meshtasticator_e2e/README.md create mode 100644 scripts/meshtasticator_e2e/run_e2e.sh diff --git a/plugin/tests/e2e/test_meshtasticator_e2e.py b/plugin/tests/e2e/test_meshtasticator_e2e.py new file mode 100644 index 0000000..7096427 --- /dev/null +++ b/plugin/tests/e2e/test_meshtasticator_e2e.py @@ -0,0 +1,227 @@ +"""End-to-end tests: hermes-meshtastic adapter against live Meshtasticator nodes. + +What these tests validate (the "radio-level" layer no contract test can): + * the adapter's real TCP/protobuf handshake against MeshtasticD (the actual + device software, not a mock) as run by the Meshtasticator interactive + simulator (https://meshtastic.org/docs/software/meshtasticator/), + * inbound text reception across the simulated LoRa mesh, + * outbound chunked sends arriving at another simulated node, + * the whole path: pubsub callback -> MessageEvent construction (real gateway + types) -> handle_message dispatch. + +What they do NOT validate (out of scope even for Meshtasticator): true RF +(silicon-level LoRa, real propagation/interference, regulatory duty cycle). + +Running +------- + 1. On a Linux host (or a machine with Docker): clone meshtastic/meshtasticator + and boot the interactive simulator (see scripts/meshtasticator_e2e/). + 2. Point the plugin's own contract venv at the booted nodes and run: + cd plugin && MESHTASTICATOR_E2E=1 E2E_PORTS=4404,4405 \ + E2E_CHANNEL_INDEX=0 .venv-contract/bin/python -m pytest tests/e2e -q + +Environment +----------- + MESHTASTICATOR_E2E = "1" enables these tests (required; without it the + module skips so ordinary unit/CI runs stay fast). + E2E_PORTS = comma-separated TCP API ports of the simulated nodes + (first = the node the adapter connects to, default 4404). + E2E_HOST = host where the simulator exposes the ports (default 127.0.0.1). + E2E_CHANNEL_INDEX = Meshtastic channel index the adapter listens/sends on + (default 0 = primary channel, matching a stock sim node). + +NOTE: this module is an initial scaffold. Structural validation (imports, +fixtures, flow) passes, but a full green run requires a Linux/Docker host with +a MeshtasticD build — tracked as the remaining verification step. +""" + +import asyncio +import os + +import pytest + +if os.environ.get("MESHTASTICATOR_E2E") != "1": + pytest.skip( + "set MESHTASTICATOR_E2E=1 to run against Meshtasticator nodes " + "(see scripts/meshtasticator_e2e/README.md)", + allow_module_level=True, + ) + +# Real Hermes gateway + real Meshtastic client library are required. +pytest.importorskip("gateway.platforms.base") +pytest.importorskip("meshtastic.tcp_interface") + +from gateway.config import PlatformConfig # noqa: E402 +from gateway.platform_registry import ( # noqa: E402 + PlatformEntry, + platform_registry, +) + +import hermes_meshtastic # noqa: E402 +from hermes_meshtastic import adapter as plugin_adapter # noqa: E402 + +E2E_HOST = os.environ.get("E2E_HOST", "127.0.0.1") +E2E_PORTS = [int(p) for p in os.environ.get("E2E_PORTS", "4404,4405").split(",") if p] +E2E_CHANNEL_INDEX = int(os.environ.get("E2E_CHANNEL_INDEX", "0")) +assert len(E2E_PORTS) >= 2, "E2E_PORTS needs at least two node ports (adapter + peer)" + +NODE0_PORT = E2E_PORTS[0] # the node our adapter connects to +NODE1_PORT = E2E_PORTS[1] # the peer node that sends / observes +CONNECT_TIMEOUT_S = 30 +RADIO_SETTLE_S = 2 + + +def _register_meshtastic_platform(): + """Register the plugin (mirrors discovery) so Platform('meshtastic') resolves.""" + if platform_registry.is_registered("meshtastic"): + return + kwargs = {} + + class _Ctx: + def register_platform(self, **kw): + kwargs.update(kw) + + hermes_meshtastic.register(_Ctx()) + platform_registry.register( + PlatformEntry( + name=kwargs["name"], + label=kwargs["label"], + adapter_factory=kwargs["adapter_factory"], + check_fn=kwargs["check_fn"], + required_env=[], + max_message_length=kwargs.get("max_message_length", 0), + emoji=kwargs.get("emoji", "🔌"), + platform_hint=kwargs.get("platform_hint", ""), + ) + ) + + +class _ProbeAdapter(plugin_adapter.get_adapter_class()): + """MeshtasticAdapter that records inbound MessageEvents instead of + dispatching to a Hermes agent handler (no agent runs in these tests).""" + + def __init__(self, config): + super().__init__(config) + self.received_events = [] + + async def handle_message(self, event): + self.received_events.append(event) + # Intentionally do not call super(): the real base would forward to a + # message handler that only exists inside a running Hermes gateway. + return None + + +@pytest.fixture(scope="module") +def adapter(): + _register_meshtastic_platform() + cfg = PlatformConfig( + enabled=True, + extra={ + "host": E2E_HOST, + "port": NODE0_PORT, + "channel_index": E2E_CHANNEL_INDEX, + }, + ) + return _ProbeAdapter(cfg) + + +@pytest.fixture(scope="module") +def node0(adapter): + """Connect the adapter to simulated node 0; disconnect afterwards.""" + async def _connect(): + return await adapter.connect() + + assert asyncio.run(_connect()) is True, ( + f"adapter failed to connect to simulated node at {E2E_HOST}:{NODE0_PORT} " + "(is the simulator running and the node reachable?)" + ) + try: + yield adapter + finally: + asyncio.run(adapter.disconnect()) + + +def _wait_for(predicate, timeout_s=15.0, interval_s=0.5, what="condition"): + """Poll until *predicate* (callable -> bool) is true, else pytest.fail.""" + import time + + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(interval_s) + pytest.fail(f"timed out waiting for {what}") + + +# --------------------------------------------------------------------------- +# Radio-level flows +# --------------------------------------------------------------------------- + + +def test_adapter_receives_broadcast_from_peer(node0): + """Peer node 1 broadcasts text over the simulated mesh; the adapter must + receive it as a real gateway MessageEvent.""" + import time + + from meshtastic import tcp_interface + + sent = f"e2e-inbound-{int(time.time())}" + + peer = tcp_interface.TCPInterface(hostname=E2E_HOST, portNumber=NODE1_PORT) + try: + time.sleep(RADIO_SETTLE_S) + + def _got(): + return any(getattr(e, "text", None) == sent for e in node0.received_events) + + # Radio delivery is not guaranteed per attempt; retry a few times. + for _ in range(3): + if _got(): + break + peer.localNode.sendText( + text=sent, destinationId="^all", channelIndex=E2E_CHANNEL_INDEX + ) + time.sleep(1) + _wait_for(_got, timeout_s=25, what="inbound broadcast from peer node") + finally: + peer.close() + + +def test_adapter_chunked_send_reaches_peer(node0): + """A long outbound message must be chunked and arrive at the peer node.""" + import time + + from meshtastic import tcp_interface + from pubsub import pub + + content = " ".join(f"word{n}" for n in range(60)) # well over 180 chars -> chunks + received = [] + + def _on_receive(packet, interface): + decoded = packet.get("decoded", {}) or {} + text = decoded.get("text", "") + if text and text.startswith("(1/"): # our chunked framing + received.append(text) + elif text: + received.append(text) + + peer = tcp_interface.TCPInterface(hostname=E2E_HOST, portNumber=NODE1_PORT) + try: + time.sleep(RADIO_SETTLE_S) + pub.subscribe(_on_receive, "meshtastic.receive.text") + + async def _send(): + result = await node0.send("channel_e2e", content) + assert result.success, f"adapter.send failed: {result.error}" + + asyncio.run(_send()) + + def _all_chunks_arrived(): + # Adapter frames multi-part sends as "(i/n) "; the final + # chunk carries the message tail. + return any(str(r).rstrip().endswith("word59") for r in received) + + _wait_for(_all_chunks_arrived, timeout_s=45, what="chunked outbound send at peer") + finally: + pub.unsubscribe(_on_receive, "meshtastic.receive.text") + peer.close() diff --git a/scripts/meshtasticator_e2e/README.md b/scripts/meshtasticator_e2e/README.md new file mode 100644 index 0000000..2537853 --- /dev/null +++ b/scripts/meshtasticator_e2e/README.md @@ -0,0 +1,99 @@ +# Meshtasticator e2e — radio-level validation for hermes-meshtastic + +This directory is the **radio-level counterpart** to the contract suite +(`plugin/tests/contract`). The contract suite guarantees the adapter matches +the Hermes gateway API; it cannot prove the adapter actually talks to a +Meshtastic node. These e2e tests boot the +[Meshtasticator](https://meshtastic.org/docs/software/meshtasticator/) +interactive simulator — which runs the **real MeshtasticD device software** +per node with a simulated LoRa PHY — and drive the adapter against a live +simulated node. + +| Layer | Validated by | +|---|---| +| Adapter ↔ Hermes gateway API | `plugin/tests/contract` (no deployment) | +| Adapter ↔ real MeshtasticD TCP/protobuf session | **this suite** | +| Inbound text across the (simulated) LoRa mesh | **this suite** | +| Outbound chunked sends arriving at another node | **this suite** | +| True RF (silicon LoRa, real propagation, duty cycle) | physical radio only | + +## Layout + +``` +scripts/meshtasticator_e2e/ + run_e2e.sh orchestrator: boots N nodes, runs pytest, tears down + README.md this file +plugin/tests/e2e/ + test_meshtasticator_e2e.py the e2e assertions (skip-gated) +``` + +## Prerequisites + +* **uv** + Python 3.13. +* A contract venv with the real gateway **and** the plugin (with deps) installed: + ```bash + scripts/contract-test.sh # creates plugin/.venv-contract (hermes-agent pinned) + uv pip install --python plugin/.venv-contract/bin/python -e plugin + ``` + (`-e plugin` installs the meshtastic client library the tests also use.) +* A clone of Meshtasticator: `git clone https://github.com/meshtastic/meshtasticator` +* Runtime for the simulator — pick one: + * **Docker** (simplest, works on macOS/Windows): the simulator builds + MeshtasticD from a container image on first run (slow once). + * **Linux native**: build MeshtasticD's `native` PlatformIO target and pass + the build dir with `--program`. + +## Run + +```bash +# Docker mode (default) +MESHTASTICATOR_DIR=/path/to/meshtasticator \ + scripts/meshtasticator_e2e/run_e2e.sh + +# Linux native mode +MESHTASTICATOR_DIR=/path/to/meshtasticator \ + scripts/meshtasticator_e2e/run_e2e.sh \ + --mode native --program /path/to/firmware/.pio/build/native + +# Tune the scenario + ... --nodes 3 --channel-index 0 --host 127.0.0.1 +``` + +The orchestrator boots the simulator, waits until `N` node TCP API ports accept +connections (Meshtasticator assigns node *n* → port `4404 + n`), then runs +`pytest plugin/tests/e2e` with `MESHTASTICATOR_E2E=1`. Simulator output lands in +`meshtasticator-sim.log` at the repo root. + +Without `MESHTASTICATOR_E2E=1` the e2e module skips, so ordinary unit/CI runs +stay fast and gateway-free. + +## What the tests do + +1. **Inbound**: peer node 1 broadcasts text; the adapter (connected to node 0) + must receive it as a real gateway `MessageEvent` (`host/port/channel_index` + come from `E2E_PORTS`/`E2E_HOST`/`E2E_CHANNEL_INDEX`). +2. **Outbound**: the adapter sends a >180-char message; it must be chunked and + the final chunk (`(i/n) … word59`) must arrive at peer node 1. + +A `_ProbeAdapter` subclass records inbound events instead of dispatching to a +Hermes agent handler (no agent runs in these tests; dispatch is already covered +by the contract suite). + +## Status / open items (v1 scaffold) + +This is an initial, structurally validated scaffold. A full green run still +needs to happen on a Linux/Docker host with a MeshtasticD build, and these +items are expected to need iteration there: + +* **Node adjacency**: the simulator's default random placement may put nodes out + of range. For deterministic tests, seed close-together coordinates (the sim + supports `--from-file` with an `out/nodeConfig.yaml`). +* **Channel semantics**: tests default to channel index 0 (stock primary + channel). Mirroring production (channel 1 / private channel) requires + configuring matching channels on the simulated nodes. +* **Reconnect flow**: killing/restarting a single simulated node needs per-node + lifecycle control (native mode subprocesses; docker mode currently runs all + nodes in one container), so the adapter's watchdog reconnect is **not** yet + exercised here. See the watchdog unit coverage in the adapter for now. +* **CI wiring**: an e2e job needs a docker-enabled runner and tolerates a long + first-run firmware build — deliberately left out of this PR. diff --git a/scripts/meshtasticator_e2e/run_e2e.sh b/scripts/meshtasticator_e2e/run_e2e.sh new file mode 100644 index 0000000..52462ef --- /dev/null +++ b/scripts/meshtasticator_e2e/run_e2e.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# Boot N Meshtasticator nodes and run the plugin's radio-level e2e tests +# (plugin/tests/e2e) against node 0. +# +# Meshtasticator's interactive simulator (https://meshtastic.org/docs/software/meshtasticator/) +# runs the real MeshtasticD device software per node and simulates the LoRa +# PHY, so these tests exercise the adapter's real TCP/protobuf session, +# inbound/outbound text flow across the simulated mesh, and chunked sends — +# without physical radios. It cannot validate silicon-level RF. +# +# Prerequisites +# * Linux host, OR Docker (macOS/Windows): the simulator auto-falls back to +# Docker on non-Linux and can build MeshtasticD itself (`-d`). +# * Native Linux mode: build MeshtasticD's 'native' target (PlatformIO, +# select 'native') and pass --program . +# * uv + python 3.13 for the contract venv (plugin/.venv-contract). +# +# Usage +# MESHTASTICATOR_DIR=/path/to/meshtasticator scripts/meshtasticator_e2e/run_e2e.sh +# # docker mode (default): ... run_e2e.sh --mode docker +# # native mode: ... run_e2e.sh --mode native --program /path/to/firmware/.pio/build/native +# # node count / channel: ... run_e2e.sh --nodes 3 --channel-index 0 +# +# Environment +# MESHTASTICATOR_DIR path to a clone of github.com/meshtastic/meshtasticator +# E2E_VENV python env with hermes-agent + plugin deps installed +# (default: plugin/.venv-contract — set it up with +# `uv pip install -e plugin/.hermes-src` equivalents via +# scripts/contract-test.sh, then `uv pip install -e plugin`) +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +SIM_DIR="${MESHTASTICATOR_DIR:?set MESHTASTICATOR_DIR to a meshtasticator clone}" +NODES=3 +MODE=docker +PROGRAM="" +CHANNEL_INDEX=0 +HOST=127.0.0.1 +VENV="${E2E_VENV:-$REPO_ROOT/plugin/.venv-contract}" + +while [ $# -gt 0 ]; do + case "$1" in + --nodes) NODES="$2"; shift 2 ;; + --mode) MODE="$2"; shift 2 ;; + --program) PROGRAM="$2"; shift 2 ;; + --channel-index) CHANNEL_INDEX="$2"; shift 2 ;; + --host) HOST="$2"; shift 2 ;; + *) echo "unknown arg: $1" >&2; exit 2 ;; + esac +done + +command -v uv >/dev/null 2>&1 || { echo "uv required" >&2; exit 1; } +[ -d "$SIM_DIR" ] || { echo "meshtasticator not found at $SIM_DIR" >&2; exit 1; } +[ -x "$VENV/bin/python" ] || { echo "e2e venv missing at $VENV (see scripts/contract-test.sh)" >&2; exit 1; } + +export MPLBACKEND=Agg # headless matplotlib (no display needed) + +SIM_LOG="$REPO_ROOT/meshtasticator-sim.log" +SIM_ARGS=("$NODES") +if [ "$MODE" = native ]; then + [ -n "$PROGRAM" ] && [ -d "$PROGRAM" ] || { echo "--mode native needs --program " >&2; exit 2; } + SIM_ARGS+=(-p "$PROGRAM") +else + SIM_ARGS+=(-d) +fi + +echo "== booting Meshtasticator ($MODE) with $NODES node(s) from $SIM_DIR ==" +( + cd "$SIM_DIR" + python3 interactiveSim.py "${SIM_ARGS[@]}" >"$SIM_LOG" 2>&1 +) & +SIM_PID=$! + +PORTS="" +cleanup() { + echo "== tearing down simulator (pid $SIM_PID) ==" + kill "$SIM_PID" 2>/dev/null || true + wait "$SIM_PID" 2>/dev/null || true + if [ "$MODE" = docker ]; then + # The sim container may outlive the process on abrupt kills. + docker ps -q --filter "ancestor=meshtastic-meshtasticd" 2>/dev/null | xargs -r docker rm -f >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT + +echo "== waiting for $NODES node TCP API port(s) to accept connections ==" +READY="" +for _ in $(seq 1 180); do # up to ~5 min: docker builds firmware on first run + READY="$("$VENV/bin/python" - "$HOST" "$NODES" <<'PY' || true +import socket, sys +host, count = sys.argv[1], int(sys.argv[2]) +found = [] +# Meshtasticator assigns node n -> TCP port starting at 4404 (see +# lib/interactive.py TCP_PORT_OFFSET); probe a small window to be safe. +for port in range(4403, 4403 + 16): + try: + with socket.create_connection((host, port), timeout=0.5): + found.append(port) + except OSError: + pass + if len(found) >= count: + break +print(",".join(map(str, found[:count]))) +PY +)" + if [ -n "$READY" ]; then + break + fi + sleep 2 +done +if [ -z "$READY" ]; then + echo "error: no Meshtasticator node ports became reachable on $HOST (see $SIM_LOG)" >&2 + exit 1 +fi +echo " node ports: $READY" + +echo "== running radio-level e2e tests ==" +( + cd "$REPO_ROOT" + MESHTASTICATOR_E2E=1 \ + E2E_HOST="$HOST" \ + E2E_PORTS="$READY" \ + E2E_CHANNEL_INDEX="$CHANNEL_INDEX" \ + "$VENV/bin/python" -m pytest plugin/tests/e2e -q "$@" +) -- 2.52.0 From 4485ec38144dca6a41d81778ab3f9b3dab388050 Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Mon, 7 Sep 2026 21:04:56 -0700 Subject: [PATCH 2/9] ci(e2e): on-demand Meshtasticator e2e job for the docker-enabled runner Adds .gitea/workflows/meshtasticator-e2e.yaml (workflow_dispatch until validated green, pull_request trigger commented out): builds the pinned Hermes contract env like release.yaml, installs the plugin with deps, checks out Meshtasticator at a pinned commit, installs the simulator's own (conflicting-version) requirements in an isolated venv, then runs scripts/meshtasticator_e2e/run_e2e.sh --mode docker. Supporting changes: - run_e2e.sh: SIM_VENV support; automatic localhost->daemon TCP forwards (scripts/meshtasticator_e2e/tcp_forward.py) when DOCKER_HOST is a tcp:// daemon, because Meshtasticator hardcodes localhost for its node control connections; node-boot settle wait; fixed docker cleanup (container 'Meshtastic', volume 'Meshtasticator'). - e2e tests: connect via a module-long event loop in a background thread (adapter requires a running loop for watchdog + inbound dispatch, so per-call asyncio.run was wrong) with retry while the node TCP API boots. - README: sim-venv guidance, remote-daemon forwarding, CI section. Docker access follows the deployment examples in eric/actions-docker (docker is available to runner jobs). --- .gitea/workflows/meshtasticator-e2e.yaml | 90 +++++++++++++++++++++ plugin/tests/e2e/test_meshtasticator_e2e.py | 61 +++++++++++--- scripts/meshtasticator_e2e/README.md | 42 +++++++++- scripts/meshtasticator_e2e/run_e2e.sh | 66 +++++++++++++-- scripts/meshtasticator_e2e/tcp_forward.py | 70 ++++++++++++++++ 5 files changed, 306 insertions(+), 23 deletions(-) create mode 100644 .gitea/workflows/meshtasticator-e2e.yaml create mode 100644 scripts/meshtasticator_e2e/tcp_forward.py diff --git a/.gitea/workflows/meshtasticator-e2e.yaml b/.gitea/workflows/meshtasticator-e2e.yaml new file mode 100644 index 0000000..5ba7feb --- /dev/null +++ b/.gitea/workflows/meshtasticator-e2e.yaml @@ -0,0 +1,90 @@ +name: meshtasticator-e2e + +# Radio-level end-to-end validation against Meshtasticator's interactive +# simulator (real MeshtasticD per node + simulated LoRa PHY). Runs on demand +# until it is validated green on the runner; afterwards switch `on` to also +# trigger on pull_request touching plugin/ and this workflow. +# +# Docker access follows the deployment examples in +# https://git.ericxliu.me/eric/actions-docker (docker is available to jobs); +# Meshtasticator's docker mode publishes node TCP API ports (4404+n) from the +# meshtastic/meshtasticd container and the simulator connects to them on +# localhost. scripts/meshtasticator_e2e/run_e2e.sh starts localhost->daemon +# TCP forwards automatically when DOCKER_HOST is a tcp:// daemon, which is how +# this job is configured. + +on: + workflow_dispatch: + # pull_request: + # paths: + # - 'plugin/**' + # - 'scripts/meshtasticator_e2e/**' + # - '.gitea/workflows/meshtasticator-e2e.yaml' + +concurrency: + # One e2e run at a time: Meshtasticator hardcodes the container/volume name + # ("Meshtastic"/"Meshtasticator") on the shared docker daemon. + group: meshtasticator-e2e + cancel-in-progress: true + +jobs: + e2e: + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + DOCKER_HOST: tcp://docker.local:2375 + MESHTASTICATOR_COMMIT: 17ceb8231079d87b070abc6132181e4c6b20202d + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python 3.13 (matches Hermes image) + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install uv + run: pip install uv + + # --- Hermes gateway contract env (mirrors .gitea/workflows/release.yaml) --- + - name: Clone pinned Hermes gateway source (deployed image tag) + run: | + git clone --depth 1 --branch v2026.8.31 \ + https://github.com/NousResearch/hermes-agent.git plugin/.hermes-src + test "$(git -C plugin/.hermes-src rev-parse HEAD)" = \ + "29112bef099274229cadff79cdff7bf7b99c4b77" + + - name: Editable install hermes-agent + pytest + plugin (meshtastic client lib) + run: | + uv venv --python 3.13 plugin/.venv-contract + uv pip install --python plugin/.venv-contract/bin/python -e plugin/.hermes-src + uv pip install --python plugin/.venv-contract/bin/python pytest + uv pip install --python plugin/.venv-contract/bin/python -e plugin + + # --- Simulator env (isolated: meshtasticator pins meshtastic~=2.6.1) --- + - name: Clone Meshtasticator at pinned commit + run: | + git clone https://github.com/meshtastic/meshtasticator.git "$HOME/meshtasticator" + git -C "$HOME/meshtasticator" checkout "$MESHTASTICATOR_COMMIT" + test "$(git -C "$HOME/meshtasticator" rev-parse HEAD)" = "$MESHTASTICATOR_COMMIT" + + - name: Simulator venv (its own requirements + docker SDK) + run: | + uv venv --python 3.13 "$HOME/sim-venv" + uv pip install --python "$HOME/sim-venv/bin/python" \ + -r "$HOME/meshtasticator/requirements.txt" docker + + - name: Run Meshtasticator e2e suite + run: | + export MESHTASTICATOR_DIR="$HOME/meshtasticator" + export SIM_VENV="$HOME/sim-venv" + export E2E_VENV="plugin/.venv-contract" + scripts/meshtasticator_e2e/run_e2e.sh --mode docker + + - name: Print simulator log (on failure) + if: failure() + run: | + echo "===== meshtasticator-sim.log =====" + tail -n 200 meshtasticator-sim.log 2>/dev/null || true + echo "===== docker ps =====" + docker ps -a 2>/dev/null || true diff --git a/plugin/tests/e2e/test_meshtasticator_e2e.py b/plugin/tests/e2e/test_meshtasticator_e2e.py index 7096427..c5b0837 100644 --- a/plugin/tests/e2e/test_meshtasticator_e2e.py +++ b/plugin/tests/e2e/test_meshtasticator_e2e.py @@ -126,19 +126,60 @@ def adapter(): @pytest.fixture(scope="module") -def node0(adapter): - """Connect the adapter to simulated node 0; disconnect afterwards.""" - async def _connect(): - return await adapter.connect() +def loop(): + """Long-lived event loop in a background thread. - assert asyncio.run(_connect()) is True, ( + The adapter mirrors the Hermes gateway contract: connect()/watchdog/ + handle_message() assume a *running* loop (the gateway's). A per-call + asyncio.run() would close the loop right after connect() and strand both + the watchdog and inbound dispatch, so the e2e module keeps one loop alive + and drives coroutines into it with run_coroutine_threadsafe. + """ + import threading + + _loop = asyncio.new_event_loop() + thread = threading.Thread(target=_loop.run_forever, daemon=True) + thread.start() + try: + yield _loop + finally: + _loop.call_soon_threadsafe(_loop.stop) + thread.join(timeout=5) + + +def _run_in_loop(loop, coro, timeout_s=CONNECT_TIMEOUT_S): + import concurrent.futures + + try: + return asyncio.run_coroutine_threadsafe(coro, loop).result(timeout=timeout_s) + except concurrent.futures.TimeoutError: + pytest.fail(f"coroutine did not finish within {timeout_s}s") + + +@pytest.fixture(scope="module") +def node0(adapter, loop): + """Connect the adapter to simulated node 0 (retrying while the node's TCP + API comes up); disconnect afterwards.""" + import time + + deadline = time.monotonic() + CONNECT_TIMEOUT_S + last_ok = False + while time.monotonic() < deadline: + last_ok = bool(_run_in_loop(loop, adapter.connect())) + if last_ok: + break + time.sleep(3) + assert last_ok, ( f"adapter failed to connect to simulated node at {E2E_HOST}:{NODE0_PORT} " "(is the simulator running and the node reachable?)" ) try: yield adapter finally: - asyncio.run(adapter.disconnect()) + try: + _run_in_loop(loop, adapter.disconnect(), timeout_s=10) + except Exception: # loop teardown must not mask test failures + pass def _wait_for(predicate, timeout_s=15.0, interval_s=0.5, what="condition"): @@ -187,7 +228,7 @@ def test_adapter_receives_broadcast_from_peer(node0): peer.close() -def test_adapter_chunked_send_reaches_peer(node0): +def test_adapter_chunked_send_reaches_peer(node0, loop): """A long outbound message must be chunked and arrive at the peer node.""" import time @@ -200,9 +241,7 @@ def test_adapter_chunked_send_reaches_peer(node0): def _on_receive(packet, interface): decoded = packet.get("decoded", {}) or {} text = decoded.get("text", "") - if text and text.startswith("(1/"): # our chunked framing - received.append(text) - elif text: + if text: received.append(text) peer = tcp_interface.TCPInterface(hostname=E2E_HOST, portNumber=NODE1_PORT) @@ -214,7 +253,7 @@ def test_adapter_chunked_send_reaches_peer(node0): result = await node0.send("channel_e2e", content) assert result.success, f"adapter.send failed: {result.error}" - asyncio.run(_send()) + _run_in_loop(loop, _send(), timeout_s=60) def _all_chunks_arrived(): # Adapter frames multi-part sends as "(i/n) "; the final diff --git a/scripts/meshtasticator_e2e/README.md b/scripts/meshtasticator_e2e/README.md index 2537853..08073b1 100644 --- a/scripts/meshtasticator_e2e/README.md +++ b/scripts/meshtasticator_e2e/README.md @@ -22,9 +22,12 @@ simulated node. ``` scripts/meshtasticator_e2e/ run_e2e.sh orchestrator: boots N nodes, runs pytest, tears down + tcp_forward.py localhost->daemon TCP forward (remote docker daemons) README.md this file plugin/tests/e2e/ test_meshtasticator_e2e.py the e2e assertions (skip-gated) +.gitea/workflows/ + meshtasticator-e2e.yaml on-demand CI job (docker-enabled runner) ``` ## Prerequisites @@ -38,10 +41,20 @@ plugin/tests/e2e/ (`-e plugin` installs the meshtastic client library the tests also use.) * A clone of Meshtasticator: `git clone https://github.com/meshtastic/meshtasticator` * Runtime for the simulator — pick one: - * **Docker** (simplest, works on macOS/Windows): the simulator builds - MeshtasticD from a container image on first run (slow once). + * **Docker** (simplest, works on macOS/Windows): the simulator pulls the + `meshtastic/meshtasticd` image on first run (slow once). Node TCP API ports + are published on your docker host's localhost, which is what the simulator + itself expects. * **Linux native**: build MeshtasticD's `native` PlatformIO target and pass the build dir with `--program`. +* Optional but recommended: an isolated simulator venv, because Meshtasticator + pins old client libs (`meshtastic~=2.6.1` in its requirements.txt): + ```bash + uv venv --python 3.13 /tmp/sim-venv + uv pip install --python /tmp/sim-venv/bin/python \ + -r /requirements.txt docker + export SIM_VENV=/tmp/sim-venv # else the script uses plain `python3` + ``` ## Run @@ -64,9 +77,26 @@ connections (Meshtasticator assigns node *n* → port `4404 + n`), then runs `pytest plugin/tests/e2e` with `MESHTASTICATOR_E2E=1`. Simulator output lands in `meshtasticator-sim.log` at the repo root. +**Remote docker daemon** (`DOCKER_HOST=tcp://host:2375`): the simulator's own +control connections assume nodes listen on localhost, so the script starts a +`tcp_forward.py` per node port (localhost → daemon host) before booting the +simulator. Nothing is forwarded when the daemon is local (unset/unix +`DOCKER_HOST`). + Without `MESHTASTICATOR_E2E=1` the e2e module skips, so ordinary unit/CI runs stay fast and gateway-free. +## CI + +`.gitea/workflows/meshtasticator-e2e.yaml` runs the whole flow on a +docker-enabled runner (on demand via `workflow_dispatch` until validated, then +flip on the commented `pull_request` trigger): it builds the Hermes contract +env (pinned like `release.yaml`), installs the plugin with deps, clones +Meshtasticator at a pinned commit, installs the simulator's own requirements in +an isolated venv, and calls `run_e2e.sh --mode docker`. Simulator logs print on +failure. Only one e2e run at a time (the sim hardcodes the `Meshtastic` +container/volume name on the shared daemon). + ## What the tests do 1. **Inbound**: peer node 1 broadcasts text; the adapter (connected to node 0) @@ -95,5 +125,9 @@ items are expected to need iteration there: lifecycle control (native mode subprocesses; docker mode currently runs all nodes in one container), so the adapter's watchdog reconnect is **not** yet exercised here. See the watchdog unit coverage in the adapter for now. -* **CI wiring**: an e2e job needs a docker-enabled runner and tolerates a long - first-run firmware build — deliberately left out of this PR. +* **CI validation run**: `.gitea/workflows/meshtasticator-e2e.yaml` is drafted + and on-demand only; it still needs a first green run on the docker-enabled + runner (topology assumptions: docker reachable from the job, node ports on + the job's localhost via the automatic tcp forwards). Trigger it via + `workflow_dispatch`, watch the sim log step, and only then enable the + `pull_request` trigger. diff --git a/scripts/meshtasticator_e2e/run_e2e.sh b/scripts/meshtasticator_e2e/run_e2e.sh index 52462ef..06327aa 100644 --- a/scripts/meshtasticator_e2e/run_e2e.sh +++ b/scripts/meshtasticator_e2e/run_e2e.sh @@ -15,6 +15,12 @@ # select 'native') and pass --program . # * uv + python 3.13 for the contract venv (plugin/.venv-contract). # +# Docker on a remote daemon: if DOCKER_HOST points at a tcp:// daemon, the +# simulator's own control connections assume nodes listen on localhost, so +# this script starts lightweight localhost->daemon TCP forwarders for the node +# ports first (scripts/meshtasticator_e2e/tcp_forward.py). Nothing is +# forwarded when the daemon is local (unset/unix DOCKER_HOST). +# # Usage # MESHTASTICATOR_DIR=/path/to/meshtasticator scripts/meshtasticator_e2e/run_e2e.sh # # docker mode (default): ... run_e2e.sh --mode docker @@ -24,9 +30,11 @@ # Environment # MESHTASTICATOR_DIR path to a clone of github.com/meshtastic/meshtasticator # E2E_VENV python env with hermes-agent + plugin deps installed -# (default: plugin/.venv-contract — set it up with -# `uv pip install -e plugin/.hermes-src` equivalents via -# scripts/contract-test.sh, then `uv pip install -e plugin`) +# (default: plugin/.venv-contract — see scripts/contract-test.sh, +# then `uv pip install --python plugin/.venv-contract/bin/python -e plugin`) +# SIM_VENV optional python env holding the simulator's deps +# (meshtasticator/requirements.txt + `pip install docker`); +# defaults to plain `python3`. set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" @@ -37,6 +45,11 @@ PROGRAM="" CHANNEL_INDEX=0 HOST=127.0.0.1 VENV="${E2E_VENV:-$REPO_ROOT/plugin/.venv-contract}" +if [ -n "${SIM_VENV:-}" ]; then + SIM_PYTHON="$SIM_VENV/bin/python" +else + SIM_PYTHON="${SIM_PYTHON:-python3}" +fi while [ $# -gt 0 ]; do case "$1" in @@ -55,6 +68,30 @@ command -v uv >/dev/null 2>&1 || { echo "uv required" >&2; exit 1; } export MPLBACKEND=Agg # headless matplotlib (no display needed) +# --------------------------------------------------------------------------- +# Remote-daemon port forwarding: Meshtasticator talks to nodes on localhost. +# --------------------------------------------------------------------------- +DOCKER_TARGET="" +if [[ "${DOCKER_HOST:-}" == tcp://* ]]; then + DOCKER_TARGET="${DOCKER_HOST#tcp://}" + DOCKER_TARGET="${DOCKER_TARGET%%:*}" + if [ "$DOCKER_TARGET" = "127.0.0.1" ] || [ "$DOCKER_TARGET" = "localhost" ]; then + DOCKER_TARGET="" + fi +fi +FWD_PIDS=() +PORT_BASE=4404 # matches meshtasticator lib/interactive.py TCP_PORT_OFFSET + +start_forwards() { + for ((i = 0; i < NODES; i++)); do + port=$((PORT_BASE + i)) + "$VENV/bin/python" "$REPO_ROOT/scripts/meshtasticator_e2e/tcp_forward.py" \ + --listen "127.0.0.1:$port" --target "$DOCKER_TARGET:$port" >/dev/null 2>&1 & + FWD_PIDS+=("$!") + done + sleep 1 +} + SIM_LOG="$REPO_ROOT/meshtasticator-sim.log" SIM_ARGS=("$NODES") if [ "$MODE" = native ]; then @@ -64,28 +101,36 @@ else SIM_ARGS+=(-d) fi +if [ -n "$DOCKER_TARGET" ]; then + echo "== remote docker daemon detected ($DOCKER_TARGET): starting localhost port forwards ==" + start_forwards +fi + echo "== booting Meshtasticator ($MODE) with $NODES node(s) from $SIM_DIR ==" ( cd "$SIM_DIR" - python3 interactiveSim.py "${SIM_ARGS[@]}" >"$SIM_LOG" 2>&1 + "$SIM_PYTHON" interactiveSim.py "${SIM_ARGS[@]}" >"$SIM_LOG" 2>&1 ) & SIM_PID=$! -PORTS="" cleanup() { echo "== tearing down simulator (pid $SIM_PID) ==" kill "$SIM_PID" 2>/dev/null || true wait "$SIM_PID" 2>/dev/null || true + for pid in "${FWD_PIDS[@]:-}"; do + kill "$pid" 2>/dev/null || true + done if [ "$MODE" = docker ]; then - # The sim container may outlive the process on abrupt kills. - docker ps -q --filter "ancestor=meshtastic-meshtasticd" 2>/dev/null | xargs -r docker rm -f >/dev/null 2>&1 || true + # The sim names its container/volume (auto_remove only fires on stop). + docker rm -f Meshtastic >/dev/null 2>&1 || true + docker volume rm Meshtasticator >/dev/null 2>&1 || true fi } trap cleanup EXIT echo "== waiting for $NODES node TCP API port(s) to accept connections ==" READY="" -for _ in $(seq 1 180); do # up to ~5 min: docker builds firmware on first run +for _ in $(seq 1 180); do # up to ~5 min: docker pulls meshtastic/meshtasticd on first run READY="$("$VENV/bin/python" - "$HOST" "$NODES" <<'PY' || true import socket, sys host, count = sys.argv[1], int(sys.argv[2]) @@ -114,6 +159,11 @@ if [ -z "$READY" ]; then fi echo " node ports: $READY" +# Ports accept before the node firmware finished booting inside the container; +# give meshtasticd a moment to open its TCP API for real. +echo "== settling (node firmware boot) ==" +sleep 15 + echo "== running radio-level e2e tests ==" ( cd "$REPO_ROOT" diff --git a/scripts/meshtasticator_e2e/tcp_forward.py b/scripts/meshtasticator_e2e/tcp_forward.py new file mode 100644 index 0000000..b543120 --- /dev/null +++ b/scripts/meshtasticator_e2e/tcp_forward.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Minimal TCP forwarder used by run_e2e.sh when the Docker daemon is remote. + +Meshtasticator's interactive simulator always connects to its simulated nodes +at localhost: (lib/interactive.py init_communication). On a Gitea +Actions runner whose Docker daemon is remote (e.g. DOCKER_HOST=tcp://docker.local:2375) +the container-published node ports live on the *daemon* host, not the job's +localhost — so we bind 127.0.0.1: in the job and forward to the daemon +host, restoring the localhost view the simulator (and our e2e tests) expect. + +Usage: tcp_forward.py --listen 127.0.0.1:4404 --target docker.local:4404 +""" + +import argparse +import socket +import threading + + +def _forward(src: socket.socket, dst: socket.socket): + try: + while True: + data = src.recv(65536) + if not data: + break + dst.sendall(data) + except OSError: + pass + finally: + try: + dst.shutdown(socket.SHUT_WR) + except OSError: + pass + + +def _handle(client: socket.socket, target_addr): + try: + upstream = socket.create_connection(target_addr, timeout=10) + except OSError: + client.close() + return + t = threading.Thread(target=_forward, args=(client, upstream), daemon=True) + t.start() + _forward(upstream, client) + client.close() + upstream.close() + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--listen", required=True, help="local bind, e.g. 127.0.0.1:4404") + ap.add_argument("--target", required=True, help="forward target, e.g. docker.local:4404") + args = ap.parse_args() + + lhost, lport = args.listen.rsplit(":", 1) + thost, tport = args.target.rsplit(":", 1) + target_addr = (thost, int(tport)) + + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind((lhost, int(lport))) + listener.listen(16) + print(f"forwarding {args.listen} -> {args.target}", flush=True) + + while True: + client, _ = listener.accept() + threading.Thread(target=_handle, args=(client, target_addr), daemon=True).start() + + +if __name__ == "__main__": + main() -- 2.52.0 From 58b18d6260de4ba93f619c32781494e9e67ef5a6 Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Mon, 7 Sep 2026 21:30:01 -0700 Subject: [PATCH 3/9] fix(ci): make run_e2e.sh executable; invoke via bash in workflow First dispatch (run 10535) failed instantly with Permission denied on the orchestrator: the file lost its executable bit in the branch move. Set mode 100755 and call through bash so a future mode regression cannot break the job. --- .gitea/workflows/meshtasticator-e2e.yaml | 2 +- scripts/meshtasticator_e2e/run_e2e.sh | 0 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 scripts/meshtasticator_e2e/run_e2e.sh diff --git a/.gitea/workflows/meshtasticator-e2e.yaml b/.gitea/workflows/meshtasticator-e2e.yaml index 5ba7feb..6cdcad8 100644 --- a/.gitea/workflows/meshtasticator-e2e.yaml +++ b/.gitea/workflows/meshtasticator-e2e.yaml @@ -79,7 +79,7 @@ jobs: export MESHTASTICATOR_DIR="$HOME/meshtasticator" export SIM_VENV="$HOME/sim-venv" export E2E_VENV="plugin/.venv-contract" - scripts/meshtasticator_e2e/run_e2e.sh --mode docker + bash scripts/meshtasticator_e2e/run_e2e.sh --mode docker - name: Print simulator log (on failure) if: failure() diff --git a/scripts/meshtasticator_e2e/run_e2e.sh b/scripts/meshtasticator_e2e/run_e2e.sh old mode 100644 new mode 100755 -- 2.52.0 From ba31d79368ca0ba4c498d3d9a15169bea0693eee Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Mon, 7 Sep 2026 21:33:49 -0700 Subject: [PATCH 4/9] fix(e2e): headless TkAgg crash + readiness false-positive on forwards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run 10536 reached pytest but every connect was reset: the simulator died at import with 'Tkinter is needed' — lib/gui.py calls matplotlib.use("TkAgg") unconditionally (an explicit use() overrides MPLBACKEND=Agg), and the port probe passed instantly because it was accepting the localhost forwarders, not the nodes. - run_e2e.sh: patch meshtasticator lib/gui.py TkAgg->Agg before boot; probe readiness on the upstream host (docker daemon) when forwards are active; docker pull meshtastic/meshtasticd up front so the pull is off the simulator's node-boot path. - e2e tests: cancel leftover loop tasks (watchdog) at loop teardown. --- plugin/tests/e2e/test_meshtasticator_e2e.py | 10 ++++++++ scripts/meshtasticator_e2e/README.md | 10 +++++++- scripts/meshtasticator_e2e/run_e2e.sh | 28 ++++++++++++++++++--- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/plugin/tests/e2e/test_meshtasticator_e2e.py b/plugin/tests/e2e/test_meshtasticator_e2e.py index c5b0837..5476e48 100644 --- a/plugin/tests/e2e/test_meshtasticator_e2e.py +++ b/plugin/tests/e2e/test_meshtasticator_e2e.py @@ -143,6 +143,16 @@ def loop(): try: yield _loop finally: + # Cancel leftover tasks (e.g. a watchdog started by a failed connect + # whose disconnect teardown never ran) before closing the loop. + async def _shutdown(): + for task in asyncio.all_tasks(_loop): + task.cancel() + + try: + asyncio.run_coroutine_threadsafe(_shutdown(), _loop).result(timeout=5) + except Exception: + pass _loop.call_soon_threadsafe(_loop.stop) thread.join(timeout=5) diff --git a/scripts/meshtasticator_e2e/README.md b/scripts/meshtasticator_e2e/README.md index 08073b1..eebbb24 100644 --- a/scripts/meshtasticator_e2e/README.md +++ b/scripts/meshtasticator_e2e/README.md @@ -81,7 +81,15 @@ connections (Meshtasticator assigns node *n* → port `4404 + n`), then runs control connections assume nodes listen on localhost, so the script starts a `tcp_forward.py` per node port (localhost → daemon host) before booting the simulator. Nothing is forwarded when the daemon is local (unset/unix -`DOCKER_HOST`). +`DOCKER_HOST`). Readiness is probed on the *upstream* host (the daemon), never +on the forwarders themselves. + +Other headless adaptations the script makes automatically: +* patches `lib/gui.py` `matplotlib.use("TkAgg")` → `"Agg"` in the pinned + Meshtasticator checkout — the TkAgg call crashes headless runs even under + `MPLBACKEND=Agg` (only the interactive GUI mode is lost), +* warms the `meshtastic/meshtasticd` image (`docker pull`) before booting the + simulator so the pull is not on the simulator's node-boot critical path. Without `MESHTASTICATOR_E2E=1` the e2e module skips, so ordinary unit/CI runs stay fast and gateway-free. diff --git a/scripts/meshtasticator_e2e/run_e2e.sh b/scripts/meshtasticator_e2e/run_e2e.sh index 06327aa..a4fe846 100755 --- a/scripts/meshtasticator_e2e/run_e2e.sh +++ b/scripts/meshtasticator_e2e/run_e2e.sh @@ -104,6 +104,26 @@ fi if [ -n "$DOCKER_TARGET" ]; then echo "== remote docker daemon detected ($DOCKER_TARGET): starting localhost port forwards ==" start_forwards + PROBE_HOST="$DOCKER_TARGET" +else + PROBE_HOST="$HOST" +fi + +# The simulator forces TkAgg at import (lib/gui.py: matplotlib.use("TkAgg")), +# which crashes headless runs ("Tkinter is needed") even under MPLBACKEND=Agg — +# an explicit use() overrides the env var. Neutralize it in the pinned +# checkout; only the interactive GUI mode (never used by this harness) is lost. +if grep -q 'matplotlib.use("TkAgg")' "$SIM_DIR/lib/gui.py"; then + echo "== patching meshtasticator lib/gui.py: TkAgg -> Agg (headless) ==" + sed -i.bak 's/matplotlib.use("TkAgg")/matplotlib.use("Agg")/' "$SIM_DIR/lib/gui.py" +fi + +if [ "$MODE" = docker ]; then + # Warm the node image on the (possibly remote) daemon first: the simulator + # starts node processes ~4s after the container is created and control + # connections must succeed quickly, so the pull cannot sit in that path. + echo "== warming meshtastic/meshtasticd image ==" + docker pull meshtastic/meshtasticd fi echo "== booting Meshtasticator ($MODE) with $NODES node(s) from $SIM_DIR ==" @@ -131,7 +151,9 @@ trap cleanup EXIT echo "== waiting for $NODES node TCP API port(s) to accept connections ==" READY="" for _ in $(seq 1 180); do # up to ~5 min: docker pulls meshtastic/meshtasticd on first run - READY="$("$VENV/bin/python" - "$HOST" "$NODES" <<'PY' || true + # Probe the upstream node host (the docker daemon when forwards are active, + # localhost otherwise) — probing the forwarders themselves would always pass. + READY="$("$VENV/bin/python" - "$PROBE_HOST" "$NODES" <<'PY' || true import socket, sys host, count = sys.argv[1], int(sys.argv[2]) found = [] @@ -154,10 +176,10 @@ PY sleep 2 done if [ -z "$READY" ]; then - echo "error: no Meshtasticator node ports became reachable on $HOST (see $SIM_LOG)" >&2 + echo "error: no Meshtasticator node ports became reachable on $PROBE_HOST (see $SIM_LOG)" >&2 exit 1 fi -echo " node ports: $READY" +echo " node ports: $READY (probed on $PROBE_HOST)" # Ports accept before the node firmware finished booting inside the container; # give meshtasticd a moment to open its TCP API for real. -- 2.52.0 From eff352b2eac66b8ecf36b855c7e7cc1438ee2335 Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Mon, 7 Sep 2026 21:41:50 -0700 Subject: [PATCH 5/9] fix(e2e): headless Graph construction + keep simulator cmdloop alive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run 10537 reached node boot but the simulator crashed again at Graph construction: lib/gui.py move_figure() unconditionally touches the Tk window manager (canvas.manager.window), which does not exist under Agg. Patch it to a no-op guard alongside the TkAgg->Agg backend swap. Also feed the simulator's interactive cmdloop from a FIFO held open until teardown — CI stdin is EOF, so the sim would otherwise exit immediately after booting the nodes. --- scripts/meshtasticator_e2e/README.md | 13 +++++--- scripts/meshtasticator_e2e/run_e2e.sh | 45 +++++++++++++++++++++------ 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/scripts/meshtasticator_e2e/README.md b/scripts/meshtasticator_e2e/README.md index eebbb24..78e3710 100644 --- a/scripts/meshtasticator_e2e/README.md +++ b/scripts/meshtasticator_e2e/README.md @@ -85,11 +85,16 @@ simulator. Nothing is forwarded when the daemon is local (unset/unix on the forwarders themselves. Other headless adaptations the script makes automatically: -* patches `lib/gui.py` `matplotlib.use("TkAgg")` → `"Agg"` in the pinned - Meshtasticator checkout — the TkAgg call crashes headless runs even under - `MPLBACKEND=Agg` (only the interactive GUI mode is lost), +* patches `lib/gui.py` in the pinned Meshtasticator checkout: forces the Agg + backend (the code calls `matplotlib.use("TkAgg")` at import) and makes + `move_figure()` a no-op when no Tk window manager exists (its unconditional + `canvas.manager.window` access crashes under Agg at Graph construction). + Only the interactive GUI modes, never used by this harness, are lost, * warms the `meshtastic/meshtasticd` image (`docker pull`) before booting the - simulator so the pull is not on the simulator's node-boot critical path. + simulator so the pull is not on the simulator's node-boot critical path, +* feeds the simulator's interactive `cmdloop` from a FIFO held open until + teardown — an EOF stdin (e.g. CI) would otherwise make it exit right after + booting the nodes. Without `MESHTASTICATOR_E2E=1` the e2e module skips, so ordinary unit/CI runs stay fast and gateway-free. diff --git a/scripts/meshtasticator_e2e/run_e2e.sh b/scripts/meshtasticator_e2e/run_e2e.sh index a4fe846..3a527dc 100755 --- a/scripts/meshtasticator_e2e/run_e2e.sh +++ b/scripts/meshtasticator_e2e/run_e2e.sh @@ -109,14 +109,33 @@ else PROBE_HOST="$HOST" fi -# The simulator forces TkAgg at import (lib/gui.py: matplotlib.use("TkAgg")), -# which crashes headless runs ("Tkinter is needed") even under MPLBACKEND=Agg — -# an explicit use() overrides the env var. Neutralize it in the pinned -# checkout; only the interactive GUI mode (never used by this harness) is lost. -if grep -q 'matplotlib.use("TkAgg")' "$SIM_DIR/lib/gui.py"; then - echo "== patching meshtasticator lib/gui.py: TkAgg -> Agg (headless) ==" - sed -i.bak 's/matplotlib.use("TkAgg")/matplotlib.use("Agg")/' "$SIM_DIR/lib/gui.py" -fi +# The simulator assumes an interactive Tk desktop: lib/gui.py forces +# matplotlib.use("TkAgg") at import and move_figure() touches the Tk window +# manager at Graph construction — both crash headless runs even under +# MPLBACKEND=Agg. Neutralize them in the pinned checkout (only the interactive +# GUI modes, never used by this harness, are lost). +echo "== patching meshtasticator lib/gui.py for headless runs ==" +"$VENV/bin/python" - "$SIM_DIR" <<'PY' +import sys +from pathlib import Path + +gui = Path(sys.argv[1]) / "lib/gui.py" +src = gui.read_text() + +src = src.replace('matplotlib.use("TkAgg")', 'matplotlib.use("Agg")') + +old = 'def move_figure(fig, x, y):\n fig.canvas.manager.window.wm_geometry("+%d+%d" % (x, y))' +new = ('def move_figure(fig, x, y):\n' + ' # Headless (Agg) backends have no Tk window manager.\n' + ' try:\n' + ' fig.canvas.manager.window.wm_geometry("+%d+%d" % (x, y))\n' + ' except AttributeError:\n' + ' pass') +assert old in src, "move_figure source no longer matches the pinned meshtasticator" +src = src.replace(old, new) +gui.write_text(src) +print("patched lib/gui.py") +PY if [ "$MODE" = docker ]; then # Warm the node image on the (possibly remote) daemon first: the simulator @@ -127,14 +146,22 @@ if [ "$MODE" = docker ]; then fi echo "== booting Meshtasticator ($MODE) with $NODES node(s) from $SIM_DIR ==" +# The simulator ends in an interactive cmdloop reading stdin; an EOF there +# (e.g. CI) would make it exit right after booting the nodes. Feed it from a +# FIFO whose write end this shell keeps open until cleanup. +SIM_FIFO="$(mktemp -u /tmp/meshtasticator-XXXXXX.fifo)" +mkfifo "$SIM_FIFO" ( cd "$SIM_DIR" - "$SIM_PYTHON" interactiveSim.py "${SIM_ARGS[@]}" >"$SIM_LOG" 2>&1 + "$SIM_PYTHON" interactiveSim.py "${SIM_ARGS[@]}" <"$SIM_FIFO" >"$SIM_LOG" 2>&1 ) & SIM_PID=$! +exec 9>"$SIM_FIFO" # hold the write end open -> sim stdin never hits EOF +rm -f "$SIM_FIFO" cleanup() { echo "== tearing down simulator (pid $SIM_PID) ==" + exec 9>&- 2>/dev/null || true # release the FIFO write end (sim stdin EOF) kill "$SIM_PID" 2>/dev/null || true wait "$SIM_PID" 2>/dev/null || true for pid in "${FWD_PIDS[@]:-}"; do -- 2.52.0 From 6125130e6ac2b80cd7c770561cbb3e5ef941b9cc Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Mon, 7 Sep 2026 21:46:25 -0700 Subject: [PATCH 6/9] fix(e2e): peer sendText moved to Interface; add RF failure diagnostics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run 10538 booted the full simulator and ran pytest — both failures were test-side: the installed meshtastic client moved sendText from Node to the Interface, and the outbound chunked message never arrived at the peer (cause still unknown). Fix the sendText call and add in-run diagnostics so the next failure is self-explaining: - e2e: outbound-timeout failure now reports what arrived at the peer and the simulator log tail; sim now boots with -v (meshtastic debug logging). - run_e2e.sh: on pytest failure in docker mode, dump per-node meshtasticd logs (meshtasticator-nodes.log) before cleanup removes the container. --- plugin/tests/e2e/test_meshtasticator_e2e.py | 22 ++++++++++++++++++++- scripts/meshtasticator_e2e/run_e2e.sh | 13 +++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/plugin/tests/e2e/test_meshtasticator_e2e.py b/plugin/tests/e2e/test_meshtasticator_e2e.py index 5476e48..5d19278 100644 --- a/plugin/tests/e2e/test_meshtasticator_e2e.py +++ b/plugin/tests/e2e/test_meshtasticator_e2e.py @@ -226,10 +226,11 @@ def test_adapter_receives_broadcast_from_peer(node0): return any(getattr(e, "text", None) == sent for e in node0.received_events) # Radio delivery is not guaranteed per attempt; retry a few times. + # (meshtastic client >=2.8: sendText lives on the Interface, not Node.) for _ in range(3): if _got(): break - peer.localNode.sendText( + peer.sendText( text=sent, destinationId="^all", channelIndex=E2E_CHANNEL_INDEX ) time.sleep(1) @@ -271,6 +272,25 @@ def test_adapter_chunked_send_reaches_peer(node0, loop): return any(str(r).rstrip().endswith("word59") for r in received) _wait_for(_all_chunks_arrived, timeout_s=45, what="chunked outbound send at peer") + + except Exception as exc: + # Fast-fail diagnostics: what arrived at the peer, and what the + # simulator itself logged (RF forwarding/routing noise). + tail = "" + try: + sim_log = os.path.join(os.getcwd(), "..", "..", "meshtasticator-sim.log") + if not os.path.exists(sim_log): + sim_log = os.path.join(os.getcwd(), "meshtasticator-sim.log") + if os.path.exists(sim_log): + tail = "\n".join( + open(sim_log, errors="replace").read().splitlines()[-40:] + ) + except OSError: + pass + raise AssertionError( + f"{exc}\nreceived at peer so far: {received[-10:]!r}\n" + f"--- meshtasticator-sim.log tail ---\n{tail}" + ) from exc finally: pub.unsubscribe(_on_receive, "meshtastic.receive.text") peer.close() diff --git a/scripts/meshtasticator_e2e/run_e2e.sh b/scripts/meshtasticator_e2e/run_e2e.sh index 3a527dc..e3f5da4 100755 --- a/scripts/meshtasticator_e2e/run_e2e.sh +++ b/scripts/meshtasticator_e2e/run_e2e.sh @@ -93,7 +93,7 @@ start_forwards() { } SIM_LOG="$REPO_ROOT/meshtasticator-sim.log" -SIM_ARGS=("$NODES") +SIM_ARGS=("$NODES" -v) # -v: meshtastic/debug logging for RF diagnostics if [ "$MODE" = native ]; then [ -n "$PROGRAM" ] && [ -d "$PROGRAM" ] || { echo "--mode native needs --program " >&2; exit 2; } SIM_ARGS+=(-p "$PROGRAM") @@ -222,3 +222,14 @@ echo "== running radio-level e2e tests ==" E2E_CHANNEL_INDEX="$CHANNEL_INDEX" \ "$VENV/bin/python" -m pytest plugin/tests/e2e -q "$@" ) +RC=$? + +if [ $RC -ne 0 ] && [ "$MODE" = docker ]; then + # Capture per-node meshtasticd logs before cleanup removes the container. + echo "== dumping node logs (pytest rc=$RC) ==" + docker exec Meshtastic sh -c 'for f in /home/out_*.log; do echo "----- $f -----"; tail -n 60 "$f" 2>/dev/null; done' \ + >"$REPO_ROOT/meshtasticator-nodes.log" 2>&1 || true + echo " node logs: $REPO_ROOT/meshtasticator-nodes.log" +fi + +exit $RC -- 2.52.0 From cbef4635cbbf717203473630a2438bef06312e88 Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Mon, 7 Sep 2026 21:50:26 -0700 Subject: [PATCH 7/9] fix(e2e): instrument sim RF path; graceful shutdown; richer failure logs Both e2e directions time out: nothing is delivered between nodes although the adapter connects fine and bridging code is equivalent to meshtastic's own firmware_harness. Before concluding, instrument the pinned meshtasticator: - on_receive now prints every observed packet (topic keys) to sim.log, - a [sim-text] listener prints any text packet any sim interface sees, - cleanup sends 'exit' through the cmdloop (graceful, no EOF-spam flood that was drowning the log tail), - workflow failure step prints 400 sim-log lines + the node logs dump. --- .gitea/workflows/meshtasticator-e2e.yaml | 6 +++-- scripts/meshtasticator_e2e/run_e2e.sh | 30 ++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/.gitea/workflows/meshtasticator-e2e.yaml b/.gitea/workflows/meshtasticator-e2e.yaml index 6cdcad8..d4a0dc0 100644 --- a/.gitea/workflows/meshtasticator-e2e.yaml +++ b/.gitea/workflows/meshtasticator-e2e.yaml @@ -84,7 +84,9 @@ jobs: - name: Print simulator log (on failure) if: failure() run: | - echo "===== meshtasticator-sim.log =====" - tail -n 200 meshtasticator-sim.log 2>/dev/null || true + echo "===== meshtasticator-sim.log (tail) =====" + tail -n 400 meshtasticator-sim.log 2>/dev/null || true + echo "===== meshtasticator-nodes.log (if any) =====" + tail -n 200 meshtasticator-nodes.log 2>/dev/null || true echo "===== docker ps =====" docker ps -a 2>/dev/null || true diff --git a/scripts/meshtasticator_e2e/run_e2e.sh b/scripts/meshtasticator_e2e/run_e2e.sh index e3f5da4..2a5a034 100755 --- a/scripts/meshtasticator_e2e/run_e2e.sh +++ b/scripts/meshtasticator_e2e/run_e2e.sh @@ -135,6 +135,32 @@ assert old in src, "move_figure source no longer matches the pinned meshtasticat src = src.replace(old, new) gui.write_text(src) print("patched lib/gui.py") + +# --- RF instrumentation (prints land in meshtasticator-sim.log) --------------- +# Why nothing is delivered between nodes is not yet known; log every packet the +# sim's per-node interfaces observe so the next failing run explains itself. +interactive = Path(sys.argv[1]) / "lib" / "interactive.py" +src = interactive.read_text() + +old = ' def on_receive(self, interface, packet):\n' +new = old + ( + ' print(f"[sim] on_receive port={getattr(interface, \'portNumber\', None)} ' + 'decoded_keys={list(packet.get(\'decoded\', {}).keys())}")\n' +) +assert old in src, "on_receive signature moved in the pinned meshtasticator" +src = src.replace(old, new, 1) + +old = ' pub.subscribe(self.on_receive, "meshtastic.receive.simulator")' +new = old + ( + '\n pub.subscribe(' + 'lambda interface, packet: print(f"[sim-text] port={getattr(interface, \'portNumber\', None)} ' + 'text={packet.get(\'decoded\', {}).get(\'text\')!r}"), "meshtastic.receive.text")' +) +assert old in src, "simulator subscription moved in the pinned meshtasticator" +src = src.replace(old, new, 1) + +interactive.write_text(src) +print("patched lib/interactive.py (RF instrumentation)") PY if [ "$MODE" = docker ]; then @@ -161,6 +187,10 @@ rm -f "$SIM_FIFO" cleanup() { echo "== tearing down simulator (pid $SIM_PID) ==" + # Graceful 'exit' through the cmdloop (avoids hundreds of EOF spam lines in + # sim.log and lets the sim close its own nodes/container). + echo exit >&9 2>/dev/null || true + sleep 2 exec 9>&- 2>/dev/null || true # release the FIFO write end (sim stdin EOF) kill "$SIM_PID" 2>/dev/null || true wait "$SIM_PID" 2>/dev/null || true -- 2.52.0 From 21987411324198cf05cced29955e73fbb3a21d94 Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Mon, 7 Sep 2026 21:53:31 -0700 Subject: [PATCH 8/9] ci(e2e): run simulator with current meshtastic client, not the stale 2.6.1 pin Meshtasticator's requirements.txt pins meshtastic~=2.6.1, which predates the meshtastic/meshtasticd image it drives; with the old client the simulator never observed node TX packets (both e2e directions timed out). The official meshtastic firmware_harness uses the current client against the same simulator-mode meshtasticd, so install the sim deps with meshtastic floating instead of the pinned requirements file. --- .gitea/workflows/meshtasticator-e2e.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/meshtasticator-e2e.yaml b/.gitea/workflows/meshtasticator-e2e.yaml index d4a0dc0..6f97825 100644 --- a/.gitea/workflows/meshtasticator-e2e.yaml +++ b/.gitea/workflows/meshtasticator-e2e.yaml @@ -71,8 +71,15 @@ jobs: - name: Simulator venv (its own requirements + docker SDK) run: | uv venv --python 3.13 "$HOME/sim-venv" + # NOTE: meshtasticator/requirements.txt pins meshtastic~=2.6.1, which + # predates the meshtastic/meshtasticd image it talks to and never + # observed TX packets (both e2e directions timed out). Install the + # simulator's deps but let meshtastic float to the current client — + # the version the official firmware_harness uses against the same + # meshtasticd simulator mode. uv pip install --python "$HOME/sim-venv/bin/python" \ - -r "$HOME/meshtasticator/requirements.txt" docker + meshtastic numpy matplotlib pandas \ + 'PyPubSub==4.0.3' simpy PyYAML protobuf docker - name: Run Meshtasticator e2e suite run: | -- 2.52.0 From b3063239f7e43dab120246f8de9a07bb4f119e96 Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Mon, 7 Sep 2026 21:56:48 -0700 Subject: [PATCH 9/9] fix(e2e): deterministic in-range node scenario + log receiver decisions The sim now observes TX from both nodes (current client fixed observation) but nothing is ever delivered to the other node. Suspect: random placement leaving nodes out of range, or the receiver decision. Provide a --from-file scenario with nodes spaced 10 m apart (guaranteed adjacency) and print the transmitter -> receivers decision per TX in the sim log. --- scripts/meshtasticator_e2e/run_e2e.sh | 34 ++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/scripts/meshtasticator_e2e/run_e2e.sh b/scripts/meshtasticator_e2e/run_e2e.sh index 2a5a034..df34a94 100755 --- a/scripts/meshtasticator_e2e/run_e2e.sh +++ b/scripts/meshtasticator_e2e/run_e2e.sh @@ -93,7 +93,31 @@ start_forwards() { } SIM_LOG="$REPO_ROOT/meshtasticator-sim.log" -SIM_ARGS=("$NODES" -v) # -v: meshtastic/debug logging for RF diagnostics +# Deterministic topology: random placement may leave nodes out of range +# (nothing was ever delivered between nodes). Provide a --from-file scenario +# with all nodes within ~10 m of each other so RF adjacency is guaranteed. +NODE_CONF_DIR="$SIM_DIR/out" +mkdir -p "$NODE_CONF_DIR" +"$VENV/bin/python" - "$NODE_CONF_DIR/nodeConfig.yaml" "$NODES" <<'PY' +import sys +from pathlib import Path + +path, count = sys.argv[1], int(sys.argv[2]) +conf = {} +for i in range(count): + x = float((i - (count - 1) / 2) * 10.0) # 10 m spacing around the origin + conf[i] = { + "x": x, "y": 0.0, "z": 1.0, + "isRouter": False, "isRepeater": False, "isClientMute": False, + "hopLimit": 3, "antennaGain": 0, "neighborInfo": False, + } +Path(path).write_text( + "".join(f"{k}:\n" + "".join(f" {ck}: {cv}\n" for ck, cv in v.items()) + for k, v in conf.items()) +) +print(f"wrote deterministic node scenario ({count} nodes) to {path}") +PY +SIM_ARGS=("--from-file" -v) # node count/positions come from nodeConfig.yaml if [ "$MODE" = native ]; then [ -n "$PROGRAM" ] && [ -d "$PROGRAM" ] || { echo "--mode native needs --program " >&2; exit 2; } SIM_ARGS+=(-p "$PROGRAM") @@ -159,6 +183,14 @@ new = old + ( assert old in src, "simulator subscription moved in the pinned meshtasticator" src = src.replace(old, new, 1) +old = ' rxs, rssis, snrs = self.calc_receivers(transmitter, receivers)' +new = old + ( + '\n print(f"[sim] TX node={transmitter.nodeid} candidates={[n.nodeid for n in receivers]} ' + 'rxs={[n.nodeid for n in rxs]} rssis={rssis}")' +) +assert old in src, "calc_receivers call moved in the pinned meshtasticator" +src = src.replace(old, new, 1) + interactive.write_text(src) print("patched lib/interactive.py (RF instrumentation)") PY -- 2.52.0