From a4abb5d15d2969a2da1ed681a53a69eb4b408c15 Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Mon, 7 Sep 2026 20:56:28 -0700 Subject: [PATCH] feat(e2e): Meshtasticator radio-level validation harness for the adapter Adds a skip-gated end-to-end suite (plugin/tests/e2e) that drives the adapter against live Meshtasticator simulated nodes - real MeshtasticD instances speaking the actual TCP/protobuf API - covering the layer no contract test can reach: TCP handshake, inbound text over the simulated mesh, and chunked outbound sends arriving at a peer node. scripts/meshtasticator_e2e/run_e2e.sh boots N nodes (Docker or Linux native MeshtasticD), discovers their TCP API ports, and runs the suite. Radio-level reconnect and CI wiring are documented as open items pending a Linux/Docker validation run. --- plugin/tests/e2e/test_meshtasticator_e2e.py | 227 ++++++++++++++++++++ scripts/meshtasticator_e2e/README.md | 99 +++++++++ scripts/meshtasticator_e2e/run_e2e.sh | 125 +++++++++++ 3 files changed, 451 insertions(+) create mode 100644 plugin/tests/e2e/test_meshtasticator_e2e.py create mode 100644 scripts/meshtasticator_e2e/README.md create mode 100644 scripts/meshtasticator_e2e/run_e2e.sh diff --git a/plugin/tests/e2e/test_meshtasticator_e2e.py b/plugin/tests/e2e/test_meshtasticator_e2e.py new file mode 100644 index 0000000..7096427 --- /dev/null +++ b/plugin/tests/e2e/test_meshtasticator_e2e.py @@ -0,0 +1,227 @@ +"""End-to-end tests: hermes-meshtastic adapter against live Meshtasticator nodes. + +What these tests validate (the "radio-level" layer no contract test can): + * the adapter's real TCP/protobuf handshake against MeshtasticD (the actual + device software, not a mock) as run by the Meshtasticator interactive + simulator (https://meshtastic.org/docs/software/meshtasticator/), + * inbound text reception across the simulated LoRa mesh, + * outbound chunked sends arriving at another simulated node, + * the whole path: pubsub callback -> MessageEvent construction (real gateway + types) -> handle_message dispatch. + +What they do NOT validate (out of scope even for Meshtasticator): true RF +(silicon-level LoRa, real propagation/interference, regulatory duty cycle). + +Running +------- + 1. On a Linux host (or a machine with Docker): clone meshtastic/meshtasticator + and boot the interactive simulator (see scripts/meshtasticator_e2e/). + 2. Point the plugin's own contract venv at the booted nodes and run: + cd plugin && MESHTASTICATOR_E2E=1 E2E_PORTS=4404,4405 \ + E2E_CHANNEL_INDEX=0 .venv-contract/bin/python -m pytest tests/e2e -q + +Environment +----------- + MESHTASTICATOR_E2E = "1" enables these tests (required; without it the + module skips so ordinary unit/CI runs stay fast). + E2E_PORTS = comma-separated TCP API ports of the simulated nodes + (first = the node the adapter connects to, default 4404). + E2E_HOST = host where the simulator exposes the ports (default 127.0.0.1). + E2E_CHANNEL_INDEX = Meshtastic channel index the adapter listens/sends on + (default 0 = primary channel, matching a stock sim node). + +NOTE: this module is an initial scaffold. Structural validation (imports, +fixtures, flow) passes, but a full green run requires a Linux/Docker host with +a MeshtasticD build — tracked as the remaining verification step. +""" + +import asyncio +import os + +import pytest + +if os.environ.get("MESHTASTICATOR_E2E") != "1": + pytest.skip( + "set MESHTASTICATOR_E2E=1 to run against Meshtasticator nodes " + "(see scripts/meshtasticator_e2e/README.md)", + allow_module_level=True, + ) + +# Real Hermes gateway + real Meshtastic client library are required. +pytest.importorskip("gateway.platforms.base") +pytest.importorskip("meshtastic.tcp_interface") + +from gateway.config import PlatformConfig # noqa: E402 +from gateway.platform_registry import ( # noqa: E402 + PlatformEntry, + platform_registry, +) + +import hermes_meshtastic # noqa: E402 +from hermes_meshtastic import adapter as plugin_adapter # noqa: E402 + +E2E_HOST = os.environ.get("E2E_HOST", "127.0.0.1") +E2E_PORTS = [int(p) for p in os.environ.get("E2E_PORTS", "4404,4405").split(",") if p] +E2E_CHANNEL_INDEX = int(os.environ.get("E2E_CHANNEL_INDEX", "0")) +assert len(E2E_PORTS) >= 2, "E2E_PORTS needs at least two node ports (adapter + peer)" + +NODE0_PORT = E2E_PORTS[0] # the node our adapter connects to +NODE1_PORT = E2E_PORTS[1] # the peer node that sends / observes +CONNECT_TIMEOUT_S = 30 +RADIO_SETTLE_S = 2 + + +def _register_meshtastic_platform(): + """Register the plugin (mirrors discovery) so Platform('meshtastic') resolves.""" + if platform_registry.is_registered("meshtastic"): + return + kwargs = {} + + class _Ctx: + def register_platform(self, **kw): + kwargs.update(kw) + + hermes_meshtastic.register(_Ctx()) + platform_registry.register( + PlatformEntry( + name=kwargs["name"], + label=kwargs["label"], + adapter_factory=kwargs["adapter_factory"], + check_fn=kwargs["check_fn"], + required_env=[], + max_message_length=kwargs.get("max_message_length", 0), + emoji=kwargs.get("emoji", "🔌"), + platform_hint=kwargs.get("platform_hint", ""), + ) + ) + + +class _ProbeAdapter(plugin_adapter.get_adapter_class()): + """MeshtasticAdapter that records inbound MessageEvents instead of + dispatching to a Hermes agent handler (no agent runs in these tests).""" + + def __init__(self, config): + super().__init__(config) + self.received_events = [] + + async def handle_message(self, event): + self.received_events.append(event) + # Intentionally do not call super(): the real base would forward to a + # message handler that only exists inside a running Hermes gateway. + return None + + +@pytest.fixture(scope="module") +def adapter(): + _register_meshtastic_platform() + cfg = PlatformConfig( + enabled=True, + extra={ + "host": E2E_HOST, + "port": NODE0_PORT, + "channel_index": E2E_CHANNEL_INDEX, + }, + ) + return _ProbeAdapter(cfg) + + +@pytest.fixture(scope="module") +def node0(adapter): + """Connect the adapter to simulated node 0; disconnect afterwards.""" + async def _connect(): + return await adapter.connect() + + assert asyncio.run(_connect()) is True, ( + f"adapter failed to connect to simulated node at {E2E_HOST}:{NODE0_PORT} " + "(is the simulator running and the node reachable?)" + ) + try: + yield adapter + finally: + asyncio.run(adapter.disconnect()) + + +def _wait_for(predicate, timeout_s=15.0, interval_s=0.5, what="condition"): + """Poll until *predicate* (callable -> bool) is true, else pytest.fail.""" + import time + + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(interval_s) + pytest.fail(f"timed out waiting for {what}") + + +# --------------------------------------------------------------------------- +# Radio-level flows +# --------------------------------------------------------------------------- + + +def test_adapter_receives_broadcast_from_peer(node0): + """Peer node 1 broadcasts text over the simulated mesh; the adapter must + receive it as a real gateway MessageEvent.""" + import time + + from meshtastic import tcp_interface + + sent = f"e2e-inbound-{int(time.time())}" + + peer = tcp_interface.TCPInterface(hostname=E2E_HOST, portNumber=NODE1_PORT) + try: + time.sleep(RADIO_SETTLE_S) + + def _got(): + return any(getattr(e, "text", None) == sent for e in node0.received_events) + + # Radio delivery is not guaranteed per attempt; retry a few times. + for _ in range(3): + if _got(): + break + peer.localNode.sendText( + text=sent, destinationId="^all", channelIndex=E2E_CHANNEL_INDEX + ) + time.sleep(1) + _wait_for(_got, timeout_s=25, what="inbound broadcast from peer node") + finally: + peer.close() + + +def test_adapter_chunked_send_reaches_peer(node0): + """A long outbound message must be chunked and arrive at the peer node.""" + import time + + from meshtastic import tcp_interface + from pubsub import pub + + content = " ".join(f"word{n}" for n in range(60)) # well over 180 chars -> chunks + received = [] + + def _on_receive(packet, interface): + decoded = packet.get("decoded", {}) or {} + text = decoded.get("text", "") + if text and text.startswith("(1/"): # our chunked framing + received.append(text) + elif text: + received.append(text) + + peer = tcp_interface.TCPInterface(hostname=E2E_HOST, portNumber=NODE1_PORT) + try: + time.sleep(RADIO_SETTLE_S) + pub.subscribe(_on_receive, "meshtastic.receive.text") + + async def _send(): + result = await node0.send("channel_e2e", content) + assert result.success, f"adapter.send failed: {result.error}" + + asyncio.run(_send()) + + def _all_chunks_arrived(): + # Adapter frames multi-part sends as "(i/n) "; the final + # chunk carries the message tail. + return any(str(r).rstrip().endswith("word59") for r in received) + + _wait_for(_all_chunks_arrived, timeout_s=45, what="chunked outbound send at peer") + finally: + pub.unsubscribe(_on_receive, "meshtastic.receive.text") + peer.close() diff --git a/scripts/meshtasticator_e2e/README.md b/scripts/meshtasticator_e2e/README.md new file mode 100644 index 0000000..2537853 --- /dev/null +++ b/scripts/meshtasticator_e2e/README.md @@ -0,0 +1,99 @@ +# Meshtasticator e2e — radio-level validation for hermes-meshtastic + +This directory is the **radio-level counterpart** to the contract suite +(`plugin/tests/contract`). The contract suite guarantees the adapter matches +the Hermes gateway API; it cannot prove the adapter actually talks to a +Meshtastic node. These e2e tests boot the +[Meshtasticator](https://meshtastic.org/docs/software/meshtasticator/) +interactive simulator — which runs the **real MeshtasticD device software** +per node with a simulated LoRa PHY — and drive the adapter against a live +simulated node. + +| Layer | Validated by | +|---|---| +| Adapter ↔ Hermes gateway API | `plugin/tests/contract` (no deployment) | +| Adapter ↔ real MeshtasticD TCP/protobuf session | **this suite** | +| Inbound text across the (simulated) LoRa mesh | **this suite** | +| Outbound chunked sends arriving at another node | **this suite** | +| True RF (silicon LoRa, real propagation, duty cycle) | physical radio only | + +## Layout + +``` +scripts/meshtasticator_e2e/ + run_e2e.sh orchestrator: boots N nodes, runs pytest, tears down + README.md this file +plugin/tests/e2e/ + test_meshtasticator_e2e.py the e2e assertions (skip-gated) +``` + +## Prerequisites + +* **uv** + Python 3.13. +* A contract venv with the real gateway **and** the plugin (with deps) installed: + ```bash + scripts/contract-test.sh # creates plugin/.venv-contract (hermes-agent pinned) + uv pip install --python plugin/.venv-contract/bin/python -e plugin + ``` + (`-e plugin` installs the meshtastic client library the tests also use.) +* A clone of Meshtasticator: `git clone https://github.com/meshtastic/meshtasticator` +* Runtime for the simulator — pick one: + * **Docker** (simplest, works on macOS/Windows): the simulator builds + MeshtasticD from a container image on first run (slow once). + * **Linux native**: build MeshtasticD's `native` PlatformIO target and pass + the build dir with `--program`. + +## Run + +```bash +# Docker mode (default) +MESHTASTICATOR_DIR=/path/to/meshtasticator \ + scripts/meshtasticator_e2e/run_e2e.sh + +# Linux native mode +MESHTASTICATOR_DIR=/path/to/meshtasticator \ + scripts/meshtasticator_e2e/run_e2e.sh \ + --mode native --program /path/to/firmware/.pio/build/native + +# Tune the scenario + ... --nodes 3 --channel-index 0 --host 127.0.0.1 +``` + +The orchestrator boots the simulator, waits until `N` node TCP API ports accept +connections (Meshtasticator assigns node *n* → port `4404 + n`), then runs +`pytest plugin/tests/e2e` with `MESHTASTICATOR_E2E=1`. Simulator output lands in +`meshtasticator-sim.log` at the repo root. + +Without `MESHTASTICATOR_E2E=1` the e2e module skips, so ordinary unit/CI runs +stay fast and gateway-free. + +## What the tests do + +1. **Inbound**: peer node 1 broadcasts text; the adapter (connected to node 0) + must receive it as a real gateway `MessageEvent` (`host/port/channel_index` + come from `E2E_PORTS`/`E2E_HOST`/`E2E_CHANNEL_INDEX`). +2. **Outbound**: the adapter sends a >180-char message; it must be chunked and + the final chunk (`(i/n) … word59`) must arrive at peer node 1. + +A `_ProbeAdapter` subclass records inbound events instead of dispatching to a +Hermes agent handler (no agent runs in these tests; dispatch is already covered +by the contract suite). + +## Status / open items (v1 scaffold) + +This is an initial, structurally validated scaffold. A full green run still +needs to happen on a Linux/Docker host with a MeshtasticD build, and these +items are expected to need iteration there: + +* **Node adjacency**: the simulator's default random placement may put nodes out + of range. For deterministic tests, seed close-together coordinates (the sim + supports `--from-file` with an `out/nodeConfig.yaml`). +* **Channel semantics**: tests default to channel index 0 (stock primary + channel). Mirroring production (channel 1 / private channel) requires + configuring matching channels on the simulated nodes. +* **Reconnect flow**: killing/restarting a single simulated node needs per-node + lifecycle control (native mode subprocesses; docker mode currently runs all + nodes in one container), so the adapter's watchdog reconnect is **not** yet + exercised here. See the watchdog unit coverage in the adapter for now. +* **CI wiring**: an e2e job needs a docker-enabled runner and tolerates a long + first-run firmware build — deliberately left out of this PR. diff --git a/scripts/meshtasticator_e2e/run_e2e.sh b/scripts/meshtasticator_e2e/run_e2e.sh new file mode 100644 index 0000000..52462ef --- /dev/null +++ b/scripts/meshtasticator_e2e/run_e2e.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# Boot N Meshtasticator nodes and run the plugin's radio-level e2e tests +# (plugin/tests/e2e) against node 0. +# +# Meshtasticator's interactive simulator (https://meshtastic.org/docs/software/meshtasticator/) +# runs the real MeshtasticD device software per node and simulates the LoRa +# PHY, so these tests exercise the adapter's real TCP/protobuf session, +# inbound/outbound text flow across the simulated mesh, and chunked sends — +# without physical radios. It cannot validate silicon-level RF. +# +# Prerequisites +# * Linux host, OR Docker (macOS/Windows): the simulator auto-falls back to +# Docker on non-Linux and can build MeshtasticD itself (`-d`). +# * Native Linux mode: build MeshtasticD's 'native' target (PlatformIO, +# select 'native') and pass --program . +# * uv + python 3.13 for the contract venv (plugin/.venv-contract). +# +# Usage +# MESHTASTICATOR_DIR=/path/to/meshtasticator scripts/meshtasticator_e2e/run_e2e.sh +# # docker mode (default): ... run_e2e.sh --mode docker +# # native mode: ... run_e2e.sh --mode native --program /path/to/firmware/.pio/build/native +# # node count / channel: ... run_e2e.sh --nodes 3 --channel-index 0 +# +# Environment +# MESHTASTICATOR_DIR path to a clone of github.com/meshtastic/meshtasticator +# E2E_VENV python env with hermes-agent + plugin deps installed +# (default: plugin/.venv-contract — set it up with +# `uv pip install -e plugin/.hermes-src` equivalents via +# scripts/contract-test.sh, then `uv pip install -e plugin`) +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +SIM_DIR="${MESHTASTICATOR_DIR:?set MESHTASTICATOR_DIR to a meshtasticator clone}" +NODES=3 +MODE=docker +PROGRAM="" +CHANNEL_INDEX=0 +HOST=127.0.0.1 +VENV="${E2E_VENV:-$REPO_ROOT/plugin/.venv-contract}" + +while [ $# -gt 0 ]; do + case "$1" in + --nodes) NODES="$2"; shift 2 ;; + --mode) MODE="$2"; shift 2 ;; + --program) PROGRAM="$2"; shift 2 ;; + --channel-index) CHANNEL_INDEX="$2"; shift 2 ;; + --host) HOST="$2"; shift 2 ;; + *) echo "unknown arg: $1" >&2; exit 2 ;; + esac +done + +command -v uv >/dev/null 2>&1 || { echo "uv required" >&2; exit 1; } +[ -d "$SIM_DIR" ] || { echo "meshtasticator not found at $SIM_DIR" >&2; exit 1; } +[ -x "$VENV/bin/python" ] || { echo "e2e venv missing at $VENV (see scripts/contract-test.sh)" >&2; exit 1; } + +export MPLBACKEND=Agg # headless matplotlib (no display needed) + +SIM_LOG="$REPO_ROOT/meshtasticator-sim.log" +SIM_ARGS=("$NODES") +if [ "$MODE" = native ]; then + [ -n "$PROGRAM" ] && [ -d "$PROGRAM" ] || { echo "--mode native needs --program " >&2; exit 2; } + SIM_ARGS+=(-p "$PROGRAM") +else + SIM_ARGS+=(-d) +fi + +echo "== booting Meshtasticator ($MODE) with $NODES node(s) from $SIM_DIR ==" +( + cd "$SIM_DIR" + python3 interactiveSim.py "${SIM_ARGS[@]}" >"$SIM_LOG" 2>&1 +) & +SIM_PID=$! + +PORTS="" +cleanup() { + echo "== tearing down simulator (pid $SIM_PID) ==" + kill "$SIM_PID" 2>/dev/null || true + wait "$SIM_PID" 2>/dev/null || true + if [ "$MODE" = docker ]; then + # The sim container may outlive the process on abrupt kills. + docker ps -q --filter "ancestor=meshtastic-meshtasticd" 2>/dev/null | xargs -r docker rm -f >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT + +echo "== waiting for $NODES node TCP API port(s) to accept connections ==" +READY="" +for _ in $(seq 1 180); do # up to ~5 min: docker builds firmware on first run + READY="$("$VENV/bin/python" - "$HOST" "$NODES" <<'PY' || true +import socket, sys +host, count = sys.argv[1], int(sys.argv[2]) +found = [] +# Meshtasticator assigns node n -> TCP port starting at 4404 (see +# lib/interactive.py TCP_PORT_OFFSET); probe a small window to be safe. +for port in range(4403, 4403 + 16): + try: + with socket.create_connection((host, port), timeout=0.5): + found.append(port) + except OSError: + pass + if len(found) >= count: + break +print(",".join(map(str, found[:count]))) +PY +)" + if [ -n "$READY" ]; then + break + fi + sleep 2 +done +if [ -z "$READY" ]; then + echo "error: no Meshtasticator node ports became reachable on $HOST (see $SIM_LOG)" >&2 + exit 1 +fi +echo " node ports: $READY" + +echo "== running radio-level e2e tests ==" +( + cd "$REPO_ROOT" + MESHTASTICATOR_E2E=1 \ + E2E_HOST="$HOST" \ + E2E_PORTS="$READY" \ + E2E_CHANNEL_INDEX="$CHANNEL_INDEX" \ + "$VENV/bin/python" -m pytest plugin/tests/e2e -q "$@" +)