feat(e2e): Meshtasticator radio-level validation harness for the adapter
Adds a skip-gated end-to-end suite (plugin/tests/e2e) that drives the adapter against live Meshtasticator simulated nodes - real MeshtasticD instances speaking the actual TCP/protobuf API - covering the layer no contract test can reach: TCP handshake, inbound text over the simulated mesh, and chunked outbound sends arriving at a peer node. scripts/meshtasticator_e2e/run_e2e.sh boots N nodes (Docker or Linux native MeshtasticD), discovers their TCP API ports, and runs the suite. Radio-level reconnect and CI wiring are documented as open items pending a Linux/Docker validation run.
This commit is contained in:
@@ -0,0 +1,227 @@
|
||||
"""End-to-end tests: hermes-meshtastic adapter against live Meshtasticator nodes.
|
||||
|
||||
What these tests validate (the "radio-level" layer no contract test can):
|
||||
* the adapter's real TCP/protobuf handshake against MeshtasticD (the actual
|
||||
device software, not a mock) as run by the Meshtasticator interactive
|
||||
simulator (https://meshtastic.org/docs/software/meshtasticator/),
|
||||
* inbound text reception across the simulated LoRa mesh,
|
||||
* outbound chunked sends arriving at another simulated node,
|
||||
* the whole path: pubsub callback -> MessageEvent construction (real gateway
|
||||
types) -> handle_message dispatch.
|
||||
|
||||
What they do NOT validate (out of scope even for Meshtasticator): true RF
|
||||
(silicon-level LoRa, real propagation/interference, regulatory duty cycle).
|
||||
|
||||
Running
|
||||
-------
|
||||
1. On a Linux host (or a machine with Docker): clone meshtastic/meshtasticator
|
||||
and boot the interactive simulator (see scripts/meshtasticator_e2e/).
|
||||
2. Point the plugin's own contract venv at the booted nodes and run:
|
||||
cd plugin && MESHTASTICATOR_E2E=1 E2E_PORTS=4404,4405 \
|
||||
E2E_CHANNEL_INDEX=0 .venv-contract/bin/python -m pytest tests/e2e -q
|
||||
|
||||
Environment
|
||||
-----------
|
||||
MESHTASTICATOR_E2E = "1" enables these tests (required; without it the
|
||||
module skips so ordinary unit/CI runs stay fast).
|
||||
E2E_PORTS = comma-separated TCP API ports of the simulated nodes
|
||||
(first = the node the adapter connects to, default 4404).
|
||||
E2E_HOST = host where the simulator exposes the ports (default 127.0.0.1).
|
||||
E2E_CHANNEL_INDEX = Meshtastic channel index the adapter listens/sends on
|
||||
(default 0 = primary channel, matching a stock sim node).
|
||||
|
||||
NOTE: this module is an initial scaffold. Structural validation (imports,
|
||||
fixtures, flow) passes, but a full green run requires a Linux/Docker host with
|
||||
a MeshtasticD build — tracked as the remaining verification step.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
if os.environ.get("MESHTASTICATOR_E2E") != "1":
|
||||
pytest.skip(
|
||||
"set MESHTASTICATOR_E2E=1 to run against Meshtasticator nodes "
|
||||
"(see scripts/meshtasticator_e2e/README.md)",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
# Real Hermes gateway + real Meshtastic client library are required.
|
||||
pytest.importorskip("gateway.platforms.base")
|
||||
pytest.importorskip("meshtastic.tcp_interface")
|
||||
|
||||
from gateway.config import PlatformConfig # noqa: E402
|
||||
from gateway.platform_registry import ( # noqa: E402
|
||||
PlatformEntry,
|
||||
platform_registry,
|
||||
)
|
||||
|
||||
import hermes_meshtastic # noqa: E402
|
||||
from hermes_meshtastic import adapter as plugin_adapter # noqa: E402
|
||||
|
||||
E2E_HOST = os.environ.get("E2E_HOST", "127.0.0.1")
|
||||
E2E_PORTS = [int(p) for p in os.environ.get("E2E_PORTS", "4404,4405").split(",") if p]
|
||||
E2E_CHANNEL_INDEX = int(os.environ.get("E2E_CHANNEL_INDEX", "0"))
|
||||
assert len(E2E_PORTS) >= 2, "E2E_PORTS needs at least two node ports (adapter + peer)"
|
||||
|
||||
NODE0_PORT = E2E_PORTS[0] # the node our adapter connects to
|
||||
NODE1_PORT = E2E_PORTS[1] # the peer node that sends / observes
|
||||
CONNECT_TIMEOUT_S = 30
|
||||
RADIO_SETTLE_S = 2
|
||||
|
||||
|
||||
def _register_meshtastic_platform():
|
||||
"""Register the plugin (mirrors discovery) so Platform('meshtastic') resolves."""
|
||||
if platform_registry.is_registered("meshtastic"):
|
||||
return
|
||||
kwargs = {}
|
||||
|
||||
class _Ctx:
|
||||
def register_platform(self, **kw):
|
||||
kwargs.update(kw)
|
||||
|
||||
hermes_meshtastic.register(_Ctx())
|
||||
platform_registry.register(
|
||||
PlatformEntry(
|
||||
name=kwargs["name"],
|
||||
label=kwargs["label"],
|
||||
adapter_factory=kwargs["adapter_factory"],
|
||||
check_fn=kwargs["check_fn"],
|
||||
required_env=[],
|
||||
max_message_length=kwargs.get("max_message_length", 0),
|
||||
emoji=kwargs.get("emoji", "🔌"),
|
||||
platform_hint=kwargs.get("platform_hint", ""),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class _ProbeAdapter(plugin_adapter.get_adapter_class()):
|
||||
"""MeshtasticAdapter that records inbound MessageEvents instead of
|
||||
dispatching to a Hermes agent handler (no agent runs in these tests)."""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.received_events = []
|
||||
|
||||
async def handle_message(self, event):
|
||||
self.received_events.append(event)
|
||||
# Intentionally do not call super(): the real base would forward to a
|
||||
# message handler that only exists inside a running Hermes gateway.
|
||||
return None
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def adapter():
|
||||
_register_meshtastic_platform()
|
||||
cfg = PlatformConfig(
|
||||
enabled=True,
|
||||
extra={
|
||||
"host": E2E_HOST,
|
||||
"port": NODE0_PORT,
|
||||
"channel_index": E2E_CHANNEL_INDEX,
|
||||
},
|
||||
)
|
||||
return _ProbeAdapter(cfg)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def node0(adapter):
|
||||
"""Connect the adapter to simulated node 0; disconnect afterwards."""
|
||||
async def _connect():
|
||||
return await adapter.connect()
|
||||
|
||||
assert asyncio.run(_connect()) is True, (
|
||||
f"adapter failed to connect to simulated node at {E2E_HOST}:{NODE0_PORT} "
|
||||
"(is the simulator running and the node reachable?)"
|
||||
)
|
||||
try:
|
||||
yield adapter
|
||||
finally:
|
||||
asyncio.run(adapter.disconnect())
|
||||
|
||||
|
||||
def _wait_for(predicate, timeout_s=15.0, interval_s=0.5, what="condition"):
|
||||
"""Poll until *predicate* (callable -> bool) is true, else pytest.fail."""
|
||||
import time
|
||||
|
||||
deadline = time.monotonic() + timeout_s
|
||||
while time.monotonic() < deadline:
|
||||
if predicate():
|
||||
return
|
||||
time.sleep(interval_s)
|
||||
pytest.fail(f"timed out waiting for {what}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Radio-level flows
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_adapter_receives_broadcast_from_peer(node0):
|
||||
"""Peer node 1 broadcasts text over the simulated mesh; the adapter must
|
||||
receive it as a real gateway MessageEvent."""
|
||||
import time
|
||||
|
||||
from meshtastic import tcp_interface
|
||||
|
||||
sent = f"e2e-inbound-{int(time.time())}"
|
||||
|
||||
peer = tcp_interface.TCPInterface(hostname=E2E_HOST, portNumber=NODE1_PORT)
|
||||
try:
|
||||
time.sleep(RADIO_SETTLE_S)
|
||||
|
||||
def _got():
|
||||
return any(getattr(e, "text", None) == sent for e in node0.received_events)
|
||||
|
||||
# Radio delivery is not guaranteed per attempt; retry a few times.
|
||||
for _ in range(3):
|
||||
if _got():
|
||||
break
|
||||
peer.localNode.sendText(
|
||||
text=sent, destinationId="^all", channelIndex=E2E_CHANNEL_INDEX
|
||||
)
|
||||
time.sleep(1)
|
||||
_wait_for(_got, timeout_s=25, what="inbound broadcast from peer node")
|
||||
finally:
|
||||
peer.close()
|
||||
|
||||
|
||||
def test_adapter_chunked_send_reaches_peer(node0):
|
||||
"""A long outbound message must be chunked and arrive at the peer node."""
|
||||
import time
|
||||
|
||||
from meshtastic import tcp_interface
|
||||
from pubsub import pub
|
||||
|
||||
content = " ".join(f"word{n}" for n in range(60)) # well over 180 chars -> chunks
|
||||
received = []
|
||||
|
||||
def _on_receive(packet, interface):
|
||||
decoded = packet.get("decoded", {}) or {}
|
||||
text = decoded.get("text", "")
|
||||
if text and text.startswith("(1/"): # our chunked framing
|
||||
received.append(text)
|
||||
elif text:
|
||||
received.append(text)
|
||||
|
||||
peer = tcp_interface.TCPInterface(hostname=E2E_HOST, portNumber=NODE1_PORT)
|
||||
try:
|
||||
time.sleep(RADIO_SETTLE_S)
|
||||
pub.subscribe(_on_receive, "meshtastic.receive.text")
|
||||
|
||||
async def _send():
|
||||
result = await node0.send("channel_e2e", content)
|
||||
assert result.success, f"adapter.send failed: {result.error}"
|
||||
|
||||
asyncio.run(_send())
|
||||
|
||||
def _all_chunks_arrived():
|
||||
# Adapter frames multi-part sends as "(i/n) <chunk>"; the final
|
||||
# chunk carries the message tail.
|
||||
return any(str(r).rstrip().endswith("word59") for r in received)
|
||||
|
||||
_wait_for(_all_chunks_arrived, timeout_s=45, what="chunked outbound send at peer")
|
||||
finally:
|
||||
pub.unsubscribe(_on_receive, "meshtastic.receive.text")
|
||||
peer.close()
|
||||
@@ -0,0 +1,99 @@
|
||||
# Meshtasticator e2e — radio-level validation for hermes-meshtastic
|
||||
|
||||
This directory is the **radio-level counterpart** to the contract suite
|
||||
(`plugin/tests/contract`). The contract suite guarantees the adapter matches
|
||||
the Hermes gateway API; it cannot prove the adapter actually talks to a
|
||||
Meshtastic node. These e2e tests boot the
|
||||
[Meshtasticator](https://meshtastic.org/docs/software/meshtasticator/)
|
||||
interactive simulator — which runs the **real MeshtasticD device software**
|
||||
per node with a simulated LoRa PHY — and drive the adapter against a live
|
||||
simulated node.
|
||||
|
||||
| Layer | Validated by |
|
||||
|---|---|
|
||||
| Adapter ↔ Hermes gateway API | `plugin/tests/contract` (no deployment) |
|
||||
| Adapter ↔ real MeshtasticD TCP/protobuf session | **this suite** |
|
||||
| Inbound text across the (simulated) LoRa mesh | **this suite** |
|
||||
| Outbound chunked sends arriving at another node | **this suite** |
|
||||
| True RF (silicon LoRa, real propagation, duty cycle) | physical radio only |
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
scripts/meshtasticator_e2e/
|
||||
run_e2e.sh orchestrator: boots N nodes, runs pytest, tears down
|
||||
README.md this file
|
||||
plugin/tests/e2e/
|
||||
test_meshtasticator_e2e.py the e2e assertions (skip-gated)
|
||||
```
|
||||
|
||||
## Prerequisites
|
||||
|
||||
* **uv** + Python 3.13.
|
||||
* A contract venv with the real gateway **and** the plugin (with deps) installed:
|
||||
```bash
|
||||
scripts/contract-test.sh # creates plugin/.venv-contract (hermes-agent pinned)
|
||||
uv pip install --python plugin/.venv-contract/bin/python -e plugin
|
||||
```
|
||||
(`-e plugin` installs the meshtastic client library the tests also use.)
|
||||
* A clone of Meshtasticator: `git clone https://github.com/meshtastic/meshtasticator`
|
||||
* Runtime for the simulator — pick one:
|
||||
* **Docker** (simplest, works on macOS/Windows): the simulator builds
|
||||
MeshtasticD from a container image on first run (slow once).
|
||||
* **Linux native**: build MeshtasticD's `native` PlatformIO target and pass
|
||||
the build dir with `--program`.
|
||||
|
||||
## Run
|
||||
|
||||
```bash
|
||||
# Docker mode (default)
|
||||
MESHTASTICATOR_DIR=/path/to/meshtasticator \
|
||||
scripts/meshtasticator_e2e/run_e2e.sh
|
||||
|
||||
# Linux native mode
|
||||
MESHTASTICATOR_DIR=/path/to/meshtasticator \
|
||||
scripts/meshtasticator_e2e/run_e2e.sh \
|
||||
--mode native --program /path/to/firmware/.pio/build/native
|
||||
|
||||
# Tune the scenario
|
||||
... --nodes 3 --channel-index 0 --host 127.0.0.1
|
||||
```
|
||||
|
||||
The orchestrator boots the simulator, waits until `N` node TCP API ports accept
|
||||
connections (Meshtasticator assigns node *n* → port `4404 + n`), then runs
|
||||
`pytest plugin/tests/e2e` with `MESHTASTICATOR_E2E=1`. Simulator output lands in
|
||||
`meshtasticator-sim.log` at the repo root.
|
||||
|
||||
Without `MESHTASTICATOR_E2E=1` the e2e module skips, so ordinary unit/CI runs
|
||||
stay fast and gateway-free.
|
||||
|
||||
## What the tests do
|
||||
|
||||
1. **Inbound**: peer node 1 broadcasts text; the adapter (connected to node 0)
|
||||
must receive it as a real gateway `MessageEvent` (`host/port/channel_index`
|
||||
come from `E2E_PORTS`/`E2E_HOST`/`E2E_CHANNEL_INDEX`).
|
||||
2. **Outbound**: the adapter sends a >180-char message; it must be chunked and
|
||||
the final chunk (`(i/n) … word59`) must arrive at peer node 1.
|
||||
|
||||
A `_ProbeAdapter` subclass records inbound events instead of dispatching to a
|
||||
Hermes agent handler (no agent runs in these tests; dispatch is already covered
|
||||
by the contract suite).
|
||||
|
||||
## Status / open items (v1 scaffold)
|
||||
|
||||
This is an initial, structurally validated scaffold. A full green run still
|
||||
needs to happen on a Linux/Docker host with a MeshtasticD build, and these
|
||||
items are expected to need iteration there:
|
||||
|
||||
* **Node adjacency**: the simulator's default random placement may put nodes out
|
||||
of range. For deterministic tests, seed close-together coordinates (the sim
|
||||
supports `--from-file` with an `out/nodeConfig.yaml`).
|
||||
* **Channel semantics**: tests default to channel index 0 (stock primary
|
||||
channel). Mirroring production (channel 1 / private channel) requires
|
||||
configuring matching channels on the simulated nodes.
|
||||
* **Reconnect flow**: killing/restarting a single simulated node needs per-node
|
||||
lifecycle control (native mode subprocesses; docker mode currently runs all
|
||||
nodes in one container), so the adapter's watchdog reconnect is **not** yet
|
||||
exercised here. See the watchdog unit coverage in the adapter for now.
|
||||
* **CI wiring**: an e2e job needs a docker-enabled runner and tolerates a long
|
||||
first-run firmware build — deliberately left out of this PR.
|
||||
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env bash
|
||||
# Boot N Meshtasticator nodes and run the plugin's radio-level e2e tests
|
||||
# (plugin/tests/e2e) against node 0.
|
||||
#
|
||||
# Meshtasticator's interactive simulator (https://meshtastic.org/docs/software/meshtasticator/)
|
||||
# runs the real MeshtasticD device software per node and simulates the LoRa
|
||||
# PHY, so these tests exercise the adapter's real TCP/protobuf session,
|
||||
# inbound/outbound text flow across the simulated mesh, and chunked sends —
|
||||
# without physical radios. It cannot validate silicon-level RF.
|
||||
#
|
||||
# Prerequisites
|
||||
# * Linux host, OR Docker (macOS/Windows): the simulator auto-falls back to
|
||||
# Docker on non-Linux and can build MeshtasticD itself (`-d`).
|
||||
# * Native Linux mode: build MeshtasticD's 'native' target (PlatformIO,
|
||||
# select 'native') and pass --program <dir containing binary 'program'>.
|
||||
# * uv + python 3.13 for the contract venv (plugin/.venv-contract).
|
||||
#
|
||||
# Usage
|
||||
# MESHTASTICATOR_DIR=/path/to/meshtasticator scripts/meshtasticator_e2e/run_e2e.sh
|
||||
# # docker mode (default): ... run_e2e.sh --mode docker
|
||||
# # native mode: ... run_e2e.sh --mode native --program /path/to/firmware/.pio/build/native
|
||||
# # node count / channel: ... run_e2e.sh --nodes 3 --channel-index 0
|
||||
#
|
||||
# Environment
|
||||
# MESHTASTICATOR_DIR path to a clone of github.com/meshtastic/meshtasticator
|
||||
# E2E_VENV python env with hermes-agent + plugin deps installed
|
||||
# (default: plugin/.venv-contract — set it up with
|
||||
# `uv pip install -e plugin/.hermes-src` equivalents via
|
||||
# scripts/contract-test.sh, then `uv pip install -e plugin`)
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
SIM_DIR="${MESHTASTICATOR_DIR:?set MESHTASTICATOR_DIR to a meshtasticator clone}"
|
||||
NODES=3
|
||||
MODE=docker
|
||||
PROGRAM=""
|
||||
CHANNEL_INDEX=0
|
||||
HOST=127.0.0.1
|
||||
VENV="${E2E_VENV:-$REPO_ROOT/plugin/.venv-contract}"
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--nodes) NODES="$2"; shift 2 ;;
|
||||
--mode) MODE="$2"; shift 2 ;;
|
||||
--program) PROGRAM="$2"; shift 2 ;;
|
||||
--channel-index) CHANNEL_INDEX="$2"; shift 2 ;;
|
||||
--host) HOST="$2"; shift 2 ;;
|
||||
*) echo "unknown arg: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
command -v uv >/dev/null 2>&1 || { echo "uv required" >&2; exit 1; }
|
||||
[ -d "$SIM_DIR" ] || { echo "meshtasticator not found at $SIM_DIR" >&2; exit 1; }
|
||||
[ -x "$VENV/bin/python" ] || { echo "e2e venv missing at $VENV (see scripts/contract-test.sh)" >&2; exit 1; }
|
||||
|
||||
export MPLBACKEND=Agg # headless matplotlib (no display needed)
|
||||
|
||||
SIM_LOG="$REPO_ROOT/meshtasticator-sim.log"
|
||||
SIM_ARGS=("$NODES")
|
||||
if [ "$MODE" = native ]; then
|
||||
[ -n "$PROGRAM" ] && [ -d "$PROGRAM" ] || { echo "--mode native needs --program <dir>" >&2; exit 2; }
|
||||
SIM_ARGS+=(-p "$PROGRAM")
|
||||
else
|
||||
SIM_ARGS+=(-d)
|
||||
fi
|
||||
|
||||
echo "== booting Meshtasticator ($MODE) with $NODES node(s) from $SIM_DIR =="
|
||||
(
|
||||
cd "$SIM_DIR"
|
||||
python3 interactiveSim.py "${SIM_ARGS[@]}" >"$SIM_LOG" 2>&1
|
||||
) &
|
||||
SIM_PID=$!
|
||||
|
||||
PORTS=""
|
||||
cleanup() {
|
||||
echo "== tearing down simulator (pid $SIM_PID) =="
|
||||
kill "$SIM_PID" 2>/dev/null || true
|
||||
wait "$SIM_PID" 2>/dev/null || true
|
||||
if [ "$MODE" = docker ]; then
|
||||
# The sim container may outlive the process on abrupt kills.
|
||||
docker ps -q --filter "ancestor=meshtastic-meshtasticd" 2>/dev/null | xargs -r docker rm -f >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "== waiting for $NODES node TCP API port(s) to accept connections =="
|
||||
READY=""
|
||||
for _ in $(seq 1 180); do # up to ~5 min: docker builds firmware on first run
|
||||
READY="$("$VENV/bin/python" - "$HOST" "$NODES" <<'PY' || true
|
||||
import socket, sys
|
||||
host, count = sys.argv[1], int(sys.argv[2])
|
||||
found = []
|
||||
# Meshtasticator assigns node n -> TCP port starting at 4404 (see
|
||||
# lib/interactive.py TCP_PORT_OFFSET); probe a small window to be safe.
|
||||
for port in range(4403, 4403 + 16):
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=0.5):
|
||||
found.append(port)
|
||||
except OSError:
|
||||
pass
|
||||
if len(found) >= count:
|
||||
break
|
||||
print(",".join(map(str, found[:count])))
|
||||
PY
|
||||
)"
|
||||
if [ -n "$READY" ]; then
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
if [ -z "$READY" ]; then
|
||||
echo "error: no Meshtasticator node ports became reachable on $HOST (see $SIM_LOG)" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo " node ports: $READY"
|
||||
|
||||
echo "== running radio-level e2e tests =="
|
||||
(
|
||||
cd "$REPO_ROOT"
|
||||
MESHTASTICATOR_E2E=1 \
|
||||
E2E_HOST="$HOST" \
|
||||
E2E_PORTS="$READY" \
|
||||
E2E_CHANNEL_INDEX="$CHANNEL_INDEX" \
|
||||
"$VENV/bin/python" -m pytest plugin/tests/e2e -q "$@"
|
||||
)
|
||||
Reference in New Issue
Block a user