fix(e2e): deterministic in-range node scenario + log receiver decisions

The sim now observes TX from both nodes (current client fixed observation)
but nothing is ever delivered to the other node. Suspect: random placement
leaving nodes out of range, or the receiver decision. Provide a --from-file
scenario with nodes spaced 10 m apart (guaranteed adjacency) and print the
transmitter -> receivers decision per TX in the sim log.

.gitea/workflows/meshtasticator-e2e.yaml

@@ -71,20 +71,29 @@ jobs:
       - name: Simulator venv (its own requirements + docker SDK)
         run: |
           uv venv --python 3.13 "$HOME/sim-venv"
+          # NOTE: meshtasticator/requirements.txt pins meshtastic~=2.6.1, which
+          # predates the meshtastic/meshtasticd image it talks to and never
+          # observed TX packets (both e2e directions timed out). Install the
+          # simulator's deps but let meshtastic float to the current client —
+          # the version the official firmware_harness uses against the same
+          # meshtasticd simulator mode.
           uv pip install --python "$HOME/sim-venv/bin/python" \
-            -r "$HOME/meshtasticator/requirements.txt" docker
+            meshtastic numpy matplotlib pandas \
+            'PyPubSub==4.0.3' simpy PyYAML protobuf docker
 
       - name: Run Meshtasticator e2e suite
         run: |
           export MESHTASTICATOR_DIR="$HOME/meshtasticator"
           export SIM_VENV="$HOME/sim-venv"
           export E2E_VENV="plugin/.venv-contract"
-          scripts/meshtasticator_e2e/run_e2e.sh --mode docker
+          bash scripts/meshtasticator_e2e/run_e2e.sh --mode docker
 
       - name: Print simulator log (on failure)
         if: failure()
         run: |
-          echo "===== meshtasticator-sim.log ====="
+          echo "===== meshtasticator-sim.log (tail) ====="
-          tail -n 200 meshtasticator-sim.log 2>/dev/null || true
+          tail -n 400 meshtasticator-sim.log 2>/dev/null || true
+          echo "===== meshtasticator-nodes.log (if any) ====="
+          tail -n 200 meshtasticator-nodes.log 2>/dev/null || true
           echo "===== docker ps ====="
           docker ps -a 2>/dev/null || true

plugin/tests/e2e/test_meshtasticator_e2e.py

@@ -143,6 +143,16 @@ def loop():
     try:
         yield _loop
     finally:
+        # Cancel leftover tasks (e.g. a watchdog started by a failed connect
+        # whose disconnect teardown never ran) before closing the loop.
+        async def _shutdown():
+            for task in asyncio.all_tasks(_loop):
+                task.cancel()
+
+        try:
+            asyncio.run_coroutine_threadsafe(_shutdown(), _loop).result(timeout=5)
+        except Exception:
+            pass
         _loop.call_soon_threadsafe(_loop.stop)
         thread.join(timeout=5)
 
@@ -216,10 +226,11 @@ def test_adapter_receives_broadcast_from_peer(node0):
             return any(getattr(e, "text", None) == sent for e in node0.received_events)
 
         # Radio delivery is not guaranteed per attempt; retry a few times.
+        # (meshtastic client >=2.8: sendText lives on the Interface, not Node.)
         for _ in range(3):
             if _got():
                 break
-            peer.localNode.sendText(
+            peer.sendText(
                 text=sent, destinationId="^all", channelIndex=E2E_CHANNEL_INDEX
             )
             time.sleep(1)
@@ -261,6 +272,25 @@ def test_adapter_chunked_send_reaches_peer(node0, loop):
             return any(str(r).rstrip().endswith("word59") for r in received)
 
         _wait_for(_all_chunks_arrived, timeout_s=45, what="chunked outbound send at peer")
+
+    except Exception as exc:
+        # Fast-fail diagnostics: what arrived at the peer, and what the
+        # simulator itself logged (RF forwarding/routing noise).
+        tail = ""
+        try:
+            sim_log = os.path.join(os.getcwd(), "..", "..", "meshtasticator-sim.log")
+            if not os.path.exists(sim_log):
+                sim_log = os.path.join(os.getcwd(), "meshtasticator-sim.log")
+            if os.path.exists(sim_log):
+                tail = "\n".join(
+                    open(sim_log, errors="replace").read().splitlines()[-40:]
+                )
+        except OSError:
+            pass
+        raise AssertionError(
+            f"{exc}\nreceived at peer so far: {received[-10:]!r}\n"
+            f"--- meshtasticator-sim.log tail ---\n{tail}"
+        ) from exc
     finally:
         pub.unsubscribe(_on_receive, "meshtastic.receive.text")
         peer.close()

scripts/meshtasticator_e2e/README.md

@@ -81,7 +81,20 @@ connections (Meshtasticator assigns node *n* → port `4404 + n`), then runs
 control connections assume nodes listen on localhost, so the script starts a
 `tcp_forward.py` per node port (localhost → daemon host) before booting the
 simulator. Nothing is forwarded when the daemon is local (unset/unix
-`DOCKER_HOST`).
+`DOCKER_HOST`). Readiness is probed on the *upstream* host (the daemon), never
+on the forwarders themselves.
+
+Other headless adaptations the script makes automatically:
+* patches `lib/gui.py` in the pinned Meshtasticator checkout: forces the Agg
+  backend (the code calls `matplotlib.use("TkAgg")` at import) and makes
+  `move_figure()` a no-op when no Tk window manager exists (its unconditional
+  `canvas.manager.window` access crashes under Agg at Graph construction).
+  Only the interactive GUI modes, never used by this harness, are lost,
+* warms the `meshtastic/meshtasticd` image (`docker pull`) before booting the
+  simulator so the pull is not on the simulator's node-boot critical path,
+* feeds the simulator's interactive `cmdloop` from a FIFO held open until
+  teardown — an EOF stdin (e.g. CI) would otherwise make it exit right after
+  booting the nodes.
 
 Without `MESHTASTICATOR_E2E=1` the e2e module skips, so ordinary unit/CI runs
 stay fast and gateway-free.

scripts/meshtasticator_e2e/run_e2e.sh

@@ -93,7 +93,31 @@ start_forwards() {
 }
 
 SIM_LOG="$REPO_ROOT/meshtasticator-sim.log"
-SIM_ARGS=("$NODES")
+# Deterministic topology: random placement may leave nodes out of range
+# (nothing was ever delivered between nodes). Provide a --from-file scenario
+# with all nodes within ~10 m of each other so RF adjacency is guaranteed.
+NODE_CONF_DIR="$SIM_DIR/out"
+mkdir -p "$NODE_CONF_DIR"
+"$VENV/bin/python" - "$NODE_CONF_DIR/nodeConfig.yaml" "$NODES" <<'PY'
+import sys
+from pathlib import Path
+
+path, count = sys.argv[1], int(sys.argv[2])
+conf = {}
+for i in range(count):
+    x = float((i - (count - 1) / 2) * 10.0)   # 10 m spacing around the origin
+    conf[i] = {
+        "x": x, "y": 0.0, "z": 1.0,
+        "isRouter": False, "isRepeater": False, "isClientMute": False,
+        "hopLimit": 3, "antennaGain": 0, "neighborInfo": False,
+    }
+Path(path).write_text(
+    "".join(f"{k}:\n" + "".join(f"  {ck}: {cv}\n" for ck, cv in v.items())
+            for k, v in conf.items())
+)
+print(f"wrote deterministic node scenario ({count} nodes) to {path}")
+PY
+SIM_ARGS=("--from-file" -v)   # node count/positions come from nodeConfig.yaml
 if [ "$MODE" = native ]; then
   [ -n "$PROGRAM" ] && [ -d "$PROGRAM" ] || { echo "--mode native needs --program <dir>" >&2; exit 2; }
   SIM_ARGS+=(-p "$PROGRAM")
@@ -104,17 +128,102 @@ fi
 if [ -n "$DOCKER_TARGET" ]; then
   echo "== remote docker daemon detected ($DOCKER_TARGET): starting localhost port forwards =="
   start_forwards
+  PROBE_HOST="$DOCKER_TARGET"
+else
+  PROBE_HOST="$HOST"
+fi
+
+# The simulator assumes an interactive Tk desktop: lib/gui.py forces
+# matplotlib.use("TkAgg") at import and move_figure() touches the Tk window
+# manager at Graph construction — both crash headless runs even under
+# MPLBACKEND=Agg. Neutralize them in the pinned checkout (only the interactive
+# GUI modes, never used by this harness, are lost).
+echo "== patching meshtasticator lib/gui.py for headless runs =="
+"$VENV/bin/python" - "$SIM_DIR" <<'PY'
+import sys
+from pathlib import Path
+
+gui = Path(sys.argv[1]) / "lib/gui.py"
+src = gui.read_text()
+
+src = src.replace('matplotlib.use("TkAgg")', 'matplotlib.use("Agg")')
+
+old = 'def move_figure(fig, x, y):\n    fig.canvas.manager.window.wm_geometry("+%d+%d" % (x, y))'
+new = ('def move_figure(fig, x, y):\n'
+       '    # Headless (Agg) backends have no Tk window manager.\n'
+       '    try:\n'
+       '        fig.canvas.manager.window.wm_geometry("+%d+%d" % (x, y))\n'
+       '    except AttributeError:\n'
+       '        pass')
+assert old in src, "move_figure source no longer matches the pinned meshtasticator"
+src = src.replace(old, new)
+gui.write_text(src)
+print("patched lib/gui.py")
+
+# --- RF instrumentation (prints land in meshtasticator-sim.log) ---------------
+# Why nothing is delivered between nodes is not yet known; log every packet the
+# sim's per-node interfaces observe so the next failing run explains itself.
+interactive = Path(sys.argv[1]) / "lib" / "interactive.py"
+src = interactive.read_text()
+
+old = '    def on_receive(self, interface, packet):\n'
+new = old + (
+    '        print(f"[sim] on_receive port={getattr(interface, \'portNumber\', None)} '
+    'decoded_keys={list(packet.get(\'decoded\', {}).keys())}")\n'
+)
+assert old in src, "on_receive signature moved in the pinned meshtasticator"
+src = src.replace(old, new, 1)
+
+old = '            pub.subscribe(self.on_receive, "meshtastic.receive.simulator")'
+new = old + (
+    '\n            pub.subscribe('
+    'lambda interface, packet: print(f"[sim-text] port={getattr(interface, \'portNumber\', None)} '
+    'text={packet.get(\'decoded\', {}).get(\'text\')!r}"), "meshtastic.receive.text")'
+)
+assert old in src, "simulator subscription moved in the pinned meshtasticator"
+src = src.replace(old, new, 1)
+
+old = '            rxs, rssis, snrs = self.calc_receivers(transmitter, receivers)'
+new = old + (
+    '\n            print(f"[sim] TX node={transmitter.nodeid} candidates={[n.nodeid for n in receivers]} '
+    'rxs={[n.nodeid for n in rxs]} rssis={rssis}")'
+)
+assert old in src, "calc_receivers call moved in the pinned meshtasticator"
+src = src.replace(old, new, 1)
+
+interactive.write_text(src)
+print("patched lib/interactive.py (RF instrumentation)")
+PY
+
+if [ "$MODE" = docker ]; then
+  # Warm the node image on the (possibly remote) daemon first: the simulator
+  # starts node processes ~4s after the container is created and control
+  # connections must succeed quickly, so the pull cannot sit in that path.
+  echo "== warming meshtastic/meshtasticd image =="
+  docker pull meshtastic/meshtasticd
 fi
 
 echo "== booting Meshtasticator ($MODE) with $NODES node(s) from $SIM_DIR =="
+# The simulator ends in an interactive cmdloop reading stdin; an EOF there
+# (e.g. CI) would make it exit right after booting the nodes. Feed it from a
+# FIFO whose write end this shell keeps open until cleanup.
+SIM_FIFO="$(mktemp -u /tmp/meshtasticator-XXXXXX.fifo)"
+mkfifo "$SIM_FIFO"
 (
   cd "$SIM_DIR"
-  "$SIM_PYTHON" interactiveSim.py "${SIM_ARGS[@]}" >"$SIM_LOG" 2>&1
+  "$SIM_PYTHON" interactiveSim.py "${SIM_ARGS[@]}" <"$SIM_FIFO" >"$SIM_LOG" 2>&1
 ) &
 SIM_PID=$!
+exec 9>"$SIM_FIFO"   # hold the write end open -> sim stdin never hits EOF
+rm -f "$SIM_FIFO"
 
 cleanup() {
   echo "== tearing down simulator (pid $SIM_PID) =="
+  # Graceful 'exit' through the cmdloop (avoids hundreds of EOF spam lines in
+  # sim.log and lets the sim close its own nodes/container).
+  echo exit >&9 2>/dev/null || true
+  sleep 2
+  exec 9>&- 2>/dev/null || true   # release the FIFO write end (sim stdin EOF)
   kill "$SIM_PID" 2>/dev/null || true
   wait "$SIM_PID" 2>/dev/null || true
   for pid in "${FWD_PIDS[@]:-}"; do
@@ -131,7 +240,9 @@ trap cleanup EXIT
 echo "== waiting for $NODES node TCP API port(s) to accept connections =="
 READY=""
 for _ in $(seq 1 180); do   # up to ~5 min: docker pulls meshtastic/meshtasticd on first run
-  READY="$("$VENV/bin/python" - "$HOST" "$NODES" <<'PY' || true
+  # Probe the upstream node host (the docker daemon when forwards are active,
+  # localhost otherwise) — probing the forwarders themselves would always pass.
+  READY="$("$VENV/bin/python" - "$PROBE_HOST" "$NODES" <<'PY' || true
 import socket, sys
 host, count = sys.argv[1], int(sys.argv[2])
 found = []
@@ -154,10 +265,10 @@ PY
   sleep 2
 done
 if [ -z "$READY" ]; then
-  echo "error: no Meshtasticator node ports became reachable on $HOST (see $SIM_LOG)" >&2
+  echo "error: no Meshtasticator node ports became reachable on $PROBE_HOST (see $SIM_LOG)" >&2
   exit 1
 fi
-echo "   node ports: $READY"
+echo "   node ports: $READY (probed on $PROBE_HOST)"
 
 # Ports accept before the node firmware finished booting inside the container;
 # give meshtasticd a moment to open its TCP API for real.
@@ -173,3 +284,14 @@ echo "== running radio-level e2e tests =="
   E2E_CHANNEL_INDEX="$CHANNEL_INDEX" \
     "$VENV/bin/python" -m pytest plugin/tests/e2e -q "$@"
 )
+RC=$?
+
+if [ $RC -ne 0 ] && [ "$MODE" = docker ]; then
+  # Capture per-node meshtasticd logs before cleanup removes the container.
+  echo "== dumping node logs (pytest rc=$RC) =="
+  docker exec Meshtastic sh -c 'for f in /home/out_*.log; do echo "----- $f -----"; tail -n 60 "$f" 2>/dev/null; done' \
+    >"$REPO_ROOT/meshtasticator-nodes.log" 2>&1 || true
+  echo "   node logs: $REPO_ROOT/meshtasticator-nodes.log"
+fi
+
+exit $RC