From 5c3f23735e1fbabc5dfa853bf40386975d1f8ff1 Mon Sep 17 00:00:00 2001 From: Michael Pham <61564344+Mikefly123@users.noreply.github.com> Date: Mon, 6 Apr 2026 22:34:33 -0700 Subject: [PATCH 01/15] Incremental Progress --- .gitignore | 5 +++++ Makefile | 45 ++++++++++++++++++++++++++++++++++++++++++++- requirements.txt | 4 ++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 4579e25d..94fcbd03 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,11 @@ Framing/src/sequence_number.bin lib/fprime-extras/ +# YAMCS generated dictionary (built from build-artifacts via make yamcs-dict) +yamcs/yamcs-data/mdb/*.xml +# YAMCS runtime data directory (created at startup by fprime-yamcs) +yamcs/yamcs-runtime/ + # CircuitPython passthrough /circuit-python-passthrough/firmware.uf2 /circuit-python-passthrough/lib/ diff --git a/Makefile b/Makefile index c59c017f..a55c08a2 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ export VIRTUAL_ENV ?= $(shell pwd)/fprime-venv .PHONY: fprime-venv fprime-venv: uv ## Create a virtual environment @$(UV) venv fprime-venv --allow-existing - @$(UV) pip install --prerelease=allow --requirement requirements.txt + @$(UV) pip install --prerelease=allow --requirement requirements.txt --overrides yamcs/pyyaml-override.txt .PHONY: zephyr-setup @@ -212,6 +212,49 @@ sync-sequence-number: fprime-venv ## Synchronize sequence number between GDS and clean: ## Remove all gitignored files git clean -dfX +##@ YAMCS + +.PHONY: yamcs-dict +yamcs-dict: fprime-venv ## Generate XTCE dictionary for YAMCS (requires build-artifacts; run 'make build' first) + @mkdir -p yamcs/yamcs-data/mdb + @DICT=$$(find build-artifacts -name "*TopologyDictionary.json" | head -1); \ + if [ -z "$$DICT" ]; then echo "Error: run 'make build' first"; exit 1; fi; \ + echo "Generating XTCE from $$DICT"; \ + $(UV_RUN) fprime-to-xtce "$$DICT" -o yamcs/yamcs-data/mdb/fprime.xtce.xml + @echo "XTCE dictionary at yamcs/yamcs-data/mdb/fprime.xtce.xml" + +.PHONY: yamcs +yamcs: fprime-venv yamcs-dict ## Run YAMCS with serial adapter (Use Case 1: UART_DEVICE=/dev/ttyXXX) + @if [ -z "$(UART_DEVICE)" ]; then echo "Error: set UART_DEVICE=/dev/ttyXXX"; exit 1; fi + @echo "Starting YAMCS (requires Java 11+)..." + @mkdir -p $(shell pwd)/yamcs/yamcs-runtime + FPRIME_GDS_CONFIG_PATH=$(shell pwd)/yamcs/fprime-gds.yml \ + $(UV_RUN) fprime-yamcs \ + -d $(shell pwd)/build-artifacts/zephyr/fprime-zephyr-deployment \ + --no-app \ + --communication-selection none \ + --no-convert-dictionary \ + --yamcs-config-dir $(shell pwd)/yamcs/yamcs-data \ + --yamcs-data-dir $(shell pwd)/yamcs/yamcs-runtime & + @sleep 3 + @echo "Starting serial adapter on $(UART_DEVICE)..." + $(UV_RUN) python tools/yamcs/proves_adapter.py \ + --mode serial \ + --uart-device $(UART_DEVICE) \ + --uart-baud 115200 + +.PHONY: yamcs-server +yamcs-server: yamcs-dict ## Start YAMCS server via Docker (Use Case 2: remote deployment) + docker compose -f yamcs/docker-compose.yml up + +.PHONY: yamcs-adapter-tcp +yamcs-adapter-tcp: fprime-venv ## Start TCP adapter for bent-pipe (GS_HOST=, GS_PORT=, YAMCS_HOST=) + $(UV_RUN) python tools/yamcs/proves_adapter.py \ + --mode tcp \ + --tcp-host $(GS_HOST) \ + --tcp-port $(GS_PORT) \ + --yamcs-host $(YAMCS_HOST) + ##@ Operations GDS_COMMAND ?= $(UV_RUN) fprime-gds diff --git a/requirements.txt b/requirements.txt index 8980729d..8cd0407b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,7 @@ semver tqdm -r lib/fprime/requirements.txt -r lib/zephyr-workspace/zephyr/scripts/requirements.txt +yamcs-client==1.12.1 +fprime-xtce==0.1.0 +fprime-yamcs==0.1.1 +pyserial>=3.5 From 4e69dcf1b59959d9f2753b313037edcdc685e1d1 Mon Sep 17 00:00:00 2001 From: Michael Pham <61564344+Mikefly123@users.noreply.github.com> Date: Tue, 7 Apr 2026 19:27:04 -0700 Subject: [PATCH 02/15] Add YAMCS integration (Option B auth bridge) - yamcs/yamcs-data/etc/: YAMCS server config (global + fprime-project instance) with spacecraftId=68, frameLength=248, UDP ports 50000/50001 - yamcs/yamcs-data/etc/processor.yaml: YAMCS realtime processor config - yamcs/yamcs-data/mdb/.gitkeep: placeholder for generated fprime.xtce.xml - yamcs/fprime-gds.yml: yamcs-specific GDS config (omits options unknown to fprime-yamcs such as output-unframed-data and file-uplink-cooldown) - yamcs/pyyaml-override.txt: uv override to satisfy fprime-yamcs>=0.1.0 which requires PyYAML>=6.0.3 while fprime submodule pins ==6.0.2 - yamcs/docker-compose.yml: Docker Compose for remote server deployment - tools/yamcs/proves_adapter.py: auth bridge; serial/TCP <-> YAMCS UDP with HMAC-SHA256 wrapping via AuthenticateFramer (Option B) - Makefile: yamcs-dict, yamcs, yamcs-server, yamcs-adapter-tcp targets; fprime-yamcs-events launched alongside YAMCS; pyyaml uv override wired in - .codespell-ignore-words.txt: add 'ser' (pyserial handle variable) Verified: YAMCS 5.12.0 boots to RUNNING, all UDP data links OK, Beacon container present in MDB from generated XTCE. Co-Authored-By: Claude Sonnet 4.6 --- .codespell-ignore-words.txt | 1 + Makefile | 4 +- tools/yamcs/proves_adapter.py | 217 ++++++++++++++++++ yamcs/docker-compose.yml | 10 + yamcs/fprime-gds.yml | 9 + yamcs/pyyaml-override.txt | 3 + yamcs/yamcs-data/etc/processor.yaml | 44 ++++ .../yamcs-data/etc/yamcs.fprime-project.yaml | 71 ++++++ yamcs/yamcs-data/etc/yamcs.yaml | 15 ++ yamcs/yamcs-data/mdb/.gitkeep | 0 10 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 tools/yamcs/proves_adapter.py create mode 100644 yamcs/docker-compose.yml create mode 100644 yamcs/fprime-gds.yml create mode 100644 yamcs/pyyaml-override.txt create mode 100644 yamcs/yamcs-data/etc/processor.yaml create mode 100644 yamcs/yamcs-data/etc/yamcs.fprime-project.yaml create mode 100644 yamcs/yamcs-data/etc/yamcs.yaml create mode 100644 yamcs/yamcs-data/mdb/.gitkeep diff --git a/.codespell-ignore-words.txt b/.codespell-ignore-words.txt index c5fffd7e..9c69ee4e 100644 --- a/.codespell-ignore-words.txt +++ b/.codespell-ignore-words.txt @@ -4,3 +4,4 @@ bufferIn commandIn Ines rsource +ser diff --git a/Makefile b/Makefile index a55c08a2..9647f3be 100644 --- a/Makefile +++ b/Makefile @@ -236,7 +236,9 @@ yamcs: fprime-venv yamcs-dict ## Run YAMCS with serial adapter (Use Case 1: UART --no-convert-dictionary \ --yamcs-config-dir $(shell pwd)/yamcs/yamcs-data \ --yamcs-data-dir $(shell pwd)/yamcs/yamcs-runtime & - @sleep 3 + @sleep 5 + @echo "Starting fprime-yamcs-events bridge..." + $(UV_RUN) fprime-yamcs-events & @echo "Starting serial adapter on $(UART_DEVICE)..." $(UV_RUN) python tools/yamcs/proves_adapter.py \ --mode serial \ diff --git a/tools/yamcs/proves_adapter.py b/tools/yamcs/proves_adapter.py new file mode 100644 index 00000000..db2375fe --- /dev/null +++ b/tools/yamcs/proves_adapter.py @@ -0,0 +1,217 @@ +""" +PROVES YAMCS Adapter — Option B authentication bridge. + +Routes TM frames from the spacecraft to YAMCS (via UDP) and TC frames from +YAMCS (via UDP) back to the spacecraft after wrapping them with the HMAC-SHA256 +authentication header/trailer required by the FSW Authenticate component. + +Use Case 1 (local UART): + python proves_adapter.py --mode serial --uart-device /dev/ttyUSB0 + +Use Case 2 (remote TCP, skeleton): + python proves_adapter.py --mode tcp --tcp-host --tcp-port 5000 \ + --yamcs-host +""" + +import argparse +import socket +import sys +import threading +from pathlib import Path + +# Allow importing authenticate_plugin from the Framing package without installing it. +sys.path.insert(0, str(Path(__file__).parents[2] / "Framing" / "src")) + +from authenticate_plugin import ( # noqa: E402 + AuthenticateFramer, + get_default_auth_key_from_header, +) + +# --------------------------------------------------------------------------- +# TM path helpers +# --------------------------------------------------------------------------- + + +def _forward_tm_serial(ser, tm_sock, yamcs_host: str, tm_port: int, frame_length: int): + """Read fixed-length TM frames from serial and forward to YAMCS via UDP.""" + print(f"[TM] serial → UDP {yamcs_host}:{tm_port} (frame_length={frame_length})") + while True: + frame = ser.read(frame_length) + if frame: + tm_sock.sendto(frame, (yamcs_host, tm_port)) + + +def _forward_tm_tcp( + tcp_sock, tm_sock, yamcs_host: str, tm_port: int, frame_length: int +): + """Read fixed-length TM frames from a TCP connection and forward to YAMCS via UDP.""" + print(f"[TM] TCP → UDP {yamcs_host}:{tm_port} (frame_length={frame_length})") + buf = b"" + while True: + chunk = tcp_sock.recv(4096) + if not chunk: + print("[TM] TCP connection closed.") + break + buf += chunk + while len(buf) >= frame_length: + frame, buf = buf[:frame_length], buf[frame_length:] + tm_sock.sendto(frame, (yamcs_host, tm_port)) + + +# --------------------------------------------------------------------------- +# TC path helpers +# --------------------------------------------------------------------------- + + +def _forward_tc_serial(tc_sock, ser, auth_framer: AuthenticateFramer): + """Receive TC datagrams from YAMCS, wrap with auth, write to serial.""" + print("[TC] UDP → authenticate → serial") + while True: + tc_frame, _ = tc_sock.recvfrom(4096) + wrapped = auth_framer.frame(tc_frame) + ser.write(wrapped) + + +def _forward_tc_tcp(tc_sock, tcp_sock, auth_framer: AuthenticateFramer): + """Receive TC datagrams from YAMCS, wrap with auth, send over TCP.""" + print("[TC] UDP → authenticate → TCP") + while True: + tc_frame, _ = tc_sock.recvfrom(4096) + wrapped = auth_framer.frame(tc_frame) + tcp_sock.sendall(wrapped) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def parse_args(): + p = argparse.ArgumentParser( + description="PROVES YAMCS authentication adapter (Option B)", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + p.add_argument( + "--mode", + choices=["serial", "tcp"], + default="serial", + help="Transport mode: 'serial' for USB-UART (Use Case 1), 'tcp' for bent-pipe (Use Case 2)", + ) + + # Serial options + p.add_argument("--uart-device", default="/dev/ttyUSB0", help="Serial device path") + p.add_argument("--uart-baud", type=int, default=115200, help="Serial baud rate") + + # TCP options (Use Case 2) + p.add_argument( + "--tcp-host", default="127.0.0.1", help="Ground station host (tcp mode)" + ) + p.add_argument( + "--tcp-port", type=int, default=5000, help="Ground station port (tcp mode)" + ) + + # YAMCS UDP endpoints + p.add_argument("--yamcs-host", default="127.0.0.1", help="YAMCS host") + p.add_argument( + "--yamcs-tm-port", + type=int, + default=50000, + help="YAMCS TM UDP port (adapter sends TM here)", + ) + p.add_argument( + "--yamcs-tc-port", + type=int, + default=50001, + help="YAMCS TC UDP port (adapter receives TC from here)", + ) + + # Auth options + p.add_argument( + "--auth-key", + default=None, + help="HMAC key as hex string (no 0x prefix). Defaults to key from AuthDefaultKey.h.", + ) + + # Frame size + p.add_argument( + "--frame-length", + type=int, + default=248, + help="TM frame length in bytes (must match TmFrameFixedSize / YAMCS frameLength)", + ) + + return p.parse_args() + + +def main(): + args = parse_args() + + # Resolve auth key + auth_key = args.auth_key + if auth_key is None: + auth_key = get_default_auth_key_from_header() + print("[auth] Loaded key from AuthDefaultKey.h") + + auth_framer = AuthenticateFramer(authentication_key=auth_key) + + # Shared UDP sockets + tm_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + tc_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + tc_sock.bind(("0.0.0.0", args.yamcs_tc_port)) + + if args.mode == "serial": + import serial # pyserial — installed via requirements.txt + + print(f"[serial] Opening {args.uart_device} @ {args.uart_baud} baud") + ser = serial.Serial(args.uart_device, args.uart_baud, timeout=0.1) + + tm_thread = threading.Thread( + target=_forward_tm_serial, + args=(ser, tm_sock, args.yamcs_host, args.yamcs_tm_port, args.frame_length), + daemon=True, + ) + tc_thread = threading.Thread( + target=_forward_tc_serial, + args=(tc_sock, ser, auth_framer), + daemon=True, + ) + + elif args.mode == "tcp": + print(f"[tcp] Connecting to ground station at {args.tcp_host}:{args.tcp_port}") + tcp_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + tcp_sock.connect((args.tcp_host, args.tcp_port)) + print("[tcp] Connected.") + + tm_thread = threading.Thread( + target=_forward_tm_tcp, + args=( + tcp_sock, + tm_sock, + args.yamcs_host, + args.yamcs_tm_port, + args.frame_length, + ), + daemon=True, + ) + tc_thread = threading.Thread( + target=_forward_tc_tcp, + args=(tc_sock, tcp_sock, auth_framer), + daemon=True, + ) + + print( + f"[adapter] Starting in '{args.mode}' mode. YAMCS at {args.yamcs_host} (TM→:{args.yamcs_tm_port}, TC←:{args.yamcs_tc_port})" + ) + tm_thread.start() + tc_thread.start() + + try: + tm_thread.join() + tc_thread.join() + except KeyboardInterrupt: + print("\n[adapter] Interrupted. Shutting down.") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/yamcs/docker-compose.yml b/yamcs/docker-compose.yml new file mode 100644 index 00000000..7c82b52c --- /dev/null +++ b/yamcs/docker-compose.yml @@ -0,0 +1,10 @@ +services: + yamcs: + image: yamcs/yamcs:latest + ports: + - "8090:8090" + - "50000:50000/udp" + - "50001:50001/udp" + volumes: + - ./yamcs-data:/yamcs-data:ro + command: ["yamcs", "--etc", "/yamcs-data/etc"] diff --git a/yamcs/fprime-gds.yml b/yamcs/fprime-gds.yml new file mode 100644 index 00000000..6fce4a8b --- /dev/null +++ b/yamcs/fprime-gds.yml @@ -0,0 +1,9 @@ +# fprime-gds config for fprime-yamcs (--communication-selection none mode). +# Does NOT include output-unframed-data — that option is for the plain GDS +# and is not recognised by fprime-yamcs's argument parser. +command-line-options: + communication-selection: none + no-app: + dictionary: build-artifacts/zephyr/fprime-zephyr-deployment/dict/ReferenceDeploymentTopologyDictionary.json + frame-size: 248 + framing-selection: authenticate-space-data-link diff --git a/yamcs/pyyaml-override.txt b/yamcs/pyyaml-override.txt new file mode 100644 index 00000000..37cbc627 --- /dev/null +++ b/yamcs/pyyaml-override.txt @@ -0,0 +1,3 @@ +# Override fprime's PyYAML==6.0.2 pin to satisfy fprime-yamcs>=0.1.0 (requires PyYAML>=6.0.3). +# PyYAML 6.0.x is API-stable; the patch bump does not affect fprime-gds behaviour. +PyYAML>=6.0.3 diff --git a/yamcs/yamcs-data/etc/processor.yaml b/yamcs/yamcs-data/etc/processor.yaml new file mode 100644 index 00000000..e3a58fb5 --- /dev/null +++ b/yamcs/yamcs-data/etc/processor.yaml @@ -0,0 +1,44 @@ +realtime: + services: + - class: org.yamcs.StreamTmPacketProvider + - class: org.yamcs.StreamTcCommandReleaser + - class: org.yamcs.tctm.StreamParameterProvider + args: + streams: ["pp_realtime", "sys_param"] + - class: org.yamcs.algorithms.AlgorithmManager + - class: org.yamcs.parameter.LocalParameterManager + config: + subscribeAll: true + persistParameters: true + # Check alarms and also enable the alarm server (that keeps track of unacknowledged alarms) + alarm: + parameterCheck: true + parameterServer: enabled + tmProcessor: + # If container entries fit outside the binary packet, setting this to true causes the error + # to be ignored, otherwise an exception will be printed in Yamcs log output + ignoreOutOfContainerEntries: false + # Record all the parameters that have initial values at the start of the processor + recordInitialValues: true + # Record the local values + recordLocalValues: true + +# Used to perform step-by-step archive replays to displays, etc +Archive: + services: + - class: org.yamcs.tctm.ReplayService + - class: org.yamcs.algorithms.AlgorithmManager + +# Used by the ParameterArchive when rebuilding the parameter archive +ParameterArchive: + services: + - class: org.yamcs.tctm.ReplayService + - class: org.yamcs.algorithms.AlgorithmManager + +# Used for performing archive retrievals via replays (e.g. GET /api/archive/{instance}/parameters/{name*}?source=replay) +ArchiveRetrieval: + services: + - class: org.yamcs.tctm.ReplayService + - class: org.yamcs.algorithms.AlgorithmManager + config: + subscribeContainerArchivePartitions: false diff --git a/yamcs/yamcs-data/etc/yamcs.fprime-project.yaml b/yamcs/yamcs-data/etc/yamcs.fprime-project.yaml new file mode 100644 index 00000000..cd249d72 --- /dev/null +++ b/yamcs/yamcs-data/etc/yamcs.fprime-project.yaml @@ -0,0 +1,71 @@ +services: + - class: org.yamcs.archive.XtceTmRecorder + - class: org.yamcs.archive.ParameterRecorder + - class: org.yamcs.archive.AlarmRecorder + - class: org.yamcs.archive.EventRecorder + - class: org.yamcs.archive.ReplayServer + - class: org.yamcs.parameter.SystemParametersService + args: + producers: + - fs + - jvm + - class: org.yamcs.ProcessorCreatorService + args: + name: realtime + type: realtime + - class: org.yamcs.archive.CommandHistoryRecorder + - class: org.yamcs.parameterarchive.ParameterArchive + args: + realtimeFiller: + enabled: true + - class: org.yamcs.plists.ParameterListService + - class: org.yamcs.timeline.TimelineService +dataLinks: + - name: UDP_TM_IN + class: org.yamcs.tctm.ccsds.UdpTmFrameLink + port: 50000 + frameType: "TM" + spacecraftId: 68 # ComCfg.fpp: SpacecraftId = 0x0044 + frameLength: 248 # ComCfg.fpp: TmFrameFixedSize = 248 + errorDetection: CRC16 + virtualChannels: + - vcId: 1 + ocfPresent: false + service: "PACKET" + maxPacketLength: 248 + packetPreprocessorClassName: com.example.myproject.MyPacketPreprocessor + stream: "tm_realtime" + + - name: UDP_TC_OUT + class: org.yamcs.tctm.ccsds.UdpTcFrameLink + host: localhost + port: 50001 + spacecraftId: 68 + maxFrameLength: 248 + errorDetection: CRC16 + virtualChannels: + - vcId: 1 + ocfPresent: false + service: "PACKET" + priority: 1 + commandPostprocessorClassName: com.example.myproject.MyCommandPostprocessor + stream: "tc_realtime" + useCop1: false + +mdb: + - type: xtce + args: + file: mdb/fprime.xtce.xml + +streamConfig: + tm: + - name: "tm_realtime" + processor: "realtime" + - name: "tm_dump" + cmdHist: ["cmdhist_realtime", "cmdhist_dump"] + event: ["events_realtime", "events_dump"] + param: ["pp_realtime", "pp_dump", "sys_param", "proc_param"] + parameterAlarm: ["alarms_realtime"] + tc: + - name: "tc_realtime" + processor: "realtime" diff --git a/yamcs/yamcs-data/etc/yamcs.yaml b/yamcs/yamcs-data/etc/yamcs.yaml new file mode 100644 index 00000000..e236f204 --- /dev/null +++ b/yamcs/yamcs-data/etc/yamcs.yaml @@ -0,0 +1,15 @@ +services: + - class: org.yamcs.http.HttpServer + args: + port: 8090 + address: "0.0.0.0" + cors: + allowOrigin: "*" + allowCredentials: false + +dataDir: yamcs-data + +instances: + - fprime-project + +secretKey: changeme diff --git a/yamcs/yamcs-data/mdb/.gitkeep b/yamcs/yamcs-data/mdb/.gitkeep new file mode 100644 index 00000000..e69de29b From 27ad83a5547f70decc1dd566befd42d24097b4c5 Mon Sep 17 00:00:00 2001 From: Michael Pham <61564344+Mikefly123@users.noreply.github.com> Date: Tue, 7 Apr 2026 19:44:07 -0700 Subject: [PATCH 03/15] patches: add fprime-yamcs-noapp-path.patch and wire into fprime-venv fprime-yamcs 0.1.1 has two bugs when used with --no-app and --communication-selection none: 1. launch_app() is called unconditionally even when --no-app is set, crashing on parsed_args.app.name (NoneType). Fix: skip launch_app when noapp=True. 2. fprime-yamcs-events (run by YAMCS ProcessRunner, now by the Makefile) and other venv binaries are not on PATH for Java subprocesses spawned by Maven. Fix: prepend sys.executable's parent (venv bin) to PATH in launch_yamcs(). patches/fprime-yamcs-noapp-path.patch encodes both fixes. The fprime-venv Makefile target applies it after pip install, using a grep sentinel to detect whether it has already been applied (idempotent). Co-Authored-By: Claude Sonnet 4.6 --- Makefile | 12 ++++++++++++ patches/fprime-yamcs-noapp-path.patch | 22 ++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 patches/fprime-yamcs-noapp-path.patch diff --git a/Makefile b/Makefile index 9647f3be..cfed1a3a 100644 --- a/Makefile +++ b/Makefile @@ -24,9 +24,21 @@ submodules: ## Initialize and update git submodules export VIRTUAL_ENV ?= $(shell pwd)/fprime-venv .PHONY: fprime-venv +FPRIME_YAMCS_MAIN ?= $(shell pwd)/fprime-venv/lib/python*/site-packages/fprime_yamcs/__main__.py fprime-venv: uv ## Create a virtual environment @$(UV) venv fprime-venv --allow-existing @$(UV) pip install --prerelease=allow --requirement requirements.txt --overrides yamcs/pyyaml-override.txt + @echo "Applying fprime-yamcs noapp/path patch..." + @TARGET=$$(ls $(FPRIME_YAMCS_MAIN) 2>/dev/null | head -1); \ + if [ -z "$$TARGET" ]; then echo "⚠ fprime-yamcs not found, skipping patch"; exit 0; fi; \ + PATCH_DIR=$$(dirname $$TARGET); \ + if grep -q 'venv_bin = str(Path(sys.executable).parent)' $$TARGET; then \ + echo "⚠ fprime-yamcs patch already applied"; \ + elif patch --dry-run -p2 -d $$PATCH_DIR < patches/fprime-yamcs-noapp-path.patch > /dev/null 2>&1; then \ + patch -p2 -d $$PATCH_DIR < patches/fprime-yamcs-noapp-path.patch && echo "✓ Applied fprime-yamcs patch"; \ + else \ + echo "❌ Error: Unable to apply fprime-yamcs patch. Run 'ls $$TARGET' to check."; exit 1; \ + fi .PHONY: zephyr-setup diff --git a/patches/fprime-yamcs-noapp-path.patch b/patches/fprime-yamcs-noapp-path.patch new file mode 100644 index 00000000..a9b56686 --- /dev/null +++ b/patches/fprime-yamcs-noapp-path.patch @@ -0,0 +1,22 @@ +diff --git a/fprime_yamcs/__main__.py b/fprime_yamcs/__main__.py +--- a/fprime_yamcs/__main__.py ++++ b/fprime_yamcs/__main__.py +@@ -147,6 +147,9 @@ def launch_yamcs(parsed_args): + environment = os.environ.copy() + environment["FPRIME_DICTIONARY"] = parsed_args.dictionary + environment["FPRIME_YAMCS_INSTANCE"] = parsed_args.yamcs_events_instance ++ # Ensure the venv bin (fprime-yamcs-events, etc.) is on PATH for Java subprocesses ++ venv_bin = str(Path(sys.executable).parent) ++ environment["PATH"] = venv_bin + os.pathsep + environment.get("PATH", "") + + print(f"[INFO] Using FPRIME_DICTIONARY: {environment['FPRIME_DICTIONARY']}") + print(f"[INFO] Using FPRIME_YAMCS_INSTANCE: {environment['FPRIME_YAMCS_INSTANCE']}") +@@ -205,7 +208,9 @@ def main(): + if parsed_args.yamcs_events_instance is None: + parsed_args.yamcs_events_instance = events_instance +- processes = [launcher(parsed_args) for launcher in [launch_app, launch_yamcs]] ++ launchers = [] if getattr(parsed_args, "noapp", False) else [launch_app] ++ launchers.append(launch_yamcs) ++ processes = [launcher(parsed_args) for launcher in launchers] + print("[INFO] F Prime/YAMCS is now running. CTRL-C to shutdown all components.") + processes[-1].wait() From 584c8f3547dd0a123ff64b1f3cc5ee08a8c4c835 Mon Sep 17 00:00:00 2001 From: Michael Pham <61564344+Mikefly123@users.noreply.github.com> Date: Tue, 7 Apr 2026 20:06:00 -0700 Subject: [PATCH 04/15] Fix fprime-yamcs-events missing dict arg and short TM frame reads - Pass --dictionary to fprime-yamcs-events in Makefile yamcs target so the events bridge can resolve F Prime event IDs (previously failing with "Supply --dictionary or set the FPRIME_DICTIONARY env var") - Fix _forward_tm_serial() to accumulate bytes in a loop until exactly frame_length bytes are available before forwarding; pyserial read() with a short timeout returns partial data, causing YAMCS to receive truncated frames Co-Authored-By: Claude Sonnet 4.6 --- Makefile | 2 +- tools/yamcs/proves_adapter.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index cfed1a3a..c4012adc 100644 --- a/Makefile +++ b/Makefile @@ -250,7 +250,7 @@ yamcs: fprime-venv yamcs-dict ## Run YAMCS with serial adapter (Use Case 1: UART --yamcs-data-dir $(shell pwd)/yamcs/yamcs-runtime & @sleep 5 @echo "Starting fprime-yamcs-events bridge..." - $(UV_RUN) fprime-yamcs-events & + $(UV_RUN) fprime-yamcs-events --dictionary $(shell pwd)/build-artifacts/zephyr/fprime-zephyr-deployment/dict/ReferenceDeploymentTopologyDictionary.json & @echo "Starting serial adapter on $(UART_DEVICE)..." $(UV_RUN) python tools/yamcs/proves_adapter.py \ --mode serial \ diff --git a/tools/yamcs/proves_adapter.py b/tools/yamcs/proves_adapter.py index db2375fe..8277a85b 100644 --- a/tools/yamcs/proves_adapter.py +++ b/tools/yamcs/proves_adapter.py @@ -36,9 +36,15 @@ def _forward_tm_serial(ser, tm_sock, yamcs_host: str, tm_port: int, frame_length """Read fixed-length TM frames from serial and forward to YAMCS via UDP.""" print(f"[TM] serial → UDP {yamcs_host}:{tm_port} (frame_length={frame_length})") while True: - frame = ser.read(frame_length) - if frame: - tm_sock.sendto(frame, (yamcs_host, tm_port)) + # Accumulate bytes until a complete frame is assembled. pyserial's + # read(n) with a short timeout may return fewer than n bytes, so loop + # until we have exactly frame_length bytes before forwarding. + buf = b"" + while len(buf) < frame_length: + chunk = ser.read(frame_length - len(buf)) + if chunk: + buf += chunk + tm_sock.sendto(buf, (yamcs_host, tm_port)) def _forward_tm_tcp( From f92c02e35cd9a0856f800ce4805c1a314bf05f08 Mon Sep 17 00:00:00 2001 From: Michael Pham <61564344+Mikefly123@users.noreply.github.com> Date: Tue, 7 Apr 2026 22:00:56 -0700 Subject: [PATCH 05/15] Fix YAMCS adapter steady-state misalignment and process cleanup - proves_adapter.py: add per-frame CRC validation in the steady-state read loop; re-sync byte-by-byte on failure instead of forwarding misaligned frames to YAMCS permanently. Prints first 6 bytes of any failed frame for diagnosis. Add sys.stdout line-buffering so diagnostics flush immediately to logs. - Makefile yamcs-stop: kill fprime_yamcs Python wrapper and Maven in addition to the JVM; poll until the JVM exits (up to 5 s, then SIGKILL) before returning, preventing port-8090 conflicts on restart. - fprime-yamcs-events busy-wait fix: replace the 100% CPU while True: pass loop with subscription.result(); persist via tools/apply-events-cpu-fix.py wired into the fprime-venv Make target. Co-Authored-By: Claude Sonnet 4.6 --- Makefile | 23 +++++- tools/apply-events-cpu-fix.py | 26 +++++++ tools/yamcs/proves_adapter.py | 129 +++++++++++++++++++++++++++++++--- 3 files changed, 165 insertions(+), 13 deletions(-) create mode 100644 tools/apply-events-cpu-fix.py diff --git a/Makefile b/Makefile index c4012adc..095e1c77 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,10 @@ fprime-venv: uv ## Create a virtual environment else \ echo "❌ Error: Unable to apply fprime-yamcs patch. Run 'ls $$TARGET' to check."; exit 1; \ fi + @echo "Applying fprime-yamcs-events CPU fix..." + @EVENTS_PROC=$$(ls $(shell pwd)/fprime-venv/lib/python*/site-packages/fprime_yamcs/events/processor.py 2>/dev/null | head -1); \ + if [ -z "$$EVENTS_PROC" ]; then echo "⚠ events processor not found, skipping"; exit 0; fi; \ + $(VIRTUAL_ENV)/bin/python tools/apply-events-cpu-fix.py "$$EVENTS_PROC" .PHONY: zephyr-setup @@ -235,9 +239,24 @@ yamcs-dict: fprime-venv ## Generate XTCE dictionary for YAMCS (requires build-ar $(UV_RUN) fprime-to-xtce "$$DICT" -o yamcs/yamcs-data/mdb/fprime.xtce.xml @echo "XTCE dictionary at yamcs/yamcs-data/mdb/fprime.xtce.xml" +.PHONY: yamcs-stop +yamcs-stop: ## Stop all YAMCS-related processes (YAMCS server, events bridge, adapter) + @echo "Stopping YAMCS processes..." + @pkill -f 'proves_adapter.py' 2>/dev/null && echo " stopped serial adapter" || true + @pkill -f 'fprime-yamcs-events' 2>/dev/null && echo " stopped fprime-yamcs-events" || true + @pkill -f 'fprime_yamcs' 2>/dev/null && echo " stopped fprime-yamcs wrapper" || true + @pkill -f 'mvn.*yamcs' 2>/dev/null && echo " stopped Maven yamcs runner" || true + @pkill -f 'org.yamcs.YamcsServer' 2>/dev/null && echo " stopped YAMCS server" || true + @i=0; while pgrep -f 'org.yamcs.YamcsServer' > /dev/null 2>&1; do \ + sleep 0.5; i=$$((i+1)); if [ $$i -ge 10 ]; then \ + echo " Warning: YAMCS server still running after 5s, forcing..."; \ + pkill -9 -f 'org.yamcs.YamcsServer' 2>/dev/null; break; fi; done + @echo "Done." + .PHONY: yamcs yamcs: fprime-venv yamcs-dict ## Run YAMCS with serial adapter (Use Case 1: UART_DEVICE=/dev/ttyXXX) @if [ -z "$(UART_DEVICE)" ]; then echo "Error: set UART_DEVICE=/dev/ttyXXX"; exit 1; fi + @$(MAKE) yamcs-stop @echo "Starting YAMCS (requires Java 11+)..." @mkdir -p $(shell pwd)/yamcs/yamcs-runtime FPRIME_GDS_CONFIG_PATH=$(shell pwd)/yamcs/fprime-gds.yml \ @@ -252,7 +271,7 @@ yamcs: fprime-venv yamcs-dict ## Run YAMCS with serial adapter (Use Case 1: UART @echo "Starting fprime-yamcs-events bridge..." $(UV_RUN) fprime-yamcs-events --dictionary $(shell pwd)/build-artifacts/zephyr/fprime-zephyr-deployment/dict/ReferenceDeploymentTopologyDictionary.json & @echo "Starting serial adapter on $(UART_DEVICE)..." - $(UV_RUN) python tools/yamcs/proves_adapter.py \ + $(VIRTUAL_ENV)/bin/python tools/yamcs/proves_adapter.py \ --mode serial \ --uart-device $(UART_DEVICE) \ --uart-baud 115200 @@ -263,7 +282,7 @@ yamcs-server: yamcs-dict ## Start YAMCS server via Docker (Use Case 2: remote de .PHONY: yamcs-adapter-tcp yamcs-adapter-tcp: fprime-venv ## Start TCP adapter for bent-pipe (GS_HOST=, GS_PORT=, YAMCS_HOST=) - $(UV_RUN) python tools/yamcs/proves_adapter.py \ + $(VIRTUAL_ENV)/bin/python tools/yamcs/proves_adapter.py \ --mode tcp \ --tcp-host $(GS_HOST) \ --tcp-port $(GS_PORT) \ diff --git a/tools/apply-events-cpu-fix.py b/tools/apply-events-cpu-fix.py new file mode 100644 index 00000000..759309b1 --- /dev/null +++ b/tools/apply-events-cpu-fix.py @@ -0,0 +1,26 @@ +"""Apply the fprime-yamcs-events busy-wait CPU fix to the installed package.""" + +import re +import sys + +path = sys.argv[1] +content = open(path).read() + +if "subscription.result()" in content: + print("⚠ fprime-yamcs-events CPU fix already applied") + sys.exit(0) + +fixed = re.sub( + r"# Keep the script running\s*\n\s+while True:\s*\n\s+pass", + "# Block until the WebSocket subscription ends (no CPU spin)\n subscription.result()", + content, +) + +if fixed == content: + print( + "❌ Error: pattern not found in events processor — fix may not be needed or file changed" + ) + sys.exit(1) + +open(path, "w").write(fixed) +print("✓ Applied fprime-yamcs-events CPU fix") diff --git a/tools/yamcs/proves_adapter.py b/tools/yamcs/proves_adapter.py index 8277a85b..19d948d4 100644 --- a/tools/yamcs/proves_adapter.py +++ b/tools/yamcs/proves_adapter.py @@ -19,6 +19,24 @@ import threading from pathlib import Path +# Flush stdout on every print so diagnostic messages appear immediately even +# when the adapter is run as a background process or piped to a log file. +sys.stdout.reconfigure(line_buffering=True) + +# --------------------------------------------------------------------------- +# CRC16-CCITT (CCSDS standard: poly 0x1021, init 0xFFFF, no reflection) +# --------------------------------------------------------------------------- + + +def _crc16_ccitt(data: bytes) -> int: + crc = 0xFFFF + for byte in data: + crc ^= byte << 8 + for _ in range(8): + crc = ((crc << 1) ^ 0x1021 if crc & 0x8000 else crc << 1) & 0xFFFF + return crc + + # Allow importing authenticate_plugin from the Framing package without installing it. sys.path.insert(0, str(Path(__file__).parents[2] / "Framing" / "src")) @@ -32,19 +50,88 @@ # --------------------------------------------------------------------------- -def _forward_tm_serial(ser, tm_sock, yamcs_host: str, tm_port: int, frame_length: int): +def _serial_read_exact(ser, n: int) -> bytes: + """Read exactly n bytes from serial, blocking until all bytes are available.""" + buf = b"" + while len(buf) < n: + chunk = ser.read(n - len(buf)) + if chunk: + buf += chunk + return buf + + +def _sync_serial_frame(ser, frame_length: int, sync_header: bytes) -> bytes: + """Scan the serial stream for a valid frame boundary and return the first complete frame. + + The OS serial buffer may contain a partial frame from before the adapter + started (or the header bytes may appear in payload data as a false positive). + Scan byte by byte: each time the header pattern is found, read a full candidate + frame and validate its CRC. Only accept when the CRC is correct. + """ + print(f"[TM] Searching for frame sync (header={sync_header.hex(' ')})...") + header_len = len(sync_header) + window = b"" + attempts = 0 + while True: + b = ser.read(1) + if not b: + continue + window = (window + b)[-header_len:] + if window != sync_header: + continue + # Candidate sync point — read the rest of the frame and validate CRC. + remainder = _serial_read_exact(ser, frame_length - header_len) + frame = sync_header + remainder + if _crc16_ccitt(frame[:-2]) == int.from_bytes(frame[-2:], "big"): + print(f"[TM] Frame sync acquired (after {attempts} false positives).") + return frame + # CRC mismatch — false positive, keep scanning from the next byte. + attempts += 1 + window = b"" # reset window so we re-scan from inside the false frame + + +def _forward_tm_serial( + ser, + tm_sock, + yamcs_host: str, + tm_port: int, + frame_length: int, + spacecraft_id: int = 68, + vc_id: int = 1, +): """Read fixed-length TM frames from serial and forward to YAMCS via UDP.""" print(f"[TM] serial → UDP {yamcs_host}:{tm_port} (frame_length={frame_length})") + # Compute the expected first 2 bytes of the CCSDS TM primary header: + # bits 15-14: version=0, bits 13-4: spacecraft_id, bits 3-1: vc_id, bit 0: OCF=0 + word0 = (spacecraft_id << 4) | (vc_id << 1) + sync_header = bytes([(word0 >> 8) & 0xFF, word0 & 0xFF]) + + # Scan to the first frame boundary before entering the steady-state read loop. + frame = _sync_serial_frame(ser, frame_length, sync_header) + tm_sock.sendto(frame, (yamcs_host, tm_port)) + frames_sent = 1 + crc_errors = 0 + while True: # Accumulate bytes until a complete frame is assembled. pyserial's - # read(n) with a short timeout may return fewer than n bytes, so loop - # until we have exactly frame_length bytes before forwarding. - buf = b"" - while len(buf) < frame_length: - chunk = ser.read(frame_length - len(buf)) - if chunk: - buf += chunk - tm_sock.sendto(buf, (yamcs_host, tm_port)) + # read(n) with a short timeout may return fewer than n bytes. + frame = _serial_read_exact(ser, frame_length) + + # Validate CRC in steady state to detect alignment drift (serial glitches, + # board reset, USB hiccup, etc.). Re-sync byte-by-byte on failure rather + # than forwarding bad data to YAMCS and staying permanently misaligned. + if _crc16_ccitt(frame[:-2]) != int.from_bytes(frame[-2:], "big"): + crc_errors += 1 + print( + f"[TM] CRC error in steady state (#{crc_errors} after {frames_sent} " + f"good frames) — first 6 bytes: {frame[:6].hex(' ')} — resyncing..." + ) + frame = _sync_serial_frame(ser, frame_length, sync_header) + frames_sent = 0 + crc_errors = 0 + + tm_sock.sendto(frame, (yamcs_host, tm_port)) + frames_sent += 1 def _forward_tm_tcp( @@ -138,13 +225,25 @@ def parse_args(): help="HMAC key as hex string (no 0x prefix). Defaults to key from AuthDefaultKey.h.", ) - # Frame size + # Frame size and CCSDS identifiers p.add_argument( "--frame-length", type=int, default=248, help="TM frame length in bytes (must match TmFrameFixedSize / YAMCS frameLength)", ) + p.add_argument( + "--spacecraft-id", + type=int, + default=68, + help="CCSDS spacecraft ID (10-bit, used for frame sync header)", + ) + p.add_argument( + "--vc-id", + type=int, + default=1, + help="CCSDS virtual channel ID (3-bit, used for frame sync header)", + ) return p.parse_args() @@ -173,7 +272,15 @@ def main(): tm_thread = threading.Thread( target=_forward_tm_serial, - args=(ser, tm_sock, args.yamcs_host, args.yamcs_tm_port, args.frame_length), + args=( + ser, + tm_sock, + args.yamcs_host, + args.yamcs_tm_port, + args.frame_length, + args.spacecraft_id, + args.vc_id, + ), daemon=True, ) tc_thread = threading.Thread( From f0721334e61e27cee057849863bb49f9a61d2c9b Mon Sep 17 00:00:00 2001 From: Michael Pham <61564344+Mikefly123@users.noreply.github.com> Date: Tue, 7 Apr 2026 22:26:24 -0700 Subject: [PATCH 06/15] Fix YAMCS packet preprocessor so Space Packets appear in Packets tab MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - yamcs.fprime-project.yaml: replace placeholder MyPacketPreprocessor with org.yamcs.tctm.cfs.CfsPacketPreprocessor + useLocalGenerationTime. CcsdsPacketPreprocessor is abstract; GenericPacketPreprocessor requires timestampOffset/seqCountOffset args; CfsPacketPreprocessor handles the standard CCSDS primary header and, with useLocalGenerationTime: true, uses reception time instead of a CFS secondary header — correct for F Prime packets. Also remove MyCommandPostprocessor placeholder (not required for TC frames). - tools/apply-yamcs-instance-config-fix.py: new script that patches the fprime-yamcs venv template (which is what the yamcs-maven-plugin actually reads) with the same fixes: correct preprocessor, frame length 248, remove ProcessRunner for fprime-yamcs-events. - Makefile fprime-venv: wire apply-yamcs-instance-config-fix.py into the venv setup so the template is re-patched on every reinstall. Co-Authored-By: Claude Sonnet 4.6 --- Makefile | 4 ++ tools/apply-yamcs-instance-config-fix.py | 62 +++++++++++++++++++ .../yamcs-data/etc/yamcs.fprime-project.yaml | 5 +- 3 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 tools/apply-yamcs-instance-config-fix.py diff --git a/Makefile b/Makefile index 095e1c77..22f8e8d8 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,10 @@ fprime-venv: uv ## Create a virtual environment @EVENTS_PROC=$$(ls $(shell pwd)/fprime-venv/lib/python*/site-packages/fprime_yamcs/events/processor.py 2>/dev/null | head -1); \ if [ -z "$$EVENTS_PROC" ]; then echo "⚠ events processor not found, skipping"; exit 0; fi; \ $(VIRTUAL_ENV)/bin/python tools/apply-events-cpu-fix.py "$$EVENTS_PROC" + @echo "Applying fprime-yamcs instance config fix..." + @INST_CFG=$$(ls $(shell pwd)/fprime-venv/lib/python*/site-packages/fprime_yamcs/yamcs/src/main/yamcs/etc/yamcs.fprime-project.yaml 2>/dev/null | head -1); \ + if [ -z "$$INST_CFG" ]; then echo "⚠ instance config not found, skipping"; exit 0; fi; \ + $(VIRTUAL_ENV)/bin/python tools/apply-yamcs-instance-config-fix.py "$$INST_CFG" .PHONY: zephyr-setup diff --git a/tools/apply-yamcs-instance-config-fix.py b/tools/apply-yamcs-instance-config-fix.py new file mode 100644 index 00000000..13396d0e --- /dev/null +++ b/tools/apply-yamcs-instance-config-fix.py @@ -0,0 +1,62 @@ +"""Apply PROVES-specific fixes to the fprime-yamcs instance config template. + +Replaces placeholder preprocessor/postprocessor class names with the correct +YAMCS built-in CCSDS classes, corrects the frame/packet length from the +generic 1024 default to 248 bytes, and removes the ProcessRunner service +(we launch fprime-yamcs-events separately with --dictionary). +""" + +import sys + +path = sys.argv[1] +content = open(path).read() + +if "frameLength: 248" in content and "CfsPacketPreprocessor" in content: + print("⚠ fprime-yamcs instance config fix already applied") + sys.exit(0) + +fixes = [ + # Replace placeholder preprocessor with CfsPacketPreprocessor + useLocalGenerationTime. + # YAMCS 5.12 requires packetPreprocessorClassName (mandatory field in VcDownlinkManagedParameters). + # CcsdsPacketPreprocessor is abstract; GenericPacketPreprocessor requires timestampOffset/seqCountOffset. + # CfsPacketPreprocessor handles standard CCSDS primary headers and, with useLocalGenerationTime: true, + # uses reception time instead of reading a CFS secondary header — correct for F Prime packets. + ( + " packetPreprocessorClassName: com.example.myproject.MyPacketPreprocessor\n", + " packetPreprocessorClassName: org.yamcs.tctm.cfs.CfsPacketPreprocessor\n" + " packetPreprocessorArgs:\n" + " useLocalGenerationTime: true\n", + ), + # Remove placeholder postprocessor class (not required for TC) + ( + " commandPostprocessorClassName: com.example.myproject.MyCommandPostprocessor\n", + "", + ), + # Correct frame length (template default 1024 is wrong for PROVES: 248 bytes) + ( + " frameLength: 1024\n", + " frameLength: 248 # ComCfg.fpp: TmFrameFixedSize = 248\n", + ), + (" maxPacketLength: 1024\n", " maxPacketLength: 248\n"), + (" maxFrameLength: 1024\n", " maxFrameLength: 248\n"), + # Remove ProcessRunner for fprime-yamcs-events (we start it separately with --dictionary) + ( + " - class: org.yamcs.ProcessRunner\n" + " args:\n" + ' command: ["fprime-yamcs-events"]\n' + ' restart: "always"\n' + " logLevel: INFO\n", + "", + ), +] + +fixed = content +for old, new in fixes: + fixed = fixed.replace(old, new) + +if fixed == content: + print("❌ Error: no changes made — template may have changed upstream") + sys.exit(1) + +open(path, "w").write(fixed) +print("✓ Applied fprime-yamcs instance config fix") diff --git a/yamcs/yamcs-data/etc/yamcs.fprime-project.yaml b/yamcs/yamcs-data/etc/yamcs.fprime-project.yaml index cd249d72..f156aa0b 100644 --- a/yamcs/yamcs-data/etc/yamcs.fprime-project.yaml +++ b/yamcs/yamcs-data/etc/yamcs.fprime-project.yaml @@ -33,7 +33,9 @@ dataLinks: ocfPresent: false service: "PACKET" maxPacketLength: 248 - packetPreprocessorClassName: com.example.myproject.MyPacketPreprocessor + packetPreprocessorClassName: org.yamcs.tctm.cfs.CfsPacketPreprocessor + packetPreprocessorArgs: + useLocalGenerationTime: true stream: "tm_realtime" - name: UDP_TC_OUT @@ -48,7 +50,6 @@ dataLinks: ocfPresent: false service: "PACKET" priority: 1 - commandPostprocessorClassName: com.example.myproject.MyCommandPostprocessor stream: "tc_realtime" useCop1: false From 7a33070b0ee090d00b303110bcb6efe54bfe3d8f Mon Sep 17 00:00:00 2001 From: Michael Pham <61564344+Mikefly123@users.noreply.github.com> Date: Tue, 7 Apr 2026 22:51:47 -0700 Subject: [PATCH 07/15] Fix YAMCS parameter extraction by specifying tm_realtime rootContainer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fprime-to-xtce does not designate a root container in the generated XTCE. Mdb.getRootSequenceContainer() therefore returns null, causing StreamTmPacketProvider to log "MDB does not have a root sequence container and none was defined under streamConfig -> tm" and XtceTmExtractor to run with rootContainer=null — packets arrived in the Links/Packets tabs but zero parameters were ever extracted from any packet. Fix: add rootContainer to the tm_realtime streamConfig entry pointing to CCSDSSpacePacket, the CCSDS primary-header root container from which the full F Prime container hierarchy (FPrimeTelemetryPacket/Channel/Event → concrete per-packet/per-channel containers) descends. Also wired into tools/apply-yamcs-instance-config-fix.py so the fix survives venv reinstalls (the venv template is what yamcs-maven-plugin actually reads). Result: Beacon packet parameters (BootCount, CurrentMode, etc.) now appear and update in the YAMCS Parameters tab. Co-Authored-By: Claude Sonnet 4.6 --- tools/apply-yamcs-instance-config-fix.py | 19 ++++++++++++++++++- .../yamcs-data/etc/yamcs.fprime-project.yaml | 1 + 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/tools/apply-yamcs-instance-config-fix.py b/tools/apply-yamcs-instance-config-fix.py index 13396d0e..741c1bd1 100644 --- a/tools/apply-yamcs-instance-config-fix.py +++ b/tools/apply-yamcs-instance-config-fix.py @@ -11,7 +11,11 @@ path = sys.argv[1] content = open(path).read() -if "frameLength: 248" in content and "CfsPacketPreprocessor" in content: +if ( + "frameLength: 248" in content + and "CfsPacketPreprocessor" in content + and "rootContainer" in content +): print("⚠ fprime-yamcs instance config fix already applied") sys.exit(0) @@ -48,6 +52,19 @@ " logLevel: INFO\n", "", ), + # Add rootContainer to tm_realtime stream so XtceTmExtractor knows where to start. + # Without this, Mdb.getRootSequenceContainer() returns null (fprime-to-xtce doesn't + # set a root in the XTCE), and StreamTmPacketProvider logs a warning and does no + # container matching — packets arrive but zero parameters are extracted. + ( + ' - name: "tm_realtime"\n' + ' processor: "realtime"\n' + ' - name: "tm_dump"\n', + ' - name: "tm_realtime"\n' + ' processor: "realtime"\n' + ' rootContainer: "/ReferenceDeployment|ReferenceDeployment/CCSDSSpacePacket"\n' + ' - name: "tm_dump"\n', + ), ] fixed = content diff --git a/yamcs/yamcs-data/etc/yamcs.fprime-project.yaml b/yamcs/yamcs-data/etc/yamcs.fprime-project.yaml index f156aa0b..c08ecdfe 100644 --- a/yamcs/yamcs-data/etc/yamcs.fprime-project.yaml +++ b/yamcs/yamcs-data/etc/yamcs.fprime-project.yaml @@ -62,6 +62,7 @@ streamConfig: tm: - name: "tm_realtime" processor: "realtime" + rootContainer: "/ReferenceDeployment|ReferenceDeployment/CCSDSSpacePacket" - name: "tm_dump" cmdHist: ["cmdhist_realtime", "cmdhist_dump"] event: ["events_realtime", "events_dump"] From cc7f2bc591e527fcfdc7824de86ee2e3f372993a Mon Sep 17 00:00:00 2001 From: Michael Pham <61564344+Mikefly123@users.noreply.github.com> Date: Tue, 7 Apr 2026 23:41:07 -0700 Subject: [PATCH 08/15] =?UTF-8?q?Fix=20YAMCS=E2=86=92FSW=20command=20path:?= =?UTF-8?q?=20correct=20TC=20framing=20and=20CCSDS=20header=20fields?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The adapter's TC path had two problems preventing commands from reaching the FSW correctly: 1. Wrong framing order — the adapter wrapped auth around the entire TC Transfer Frame from YAMCS, producing Auth(SDLink(SpacePacket)). The FSW uplink pipeline (frameAccumulator → tcDeframer → authenticate → spacePacketDeframer) expects SDLink(Auth(SpacePacket)). Fix: extract the SpacePacket from YAMCS's TC Transfer Frame, auth-wrap just the SpacePacket, then re-frame in a new TC Transfer Frame using SpaceDataLinkFramerDeframer. 2. CCSDS Packet Data Length was zero — fprime-to-xtce encodes this as a FixedValueEntry of 0 in the XTCE, and no built-in YAMCS command postprocessor fills it in without also adding a CFS checksum that corrupts F Prime's FW_PACKET_COMMAND field. Fix: patch Packet Data Length and Source Sequence Count in the adapter before auth-wrapping. Co-Authored-By: Claude Sonnet 4.6 --- tools/yamcs/proves_adapter.py | 152 ++++++++++++++++++++++++++++++---- 1 file changed, 135 insertions(+), 17 deletions(-) diff --git a/tools/yamcs/proves_adapter.py b/tools/yamcs/proves_adapter.py index 19d948d4..ff38bf0f 100644 --- a/tools/yamcs/proves_adapter.py +++ b/tools/yamcs/proves_adapter.py @@ -23,6 +23,10 @@ # when the adapter is run as a background process or piped to a log file. sys.stdout.reconfigure(line_buffering=True) +# Allow importing authenticate_plugin from the Framing package. Must be set +# before the import below so authenticate_plugin.py (and fprime_gds) load OK. +sys.path.insert(0, str(Path(__file__).parents[2] / "Framing" / "src")) + # --------------------------------------------------------------------------- # CRC16-CCITT (CCSDS standard: poly 0x1021, init 0xFFFF, no reflection) # --------------------------------------------------------------------------- @@ -37,13 +41,13 @@ def _crc16_ccitt(data: bytes) -> int: return crc -# Allow importing authenticate_plugin from the Framing package without installing it. -sys.path.insert(0, str(Path(__file__).parents[2] / "Framing" / "src")) - from authenticate_plugin import ( # noqa: E402 AuthenticateFramer, get_default_auth_key_from_header, ) +from fprime_gds.common.communication.ccsds.space_data_link import ( # noqa: E402 + SpaceDataLinkFramerDeframer, +) # --------------------------------------------------------------------------- # TM path helpers @@ -155,23 +159,129 @@ def _forward_tm_tcp( # TC path helpers # --------------------------------------------------------------------------- +# CCSDS TC Transfer Frame structure: 5-byte primary header + data + 2-byte FECF (CRC) +_TC_FRAME_HEADER_SIZE = 5 +_TC_FRAME_CRC_SIZE = 2 + +# CCSDS Space Packet primary header is 6 bytes. +_SP_HEADER_SIZE = 6 + +# Thread-safe counter for CCSDS Source Sequence Count (per-APID not needed — all +# F Prime commands use APID 0). +import itertools # noqa: E402 (stdlib, grouped with runtime helpers) + +_ccsds_seq_counter = itertools.count() + + +def _fix_ccsds_primary_header(space_packet: bytearray) -> None: + """Patch CCSDS Packet Data Length and Source Sequence Count in-place. + + The XTCE generated by fprime-to-xtce uses FixedValueEntry for both fields, + encoding them as literal 0. No built-in YAMCS command postprocessor fills + them in without also adding a CFS secondary-header checksum that would + corrupt the F Prime FW_PACKET_COMMAND type field. We fix them here instead. + + CCSDS primary header layout (6 bytes / 48 bits): + bits 47-45 Version (3) + bit 44 Type (1) + bit 43 Sec Hdr Flag (1) + bits 42-32 APID (11) + bits 31-30 Sequence Flags (2) + bits 29-16 Source Sequence Count (14) + bits 15-0 Packet Data Length (16) + + Packet Data Length = (total_packet_length - primary_header(6) - 1) + """ + if len(space_packet) < _SP_HEADER_SIZE: + return + + # --- Packet Data Length (bytes 4-5) --- + pkt_data_len = len(space_packet) - _SP_HEADER_SIZE - 1 + space_packet[4] = (pkt_data_len >> 8) & 0xFF + space_packet[5] = pkt_data_len & 0xFF + + # --- Source Sequence Count (lower 14 bits of bytes 2-3) --- + seq = next(_ccsds_seq_counter) & 0x3FFF + seq_flags = space_packet[2] & 0xC0 # preserve upper 2 bits (Sequence Flags) + space_packet[2] = seq_flags | ((seq >> 8) & 0x3F) + space_packet[3] = seq & 0xFF + + +def _extract_space_packet(tc_transfer_frame: bytes) -> bytearray: + """Strip the TC Transfer Frame primary header and CRC, returning just the Space Packet. + + YAMCS UdpTcFrameLink sends fully-formed TC Transfer Frames. The FSW expects + the TC Transfer Frame payload to be auth_header + SpacePacket + HMAC — so we + must extract the SpacePacket, apply auth, then re-wrap in a new TC frame. -def _forward_tc_serial(tc_sock, ser, auth_framer: AuthenticateFramer): - """Receive TC datagrams from YAMCS, wrap with auth, write to serial.""" - print("[TC] UDP → authenticate → serial") + Returns a mutable bytearray so _fix_ccsds_primary_header can patch it in-place. + """ + min_len = _TC_FRAME_HEADER_SIZE + _TC_FRAME_CRC_SIZE + 1 + if len(tc_transfer_frame) < min_len: + raise ValueError(f"TC frame too short ({len(tc_transfer_frame)} bytes)") + return bytearray(tc_transfer_frame[_TC_FRAME_HEADER_SIZE:-_TC_FRAME_CRC_SIZE]) + + +def _wrap_tc( + tc_transfer_frame: bytes, + auth_framer: AuthenticateFramer, + sdlink_framer: SpaceDataLinkFramerDeframer, +) -> bytes: + """Extract SpacePacket from YAMCS TC frame, fix CCSDS header, auth-wrap, re-frame. + + FSW uplink pipeline: + UART → CcsdsTcFrameDetector/frameAccumulator → tcDeframer → authenticate → spacePacketDeframer + + The tcDeframer strips the outer TC Transfer Frame and passes its payload to Authenticate. + Authenticate expects: SPI(2) + SeqNum(4) + SpacePacket + HMAC(16). + So the wire format must be: TC_Frame( SPI + SeqNum + SpacePacket + HMAC ). + """ + space_packet = _extract_space_packet(tc_transfer_frame) + _fix_ccsds_primary_header(space_packet) + auth_wrapped = auth_framer.frame( + bytes(space_packet) + ) # SPI + SeqNum + SpacePacket + HMAC + return sdlink_framer.frame(auth_wrapped) # TC_header + auth_wrapped + CRC + + +def _forward_tc_serial( + tc_sock, + ser, + auth_framer: AuthenticateFramer, + sdlink_framer: SpaceDataLinkFramerDeframer, +): + """Receive TC datagrams from YAMCS, extract SpacePacket, auth-wrap, re-frame, write to serial.""" + print("[TC] UDP → extract SpacePacket → authenticate → TC frame → serial") while True: - tc_frame, _ = tc_sock.recvfrom(4096) - wrapped = auth_framer.frame(tc_frame) - ser.write(wrapped) + tc_transfer_frame, _ = tc_sock.recvfrom(4096) + try: + out_frame = _wrap_tc(tc_transfer_frame, auth_framer, sdlink_framer) + except ValueError as exc: + print( + f"[TC] Malformed TC frame from YAMCS ({len(tc_transfer_frame)} bytes): {exc}" + ) + continue + ser.write(out_frame) -def _forward_tc_tcp(tc_sock, tcp_sock, auth_framer: AuthenticateFramer): - """Receive TC datagrams from YAMCS, wrap with auth, send over TCP.""" - print("[TC] UDP → authenticate → TCP") +def _forward_tc_tcp( + tc_sock, + tcp_sock, + auth_framer: AuthenticateFramer, + sdlink_framer: SpaceDataLinkFramerDeframer, +): + """Receive TC datagrams from YAMCS, extract SpacePacket, auth-wrap, re-frame, send over TCP.""" + print("[TC] UDP → extract SpacePacket → authenticate → TC frame → TCP") while True: - tc_frame, _ = tc_sock.recvfrom(4096) - wrapped = auth_framer.frame(tc_frame) - tcp_sock.sendall(wrapped) + tc_transfer_frame, _ = tc_sock.recvfrom(4096) + try: + out_frame = _wrap_tc(tc_transfer_frame, auth_framer, sdlink_framer) + except ValueError as exc: + print( + f"[TC] Malformed TC frame from YAMCS ({len(tc_transfer_frame)} bytes): {exc}" + ) + continue + tcp_sock.sendall(out_frame) # --------------------------------------------------------------------------- @@ -259,6 +369,14 @@ def main(): auth_framer = AuthenticateFramer(authentication_key=auth_key) + # SpaceDataLinkFramerDeframer builds the TC Transfer Frame that wraps + # auth(SpacePacket) for the FSW uplink pipeline (tcDeframer → authenticate). + sdlink_framer = SpaceDataLinkFramerDeframer( + scid=args.spacecraft_id, + vcid=args.vc_id, + frame_size=args.frame_length, + ) + # Shared UDP sockets tm_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) tc_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) @@ -285,7 +403,7 @@ def main(): ) tc_thread = threading.Thread( target=_forward_tc_serial, - args=(tc_sock, ser, auth_framer), + args=(tc_sock, ser, auth_framer, sdlink_framer), daemon=True, ) @@ -308,7 +426,7 @@ def main(): ) tc_thread = threading.Thread( target=_forward_tc_tcp, - args=(tc_sock, tcp_sock, auth_framer), + args=(tc_sock, tcp_sock, auth_framer, sdlink_framer), daemon=True, ) From 8327fe797f31c3bf9bb1228509818780d811c983 Mon Sep 17 00:00:00 2001 From: Michael Pham <61564344+Mikefly123@users.noreply.github.com> Date: Wed, 8 Apr 2026 00:22:12 -0700 Subject: [PATCH 09/15] Fix TM frame loss caused by FSW console text on shared UART The FSW interleaves debug/console output (e.g. [Os::...]) on the same UART as CCSDS TM frames. The previous fixed-length read approach would read a mix of console text and partial frame data, causing CRC errors and frame loss during event bursts. Replace the fixed-length read + resync approach with a rolling buffer that scans for TM sync headers and validates CRC before accepting each frame. Non-frame bytes are efficiently skipped, so frames on both sides of console text are recovered. Also enlarge the OS serial receive buffer to 64 KB to absorb TM bursts. Co-Authored-By: Claude Sonnet 4.6 --- tools/yamcs/proves_adapter.py | 112 +++++++++++++++++++++++++++------- 1 file changed, 90 insertions(+), 22 deletions(-) diff --git a/tools/yamcs/proves_adapter.py b/tools/yamcs/proves_adapter.py index ff38bf0f..dafe3658 100644 --- a/tools/yamcs/proves_adapter.py +++ b/tools/yamcs/proves_adapter.py @@ -103,39 +103,99 @@ def _forward_tm_serial( spacecraft_id: int = 68, vc_id: int = 1, ): - """Read fixed-length TM frames from serial and forward to YAMCS via UDP.""" + """Read fixed-length TM frames from serial and forward to YAMCS via UDP. + + The FSW may interleave console/debug text (e.g. ``[Os::...]``) on the same + UART as CCSDS TM frames. Rather than reading exactly *frame_length* bytes + and hoping they form a clean frame, we maintain a rolling buffer and scan for + the 2-byte sync header. When found, we validate the CRC — if it passes the + frame is forwarded; if it fails the candidate is discarded and scanning + continues. This makes the adapter resilient to arbitrary non-frame data on + the serial link without losing the frame that immediately follows. + """ + import time + print(f"[TM] serial → UDP {yamcs_host}:{tm_port} (frame_length={frame_length})") # Compute the expected first 2 bytes of the CCSDS TM primary header: # bits 15-14: version=0, bits 13-4: spacecraft_id, bits 3-1: vc_id, bit 0: OCF=0 word0 = (spacecraft_id << 4) | (vc_id << 1) sync_header = bytes([(word0 >> 8) & 0xFF, word0 & 0xFF]) - # Scan to the first frame boundary before entering the steady-state read loop. - frame = _sync_serial_frame(ser, frame_length, sync_header) - tm_sock.sendto(frame, (yamcs_host, tm_port)) - frames_sent = 1 - crc_errors = 0 + buf = bytearray() + frames_sent = 0 + last_vc_count = -1 # not yet known + vc_frame_gaps = 0 + junk_bytes = 0 + stats_time = time.monotonic() + + print(f"[TM] Scanning for frames (sync header={sync_header.hex(' ')})...") while True: - # Accumulate bytes until a complete frame is assembled. pyserial's - # read(n) with a short timeout may return fewer than n bytes. - frame = _serial_read_exact(ser, frame_length) - - # Validate CRC in steady state to detect alignment drift (serial glitches, - # board reset, USB hiccup, etc.). Re-sync byte-by-byte on failure rather - # than forwarding bad data to YAMCS and staying permanently misaligned. - if _crc16_ccitt(frame[:-2]) != int.from_bytes(frame[-2:], "big"): - crc_errors += 1 + # Read whatever is available (up to 4 KB) to keep the OS buffer drained. + chunk = ser.read(max(1, ser.in_waiting or 1)) + if not chunk: + continue + buf.extend(chunk) + + # Scan the buffer for complete, CRC-valid frames. + while True: + # Find the sync header in the buffer. + idx = buf.find(sync_header) + if idx == -1: + # No sync header anywhere — discard everything except the last + # byte (which could be the first byte of a future sync header). + if len(buf) > 1: + junk_bytes += len(buf) - 1 + del buf[: len(buf) - 1] + break + + # Discard any non-frame bytes before the sync header. + if idx > 0: + junk_bytes += idx + del buf[:idx] + + # Need a full frame to validate. + if len(buf) < frame_length: + break # wait for more data + + candidate = bytes(buf[:frame_length]) + if _crc16_ccitt(candidate[:-2]) == int.from_bytes(candidate[-2:], "big"): + # Valid frame — forward it. + del buf[:frame_length] + + # VC frame count gap detection (byte 3 of TM primary header). + vc_count = candidate[3] + if last_vc_count >= 0: + expected_vc = (last_vc_count + 1) & 0xFF + if vc_count != expected_vc: + gap = (vc_count - last_vc_count) & 0xFF + vc_frame_gaps += gap - 1 + print( + f"[TM] VC frame gap: expected {expected_vc}, " + f"got {vc_count} ({gap - 1} frame(s) lost)" + ) + last_vc_count = vc_count + + tm_sock.sendto(candidate, (yamcs_host, tm_port)) + frames_sent += 1 + else: + # CRC mismatch — sync header was a false positive (or the + # frame was corrupted by interleaved text). Skip past the + # 2 sync-header bytes and keep scanning from the next byte. + del buf[:2] + + # Periodic stats every 30 seconds. + now = time.monotonic() + if now - stats_time >= 30.0: + elapsed = now - stats_time + rate = frames_sent / elapsed if elapsed else 0 print( - f"[TM] CRC error in steady state (#{crc_errors} after {frames_sent} " - f"good frames) — first 6 bytes: {frame[:6].hex(' ')} — resyncing..." + f"[TM] stats: {frames_sent} frames, {rate:.1f} f/s, " + f"{vc_frame_gaps} gap(s), {junk_bytes} junk bytes skipped" ) - frame = _sync_serial_frame(ser, frame_length, sync_header) + stats_time = now frames_sent = 0 - crc_errors = 0 - - tm_sock.sendto(frame, (yamcs_host, tm_port)) - frames_sent += 1 + junk_bytes = 0 def _forward_tm_tcp( @@ -388,6 +448,14 @@ def main(): print(f"[serial] Opening {args.uart_device} @ {args.uart_baud} baud") ser = serial.Serial(args.uart_device, args.uart_baud, timeout=0.1) + # Enlarge the OS receive buffer so burst TM frames from the FSW don't + # overflow while the adapter is in the sendto() or CRC-check path. + # The default on macOS is only 4 KB (~16 frames at 248 B each). + try: + ser.set_buffer_size(rx_size=65536) + except Exception: + pass # not supported on all platforms; 4 KB default still works + tm_thread = threading.Thread( target=_forward_tm_serial, args=( From c44958286791261e4efff5d4dc53d382bb383a3f Mon Sep 17 00:00:00 2001 From: Michael Pham <61564344+Mikefly123@users.noreply.github.com> Date: Wed, 8 Apr 2026 21:17:27 -0400 Subject: [PATCH 10/15] Updated README With YAMCS Instructions --- README.md | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/README.md b/README.md index c6811caa..798e3d9b 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,12 @@ cp bootable.uf2 [path-to-your-board] ``` Run ```make build``` and reflash bootable.uf2 onto the board anytime you change code. +## Ground Station: F Prime GDS or YAMCS + +You can use either the F Prime GDS or YAMCS as your mission control ground station. + +### Option A: F Prime GDS (Traditional) + If this is your first time running the gds, you must create the authentication plug: ```shell make framer-plugin @@ -92,6 +98,52 @@ Finally, run the fprime-gds. make gds ``` +### Option B: YAMCS (Alternative Mission Control System) + +[YAMCS](https://www.yamcs.org/) (Yet Another Mission Control System) is an alternative ground station interface that provides a web-based mission control system with real-time telemetry visualization, commanding, and parameter trending. + +#### Setup and First Run + +Before running YAMCS for the first time, generate the F Prime dictionary and set up the Python environment: + +```shell +make fprime-venv +``` + +This creates the YAMCS configuration and applies necessary patches (packet preprocessor configuration, TM stream root container, and CPU fixes for the event processor). + +Then start YAMCS with: + +```shell +make yamcs +``` + +YAMCS starts the following components: +1. **YAMCS Server** – web interface and mission control backbone (available at `http://localhost:8090`) +2. **F Prime Adapter** – communicates with the flight software over serial/TCP, translates telemetry frames (TM) and commands (TC) +3. **Events Bridge** – loads 655+ F Prime event definitions and streams events in real-time + +#### Web Interface + +Once YAMCS is running, open your browser to **`http://localhost:8090`** to see YAMCS running. + +#### Stopping YAMCS + +To cleanly shut down YAMCS and all its components: + +```shell +make yamcs-stop +``` + +This kills the adapter, event bridge, and JVM, ensuring clean startup on the next `make yamcs` run. + +#### Troubleshooting YAMCS + +- **No parameters appearing:** Verify the `rootContainer` in `yamcs-data/mdb/fprime.xtce.xml` matches your deployment (e.g., `ReferenceDeployment`) +- **Port 8090 in use:** Run `make yamcs-stop` to ensure previous YAMCS processes are cleaned up +- **TM frame misalignment:** Caused by FSW console text on the serial UART. The adapter's rolling buffer and CRC validation handle this automatically +- **UnsupportedPacketVersionException warnings:** Cosmetic — idle fill bytes have invalid CCSDS version but valid packets are processed correctly + #### Ensuring your authentication/signing is correct The Makefile will ensure the authentication is correct if you run the code on the same computer you flash on. However, if you switch from a computer that compiled the code you will likely have issues with authentication. Here are some things you may encounter From fa24059cb99f30f31f3c3d33194de12cda505df1 Mon Sep 17 00:00:00 2001 From: Michael Pham <61564344+Mikefly123@users.noreply.github.com> Date: Wed, 8 Apr 2026 23:45:26 -0400 Subject: [PATCH 11/15] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- Makefile | 31 ++++++++++++++++++------ README.md | 2 +- tools/apply-events-cpu-fix.py | 4 +-- tools/apply-yamcs-instance-config-fix.py | 6 +++++ yamcs/docker-compose.yml | 2 +- yamcs/yamcs-data/etc/yamcs.yaml | 6 ++--- 6 files changed, 37 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 22f8e8d8..cbbf8f16 100644 --- a/Makefile +++ b/Makefile @@ -246,15 +246,32 @@ yamcs-dict: fprime-venv ## Generate XTCE dictionary for YAMCS (requires build-ar .PHONY: yamcs-stop yamcs-stop: ## Stop all YAMCS-related processes (YAMCS server, events bridge, adapter) @echo "Stopping YAMCS processes..." - @pkill -f 'proves_adapter.py' 2>/dev/null && echo " stopped serial adapter" || true - @pkill -f 'fprime-yamcs-events' 2>/dev/null && echo " stopped fprime-yamcs-events" || true - @pkill -f 'fprime_yamcs' 2>/dev/null && echo " stopped fprime-yamcs wrapper" || true - @pkill -f 'mvn.*yamcs' 2>/dev/null && echo " stopped Maven yamcs runner" || true - @pkill -f 'org.yamcs.YamcsServer' 2>/dev/null && echo " stopped YAMCS server" || true - @i=0; while pgrep -f 'org.yamcs.YamcsServer' > /dev/null 2>&1; do \ + @find_repo_pids() { \ + marker="$$1"; \ + ps -eo pid=,args= | awk -v repo="$(CURDIR)" -v marker="$$marker" 'index($$0, repo) && index($$0, marker) { print $$1 }'; \ + }; \ + stop_repo_processes() { \ + marker="$$1"; \ + label="$$2"; \ + pids="$$(find_repo_pids "$$marker" | tr '\n' ' ' | sed 's/[[:space:]]*$$//')"; \ + if [ -n "$$pids" ]; then \ + kill $$pids 2>/dev/null || true; \ + echo " stopped $$label"; \ + fi; \ + }; \ + stop_repo_processes 'proves_adapter.py' 'serial adapter'; \ + stop_repo_processes 'fprime-yamcs-events' 'fprime-yamcs-events'; \ + stop_repo_processes 'fprime_yamcs' 'fprime-yamcs wrapper'; \ + stop_repo_processes 'mvn' 'Maven yamcs runner'; \ + stop_repo_processes 'org.yamcs.YamcsServer' 'YAMCS server'; \ + i=0; while [ -n "$$(find_repo_pids 'org.yamcs.YamcsServer')" ]; do \ sleep 0.5; i=$$((i+1)); if [ $$i -ge 10 ]; then \ echo " Warning: YAMCS server still running after 5s, forcing..."; \ - pkill -9 -f 'org.yamcs.YamcsServer' 2>/dev/null; break; fi; done + pids="$$(find_repo_pids 'org.yamcs.YamcsServer' | tr '\n' ' ' | sed 's/[[:space:]]*$$//')"; \ + [ -n "$$pids" ] && kill -9 $$pids 2>/dev/null || true; \ + break; \ + fi; \ + done @echo "Done." .PHONY: yamcs diff --git a/README.md b/README.md index 798e3d9b..c07ce8fa 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ This creates the YAMCS configuration and applies necessary patches (packet prepr Then start YAMCS with: ```shell -make yamcs +UART_DEVICE=/dev/ttyXXX make yamcs ``` YAMCS starts the following components: diff --git a/tools/apply-events-cpu-fix.py b/tools/apply-events-cpu-fix.py index 759309b1..3313ff20 100644 --- a/tools/apply-events-cpu-fix.py +++ b/tools/apply-events-cpu-fix.py @@ -11,8 +11,8 @@ sys.exit(0) fixed = re.sub( - r"# Keep the script running\s*\n\s+while True:\s*\n\s+pass", - "# Block until the WebSocket subscription ends (no CPU spin)\n subscription.result()", + r"# Keep the script running\s*\n([ \t]*)while True:\s*\n\1[ \t]+pass", + r"# Block until the WebSocket subscription ends (no CPU spin)\n\1subscription.result()", content, ) diff --git a/tools/apply-yamcs-instance-config-fix.py b/tools/apply-yamcs-instance-config-fix.py index 741c1bd1..1210247c 100644 --- a/tools/apply-yamcs-instance-config-fix.py +++ b/tools/apply-yamcs-instance-config-fix.py @@ -8,6 +8,12 @@ import sys +if len(sys.argv) != 2: + print( + f"Usage: {sys.argv[0]} ", + file=sys.stderr, + ) + sys.exit(1) path = sys.argv[1] content = open(path).read() diff --git a/yamcs/docker-compose.yml b/yamcs/docker-compose.yml index 7c82b52c..93079cdc 100644 --- a/yamcs/docker-compose.yml +++ b/yamcs/docker-compose.yml @@ -1,6 +1,6 @@ services: yamcs: - image: yamcs/yamcs:latest + image: yamcs/yamcs:5.11.7 ports: - "8090:8090" - "50000:50000/udp" diff --git a/yamcs/yamcs-data/etc/yamcs.yaml b/yamcs/yamcs-data/etc/yamcs.yaml index e236f204..07c0a2d1 100644 --- a/yamcs/yamcs-data/etc/yamcs.yaml +++ b/yamcs/yamcs-data/etc/yamcs.yaml @@ -2,9 +2,9 @@ services: - class: org.yamcs.http.HttpServer args: port: 8090 - address: "0.0.0.0" + address: "127.0.0.1" cors: - allowOrigin: "*" + allowOrigin: "http://127.0.0.1:8090" allowCredentials: false dataDir: yamcs-data @@ -12,4 +12,4 @@ dataDir: yamcs-data instances: - fprime-project -secretKey: changeme +secretKey: "7f4d1e9c2a6b8d03f1a5c7e9b2d4f6a8c1e3b5d7f9a2c4e6b8d0f2a4c6e8b1d3" From d5079a2d9bfd1f0391b4d92d0f5555f03909c50a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 9 Apr 2026 05:11:14 +0000 Subject: [PATCH 12/15] feat: add OpCode acknowledgement bridge for YAMCS command verification Agent-Logs-Url: https://github.com/Open-Source-Space-Foundation/proves-core-reference/sessions/26a2a047-98ce-4d48-8143-4d17dcf73dd8 Co-authored-by: Mikefly123 <61564344+Mikefly123@users.noreply.github.com> --- Makefile | 7 + tools/apply-opcode-ack-fix.py | 56 ++++++ tools/yamcs/opcode_ack_bridge.py | 334 +++++++++++++++++++++++++++++++ 3 files changed, 397 insertions(+) create mode 100644 tools/apply-opcode-ack-fix.py create mode 100644 tools/yamcs/opcode_ack_bridge.py diff --git a/Makefile b/Makefile index cbbf8f16..a8200b4d 100644 --- a/Makefile +++ b/Makefile @@ -47,6 +47,10 @@ fprime-venv: uv ## Create a virtual environment @INST_CFG=$$(ls $(shell pwd)/fprime-venv/lib/python*/site-packages/fprime_yamcs/yamcs/src/main/yamcs/etc/yamcs.fprime-project.yaml 2>/dev/null | head -1); \ if [ -z "$$INST_CFG" ]; then echo "⚠ instance config not found, skipping"; exit 0; fi; \ $(VIRTUAL_ENV)/bin/python tools/apply-yamcs-instance-config-fix.py "$$INST_CFG" + @echo "Applying fprime-yamcs-events opcode-ack fix..." + @EVENTS_PROC=$$(ls $(shell pwd)/fprime-venv/lib/python*/site-packages/fprime_yamcs/events/processor.py 2>/dev/null | head -1); \ + if [ -z "$$EVENTS_PROC" ]; then echo "⚠ events processor not found, skipping"; exit 0; fi; \ + $(VIRTUAL_ENV)/bin/python tools/apply-opcode-ack-fix.py "$$EVENTS_PROC" .PHONY: zephyr-setup @@ -260,6 +264,7 @@ yamcs-stop: ## Stop all YAMCS-related processes (YAMCS server, events bridge, ad fi; \ }; \ stop_repo_processes 'proves_adapter.py' 'serial adapter'; \ + stop_repo_processes 'opcode_ack_bridge.py' 'opcode-ack bridge'; \ stop_repo_processes 'fprime-yamcs-events' 'fprime-yamcs-events'; \ stop_repo_processes 'fprime_yamcs' 'fprime-yamcs wrapper'; \ stop_repo_processes 'mvn' 'Maven yamcs runner'; \ @@ -291,6 +296,8 @@ yamcs: fprime-venv yamcs-dict ## Run YAMCS with serial adapter (Use Case 1: UART @sleep 5 @echo "Starting fprime-yamcs-events bridge..." $(UV_RUN) fprime-yamcs-events --dictionary $(shell pwd)/build-artifacts/zephyr/fprime-zephyr-deployment/dict/ReferenceDeploymentTopologyDictionary.json & + @echo "Starting OpCode acknowledgement bridge..." + $(VIRTUAL_ENV)/bin/python tools/yamcs/opcode_ack_bridge.py --dictionary $(shell pwd)/build-artifacts/zephyr/fprime-zephyr-deployment/dict/ReferenceDeploymentTopologyDictionary.json & @echo "Starting serial adapter on $(UART_DEVICE)..." $(VIRTUAL_ENV)/bin/python tools/yamcs/proves_adapter.py \ --mode serial \ diff --git a/tools/apply-opcode-ack-fix.py b/tools/apply-opcode-ack-fix.py new file mode 100644 index 00000000..a44f8f21 --- /dev/null +++ b/tools/apply-opcode-ack-fix.py @@ -0,0 +1,56 @@ +"""Patch FPrimeEventProcessor to include event args in the send_event extra field. + +The extra field is a free-form str→str mapping that YAMCS stores alongside each +event. By including event arguments there, the opcode_ack_bridge can recover the +opcode value from OpCodeCompleted events without needing to re-decode the raw +CCSDS packet or parse the human-readable message string. +""" + +import sys + +if len(sys.argv) != 2: + print( + f"Usage: {sys.argv[0]} ", + file=sys.stderr, + ) + sys.exit(1) + +path = sys.argv[1] +content = open(path).read() + +SENTINEL = "extra=event_args if event_args else None," + +if SENTINEL in content: + print("⚠ fprime-yamcs-events opcode-ack fix already applied") + sys.exit(0) + +OLD = ( + " self.yamcs_client.send_event(\n" + " instance=self.yamcs_instance,\n" + " source='FPrimeEventProcessor',\n" + " event_type=event_name,\n" + " severity=yamcs_severity,\n" + " message=message,\n" + " )\n" +) +NEW = ( + " self.yamcs_client.send_event(\n" + " instance=self.yamcs_instance,\n" + " source='FPrimeEventProcessor',\n" + " event_type=event_name,\n" + " severity=yamcs_severity,\n" + " message=message,\n" + " extra=event_args if event_args else None,\n" + " )\n" +) + +fixed = content.replace(OLD, NEW) + +if fixed == content: + print( + "❌ Error: pattern not found in events processor — fix may not apply or upstream changed" + ) + sys.exit(1) + +open(path, "w").write(fixed) +print("✓ Applied fprime-yamcs-events opcode-ack fix") diff --git a/tools/yamcs/opcode_ack_bridge.py b/tools/yamcs/opcode_ack_bridge.py new file mode 100644 index 00000000..2227b015 --- /dev/null +++ b/tools/yamcs/opcode_ack_bridge.py @@ -0,0 +1,334 @@ +"""OpCode Acknowledgement Bridge — PROVES YAMCS Integration. + +Subscribes to the YAMCS event stream and command history, then bridges +F Prime OpCodeCompleted events to YAMCS command acknowledgements. + +How it works +------------ +1. The FPrimeEventProcessor (fprime-yamcs-events) decodes raw F Prime event + packets and publishes them to YAMCS. For OpCodeCompleted events it also + stores the raw opcode value in the YAMCS event's ``extra`` dict under the + key ``opCode`` (applied via tools/apply-opcode-ack-fix.py). +2. This bridge subscribes to YAMCS command history to build a queue of + pending (issued-but-not-yet-acknowledged) commands keyed by their F Prime + component.command name. +3. It also subscribes to YAMCS events and listens for events whose source is + ``FPrimeEventProcessor`` and whose type ends with ``OpCodeCompleted``. +4. On every such event it: + a. Reads the opcode from ``event.extra["opCode"]``. + b. Cross-references the F Prime GDS dictionary (loaded from the topology + dictionary JSON) to resolve the opcode to the fully-qualified F Prime + command name (e.g. ``cmdDisp.CMD_NO_OP``). + c. Pops the oldest pending YAMCS command with that name from the queue. + d. Posts a ``Completed_Status = OK`` acknowledgement via the YAMCS + UpdateCommandHistory REST API. + +Usage +----- + python opcode_ack_bridge.py \\ + --dictionary /path/to/ReferenceDeploymentTopologyDictionary.json +""" + +import argparse +import logging +import os +import sys +import threading +from collections import deque +from pathlib import Path +from typing import Optional + +# Ensure the Framing/src directory is on sys.path so that fprime_gds can be +# imported when the script is run outside the virtual environment. +sys.path.insert(0, str(Path(__file__).parents[2] / "Framing" / "src")) + +# Flush stdout immediately so log lines appear in real time when used as a +# background process (consistent with proves_adapter.py). +sys.stdout.reconfigure(line_buffering=True) + +from yamcs.client import YamcsClient # noqa: E402 +from yamcs.protobuf.commanding import ( # noqa: E402 + commanding_pb2, + commands_service_pb2, +) +from yamcs.protobuf import yamcs_pb2 # noqa: E402 +from fprime_gds.common.loaders.cmd_json_loader import CmdJsonLoader # noqa: E402 + +logger = logging.getLogger("opcode-ack-bridge") + + +class OpcodeAckBridge: + """Bridge that recognises F Prime OpCodeCompleted events as YAMCS command acks. + + Thread-safety + ~~~~~~~~~~~~~ + Both the command-history subscription callback and the event subscription + callback run in background threads managed by the yamcs-client library. + Access to ``_pending`` is protected by ``_lock``. + """ + + # Source label set by FPrimeEventProcessor when publishing events. + EVENT_SOURCE = "FPrimeEventProcessor" + # F Prime event type suffix that signals command completion. + OPCODE_COMPLETED_SUFFIX = "OpCodeCompleted" + # YAMCS command history attribute name for completion stage. + ACK_STAGE = "Completed" + + def __init__(self, yamcs_url: str, yamcs_instance: str, dictionary_path: str): + self.yamcs_url = yamcs_url + self.yamcs_instance = yamcs_instance + self._lock = threading.Lock() + # {fprime_cmd_name: deque([(yamcs_cmd_id, yamcs_cmd_qualified_name)])} + self._pending: dict = {} + + # Build opcode (int) → CmdTemplate mapping from the F Prime GDS dictionary. + self._opcode_map = self._load_cmd_dict(dictionary_path) + logger.info( + f"Loaded {len(self._opcode_map)} command definition(s) from {dictionary_path}" + ) + + # Connect to YAMCS. + logger.info( + f"Connecting to YAMCS at {yamcs_url}, instance={yamcs_instance!r}" + ) + self.yamcs_client = YamcsClient(yamcs_url) + self.processor_client = self.yamcs_client.get_processor( + yamcs_instance, "realtime" + ) + + # ------------------------------------------------------------------ + # Initialisation helpers + # ------------------------------------------------------------------ + + @staticmethod + def _load_cmd_dict(dictionary_path: str) -> dict: + """Return ``{opcode (int): CmdTemplate}`` from the topology dictionary.""" + loader = CmdJsonLoader(Path(dictionary_path)) + id_dict, _, _ = loader.get_id_dict(dictionary_path) + return id_dict + + # ------------------------------------------------------------------ + # Name conversion utilities + # ------------------------------------------------------------------ + + @staticmethod + def _yamcs_to_fprime_name(yamcs_name: str) -> Optional[str]: + """Convert a YAMCS XTCE-qualified command name to an F Prime ``component.cmd`` name. + + The XTCE generated by fprime-to-xtce uses the convention:: + + /SpaceSystemName|TopologyName/TopologyName|componentName|CMD_MNEMONIC + + The corresponding F Prime fully-qualified name is:: + + componentName.CMD_MNEMONIC + + Example:: + + /ReferenceDeployment|ReferenceDeployment/ReferenceDeployment|cmdDisp|CMD_NO_OP + → cmdDisp.CMD_NO_OP + """ + # Take everything after the last '/' separator. + last_segment = yamcs_name.rsplit("/", 1)[-1] + parts = last_segment.split("|") + if len(parts) >= 2: + return ".".join(parts[-2:]) + return None + + # ------------------------------------------------------------------ + # Subscription callbacks + # ------------------------------------------------------------------ + + def _on_cmd_history(self, cmdhist) -> None: + """Track newly issued commands that have not yet been acknowledged.""" + cmd_id: str = cmdhist.id + yamcs_name: str = cmdhist.name + + # Skip commands that have already received a Completed acknowledgement. + if f"{self.ACK_STAGE}_Status" in (cmdhist.attributes or {}): + return + + fprime_name = self._yamcs_to_fprime_name(yamcs_name) + if fprime_name is None: + return + + with self._lock: + queue = self._pending.setdefault(fprime_name, deque()) + # Avoid duplicates on repeated cmdhist updates for the same command. + if not any(entry[0] == cmd_id for entry in queue): + queue.append((cmd_id, yamcs_name)) + logger.debug( + f"Tracking pending command: {fprime_name!r} id={cmd_id}" + ) + + def _on_event(self, event) -> None: + """Recognise OpCodeCompleted YAMCS events and post command acknowledgements.""" + if event.source != self.EVENT_SOURCE: + return + if not (event.event_type or "").endswith(self.OPCODE_COMPLETED_SUFFIX): + return + + extra = event.extra or {} + opcode_str = extra.get("opCode") + if opcode_str is None: + logger.warning( + f"Received OpCodeCompleted event without 'opCode' in extra — " + f"ensure tools/apply-opcode-ack-fix.py has been applied. " + f"Message: {event.message!r}" + ) + return + + try: + opcode = int(opcode_str) + except ValueError: + logger.warning(f"Malformed opCode value in event extra: {opcode_str!r}") + return + + cmd_template = self._opcode_map.get(opcode) + if cmd_template is None: + logger.warning( + f"OpCodeCompleted for opcode {opcode:#010x} — " + f"opcode not found in F Prime GDS dictionary" + ) + return + + fprime_name: str = cmd_template.get_full_name() + logger.info( + f"OpCodeCompleted → opcode={opcode:#010x} command={fprime_name!r}" + ) + + with self._lock: + queue = self._pending.get(fprime_name) + if not queue: + logger.warning( + f"No pending YAMCS command found for {fprime_name!r} " + f"(opcode={opcode:#010x}). " + "The command may have been issued before this bridge started." + ) + return + cmd_id, yamcs_name = queue.popleft() + + self._post_completed_ack(cmd_id, yamcs_name) + + # ------------------------------------------------------------------ + # YAMCS REST API call + # ------------------------------------------------------------------ + + def _post_completed_ack(self, cmd_id: str, yamcs_name: str) -> None: + """Post a ``Completed_Status = OK`` attribute to the YAMCS command history. + + YAMCS command acknowledgement stages follow the naming convention + ``{stage}_Status`` (string) and ``{stage}_Time`` (timestamp). Setting + ``Completed_Status = OK`` marks the command as successfully completed in + both the YAMCS UI and the command history archive. + """ + req = commands_service_pb2.UpdateCommandHistoryRequest() + req.instance = self.yamcs_instance + req.processor = "realtime" + req.name = yamcs_name + req.id = cmd_id + + attr = req.attributes.add() + attr.name = f"{self.ACK_STAGE}_Status" + attr.value.type = yamcs_pb2.Value.STRING + attr.value.stringValue = "OK" + + # The context path is relative to the YAMCS /api root. + url = f"/processors/{self.yamcs_instance}/realtime/commandhistory{yamcs_name}" + try: + self.yamcs_client.ctx.post_proto(url, data=req.SerializeToString()) + logger.info( + f"✓ Acknowledged {yamcs_name!r} (id={cmd_id}) as {self.ACK_STAGE}=OK" + ) + except Exception as exc: + logger.error( + f"Failed to post command acknowledgement for {yamcs_name!r}: {exc}" + ) + + # ------------------------------------------------------------------ + # Main loop + # ------------------------------------------------------------------ + + def start(self) -> None: + """Subscribe to YAMCS and block until the connection ends or Ctrl-C.""" + logger.info("Subscribing to YAMCS command history…") + self.processor_client.create_command_history_subscription( + on_data=self._on_cmd_history + ) + + logger.info("Subscribing to YAMCS event stream…") + event_sub = self.yamcs_client.create_event_subscription( + instance=self.yamcs_instance, + on_data=self._on_event, + ) + + logger.info( + f"OpCode Ack Bridge running — " + f"YAMCS={self.yamcs_url} instance={self.yamcs_instance!r} " + "Press Ctrl-C to stop." + ) + try: + # Block until the WebSocket subscription ends (no CPU spin). + event_sub.result() + except KeyboardInterrupt: + logger.info("Shutting down OpCode Ack Bridge.") + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Bridge F Prime OpCodeCompleted events to YAMCS command acknowledgements. " + "Cross-references the F Prime GDS dictionary to resolve opcodes to command " + "names and posts Completed_Status=OK to the YAMCS command history." + ), + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--yamcs-url", + default="http://localhost:8090", + help="YAMCS server base URL", + ) + parser.add_argument( + "--instance", + default=os.environ.get("FPRIME_YAMCS_INSTANCE", "fprime-project"), + help="YAMCS instance name", + ) + parser.add_argument( + "--dictionary", + default=os.environ.get("FPRIME_DICTIONARY"), + help="Path to the F Prime topology dictionary JSON file", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Enable DEBUG-level logging", + ) + args = parser.parse_args() + if args.dictionary is None: + parser.error( + "Supply --dictionary or set the FPRIME_DICTIONARY environment variable" + ) + return args + + +def main() -> None: + args = parse_args() + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s: %(message)s", + ) + bridge = OpcodeAckBridge( + yamcs_url=args.yamcs_url, + yamcs_instance=args.instance, + dictionary_path=args.dictionary, + ) + bridge.start() + + +if __name__ == "__main__": + main() From 7b54bf914bcec73aaf1e31d3915b2d7fe7c0790f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 9 Apr 2026 05:12:40 +0000 Subject: [PATCH 13/15] refactor: address code review feedback on opcode-ack bridge Agent-Logs-Url: https://github.com/Open-Source-Space-Foundation/proves-core-reference/sessions/26a2a047-98ce-4d48-8143-4d17dcf73dd8 Co-authored-by: Mikefly123 <61564344+Mikefly123@users.noreply.github.com> --- tools/apply-opcode-ack-fix.py | 6 ++++-- tools/yamcs/opcode_ack_bridge.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/apply-opcode-ack-fix.py b/tools/apply-opcode-ack-fix.py index a44f8f21..d17682c5 100644 --- a/tools/apply-opcode-ack-fix.py +++ b/tools/apply-opcode-ack-fix.py @@ -16,7 +16,8 @@ sys.exit(1) path = sys.argv[1] -content = open(path).read() +with open(path) as f: + content = f.read() SENTINEL = "extra=event_args if event_args else None," @@ -52,5 +53,6 @@ ) sys.exit(1) -open(path, "w").write(fixed) +with open(path, "w") as f: + f.write(fixed) print("✓ Applied fprime-yamcs-events opcode-ack fix") diff --git a/tools/yamcs/opcode_ack_bridge.py b/tools/yamcs/opcode_ack_bridge.py index 2227b015..ef58c473 100644 --- a/tools/yamcs/opcode_ack_bridge.py +++ b/tools/yamcs/opcode_ack_bridge.py @@ -79,7 +79,7 @@ def __init__(self, yamcs_url: str, yamcs_instance: str, dictionary_path: str): self.yamcs_instance = yamcs_instance self._lock = threading.Lock() # {fprime_cmd_name: deque([(yamcs_cmd_id, yamcs_cmd_qualified_name)])} - self._pending: dict = {} + self._pending: dict[str, deque[tuple[str, str]]] = {} # Build opcode (int) → CmdTemplate mapping from the F Prime GDS dictionary. self._opcode_map = self._load_cmd_dict(dictionary_path) @@ -268,7 +268,9 @@ def start(self) -> None: "Press Ctrl-C to stop." ) try: - # Block until the WebSocket subscription ends (no CPU spin). + # result() blocks until the WebSocket subscription is closed by + # the server or by calling subscription.cancel(), avoiding a + # busy-wait loop. A KeyboardInterrupt (Ctrl-C) breaks out cleanly. event_sub.result() except KeyboardInterrupt: logger.info("Shutting down OpCode Ack Bridge.") From 10ac413b47c8ea180bd135eeda35426cbded0556 Mon Sep 17 00:00:00 2001 From: Michael Pham <61564344+Mikefly123@users.noreply.github.com> Date: Wed, 15 Apr 2026 23:25:41 -0700 Subject: [PATCH 14/15] Fixed ACK Bridge --- tools/yamcs/opcode_ack_bridge.py | 55 ++++++++++++++------------------ 1 file changed, 24 insertions(+), 31 deletions(-) diff --git a/tools/yamcs/opcode_ack_bridge.py b/tools/yamcs/opcode_ack_bridge.py index ef58c473..1c672d81 100644 --- a/tools/yamcs/opcode_ack_bridge.py +++ b/tools/yamcs/opcode_ack_bridge.py @@ -8,14 +8,14 @@ 1. The FPrimeEventProcessor (fprime-yamcs-events) decodes raw F Prime event packets and publishes them to YAMCS. For OpCodeCompleted events it also stores the raw opcode value in the YAMCS event's ``extra`` dict under the - key ``opCode`` (applied via tools/apply-opcode-ack-fix.py). + key ``Opcode`` (applied via tools/apply-opcode-ack-fix.py). 2. This bridge subscribes to YAMCS command history to build a queue of pending (issued-but-not-yet-acknowledged) commands keyed by their F Prime component.command name. 3. It also subscribes to YAMCS events and listens for events whose source is ``FPrimeEventProcessor`` and whose type ends with ``OpCodeCompleted``. 4. On every such event it: - a. Reads the opcode from ``event.extra["opCode"]``. + a. Reads the opcode from ``event.extra["Opcode"]``. b. Cross-references the F Prime GDS dictionary (loaded from the topology dictionary JSON) to resolve the opcode to the fully-qualified F Prime command name (e.g. ``cmdDisp.CMD_NO_OP``). @@ -46,13 +46,12 @@ # background process (consistent with proves_adapter.py). sys.stdout.reconfigure(line_buffering=True) +from fprime_gds.common.loaders.cmd_json_loader import CmdJsonLoader # noqa: E402 from yamcs.client import YamcsClient # noqa: E402 +from yamcs.protobuf import yamcs_pb2 # noqa: E402 from yamcs.protobuf.commanding import ( # noqa: E402 - commanding_pb2, commands_service_pb2, ) -from yamcs.protobuf import yamcs_pb2 # noqa: E402 -from fprime_gds.common.loaders.cmd_json_loader import CmdJsonLoader # noqa: E402 logger = logging.getLogger("opcode-ack-bridge") @@ -88,9 +87,7 @@ def __init__(self, yamcs_url: str, yamcs_instance: str, dictionary_path: str): ) # Connect to YAMCS. - logger.info( - f"Connecting to YAMCS at {yamcs_url}, instance={yamcs_instance!r}" - ) + logger.info(f"Connecting to YAMCS at {yamcs_url}, instance={yamcs_instance!r}") self.yamcs_client = YamcsClient(yamcs_url) self.processor_client = self.yamcs_client.get_processor( yamcs_instance, "realtime" @@ -104,7 +101,7 @@ def __init__(self, yamcs_url: str, yamcs_instance: str, dictionary_path: str): def _load_cmd_dict(dictionary_path: str) -> dict: """Return ``{opcode (int): CmdTemplate}`` from the topology dictionary.""" loader = CmdJsonLoader(Path(dictionary_path)) - id_dict, _, _ = loader.get_id_dict(dictionary_path) + id_dict = loader.get_id_dict(dictionary_path) return id_dict # ------------------------------------------------------------------ @@ -115,25 +112,25 @@ def _load_cmd_dict(dictionary_path: str) -> dict: def _yamcs_to_fprime_name(yamcs_name: str) -> Optional[str]: """Convert a YAMCS XTCE-qualified command name to an F Prime ``component.cmd`` name. - The XTCE generated by fprime-to-xtce uses the convention:: + The XTCE generated by fprime-to-xtce nests SpaceSystem elements that + YAMCS represents as ``/``-separated qualified names:: - /SpaceSystemName|TopologyName/TopologyName|componentName|CMD_MNEMONIC + /ReferenceDeployment_ReferenceDeployment/CdhCore/cmdDisp/CMD_NO_OP The corresponding F Prime fully-qualified name is:: - componentName.CMD_MNEMONIC - - Example:: + CdhCore.cmdDisp.CMD_NO_OP - /ReferenceDeployment|ReferenceDeployment/ReferenceDeployment|cmdDisp|CMD_NO_OP - → cmdDisp.CMD_NO_OP + The first segment is the root SpaceSystem (topology name) and is + stripped; the remaining segments are joined with ``.``. """ - # Take everything after the last '/' separator. - last_segment = yamcs_name.rsplit("/", 1)[-1] - parts = last_segment.split("|") - if len(parts) >= 2: - return ".".join(parts[-2:]) - return None + # Strip leading '/' and split on '/'. + parts = yamcs_name.strip("/").split("/") + # Need at least the root SpaceSystem + one more segment. + if len(parts) < 2: + return None + # Drop the root SpaceSystem (topology name), join the rest. + return ".".join(parts[1:]) # ------------------------------------------------------------------ # Subscription callbacks @@ -157,9 +154,7 @@ def _on_cmd_history(self, cmdhist) -> None: # Avoid duplicates on repeated cmdhist updates for the same command. if not any(entry[0] == cmd_id for entry in queue): queue.append((cmd_id, yamcs_name)) - logger.debug( - f"Tracking pending command: {fprime_name!r} id={cmd_id}" - ) + logger.debug(f"Tracking pending command: {fprime_name!r} id={cmd_id}") def _on_event(self, event) -> None: """Recognise OpCodeCompleted YAMCS events and post command acknowledgements.""" @@ -169,10 +164,10 @@ def _on_event(self, event) -> None: return extra = event.extra or {} - opcode_str = extra.get("opCode") + opcode_str = extra.get("Opcode") if opcode_str is None: logger.warning( - f"Received OpCodeCompleted event without 'opCode' in extra — " + f"Received OpCodeCompleted event without 'Opcode' in extra — " f"ensure tools/apply-opcode-ack-fix.py has been applied. " f"Message: {event.message!r}" ) @@ -181,7 +176,7 @@ def _on_event(self, event) -> None: try: opcode = int(opcode_str) except ValueError: - logger.warning(f"Malformed opCode value in event extra: {opcode_str!r}") + logger.warning(f"Malformed Opcode value in event extra: {opcode_str!r}") return cmd_template = self._opcode_map.get(opcode) @@ -193,9 +188,7 @@ def _on_event(self, event) -> None: return fprime_name: str = cmd_template.get_full_name() - logger.info( - f"OpCodeCompleted → opcode={opcode:#010x} command={fprime_name!r}" - ) + logger.info(f"OpCodeCompleted → opcode={opcode:#010x} command={fprime_name!r}") with self._lock: queue = self._pending.get(fprime_name) From ae1f2517dba3205812e71f35d1967e6c5288859e Mon Sep 17 00:00:00 2001 From: Michael Pham <61564344+Mikefly123@users.noreply.github.com> Date: Wed, 15 Apr 2026 23:59:20 -0700 Subject: [PATCH 15/15] fix opcode ack bridge: correct key lookup, XTCE name parsing, and add dual-stage acks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three bugs fixed: - get_id_dict() returns a single dict, not a 3-tuple (ValueError on startup) - Event extra key was "opCode" but F Prime names it "Opcode" (acks never fired) - YAMCS XTCE uses '/'-separated SpaceSystem paths, not '|' (pending queue always empty) Added OpCodeDispatched → Acknowledged ack alongside existing OpCodeCompleted → CommandComplete (Completion). Timestamps sent via both _Time STRING attribute and protobuf attr.time field. Co-Authored-By: Claude Opus 4.6 --- tools/yamcs/opcode_ack_bridge.py | 82 +++++++++++++++++++------------- 1 file changed, 50 insertions(+), 32 deletions(-) diff --git a/tools/yamcs/opcode_ack_bridge.py b/tools/yamcs/opcode_ack_bridge.py index 1c672d81..3631411a 100644 --- a/tools/yamcs/opcode_ack_bridge.py +++ b/tools/yamcs/opcode_ack_bridge.py @@ -13,15 +13,18 @@ pending (issued-but-not-yet-acknowledged) commands keyed by their F Prime component.command name. 3. It also subscribes to YAMCS events and listens for events whose source is - ``FPrimeEventProcessor`` and whose type ends with ``OpCodeCompleted``. + ``FPrimeEventProcessor`` and whose type ends with ``OpCodeDispatched`` + or ``OpCodeCompleted``. 4. On every such event it: a. Reads the opcode from ``event.extra["Opcode"]``. b. Cross-references the F Prime GDS dictionary (loaded from the topology dictionary JSON) to resolve the opcode to the fully-qualified F Prime command name (e.g. ``cmdDisp.CMD_NO_OP``). - c. Pops the oldest pending YAMCS command with that name from the queue. - d. Posts a ``Completed_Status = OK`` acknowledgement via the YAMCS - UpdateCommandHistory REST API. + c. For OpCodeDispatched: posts ``Acknowledged_Status = OK`` (command was + dispatched to the target component). + d. For OpCodeCompleted: posts ``CommandComplete_Status = OK`` (populates + the YAMCS Completion section) and removes the command from the pending + queue. Usage ----- @@ -35,6 +38,7 @@ import sys import threading from collections import deque +from datetime import datetime, timezone from pathlib import Path from typing import Optional @@ -68,10 +72,9 @@ class OpcodeAckBridge: # Source label set by FPrimeEventProcessor when publishing events. EVENT_SOURCE = "FPrimeEventProcessor" - # F Prime event type suffix that signals command completion. + # F Prime event type suffixes for command lifecycle events. + OPCODE_DISPATCHED_SUFFIX = "OpCodeDispatched" OPCODE_COMPLETED_SUFFIX = "OpCodeCompleted" - # YAMCS command history attribute name for completion stage. - ACK_STAGE = "Completed" def __init__(self, yamcs_url: str, yamcs_instance: str, dictionary_path: str): self.yamcs_url = yamcs_url @@ -141,8 +144,8 @@ def _on_cmd_history(self, cmdhist) -> None: cmd_id: str = cmdhist.id yamcs_name: str = cmdhist.name - # Skip commands that have already received a Completed acknowledgement. - if f"{self.ACK_STAGE}_Status" in (cmdhist.attributes or {}): + # Skip commands that have already received a completion acknowledgement. + if "CommandComplete_Status" in (cmdhist.attributes or {}): return fprime_name = self._yamcs_to_fprime_name(yamcs_name) @@ -157,17 +160,25 @@ def _on_cmd_history(self, cmdhist) -> None: logger.debug(f"Tracking pending command: {fprime_name!r} id={cmd_id}") def _on_event(self, event) -> None: - """Recognise OpCodeCompleted YAMCS events and post command acknowledgements.""" + """Recognise OpCodeDispatched/OpCodeCompleted events and post acks.""" if event.source != self.EVENT_SOURCE: return - if not (event.event_type or "").endswith(self.OPCODE_COMPLETED_SUFFIX): + + event_type = event.event_type or "" + if event_type.endswith(self.OPCODE_DISPATCHED_SUFFIX): + stage = "Acknowledged" + pop_pending = False + elif event_type.endswith(self.OPCODE_COMPLETED_SUFFIX): + stage = "CommandComplete" + pop_pending = True + else: return extra = event.extra or {} opcode_str = extra.get("Opcode") if opcode_str is None: logger.warning( - f"Received OpCodeCompleted event without 'Opcode' in extra — " + f"Received {event_type} event without 'Opcode' in extra — " f"ensure tools/apply-opcode-ack-fix.py has been applied. " f"Message: {event.message!r}" ) @@ -182,13 +193,13 @@ def _on_event(self, event) -> None: cmd_template = self._opcode_map.get(opcode) if cmd_template is None: logger.warning( - f"OpCodeCompleted for opcode {opcode:#010x} — " + f"{event_type} for opcode {opcode:#010x} — " f"opcode not found in F Prime GDS dictionary" ) return fprime_name: str = cmd_template.get_full_name() - logger.info(f"OpCodeCompleted → opcode={opcode:#010x} command={fprime_name!r}") + logger.info(f"{event_type} → opcode={opcode:#010x} command={fprime_name!r}") with self._lock: queue = self._pending.get(fprime_name) @@ -199,21 +210,22 @@ def _on_event(self, event) -> None: "The command may have been issued before this bridge started." ) return - cmd_id, yamcs_name = queue.popleft() + if pop_pending: + cmd_id, yamcs_name = queue.popleft() + else: + cmd_id, yamcs_name = queue[0] - self._post_completed_ack(cmd_id, yamcs_name) + self._post_ack(cmd_id, yamcs_name, stage) # ------------------------------------------------------------------ # YAMCS REST API call # ------------------------------------------------------------------ - def _post_completed_ack(self, cmd_id: str, yamcs_name: str) -> None: - """Post a ``Completed_Status = OK`` attribute to the YAMCS command history. + def _post_ack(self, cmd_id: str, yamcs_name: str, stage: str) -> None: + """Post a ``{stage}_Status = OK`` attribute to the YAMCS command history. YAMCS command acknowledgement stages follow the naming convention - ``{stage}_Status`` (string) and ``{stage}_Time`` (timestamp). Setting - ``Completed_Status = OK`` marks the command as successfully completed in - both the YAMCS UI and the command history archive. + ``{stage}_Status`` (string) and ``{stage}_Time`` (timestamp). """ req = commands_service_pb2.UpdateCommandHistoryRequest() req.instance = self.yamcs_instance @@ -221,22 +233,28 @@ def _post_completed_ack(self, cmd_id: str, yamcs_name: str) -> None: req.name = yamcs_name req.id = cmd_id - attr = req.attributes.add() - attr.name = f"{self.ACK_STAGE}_Status" - attr.value.type = yamcs_pb2.Value.STRING - attr.value.stringValue = "OK" + now = datetime.now(timezone.utc) + now_ms = int(now.timestamp() * 1000) + now_iso = now.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + attr_status = req.attributes.add() + attr_status.name = f"{stage}_Status" + attr_status.value.type = yamcs_pb2.Value.STRING + attr_status.value.stringValue = "OK" + attr_status.time = now_ms + + attr_time = req.attributes.add() + attr_time.name = f"{stage}_Time" + attr_time.value.type = yamcs_pb2.Value.STRING + attr_time.value.stringValue = now_iso + attr_time.time = now_ms - # The context path is relative to the YAMCS /api root. url = f"/processors/{self.yamcs_instance}/realtime/commandhistory{yamcs_name}" try: self.yamcs_client.ctx.post_proto(url, data=req.SerializeToString()) - logger.info( - f"✓ Acknowledged {yamcs_name!r} (id={cmd_id}) as {self.ACK_STAGE}=OK" - ) + logger.info(f"✓ {stage}=OK for {yamcs_name!r} (id={cmd_id})") except Exception as exc: - logger.error( - f"Failed to post command acknowledgement for {yamcs_name!r}: {exc}" - ) + logger.error(f"Failed to post {stage} ack for {yamcs_name!r}: {exc}") # ------------------------------------------------------------------ # Main loop