From 4b828d74ca05f18bf7af996346386bc1841546e5 Mon Sep 17 00:00:00 2001 From: cj-vana Date: Thu, 14 May 2026 21:52:07 -0600 Subject: [PATCH] =?UTF-8?q?forge:=20step=2010=20=E2=80=94=20add=20CI=20saf?= =?UTF-8?q?ety=20checks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/firmware-build.yml | 8 +- scripts/build-representative.sh | 19 ++++- scripts/check-bot-safety.sh | 86 +++++++++++++++++++++ scripts/parse-size-report.py | 109 +++++++++++++++++++++++++++ scripts/verify.sh | 61 +++++++++++++++ 5 files changed, 281 insertions(+), 2 deletions(-) create mode 100644 scripts/check-bot-safety.sh create mode 100644 scripts/verify.sh diff --git a/.github/workflows/firmware-build.yml b/.github/workflows/firmware-build.yml index 835eb9e..b3d3639 100644 --- a/.github/workflows/firmware-build.yml +++ b/.github/workflows/firmware-build.yml @@ -32,8 +32,14 @@ jobs: - name: Install PlatformIO run: python3 -m pip install --upgrade platformio + - name: Run host and safety checks + run: bash scripts/verify.sh --no-build + - name: Build representative firmware - run: bash scripts/build-representative.sh --baseline + run: | + MESHCORE_SKIP_APPLY_PATCHES=1 \ + SIZE_REPORT=out/size/report.txt \ + bash scripts/build-representative.sh --baseline - name: Upload size reports uses: actions/upload-artifact@v4 diff --git a/scripts/build-representative.sh b/scripts/build-representative.sh index 7a8e7cf..29eafbb 100644 --- a/scripts/build-representative.sh +++ b/scripts/build-representative.sh @@ -10,6 +10,10 @@ Usage: scripts/build-representative.sh [--baseline|--compare ] Applies the MeshCore patch queue, builds the representative companion firmware environments, copies firmware artifacts to out/firmware, and writes a size summary to out/size/summary.json. + +To intentionally refresh a baseline, run --baseline and review out/size/summary.json +before copying it to the tracked baseline path. Optional environment variables: +SIZE_THRESHOLDS=, SIZE_REPORT=, SIZE_ENFORCE_THRESHOLDS=1. EOF } @@ -87,7 +91,11 @@ if [ ! -f "${MESHCORE_DIR}/build.sh" ]; then exit 1 fi -"${MESHCORE_FW_ROOT}/scripts/apply-patches.sh" +if [ "${MESHCORE_SKIP_APPLY_PATCHES:-0}" = "1" ]; then + echo "Skipping MeshCore patch application; using current vendor/MeshCore tree." +else + "${MESHCORE_FW_ROOT}/scripts/apply-patches.sh" +fi OUT_DIR="${MESHCORE_FW_ROOT}/out" LOG_DIR="${OUT_DIR}/size" @@ -121,6 +129,15 @@ parser_args=( for env in "${REPRESENTATIVE_ENVS[@]}"; do parser_args+=(--env "$env") done +if [ -n "${SIZE_THRESHOLDS:-}" ]; then + parser_args+=(--thresholds "$SIZE_THRESHOLDS") +fi +if [ -n "${SIZE_REPORT:-}" ]; then + parser_args+=(--report "$SIZE_REPORT") +fi +if [ "${SIZE_ENFORCE_THRESHOLDS:-0}" = "1" ]; then + parser_args+=(--enforce-thresholds) +fi case "$mode" in baseline) diff --git a/scripts/check-bot-safety.sh b/scripts/check-bot-safety.sh new file mode 100644 index 0000000..518e428 --- /dev/null +++ b/scripts/check-bot-safety.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -euo pipefail + +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/meshcore-env.sh" + +bot_sources=( + "${MESHCORE_DIR}/examples/companion_radio/BotTypes.h" + "${MESHCORE_DIR}/examples/companion_radio/FirmwareBot.h" + "${MESHCORE_DIR}/examples/companion_radio/FirmwareBot.cpp" + "${MESHCORE_DIR}/examples/companion_radio/BotCommands.h" + "${MESHCORE_DIR}/examples/companion_radio/BotCommands.cpp" + "${MESHCORE_DIR}/examples/companion_radio/BotPolicy.h" + "${MESHCORE_DIR}/examples/companion_radio/BotPolicy.cpp" + "${MESHCORE_DIR}/examples/companion_radio/BotPrefs.h" + "${MESHCORE_DIR}/examples/companion_radio/BotPrefs.cpp" + "${MESHCORE_DIR}/examples/companion_radio/EmergencyForwarder.h" + "${MESHCORE_DIR}/examples/companion_radio/EmergencyForwarder.cpp" + "${MESHCORE_DIR}/examples/companion_radio/KnownBotRegistry.h" + "${MESHCORE_DIR}/examples/companion_radio/KnownBotRegistry.cpp" + "${MESHCORE_DIR}/examples/companion_radio/ResponseCoordinator.h" + "${MESHCORE_DIR}/examples/companion_radio/ResponseCoordinator.cpp" +) + +missing=0 +for file in "${bot_sources[@]}"; do + if [ ! -f "$file" ]; then + echo "Missing bot source: ${file#${MESHCORE_FW_ROOT}/}" >&2 + missing=1 + fi +done +if [ "$missing" -ne 0 ]; then + exit 1 +fi + +forbidden='\bString\b|std::|\bnew[[:space:]]|\bdelete[[:space:]]|\b(malloc|calloc|realloc|free)[[:space:]]*\(|ArduinoJson|JsonDocument|HTTPClient|WiFiClient|WebServer|AsyncWebServer|socket[[:space:]]*\(' +if grep -n -E "$forbidden" "${bot_sources[@]}"; then + echo "Forbidden dynamic allocation, container, JSON, or network API found in firmware bot sources." >&2 + exit 1 +fi + +forbidden_timing='\b(txdelay|rxdelay)\b|direct[._-]?tx[._-]?delay|airtime_factor|rx_delay_base' +if grep -n -E -i "$forbidden_timing" "${bot_sources[@]}"; then + echo "Forbidden lower-layer timing dependency found in firmware bot sources." >&2 + exit 1 +fi + +require_pattern() { + local pattern="$1" + shift + local label="$1" + shift + if ! grep -q -E "$pattern" "$@"; then + echo "Missing safety marker: $label" >&2 + exit 1 + fi +} + +require_pattern 'BOT_POLICY_IGNORE' 'normal Public bot traffic is ignored silently' \ + "${MESHCORE_DIR}/examples/companion_radio/BotPolicy.cpp" \ + "${MESHCORE_DIR}/examples/companion_radio/BotTypes.h" +require_pattern 'BOT_POLICY_EMERGENCY_FORWARD' 'emergency traffic forwarding policy exists' \ + "${MESHCORE_DIR}/examples/companion_radio/BotPolicy.cpp" \ + "${MESHCORE_DIR}/examples/companion_radio/BotPolicy.h" +require_pattern 'recordBotObservation\(|sendQueuedEmergencyForwards\(|tickBot\(' 'emergency path is wired in MyMesh' \ + "${MESHCORE_DIR}/examples/companion_radio/MyMesh.cpp" +require_pattern '_prefs\.path_hash_mode[[:space:]]*=[[:space:]]*1' 'bot firmware defaults to two-byte path hashes' \ + "${MESHCORE_DIR}/examples/companion_radio/MyMesh.cpp" +require_pattern 'CMESH_BOT_ENABLED=1' 'production bot build flag is enabled' \ + "${MESHCORE_DIR}/platformio.ini" +require_pattern 'ENABLE_PRIVATE_KEY_IMPORT=0' 'private key import disabled in production bot flags' \ + "${MESHCORE_DIR}/platformio.ini" +require_pattern 'ENABLE_PRIVATE_KEY_EXPORT=0' 'private key export disabled in production bot flags' \ + "${MESHCORE_DIR}/platformio.ini" +require_pattern '\$\{cmesh_bot_production\.build_flags\}' 'representative envs include production bot flags' \ + "${MESHCORE_DIR}/variants/heltec_v3/platformio.ini" \ + "${MESHCORE_DIR}/variants/rak4631/platformio.ini" + +count="$(grep -h -E '\$\{cmesh_bot_production\.build_flags\}' \ + "${MESHCORE_DIR}/variants/heltec_v3/platformio.ini" \ + "${MESHCORE_DIR}/variants/rak4631/platformio.ini" | wc -l | tr -d ' ')" +if [ "$count" -lt 4 ]; then + echo "Expected production bot flags in four representative envs, found $count." >&2 + exit 1 +fi + +echo "Firmware bot safety checks passed." diff --git a/scripts/parse-size-report.py b/scripts/parse-size-report.py index 932b84b..d84836b 100644 --- a/scripts/parse-size-report.py +++ b/scripts/parse-size-report.py @@ -98,6 +98,102 @@ def add_delta(entry, baseline_by_env): entry["delta"] = delta +def metric_limit(config, metric, level): + value = config.get(metric, {}) + if isinstance(value, dict): + return value.get(f"{level}_delta", value.get(level)) + return None + + +def env_threshold_config(thresholds, env): + config = dict(thresholds.get("defaults", {})) + config.update(thresholds.get("environments", {}).get(env, {})) + return config + + +def evaluate_thresholds(summary, thresholds): + checks = [] + status = "ok" + for entry in summary["environments"]: + delta = entry.get("delta") + if not delta: + checks.append({"env": entry["env"], "status": "not_evaluated", "reason": "no baseline delta"}) + continue + + config = env_threshold_config(thresholds, entry["env"]) + for metric in ("flash_used", "ram_used"): + value = delta.get(metric) + if value is None: + checks.append({"env": entry["env"], "metric": metric, "status": "not_evaluated", "reason": "missing delta"}) + continue + warn_limit = metric_limit(config, metric, "warn") + fail_limit = metric_limit(config, metric, "fail") + check_status = "ok" + if fail_limit is not None and value > fail_limit: + check_status = "fail" + status = "fail" + elif warn_limit is not None and value > warn_limit: + check_status = "warn" + if status == "ok": + status = "warn" + checks.append({ + "env": entry["env"], + "metric": metric, + "delta": value, + "warn_delta": warn_limit, + "fail_delta": fail_limit, + "status": check_status, + }) + + artifact_config = config.get("artifact_bytes_by_kind", {}) + for kind, value in delta.get("artifact_bytes_by_kind", {}).items(): + limits = artifact_config.get(kind, {}) + warn_limit = limits.get("warn_delta", limits.get("warn")) + fail_limit = limits.get("fail_delta", limits.get("fail")) + check_status = "ok" + if fail_limit is not None and value > fail_limit: + check_status = "fail" + status = "fail" + elif warn_limit is not None and value > warn_limit: + check_status = "warn" + if status == "ok": + status = "warn" + checks.append({ + "env": entry["env"], + "metric": f"artifact_bytes_by_kind.{kind}", + "delta": value, + "warn_delta": warn_limit, + "fail_delta": fail_limit, + "status": check_status, + }) + + return {"status": status, "checks": checks} + + +def report_lines(summary): + lines = [f"Size summary ({summary['mode']})"] + for entry in summary["environments"]: + lines.append(f"- {entry['env']}:") + lines.append(f" RAM: {entry['ram_used']} / {entry['ram_total']} bytes") + lines.append(f" Flash: {entry['flash_used']} / {entry['flash_total']} bytes") + if entry.get("delta"): + delta = entry["delta"] + lines.append(f" Delta: RAM {delta.get('ram_used')} bytes, flash {delta.get('flash_used')} bytes") + for name, size in sorted(entry.get("artifact_bytes", {}).items()): + lines.append(f" Artifact: {name} = {size} bytes") + thresholds = summary.get("thresholds") + if thresholds: + lines.append(f"Threshold status: {thresholds['status']}") + for check in thresholds["checks"]: + if check["status"] == "ok": + continue + reason = check.get("reason", "") + metric = check.get("metric", "all") + delta = check.get("delta", "n/a") + lines.append(f"- {check['env']} {metric}: {check['status']} delta={delta} {reason}".rstrip()) + return lines + + def build_summary(args): log_dir = args.logs artifact_dir = args.artifacts @@ -136,16 +232,29 @@ def main(): parser.add_argument("--env", action="append", help="Environment name to include; may be passed more than once") parser.add_argument("--baseline", action="store_true", help="Mark output as a baseline summary") parser.add_argument("--compare", type=Path, help="Baseline JSON to compare against") + parser.add_argument("--thresholds", type=Path, help="JSON file with warn/fail size delta thresholds") + parser.add_argument("--enforce-thresholds", action="store_true", help="Exit nonzero when threshold status is fail") + parser.add_argument("--report", type=Path, help="Write a human-readable size report to this path") parser.add_argument("--output", type=Path, help="Write JSON summary to this path") args = parser.parse_args() summary = build_summary(args) + if args.thresholds: + thresholds = json.loads(args.thresholds.read_text()) + summary["thresholds"] = evaluate_thresholds(summary, thresholds) + text = json.dumps(summary, indent=2, sort_keys=True) if args.output: args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(text + "\n") + if args.report: + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text("\n".join(report_lines(summary)) + "\n") print(text) + if args.enforce_thresholds and summary.get("thresholds", {}).get("status") == "fail": + raise SystemExit(1) + if __name__ == "__main__": main() diff --git a/scripts/verify.sh b/scripts/verify.sh new file mode 100644 index 0000000..b6b7ed8 --- /dev/null +++ b/scripts/verify.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail + +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/meshcore-env.sh" + +usage() { + cat <<'EOF' +Usage: scripts/verify.sh [--no-build|--build] + +Runs local pre-review checks. By default, this applies the MeshCore patch queue +when needed, runs host tests, runs firmware bot safety checks, and compiles +Python helper scripts. Use --build to also run representative firmware builds. +EOF +} + +run_build=0 +while [ "$#" -gt 0 ]; do + case "$1" in + --help|-h) + usage + exit 0 + ;; + --no-build) + run_build=0 + shift + ;; + --build) + run_build=1 + shift + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [ ! -d "${MESHCORE_DIR}/.git" ] && [ ! -f "${MESHCORE_DIR}/.git" ]; then + echo "MeshCore submodule is missing at ${MESHCORE_DIR}" >&2 + echo "Run: git submodule update --init --recursive" >&2 + exit 1 +fi + +if ! git -C "${MESHCORE_DIR}" diff --quiet --cached || ! git -C "${MESHCORE_DIR}" diff --quiet || [ -n "$(git -C "${MESHCORE_DIR}" ls-files --others --exclude-standard)" ]; then + echo "MeshCore submodule is dirty; verify existing patched tree without applying patches." +else + "${MESHCORE_FW_ROOT}/scripts/apply-patches.sh" +fi + +python3 "${MESHCORE_FW_ROOT}/tests/firmware_bot/run_tests.py" +bash "${MESHCORE_FW_ROOT}/scripts/check-bot-safety.sh" +python3 -m py_compile \ + "${MESHCORE_FW_ROOT}/scripts/parse-size-report.py" \ + "${MESHCORE_FW_ROOT}/tests/firmware_bot/run_tests.py" + +if [ "$run_build" -eq 1 ]; then + MESHCORE_SKIP_APPLY_PATCHES=1 bash "${MESHCORE_FW_ROOT}/scripts/build-representative.sh" --baseline +fi + +echo "Verification complete."