diff --git a/scripts/airplanes-diagnostics.sh b/scripts/airplanes-diagnostics.sh index 1b8c94a..7fc604a 100755 --- a/scripts/airplanes-diagnostics.sh +++ b/scripts/airplanes-diagnostics.sh @@ -77,27 +77,6 @@ log() { printf '%s level=%s %s host=%s\n' "$SCRIPT_NAME" "$level" "$*" "$WEBSITE_HOST" >&2 } -# parse_report_status RAW -# echoes one of: enabled, disabled, invalid, empty -# "empty" means the key was not set in feed.env (treated as enabled). -parse_report_status() { - local raw="$1" - if [[ -z "$raw" ]]; then - printf '%s' 'empty' - return - fi - local lower - lower="$(printf '%s' "$raw" | tr '[:upper:]' '[:lower:]')" - # strip leading/trailing whitespace - lower="${lower#"${lower%%[![:space:]]*}"}" - lower="${lower%"${lower##*[![:space:]]}"}" - case "$lower" in - true|yes|1|on) printf '%s' 'enabled' ;; - false|no|0|off) printf '%s' 'disabled' ;; - *) printf '%s' 'invalid' ;; - esac -} - # airplanes_diagnostics_read_intent_ack # echoes one of: "true", "false", or empty. # The ack file records the last diagnostics_enabled state the server @@ -502,83 +481,6 @@ build_pi_throttle_json() { printf 'null' } -# Build the feeder-attested reception block from the forwarder's readsb JSON -# (aircraft.json + stats.json under /run/airplanes-feed, written by -# airplanes-feed.sh's --write-json). Prints "null" — pruned from the payload — -# when those files are absent/unreadable, jq is missing, or the data is stale. -# aircraft.json is rewritten continuously, so a stale `now` means the forwarder -# is down; we don't report its last gasp. max_range_nmi is gated on a real, -# non-zero lat/lon (range from null island is meaningless); counts and signal -# are reported regardless of location. -build_reception_json() { - local json_dir aircraft_file stats_file now_epoch lat lon have_geo out - json_dir="$(root_path /run/airplanes-feed)" - aircraft_file="$json_dir/aircraft.json" - stats_file="$json_dir/stats.json" - [[ -r "$aircraft_file" && -r "$stats_file" ]] || { printf 'null'; return; } - command -v jq >/dev/null 2>&1 || { printf 'null'; return; } - - now_epoch="$(date +%s 2>/dev/null)" - [[ "$now_epoch" =~ ^[0-9]+$ ]] || { printf 'null'; return; } - - lat="$(feed_env_get LATITUDE 2>/dev/null || true)" - lon="$(feed_env_get LONGITUDE 2>/dev/null || true)" - # Geo gate: report range only with a valid, non-zero lat/lon. Both - # coordinates zero (0, +0, -0, 0.0, …) is the unconfigured "null island" - # default; a single zero is legitimate (equator / prime meridian). A value - # that isn't a plain decimal number is treated as no-geo — we won't trust - # range derived from a coordinate we can't parse. - have_geo=false - if [[ "$lat" =~ ^[+-]?[0-9]+(\.[0-9]+)?$ && "$lon" =~ ^[+-]?[0-9]+(\.[0-9]+)?$ ]]; then - if [[ "$lat" =~ ^[+-]?0+(\.0*)?$ && "$lon" =~ ^[+-]?0+(\.0*)?$ ]]; then - have_geo=false - else - have_geo=true - fi - fi - - # timeout: a pathological aircraft.json must never hang the diagnostics - # run. An empty file slurps to [] → $a is null → null result; a malformed - # file makes jq exit nonzero, caught by the `if` and reported as null. - # `num` coerces non-number fields to null so a wrong-typed value can't - # produce string concatenation or a phantom zero. rssi: drop readsb's - # ~-49.5 dBFS no-signal floor before averaging. - if out="$(timeout 3s jq -nc \ - --argjson now_epoch "$now_epoch" \ - --argjson have_geo "$have_geo" \ - --slurpfile ac "$aircraft_file" \ - --slurpfile st "$stats_file" \ - ' - def num: if type == "number" then . else null end; - ($ac[0]) as $a | ($st[0]) as $s - | if ($a | type) != "object" or ($s | type) != "object" then null - elif ($a.now | num) == null - or (($now_epoch - $a.now) > 120) or (($a.now - $now_epoch) > 120) then null - else - ([ $a.aircraft[]? | .rssi? | select(type == "number" and . > -49.4) ]) as $rssi - | ($s.aircraft_with_pos | num) as $wp - | ($s.aircraft_without_pos | num) as $wo - | ($s.total.max_distance | num) as $md - | { - aircraft_total: (if $wp != null and $wo != null then $wp + $wo else null end), - aircraft_with_pos: $wp, - max_range_nmi: (if $have_geo and $md != null - then (($md / 1852.0) | (. * 10 | round) / 10) - else null end), - rssi_avg_dbfs: (if ($rssi | length) > 0 - then (($rssi | add) / ($rssi | length) | (. * 10 | round) / 10) - else null end), - rssi_max_dbfs: (if ($rssi | length) > 0 then ($rssi | max) else null end) - } - | with_entries(select(.value != null)) - end - ' 2>/dev/null)" && [[ -n "$out" ]]; then - printf '%s' "$out" - else - printf 'null' - fi -} - # nullable_num VALUE — echoes the value if non-empty, otherwise "null". # Used with `jq --argjson` so missing numerics become JSON null and the # `del(.. | nulls?)` pass strips them from the payload. @@ -646,27 +548,18 @@ main() { # between enabled/disabled gets a one-shot POST (full payload or # minimal goodbye) on the next tick, retrying naturally on the # timer cadence until the server acks. - local report_status_raw + local report_status_raw toggle report_status_raw="$(feed_env_get REPORT_STATUS 2>/dev/null || true)" - # A REPORT_STATUS line the strict reader refuses (same-line comment, - # broken quoting) must not fail open to the enabled default: a key - # that is present but unreadable gets the same bad-config exit as a - # parseable-but-invalid value. Privacy toggles fail closed. - if [[ -z "$report_status_raw" ]]; then - local _rs_path - while IFS= read -r _rs_path; do - [[ -f "$_rs_path" ]] || continue - if grep -q '^[[:space:]]*REPORT_STATUS=.' "$_rs_path"; then - log error "status=bad_config key=REPORT_STATUS value=unreadable" - exit "$EXIT_BAD_CONFIG" - fi - done < <(feed_env_paths) - fi - local toggle - toggle="$(parse_report_status "$report_status_raw")" + # Privacy toggles fail closed: report_status_consent (common.sh) returns + # "invalid" both for a parseable-but-invalid value AND for a REPORT_STATUS + # line present but refused by the strict reader, so an unreadable toggle + # gets the same bad-config exit as a bad value rather than failing open to + # enabled. ${report_status_raw:-unreadable} reproduces the two historical + # log values (an empty raw means the present-but-unreadable case). + toggle="$(report_status_consent)" case "$toggle" in invalid) - log error "status=bad_config key=REPORT_STATUS value=${report_status_raw}" + log error "status=bad_config key=REPORT_STATUS value=${report_status_raw:-unreadable}" exit "$EXIT_BAD_CONFIG" ;; esac @@ -790,15 +683,6 @@ main() { # separately at ingest. ntp_sync_json="$(collect_ntp_sync 2>/dev/null || printf 'null')" - # Feeder-attested reception stats from the forwarder's readsb JSON, - # POSTed separately to /api/feeders/reception below (its own lane, not - # folded into the diagnostics payload). "null" — and the reception POST - # is skipped — until the forwarder writes JSON, or when the data is - # stale / jq is unavailable. - local reception_json - reception_json="$(build_reception_json 2>/dev/null || true)" - [[ -n "$reception_json" ]] || reception_json='null' - local feed_scripts_version os_pretty_name os_id os_version_id kernel architecture image_release feed_scripts_version="$(get_feed_scripts_version || true)" os_pretty_name="$(get_os_release_field PRETTY_NAME || true)" @@ -947,36 +831,6 @@ main() { curl_rc=$? set -e - # Feeder-attested reception stats go to their own endpoint — a separate - # lane from device-health diagnostics. Best-effort and independent: a - # failure here is logged but never changes the diagnostics result. Rides - # the same REPORT_STATUS consent (full mode only) and is sent only when the - # forwarder produced fresh data (reception_json != null). Flattened onto the - # envelope so the server's reception sanitizer reads scalars top-level. - # Skipped when the diagnostics POST hit a transport error — the same dead - # network would just stall another curl timeout for nothing. - if (( curl_rc == 0 )) \ - && [[ "$mode" == "full" && "${reception_json:-null}" != "null" ]]; then - local reception_payload reception_response reception_status - reception_payload="$(jq -nc \ - --arg ts "$ts" \ - --arg uuid "$uuid" \ - --argjson reception "$reception_json" \ - '{schema_version: 1, ts: $ts, uuid: $uuid} + $reception')" \ - || reception_payload='' - if [[ -n "$reception_payload" ]]; then - reception_response="$(new_tmp_file)" - set +e - reception_status="$(post_json_bearer "$token" '/api/feeders/reception' "$reception_payload" "$reception_response")" - set -e - if [[ "$reception_status" == 2* ]]; then - log info "status=reception_ok http=$reception_status" - else - log warn "status=reception_failed http=${reception_status:-none}" - fi - fi - fi - # Wipe the token before any further work — it's no longer needed. token='' diff --git a/scripts/airplanes-feed.sh b/scripts/airplanes-feed.sh index c7c2bef..53d9862 100755 --- a/scripts/airplanes-feed.sh +++ b/scripts/airplanes-feed.sh @@ -150,12 +150,12 @@ airplanes_write_state "$STATE_FILE" \ "target_is_default=$TARGET_IS_DEFAULT" \ "feed_bin=$FEED_BIN" || true -# Reception JSON (aircraft.json / stats.json / outline.json) for the diagnostics -# stats uploader. Written into the forwarder's own RuntimeDirectory, NOT -# /run/readsb (the image decoder's output dir — a second writer there is what +# Stats JSON (aircraft.json / stats.json / outline.json) for the airplanes-stats +# uploader (airplanes-stats.sh). Written into the forwarder's own RuntimeDirectory, +# NOT /run/readsb (the image decoder's output dir — a second writer there is what # test_image_runtime_scripts.bats guards against). The unit sets # RuntimeDirectoryPreserve=yes, so clear a previous run's outputs before exec to -# avoid serving stale reception after a crash; the `state` file has no .json +# avoid serving stale stats after a crash; the `state` file has no .json # suffix and is left intact. The managed --write-json is appended LAST so a # legacy/operator JSON_OPTIONS can't redirect output elsewhere (readsb argp is # last-wins). diff --git a/scripts/airplanes-stats.service b/scripts/airplanes-stats.service new file mode 100644 index 0000000..5166e18 --- /dev/null +++ b/scripts/airplanes-stats.service @@ -0,0 +1,20 @@ +[Unit] +Description=airplanes.live feeder stats push +Wants=network-online.target +After=network-online.target airplanes-feed.service + +[Service] +Type=oneshot +# Runs as the forwarder's user so it can read the 0640 airplanes-feed:airplanes-feed +# claim secret and the forwarder's readsb JSON under /run/airplanes-feed. It needs +# no directory of its own: it only READS /run/airplanes-feed (owned by +# airplanes-feed.service) and writes scratch files to its private /tmp. Declaring +# RuntimeDirectory=airplanes-feed would clobber the forwarder's dir, so none is set. +User=airplanes-feed +ExecStart=/usr/local/share/airplanes/airplanes-stats.sh +TimeoutStartSec=60s +SyslogIdentifier=airplanes-stats +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes diff --git a/scripts/airplanes-stats.sh b/scripts/airplanes-stats.sh new file mode 100755 index 0000000..c2deab4 --- /dev/null +++ b/scripts/airplanes-stats.sh @@ -0,0 +1,296 @@ +#!/usr/bin/env bash +# airplanes-stats.sh — forward the feeder's raw readsb JSON (aircraft.json / +# stats.json / outline.json, written by airplanes-feed.sh's --write-json) to the +# airplanes.live backend, which extracts per-feeder reception metrics. Invoked +# every ~120s by airplanes-stats.timer, independent of the diagnostics push. +# +# The server owns metric extraction (see /api/feeders/stats), so the feeder +# forwards the documents verbatim and new statistics can be added without a fleet +# update. The body is the envelope {schema_version, ts, uuid, aircraft, stats, +# outline?}, gzip-encoded. +# +# Exit codes +# 0 success, REPORT_STATUS=disabled (deliberate skip), not-yet-claimed, +# forwarder JSON absent/stale, oversize, or a transient HTTP/transport +# failure (logged; no systemd backoff) +# 64 REPORT_STATUS has an unrecognized or unreadable value — systemd marks +# the unit failed so the config error surfaces to the operator +# +# Best-effort by design: any failure short of a config error exits 0 so the +# timer keeps the unit green and simply retries on the next tick. + +set -uo pipefail + +SCRIPT_NAME="airplanes-stats" +EXIT_OK=0 +EXIT_BAD_CONFIG=64 + +# Endpoint + structural limits. The size caps mirror the server's +# /api/feeders/stats decoder: a gzip body must be <= 512 KiB compressed AND +# expand to <= 4 MiB; exceeding either yields a 413, so the client checks both +# before sending. FRESHNESS_MAX_SKEW_SEC: aircraft.json is rewritten ~1s, so a +# `now` further than this from wall-clock means the forwarder is down — we don't +# ship its last gasp. +STATS_ENDPOINT='/api/feeders/stats' +# The AIRPLANES_STATS_TEST_* overrides are test-only knobs (the bats suite sets a +# tiny cap to exercise the outline-drop fallback + oversize-skip deterministically +# without generating multi-MB payloads). The TEST prefix is deliberate so a stray +# export in a production shell is obvious; production never sets them. +MAX_RAW_BYTES="${AIRPLANES_STATS_TEST_MAX_RAW_BYTES:-$((4 * 1024 * 1024))}" +MAX_GZIP_BYTES="${AIRPLANES_STATS_TEST_MAX_GZIP_BYTES:-$((512 * 1024))}" +FRESHNESS_MAX_SKEW_SEC=120 + +INSTALL_DIR="${AIRPLANES_STATS_INSTALL_DIR:-}" + +_resolve_install_dir() { + if [[ -n "$INSTALL_DIR" ]]; then + printf '%s' "$INSTALL_DIR" + return + fi + local self_dir + self_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + printf '%s' "$self_dir" +} + +_INSTALL_DIR="$(_resolve_install_dir)" + +# Source helpers from apl-feed/ (production /usr/local/share/airplanes/apl-feed/, +# source tree feed/scripts/apl-feed/). feed-env-apply.sh sits in the sibling lib/ +# dir and is required: common.sh's feed_env_get delegates to its strict reader. +# Mirrors airplanes-diagnostics.sh's bootstrap. +for _candidate in \ + "$_INSTALL_DIR/apl-feed/common.sh" \ + "$_INSTALL_DIR/../scripts/apl-feed/common.sh"; do + if [[ -r "$_candidate" ]]; then + _COMMON_SH="$_candidate" + _HTTP_SH="$(dirname "$_candidate")/http.sh" + _FEED_ENV_APPLY_SH="$(dirname "$_candidate")/../lib/feed-env-apply.sh" + break + fi +done + +if [[ -z "${_COMMON_SH:-}" ]] || [[ ! -r "${_HTTP_SH:-}" ]] \ + || [[ ! -r "${_FEED_ENV_APPLY_SH:-}" ]]; then + printf '%s level=error status=fatal reason=helpers_missing install_dir=%s\n' \ + "$SCRIPT_NAME" "$_INSTALL_DIR" >&2 + exit "$EXIT_BAD_CONFIG" +fi + +# shellcheck source=lib/feed-env-apply.sh +source "$_FEED_ENV_APPLY_SH" +# shellcheck source=apl-feed/common.sh +source "$_COMMON_SH" +# shellcheck source=apl-feed/http.sh +source "$_HTTP_SH" + +# common.sh unconditionally sets ROOT='/' on source. Reapply the override after +# sourcing so tests / chroot smokes can re-root the script's filesystem reads. +ROOT="${AIRPLANES_STATS_ROOT:-/}" + +log() { + local level="$1"; shift + printf '%s level=%s %s host=%s\n' "$SCRIPT_NAME" "$level" "$*" "$WEBSITE_HOST" >&2 +} + +# validate_doc SRC DEST +# Write SRC compacted to DEST iff SRC parses to a JSON object; return 0 on +# success, 1 if SRC is missing/unreadable/non-object/parse-fail (e.g. caught +# mid-write by the forwarder). Each document is validated independently so one +# bad file (typically a truncated outline.json) doesn't sink the others. +# `timeout` bounds a pathological file so it can't hang the run. +validate_doc() { + local src="$1" dest="$2" + [[ -r "$src" ]] || return 1 + # Slurp the whole file and accept ONLY a single JSON object. A multi-document + # stream (a corrupt/half-rewritten readsb file can concatenate two objects) + # or a non-object is rejected, so the run skips rather than uploading the + # first fragment. `timeout` bounds a pathological file. + timeout 3s jq -sce \ + 'if length == 1 and (.[0] | type == "object") then .[0] else error("not one object") end' \ + "$src" > "$dest" 2>/dev/null +} + +# build_envelope WITH_OUTLINE OUT_FILE +# Assemble the stats envelope from the validated compact tmpfiles into OUT_FILE. +# Reads the caller's locals (ts, uuid, aircraft_tmp, stats_tmp, outline_tmp) via +# bash dynamic scope. WITH_OUTLINE=1 includes the outline doc (only if +# outline_tmp is set); 0 omits it (the size-fallback path). schema_version is +# the integer 1 the server requires. Returns jq's exit code. +build_envelope() { + local with_outline="$1" out_file="$2" + local -a args=( + --arg ts "$ts" + --arg uuid "$uuid" + --slurpfile aircraft "$aircraft_tmp" + --slurpfile stats "$stats_tmp" + ) + local filter='{schema_version: 1, ts: $ts, uuid: $uuid, aircraft: $aircraft[0], stats: $stats[0]}' + if [[ "$with_outline" == "1" && -n "$outline_tmp" ]]; then + args+=(--slurpfile outline "$outline_tmp") + filter='{schema_version: 1, ts: $ts, uuid: $uuid, aircraft: $aircraft[0], stats: $stats[0], outline: $outline[0]}' + fi + timeout 3s jq -nc "${args[@]}" "$filter" > "$out_file" 2>/dev/null +} + +main() { + # 1. Consent. report_status_consent (common.sh) fails CLOSED: an unreadable + # REPORT_STATUS line is "invalid", same bad-config exit as a bad value. + local report_status_raw toggle + report_status_raw="$(feed_env_get REPORT_STATUS 2>/dev/null || true)" + toggle="$(report_status_consent)" + case "$toggle" in + invalid) + log error "status=bad_config key=REPORT_STATUS value=${report_status_raw:-unreadable}" + exit "$EXIT_BAD_CONFIG" + ;; + disabled) + log info "status=disabled" + exit "$EXIT_OK" + ;; + esac + + # 2. Identity. Either piece missing/invalid means the feeder isn't claimed + # yet; the timer fires again in ~120s once claim has run. read_uuid / + # read_secret_file die() on a malformed file, caught by the `||` here + # (the script runs without set -e). + local uuid secret + uuid="$(read_uuid 2>/dev/null)" || { + log info "status=not_configured reason=no_uuid" + exit "$EXIT_OK" + } + if ! uuid="$(canonicalize_uuid "$uuid" 2>/dev/null)" || [[ -z "$uuid" ]]; then + log info "status=not_configured reason=bad_uuid" + exit "$EXIT_OK" + fi + secret="$(read_secret_file "$(secret_final_path)" 2>/dev/null)" || { + log info "status=not_configured reason=no_secret" + exit "$EXIT_OK" + } + if ! validate_secret "$secret"; then + log info "status=not_configured reason=bad_secret" + exit "$EXIT_OK" + fi + + # 3. jq is required to validate + assemble the envelope. Absent → skip. + command -v jq >/dev/null 2>&1 || { + log info "status=skip reason=no_jq" + exit "$EXIT_OK" + } + + # 4. Locate + validate the forwarder's documents. aircraft + stats are core + # (skip the whole run if either is missing/invalid — never POST a partial + # envelope); outline is optional (omitted if absent/invalid, e.g. a quiet + # receiver with no range ring). + local json_dir aircraft_file stats_file outline_file + json_dir="$(root_path /run/airplanes-feed)" + aircraft_file="$json_dir/aircraft.json" + stats_file="$json_dir/stats.json" + outline_file="$json_dir/outline.json" + + local aircraft_tmp stats_tmp outline_tmp outline_candidate + aircraft_tmp="$(new_tmp_file)" + stats_tmp="$(new_tmp_file)" + if ! validate_doc "$aircraft_file" "$aircraft_tmp"; then + log info "status=skip reason=aircraft_unavailable" + exit "$EXIT_OK" + fi + if ! validate_doc "$stats_file" "$stats_tmp"; then + log info "status=skip reason=stats_unavailable" + exit "$EXIT_OK" + fi + outline_tmp='' + outline_candidate="$(new_tmp_file)" + if validate_doc "$outline_file" "$outline_candidate"; then + outline_tmp="$outline_candidate" + fi + + # 5. Freshness. A stale/missing `now` in aircraft.json means the forwarder is + # down — don't ship its last gasp. + local now_val epoch + # Only a JSON *number* now is trusted — a string "now" is a malformed doc. + now_val="$(jq -r 'if (.now | type) == "number" then .now else empty end' "$aircraft_tmp" 2>/dev/null)" + if [[ -z "$now_val" || ! "$now_val" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then + log info "status=skip reason=no_now" + exit "$EXIT_OK" + fi + epoch="$(date +%s 2>/dev/null)" + if [[ ! "$epoch" =~ ^[0-9]+$ ]]; then + log info "status=skip reason=no_epoch" + exit "$EXIT_OK" + fi + if awk -v a="$now_val" -v b="$epoch" -v m="$FRESHNESS_MAX_SKEW_SEC" \ + 'BEGIN { d = a - b; if (d < 0) d = -d; exit !(d > m) }'; then + log info "status=skip reason=stale now=$now_val epoch=$epoch" + exit "$EXIT_OK" + fi + + # 6. Assemble + size-check against BOTH server caps. On a dense receiver the + # full snapshot can exceed the gzip/raw caps; drop the (largest) outline doc + # and retry, then skip if still over — keeps core stats flowing rather than + # 413ing every tick. + local ts envelope_file gzip_file raw_size gz_size with_outline + ts="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" + envelope_file="$(new_tmp_file)" + gzip_file="$(new_tmp_file)" + with_outline=0 + [[ -n "$outline_tmp" ]] && with_outline=1 + + if ! build_envelope "$with_outline" "$envelope_file" \ + || ! gzip -c "$envelope_file" > "$gzip_file" 2>/dev/null; then + log warn "status=build_failed" + exit "$EXIT_OK" + fi + raw_size="$(wc -c < "$envelope_file" 2>/dev/null || echo 0)" + gz_size="$(wc -c < "$gzip_file" 2>/dev/null || echo 0)" + + if (( raw_size > MAX_RAW_BYTES || gz_size > MAX_GZIP_BYTES )); then + if (( with_outline == 1 )); then + with_outline=0 + if ! build_envelope 0 "$envelope_file" \ + || ! gzip -c "$envelope_file" > "$gzip_file" 2>/dev/null; then + log warn "status=build_failed" + exit "$EXIT_OK" + fi + raw_size="$(wc -c < "$envelope_file" 2>/dev/null || echo 0)" + gz_size="$(wc -c < "$gzip_file" 2>/dev/null || echo 0)" + fi + if (( raw_size > MAX_RAW_BYTES || gz_size > MAX_GZIP_BYTES )); then + log warn "status=oversize raw=$raw_size gz=$gz_size" + exit "$EXIT_OK" + fi + fi + + # 7. Re-check consent right before the POST so a near-simultaneous opt-out + # is honored on this tick rather than after the upload. + local late_toggle + late_toggle="$(report_status_consent)" + if [[ "$late_toggle" != "enabled" ]]; then + log info "status=consent_changed toggle=$late_toggle" + exit "$EXIT_OK" + fi + + # 8. POST. Bearer = alv1.., carried in a 0600 curl --config + # file (not argv). Best-effort: a non-2xx / transport error is logged and the + # next tick retries; the exit stays 0 so the unit doesn't flap to failed. + local token response_file status curl_rc + token="$(apl_auth_token "$uuid" "$secret")" || { + log warn "status=token_failed" + exit "$EXIT_OK" + } + response_file="$(new_tmp_file)" + status="$(post_gzip_bearer "$token" "$STATS_ENDPOINT" "$gzip_file" "$response_file")" + curl_rc=$? + token='' + + if (( curl_rc != 0 )); then + log warn "status=transport_error curl_rc=$curl_rc" + exit "$EXIT_OK" + fi + case "$status" in + 2*) log info "status=stats_ok http=$status outline=$with_outline" ;; + *) log warn "status=stats_failed http=${status:-none}" ;; + esac + exit "$EXIT_OK" +} + +main "$@" diff --git a/scripts/airplanes-stats.timer b/scripts/airplanes-stats.timer new file mode 100644 index 0000000..5bf9b1d --- /dev/null +++ b/scripts/airplanes-stats.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Push feeder stats to airplanes.live every 2 minutes + +[Timer] +OnBootSec=3min +OnUnitActiveSec=2min +RandomizedDelaySec=45s +Unit=airplanes-stats.service +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/scripts/apl-feed/common.sh b/scripts/apl-feed/common.sh index bcfc7c8..0947b4b 100644 --- a/scripts/apl-feed/common.sh +++ b/scripts/apl-feed/common.sh @@ -410,6 +410,55 @@ feed_env_get() { printf '%s\n' "$value" } +# parse_report_status RAW +# echoes one of: enabled, disabled, invalid, empty +# "empty" means the raw value was empty (key unset on the line read). +# Treated as enabled by report_status_consent. +parse_report_status() { + local raw="$1" + if [[ -z "$raw" ]]; then + printf '%s' 'empty' + return + fi + local lower + lower="$(printf '%s' "$raw" | tr '[:upper:]' '[:lower:]')" + # strip leading/trailing whitespace + lower="${lower#"${lower%%[![:space:]]*}"}" + lower="${lower%"${lower##*[![:space:]]}"}" + case "$lower" in + true|yes|1|on) printf '%s' 'enabled' ;; + false|no|0|off) printf '%s' 'disabled' ;; + *) printf '%s' 'invalid' ;; + esac +} + +# report_status_consent +# Resolve the REPORT_STATUS privacy toggle across the effective feed.env +# files. Echoes one of: enabled | disabled | invalid. +# - unset/absent → enabled (the opt-out default) +# - present but the strict reader refuses it (same-line comment, broken +# quoting) → invalid: privacy toggles fail CLOSED, never to enabled +# - parseable value → enabled/disabled/invalid via parse_report_status +# Single-sourced here so airplanes-diagnostics.sh and airplanes-stats.sh +# apply the identical fail-closed rule — a drift between them is a privacy bug. +report_status_consent() { + local raw + raw="$(feed_env_get REPORT_STATUS 2>/dev/null || true)" + if [[ -z "$raw" ]]; then + local _rs_path + while IFS= read -r _rs_path; do + [[ -f "$_rs_path" ]] || continue + if grep -q '^[[:space:]]*REPORT_STATUS=.' "$_rs_path"; then + printf '%s' 'invalid' + return + fi + done < <(feed_env_paths) + printf '%s' 'enabled' + return + fi + parse_report_status "$raw" +} + apl_auth_token() { # Build the v1 Authorization: Bearer token shape `alv1..` # from a raw uuid + secret pair. Canonicalizes both internally so call diff --git a/scripts/apl-feed/http.sh b/scripts/apl-feed/http.sh index b2b7438..cdbea0d 100644 --- a/scripts/apl-feed/http.sh +++ b/scripts/apl-feed/http.sh @@ -77,6 +77,52 @@ post_json_bearer() { return "$rc" } +# post_gzip_bearer +# +# Like post_json_bearer, but uploads a PRE-gzipped request body read from a file +# and sets Content-Encoding: gzip so the server gunzips it. Compression and the +# size check stay in the caller (airplanes-stats.sh needs the compressed size +# for its outline-drop fallback), so this helper owns only transport. The bearer +# goes into a 0600 curl --config file (not argv), same as post_json_bearer. +# --max-time is larger than the JSON helpers' because the raw-snapshot upload is +# bigger than the sub-KiB control bodies. Response capped at 128 KiB. +# +# Returns curl's exit code; echoes the HTTP status to stdout (empty on transport +# failure). +post_gzip_bearer() { + local token="$1" + local path="$2" + local gzip_file="$3" + local response_file="$4" + + local cfg rc + cfg="$(mktemp -t apl-feed-curlcfg.XXXXXX)" || return 1 + chmod 0600 "$cfg" || { rm -f "$cfg"; return 1; } + # Same cleanup contract as post_json_bearer: explicit rm at return, with + # TMP_FILES as the EXIT-trap safety net for direct (non-$()) callers. + TMP_FILES+=("$cfg") + local escaped="${token//\\/\\\\}" + escaped="${escaped//\"/\\\"}" + if ! printf 'header = "Authorization: Bearer %s"\n' "$escaped" > "$cfg"; then + rm -f "$cfg" + return 1 + fi + + curl --silent --show-error \ + --connect-timeout 10 --max-time 60 --max-filesize 131072 \ + --request POST \ + --header 'Content-Type: application/json' \ + --header 'Content-Encoding: gzip' \ + --config "$cfg" \ + --data-binary @"$gzip_file" \ + --output "$response_file" \ + --write-out '%{http_code}' \ + "$WEBSITE_URL$path" + rc=$? + rm -f "$cfg" + return "$rc" +} + body_preview() { local file="$1" head -c 200 "$file" || true diff --git a/test/test_airplanes_diagnostics.bats b/test/test_airplanes_diagnostics.bats index a293a2b..abb013e 100644 --- a/test/test_airplanes_diagnostics.bats +++ b/test/test_airplanes_diagnostics.bats @@ -11,7 +11,6 @@ setup() { STUB_DIR="$ROOT_DIR/bin" COMMAND_LOG="$ROOT_DIR/cmd.log" BODY_LOG="$ROOT_DIR/body.log" - RECEPTION_BODY_LOG="$ROOT_DIR/reception-body.log" HEADER_LOG="$ROOT_DIR/header.log" mkdir -p "$STUB_DIR" "$ROOT_DIR/var/lib" @@ -33,19 +32,8 @@ for arg in "$@"; do fi prev="$arg" done -# Drain stdin (the JSON body) so callers can inspect what was sent. Route by -# endpoint so the diagnostics and reception POST bodies stay separable. -_recv_target='' -for arg in "$@"; do - case "$arg" in - */api/feeders/reception) _recv_target="${RECEPTION_BODY_LOG:-}" ;; - esac -done -if [[ -n "$_recv_target" ]]; then - cat > "$_recv_target" -else - cat > "$BODY_LOG" -fi +# Drain stdin (the JSON body) so callers can inspect what was sent. +cat > "$BODY_LOG" if [[ -n "$output_file" ]]; then printf '%s' "${CURL_RESPONSE:-{\"ok\":true\}}" > "$output_file" fi @@ -189,7 +177,6 @@ run_script() { APL_FEED_WEBSITE_URL="${APL_FEED_WEBSITE_URL:-http://127.0.0.1:0}" \ COMMAND_LOG="$COMMAND_LOG" \ BODY_LOG="$BODY_LOG" \ - RECEPTION_BODY_LOG="$RECEPTION_BODY_LOG" \ HEADER_LOG="$HEADER_LOG" \ CURL_STATUS="${CURL_STATUS:-200}" \ CURL_RC="${CURL_RC:-0}" \ @@ -1064,156 +1051,22 @@ SH [ ! -f "$COMMAND_LOG" ] } -# ---- reception stats (forwarder readsb JSON) ---- - -# Seed the forwarder's readsb JSON outputs under the rooted /run dir. Kept -# rssi after the no-signal floor filter is {-10,-20,-6} → avg -12, max -6. -_seed_forwarder_json() { - local now_val="$1" - local dir="$ROOT_DIR/run/airplanes-feed" - mkdir -p "$dir" - cat > "$dir/aircraft.json" < "$dir/stats.json" < "$ROOT_DIR/etc/airplanes/feed.env" - _seed_forwarder_json "$(date +%s)" - run_script - [ "$status" -eq 0 ] - # Both endpoints hit; reception travels in its own POST, not diagnostics. - grep -q -- '/api/feeders/diagnostics' "$COMMAND_LOG" - grep -q -- '/api/feeders/reception' "$COMMAND_LOG" - # Reception body is the envelope with scalars flattened to top level. - run jq -e '.schema_version == 1' "$RECEPTION_BODY_LOG" - [ "$status" -eq 0 ] - run jq -e '.aircraft_total == 10' "$RECEPTION_BODY_LOG" - [ "$status" -eq 0 ] - run jq -e '.aircraft_with_pos == 7' "$RECEPTION_BODY_LOG" - [ "$status" -eq 0 ] - # 185200 m / 1852 = 100 nmi - run jq -e '.max_range_nmi == 100' "$RECEPTION_BODY_LOG" - [ "$status" -eq 0 ] - run jq -e '.rssi_max_dbfs == -6' "$RECEPTION_BODY_LOG" - [ "$status" -eq 0 ] - # avg over kept rssi {-10,-20,-6} = -12; the -49.5 floor is excluded - run jq -e '.rssi_avg_dbfs == -12' "$RECEPTION_BODY_LOG" - [ "$status" -eq 0 ] - # Diagnostics payload no longer carries reception. - run jq -e 'has("reception") | not' "$BODY_LOG" - [ "$status" -eq 0 ] -} - -@test "reception: stale forwarder JSON (old now) → no reception POST" { - printf 'REPORT_STATUS=true\nLATITUDE=52.52\nLONGITUDE=13.405\n' > "$ROOT_DIR/etc/airplanes/feed.env" - _seed_forwarder_json "$(( $(date +%s) - 9999 ))" - run_script - [ "$status" -eq 0 ] - grep -q -- '/api/feeders/diagnostics' "$COMMAND_LOG" - if grep -q -- '/api/feeders/reception' "$COMMAND_LOG"; then return 1; fi -} +# ---- stats lane moved out of diagnostics ---- -@test "reception: absent forwarder JSON → no reception POST (diagnostics still sent)" { - # No _seed_forwarder_json — /run/airplanes-feed has no JSON files yet. +@test "diagnostics no longer posts to /api/feeders/reception" { + # The feeder-attested stats lane moved to airplanes-stats.sh (its own + # timer + the /api/feeders/stats endpoint). Diagnostics must POST only its + # device-health payload, never the retired reception endpoint — even with + # forwarder JSON present on disk. + mkdir -p "$ROOT_DIR/run/airplanes-feed" + printf '{"now": %s, "aircraft": []}\n' "$(date +%s)" \ + > "$ROOT_DIR/run/airplanes-feed/aircraft.json" + printf '{"total": {"max_distance": 0}}\n' \ + > "$ROOT_DIR/run/airplanes-feed/stats.json" run_script [ "$status" -eq 0 ] grep -q -- '/api/feeders/diagnostics' "$COMMAND_LOG" if grep -q -- '/api/feeders/reception' "$COMMAND_LOG"; then return 1; fi -} - -@test "reception: 0/0 geo → reception POST omits max_range_nmi" { - printf 'REPORT_STATUS=true\nLATITUDE=0\nLONGITUDE=0\n' > "$ROOT_DIR/etc/airplanes/feed.env" - _seed_forwarder_json "$(date +%s)" - run_script - [ "$status" -eq 0 ] - grep -q -- '/api/feeders/reception' "$COMMAND_LOG" - run jq -e '.aircraft_total == 10' "$RECEPTION_BODY_LOG" - [ "$status" -eq 0 ] - run jq -e '.rssi_max_dbfs == -6' "$RECEPTION_BODY_LOG" - [ "$status" -eq 0 ] - run jq -e 'has("max_range_nmi") | not' "$RECEPTION_BODY_LOG" - [ "$status" -eq 0 ] -} - -@test "reception: malformed forwarder JSON → no reception POST (diagnostics still sent)" { - printf 'REPORT_STATUS=true\nLATITUDE=52.52\nLONGITUDE=13.405\n' > "$ROOT_DIR/etc/airplanes/feed.env" - local dir="$ROOT_DIR/run/airplanes-feed" - mkdir -p "$dir" - printf '{ this is not json' > "$dir/aircraft.json" - printf '{"now": %s, "aircraft_with_pos": 1, "aircraft_without_pos": 0, "total": {"max_distance": 0}}\n' \ - "$(date +%s)" > "$dir/stats.json" - run_script - [ "$status" -eq 0 ] - grep -q -- '/api/feeders/diagnostics' "$COMMAND_LOG" - if grep -q -- '/api/feeders/reception' "$COMMAND_LOG"; then return 1; fi -} - -@test "reception: all RSSI at the no-signal floor → reception POST omits signal fields" { - printf 'REPORT_STATUS=true\nLATITUDE=52.52\nLONGITUDE=13.405\n' > "$ROOT_DIR/etc/airplanes/feed.env" - local dir="$ROOT_DIR/run/airplanes-feed" - mkdir -p "$dir" - cat > "$dir/aircraft.json" < "$dir/stats.json" - run_script - [ "$status" -eq 0 ] - grep -q -- '/api/feeders/reception' "$COMMAND_LOG" - run jq -e '.aircraft_total == 2' "$RECEPTION_BODY_LOG" - [ "$status" -eq 0 ] - run jq -e 'has("rssi_avg_dbfs") | not' "$RECEPTION_BODY_LOG" - [ "$status" -eq 0 ] - run jq -e 'has("rssi_max_dbfs") | not' "$RECEPTION_BODY_LOG" - [ "$status" -eq 0 ] -} - -@test "reception: missing stats count omits total; max_range rounds to 1 dp" { - printf 'REPORT_STATUS=true\nLATITUDE=52.52\nLONGITUDE=13.405\n' > "$ROOT_DIR/etc/airplanes/feed.env" - local dir="$ROOT_DIR/run/airplanes-feed" - mkdir -p "$dir" - cat > "$dir/aircraft.json" < "$dir/stats.json" - run_script - [ "$status" -eq 0 ] - grep -q -- '/api/feeders/reception' "$COMMAND_LOG" - run jq -e '.aircraft_with_pos == 5' "$RECEPTION_BODY_LOG" - [ "$status" -eq 0 ] - run jq -e 'has("aircraft_total") | not' "$RECEPTION_BODY_LOG" - [ "$status" -eq 0 ] - run jq -e '.max_range_nmi == 100.4' "$RECEPTION_BODY_LOG" - [ "$status" -eq 0 ] -} - -@test "reception: REPORT_STATUS=false → no reception POST (rides diagnostics consent)" { - printf 'REPORT_STATUS=false\n' > "$ROOT_DIR/etc/airplanes/feed.env" - _seed_forwarder_json "$(date +%s)" - run_script - [ "$status" -eq 0 ] - # Opted out of telemetry → reception is not sent even with fresh data. - if grep -q -- '/api/feeders/reception' "$COMMAND_LOG"; then return 1; fi -} - -@test "reception: skipped when the diagnostics POST has a transport error" { - printf 'REPORT_STATUS=true\nLATITUDE=52.52\nLONGITUDE=13.405\n' > "$ROOT_DIR/etc/airplanes/feed.env" - _seed_forwarder_json "$(date +%s)" - # curl exits non-zero (e.g. couldn't connect) → don't stall another POST - # against the same dead network. - CURL_RC=7 run_script + run jq -e 'has("reception") | not' "$BODY_LOG" [ "$status" -eq 0 ] - if grep -q -- '/api/feeders/reception' "$COMMAND_LOG"; then return 1; fi } diff --git a/test/test_airplanes_stats.bats b/test/test_airplanes_stats.bats new file mode 100644 index 0000000..b5cbdd5 --- /dev/null +++ b/test/test_airplanes_stats.bats @@ -0,0 +1,298 @@ +#!/usr/bin/env bats + +# Tests for scripts/airplanes-stats.sh — the feeder stats uploader invoked every +# ~120s by airplanes-stats.timer. It forwards the forwarder's raw readsb JSON +# (aircraft/stats/outline) gzip-encoded to /api/feeders/stats. + +setup() { + REPO_ROOT="$BATS_TEST_DIRNAME/.." + SCRIPT="$REPO_ROOT/scripts/airplanes-stats.sh" + + ROOT_DIR="$(mktemp -d)" + STUB_DIR="$ROOT_DIR/bin" + COMMAND_LOG="$ROOT_DIR/cmd.log" + BODY_LOG="$ROOT_DIR/body.log" + HEADER_LOG="$ROOT_DIR/header.log" + mkdir -p "$STUB_DIR" "$ROOT_DIR/etc/airplanes" "$ROOT_DIR/run/airplanes-feed" + + # curl stub: records argv, dumps the --config file (bearer header) to + # HEADER_LOG, and GUNZIPS the uploaded body (post_gzip_bearer sends + # `--data-binary @`) into BODY_LOG so tests can inspect the JSON. + # Writes a synthetic response and exits with $CURL_STATUS / $CURL_RC. + cat > "$STUB_DIR/curl" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "$COMMAND_LOG" +prev='' +output_file='' +data_file='' +for arg in "$@"; do + if [[ "$prev" == "--config" && -r "$arg" ]]; then + cat "$arg" >> "$HEADER_LOG" + fi + [[ "$prev" == "--output" ]] && output_file="$arg" + [[ "$prev" == "--data-binary" ]] && data_file="${arg#@}" + prev="$arg" +done +if [[ -n "$data_file" && -r "$data_file" ]]; then + gunzip -c "$data_file" > "$BODY_LOG" 2>/dev/null || cp "$data_file" "$BODY_LOG" +fi +if [[ -n "$output_file" ]]; then + printf '%s' "${CURL_RESPONSE:-{\"ok\":true\}}" > "$output_file" +fi +printf '%s' "${CURL_STATUS:-200}" +exit "${CURL_RC:-0}" +SH + chmod +x "$STUB_DIR/curl" + + # Claim state + consent (claimed + opted in by default). + printf '11111111-2222-3333-4444-555555555555\n' > "$ROOT_DIR/etc/airplanes/feeder-id" + chmod 0644 "$ROOT_DIR/etc/airplanes/feeder-id" + printf 'ABCDEFGHIJKLMNOP\n' > "$ROOT_DIR/etc/airplanes/feeder-claim-secret" + chmod 0640 "$ROOT_DIR/etc/airplanes/feeder-claim-secret" + printf 'REPORT_STATUS=true\n' > "$ROOT_DIR/etc/airplanes/feed.env" +} + +teardown() { + rm -rf "$ROOT_DIR" +} + +# Seed fresh, valid forwarder JSON (aircraft + stats). $1 overrides aircraft.now. +_seed_forwarder() { + local now="${1:-$(date +%s)}" + local dir="$ROOT_DIR/run/airplanes-feed" + cat > "$dir/aircraft.json" < "$dir/stats.json" <<'EOF' +{"aircraft_with_pos": 7, "aircraft_without_pos": 3, "total": {"messages": 1000, "position_count_total": 400, "tracks": {"all": 50}, "max_distance": 185200}} +EOF +} + +_seed_outline() { + cat > "$ROOT_DIR/run/airplanes-feed/outline.json" <<'EOF' +{"actualRange": {"last24h": {"points": [[52.5, 13.4], [52.6, 13.5], [52.4, 13.3]]}}} +EOF +} + +run_script() { + run env -i \ + PATH="$STUB_DIR:/usr/bin:/bin" \ + HOME="$ROOT_DIR" \ + AIRPLANES_STATS_ROOT="$ROOT_DIR" \ + APL_FEED_WEBSITE_URL="${APL_FEED_WEBSITE_URL:-http://127.0.0.1:0}" \ + AIRPLANES_STATS_TEST_MAX_RAW_BYTES="${AIRPLANES_STATS_TEST_MAX_RAW_BYTES:-}" \ + AIRPLANES_STATS_TEST_MAX_GZIP_BYTES="${AIRPLANES_STATS_TEST_MAX_GZIP_BYTES:-}" \ + COMMAND_LOG="$COMMAND_LOG" \ + BODY_LOG="$BODY_LOG" \ + HEADER_LOG="$HEADER_LOG" \ + CURL_STATUS="${CURL_STATUS:-200}" \ + CURL_RC="${CURL_RC:-0}" \ + CURL_RESPONSE="${CURL_RESPONSE:-}" \ + bash "$SCRIPT" +} + +_posted() { grep -q -- '/api/feeders/stats' "$COMMAND_LOG"; } + +# ---- happy path ---- + +@test "fresh JSON + outline → gzip POST to /api/feeders/stats with all three docs" { + _seed_forwarder + _seed_outline + run_script + [ "$status" -eq 0 ] + _posted + grep -q -- 'Content-Encoding: gzip' "$COMMAND_LOG" + run jq -e '.schema_version == 1' "$BODY_LOG"; [ "$status" -eq 0 ] + run jq -e '.uuid == "11111111-2222-3333-4444-555555555555"' "$BODY_LOG"; [ "$status" -eq 0 ] + run jq -e 'has("ts")' "$BODY_LOG"; [ "$status" -eq 0 ] + run jq -e '.aircraft.now != null' "$BODY_LOG"; [ "$status" -eq 0 ] + run jq -e '.stats.total.max_distance == 185200' "$BODY_LOG"; [ "$status" -eq 0 ] + run jq -e '.outline.actualRange.last24h.points | length == 3' "$BODY_LOG"; [ "$status" -eq 0 ] +} + +@test "bearer travels in the curl --config header, never in argv" { + _seed_forwarder + run_script + [ "$status" -eq 0 ] + grep -q 'Authorization: Bearer alv1.11111111-2222-3333-4444-555555555555.ABCDEFGHIJKLMNOP' "$HEADER_LOG" + # The secret must never appear in the recorded curl argv. + if grep -q 'ABCDEFGHIJKLMNOP' "$COMMAND_LOG"; then return 1; fi +} + +@test "outline absent → envelope omits outline, core docs still sent" { + _seed_forwarder # no _seed_outline + run_script + [ "$status" -eq 0 ] + _posted + run jq -e 'has("outline") | not' "$BODY_LOG"; [ "$status" -eq 0 ] + run jq -e '(.aircraft != null) and (.stats != null)' "$BODY_LOG"; [ "$status" -eq 0 ] +} + +@test "outline present but truncated → outline omitted, core still sent" { + _seed_forwarder + printf '{ truncated outline' > "$ROOT_DIR/run/airplanes-feed/outline.json" + run_script + [ "$status" -eq 0 ] + _posted + run jq -e 'has("outline") | not' "$BODY_LOG"; [ "$status" -eq 0 ] + run jq -e '.aircraft != null' "$BODY_LOG"; [ "$status" -eq 0 ] +} + +@test "non-object core doc (array) → no POST, exit 0" { + _seed_forwarder + printf '[1,2,3]\n' > "$ROOT_DIR/run/airplanes-feed/stats.json" + run_script + [ "$status" -eq 0 ] + if _posted; then return 1; fi +} + +@test "multi-document core file (concatenated objects) → no POST, exit 0" { + # A half-rewritten readsb file can concatenate two objects. validate_doc must + # reject the stream rather than upload only the first object. + _seed_forwarder + printf '{"a":1}{"b":2}\n' > "$ROOT_DIR/run/airplanes-feed/stats.json" + run_script + [ "$status" -eq 0 ] + if _posted; then return 1; fi +} + +# ---- freshness ---- + +@test "stale aircraft.now (old) → no POST, exit 0" { + _seed_forwarder "$(( $(date +%s) - 9999 ))" + run_script + [ "$status" -eq 0 ] + if _posted; then return 1; fi + [[ "$output" == *"reason=stale"* ]] +} + +@test "missing aircraft.now → no POST, exit 0" { + cat > "$ROOT_DIR/run/airplanes-feed/aircraft.json" <<'EOF' +{"messages": 5, "aircraft": []} +EOF + cat > "$ROOT_DIR/run/airplanes-feed/stats.json" <<'EOF' +{"total": {"max_distance": 0}} +EOF + run_script + [ "$status" -eq 0 ] + if _posted; then return 1; fi + [[ "$output" == *"reason=no_now"* ]] +} + +@test "non-numeric aircraft.now → no POST, exit 0" { + cat > "$ROOT_DIR/run/airplanes-feed/aircraft.json" <<'EOF' +{"now": "soon", "aircraft": []} +EOF + cat > "$ROOT_DIR/run/airplanes-feed/stats.json" <<'EOF' +{"total": {"max_distance": 0}} +EOF + run_script + [ "$status" -eq 0 ] + if _posted; then return 1; fi +} + +# ---- consent + identity gating ---- + +@test "REPORT_STATUS=false → no POST, exit 0" { + printf 'REPORT_STATUS=false\n' > "$ROOT_DIR/etc/airplanes/feed.env" + _seed_forwarder + run_script + [ "$status" -eq 0 ] + if _posted; then return 1; fi + [[ "$output" == *"status=disabled"* ]] +} + +@test "REPORT_STATUS garbage → exit 64, no POST" { + printf 'REPORT_STATUS=perhaps\n' > "$ROOT_DIR/etc/airplanes/feed.env" + _seed_forwarder + run_script + [ "$status" -eq 64 ] + [[ "$output" == *"status=bad_config"* ]] + [ ! -f "$COMMAND_LOG" ] +} + +@test "unclaimed (no secret) → no POST, exit 0" { + rm -f "$ROOT_DIR/etc/airplanes/feeder-claim-secret" + _seed_forwarder + run_script + [ "$status" -eq 0 ] + if _posted; then return 1; fi + [[ "$output" == *"reason=no_secret"* ]] +} + +@test "invalid Feeder ID → no POST, exit 0" { + printf 'not-a-uuid\n' > "$ROOT_DIR/etc/airplanes/feeder-id" + _seed_forwarder + run_script + [ "$status" -eq 0 ] + if _posted; then return 1; fi +} + +@test "forwarder JSON entirely absent → no POST, exit 0" { + # No _seed_forwarder — /run/airplanes-feed is empty. + run_script + [ "$status" -eq 0 ] + if _posted; then return 1; fi + [[ "$output" == *"reason=aircraft_unavailable"* ]] +} + +# ---- size policy ---- + +@test "oversize with outline → drop outline and resend core only" { + _seed_forwarder + # A large outline pushes the raw envelope over the test cap; the core-only + # envelope is well under it, so the uploader drops outline and still POSTs. + python3 - "$ROOT_DIR/run/airplanes-feed/outline.json" <<'PY' +import json, sys +pts = [[round(52 + i*1e-4, 4), round(13 + i*1e-4, 4)] for i in range(400)] +json.dump({"actualRange": {"last24h": {"points": pts}}}, open(sys.argv[1], "w")) +PY + AIRPLANES_STATS_TEST_MAX_RAW_BYTES=2000 run_script + [ "$status" -eq 0 ] + _posted + run jq -e 'has("outline") | not' "$BODY_LOG"; [ "$status" -eq 0 ] + run jq -e '.aircraft != null' "$BODY_LOG"; [ "$status" -eq 0 ] +} + +@test "oversize even without outline → skip, no POST, exit 0" { + _seed_forwarder + AIRPLANES_STATS_TEST_MAX_RAW_BYTES=1 run_script + [ "$status" -eq 0 ] + if _posted; then return 1; fi + [[ "$output" == *"status=oversize"* ]] +} + +@test "gzip cap exceeded with no outline to drop → skip, no POST, exit 0" { + _seed_forwarder + AIRPLANES_STATS_TEST_MAX_GZIP_BYTES=1 run_script + [ "$status" -eq 0 ] + if _posted; then return 1; fi + [[ "$output" == *"status=oversize"* ]] +} + +# ---- POST outcomes (best-effort) ---- + +@test "server returns 500 → exit 0, logs stats_failed" { + _seed_forwarder + CURL_STATUS=500 run_script + [ "$status" -eq 0 ] + _posted + [[ "$output" == *"status=stats_failed"* ]] +} + +@test "transport error (curl rc != 0) → exit 0, logs transport_error" { + _seed_forwarder + CURL_RC=7 run_script + [ "$status" -eq 0 ] + [[ "$output" == *"status=transport_error"* ]] +} + +@test "AIRPLANES_STATS_ROOT honoured: reads the rooted /run path" { + # Forwarder JSON under the rooted path is found; an absent rooted path means + # no data (already covered) — this asserts the root override is actually used + # by writing only under ROOT and expecting a successful POST. + _seed_forwarder + run_script + [ "$status" -eq 0 ] + _posted +} diff --git a/test/test_apl_feed_common.bats b/test/test_apl_feed_common.bats index ee8205d..870db44 100644 --- a/test/test_apl_feed_common.bats +++ b/test/test_apl_feed_common.bats @@ -854,3 +854,61 @@ STUB _set_website_host [ "$WEBSITE_HOST" = "invalid" ] } + +# --- parse_report_status --- + +@test "parse_report_status: truthy spellings → enabled" { + for v in true TRUE yes Yes 1 on ON; do + run parse_report_status "$v" + [ "$output" = enabled ] || { echo "got '$output' for '$v'"; return 1; } + done +} + +@test "parse_report_status: falsey spellings → disabled" { + for v in false FALSE no No 0 off OFF; do + run parse_report_status "$v" + [ "$output" = disabled ] || { echo "got '$output' for '$v'"; return 1; } + done +} + +@test "parse_report_status: empty → empty, junk → invalid" { + run parse_report_status "" + [ "$output" = empty ] + run parse_report_status "maybe" + [ "$output" = invalid ] +} + +# --- report_status_consent (shared by diagnostics + stats) --- + +@test "report_status_consent: REPORT_STATUS unset → enabled (opt-out default)" { + printf 'LATITUDE=1\n' > "$ROOT_DIR/etc/airplanes/feed.env" + run report_status_consent + [ "$output" = enabled ] +} + +@test "report_status_consent: REPORT_STATUS=true → enabled" { + printf 'REPORT_STATUS=true\n' > "$ROOT_DIR/etc/airplanes/feed.env" + run report_status_consent + [ "$output" = enabled ] +} + +@test "report_status_consent: REPORT_STATUS=false → disabled" { + printf 'REPORT_STATUS=false\n' > "$ROOT_DIR/etc/airplanes/feed.env" + run report_status_consent + [ "$output" = disabled ] +} + +@test "report_status_consent: garbage value → invalid" { + printf 'REPORT_STATUS=perhaps\n' > "$ROOT_DIR/etc/airplanes/feed.env" + run report_status_consent + [ "$output" = invalid ] +} + +@test "report_status_consent: present but strict-reader-refused → invalid (fail closed)" { + # A same-line comment makes the strict reader drop the key (feed_env_get + # returns empty), but the key IS present on the line → must fail CLOSED to + # invalid, never to the enabled default. + printf 'REPORT_STATUS=true # inline comment\n' > "$ROOT_DIR/etc/airplanes/feed.env" + run report_status_consent + [ "$output" = invalid ] +} diff --git a/test/test_apl_feed_http.bats b/test/test_apl_feed_http.bats index b20a5b3..f99dfce 100644 --- a/test/test_apl_feed_http.bats +++ b/test/test_apl_feed_http.bats @@ -57,7 +57,11 @@ class H(http.server.BaseHTTPRequestHandler): def do_POST(self): raw = self.rfile.read(int(self.headers.get("Content-Length", 0))) with open(req_log, "a") as f: - f.write(f"POST {self.path} ct={self.headers.get('Content-Type','')}\n") + f.write( + f"POST {self.path} ct={self.headers.get('Content-Type','')} " + f"ce={self.headers.get('Content-Encoding','')} " + f"auth={self.headers.get('Authorization','')}\n" + ) with open(req_body_file, "wb") as f: f.write(raw) self.send_response(status) @@ -105,6 +109,45 @@ mock_url() { grep -q 'POST /api/feeders/status ct=application/json' "$MOCK_REQUEST_LOG" } +# --- post_gzip_bearer --- + +@test "post_gzip_bearer: Content-Encoding gzip, body gunzips back to the input" { + start_mock_server 200 '{"ok":true}' + WEBSITE_URL="$(mock_url)" + response_file="$TMPDIR/resp" + gz="$TMPDIR/body.gz" + body='{"schema_version":1,"stats":{"aircraft_with_pos":7}}' + printf '%s' "$body" | gzip -c > "$gz" + run post_gzip_bearer 'alv1.11111111-2222-3333-4444-555555555555.ABCDEFGHIJKLMNOP' \ + '/api/feeders/stats' "$gz" "$response_file" + [ "$status" -eq 0 ] + [ "$output" = '200' ] + grep -q 'POST /api/feeders/stats ct=application/json ce=gzip' "$MOCK_REQUEST_LOG" + # The server received the gzip bytes verbatim; they expand to the input. + [ "$(gunzip -c "$MOCK_REQUEST_BODY")" = "$body" ] +} + +@test "post_gzip_bearer: bearer travels in the Authorization header (curl --config)" { + start_mock_server 200 '{"ok":true}' + WEBSITE_URL="$(mock_url)" + gz="$TMPDIR/body.gz" + printf '{}' | gzip -c > "$gz" + post_gzip_bearer 'alv1.11111111-2222-3333-4444-555555555555.ABCDEFGHIJKLMNOP' \ + '/api/feeders/stats' "$gz" "$TMPDIR/resp" >/dev/null + grep -q 'auth=Bearer alv1.11111111-2222-3333-4444-555555555555.ABCDEFGHIJKLMNOP' \ + "$MOCK_REQUEST_LOG" +} + +@test "post_gzip_bearer: removes its curl-config tempfile" { + start_mock_server 200 '{}' + WEBSITE_URL="$(mock_url)" + gz="$TMPDIR/body.gz" + printf '{}' | gzip -c > "$gz" + post_gzip_bearer 'alv1.11111111-2222-3333-4444-555555555555.ABCDEFGHIJKLMNOP' \ + '/api/feeders/stats' "$gz" "$TMPDIR/resp" >/dev/null || true + [ -z "$(find "$TMPDIR" -name 'apl-feed-curlcfg.*' 2>/dev/null)" ] +} + # --- body_preview --- @test "body_preview: caps at 200 chars" { diff --git a/test/test_image_runtime_scripts.bats b/test/test_image_runtime_scripts.bats index 3b089bf..6ecce82 100644 --- a/test/test_image_runtime_scripts.bats +++ b/test/test_image_runtime_scripts.bats @@ -130,8 +130,8 @@ SH if grep -q -- '--forward-mlat' "$arg_log"; then return 1 fi - # --write-json now feeds the reception-stats uploader. It must target the - # forwarder's own RuntimeDirectory, never the image decoder's /run/readsb, + # --write-json feeds the stats uploader (airplanes-stats.sh). It must target + # the forwarder's own RuntimeDirectory, never the image decoder's /run/readsb, # and must not pull in globe-index shards. grep -q -- "--write-json $root/run/airplanes-feed" "$arg_log" if grep -q -- '/run/readsb' "$arg_log"; then @@ -148,12 +148,12 @@ SH fi } -@test "airplanes-feed.sh writes reception JSON on a manual (non-image) install" { +@test "airplanes-feed.sh writes stats JSON on a manual (non-image) install" { local root="$ROOT_DIR/root" local arg_log="$ROOT_DIR/args-manual.log" # No image feed binary and no image-install marker → IMAGE_INSTALL=0, the # forwarder sources /etc/airplanes/feed.env and uses the built feed-airplanes - # binary. The reception JSON contract must be identical to the image branch. + # binary. The stats JSON contract must be identical to the image branch. mkdir -p "$root/etc/airplanes" "$root/usr/local/share/airplanes" cat > "$root/etc/airplanes/feed.env" <<'EOF' LATITUDE="52.52" diff --git a/test/test_install_uninstall_symmetry.bats b/test/test_install_uninstall_symmetry.bats index 4cd660f..d8800a3 100644 --- a/test/test_install_uninstall_symmetry.bats +++ b/test/test_install_uninstall_symmetry.bats @@ -121,6 +121,8 @@ stage_install_footprint() { : > "$ROOT_DIR/etc/systemd/system/airplanes-mlat.service" : > "$ROOT_DIR/etc/systemd/system/airplanes-diagnostics.service" : > "$ROOT_DIR/etc/systemd/system/airplanes-diagnostics.timer" + : > "$ROOT_DIR/etc/systemd/system/airplanes-stats.service" + : > "$ROOT_DIR/etc/systemd/system/airplanes-stats.timer" : > "$ROOT_DIR/etc/airplanes/image-install" else mkdir -p "$ROOT_DIR/lib/systemd/system" @@ -128,6 +130,8 @@ stage_install_footprint() { : > "$ROOT_DIR/lib/systemd/system/airplanes-mlat.service" : > "$ROOT_DIR/lib/systemd/system/airplanes-diagnostics.service" : > "$ROOT_DIR/lib/systemd/system/airplanes-diagnostics.timer" + : > "$ROOT_DIR/lib/systemd/system/airplanes-stats.service" + : > "$ROOT_DIR/lib/systemd/system/airplanes-stats.timer" fi # Diagnostics state directory — systemd's StateDirectory=airplanes-diagnostics @@ -172,6 +176,8 @@ run_uninstall() { [ ! -e "$ROOT_DIR/lib/systemd/system/airplanes-mlat.service" ] [ ! -e "$ROOT_DIR/lib/systemd/system/airplanes-diagnostics.service" ] [ ! -e "$ROOT_DIR/lib/systemd/system/airplanes-diagnostics.timer" ] + [ ! -e "$ROOT_DIR/lib/systemd/system/airplanes-stats.service" ] + [ ! -e "$ROOT_DIR/lib/systemd/system/airplanes-stats.timer" ] } @test "after install footprint, uninstall removes /var/lib/airplanes-diagnostics state dir" { @@ -300,6 +306,8 @@ run_uninstall() { [ ! -e "$ROOT_DIR/etc/systemd/system/airplanes-mlat.service" ] [ ! -e "$ROOT_DIR/etc/systemd/system/airplanes-diagnostics.service" ] [ ! -e "$ROOT_DIR/etc/systemd/system/airplanes-diagnostics.timer" ] + [ ! -e "$ROOT_DIR/etc/systemd/system/airplanes-stats.service" ] + [ ! -e "$ROOT_DIR/etc/systemd/system/airplanes-stats.timer" ] [ ! -e "$ROOT_DIR/etc/airplanes/image-install" ] } diff --git a/test/test_uninstall_script.bats b/test/test_uninstall_script.bats index adcad0a..5b5aaca 100644 --- a/test/test_uninstall_script.bats +++ b/test/test_uninstall_script.bats @@ -57,30 +57,31 @@ SH [ -f "$SYSTEMCTL_LOG" ] } -@test "uninstall.sh disables services in mlat -> mlat2 -> feed -> diagnostics -> config-sync order, then daemon-reloads, exactly eight calls" { +@test "uninstall.sh disables services in mlat -> mlat2 -> feed -> diagnostics -> stats -> config-sync order, then daemon-reloads, exactly ten calls" { run_uninstall [ "$status" -eq 0 ] - local first second third fourth fifth sixth seventh eighth count - first="$(sed -n '1p' "$SYSTEMCTL_LOG")" - second="$(sed -n '2p' "$SYSTEMCTL_LOG")" - third="$(sed -n '3p' "$SYSTEMCTL_LOG")" - fourth="$(sed -n '4p' "$SYSTEMCTL_LOG")" - fifth="$(sed -n '5p' "$SYSTEMCTL_LOG")" - sixth="$(sed -n '6p' "$SYSTEMCTL_LOG")" - seventh="$(sed -n '7p' "$SYSTEMCTL_LOG")" - eighth="$(sed -n '8p' "$SYSTEMCTL_LOG")" + local expected=( + "disable --now airplanes-mlat" + "disable --now airplanes-mlat2" + "disable --now airplanes-feed" + "disable --now airplanes-diagnostics.timer" + "disable --now airplanes-diagnostics.service" + "disable --now airplanes-stats.timer" + "disable --now airplanes-stats.service" + "disable --now airplanes-config-sync.timer" + "disable --now airplanes-config-sync.service" + "daemon-reload" + ) + local i line + for i in "${!expected[@]}"; do + line="$(sed -n "$((i + 1))p" "$SYSTEMCTL_LOG")" + [ "$line" = "${expected[$i]}" ] || { echo "line $((i+1)): '$line' != '${expected[$i]}'"; return 1; } + done + local count count="$(wc -l < "$SYSTEMCTL_LOG" | tr -d '[:space:]')" - [ "$first" = "disable --now airplanes-mlat" ] - [ "$second" = "disable --now airplanes-mlat2" ] - [ "$third" = "disable --now airplanes-feed" ] - [ "$fourth" = "disable --now airplanes-diagnostics.timer" ] - [ "$fifth" = "disable --now airplanes-diagnostics.service" ] - [ "$sixth" = "disable --now airplanes-config-sync.timer" ] - [ "$seventh" = "disable --now airplanes-config-sync.service" ] - [ "$eighth" = "daemon-reload" ] - [ "$count" = "8" ] + [ "$count" = "${#expected[@]}" ] } @test "uninstall.sh removes the airplanes systemd unit files and leaves others alone" { diff --git a/test/test_update_script.bats b/test/test_update_script.bats index d24c8f1..ef18f18 100644 --- a/test/test_update_script.bats +++ b/test/test_update_script.bats @@ -512,6 +512,10 @@ SH [ -f "$root/etc/airplanes/image-install" ] grep -q 'systemctl enable airplanes-feed' "$ROOT_DIR/commands.log" grep -q 'systemctl enable airplanes-mlat' "$ROOT_DIR/commands.log" + # Stats uploader unit is copied and its timer enabled (build-mode record). + [ -f "$root/etc/systemd/system/airplanes-stats.service" ] + [ -f "$root/etc/systemd/system/airplanes-stats.timer" ] + grep -q 'systemctl enable airplanes-stats.timer' "$ROOT_DIR/commands.log" ! grep -q 'systemctl restart' "$ROOT_DIR/commands.log" ! grep -q 'systemctl stop' "$ROOT_DIR/commands.log" ! grep -q 'systemctl is-active' "$ROOT_DIR/commands.log" diff --git a/test/test_wire_endpoints.bats b/test/test_wire_endpoints.bats index dda1324..87d082c 100644 --- a/test/test_wire_endpoints.bats +++ b/test/test_wire_endpoints.bats @@ -57,3 +57,13 @@ setup() { # key. We match the JSON-key shape `"current_secret":` only.) [ "$(grep -cE '"current_secret":' "$REPO_ROOT/scripts/apl-feed/claim.sh")" -eq 0 ] } + +@test "stats uploader posts to /api/feeders/stats" { + grep -q -- "/api/feeders/stats" "$REPO_ROOT/scripts/airplanes-stats.sh" +} + +@test "no script posts to the retired /api/feeders/reception endpoint" { + # The scalar reception lane was replaced by the raw-snapshot /stats lane. + # A stray /reception literal would silently 404/410 on every feeder tick. + [ "$(grep -rlF -- '/api/feeders/reception' "$REPO_ROOT/scripts" | wc -l)" -eq 0 ] +} diff --git a/uninstall.sh b/uninstall.sh index 6bba255..192c2fb 100755 --- a/uninstall.sh +++ b/uninstall.sh @@ -33,6 +33,8 @@ systemctl disable --now airplanes-mlat2 &>/dev/null systemctl disable --now airplanes-feed systemctl disable --now airplanes-diagnostics.timer &>/dev/null systemctl disable --now airplanes-diagnostics.service &>/dev/null +systemctl disable --now airplanes-stats.timer &>/dev/null +systemctl disable --now airplanes-stats.service &>/dev/null systemctl disable --now airplanes-config-sync.timer &>/dev/null systemctl disable --now airplanes-config-sync.service &>/dev/null @@ -49,6 +51,8 @@ for _systemd_dir in "${SYSTEMD_UNIT_DIRS[@]}"; do rm -f "$_systemd_dir/airplanes-feed.service" rm -f "$_systemd_dir/airplanes-diagnostics.service" rm -f "$_systemd_dir/airplanes-diagnostics.timer" + rm -f "$_systemd_dir/airplanes-stats.service" + rm -f "$_systemd_dir/airplanes-stats.timer" rm -f "$_systemd_dir/airplanes-config-sync.service" rm -f "$_systemd_dir/airplanes-config-sync.timer" done @@ -64,6 +68,7 @@ rm -f "$SYSTEMD_ETC/default.target.wants/airplanes-mlat.service" rm -f "$SYSTEMD_ETC/default.target.wants/airplanes-mlat2.service" rm -f "$SYSTEMD_ETC/multi-user.target.wants/airplanes-mlat2.service" rm -f "$SYSTEMD_ETC/timers.target.wants/airplanes-diagnostics.timer" +rm -f "$SYSTEMD_ETC/timers.target.wants/airplanes-stats.timer" rm -f "$SYSTEMD_ETC/timers.target.wants/airplanes-config-sync.timer" systemctl daemon-reload || true diff --git a/update.sh b/update.sh index a31af92..e4af596 100755 --- a/update.sh +++ b/update.sh @@ -563,6 +563,7 @@ historical_top_level_scripts=( airplanes-feed.sh airplanes-mlat.sh airplanes-diagnostics.sh + airplanes-stats.sh apl-feed.sh second-mlat.sh ) @@ -652,6 +653,8 @@ cp "$GIT"/scripts/airplanes-mlat.service "$SYSTEMD_DIR" cp "$GIT"/scripts/airplanes-feed.service "$SYSTEMD_DIR" cp "$GIT"/scripts/airplanes-diagnostics.service "$SYSTEMD_DIR" cp "$GIT"/scripts/airplanes-diagnostics.timer "$SYSTEMD_DIR" +cp "$GIT"/scripts/airplanes-stats.service "$SYSTEMD_DIR" +cp "$GIT"/scripts/airplanes-stats.timer "$SYSTEMD_DIR" cp "$GIT"/scripts/airplanes-config-sync.service "$SYSTEMD_DIR" cp "$GIT"/scripts/airplanes-config-sync.timer "$SYSTEMD_DIR" @@ -780,6 +783,22 @@ else echo "airplanes-diagnostics.timer could not be enabled; diagnostics push will not run until 'systemctl enable --now airplanes-diagnostics.timer'." fi +# Stats push: enable the timer so airplanes-stats.sh forwards the forwarder's +# readsb JSON every ~120s once a claim secret is present. Independent of the +# diagnostics timer (different cadence, separate lane). The uploader self-gates +# on REPORT_STATUS and exits silently when unclaimed or when the forwarder JSON +# isn't ready, so wiring the timer here is safe on a fresh box. +if airplanes_is_build_mode; then + # In build mode there's no live systemd; just record the enable so the + # baked rootfs comes up with the timer wired into timers.target. + systemctl enable airplanes-stats.timer >> "$LOGFILE" || true +elif is_unit_masked airplanes-stats.timer; then + echo "airplanes-stats.timer is masked; skipping enable." +else + systemctl enable --now airplanes-stats.timer >> "$LOGFILE" 2>&1 || \ + echo "airplanes-stats.timer could not be enabled; stats push will not run until 'systemctl enable --now airplanes-stats.timer'." +fi + # Remote-config sync: enable the timer so apl-feed config sync fires every # ~60s once a claim secret is present. The service is gated on # ConditionPathExists=/etc/airplanes/feeder-claim-secret so a freshly-