From c933376deb4231c10b28a3a7c31b94c6ef6f912f Mon Sep 17 00:00:00 2001 From: jokob-sk Date: Sun, 24 May 2026 22:09:09 +1000 Subject: [PATCH 1/4] BE: SYNC API logging --- docs/SECURITY.md | 2 +- server/api_server/sync_endpoint.py | 100 +++++++++++++++++++++++------ server/utils/datetime_utils.py | 2 +- 3 files changed, 83 insertions(+), 21 deletions(-) diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 35612495c..1d1d9a3da 100755 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -8,7 +8,7 @@ This includes (but is not limited to): - Running NetAlertX only on networks where you have legal authorization - Keeping your deployment up to date with the latest patches -> NetAlertX is not responsible for misuse, misconfiguration, or unsecure deployments. Always test and secure your setup before exposing it to the outside world. +> NetAlertX is not responsible for misuse, misconfiguration, or insecure deployments. Always test and secure your setup before exposing it to the outside world. Users interacting with the UI are treated as trusted actors within the deployment model. Always properly secure and isolate your deployment before exposing it externally. # 🔐 Securing Your NetAlertX Instance diff --git a/server/api_server/sync_endpoint.py b/server/api_server/sync_endpoint.py index fe086a2a7..36c468402 100755 --- a/server/api_server/sync_endpoint.py +++ b/server/api_server/sync_endpoint.py @@ -47,40 +47,102 @@ def handle_sync_get(): def handle_sync_post(): """Handle POST requests for SYNC (HUB receiving from NODE).""" - body = request.get_json(silent=True) or {} + + mylog("verbose", [ + "[SYNC API] ENTER handle_sync_post", + f"method={request.method}", + f"content_type={request.content_type}", + f"content_length={request.content_length}", + f"remote_addr={request.remote_addr}" + ]) + + # ---- RAW BODY (critical for debugging encoding / encryption issues) + try: + raw = request.get_data(cache=False) + mylog("verbose", [ + f"[SYNC API] raw_bytes_len={len(raw)}", + f"[SYNC API] raw_preview={raw[:200]}" + ]) + except Exception as e: + mylog("none", [f"[SYNC API] FAILED reading raw body: {e}"]) + return jsonify({"error": "failed reading body"}), 400 + + # ---- JSON PARSE (this is a very common failure point) + try: + body = request.get_json(force=False, silent=False) + mylog("verbose", [f"[SYNC API] parsed_json={body}"]) + except Exception as e: + mylog("none", [f"[SYNC API] JSON_PARSE_FAILED={e}"]) + return jsonify({"error": "invalid json"}), 400 + + # ---- EXTRACT FIELDS data = body.get("data", "") node_name = body.get("node_name", "") plugin = body.get("plugin", "") + mylog("verbose", [ + f"[SYNC API] node_name={repr(node_name)}", + f"[SYNC API] plugin={repr(plugin)}", + f"[SYNC API] data_type={type(data).__name__}", + f"[SYNC API] data_len={len(data) if isinstance(data, str) else 'non-string'}" + ]) + storage_path = INSTALL_PATH + "/log/plugins" - os.makedirs(storage_path, exist_ok=True) - - encoded_files = [ - f - for f in os.listdir(storage_path) - if f.startswith(f"last_result.{plugin}.encoded.{node_name}") - ] - decoded_files = [ - f - for f in os.listdir(storage_path) - if f.startswith(f"last_result.{plugin}.decoded.{node_name}") - ] - file_count = len(encoded_files + decoded_files) + 1 + try: + os.makedirs(storage_path, exist_ok=True) + mylog("verbose", [f"[SYNC API] storage_path_ready={storage_path}"]) + except Exception as e: + mylog("none", [f"[SYNC API] MKDIR_FAILED={e}"]) + return jsonify({"error": "storage path error"}), 500 + + # ---- FILE COUNT LOGIC + try: + encoded_files = [ + f for f in os.listdir(storage_path) + if f.startswith(f"last_result.{plugin}.encoded.{node_name}") + ] + decoded_files = [ + f for f in os.listdir(storage_path) + if f.startswith(f"last_result.{plugin}.decoded.{node_name}") + ] + file_count = len(encoded_files + decoded_files) + 1 + + mylog("verbose", [ + f"[SYNC API] encoded_files={len(encoded_files)}", + f"[SYNC API] decoded_files={len(decoded_files)}", + f"[SYNC API] file_count={file_count}" + ]) + except Exception as e: + mylog("none", [f"[SYNC API] LISTDIR_FAILED={e}"]) + return jsonify({"error": "listdir failed"}), 500 + + # ---- FILE PATH file_path_new = os.path.join( - storage_path, f"last_result.{plugin}.encoded.{node_name}.{file_count}.log" + storage_path, + f"last_result.{plugin}.encoded.{node_name}.{file_count}.log" ) + mylog("verbose", [f"[SYNC API] file_path_new={file_path_new}"]) + + # ---- WRITE FILE (final critical point) try: + if not isinstance(data, str): + data = str(data) + with open(file_path_new, "w") as f: f.write(data) + except Exception as e: - msg = f"[Plugin: SYNC] Failed to store data: {e}" - write_notification(msg, "alert", timeNowUTC()) - mylog("verbose", [msg]) - return jsonify({"error": msg}), 500 + import traceback + mylog("none", [ + f"[SYNC API] WRITE_FAILED={e}", + traceback.format_exc() + ]) + return jsonify({"error": str(e)}), 500 msg = f"[Plugin: SYNC] Data received ({file_path_new})" write_notification(msg, "info", timeNowUTC()) mylog("verbose", [msg]) + return jsonify({"message": "Data received and stored successfully"}), 200 diff --git a/server/utils/datetime_utils.py b/server/utils/datetime_utils.py index 4366e7c06..d30238dce 100644 --- a/server/utils/datetime_utils.py +++ b/server/utils/datetime_utils.py @@ -6,7 +6,7 @@ import re import pytz from typing import Union, Optional -from zoneinfo import ZoneInfo, ZoneInfoNotFoundError +from zoneinfo import ZoneInfo import email.utils import conf # from const import * From c20891d1765e54a91c55b4f93f7f32ae9751e4be Mon Sep 17 00:00:00 2001 From: "Jokob @NetAlertX" <96159884+jokob-sk@users.noreply.github.com> Date: Sun, 24 May 2026 21:28:29 +0000 Subject: [PATCH 2/4] Refactor storage path initialization in sync endpoint to use environment variable --- server/api_server/sync_endpoint.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/api_server/sync_endpoint.py b/server/api_server/sync_endpoint.py index fe086a2a7..cfa44c8a7 100755 --- a/server/api_server/sync_endpoint.py +++ b/server/api_server/sync_endpoint.py @@ -6,8 +6,6 @@ from utils.datetime_utils import timeNowUTC from messaging.in_app import write_notification -INSTALL_PATH = os.getenv("NETALERTX_APP", "/app") - # Make sure log level is initialized correctly lggr = Logger(get_setting_value('LOG_LEVEL')) @@ -52,7 +50,7 @@ def handle_sync_post(): node_name = body.get("node_name", "") plugin = body.get("plugin", "") - storage_path = INSTALL_PATH + "/log/plugins" + storage_path = os.getenv("NETALERTX_PLUGINS_LOG", "/tmp/log/plugins") os.makedirs(storage_path, exist_ok=True) encoded_files = [ From cc5fc0caae825ac62e199a090fade162a2ba4470 Mon Sep 17 00:00:00 2001 From: "Jokob @NetAlertX" <96159884+jokob-sk@users.noreply.github.com> Date: Sun, 24 May 2026 22:25:17 +0000 Subject: [PATCH 3/4] Enhance node name extraction logic to robustly handle dots in identifiers for PUSH and PULL modes --- front/plugins/sync/sync.py | 15 ++++++++----- test/plugins/test_sync_protocol.py | 35 ++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/front/plugins/sync/sync.py b/front/plugins/sync/sync.py index c778fc727..9ed0c3862 100755 --- a/front/plugins/sync/sync.py +++ b/front/plugins/sync/sync.py @@ -182,13 +182,16 @@ def main(): # are pipe-delimited — catch and skip them via the JSONDecodeError guard below. parts = file_name.split('.') if len(parts) > 2: - # Extract node name: - # decoded/encoded: last_result.PLUGIN.decoded.NodeName.N.log → parts[3] - # pull mode: last_result.NodeName.log → parts[1] - if 'decoded' in file_name or 'encoded' in file_name: - syncHubNodeName = parts[3] + # Extract node name robustly, handling dots in plugin/node identifiers. + # PUSH (decoded/encoded): split on the '.decoded.'/.encoded.' marker; + # strip the trailing .N.log counter with rsplit from the right. + # PULL: strip the known 'last_result.' prefix and '.log' suffix. + if '.decoded.' in file_name or '.encoded.' in file_name: + _marker = '.decoded.' if '.decoded.' in file_name else '.encoded.' + _, _after = file_name.split(_marker, 1) + syncHubNodeName = _after.rsplit('.', 2)[0] else: - syncHubNodeName = parts[1] + syncHubNodeName = file_name[len('last_result.'):-len('.log')] file_path = f"{LOG_PATH}/{file_name}" diff --git a/test/plugins/test_sync_protocol.py b/test/plugins/test_sync_protocol.py index fc0a7bdba..c765394fe 100644 --- a/test/plugins/test_sync_protocol.py +++ b/test/plugins/test_sync_protocol.py @@ -71,11 +71,18 @@ def _node_name_from_filename(file_name: str) -> str: """Mirror of the node-name extraction in sync.main() (Mode 3). Real file formats produced by the system: - PUSH (post-decode): last_result.PLUGIN.decoded.NodeName.N.log → parts[3] - PULL: last_result.NodeName.log → parts[1] + PUSH (post-decode): last_result.PLUGIN.decoded.NodeName.N.log + — split on '.decoded.' marker, strip .N.log with rsplit from the right + PULL: last_result.NodeName.log + — strip 'last_result.' prefix and '.log' suffix + + Both forms handle dots anywhere in PLUGIN or NodeName. """ - parts = file_name.split(".") - return parts[3] if ("decoded" in file_name or "encoded" in file_name) else parts[1] + if '.decoded.' in file_name or '.encoded.' in file_name: + marker = '.decoded.' if '.decoded.' in file_name else '.encoded.' + _, after = file_name.split(marker, 1) + return after.rsplit('.', 2)[0] + return file_name[len('last_result.'):-len('.log')] def _should_delete_after_process(filename: str) -> bool: @@ -339,6 +346,26 @@ def test_push_decoded_different_plugins(self): assert _node_name_from_filename(fname) == "HubNode", \ f"Expected 'HubNode' from {fname}" + # --- dot-in-identifier regression (fragile parts[3] fix) --- + + def test_pull_node_name_with_dots(self): + # PULL mode: node name set to e.g. "node.home" or an IP like "192.168.1.82" + assert _node_name_from_filename("last_result.node.home.log") == "node.home" + assert _node_name_from_filename("last_result.192.168.1.82.log") == "192.168.1.82" + + def test_push_decoded_node_name_with_dots(self): + # Node name "Node.Vlan01" must survive the filename round-trip intact + assert _node_name_from_filename("last_result.ARPSCAN.decoded.Node.Vlan01.1.log") == "Node.Vlan01" + + def test_push_decoded_plugin_name_with_dots(self): + # Hypothetical plugin with a dot in its name must not shift the node index + assert _node_name_from_filename("last_result.MY.PLUGIN.decoded.NodeA.1.log") == "NodeA" + + def test_push_both_identifiers_with_dots(self): + assert _node_name_from_filename( + "last_result.A.B.decoded.x.y.z.1.log" + ) == "x.y.z" + # =========================================================================== # CurrentScan candidates filter (Mode 3 – RECEIVE) From e28ec5ca2e6fafb2a1b5977ab170a1f44f6cd2ac Mon Sep 17 00:00:00 2001 From: jokob-sk Date: Mon, 25 May 2026 09:01:17 +1000 Subject: [PATCH 4/4] BE: SYNC API --- front/plugins/sync/sync.py | 98 +++++++++++++++++++++++++++++++------- 1 file changed, 81 insertions(+), 17 deletions(-) diff --git a/front/plugins/sync/sync.py b/front/plugins/sync/sync.py index 4b34b20ac..5e6bb1fd3 100755 --- a/front/plugins/sync/sync.py +++ b/front/plugins/sync/sync.py @@ -182,13 +182,30 @@ def main(): # are pipe-delimited — catch and skip them via the JSONDecodeError guard below. parts = file_name.split('.') if len(parts) > 2: - # Extract node name: - # decoded/encoded: last_result.PLUGIN.decoded.NodeName.N.log → parts[3] - # pull mode: last_result.NodeName.log → parts[1] - if 'decoded' in file_name or 'encoded' in file_name: - syncHubNodeName = parts[3] + # PUSH artifacts: + # last_result.PLUGIN.decoded.NodeName.N.log + # last_result.PLUGIN.encoded.NodeName.N.log + # + # Require BOTH: + # 1. decoded/encoded marker + # 2. trailing "..log" shape + # + # This prevents PULL filenames like: + # last_result.office.encoded.lab.log + # from being incorrectly parsed as PUSH artifacts. + is_push_artifact = ( + ('.decoded.' in file_name or '.encoded.' in file_name) + and file_name.rsplit('.', 2)[1].isdigit() + ) + + if is_push_artifact: + _marker = '.decoded.' if '.decoded.' in file_name else '.encoded.' + _, _after = file_name.split(_marker, 1) + syncHubNodeName = _after.rsplit('.', 2)[0] else: - syncHubNodeName = parts[1] + # PULL artifact: + # last_result.NodeName.log + syncHubNodeName = file_name[len('last_result.'):-len('.log')] file_path = f"{LOG_PATH}/{file_name}" @@ -284,7 +301,6 @@ def main(): return 0 -# ------------------------------------------------------------------ # Data retrieval methods api_endpoints = [ "/sync", # New Python-based endpoint @@ -293,39 +309,87 @@ def main(): # send data to the HUB def send_data(api_token, file_content, encryption_key, file_path, node_name, pref, hub_url): - """Send encrypted data to HUB, preferring /sync endpoint and falling back to PHP version.""" + """ + Sends encrypted plugin output from NODE → HUB. + + Flow: + 1. Encrypt plugin output locally + 2. Build payload (data + metadata) + 3. Try each configured HUB endpoint in order + 4. On success (200) → stop immediately + 5. On failure → log HUB response + continue fallback + 6. If all endpoints fail → alert user + """ + + # STEP 1: Encrypt raw plugin output before transmission encrypted_data = encrypt_data(file_content, encryption_key) - mylog('verbose', [f'[{pluginName}] Sending encrypted_data: "{encrypted_data}"']) + mylog('verbose', [f"[{pluginName}] Encrypted payload prepared type={type(encrypted_data).__name__}"]) + + # STEP 2: Build request payload for HUB sync API data = { 'data': encrypted_data, 'file_path': file_path, 'plugin': pref, 'node_name': node_name } - headers = {'Authorization': f'Bearer {api_token}'} + headers = { + 'Authorization': f'Bearer {api_token}' + } + + # STEP 3: Attempt delivery to each configured endpoint for endpoint in api_endpoints: final_endpoint = hub_url + endpoint try: - response = requests.post(final_endpoint, json=data, headers=headers, timeout=5) - mylog('verbose', [f'[{pluginName}] Tried endpoint: {final_endpoint}, status: {response.status_code}']) + # STEP 4: Send request to HUB sync endpoint + response = requests.post( + final_endpoint, + json=data, + headers=headers, + timeout=5 + ) + + # STEP 5a: Success path (HUB accepted payload) if response.status_code == 200: - message = f'[{pluginName}] Data for "{file_path}" sent successfully via {final_endpoint}' + message = (f'[{pluginName}] Sync success for "{file_path}" via {final_endpoint}') mylog('verbose', [message]) write_notification(message, 'info', timeNowUTC()) return True + # STEP 5b: HUB returned error (e.g. 500, 400) + try: + response_json = response.json() + except Exception: + response_json = {} + + # Extract best available error message + error_msg = ( + response_json.get("error") or response_json.get("message") or response.text + ) + + msg = (f'[{pluginName}] HUB error on {final_endpoint} [{response.status_code}]: {error_msg}') + + mylog('none', [msg]) + write_notification(msg, 'alert', timeNowUTC()) + + mylog('verbose', [f'[{pluginName}] Endpoint attempted: {final_endpoint} status={response.status_code}']) + except requests.RequestException as e: - mylog('verbose', [f'[{pluginName}] Error calling {final_endpoint}: {e}']) + # STEP 5c: Network-level failure (timeout, DNS, etc.) + mylog('verbose', [f'[{pluginName}] Request exception calling {final_endpoint} error={type(e).__name__}: {e}']) - # If all endpoints fail - message = f'[{pluginName}] Failed to send data for "{file_path}" via all endpoints' - mylog('verbose', [message]) + # STEP 6: All endpoints failed → final fallback alert + message = ( + f'[{pluginName}] All HUB endpoints failed for "{file_path}"' + ) + + mylog('none', [message]) write_notification(message, 'alert', timeNowUTC()) + return False