Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions PLUGINS.md
Original file line number Diff line number Diff line change
Expand Up @@ -310,3 +310,34 @@ python scripts/validate_plugin.py --plugin plugins/nmap

The validation checks metadata JSON, required fields, checksums, and custom
parser imports when applicable.

### Metadata quality lint rules

Two additional lint checks help maintain high-quality plugin metadata:

1. **Missing field help text** — Each field in the `fields` array should include
a `help` string that provides a brief user-facing description of the input.
Fields without `help` text produce a lint **warning** (the plugin is still valid).

```json
// Good — has help text
{ "id": "target", "label": "Target", "type": "text", "help": "IP address or hostname to scan" }

// Bad — missing help text (lint warning)
{ "id": "target", "label": "Target", "type": "text" }
```

2. **Ambiguous category** — Each plugin's `category` must be one of the
recognized categories: `recon`, `vulnerability`, `web`, `exploit`, `network`,
`expert`, `code`, `forensics`, `utils`, `execution`, `security`, `robots`.
Unknown or misspelled categories cause a validation **error** and block
the plugin from being loaded.

```bash
# Run the linter
python scripts/validate_plugins.py
```

Existing plugins can be brought into compliance incrementally — the help
text check is a non-blocking warning, and unknown categories cause a
clear error message identifying the problem.
28 changes: 28 additions & 0 deletions backend/secuscan/plugin_validator.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,12 @@
VALID_FIELD_TYPES = {"string","integer","text", "number", "boolean", "select", "multiselect", "textarea"}
VALID_PARSER_TYPES = {"json", "text", "custom", "none"}

VALID_CATEGORIES = {
"recon", "vulnerability", "web", "exploit", "network",
"expert", "code", "forensics", "utils", "execution",
"security", "robots",
}

REQUIRED_TOP_LEVEL_FIELDS = [
"id",
"name",
Expand Down Expand Up @@ -58,6 +64,7 @@ class ValidationResult:
plugin_id: str
plugin_dir: Path
errors: list = field(default_factory=list)
warnings: list = field(default_factory=list)

@property
def valid(self) -> bool:
Expand All @@ -66,6 +73,9 @@ def valid(self) -> bool:
def add(self, path: str, message: str) -> None:
self.errors.append(ValidationError(self.plugin_id, path, message))

def add_warning(self, path: str, message: str) -> None:
self.warnings.append(ValidationError(self.plugin_id, path, message))


# ---------------------------------------------------------------------------
# Core validator
Expand Down Expand Up @@ -109,6 +119,7 @@ def validate(self) -> ValidationResult:
result = ValidationResult(plugin_id=plugin_id, plugin_dir=self.plugin_dir)

self._check_required_fields(data, result)
self._check_category(data, result)
self._check_engine(data, result)
self._check_command_template(data, result)
self._check_fields(data, result)
Expand All @@ -126,6 +137,17 @@ def _check_required_fields(self, data: dict, result: ValidationResult) -> None:
if key not in data or data[key] in (None, "", [], {}):
result.add(key, f"Required field '{key}' is missing or empty")

def _check_category(self, data: dict, result: ValidationResult) -> None:
cat = data.get("category")
if not cat:
return
if cat not in VALID_CATEGORIES:
result.add(
"category",
f"'{cat}' is not a recognized category — "
f"must be one of: {sorted(VALID_CATEGORIES)}",
)

def _check_engine(self, data: dict, result: ValidationResult) -> None:
engine = data.get("engine")
if not isinstance(engine, dict):
Expand Down Expand Up @@ -214,6 +236,12 @@ def _check_fields(self, data: dict, result: ValidationResult) -> None:
f"Field '{fid}' is type '{ftype}' and must have a non-empty 'options' list",
)

if not f.get("help"):
result.add_warning(
f"{prefix}.help",
f"Field '{fid}' is missing 'help' text — add a brief user-facing description",
)

def _check_output(self, data: dict, result: ValidationResult) -> None:
output = data.get("output")
if not isinstance(output, dict):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,16 @@
"label": "Target Host",
"type": "text",
"required": true,
"placeholder": "192.168.1.1"
"placeholder": "192.168.1.1",
"help": "IP address or hostname to ping"
},
{
"id": "count",
"label": "Packet Count",
"type": "number",
"required": false,
"default": 4
"default": 4,
"help": "Number of ICMP packets to send"
}
],
"output": {
Expand Down
71 changes: 67 additions & 4 deletions testing/backend/unit/test_plugin_validator.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
VALID_SAFETY_LEVELS,
VALID_FIELD_TYPES,
VALID_PARSER_TYPES,
VALID_CATEGORIES,
)

FIXTURES_DIR = Path(__file__).resolve().parent / "fixtures" / "plugins"
Expand All @@ -48,7 +49,7 @@ def _error_messages(result: ValidationResult) -> list[str]:

def _write_metadata(tmp_path: Path, data: dict) -> Path:
plugin_dir = tmp_path / "my_plugin"
plugin_dir.mkdir()
plugin_dir.mkdir(exist_ok=True)
(plugin_dir / "metadata.json").write_text(json.dumps(data), encoding="utf-8")
return plugin_dir

Expand All @@ -65,8 +66,8 @@ def _minimal_valid() -> dict:
"engine": {"type": "cli", "binary": "ping"},
"command_template": ["ping", "-c", "{count}", "{target}"],
"fields": [
{"id": "target", "label": "Target Host", "type": "text"},
{"id": "count", "label": "Count", "type": "number"},
{"id": "target", "label": "Target Host", "type": "text", "help": "IP address or hostname"},
{"id": "count", "label": "Count", "type": "number", "help": "Number of packets"},
],
"output": {"parser": "text"},
"safety": {"level": "safe", "requires_consent": False},
Expand Down Expand Up @@ -135,6 +136,11 @@ def test_invalid_fixture_catches_unknown_placeholder(self):
placeholder_errors = [e for e in result.errors if "Placeholder" in e.message]
assert placeholder_errors, "Expected placeholder-mismatch error"

def test_invalid_fixture_catches_missing_help_text(self):
result = validate_one_plugin(INVALID_FIXTURE)
help_warnings = [e for e in result.warnings if e.path.endswith(".help")]
assert len(help_warnings) >= 2, "Expected help text warnings for both fields"


# ===========================================================================
# Required fields
Expand Down Expand Up @@ -536,4 +542,61 @@ def test_error_display_format(self, tmp_path):
result = validate_one_plugin(plugin_dir)
err = next(e for e in result.errors if e.path == "safety.level")
display = err.display()
assert "[" in display and "safety.level" in display and "→" in display
assert "[" in display and "safety.level" in display and "→" in display


# ===========================================================================
# Metadata quality lint checks
# ===========================================================================


class TestMetadataQualityLint:
def test_missing_field_help_text_reported_as_warning(self, tmp_path):
data = _minimal_valid()
data["fields"] = [
{"id": "target", "label": "Target", "type": "text"},
]
data["command_template"] = ["ping", "{target}"]
plugin_dir = _write_metadata(tmp_path, data)
result = validate_one_plugin(plugin_dir)
help_warnings = [e for e in result.warnings if e.path == "fields[0].help"]
assert len(help_warnings) == 1
assert "help" in help_warnings[0].message

def test_field_help_text_present_no_warning(self, tmp_path):
data = _minimal_valid()
data["fields"] = [
{"id": "target", "label": "Target", "type": "text", "help": "The target IP or hostname"},
]
plugin_dir = _write_metadata(tmp_path, data)
result = validate_one_plugin(plugin_dir)
help_warnings = [e for e in result.warnings if e.path.startswith("fields[0].help")]
assert help_warnings == []

def test_invalid_category_reported(self, tmp_path):
data = _minimal_valid()
data["category"] = "unknown_category"
plugin_dir = _write_metadata(tmp_path, data)
result = validate_one_plugin(plugin_dir)
assert "category" in _error_paths(result)
cat_errors = [e for e in result.errors if e.path == "category"]
assert len(cat_errors) == 1
assert "not a recognized category" in cat_errors[0].message

def test_valid_categories_accepted(self, tmp_path):
for cat in sorted(VALID_CATEGORIES):
data = _minimal_valid()
data["category"] = cat
plugin_dir = _write_metadata(tmp_path, data)
result = validate_one_plugin(plugin_dir)
cat_errors = [e for e in result.errors if e.path == "category"]
assert cat_errors == [], f"Category '{cat}' should be valid"

def test_missing_category_is_not_flagged(self, tmp_path):
data = _minimal_valid()
del data["category"]
plugin_dir = _write_metadata(tmp_path, data)
result = validate_one_plugin(plugin_dir)
cat_errors = [e for e in result.errors if e.path == "category"]
assert len(cat_errors) == 1
assert "Required" in cat_errors[0].message
Loading