From f0760906a8026ac54eab4199f3f9541bbed59519 Mon Sep 17 00:00:00 2001 From: Suchitra Swain Date: Wed, 25 Mar 2026 22:02:48 +0100 Subject: [PATCH 1/4] Monorepo for nightops --- monorepo/README.md | 137 +++++++++++++++++++++++++++++++ monorepo/packages/nightops | 1 + pyproject.toml | 3 +- src/core/config.py | 37 +++++++++ src/remediation/policy_engine.py | 20 +++++ 5 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 monorepo/README.md create mode 120000 monorepo/packages/nightops diff --git a/monorepo/README.md b/monorepo/README.md new file mode 100644 index 0000000..5e6f257 --- /dev/null +++ b/monorepo/README.md @@ -0,0 +1,137 @@ +# Monorepo Layout (Local Workspace) + +This repo is already packaged as a Python distribution named `nightops` (see `/Users/suchitraswain/Documents/google/TheNightOps/pyproject.toml`). + +To let you use it inside a monorepo without moving your existing code, we created this local workspace folder: + +- `monorepo/packages/nightops` is a symlink to the existing `TheNightOps` project root. + +## Setup + +From the `monorepo/` directory: + +```bash +cd monorepo +python3 -m venv .venv +source .venv/bin/activate + +# Install TheNightOps as a package +pip install -e "packages/nightops[dev]" +``` + +## Using the CLI from the monorepo + +The CLI loads config from a path you pass via `--config` (or from `config/nightops.yaml` relative to your current working directory). +So, when you run from `monorepo/`, prefer passing an explicit config path: + +```bash +nightops verify --config packages/nightops/config/nightops.yaml +nightops agent run --simple --incident "pod OOMKilled" --config packages/nightops/config/nightops.yaml +``` + +Alternative: `cd packages/nightops` before running `nightops ...` so `config/nightops.yaml` resolves naturally. + +## Adding more packages later + +For additional Python packages, create new folders under `monorepo/packages//` and give each package its own `pyproject.toml`. + +--- + +## Publish `nightops` (so other repos can install it) + +From the `TheNightOps` project root (this repo): + +### Step 1: Build artifacts +```bash +cd "/Users/suchitraswain/Documents/google/TheNightOps" + +python3 -m pip install -U pip build twine +python3 -m build +``` + +This generates files in `dist/` (a `.whl` and a `.tar.gz`). + +If `python -m build` fails with an error like `Unknown license exception: 'Commons-Clause-1.0'`, make sure `pyproject.toml` uses: +`license = { file = "LICENSE" }` +(this repo has been updated accordingly). + +### Step 2: Upload to PyPI (or TestPyPI) + +For PyPI: +```bash +twine upload dist/* +``` + +For TestPyPI: +```bash +twine upload --repository testpypi dist/* +``` + +You need an API token set up in `~/.pypirc` or via `TWINE_USERNAME` / `TWINE_PASSWORD`. + +### Step 3 (before re-publishing): bump version + +Update `version = "..."` in `pyproject.toml` and then repeat the build/upload steps. + +--- + +## Install `nightops` in another project + +### Step 1: Create/activate a virtualenv +```bash +python3 -m venv .venv +source .venv/bin/activate +``` + +### Step 2: Install the published package +```bash +pip install nightops +``` + +Or pin a version: +```bash +pip install "nightops==0.1.0b1" +``` + +### Step 2b (if using TestPyPI) +```bash +pip install --index-url https://test.pypi.org/simple/ "nightops==0.1.0b1" +``` + +--- + +## Using `nightops` in the other project + +The CLI expects a config file path via `--config` (or it will look for `config/nightops.yaml` relative to your current working directory). + +### Step 1: Copy/create config in the other repo +Create something like: +```text +other-repo/ + config/ + nightops.yaml +``` + +### Step 2: Run +```bash +nightops verify --config /absolute/path/to/other-repo/config/nightops.yaml + +nightops agent run --simple \ + --incident "pod OOMKilled" \ + --config /absolute/path/to/other-repo/config/nightops.yaml +``` + +### Step 3: Optional: run watch mode +```bash +nightops agent watch --simple \ + --config /absolute/path/to/other-repo/config/nightops.yaml +``` + +--- + +## Important note about config files + +`nightops` ships default YAML files inside the installed package, so the CLI can start even if the consuming repo doesn’t provide `config/nightops.yaml`. + +In practice, you will still want to provide your own `config/nightops.yaml` in the consuming repo (so you can set your real GCP/Grafana/Slack values) and pass `--config` to override defaults. + diff --git a/monorepo/packages/nightops b/monorepo/packages/nightops new file mode 120000 index 0000000..6581736 --- /dev/null +++ b/monorepo/packages/nightops @@ -0,0 +1 @@ +../../ \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 32c8d78..4d2e79d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "nightops" version = "0.1.0b1" description = "Autonomous SRE Agent built on Google ADK and Remote MCP — with proactive detection, incident memory, and graduated remediation" readme = "README.md" -license = "Apache-2.0 WITH Commons-Clause-1.0" +license = { file = "LICENSE" } requires-python = ">=3.11" authors = [ { name = "TheNightOps Contributors" }, @@ -59,6 +59,7 @@ nightops = "nightops.cli:app" [tool.hatch.build.targets.wheel.force-include] "src" = "nightops" +"config" = "nightops/config" [tool.ruff] target-version = "py311" diff --git a/src/core/config.py b/src/core/config.py index 173949a..fe5ddb5 100644 --- a/src/core/config.py +++ b/src/core/config.py @@ -7,6 +7,8 @@ from pathlib import Path from typing import Literal, Optional +from importlib.resources import files as pkg_files + import yaml from pydantic import Field, field_validator from pydantic_settings import BaseSettings @@ -367,6 +369,32 @@ def from_yaml(cls, path: str | Path) -> NightOpsConfig: return cls(**data) + @classmethod + def from_yaml_text(cls, raw: str) -> NightOpsConfig: + """Load configuration from YAML text with environment variable substitution. + + This is used for packaged-in defaults where there is no filesystem path + alongside the YAML file (so we can't load an adjacent `config/.env`). + """ + # Substitute environment variables (${VAR_NAME} syntax) + for key, value in os.environ.items(): + raw = raw.replace(f"${{{key}}}", value) + + # Replace any remaining unresolved ${VAR} references with empty string + raw = re.sub(r"\$\{[A-Za-z_][A-Za-z0-9_]*\}", "", raw) + + data = yaml.safe_load(raw) or {} + + # Handle nested mcp_servers key if present in YAML + if "mcp_servers" in data: + mcp = data.pop("mcp_servers") + # Map YAML key cloud_logging → cloud_logging_custom to avoid conflict + if "cloud_logging" in mcp: + mcp["cloud_logging_custom"] = mcp.pop("cloud_logging") + data.update(mcp) + + return cls(**data) + @classmethod def load(cls, config_path: Optional[str | Path] = None) -> NightOpsConfig: """Load config from file or use defaults with environment variables.""" @@ -378,6 +406,15 @@ def load(cls, config_path: Optional[str | Path] = None) -> NightOpsConfig: if Path(default_path).exists(): return cls.from_yaml(default_path) + # Fall back to packaged defaults (so consuming projects don't need to ship config/) + try: + packaged = pkg_files("nightops") / "config" / "nightops.yaml" + if packaged.is_file(): + return cls.from_yaml_text(packaged.read_text()) + except Exception: + # Best-effort fallback only; remaining logic uses env vars/defaults. + pass + # Fall back to environment variables and defaults project_id = os.getenv("GCP_PROJECT_ID", "") return cls( diff --git a/src/remediation/policy_engine.py b/src/remediation/policy_engine.py index f2f458f..28d27de 100644 --- a/src/remediation/policy_engine.py +++ b/src/remediation/policy_engine.py @@ -18,6 +18,8 @@ from pathlib import Path from typing import Any, Optional +from importlib.resources import files as pkg_files + import yaml from nightops.core.models import RemediationAction, RemediationPolicy, Severity @@ -140,6 +142,24 @@ def _load_policies(self, path: str) -> None: """Load policies from a YAML file, merging with defaults.""" policy_file = Path(path) if not policy_file.exists(): + # If the consuming project doesn't ship the config file, try + # loading it from the packaged defaults. + try: + packaged = pkg_files("nightops") / path + if packaged.is_file(): + data = yaml.safe_load(packaged.read_text()) or {} + policies = data.get("policies", {}) + for action_type, policy_data in policies.items(): + self._policies[action_type] = policy_data + logger.info( + "Loaded %d remediation policies from packaged default %s", + len(policies), + path, + ) + return + except Exception: + pass + logger.info("No policy file at %s, using defaults", path) return From bcb7b32c5eeeb064cbf49c42bdfc28eb039296a3 Mon Sep 17 00:00:00 2001 From: Suchitra Swain Date: Wed, 25 Mar 2026 23:16:51 +0100 Subject: [PATCH 2/4] comment resolved --- src/core/config.py | 6 ------ src/remediation/policy_engine.py | 2 -- 2 files changed, 8 deletions(-) diff --git a/src/core/config.py b/src/core/config.py index fe5ddb5..16ceeb3 100644 --- a/src/core/config.py +++ b/src/core/config.py @@ -350,19 +350,15 @@ def from_yaml(cls, path: str | Path) -> NightOpsConfig: with open(path) as f: raw = f.read() - # Substitute environment variables (${VAR_NAME} syntax) for key, value in os.environ.items(): raw = raw.replace(f"${{{key}}}", value) - # Replace any remaining unresolved ${VAR} references with empty string raw = re.sub(r"\$\{[A-Za-z_][A-Za-z0-9_]*\}", "", raw) data = yaml.safe_load(raw) - # Handle nested mcp_servers key if present in YAML if "mcp_servers" in data: mcp = data.pop("mcp_servers") - # Map YAML key cloud_logging → cloud_logging_custom to avoid conflict if "cloud_logging" in mcp: mcp["cloud_logging_custom"] = mcp.pop("cloud_logging") data.update(mcp) @@ -406,13 +402,11 @@ def load(cls, config_path: Optional[str | Path] = None) -> NightOpsConfig: if Path(default_path).exists(): return cls.from_yaml(default_path) - # Fall back to packaged defaults (so consuming projects don't need to ship config/) try: packaged = pkg_files("nightops") / "config" / "nightops.yaml" if packaged.is_file(): return cls.from_yaml_text(packaged.read_text()) except Exception: - # Best-effort fallback only; remaining logic uses env vars/defaults. pass # Fall back to environment variables and defaults diff --git a/src/remediation/policy_engine.py b/src/remediation/policy_engine.py index 28d27de..ebce1ca 100644 --- a/src/remediation/policy_engine.py +++ b/src/remediation/policy_engine.py @@ -142,8 +142,6 @@ def _load_policies(self, path: str) -> None: """Load policies from a YAML file, merging with defaults.""" policy_file = Path(path) if not policy_file.exists(): - # If the consuming project doesn't ship the config file, try - # loading it from the packaged defaults. try: packaged = pkg_files("nightops") / path if packaged.is_file(): From cbba69cf4fc403c7f1455fd4f16b183e9faab978 Mon Sep 17 00:00:00 2001 From: Mehul Patel <11514627+nomadicmehul@users.noreply.github.com> Date: Thu, 26 Mar 2026 18:39:01 +0100 Subject: [PATCH 3/4] Update monorepo/README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- monorepo/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/monorepo/README.md b/monorepo/README.md index 5e6f257..c28d4e5 100644 --- a/monorepo/README.md +++ b/monorepo/README.md @@ -43,8 +43,7 @@ From the `TheNightOps` project root (this repo): ### Step 1: Build artifacts ```bash -cd "/Users/suchitraswain/Documents/google/TheNightOps" - +# from the TheNightOps repository root python3 -m pip install -U pip build twine python3 -m build ``` From a6f256bfb949f7608eb5d6109520b359fc7dee4e Mon Sep 17 00:00:00 2001 From: Suchitra Swain Date: Thu, 26 Mar 2026 20:43:26 +0100 Subject: [PATCH 4/4] copilot review fixed --- monorepo/README.md | 26 ++++++-- pyproject.toml | 8 +++ src/core/config.py | 100 +++++++++++++++++++------------ src/remediation/policy_engine.py | 31 +++++++++- 4 files changed, 119 insertions(+), 46 deletions(-) diff --git a/monorepo/README.md b/monorepo/README.md index c28d4e5..f1a2b01 100644 --- a/monorepo/README.md +++ b/monorepo/README.md @@ -1,10 +1,10 @@ # Monorepo Layout (Local Workspace) -This repo is already packaged as a Python distribution named `nightops` (see `/Users/suchitraswain/Documents/google/TheNightOps/pyproject.toml`). +This repo is already packaged as a Python distribution named `nightops` (see `pyproject.toml` at the **repository root**). To let you use it inside a monorepo without moving your existing code, we created this local workspace folder: -- `monorepo/packages/nightops` is a symlink to the existing `TheNightOps` project root. +- `monorepo/packages/nightops` is a symlink to the repository root (parent of `monorepo/`). ## Setup @@ -39,7 +39,12 @@ For additional Python packages, create new folders under `monorepo/packages/ dict[str, Any]: + """Normalize yaml.safe_load output: None/empty document -> {}, require a mapping.""" + if data is None: + return {} + if not isinstance(data, dict): + raise ValueError(f"YAML root must be a mapping, got {type(data).__name__}") + return data + + +def _merge_mcp_servers_block(data: dict[str, Any]) -> None: + """Inline `mcp_servers:` into flat fields; map cloud_logging -> cloud_logging_custom.""" + if "mcp_servers" not in data: + return + mcp = data.pop("mcp_servers") + if not isinstance(mcp, dict): + raise ValueError("mcp_servers must be a mapping") + if "cloud_logging" in mcp: + mcp["cloud_logging_custom"] = mcp.pop("cloud_logging") + data.update(mcp) + + +def _parse_yaml_config_text(raw: str) -> dict[str, Any]: + """Substitute `${VAR}` from the environment, parse YAML, normalize `mcp_servers`. + + Shared by `from_yaml` (after optional `config/.env` load) and `from_yaml_text` + (packaged defaults with no adjacent `.env`). + """ + for key, value in os.environ.items(): + raw = raw.replace(f"${{{key}}}", value) + raw = re.sub(r"\$\{[A-Za-z_][A-Za-z0-9_]*\}", "", raw) + data = _coerce_yaml_root(yaml.safe_load(raw)) + _merge_mcp_servers_block(data) + return data + # ── Official Google Cloud MCP Server Configs ──────────────────────── @@ -350,20 +388,7 @@ def from_yaml(cls, path: str | Path) -> NightOpsConfig: with open(path) as f: raw = f.read() - for key, value in os.environ.items(): - raw = raw.replace(f"${{{key}}}", value) - - raw = re.sub(r"\$\{[A-Za-z_][A-Za-z0-9_]*\}", "", raw) - - data = yaml.safe_load(raw) - - if "mcp_servers" in data: - mcp = data.pop("mcp_servers") - if "cloud_logging" in mcp: - mcp["cloud_logging_custom"] = mcp.pop("cloud_logging") - data.update(mcp) - - return cls(**data) + return cls(**_parse_yaml_config_text(raw)) @classmethod def from_yaml_text(cls, raw: str) -> NightOpsConfig: @@ -372,24 +397,7 @@ def from_yaml_text(cls, raw: str) -> NightOpsConfig: This is used for packaged-in defaults where there is no filesystem path alongside the YAML file (so we can't load an adjacent `config/.env`). """ - # Substitute environment variables (${VAR_NAME} syntax) - for key, value in os.environ.items(): - raw = raw.replace(f"${{{key}}}", value) - - # Replace any remaining unresolved ${VAR} references with empty string - raw = re.sub(r"\$\{[A-Za-z_][A-Za-z0-9_]*\}", "", raw) - - data = yaml.safe_load(raw) or {} - - # Handle nested mcp_servers key if present in YAML - if "mcp_servers" in data: - mcp = data.pop("mcp_servers") - # Map YAML key cloud_logging → cloud_logging_custom to avoid conflict - if "cloud_logging" in mcp: - mcp["cloud_logging_custom"] = mcp.pop("cloud_logging") - data.update(mcp) - - return cls(**data) + return cls(**_parse_yaml_config_text(raw)) @classmethod def load(cls, config_path: Optional[str | Path] = None) -> NightOpsConfig: @@ -404,10 +412,28 @@ def load(cls, config_path: Optional[str | Path] = None) -> NightOpsConfig: try: packaged = pkg_files("nightops") / "config" / "nightops.yaml" + except ModuleNotFoundError as exc: + logger.warning( + "Could not resolve packaged config nightops/config/nightops.yaml: %s", + exc, + ) + else: if packaged.is_file(): - return cls.from_yaml_text(packaged.read_text()) - except Exception: - pass + try: + return cls.from_yaml_text(packaged.read_text()) + except ( + FileNotFoundError, + IsADirectoryError, + OSError, + UnicodeDecodeError, + yaml.YAMLError, + ValueError, + ValidationError, + ) as exc: + logger.warning( + "Failed to load packaged default config nightops.yaml: %s", + exc, + ) # Fall back to environment variables and defaults project_id = os.getenv("GCP_PROJECT_ID", "") diff --git a/src/remediation/policy_engine.py b/src/remediation/policy_engine.py index ebce1ca..f845799 100644 --- a/src/remediation/policy_engine.py +++ b/src/remediation/policy_engine.py @@ -144,7 +144,17 @@ def _load_policies(self, path: str) -> None: if not policy_file.exists(): try: packaged = pkg_files("nightops") / path - if packaged.is_file(): + except ModuleNotFoundError as exc: + logger.warning( + "Could not resolve packaged remediation policies path %s: %s", + path, + exc, + ) + logger.info("No policy file at %s, using defaults", path) + return + + if packaged.is_file(): + try: data = yaml.safe_load(packaged.read_text()) or {} policies = data.get("policies", {}) for action_type, policy_data in policies.items(): @@ -155,8 +165,23 @@ def _load_policies(self, path: str) -> None: path, ) return - except Exception: - pass + except ( + ModuleNotFoundError, + FileNotFoundError, + IsADirectoryError, + OSError, + UnicodeDecodeError, + yaml.YAMLError, + ValueError, + TypeError, + AttributeError, + ) as exc: + logger.warning( + "Failed to load packaged default remediation policies from %s: %s", + path, + exc, + ) + return logger.info("No policy file at %s, using defaults", path) return