From ebb374a2dec90d8334114606ae67a78bee3cedd5 Mon Sep 17 00:00:00 2001
From: pateldha <pateldha@us.ibm.com>
Date: Tue, 16 Jun 2026 18:36:26 -0400
Subject: [PATCH 1/9] revised code

Signed-off-by: Dhaval Patel <pateldha@us.ibm.com>
---
 .github/workflows/.pre-commit-config.yaml | 31 +++++++++++++++
 .github/workflows/secret-scan.yml         | 47 +++++++++++++++++++++++
 .gitleaks.toml                            | 24 ++++++++++++
 uv.lock                                   |  4 +-
 4 files changed, 105 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/.pre-commit-config.yaml
 create mode 100644 .github/workflows/secret-scan.yml
 create mode 100644 .gitleaks.toml

diff --git a/.github/workflows/.pre-commit-config.yaml b/.github/workflows/.pre-commit-config.yaml
new file mode 100644
index 000000000..223d9739d
--- /dev/null
+++ b/.github/workflows/.pre-commit-config.yaml
@@ -0,0 +1,31 @@
+# >>> Rename this file to ".pre-commit-config.yaml" at the repo root <<<
+#
+# Pre-commit hooks to block secrets before they are committed.
+# Install once per clone:  pip install pre-commit && pre-commit install
+# Run against all files:    pre-commit run --all-files
+#
+# Two complementary scanners are used:
+#   - gitleaks      : fast, regex + entropy based, large rule set
+#   - detect-secrets: IBM-maintained, baseline-driven (won't re-flag known/allowed values)
+
+repos:
+  - repo: https://github.com/gitleaks/gitleaks
+    rev: v8.21.2
+    hooks:
+      - id: gitleaks
+
+  - repo: https://github.com/Yelp/detect-secrets
+    rev: v1.5.0
+    hooks:
+      - id: detect-secrets
+        args: ['--baseline', '.secrets.baseline']
+        # exclude lockfiles / generated assets from scanning if needed:
+        # exclude: package.lock.json
+
+  # Optional general hygiene hooks (uncomment to enable)
+  # - repo: https://github.com/pre-commit/pre-commit-hooks
+  #   rev: v5.0.0
+  #   hooks:
+  #     - id: check-added-large-files
+  #     - id: end-of-file-fixer
+  #     - id: trailing-whitespace
diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml
new file mode 100644
index 000000000..09d43df7b
--- /dev/null
+++ b/.github/workflows/secret-scan.yml
@@ -0,0 +1,47 @@
+# Server-side secret scanning. Runs on every push and pull request and on a
+# weekly schedule. Cannot be bypassed by contributors who skip local hooks.
+# Place this file at: .github/workflows/secret-scan.yml
+name: Secret Scan
+
+on:
+  push:
+    branches: ['**']
+  pull_request:
+    branches: ['**']
+  schedule:
+    - cron: '0 6 * * 1'   # weekly full-history sweep, Mondays 06:00 UTC
+
+permissions:
+  contents: read
+
+jobs:
+  gitleaks:
+    name: Gitleaks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # full history so the scan covers all commits in a PR, not just the tip
+          fetch-depth: 0
+
+      - name: Run gitleaks
+        uses: gitleaks/gitleaks-action@v2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # If using the free OSS action no license is needed for public repos.
+          # For org-owned private repos, set GITLEAKS_LICENSE in repo secrets:
+          # GITLEAKS_LICENSE: ${{ secrets.GITLEAKS_LICENSE }}
+
+  # Second, independent scanner for defense in depth.
+  trufflehog:
+    name: TruffleHog
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Run TruffleHog
+        uses: trufflesecurity/trufflehog@main
+        with:
+          extra_args: --results=verified,unknown
diff --git a/.gitleaks.toml b/.gitleaks.toml
new file mode 100644
index 000000000..3ebee76db
--- /dev/null
+++ b/.gitleaks.toml
@@ -0,0 +1,24 @@
+# Gitleaks configuration. Extends the built-in default rule set and adds
+# project-specific allowlisting so test fixtures / sample values don't cause
+# false positives. Place at repo root as .gitleaks.toml
+[extend]
+useDefault = true
+
+[allowlist]
+description = "Global allowlist for AssetOpsBench"
+# Files/paths that legitimately contain dummy or example secrets.
+paths = [
+  '''(.*?)(test|tests|fixtures|examples|docs)(/|\\).*''',
+  '''.*\.md$''',
+]
+# Known-safe placeholder values (regex). Add real false positives here.
+regexes = [
+  '''(?i)(your[_-]?api[_-]?key|example|dummy|placeholder|changeme|xxxx+)''',
+]
+
+# Example of an extra custom rule (uncomment / adapt as needed):
+# [[rules]]
+# id = "ibm-cloud-api-key"
+# description = "IBM Cloud IAM API key"
+# regex = '''(?i)(ibm)?[_-]?api[_-]?key['"\s:=]+[A-Za-z0-9_\-]{44}'''
+# keywords = ["api_key", "apikey"]
diff --git a/uv.lock b/uv.lock
index b0009abdf..82310d039 100644
--- a/uv.lock
+++ b/uv.lock
@@ -181,6 +181,7 @@ dependencies = [
     { name = "litellm" },
     { name = "mcp", extra = ["cli"] },
     { name = "numpy" },
+    { name = "openai" },
     { name = "openai-agents" },
     { name = "pandas" },
     { name = "pendulum" },
@@ -222,6 +223,7 @@ requires-dist = [
     { name = "litellm", specifier = "==1.81.13" },
     { name = "mcp", extras = ["cli"], specifier = ">=1.26.0" },
     { name = "numpy", specifier = ">=1.24" },
+    { name = "openai", specifier = ">=1.40.0" },
     { name = "openai-agents", specifier = ">=0.0.7" },
     { name = "pandas", specifier = ">=2.0" },
     { name = "pendulum", specifier = ">=3.2.0" },
@@ -230,7 +232,7 @@ requires-dist = [
     { name = "pyyaml", specifier = ">=6.0" },
     { name = "requests", specifier = ">=2.32.5" },
     { name = "scipy", specifier = ">=1.10.0" },
-    { name = "stirrup", extras = ["mcp", "litellm", "docker"], specifier = ">=0.1.7" },
+    { name = "stirrup", extras = ["docker", "litellm", "mcp"], specifier = ">=0.1.7" },
 ]
 
 [package.metadata.requires-dev]

From 12a1aef9455e4bfe7ae4371306e2095c31737f02 Mon Sep 17 00:00:00 2001
From: pateldha <pateldha@us.ibm.com>
Date: Tue, 16 Jun 2026 19:10:25 -0400
Subject: [PATCH 2/9] Done

Signed-off-by: Dhaval Patel <pateldha@us.ibm.com>
---
 .gitleaks.toml                                |   2 +-
 ...mit-config.yaml => .pre-commit-config.yaml |   0
 .secrets.baseline                             | 173 ++++++++++++++++++
 SECRET_SCANNING_SETUP.md                      |  93 ++++++++++
 4 files changed, 267 insertions(+), 1 deletion(-)
 rename .github/workflows/.pre-commit-config.yaml => .pre-commit-config.yaml (100%)
 create mode 100644 .secrets.baseline
 create mode 100644 SECRET_SCANNING_SETUP.md

diff --git a/.gitleaks.toml b/.gitleaks.toml
index 3ebee76db..a05de91ae 100644
--- a/.gitleaks.toml
+++ b/.gitleaks.toml
@@ -6,8 +6,8 @@ useDefault = true
 
 [allowlist]
 description = "Global allowlist for AssetOpsBench"
-# Files/paths that legitimately contain dummy or example secrets.
 paths = [
+  '''\.secrets\.baseline$''',
   '''(.*?)(test|tests|fixtures|examples|docs)(/|\\).*''',
   '''.*\.md$''',
 ]
diff --git a/.github/workflows/.pre-commit-config.yaml b/.pre-commit-config.yaml
similarity index 100%
rename from .github/workflows/.pre-commit-config.yaml
rename to .pre-commit-config.yaml
diff --git a/.secrets.baseline b/.secrets.baseline
new file mode 100644
index 000000000..b2351abf3
--- /dev/null
+++ b/.secrets.baseline
@@ -0,0 +1,173 @@
+{
+  "version": "1.5.0",
+  "plugins_used": [
+    {
+      "name": "ArtifactoryDetector"
+    },
+    {
+      "name": "AWSKeyDetector"
+    },
+    {
+      "name": "AzureStorageKeyDetector"
+    },
+    {
+      "name": "Base64HighEntropyString",
+      "limit": 4.5
+    },
+    {
+      "name": "BasicAuthDetector"
+    },
+    {
+      "name": "CloudantDetector"
+    },
+    {
+      "name": "DiscordBotTokenDetector"
+    },
+    {
+      "name": "GitHubTokenDetector"
+    },
+    {
+      "name": "GitLabTokenDetector"
+    },
+    {
+      "name": "HexHighEntropyString",
+      "limit": 3.0
+    },
+    {
+      "name": "IbmCloudIamDetector"
+    },
+    {
+      "name": "IbmCosHmacDetector"
+    },
+    {
+      "name": "IPPublicDetector"
+    },
+    {
+      "name": "JwtTokenDetector"
+    },
+    {
+      "name": "KeywordDetector",
+      "keyword_exclude": ""
+    },
+    {
+      "name": "MailchimpDetector"
+    },
+    {
+      "name": "NpmDetector"
+    },
+    {
+      "name": "OpenAIDetector"
+    },
+    {
+      "name": "PrivateKeyDetector"
+    },
+    {
+      "name": "PypiTokenDetector"
+    },
+    {
+      "name": "SendGridDetector"
+    },
+    {
+      "name": "SlackDetector"
+    },
+    {
+      "name": "SoftlayerDetector"
+    },
+    {
+      "name": "SquareOAuthDetector"
+    },
+    {
+      "name": "StripeDetector"
+    },
+    {
+      "name": "TelegramBotTokenDetector"
+    },
+    {
+      "name": "TwilioKeyDetector"
+    }
+  ],
+  "filters_used": [
+    {
+      "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
+      "min_level": 2
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_indirect_reference"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_likely_id_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_lock_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_potential_uuid"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_sequential_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_swagger_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_templated_secret"
+    }
+  ],
+  "results": {
+    ".env.public": [
+      {
+        "type": "Secret Keyword",
+        "filename": ".env.public",
+        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
+        "is_verified": false,
+        "line_number": 4
+      }
+    ],
+    "src/agent/claude_agent/tests/test_runner.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "src/agent/claude_agent/tests/test_runner.py",
+        "hashed_secret": "18176482d2532398c7b84c22c6f8d2e59e55505c",
+        "is_verified": false,
+        "line_number": 32
+      }
+    ],
+    "src/couchdb/docker-compose.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "src/couchdb/docker-compose.yaml",
+        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
+        "is_verified": false,
+        "line_number": 6
+      }
+    ],
+    "src/llm/tests/test_backends.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "src/llm/tests/test_backends.py",
+        "hashed_secret": "ef219439b755958216dbdf4b1e3b645b1f54565e",
+        "is_verified": false,
+        "line_number": 67
+      }
+    ],
+    "src/llm/tests/test_routers.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "src/llm/tests/test_routers.py",
+        "hashed_secret": "ef219439b755958216dbdf4b1e3b645b1f54565e",
+        "is_verified": false,
+        "line_number": 60
+      }
+    ]
+  },
+  "generated_at": "2026-06-16T23:03:00Z"
+}
diff --git a/SECRET_SCANNING_SETUP.md b/SECRET_SCANNING_SETUP.md
new file mode 100644
index 000000000..da29d1f81
--- /dev/null
+++ b/SECRET_SCANNING_SETUP.md
@@ -0,0 +1,93 @@
+# Blocking Secrets in IBM/AssetOpsBench — Setup Guide
+
+A layered ("defense in depth") setup so sensitive values (API keys, tokens,
+passwords, private keys) are caught at three points: the developer's machine,
+the CI pipeline, and GitHub's own push gateway.
+
+## File placement
+
+| File in this folder        | Put it at repo path             |
+|----------------------------|---------------------------------|
+| `pre-commit-config.yaml`   | `.pre-commit-config.yaml` (rename) |
+| `.gitleaks.toml`           | `.gitleaks.toml`                |
+| `secret-scan.yml`          | `.github/workflows/secret-scan.yml` |
+
+---
+
+## Layer 1 — Local pre-commit hook (catches secrets before a commit exists)
+
+```bash
+# one-time, per contributor
+pip install pre-commit detect-secrets
+
+# generate the baseline of currently-known/allowed values
+detect-secrets scan > .secrets.baseline
+
+# install the git hook into this clone
+pre-commit install
+
+# (optional) test against the whole repo right now
+pre-commit run --all-files
+```
+
+After this, every `git commit` runs gitleaks + detect-secrets on staged changes
+and aborts the commit if a secret is found. Commit `.pre-commit-config.yaml`,
+`.gitleaks.toml`, and `.secrets.baseline` to the repo so the whole team shares
+the config. 
+
+---
+
+## Layer 2 — GitHub Actions CI check (server-side, blocks PR merges)
+
+`secret-scan.yml` runs gitleaks **and** TruffleHog on every push and pull
+request, plus a weekly full-history sweep. Nothing to install — just commit the
+workflow file. To make it enforce merges:
+
+1. Repo **Settings → Branches → Add branch protection rule** for `main`.
+2. Enable **Require status checks to pass before merging**.
+3. Select the **Gitleaks** and **TruffleHog** checks.
+
+Now a PR that introduces a secret cannot be merged until it's removed.
+
+---
+
+## Layer 3 — GitHub native Secret Scanning + Push Protection (no code)
+
+This is GitHub's built-in gateway that rejects a `git push` the moment it
+detects a recognized secret pattern.
+
+1. Go to the repo on GitHub → **Settings → Code security and analysis**
+   (org-level admins may set this for all repos under the IBM org).
+2. Enable **Secret scanning**.
+3. Enable **Push protection**.
+
+For private/internal repos this requires GitHub Advanced Security; public repos
+get secret scanning for free. Because IBM/AssetOpsBench is public, enable it
+directly — it's the strongest single control and takes ~30 seconds.
+
+---
+
+## If a secret was already committed (important)
+
+Rotating the secret is mandatory — scrubbing git history is not enough on its
+own, because clones and forks may already have it.
+
+1. **Revoke/rotate the leaked credential at its source immediately** (e.g.
+   regenerate the API key in the provider's dashboard). Assume it's compromised.
+2. Remove it from history with `git filter-repo` (preferred) or the BFG:
+   ```bash
+   pip install git-filter-repo
+   git filter-repo --replace-text <(echo 'THE_LEAKED_VALUE==>***REMOVED***')
+   git push --force --all
+   ```
+3. Tell collaborators to re-clone, since history was rewritten.
+
+---
+
+## Quick recap
+
+- Layer 1 stops most leaks at the keyboard (opt-in, bypassable).
+- Layer 2 enforces scanning in CI and blocks merges (can't be skipped).
+- Layer 3 is GitHub blocking the push itself (strongest, zero-maintenance).
+
+Enable all three for full coverage.

From 7e48e944e9f17223cd3386e087559079db48b491 Mon Sep 17 00:00:00 2001
From: pateldha <pateldha@us.ibm.com>
Date: Tue, 16 Jun 2026 19:37:50 -0400
Subject: [PATCH 3/9] Use binary installs for gitleaks/trufflehog to satisfy
 org action policy

Signed-off-by: Dhaval Patel <pateldha@us.ibm.com>
---
 .github/workflows/secret-scan.yml | 41 +++++++++++++++++++------------
 1 file changed, 25 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml
index 09d43df7b..e7128acf0 100644
--- a/.github/workflows/secret-scan.yml
+++ b/.github/workflows/secret-scan.yml
@@ -1,5 +1,7 @@
-# Server-side secret scanning. Runs on every push and pull request and on a
-# weekly schedule. Cannot be bypassed by contributors who skip local hooks.
+# Server-side secret scanning that uses NO third-party Actions, so it runs
+# under the IBM org policy "Allow IBM, and select non-IBM, actions".
+# Only actions/checkout (GitHub-created, already allowed) is used; the scanners
+# are installed via plain shell steps, which the org action policy does not gate.
 # Place this file at: .github/workflows/secret-scan.yml
 name: Secret Scan
 
@@ -21,18 +23,20 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          # full history so the scan covers all commits in a PR, not just the tip
-          fetch-depth: 0
+          fetch-depth: 0   # full history so the scan covers all commits
+
+      - name: Install gitleaks
+        run: |
+          VERSION=8.21.2
+          curl -sSL "https://github.com/gitleaks/gitleaks/releases/download/v${VERSION}/gitleaks_${VERSION}_linux_x64.tar.gz" \
+            | tar -xz -C /usr/local/bin gitleaks
+          gitleaks version
 
       - name: Run gitleaks
-        uses: gitleaks/gitleaks-action@v2
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          # If using the free OSS action no license is needed for public repos.
-          # For org-owned private repos, set GITLEAKS_LICENSE in repo secrets:
-          # GITLEAKS_LICENSE: ${{ secrets.GITLEAKS_LICENSE }}
-
-  # Second, independent scanner for defense in depth.
+        run: gitleaks detect --source . --redact --verbose
+        # gitleaks auto-loads .gitleaks.toml from the repo root and
+        # exits non-zero if any secret is found, failing the check.
+
   trufflehog:
     name: TruffleHog
     runs-on: ubuntu-latest
@@ -41,7 +45,12 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Run TruffleHog
-        uses: trufflesecurity/trufflehog@main
-        with:
-          extra_args: --results=verified,unknown
+      - name: Install trufflehog
+        run: |
+          curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh \
+            | sh -s -- -b /usr/local/bin v3.82.6
+          trufflehog --version
+
+      - name: Run trufflehog
+        run: trufflehog git "file://." --results=verified,unknown --fail
+        # --fail makes the job exit non-zero when verified/unknown secrets are found.
\ No newline at end of file

From 5b4403dd0b117eff7f3e4218789e812d115f3fac Mon Sep 17 00:00:00 2001
From: pateldha <pateldha@us.ibm.com>
Date: Tue, 16 Jun 2026 19:45:57 -0400
Subject: [PATCH 4/9] Triage known historical secret findings

Signed-off-by: Dhaval Patel <pateldha@us.ibm.com>
---
 .gitleaksignore | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 .gitleaksignore

diff --git a/.gitleaksignore b/.gitleaksignore
new file mode 100644
index 000000000..9e45e527c
--- /dev/null
+++ b/.gitleaksignore
@@ -0,0 +1,10 @@
+# Known, triaged historical findings. One gitleaks fingerprint per line.
+# A fingerprint here only silences the scanner — any real secret listed
+# must be ROTATED/REVOKED at its source first.
+
+# .env.public — intentional public/example value.
+f4443296d4565ba82ca3ec19303bc929362185eb:.env.public:generic-api-key:9
+
+# benchmark/docker-compose.yml — leaked GITHUB_TOKEN committed 2025-07-16.
+# ACTION REQUIRED: token must be revoked (history cannot be un-leaked).
+3ee88a4ef923d3f4729c25eccb0096bc7c805cf2:benchmark/docker-compose.yml:github-pat:9

From a65e2907226d14cee0f7a2074fea8457863c26f5 Mon Sep 17 00:00:00 2001
From: pateldha <pateldha@us.ibm.com>
Date: Tue, 16 Jun 2026 19:51:28 -0400
Subject: [PATCH 5/9] TruffleHog: gate on verified secrets only

Signed-off-by: Dhaval Patel <pateldha@us.ibm.com>
---
 .github/workflows/secret-scan.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml
index e7128acf0..05c1e3a9c 100644
--- a/.github/workflows/secret-scan.yml
+++ b/.github/workflows/secret-scan.yml
@@ -52,5 +52,5 @@ jobs:
           trufflehog --version
 
       - name: Run trufflehog
-        run: trufflehog git "file://." --results=verified,unknown --fail
+        run: trufflehog git "file://." --results=verified --fail
         # --fail makes the job exit non-zero when verified/unknown secrets are found.
\ No newline at end of file

From dbb87332bebb9a4fe181530956cd78f1a98f66bc Mon Sep 17 00:00:00 2001
From: pateldha <pateldha@us.ibm.com>
Date: Tue, 16 Jun 2026 22:23:20 -0400
Subject: [PATCH 6/9] adding security tag

Signed-off-by: Dhaval Patel <pateldha@us.ibm.com>
---
 CONTRIBUTING.md | 31 +++++++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 271273189..233b6fe8e 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -80,15 +80,38 @@ uv run ruff check --fix .
 
 ### 3. Security Scanning
 
-To protect industrial metadata and API keys, run the IBM `detect-secrets` scan:
+This repo blocks secrets (API keys, tokens, credentials) at three layers:
+a local pre-commit hook, a CI workflow, and GitHub push protection. As a
+contributor you only need to set up the local hook once per clone:
 
 ```bash
-uv pip install --upgrade "git+[https://github.com/ibm/detect-secrets.git@master#egg=detect-secrets](https://github.com/ibm/detect-secrets.git@master#egg=detect-secrets)"
-detect-secrets scan --update .secrets.baseline
-detect-secrets audit .secrets.baseline
+uv pip install pre-commit detect-secrets
+pre-commit install
+```
+
+After this, **every `git commit` automatically runs gitleaks and
+detect-secrets** on your staged changes and aborts the commit if a secret is
+found. To scan the whole repo on demand:
 
+```bash
+pre-commit run --all-files
 ```
 
+If you add a new file that legitimately contains an example/placeholder value
+flagged as a secret, update the detect-secrets baseline and audit it:
+
+```bash
+detect-secrets scan --baseline .secrets.baseline
+detect-secrets audit .secrets.baseline
+```
+
+Known, already-triaged historical findings are listed in `.gitleaksignore`
+(by fingerprint). Never add a *real* secret there — if a live credential is
+detected, rotate/revoke it at its source first, then remove it from the code.
+
+> The same scans run in CI on every pull request, so a commit that slips past
+> the local hook will still be caught before merge.
+
 ---
 
 ## Running Tests & Validation

From d64caf321f328673414a3279b46208a4d29068fb Mon Sep 17 00:00:00 2001
From: Dhaval Patel <pateldha@us.ibm.com>
Date: Tue, 16 Jun 2026 23:09:22 -0400
Subject: [PATCH 7/9] Guard all data files across src/couchdb and subfolders

Signed-off-by: Dhaval Patel <pateldha@us.ibm.com>
---
 .github/workflows/guard-couchdb-data.yml | 56 ++++++++++++++++++++++++
 src/couchdb/.allowed_datafiles           | 11 +++++
 2 files changed, 67 insertions(+)
 create mode 100644 .github/workflows/guard-couchdb-data.yml
 create mode 100644 src/couchdb/.allowed_datafiles

diff --git a/.github/workflows/guard-couchdb-data.yml b/.github/workflows/guard-couchdb-data.yml
new file mode 100644
index 000000000..1c25c8e3f
--- /dev/null
+++ b/.github/workflows/guard-couchdb-data.yml
@@ -0,0 +1,56 @@
+# Blocks NEW data files (csv/json/etc.) added under src/couchdb/ or any
+# subfolder, unless their path is listed in src/couchdb/.allowed_datafiles.
+# To permit a new data file, add its path to that allowlist in the same PR.
+# Uses only actions/checkout (GitHub-created) so it runs under the IBM org
+# action policy. Place at: .github/workflows/guard-couchdb-data.yml
+name: Guard couchdb data files
+
+on:
+  pull_request:
+    paths:
+      - 'src/couchdb/**'
+
+permissions:
+  contents: read
+
+jobs:
+  guard:
+    name: No unapproved data files
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Check for disallowed data files
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          set -euo pipefail
+          # Data-file extensions we want to gate (edit to taste).
+          EXT='\.(csv|tsv|json|jsonl|ndjson|parquet|xls|xlsx|feather|h5|hdf5|pkl|pickle|npy|npz|db|sqlite|sqlite3|avro|orc)$'
+          ALLOW="src/couchdb/.allowed_datafiles"
+
+          # Files newly ADDED in this PR under src/couchdb that look like data.
+          added=$(git diff --name-only --diff-filter=A "$BASE_SHA" "$HEAD_SHA" -- src/couchdb \
+                  | grep -iE "$EXT" || true)
+
+          violations=""
+          while IFS= read -r f; do
+            [ -z "$f" ] && continue
+            if ! grep -qxF "$f" "$ALLOW" 2>/dev/null; then
+              violations="${violations}${f}"$'\n'
+            fi
+          done <<< "$added"
+
+          if [ -n "$violations" ]; then
+            echo "::error::New data files are not allowed under src/couchdb/ unless allowlisted."
+            echo "Disallowed additions:"
+            printf '  - %s\n' $violations
+            echo ""
+            echo "If a file is intentional, add its exact path to ${ALLOW} in this PR"
+            echo "(a maintainer must review that change)."
+            exit 1
+          fi
+          echo "OK: no unapproved data files added under src/couchdb/"
\ No newline at end of file
diff --git a/src/couchdb/.allowed_datafiles b/src/couchdb/.allowed_datafiles
new file mode 100644
index 000000000..e4f4804d1
--- /dev/null
+++ b/src/couchdb/.allowed_datafiles
@@ -0,0 +1,11 @@
+src/couchdb/_design_workorders.json
+src/couchdb/collections.json
+src/couchdb/scenarios_data/default/manifest.json
+src/couchdb/scenarios_data/scenario_1/manifest.json
+src/couchdb/scenarios_data/scenario_2/manifest.json
+src/couchdb/scenarios_data/shared/failure_code/failure_code_sample.csv
+src/couchdb/scenarios_data/shared/iot/chiller_6.json
+src/couchdb/scenarios_data/shared/iot/hydraulic_pump_1.json
+src/couchdb/scenarios_data/shared/iot/metro_pump_1.json
+src/couchdb/scenarios_data/shared/iot/motor_01.json
+src/couchdb/scenarios_data/shared/work_order/workorders.csv

From 514f015fb03b25189c6d5d9412f981594b95de8c Mon Sep 17 00:00:00 2001
From: Dhaval Patel <pateldha@us.ibm.com>
Date: Tue, 16 Jun 2026 23:23:03 -0400
Subject: [PATCH 8/9] testing

Signed-off-by: Dhaval Patel <pateldha@us.ibm.com>
---
 .../shared/failure_code/failure_code_sample copy.csv  | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 src/couchdb/scenarios_data/shared/failure_code/failure_code_sample copy.csv

diff --git a/src/couchdb/scenarios_data/shared/failure_code/failure_code_sample copy.csv b/src/couchdb/scenarios_data/shared/failure_code/failure_code_sample copy.csv
new file mode 100644
index 000000000..648a52a0a
--- /dev/null
+++ b/src/couchdb/scenarios_data/shared/failure_code/failure_code_sample copy.csv	
@@ -0,0 +1,11 @@
+code,description
+FC001,equipment does not start
+FC002,equipment stops unexpectedly during operation
+FC003,abnormal noise during operation
+FC004,fluid leak observed
+FC005,"excessive vibration, shaking, or instability"
+FC006,overheating or high temperature reading
+FC007,indicator light does not illuminate
+FC008,gauge does not operate or is inaccurate
+FC009,structural damage or cracking
+FC010,part missing or loose

From f549fa95c160de4dbd1a52f7bf5abef8f3595f35 Mon Sep 17 00:00:00 2001
From: Dhaval Patel <pateldha@us.ibm.com>
Date: Tue, 16 Jun 2026 23:45:32 -0400
Subject: [PATCH 9/9] finalied local check for data upload

Signed-off-by: Dhaval Patel <pateldha@us.ibm.com>
---
 .pre-commit-config.yaml                       | 30 ++++++++-----------
 scripts/check_couchdb_data.sh                 | 26 ++++++++++++++++
 .../failure_code/failure_code_sample copy.csv | 11 -------
 3 files changed, 39 insertions(+), 28 deletions(-)
 create mode 100755 scripts/check_couchdb_data.sh
 delete mode 100644 src/couchdb/scenarios_data/shared/failure_code/failure_code_sample copy.csv

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 223d9739d..5f7567e3d 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,12 +1,9 @@
-# >>> Rename this file to ".pre-commit-config.yaml" at the repo root <<<
+# Pre-commit hooks. Install once per clone:  pre-commit install
+# Run against all files:  pre-commit run --all-files
 #
-# Pre-commit hooks to block secrets before they are committed.
-# Install once per clone:  pip install pre-commit && pre-commit install
-# Run against all files:    pre-commit run --all-files
-#
-# Two complementary scanners are used:
-#   - gitleaks      : fast, regex + entropy based, large rule set
-#   - detect-secrets: IBM-maintained, baseline-driven (won't re-flag known/allowed values)
+#   - gitleaks         : secret scanning (regex + entropy)
+#   - detect-secrets   : IBM-maintained, baseline-driven secret scanning
+#   - block-couchdb-data : blocks new data files under src/couchdb/ (local hook)
 
 repos:
   - repo: https://github.com/gitleaks/gitleaks
@@ -19,13 +16,12 @@ repos:
     hooks:
       - id: detect-secrets
         args: ['--baseline', '.secrets.baseline']
-        # exclude lockfiles / generated assets from scanning if needed:
-        # exclude: package.lock.json
 
-  # Optional general hygiene hooks (uncomment to enable)
-  # - repo: https://github.com/pre-commit/pre-commit-hooks
-  #   rev: v5.0.0
-  #   hooks:
-  #     - id: check-added-large-files
-  #     - id: end-of-file-fixer
-  #     - id: trailing-whitespace
+  - repo: local
+    hooks:
+      - id: block-couchdb-data
+        name: Block unapproved data files in src/couchdb
+        entry: scripts/check_couchdb_data.sh
+        language: script
+        pass_filenames: false
+        always_run: true
\ No newline at end of file
diff --git a/scripts/check_couchdb_data.sh b/scripts/check_couchdb_data.sh
new file mode 100755
index 000000000..2acac9a1a
--- /dev/null
+++ b/scripts/check_couchdb_data.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Blocks committing NEW data files under src/couchdb/ (any subfolder) unless
+# their path is listed in src/couchdb/.allowed_datafiles.
+# Used both as a pre-commit hook (staged files) and is safe to run manually.
+set -euo pipefail
+
+EXT='\.(csv|tsv|json|jsonl|ndjson|parquet|xls|xlsx|feather|h5|hdf5|pkl|pickle|npy|npz|db|sqlite|sqlite3|avro|orc)$'
+ALLOW="src/couchdb/.allowed_datafiles"
+
+# Newly ADDED, staged files under src/couchdb that look like data.
+staged=$(git diff --cached --name-only --diff-filter=A -- src/couchdb | grep -iE "$EXT" || true)
+
+violations=""
+while IFS= read -r f; do
+  [ -z "$f" ] && continue
+  grep -qxF "$f" "$ALLOW" 2>/dev/null || violations="${violations}${f}"$'\n'
+done <<< "$staged"
+
+if [ -n "$violations" ]; then
+  echo "BLOCKED: new data files under src/couchdb/ are not allowed unless allowlisted."
+  while IFS= read -r v; do [ -n "$v" ] && echo "  - $v"; done <<< "$violations"
+  echo ""
+  echo "If this file is intentional, add its exact path to ${ALLOW} and re-commit."
+  exit 1
+fi
+exit 0
diff --git a/src/couchdb/scenarios_data/shared/failure_code/failure_code_sample copy.csv b/src/couchdb/scenarios_data/shared/failure_code/failure_code_sample copy.csv
deleted file mode 100644
index 648a52a0a..000000000
--- a/src/couchdb/scenarios_data/shared/failure_code/failure_code_sample copy.csv	
+++ /dev/null
@@ -1,11 +0,0 @@
-code,description
-FC001,equipment does not start
-FC002,equipment stops unexpectedly during operation
-FC003,abnormal noise during operation
-FC004,fluid leak observed
-FC005,"excessive vibration, shaking, or instability"
-FC006,overheating or high temperature reading
-FC007,indicator light does not illuminate
-FC008,gauge does not operate or is inaccurate
-FC009,structural damage or cracking
-FC010,part missing or loose