From f19b876055dbc6bfed39222a696d5d752abbe92b Mon Sep 17 00:00:00 2001 From: Omar Rao Date: Wed, 26 Aug 2026 13:06:35 -0400 Subject: [PATCH] @ fix(cleanup): simple 21-day age-based retention + OAuth token; add CodeQL (closes #33) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per issue #33: - Retention default 21 days (was 90); keep the weekly Sunday schedule. - Cleanup now authenticates with the OAuth refresh token via GoogleDriveClient.createAuthClient (was an invalid GoogleAuth keyFile on the client-secret) — matches backup/restore. - Remove the GFS theater from cleanup.yml (retention_policy input, GFS_* env, and the log lines that claimed a policy the code never implemented). - Docs: corrected README + USERGUIDE 8.3/FAQ/19.6/SOX to describe the actual simple age-based (chain-aware) retention. Chain-protection safeguard retained. Security/quality: - Add CodeQL workflow (security-and-quality queries) to populate GitHub code scanning; Dependabot alerts currently 0. 105/105 tests pass, lint + headers clean, all workflow YAML valid. @ --- .github/workflows/cleanup.yml | 27 +++++---------------- .github/workflows/codeql.yml | 35 +++++++++++++++++++++++++++ README.md | 4 ++-- docs/USERGUIDE.md | 45 ++++++++++++++++------------------- src/cleanup/index.js | 15 +++++++----- 5 files changed, 73 insertions(+), 53 deletions(-) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml index 355db08..121716f 100644 --- a/.github/workflows/cleanup.yml +++ b/.github/workflows/cleanup.yml @@ -6,13 +6,9 @@ on: workflow_dispatch: inputs: retention_days: - description: 'Delete sessions older than N days (simple mode)' + description: 'Delete sessions older than N days' required: false - default: '90' - retention_policy: - description: 'Retention policy: simple or gfs (Grandfather-Father-Son)' - required: false - default: 'gfs' + default: '21' runner: description: 'Runner label' required: false @@ -50,20 +46,9 @@ jobs: GDRIVE_FOLDER_ID: ${{ secrets.GDRIVE_FOLDER_ID }} GOOGLE_CLIENT_SECRET_PATH: ./credentials/google-client-secret.json GOOGLE_TOKEN_PATH: ./credentials/google-token.json - RETENTION_DAYS: ${{ github.event.inputs.retention_days || '90' }} - RETENTION_POLICY: ${{ github.event.inputs.retention_policy || 'gfs' }} - # GFS policy: daily=7, weekly=4, monthly=12 - GFS_DAILY_KEEP: '7' - GFS_WEEKLY_KEEP: '4' - GFS_MONTHLY_KEEP: '12' + RETENTION_DAYS: ${{ github.event.inputs.retention_days || '21' }} run: | - echo "Retention policy: $RETENTION_POLICY" - if [ "$RETENTION_POLICY" = "gfs" ]; then - echo "Applying Grandfather-Father-Son retention:" - echo " Daily: keep last $GFS_DAILY_KEEP" - echo " Weekly: keep last $GFS_WEEKLY_KEEP (Sundays)" - echo " Monthly: keep last $GFS_MONTHLY_KEEP (1st of month)" - fi + echo "Simple age-based retention: deleting sessions older than $RETENTION_DAYS days" node src/cleanup/index.js - name: Commit weekly digest @@ -72,7 +57,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} script: | const fs = require('fs'); - const retentionDays = process.env.RETENTION_DAYS || '90'; + const retentionDays = process.env.RETENTION_DAYS || '21'; const date = new Date().toISOString().slice(0, 10); // Parse cleanup log if available @@ -113,7 +98,7 @@ jobs: ...(sha ? { sha } : {}) }); env: - RETENTION_DAYS: ${{ github.event.inputs.retention_days || '90' }} + RETENTION_DAYS: ${{ github.event.inputs.retention_days || '21' }} - name: Upload cleanup log if: always() diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..1928a95 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,35 @@ +name: CodeQL + +# Static analysis (GitHub code scanning). Populates the repository's +# Security → Code scanning alerts for JavaScript/Node source. + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: '0 5 * * 1' # weekly, Monday 05:00 UTC + +permissions: + contents: read + security-events: write + +jobs: + analyze: + name: Analyze (javascript) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: javascript + queries: security-and-quality + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: '/language:javascript' diff --git a/README.md b/README.md index b433a25..a90ab22 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,7 @@ The dashboard surfaces the latest session's **delta composition** (full / delta | **PAT rotation reminder** | Weekly `pat-check.yml` cron warns via Teams + email when PAT is ≤7 days from expiry | | **Repo search** | Filter the backup repo list by name directly in the Backup tab | | **Session diff** | Compare any two backup sessions side by side — added, removed, changed repos with byte deltas | -| **GFS retention** | Grandfather-Father-Son policy (daily × 7, weekly × 4, monthly × 12) in `cleanup.yml` | +| **Age-based retention** | Simple age-based cleanup (default 21 days, weekly Sunday schedule) in `cleanup.yml`; chain-aware so it never deletes a bundle a retained delta chain still needs | | **SLA breach alerts** | Hourly `sla-check.yml` posts Teams/email alert if backup age exceeds `SLA_HOURS` | | **Compliance CSV export** | One-click CSV export of full run history for audit evidence packages | | **Anomaly detection** | Auto-detects session size deviation >20% from 7-day average; dismissible dashboard banner | @@ -530,7 +530,7 @@ github-gdrive-backup/ │ ├── ci.yml — CI: lint, test, copyright, audit, secret scan │ ├── backup.yml — daily cron + manual backup (SBOM, storage target) │ ├── restore.yml — manual restore -│ ├── cleanup.yml — GFS / simple retention cleanup +│ ├── cleanup.yml — simple age-based retention cleanup (chain-aware) │ ├── notify.yml — Teams + SendGrid email digest │ ├── pat-check.yml — weekly PAT expiry reminder │ ├── sla-check.yml — hourly SLA breach alert diff --git a/docs/USERGUIDE.md b/docs/USERGUIDE.md index eed2c83..032cddc 100644 --- a/docs/USERGUIDE.md +++ b/docs/USERGUIDE.md @@ -37,7 +37,7 @@ - [8.1 Google Drive Structure](#81-google-drive-structure) - [8.2 S3 / Azure Blob / Backblaze B2](#82-s3--azure-blob--backblaze-b2) - [8.2A Multi-Destination Fan-Out (3-2-1 Rule)](#82a-multi-destination-fan-out-3-2-1-rule) - - [8.3 Retention Policy (Simple & GFS)](#83-retention-policy-simple--gfs) + - [8.3 Retention Policy (age-based)](#83-retention-policy-age-based) - [8.4 Drive Quota Monitoring](#84-drive-quota-monitoring) 9. [Notifications & Monitoring](#9-notifications--monitoring) - [9.1 Slack Webhook](#91-slack-webhook) @@ -569,31 +569,28 @@ Valid values are `s3`, `azure`, and `b2`. Each listed target's own credentials ( **Encryption.** When `BACKUP_ENCRYPTION_KEY` is set, archives are mirrored in their already-encrypted (`.zip.enc`) form, so secondary copies are protected at rest just like the primary. -### 8.3 Retention Policy (Simple & GFS) +### 8.3 Retention Policy (age-based) -`cleanup.yml` supports two retention modes selected via the `retention_policy` workflow input: +`cleanup.yml` applies **simple age-based retention**: any session older than +`RETENTION_DAYS` (default **21**) is deleted. It runs weekly on the **Sunday +03:00 UTC** schedule, or on demand. Authentication uses the same **OAuth refresh +token** as backup and restore (`GOOGLE_CLIENT_SECRET` + `GOOGLE_TOKEN`). -**Simple retention** (default if you set `retention_policy=simple`) deletes any session older than your configured `RETENTION_DAYS`. This is the easiest option for individuals. +**Chain-aware safety.** Deletion is age-based but never blindly destructive: the +pure `planCleanup()` function first reads the `backup-state.json` of every +*retained* session and protects the entire delta chain each one depends on, so an +old **base** or intermediate bundle is never removed while a newer session still +needs it to restore (see [15B](#15b-reliability--operations-v35)). Protected-but-old +sessions are logged explicitly. -**GFS — Grandfather-Father-Son** (default: `retention_policy=gfs`) applies a tiered policy that balances granularity with storage cost: - -| Tier | How many kept | Which sessions | -|---|---|---| -| Daily | 7 | The 7 most recent daily sessions | -| Weekly | 4 | The most recent session per calendar week, for 4 weeks | -| Monthly | 12 | The most recent session per calendar month, for 12 months | - -GFS is the recommended default for production use. It keeps last week's sessions at full granularity for quick rollback, provides weekly rollback points for the last month, and monthly archives going back a year — all at a fraction of the storage cost of a flat 365-day rolling window. - -To dispatch with a specific policy: +To dispatch with a custom window: ```bash -gh workflow run cleanup.yml -f retention_policy=gfs -# or -gh workflow run cleanup.yml -f retention_policy=simple +gh workflow run cleanup.yml -f retention_days=30 ``` -Choose a policy that satisfies your recovery objectives and compliance requirements (see [Section 14](#14-compliance--reporting)). +Choose a window that satisfies your recovery objectives and compliance +requirements (see [Section 14](#14-compliance--reporting)). ### 8.4 Drive Quota Monitoring @@ -910,7 +907,7 @@ Entries are searchable in the UI and exportable to CSV. The log is also a key au | Framework | Relevant project capability | |---|---| -| **SOX** | Audit log, GFS retention, change control via branch protection, CSV export | +| **SOX** | Audit log, age-based retention, change control via branch protection, CSV export | | **HIPAA** | AES-256-CBC encryption at rest, access control allow-list, integrity verification, SBOM | | **ISO 27001** | Backup/restore procedures, SLA tracking, monthly restore test, PAT rotation reminder | | **SOC 2** | Availability (SLA tracker + hourly check), confidentiality (encryption), monitoring (audit log + anomaly detection) | @@ -1115,8 +1112,8 @@ When the current backup session's total size deviates more than 20% from the 7-d **Can I install the dashboard as an app?** Yes — the dashboard is a PWA. Modern browsers will offer an Install prompt. After installation it launches in a standalone window and serves cached content offline. See [Section 15](#15-pwa--offline-support). -**What is GFS retention?** -Grandfather-Father-Son is a tiered retention policy: keep the last 7 daily sessions, the last 4 weekly sessions, and the last 12 monthly sessions. It gives strong rollback granularity without paying for a full flat 365-day window. Select it via `retention_policy=gfs` when dispatching `cleanup.yml`. +**How does retention/cleanup work?** +`cleanup.yml` uses simple age-based retention: sessions older than `RETENTION_DAYS` (default 21) are deleted on the weekly Sunday schedule. It is chain-aware — it never deletes a base/intermediate bundle a retained delta chain still needs. Override the window with `gh workflow run cleanup.yml -f retention_days=30`. **How do I get notified before my PAT expires?** Set the `PAT_EXPIRY_DATE` secret to your PAT's expiry date in `YYYY-MM-DD` format. The `pat-check.yml` workflow runs every Monday and sends a Teams card + email when the PAT is ≤7 days from expiry. @@ -1155,9 +1152,9 @@ A weekly workflow (`pat-check.yml`, Mondays 08:00 UTC) checks `PAT_EXPIRY_DATE` The **Reports** page now includes a **Session Diff** card. Select two sessions from the dropdowns and click **Compare** to see a table showing each repository's size in Session A and Session B, the delta, and whether repos were added or removed. Works in demo mode with sample data. -### 19.6 Smart GFS Retention +### 19.6 Age-Based Retention -`cleanup.yml` now accepts a `retention_policy` input (`simple` or `gfs`). When `gfs` is selected the cleanup step logs the Grandfather-Father-Son retention plan (daily × 7, weekly × 4, monthly × 12) alongside the deleted/kept counts. +`cleanup.yml` applies simple age-based retention (default 21 days) on the weekly Sunday schedule, authenticating with the OAuth refresh token. It is chain-aware — it protects any base/intermediate bundle a retained delta chain still needs — and logs deleted/kept (including chain-protected) counts. *(Earlier drafts referenced a "GFS" mode that was only ever logged, never enforced; that has been removed so the workflow no longer claims a policy the code does not implement — see issue #33.)* ### 19.7 SLA Breach Alerts diff --git a/src/cleanup/index.js b/src/cleanup/index.js index b4f4227..4814101 100644 --- a/src/cleanup/index.js +++ b/src/cleanup/index.js @@ -3,12 +3,13 @@ // This file is available under the GNU Affero General Public License v3.0 // or under a separate commercial license. const { google } = require('googleapis'); -const { GoogleAuth } = require('google-auth-library'); const fs = require('fs'); const path = require('path'); +const GoogleDriveClient = require('../backup/gdrive'); const FOLDER_ID = process.env.GDRIVE_FOLDER_ID; -const RETENTION_DAYS = parseInt(process.env.RETENTION_DAYS || '90', 10); +// Simple age-based retention (default 21 days). +const RETENTION_DAYS = parseInt(process.env.RETENTION_DAYS || '21', 10); const LOG_DIR = path.join(process.cwd(), 'logs'); if (!fs.existsSync(LOG_DIR)) fs.mkdirSync(LOG_DIR, { recursive: true }); @@ -101,10 +102,12 @@ async function main() { if (!FOLDER_ID) { log('ERROR: GDRIVE_FOLDER_ID not set'); process.exit(1); } if (RETENTION_DAYS === 0) { log('Retention disabled (0 days), skipping cleanup'); return; } - const auth = new GoogleAuth({ - keyFile: 'credentials/google-client-secret.json', - scopes: ['https://www.googleapis.com/auth/drive'], - }); + // Authenticate with the OAuth refresh token (same flow as backup/restore), + // not a service-account key file. + const auth = await GoogleDriveClient.createAuthClient( + process.env.GOOGLE_CLIENT_SECRET_PATH || './credentials/google-client-secret.json', + process.env.GOOGLE_TOKEN_PATH || './credentials/google-token.json' + ); const drive = google.drive({ version: 'v3', auth }); const cutoff = new Date(Date.now() - RETENTION_DAYS * 24 * 60 * 60 * 1000);