diff --git a/.env.example b/.env.example index 212d195..c56495d 100644 --- a/.env.example +++ b/.env.example @@ -58,6 +58,10 @@ AZURE_CLIENT_ID= AZURE_CLIENT_SECRET= AZURE_TENANT_ID= +# Field encryption (connector secrets) +# Generate with: python -c "import secrets; print(secrets.token_urlsafe(32))" +FIELD_ENCRYPTION_KEY= + # Confluence (optional) CONFLUENCE_URL= CONFLUENCE_USERNAME= diff --git a/CHANGELOG.md b/CHANGELOG.md index e637eff..40bdbba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Per-tenant encrypted connector secrets: users can now enter API keys/secrets directly in the connector creation UI, stored encrypted in the DB using Fernet with HKDF-derived per-tenant keys +- `FIELD_ENCRYPTION_KEY` setting for dedicated encryption key (falls back to `SECRET_KEY`) +- `connectors/crypto.py` module with `encrypt_secret()` / `decrypt_secret()` utilities +- `encrypted_secret` field on `ConnectorConfig` model with `set_secret()` / `get_secret()` methods +- Password input field in connector creation form (both standalone page and modal) +- `migrate_connector_secrets` management command to migrate existing env-var-based credentials into encrypted secrets +- `cryptography>=42.0` dependency - Apache 2.0 LICENSE file - `CONTRIBUTING.md`, `CHANGELOG.md`, `SECURITY.md` - `SECRET_KEY` startup validation (refuses to start with placeholder key in production) diff --git a/README.md b/README.md index 691ba08..528656f 100644 --- a/README.md +++ b/README.md @@ -196,9 +196,13 @@ AZURE_MISTRAL_DEPLOYMENT_NAME=... # Rate limits LLM_REQUESTS_PER_MINUTE=60 EMBEDDING_BATCH_SIZE=100 + +# Field encryption (connector secrets) +# Generate with: python -c "import secrets; print(secrets.token_urlsafe(32))" +FIELD_ENCRYPTION_KEY= ``` -> **Security**: In production (`DEBUG=False`), Django will refuse to start if `SECRET_KEY` is set to a placeholder value. +> **Security**: In production (`DEBUG=False`), Django will refuse to start if `SECRET_KEY` is set to a placeholder value. A dedicated `FIELD_ENCRYPTION_KEY` is recommended for encrypting connector secrets (falls back to `SECRET_KEY` if not set). For analysis tuning via `config.yaml`, see [docs/configuration.md](docs/configuration.md). @@ -255,7 +259,7 @@ The test suite covers all major modules: analysis views, audit views, chat, chun ## Dependencies -**Core:** Django 5.1, Celery 5.4, sqlite-vec 0.1.6, OpenAI SDK, tiktoken, django-allauth, whitenoise, gunicorn +**Core:** Django 5.1, Celery 5.4, sqlite-vec 0.1.6, OpenAI SDK, tiktoken, django-allauth, cryptography (Fernet encryption), whitenoise, gunicorn **ML/Analysis:** scikit-learn, HDBSCAN, datasketch (MinHash), numpy, NLTK, langid, rank-bm25 diff --git a/SECURITY.md b/SECURITY.md index 650b58e..cda5b91 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -31,8 +31,40 @@ Include the following in your report: - Never use the default `SECRET_KEY` in production - Generate a proper key: `python -c "from django.core.management.utils import get_random_secret_key; print(get_random_secret_key())"` +- Set a dedicated `FIELD_ENCRYPTION_KEY` for connector secret encryption: `python -c "import secrets; print(secrets.token_urlsafe(32))"` - Keep `.env` files out of version control (already in `.gitignore`) - Use HTTPS in production (HSTS is enabled when `DEBUG=False`) - Rotate API keys regularly - Change default credentials immediately after setup - Use a production-grade database (PostgreSQL) for concurrent deployments + +## Connector Secret Encryption + +Connector credentials (API keys, client secrets) can be stored encrypted in the database using per-tenant Fernet encryption: + +- **Key derivation:** HKDF (SHA-256) derives a unique Fernet key per tenant from the master `FIELD_ENCRYPTION_KEY` (or `SECRET_KEY` as fallback) +- **Info prefix:** `docuscore-connector-secret-v1:` followed by the tenant UUID, enabling future key rotation +- **Tenant isolation:** Each tenant's secrets are encrypted with a different derived key — a secret encrypted for tenant A cannot be decrypted by tenant B +- **Graceful fallback:** Connectors with only a `credential_ref` (env var name) continue working without encryption +- **Failure mode:** Decryption errors return an empty string and log a warning — the application does not crash +- **Migration:** Use `python manage.py migrate_connector_secrets --apply` to encrypt existing env-var-based credentials (see below) + +### Migrating Existing Credentials + +If you already have connectors configured with `credential_ref` pointing to environment variables, use the management command to encrypt them: + +```bash +# 1. Set FIELD_ENCRYPTION_KEY in .env (or rely on SECRET_KEY fallback) +# 2. Ensure the env vars referenced by credential_ref are set + +# Dry run — preview what would be migrated +python manage.py migrate_connector_secrets + +# Apply — encrypt env var values into encrypted_secret +python manage.py migrate_connector_secrets --apply + +# Optional: clear credential_ref after verifying encryption works +python manage.py migrate_connector_secrets --apply --clear-ref +``` + +The command is safe to run multiple times — it skips connectors that already have an `encrypted_secret` value. diff --git a/connectors/confluence.py b/connectors/confluence.py index 45afa05..2cd3291 100644 --- a/connectors/confluence.py +++ b/connectors/confluence.py @@ -23,7 +23,7 @@ def __init__(self, config: dict, credential: str = ""): self._url = config.get("url", os.environ.get("CONFLUENCE_URL", "")) self._space_key = config.get("space_key", "") self._username = config.get("username", os.environ.get("CONFLUENCE_USERNAME", "")) - self._api_token = os.environ.get(credential, os.environ.get("CONFLUENCE_API_TOKEN", "")) + self._api_token = credential or os.environ.get("CONFLUENCE_API_TOKEN", "") self._client = None def _get_client(self): diff --git a/connectors/crypto.py b/connectors/crypto.py new file mode 100644 index 0000000..e85fbbc --- /dev/null +++ b/connectors/crypto.py @@ -0,0 +1,58 @@ +""" +Per-tenant encryption for connector secrets. + +Uses HKDF key derivation to produce a unique Fernet key per tenant from a +single master key, so each tenant's secrets are cryptographically isolated. +""" + +import base64 +import logging + +from cryptography.fernet import Fernet, InvalidToken +from cryptography.hazmat.primitives.hashes import SHA256 +from cryptography.hazmat.primitives.kdf.hkdf import HKDF +from django.conf import settings + +logger = logging.getLogger(__name__) + +_HKDF_INFO_PREFIX = b"docuscore-connector-secret-v1:" + + +def _get_master_key() -> bytes: + """Return the master key bytes used for HKDF derivation.""" + key = getattr(settings, "FIELD_ENCRYPTION_KEY", "") or settings.SECRET_KEY + return key.encode("utf-8") if isinstance(key, str) else key + + +def _derive_fernet_key(tenant_id: str) -> bytes: + """Derive a tenant-specific Fernet key via HKDF.""" + hkdf = HKDF( + algorithm=SHA256(), + length=32, + salt=None, + info=_HKDF_INFO_PREFIX + tenant_id.encode("utf-8"), + ) + derived = hkdf.derive(_get_master_key()) + return base64.urlsafe_b64encode(derived) + + +def encrypt_secret(plain_text: str, tenant_id: str) -> str: + """Encrypt a secret string for a specific tenant. Returns base64 ciphertext.""" + if not plain_text: + return "" + key = _derive_fernet_key(tenant_id) + f = Fernet(key) + return f.encrypt(plain_text.encode("utf-8")).decode("utf-8") + + +def decrypt_secret(encrypted_text: str, tenant_id: str) -> str: + """Decrypt a tenant-specific secret. Returns empty string on failure.""" + if not encrypted_text: + return "" + try: + key = _derive_fernet_key(tenant_id) + f = Fernet(key) + return f.decrypt(encrypted_text.encode("utf-8")).decode("utf-8") + except (InvalidToken, Exception): + logger.warning("Failed to decrypt secret for tenant %s", tenant_id) + return "" diff --git a/connectors/management/__init__.py b/connectors/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/connectors/management/commands/__init__.py b/connectors/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/connectors/management/commands/migrate_connector_secrets.py b/connectors/management/commands/migrate_connector_secrets.py new file mode 100644 index 0000000..51873c4 --- /dev/null +++ b/connectors/management/commands/migrate_connector_secrets.py @@ -0,0 +1,124 @@ +""" +Management command to migrate existing connector credentials from environment +variables into per-tenant encrypted secrets. + +For production deployments that were using credential_ref (env var names), +this reads each env var value and encrypts it into the encrypted_secret field. + +Usage: + # Dry run (default) — shows what would be migrated without changing anything + python manage.py migrate_connector_secrets + + # Actually perform the migration + python manage.py migrate_connector_secrets --apply + + # Also clear credential_ref after migration (env vars no longer needed) + python manage.py migrate_connector_secrets --apply --clear-ref +""" + +import logging +import os + +from django.core.management.base import BaseCommand + +from connectors.models import ConnectorConfig + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = ( + "Migrate connector credentials from environment variables " + "into per-tenant encrypted secrets." + ) + + def add_arguments(self, parser): + parser.add_argument( + "--apply", + action="store_true", + help="Actually perform the migration. Without this flag, runs in dry-run mode.", + ) + parser.add_argument( + "--clear-ref", + action="store_true", + help="Clear credential_ref after successful encryption (env var no longer needed).", + ) + + def handle(self, *args, **options): + apply = options["apply"] + clear_ref = options["clear_ref"] + + if not apply: + self.stdout.write( + self.style.WARNING("DRY RUN — pass --apply to perform the migration.\n") + ) + + connectors = ConnectorConfig.objects.filter( + credential_ref__gt="", + ).exclude( + encrypted_secret__gt="", + ) + + total = connectors.count() + if total == 0: + self.stdout.write( + self.style.SUCCESS( + "Nothing to migrate: no connectors with credential_ref and empty encrypted_secret." + ) + ) + return + + self.stdout.write(f"Found {total} connector(s) to migrate.\n") + + migrated = 0 + skipped = 0 + errors = 0 + + for connector in connectors: + ref = connector.credential_ref + env_value = os.environ.get(ref, "") + + tenant_name = connector.tenant.name if connector.tenant else "?" + label = f" [{connector.name}] tenant={tenant_name} ref={ref}" + + if not env_value: + self.stdout.write( + self.style.WARNING(f"{label} — SKIPPED (env var '{ref}' is empty or unset)") + ) + skipped += 1 + continue + + if apply: + try: + connector.set_secret(env_value) + if clear_ref: + connector.credential_ref = "" + connector.save() + self.stdout.write(self.style.SUCCESS(f"{label} — MIGRATED")) + migrated += 1 + except Exception as e: + self.stdout.write(self.style.ERROR(f"{label} — ERROR: {e}")) + errors += 1 + else: + masked = env_value[:3] + "***" if len(env_value) > 3 else "***" + self.stdout.write(f"{label} — WOULD MIGRATE (value: {masked})") + migrated += 1 + + self.stdout.write("") + if apply: + self.stdout.write( + self.style.SUCCESS( + f"Done. Migrated: {migrated}, Skipped: {skipped}, Errors: {errors}" + ) + ) + if migrated > 0 and not clear_ref: + self.stdout.write( + self.style.NOTICE( + "Tip: re-run with --clear-ref to remove credential_ref values " + "once you've verified the encrypted secrets work correctly." + ) + ) + else: + self.stdout.write( + f"Would migrate: {migrated}, Would skip: {skipped}\nPass --apply to execute." + ) diff --git a/connectors/migrations/0003_add_encrypted_secret.py b/connectors/migrations/0003_add_encrypted_secret.py new file mode 100644 index 0000000..e3ee26e --- /dev/null +++ b/connectors/migrations/0003_add_encrypted_secret.py @@ -0,0 +1,21 @@ +# Generated by Django 5.1.15 on 2026-03-23 08:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("connectors", "0002_add_project"), + ] + + operations = [ + migrations.AddField( + model_name="connectorconfig", + name="encrypted_secret", + field=models.TextField( + blank=True, + default="", + help_text="Fernet-encrypted secret (set via set_secret, read via get_secret)", + ), + ), + ] diff --git a/connectors/models.py b/connectors/models.py index 114e883..4f176fc 100644 --- a/connectors/models.py +++ b/connectors/models.py @@ -5,12 +5,16 @@ and stores credentials securely. """ +import logging +import os import uuid from django.db import models from django.utils.translation import gettext_lazy as _ from tenants.models import ProjectScopedModel +logger = logging.getLogger(__name__) + class ConnectorConfig(ProjectScopedModel): """Configuration for a document source connector.""" @@ -36,6 +40,12 @@ class ConnectorType(models.TextChoices): blank=True, help_text="Reference to credential store (env var name or secret manager path)", ) + # Encrypted secret — per-tenant Fernet-encrypted credential value + encrypted_secret = models.TextField( + blank=True, + default="", + help_text="Fernet-encrypted secret (set via set_secret, read via get_secret)", + ) schedule_cron = models.CharField( max_length=100, @@ -54,3 +64,24 @@ class Meta: def __str__(self): return f"{self.name} ({self.connector_type})" + + def set_secret(self, plain_text: str) -> None: + """Encrypt and store a secret value for this connector's tenant.""" + from connectors.crypto import encrypt_secret + + self.encrypted_secret = encrypt_secret(plain_text, str(self.tenant_id)) + + def get_secret(self) -> str: + """Return the decrypted secret, falling back to credential_ref env var lookup.""" + if self.encrypted_secret: + from connectors.crypto import decrypt_secret + + decrypted = decrypt_secret(self.encrypted_secret, str(self.tenant_id)) + if decrypted: + return decrypted + + # Fallback: treat credential_ref as an env var name + if self.credential_ref: + return os.environ.get(self.credential_ref, "") + + return "" diff --git a/connectors/sharepoint.py b/connectors/sharepoint.py index ef16b08..6d31d0c 100644 --- a/connectors/sharepoint.py +++ b/connectors/sharepoint.py @@ -26,7 +26,7 @@ def __init__(self, config: dict, credential: str = ""): self._folder_path = config.get("folder_path", "/") self._client_id = config.get("client_id", os.environ.get("SHAREPOINT_CLIENT_ID", "")) self._tenant_id = config.get("tenant_id", os.environ.get("SHAREPOINT_TENANT_ID", "")) - self._client_secret = os.environ.get(credential, "") if credential else "" + self._client_secret = credential or "" self._access_token: str | None = None def _authenticate(self): diff --git a/connectors/templates/connectors/_connector_cards.html b/connectors/templates/connectors/_connector_cards.html index c469c19..ac706e6 100644 --- a/connectors/templates/connectors/_connector_cards.html +++ b/connectors/templates/connectors/_connector_cards.html @@ -23,6 +23,35 @@

{% trans "Connecteurs" %}

+ +
+
+
+ +
+
+
{% trans "Vos secrets sont chiffrés de bout en bout" %}
+
+ {% trans "Les clés API et secrets saisis dans SCORE sont chiffrés avec une clé unique par organisation. Ils ne sont jamais stockés en clair." %} + +
+
+
+
+
{% trans "Chiffrement Fernet (AES-128-CBC + HMAC)" %}
+
{% trans "Clé dérivée par tenant via HKDF-SHA256" %}
+
{% trans "Isolation cryptographique entre organisations" %}
+
{% trans "Compatible avec les variables d'environnement existantes" %}
+
+
+
+
+
+
+ {% if connectors %} @@ -241,11 +270,31 @@

{% trans "Connecteurs" %}

- + +
+
+
+ +
+
+
{% trans "Chiffrement de bout en bout" %}
+
{% trans "Votre secret sera chiffré avec une clé propre à votre organisation (Fernet + HKDF). Il ne sera jamais stocké en clair dans la base de données." %}
+
+
+
+ + +
+ + +
{% trans "La valeur sera chiffrée avant stockage. Laissez vide pour utiliser une variable d'environnement." %}
+
+ +
- + -
{% trans "Nom de la variable d'environnement contenant la clé API ou le secret." %}
+
{% trans "Utilisée uniquement si aucun secret n'est saisi ci-dessus." %}
diff --git a/connectors/templates/connectors/create.html b/connectors/templates/connectors/create.html index f88535c..34622b6 100644 --- a/connectors/templates/connectors/create.html +++ b/connectors/templates/connectors/create.html @@ -168,6 +168,55 @@ } .cf-footer-link:hover { text-decoration: underline; } .cf-footer-link svg { width: 14px; height: 14px; } + + /* Security banner (inline) */ + .sec-banner-inline { + position: relative; + background: linear-gradient(135deg, rgba(16,185,129,0.06) 0%, rgba(16,185,129,0.02) 100%); + border: 1px solid rgba(16,185,129,0.18); + border-left: 3px solid #10b981; + border-radius: 10px; + padding: 12px 14px; + margin-bottom: 18px; + overflow: hidden; + } + .sec-banner-inline::before { + content: ''; + position: absolute; + top: -30px; right: -30px; + width: 80px; height: 80px; + background: radial-gradient(circle, rgba(16,185,129,0.06) 0%, transparent 70%); + pointer-events: none; + } + .sec-inline-row { + display: flex; + align-items: flex-start; + gap: 10px; + position: relative; + } + .sec-inline-icon { + flex-shrink: 0; + width: 28px; height: 28px; + border-radius: 7px; + background: rgba(16,185,129,0.12); + display: flex; align-items: center; justify-content: center; + color: #10b981; + transition: transform 0.3s cubic-bezier(0.34,1.56,0.64,1); + } + .sec-banner-inline:hover .sec-inline-icon { transform: scale(1.08) rotate(-3deg); } + .sec-inline-icon svg { width: 14px; height: 14px; } + .sec-inline-body { flex: 1; min-width: 0; } + .sec-inline-title { + font-size: 12px; + font-weight: 700; + color: var(--ds-grey-100); + margin-bottom: 1px; + } + .sec-inline-text { + font-size: 11px; + color: var(--ds-text-muted); + line-height: 1.45; + } {% endblock %} @@ -258,11 +307,31 @@ - + +
+
+
+ +
+
+
{% trans "Chiffrement de bout en bout" %}
+
{% trans "Votre secret sera chiffré avec une clé propre à votre organisation (Fernet + HKDF). Il ne sera jamais stocké en clair dans la base de données." %}
+
+
+
+ + +
+ + +
{% trans "La valeur sera chiffrée avant stockage. Laissez vide pour utiliser une variable d'environnement." %}
+
+ +
- + -
{% trans "Nom de la variable d'environnement contenant la clé API ou le secret." %}
+
{% trans "Utilisée uniquement si aucun secret n'est saisi ci-dessus." %}
diff --git a/connectors/templates/connectors/list.html b/connectors/templates/connectors/list.html index 4a92b9f..b977e7c 100644 --- a/connectors/templates/connectors/list.html +++ b/connectors/templates/connectors/list.html @@ -193,6 +193,158 @@ margin-bottom: 16px; } + /* ── Security banner ── */ + .sec-banner { + position: relative; + background: linear-gradient(135deg, rgba(16,185,129,0.06) 0%, rgba(16,185,129,0.02) 100%); + border: 1px solid rgba(16,185,129,0.18); + border-left: 3px solid #10b981; + border-radius: 10px; + padding: 14px 18px; + margin-bottom: 20px; + overflow: hidden; + } + .sec-banner::before { + content: ''; + position: absolute; + top: -40px; right: -40px; + width: 120px; height: 120px; + background: radial-gradient(circle, rgba(16,185,129,0.06) 0%, transparent 70%); + pointer-events: none; + } + .sec-banner-row { + display: flex; + align-items: flex-start; + gap: 12px; + position: relative; + } + .sec-banner-icon { + flex-shrink: 0; + width: 34px; height: 34px; + border-radius: 8px; + background: rgba(16,185,129,0.12); + display: flex; align-items: center; justify-content: center; + color: #10b981; + transition: transform 0.3s cubic-bezier(0.34,1.56,0.64,1); + } + .sec-banner:hover .sec-banner-icon { transform: scale(1.08) rotate(-3deg); } + .sec-banner-icon svg { width: 18px; height: 18px; } + .sec-banner-body { flex: 1; min-width: 0; } + .sec-banner-title { + font-size: 13px; + font-weight: 700; + color: var(--ds-grey-100); + margin-bottom: 2px; + letter-spacing: 0.01em; + } + .sec-banner-text { + font-size: 12px; + color: var(--ds-text-muted); + line-height: 1.5; + } + .sec-banner-toggle { + background: none; border: none; padding: 0; + color: #10b981; + font-size: 12px; + font-weight: 600; + cursor: pointer; + display: inline-flex; align-items: center; gap: 4px; + transition: color 0.15s; + } + .sec-banner-toggle:hover { color: #34d399; } + .sec-banner-toggle svg { + width: 12px; height: 12px; + transition: transform 0.25s ease; + } + .sec-banner-toggle[aria-expanded="true"] svg { transform: rotate(180deg); } + .sec-banner-details { + display: grid; + grid-template-rows: 0fr; + transition: grid-template-rows 0.3s ease, margin-top 0.3s ease; + margin-top: 0; + } + .sec-banner-details[data-open="true"] { + grid-template-rows: 1fr; + margin-top: 10px; + } + .sec-banner-details-inner { + overflow: hidden; + } + .sec-banner-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 6px 16px; + padding: 10px 12px; + background: rgba(16,185,129,0.04); + border: 1px solid rgba(16,185,129,0.10); + border-radius: 8px; + } + @media (max-width: 640px) { .sec-banner-grid { grid-template-columns: 1fr; } } + .sec-banner-item { + display: flex; + align-items: center; + gap: 8px; + font-size: 11.5px; + color: var(--ds-text-muted); + line-height: 1.4; + } + .sec-banner-dot { + flex-shrink: 0; + width: 5px; height: 5px; + border-radius: 50%; + background: #10b981; + opacity: 0.7; + } + + /* ── Security banner (inline — modal) ── */ + .sec-banner-inline { + position: relative; + background: linear-gradient(135deg, rgba(16,185,129,0.06) 0%, rgba(16,185,129,0.02) 100%); + border: 1px solid rgba(16,185,129,0.18); + border-left: 3px solid #10b981; + border-radius: 10px; + padding: 12px 14px; + margin-bottom: 18px; + overflow: hidden; + } + .sec-banner-inline::before { + content: ''; + position: absolute; + top: -30px; right: -30px; + width: 80px; height: 80px; + background: radial-gradient(circle, rgba(16,185,129,0.06) 0%, transparent 70%); + pointer-events: none; + } + .sec-inline-row { + display: flex; + align-items: flex-start; + gap: 10px; + position: relative; + } + .sec-inline-icon { + flex-shrink: 0; + width: 28px; height: 28px; + border-radius: 7px; + background: rgba(16,185,129,0.12); + display: flex; align-items: center; justify-content: center; + color: #10b981; + transition: transform 0.3s cubic-bezier(0.34,1.56,0.64,1); + } + .sec-banner-inline:hover .sec-inline-icon { transform: scale(1.08) rotate(-3deg); } + .sec-inline-icon svg { width: 14px; height: 14px; } + .sec-inline-body { flex: 1; min-width: 0; } + .sec-inline-title { + font-size: 12px; + font-weight: 700; + color: var(--ds-grey-100); + margin-bottom: 1px; + } + .sec-inline-text { + font-size: 11px; + color: var(--ds-text-muted); + line-height: 1.45; + } + /* ── Create modal ── */ #conn-create-modal .modal-dialog { max-width: 520px; diff --git a/connectors/views.py b/connectors/views.py index b3ce9bb..051f261 100644 --- a/connectors/views.py +++ b/connectors/views.py @@ -72,7 +72,7 @@ def connector_create(request): return redirect("connector-list") if request.method == "POST": - ConnectorConfig.objects.create( + connector = ConnectorConfig.objects.create( tenant=request.tenant, project=request.project, name=request.POST["name"], @@ -84,6 +84,10 @@ def connector_create(request): }, credential_ref=request.POST.get("credential_ref", ""), ) + secret_value = request.POST.get("secret_value", "") + if secret_value: + connector.set_secret(secret_value) + connector.save() return redirect("connector-list") return render( diff --git a/docs/INGESTION_AND_ANALYSIS.md b/docs/INGESTION_AND_ANALYSIS.md index d1ada3e..1d60fbe 100644 --- a/docs/INGESTION_AND_ANALYSIS.md +++ b/docs/INGESTION_AND_ANALYSIS.md @@ -126,6 +126,16 @@ class BaseConnector(ABC): Connectors are registered via the `@register_connector("name")` decorator and instantiated at runtime by `get_connector(connector_type, config, credential)`. +#### Credential Handling + +The `credential` parameter passed to connectors is the **decrypted secret value**, resolved by `ConnectorConfig.get_secret()` at pipeline init time. The resolution order is: + +1. **Encrypted secret** — if `encrypted_secret` is set on the model, it is decrypted using the per-tenant Fernet key (derived via HKDF from `FIELD_ENCRYPTION_KEY` + tenant UUID) +2. **Environment variable fallback** — if no encrypted secret exists, `credential_ref` is treated as an env var name and looked up via `os.environ.get()` +3. **Empty string** — if neither is available + +This means connectors receive the actual secret directly and do not need to perform env var lookups themselves. See `connectors/crypto.py` for the encryption implementation. + #### `RawDocument` Dataclass The output of `fetch_document()`: diff --git a/docs/deployment.md b/docs/deployment.md index 2152b2d..39dde9f 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -27,6 +27,7 @@ docker-compose up -d | Variable | Description | |----------|-------------| | `SECRET_KEY` | Cryptographically random key (see above) | +| `FIELD_ENCRYPTION_KEY` | Key for encrypting connector secrets (falls back to `SECRET_KEY`). Generate with: `python -c "import secrets; print(secrets.token_urlsafe(32))"` | | `DEBUG` | **Must be `False` in production** | | `ALLOWED_HOSTS` | Comma-separated list of your domain(s) | | `LLM_PROVIDER` | `openai`, `azure`, or `azure_mistral` | @@ -50,6 +51,36 @@ The sample data script creates default users with weak passwords. **Change these python manage.py createsuperuser ``` +## Migrating Connector Secrets (Upgrading from Pre-Encryption Versions) + +If you are upgrading from a version that used `credential_ref` (env var names) for connector credentials, follow these steps after deploying the new code: + +```bash +# 1. Run the schema migration to add the encrypted_secret column +python manage.py migrate + +# 2. Generate and set FIELD_ENCRYPTION_KEY in .env +python -c "import secrets; print(secrets.token_urlsafe(32))" +# Add the output to .env as FIELD_ENCRYPTION_KEY= + +# 3. Ensure the env vars referenced by your connectors are still set + +# 4. Preview what will be migrated (dry run) +python manage.py migrate_connector_secrets + +# 5. Encrypt the credentials +python manage.py migrate_connector_secrets --apply + +# 6. Verify connectors still work (trigger a sync) + +# 7. Optionally remove env var references (secrets are now in the DB) +python manage.py migrate_connector_secrets --apply --clear-ref +``` + +**Important:** Back up your database before running the migration. The `FIELD_ENCRYPTION_KEY` (or `SECRET_KEY` if no dedicated key is set) is required to decrypt secrets — if you lose it, encrypted credentials cannot be recovered. + +--- + ## Database Considerations ### SQLite (Default) diff --git a/docs/project-structure.md b/docs/project-structure.md index 338fb2c..ac9a1b4 100644 --- a/docs/project-structure.md +++ b/docs/project-structure.md @@ -45,8 +45,9 @@ score/ │ └── templates/chat/ # home (chat UI) │ ├── connectors/ # Document source connectors -│ ├── models.py # ConnectorConfig +│ ├── models.py # ConnectorConfig (with encrypted_secret, set/get_secret) │ ├── base.py # BaseConnector ABC, RawDocument, connector registry +│ ├── crypto.py # Per-tenant Fernet encryption (HKDF key derivation) │ ├── generic.py # Filesystem + HTTP connector │ ├── sharepoint.py # SharePoint Online connector (optional dep) │ ├── confluence.py # Confluence connector (optional dep) diff --git a/docs/technical-reference.md b/docs/technical-reference.md index 3a9d3b5..65a22d9 100644 --- a/docs/technical-reference.md +++ b/docs/technical-reference.md @@ -76,7 +76,7 @@ Configuration is controlled by `CELERY_BROKER_BACKEND` in `.env`. Result backend |--------------|------------------------------------------------------------------| | `score` | Django project root: settings, scoring engine, CSP middleware, health check, rate limiting, utilities | | `tenants` | Multi-tenant system: Tenant, Membership, role-based access | -| `connectors` | Document source connectors with registry pattern | +| `connectors` | Document source connectors with registry pattern, per-tenant encrypted secrets | | `ingestion` | Ingestion pipeline: fetch, extract, chunk, embed, store | | `vectorstore`| sqlite-vec vector storage, KNN search, tenant-scoped queries | | `analysis` | Duplicate, contradiction, clustering, gap, hallucination detection + RAG audit | @@ -97,7 +97,7 @@ Configuration is controlled by `CELERY_BROKER_BACKEND` in `.env`. Result backend - `TenantMembership` — tenant, user, role (admin/editor/viewer) **Connectors:** -- `ConnectorConfig` — tenant, name, connector_type (sharepoint/confluence/generic), config (JSON), credential_ref, schedule_cron, last_sync_at/status +- `ConnectorConfig` — tenant, name, connector_type (sharepoint/confluence/generic), config (JSON), credential_ref, encrypted_secret (Fernet-encrypted, per-tenant key), schedule_cron, last_sync_at/status **Ingestion:** - `Document` — tenant, connector, source_id, title, author, doc_type, content_hash, source_version, version_number, status (PENDING → INGESTED → READY / ERROR / DELETED), word_count, chunk_count @@ -279,6 +279,7 @@ Tenant isolation is enforced via post-filtering on metadata tables after KNN ret - All data models inherit from `TenantScopedModel` (or `ProjectScopedModel`), which adds a foreign key to `Tenant` and a custom manager that filters by the current tenant. - Roles: **admin** (full access + settings), **editor** (create/sync connectors, run analysis), **viewer** (read-only dashboards and reports). - **Content Security Policy** middleware adds CSP headers to all responses. +- **Connector secrets**: Per-tenant encryption using HKDF-derived Fernet keys. Secrets entered in the UI are encrypted before storage; `get_secret()` decrypts at sync time or falls back to env var lookup via `credential_ref`. - **Production hardening**: HSTS, SSL redirect, secure cookies, full password validators, `SECRET_KEY` validation. --- diff --git a/ingestion/pipeline.py b/ingestion/pipeline.py index 09aec62..1b562ff 100644 --- a/ingestion/pipeline.py +++ b/ingestion/pipeline.py @@ -37,7 +37,7 @@ def __init__(self, job: IngestionJob): self.connector: BaseConnector = get_connector( self.connector_config.connector_type, self.connector_config.config, - self.connector_config.credential_ref, + self.connector_config.get_secret(), ) self.llm = get_llm_client() self.vec_store = get_vector_store() diff --git a/pyproject.toml b/pyproject.toml index 2c0577d..f1b350f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,8 @@ dependencies = [ "python-docx>=1.1", "python-pptx>=0.6", "markdown>=3.6", + # Encryption + "cryptography>=42.0", # Config "pyyaml>=6.0", # HTTP diff --git a/requirements.txt b/requirements.txt index 22492e6..583883b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,6 +39,9 @@ python-docx>=1.1 python-pptx>=0.6 markdown>=3.6 +# Encryption +cryptography>=42.0 + # Config pyyaml>=6.0 diff --git a/score/settings.py b/score/settings.py index 4f8e0c5..fc94fd1 100644 --- a/score/settings.py +++ b/score/settings.py @@ -32,6 +32,7 @@ APP_CONFIG = yaml.safe_load(f) or {} SECRET_KEY = env("SECRET_KEY") +FIELD_ENCRYPTION_KEY = env("FIELD_ENCRYPTION_KEY", default="") DEBUG = env("DEBUG") ALLOWED_HOSTS = env("ALLOWED_HOSTS") diff --git a/tests/test_connector_crypto.py b/tests/test_connector_crypto.py new file mode 100644 index 0000000..1b8ba38 --- /dev/null +++ b/tests/test_connector_crypto.py @@ -0,0 +1,169 @@ +"""Tests for per-tenant connector secret encryption.""" + +import uuid + +import pytest + +from connectors.crypto import decrypt_secret, encrypt_secret + + +# --------------------------------------------------------------------------- +# Pure crypto tests +# --------------------------------------------------------------------------- + + +class TestEncryptDecrypt: + def test_roundtrip(self, settings): + settings.FIELD_ENCRYPTION_KEY = "test-master-key-for-unit-tests" + tenant_id = str(uuid.uuid4()) + secret = "my-super-secret-api-key" + + encrypted = encrypt_secret(secret, tenant_id) + assert encrypted != secret + assert decrypt_secret(encrypted, tenant_id) == secret + + def test_empty_string_returns_empty(self, settings): + settings.FIELD_ENCRYPTION_KEY = "test-key" + tenant_id = str(uuid.uuid4()) + + assert encrypt_secret("", tenant_id) == "" + assert decrypt_secret("", tenant_id) == "" + + def test_different_tenants_produce_different_ciphertext(self, settings): + settings.FIELD_ENCRYPTION_KEY = "test-key" + secret = "same-secret" + t1 = str(uuid.uuid4()) + t2 = str(uuid.uuid4()) + + enc1 = encrypt_secret(secret, t1) + enc2 = encrypt_secret(secret, t2) + assert enc1 != enc2 + + def test_wrong_tenant_cannot_decrypt(self, settings): + settings.FIELD_ENCRYPTION_KEY = "test-key" + tenant_id = str(uuid.uuid4()) + wrong_tenant = str(uuid.uuid4()) + secret = "sensitive-value" + + encrypted = encrypt_secret(secret, tenant_id) + result = decrypt_secret(encrypted, wrong_tenant) + assert result == "" # graceful failure + + def test_falls_back_to_secret_key(self, settings): + settings.FIELD_ENCRYPTION_KEY = "" + settings.SECRET_KEY = "django-fallback-secret" + tenant_id = str(uuid.uuid4()) + secret = "fallback-test" + + encrypted = encrypt_secret(secret, tenant_id) + assert decrypt_secret(encrypted, tenant_id) == secret + + def test_corrupted_ciphertext_returns_empty(self, settings): + settings.FIELD_ENCRYPTION_KEY = "test-key" + tenant_id = str(uuid.uuid4()) + + result = decrypt_secret("not-valid-ciphertext", tenant_id) + assert result == "" + + +# --------------------------------------------------------------------------- +# Model integration tests +# --------------------------------------------------------------------------- + + +@pytest.mark.django_db +class TestConnectorConfigSecrets: + def test_set_and_get_secret(self, settings, tenant, project): + settings.FIELD_ENCRYPTION_KEY = "test-model-key" + from connectors.models import ConnectorConfig + + connector = ConnectorConfig.objects.create( + tenant=tenant, + project=project, + name="Secret Test", + connector_type="generic", + ) + connector.set_secret("my-api-token") + connector.save() + + # Reload from DB + connector.refresh_from_db() + assert connector.encrypted_secret != "" + assert connector.encrypted_secret != "my-api-token" + assert connector.get_secret() == "my-api-token" + + def test_env_var_fallback(self, settings, tenant, project, monkeypatch): + settings.FIELD_ENCRYPTION_KEY = "test-model-key" + monkeypatch.setenv("MY_TEST_SECRET", "env-secret-value") + from connectors.models import ConnectorConfig + + connector = ConnectorConfig.objects.create( + tenant=tenant, + project=project, + name="Env Fallback", + connector_type="generic", + credential_ref="MY_TEST_SECRET", + ) + # No encrypted_secret set — should fall back to env var + assert connector.get_secret() == "env-secret-value" + + def test_encrypted_secret_preferred_over_env(self, settings, tenant, project, monkeypatch): + settings.FIELD_ENCRYPTION_KEY = "test-model-key" + monkeypatch.setenv("MY_TEST_SECRET", "env-value") + from connectors.models import ConnectorConfig + + connector = ConnectorConfig.objects.create( + tenant=tenant, + project=project, + name="Prefer Encrypted", + connector_type="generic", + credential_ref="MY_TEST_SECRET", + ) + connector.set_secret("encrypted-value") + connector.save() + connector.refresh_from_db() + + assert connector.get_secret() == "encrypted-value" + + def test_cross_tenant_isolation(self, settings, db): + settings.FIELD_ENCRYPTION_KEY = "test-isolation-key" + from connectors.models import ConnectorConfig + from tenants.models import Project, Tenant + + t1 = Tenant.objects.create(name="Tenant A", slug="tenant-a") + t2 = Tenant.objects.create(name="Tenant B", slug="tenant-b") + p1 = Project.objects.create(tenant=t1, name="P1", slug="p1") + p2 = Project.objects.create(tenant=t2, name="P2", slug="p2") + + c1 = ConnectorConfig.objects.create( + tenant=t1, project=p1, name="C1", connector_type="generic" + ) + c2 = ConnectorConfig.objects.create( + tenant=t2, project=p2, name="C2", connector_type="generic" + ) + + c1.set_secret("shared-secret") + c1.save() + c2.set_secret("shared-secret") + c2.save() + + c1.refresh_from_db() + c2.refresh_from_db() + + # Same plaintext but different ciphertext (different tenant keys) + assert c1.encrypted_secret != c2.encrypted_secret + # Each can decrypt its own + assert c1.get_secret() == "shared-secret" + assert c2.get_secret() == "shared-secret" + + def test_no_secret_returns_empty(self, settings, tenant, project): + settings.FIELD_ENCRYPTION_KEY = "test-key" + from connectors.models import ConnectorConfig + + connector = ConnectorConfig.objects.create( + tenant=tenant, + project=project, + name="No Secret", + connector_type="generic", + ) + assert connector.get_secret() == "" diff --git a/tests/test_migrate_connector_secrets.py b/tests/test_migrate_connector_secrets.py new file mode 100644 index 0000000..d84f1e7 --- /dev/null +++ b/tests/test_migrate_connector_secrets.py @@ -0,0 +1,116 @@ +"""Tests for the migrate_connector_secrets management command.""" + +import pytest +from django.core.management import call_command +from io import StringIO + +from connectors.models import ConnectorConfig + + +@pytest.mark.django_db +class TestMigrateConnectorSecrets: + def test_dry_run_no_connectors(self, tenant, project): + out = StringIO() + call_command("migrate_connector_secrets", stdout=out) + assert "Nothing to migrate" in out.getvalue() + + def test_dry_run_shows_candidates(self, settings, tenant, project, monkeypatch): + settings.FIELD_ENCRYPTION_KEY = "test-migration-key" + monkeypatch.setenv("SP_SECRET", "my-sharepoint-secret") + + ConnectorConfig.objects.create( + tenant=tenant, + project=project, + name="Old SP", + connector_type="sharepoint", + credential_ref="SP_SECRET", + ) + + out = StringIO() + call_command("migrate_connector_secrets", stdout=out) + output = out.getvalue() + assert "WOULD MIGRATE" in output + assert "my-***" in output # masked value + + # Verify nothing was actually changed + connector = ConnectorConfig.objects.get(name="Old SP") + assert connector.encrypted_secret == "" + assert connector.credential_ref == "SP_SECRET" + + def test_apply_encrypts_secrets(self, settings, tenant, project, monkeypatch): + settings.FIELD_ENCRYPTION_KEY = "test-migration-key" + monkeypatch.setenv("CONF_TOKEN", "confluence-api-token") + + ConnectorConfig.objects.create( + tenant=tenant, + project=project, + name="Old Confluence", + connector_type="confluence", + credential_ref="CONF_TOKEN", + ) + + out = StringIO() + call_command("migrate_connector_secrets", "--apply", stdout=out) + output = out.getvalue() + assert "MIGRATED" in output + + connector = ConnectorConfig.objects.get(name="Old Confluence") + assert connector.encrypted_secret != "" + assert connector.get_secret() == "confluence-api-token" + # credential_ref preserved by default + assert connector.credential_ref == "CONF_TOKEN" + + def test_apply_with_clear_ref(self, settings, tenant, project, monkeypatch): + settings.FIELD_ENCRYPTION_KEY = "test-migration-key" + monkeypatch.setenv("MY_KEY", "secret-value") + + ConnectorConfig.objects.create( + tenant=tenant, + project=project, + name="Clear Ref", + connector_type="generic", + credential_ref="MY_KEY", + ) + + out = StringIO() + call_command("migrate_connector_secrets", "--apply", "--clear-ref", stdout=out) + + connector = ConnectorConfig.objects.get(name="Clear Ref") + assert connector.encrypted_secret != "" + assert connector.credential_ref == "" + assert connector.get_secret() == "secret-value" + + def test_skips_unset_env_vars(self, settings, tenant, project): + settings.FIELD_ENCRYPTION_KEY = "test-migration-key" + + ConnectorConfig.objects.create( + tenant=tenant, + project=project, + name="Missing Env", + connector_type="generic", + credential_ref="NONEXISTENT_VAR", + ) + + out = StringIO() + call_command("migrate_connector_secrets", "--apply", stdout=out) + output = out.getvalue() + assert "SKIPPED" in output + assert "empty or unset" in output + + def test_skips_already_encrypted(self, settings, tenant, project, monkeypatch): + settings.FIELD_ENCRYPTION_KEY = "test-migration-key" + monkeypatch.setenv("SOME_KEY", "some-value") + + connector = ConnectorConfig.objects.create( + tenant=tenant, + project=project, + name="Already Done", + connector_type="generic", + credential_ref="SOME_KEY", + ) + connector.set_secret("already-encrypted") + connector.save() + + out = StringIO() + call_command("migrate_connector_secrets", stdout=out) + assert "Nothing to migrate" in out.getvalue()