Monitoring: Prometheus + Grafana (black-box probes + CloudWatch)

.gitignore

@@ -6,4 +6,6 @@ dpaste/static/
 dpaste.egg-info
 dpaste.sqlite
 node_modules
-**/__pycache__/
+**/__pycache__/
+# Generated monitoring target (changes with the EC2 IP)
+monitoring/prometheus/targets.d/*.yml

monitoring/blackbox/blackbox.yml

@@ -0,0 +1,9 @@
+modules:
+  http_2xx:
+    prober: http
+    timeout: 10s
+    http:
+      method: GET
+      valid_status_codes: [200]
+      follow_redirects: true
+      preferred_ip_protocol: ip4

monitoring/docker-compose.yml

@@ -0,0 +1,71 @@
+# CloudPulse - local observability stack
+# Probes the local container AND the live AWS deployment (black-box monitoring:
+# no changes to the dpaste application code).
+# Usage (from monitoring/):
+#   powershell -ExecutionPolicy Bypass -File set-production-target.ps1
+#   docker compose up -d --build
+# Grafana: http://localhost:3000   Prometheus: http://localhost:9090
+name: cloudpulse-monitoring
+
+services:
+  dpaste:
+    build:
+      context: ..
+      dockerfile: Dockerfile.hardened
+    image: cloudpulse:local
+    container_name: cloudpulse-dpaste
+    environment:
+      SECRET_KEY: local-monitoring-demo-not-a-secret
+      DEBUG: "False"
+      ALLOWED_HOSTS: localhost,127.0.0.1,dpaste
+    ports:
+      - "127.0.0.1:8000:8000"
+    volumes:
+      - dpaste_data:/data
+    restart: unless-stopped
+
+  blackbox:
+    image: prom/blackbox-exporter:v0.25.0
+    container_name: cloudpulse-blackbox
+    command: ["--config.file=/etc/blackbox/blackbox.yml"]
+    volumes:
+      - ./blackbox/blackbox.yml:/etc/blackbox/blackbox.yml:ro
+    restart: unless-stopped
+
+
+  prometheus:
+    image: prom/prometheus:v3.2.1
+    container_name: cloudpulse-prometheus
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.retention.time=7d
+    volumes:
+      - ./prometheus:/etc/prometheus:ro
+      - prometheus_data:/prometheus
+    ports:
+      - "127.0.0.1:9090:9090"
+    restart: unless-stopped
+
+  grafana:
+    image: grafana/grafana:11.6.0
+    container_name: cloudpulse-grafana
+    environment:
+      GF_SECURITY_ADMIN_USER: admin
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-cloudpulse-local}
+      GF_USERS_ALLOW_SIGN_UP: "false"
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
+      - grafana_data:/var/lib/grafana
+      # AWS Academy session credentials, read-only, for the CloudWatch datasource
+      - ${USERPROFILE}/.aws:/usr/share/grafana/.aws:ro
+    ports:
+      - "127.0.0.1:3000:3000"
+    depends_on:
+      - prometheus
+    restart: unless-stopped
+
+volumes:
+  dpaste_data:
+  prometheus_data:
+  grafana_data:

monitoring/grafana/dashboards/cloudpulse-overview.json

@@ -0,0 +1,111 @@
+{
+  "uid": "cloudpulse-overview",
+  "title": "CloudPulse - dpaste overview",
+  "tags": ["cloudpulse"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "30s",
+  "time": { "from": "now-1h", "to": "now" },
+  "panels": [
+    {
+      "id": 1, "type": "stat", "title": "Production",
+      "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [{ "refId": "A", "expr": "probe_success{env=\"production\"}" }],
+      "fieldConfig": { "defaults": {
+        "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }],
+        "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }
+      }, "overrides": [] },
+      "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } }
+    },
+    {
+      "id": 2, "type": "stat", "title": "Local",
+      "gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [{ "refId": "A", "expr": "probe_success{env=\"local\"}" }],
+      "fieldConfig": { "defaults": {
+        "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }],
+        "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }
+      }, "overrides": [] },
+      "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } }
+    },
+    {
+      "id": 3, "type": "stat", "title": "Production HTTP status",
+      "gridPos": { "h": 4, "w": 4, "x": 8, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [{ "refId": "A", "expr": "probe_http_status_code{env=\"production\"}" }],
+      "fieldConfig": { "defaults": {
+        "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 200 }, { "color": "red", "value": 300 }] }
+      }, "overrides": [] },
+      "options": { "colorMode": "value", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } }
+    },
+    {
+      "id": 4, "type": "stat", "title": "TLS certificate expires in",
+      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [{ "refId": "A", "expr": "(probe_ssl_earliest_cert_expiry{env=\"production\"} - time()) / 86400" }],
+      "fieldConfig": { "defaults": {
+        "unit": "d", "decimals": 1,
+        "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "orange", "value": 14 }, { "color": "green", "value": 30 }] }
+      }, "overrides": [] },
+      "options": { "colorMode": "value", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } }
+    },
+    {
+      "id": 5, "type": "stat", "title": "Firing alerts",
+      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [{ "refId": "A", "expr": "count(ALERTS{alertstate=\"firing\"}) or vector(0)" }],
+      "fieldConfig": { "defaults": {
+        "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] }
+      }, "overrides": [] },
+      "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } }
+    },
+    {
+      "id": 6, "type": "timeseries", "title": "Response time (full request)",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [{ "refId": "A", "expr": "probe_duration_seconds", "legendFormat": "{{env}}" }],
+      "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] }
+    },
+    {
+      "id": 7, "type": "timeseries", "title": "Production request phases (DNS, connect, TLS, processing, transfer)",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [{ "refId": "A", "expr": "probe_http_duration_seconds{env=\"production\"}", "legendFormat": "{{phase}}" }],
+      "fieldConfig": { "defaults": { "unit": "s", "custom": { "stacking": { "mode": "normal" }, "fillOpacity": 30 } }, "overrides": [] }
+    },
+    {
+      "id": 8, "type": "timeseries", "title": "Production EC2 CPU (CloudWatch, 1-minute detailed monitoring)",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
+      "datasource": { "type": "cloudwatch", "uid": "cloudwatch" },
+      "targets": [{
+        "refId": "A", "datasource": { "type": "cloudwatch", "uid": "cloudwatch" },
+        "queryMode": "Metrics", "region": "default", "namespace": "AWS/EC2", "metricName": "CPUUtilization",
+        "dimensions": { "InstanceId": "*" }, "matchExact": false, "statistic": "Average", "period": "60",
+        "metricQueryType": 0, "metricEditorMode": 0, "id": "", "expression": ""
+      }],
+      "fieldConfig": { "defaults": { "unit": "percent", "min": 0 }, "overrides": [] }
+    },
+    {
+      "id": 9, "type": "timeseries", "title": "Production EC2 network (CloudWatch)",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
+      "datasource": { "type": "cloudwatch", "uid": "cloudwatch" },
+      "targets": [
+        {
+          "refId": "A", "datasource": { "type": "cloudwatch", "uid": "cloudwatch" },
+          "queryMode": "Metrics", "region": "default", "namespace": "AWS/EC2", "metricName": "NetworkIn",
+          "dimensions": { "InstanceId": "*" }, "matchExact": false, "statistic": "Sum", "period": "60",
+          "metricQueryType": 0, "metricEditorMode": 0, "id": "", "expression": ""
+        },
+        {
+          "refId": "B", "datasource": { "type": "cloudwatch", "uid": "cloudwatch" },
+          "queryMode": "Metrics", "region": "default", "namespace": "AWS/EC2", "metricName": "NetworkOut",
+          "dimensions": { "InstanceId": "*" }, "matchExact": false, "statistic": "Sum", "period": "60",
+          "metricQueryType": 0, "metricEditorMode": 0, "id": "", "expression": ""
+        }
+      ],
+      "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }
+    }
+  ]
+}

monitoring/grafana/provisioning/dashboards/cloudpulse.yml

@@ -0,0 +1,8 @@
+apiVersion: 1
+providers:
+  - name: CloudPulse
+    folder: CloudPulse
+    type: file
+    disableDeletion: true
+    options:
+      path: /var/lib/grafana/dashboards

monitoring/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,16 @@
+apiVersion: 1
+datasources:
+  - name: Prometheus
+    uid: prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+
+  # Uses the AWS SDK default credential chain (~/.aws mounted read-only)
+  - name: CloudWatch
+    uid: cloudwatch
+    type: cloudwatch
+    jsonData:
+      authType: default
+      defaultRegion: us-east-1

monitoring/prometheus/alerts.yml

@@ -0,0 +1,29 @@
+groups:
+  - name: cloudpulse
+    rules:
+      - alert: DpasteDown
+        expr: probe_success == 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "dpaste is down ({{ $labels.env }})"
+          description: "HTTP probe of {{ $labels.instance }} has been failing for more than 1 minute."
+
+      - alert: DpasteSlowResponse
+        expr: probe_duration_seconds > 2
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "dpaste is slow ({{ $labels.env }})"
+          description: "Probe of {{ $labels.instance }} takes more than 2s for 5 minutes."
+
+      - alert: TLSCertificateExpiringSoon
+        expr: (probe_ssl_earliest_cert_expiry - time()) / 86400 < 14
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "TLS certificate expires in less than 14 days"
+          description: "Certificate for {{ $labels.instance }} expires soon; Caddy should have renewed it."

monitoring/prometheus/prometheus.yml

@@ -0,0 +1,35 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+rule_files:
+  - /etc/prometheus/alerts.yml
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ["localhost:9090"]
+
+  # Black-box HTTP probes: local container (static) + production (file-based,
+  # written by set-production-target.ps1 because the sslip.io hostname changes with the IP)
+  - job_name: blackbox-http
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    static_configs:
+      - targets: ["http://dpaste:8000/"]
+        labels:
+          env: local
+    file_sd_configs:
+      - files: ["/etc/prometheus/targets.d/*.yml"]
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox:9115
+
+  - job_name: blackbox-exporter
+    static_configs:
+      - targets: ["blackbox:9115"]

monitoring/set-production-target.ps1

@@ -0,0 +1,9 @@
+# Point the black-box probe at the current production URL.
+# The sslip.io hostname is derived from the EC2 public IP, which changes when the
+# AWS Academy session restarts, so the target is generated from Terraform output.
+$url = (terraform -chdir="$PSScriptRoot\..\terraform" output -raw app_url)
+if ($LASTEXITCODE -ne 0 -or -not $url) { throw "Could not read app_url from Terraform (are AWS credentials valid?)" }
+$yml = "- targets: [`"$url/`"]`n  labels:`n    env: production`n"
+New-Item -ItemType Directory -Force "$PSScriptRoot\prometheus\targets.d" | Out-Null
+[System.IO.File]::WriteAllText("$PSScriptRoot\prometheus\targets.d\production.yml", $yml)
+Write-Host "Production probe target: $url/"