From a889e45dca58e85e5c199f4a4605a9104ee508f9 Mon Sep 17 00:00:00 2001 From: Rayen Mabrouk Date: Wed, 23 Sep 2026 21:54:57 +0100 Subject: [PATCH] Add Prometheus + Grafana monitoring stack with black-box probes and CloudWatch panels Blackbox exporter probes the local container and the live AWS deployment (availability, latency, HTTP status, TLS expiry, request phases) without changing dpaste code. Prometheus evaluates DpasteDown, DpasteSlowResponse and TLSCertificateExpiringSoon. Grafana datasources and dashboard are provisioned from files; production EC2 CPU and network come from CloudWatch. cAdvisor evaluated and removed: incompatible with Docker Desktop's containerd image store (cannot resolve container layer IDs). --- .gitignore | 4 +- monitoring/blackbox/blackbox.yml | 9 ++ monitoring/docker-compose.yml | 71 +++++++++++ .../dashboards/cloudpulse-overview.json | 111 ++++++++++++++++++ .../provisioning/dashboards/cloudpulse.yml | 8 ++ .../provisioning/datasources/prometheus.yml | 16 +++ monitoring/prometheus/alerts.yml | 29 +++++ monitoring/prometheus/prometheus.yml | 35 ++++++ monitoring/prometheus/targets.d/.gitkeep | 0 monitoring/set-production-target.ps1 | 9 ++ 10 files changed, 291 insertions(+), 1 deletion(-) create mode 100644 monitoring/blackbox/blackbox.yml create mode 100644 monitoring/docker-compose.yml create mode 100644 monitoring/grafana/dashboards/cloudpulse-overview.json create mode 100644 monitoring/grafana/provisioning/dashboards/cloudpulse.yml create mode 100644 monitoring/grafana/provisioning/datasources/prometheus.yml create mode 100644 monitoring/prometheus/alerts.yml create mode 100644 monitoring/prometheus/prometheus.yml create mode 100644 monitoring/prometheus/targets.d/.gitkeep create mode 100644 monitoring/set-production-target.ps1 diff --git a/.gitignore b/.gitignore index b69a35d..bd967a6 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,6 @@ dpaste/static/ dpaste.egg-info dpaste.sqlite node_modules -**/__pycache__/ \ No newline at end of file +**/__pycache__/ +# Generated monitoring target (changes with the EC2 IP) +monitoring/prometheus/targets.d/*.yml diff --git a/monitoring/blackbox/blackbox.yml b/monitoring/blackbox/blackbox.yml new file mode 100644 index 0000000..ec077e0 --- /dev/null +++ b/monitoring/blackbox/blackbox.yml @@ -0,0 +1,9 @@ +modules: + http_2xx: + prober: http + timeout: 10s + http: + method: GET + valid_status_codes: [200] + follow_redirects: true + preferred_ip_protocol: ip4 \ No newline at end of file diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml new file mode 100644 index 0000000..dcd3a7a --- /dev/null +++ b/monitoring/docker-compose.yml @@ -0,0 +1,71 @@ +# CloudPulse - local observability stack +# Probes the local container AND the live AWS deployment (black-box monitoring: +# no changes to the dpaste application code). +# Usage (from monitoring/): +# powershell -ExecutionPolicy Bypass -File set-production-target.ps1 +# docker compose up -d --build +# Grafana: http://localhost:3000 Prometheus: http://localhost:9090 +name: cloudpulse-monitoring + +services: + dpaste: + build: + context: .. + dockerfile: Dockerfile.hardened + image: cloudpulse:local + container_name: cloudpulse-dpaste + environment: + SECRET_KEY: local-monitoring-demo-not-a-secret + DEBUG: "False" + ALLOWED_HOSTS: localhost,127.0.0.1,dpaste + ports: + - "127.0.0.1:8000:8000" + volumes: + - dpaste_data:/data + restart: unless-stopped + + blackbox: + image: prom/blackbox-exporter:v0.25.0 + container_name: cloudpulse-blackbox + command: ["--config.file=/etc/blackbox/blackbox.yml"] + volumes: + - ./blackbox/blackbox.yml:/etc/blackbox/blackbox.yml:ro + restart: unless-stopped + + + prometheus: + image: prom/prometheus:v3.2.1 + container_name: cloudpulse-prometheus + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.retention.time=7d + volumes: + - ./prometheus:/etc/prometheus:ro + - prometheus_data:/prometheus + ports: + - "127.0.0.1:9090:9090" + restart: unless-stopped + + grafana: + image: grafana/grafana:11.6.0 + container_name: cloudpulse-grafana + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-cloudpulse-local} + GF_USERS_ALLOW_SIGN_UP: "false" + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana_data:/var/lib/grafana + # AWS Academy session credentials, read-only, for the CloudWatch datasource + - ${USERPROFILE}/.aws:/usr/share/grafana/.aws:ro + ports: + - "127.0.0.1:3000:3000" + depends_on: + - prometheus + restart: unless-stopped + +volumes: + dpaste_data: + prometheus_data: + grafana_data: \ No newline at end of file diff --git a/monitoring/grafana/dashboards/cloudpulse-overview.json b/monitoring/grafana/dashboards/cloudpulse-overview.json new file mode 100644 index 0000000..9b82f3c --- /dev/null +++ b/monitoring/grafana/dashboards/cloudpulse-overview.json @@ -0,0 +1,111 @@ +{ + "uid": "cloudpulse-overview", + "title": "CloudPulse - dpaste overview", + "tags": ["cloudpulse"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { "from": "now-1h", "to": "now" }, + "panels": [ + { + "id": 1, "type": "stat", "title": "Production", + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "refId": "A", "expr": "probe_success{env=\"production\"}" }], + "fieldConfig": { "defaults": { + "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }], + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } + }, "overrides": [] }, + "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } } + }, + { + "id": 2, "type": "stat", "title": "Local", + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "refId": "A", "expr": "probe_success{env=\"local\"}" }], + "fieldConfig": { "defaults": { + "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }], + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } + }, "overrides": [] }, + "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } } + }, + { + "id": 3, "type": "stat", "title": "Production HTTP status", + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "refId": "A", "expr": "probe_http_status_code{env=\"production\"}" }], + "fieldConfig": { "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 200 }, { "color": "red", "value": 300 }] } + }, "overrides": [] }, + "options": { "colorMode": "value", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } } + }, + { + "id": 4, "type": "stat", "title": "TLS certificate expires in", + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "refId": "A", "expr": "(probe_ssl_earliest_cert_expiry{env=\"production\"} - time()) / 86400" }], + "fieldConfig": { "defaults": { + "unit": "d", "decimals": 1, + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "orange", "value": 14 }, { "color": "green", "value": 30 }] } + }, "overrides": [] }, + "options": { "colorMode": "value", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } } + }, + { + "id": 5, "type": "stat", "title": "Firing alerts", + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "refId": "A", "expr": "count(ALERTS{alertstate=\"firing\"}) or vector(0)" }], + "fieldConfig": { "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] } + }, "overrides": [] }, + "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } } + }, + { + "id": 6, "type": "timeseries", "title": "Response time (full request)", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "refId": "A", "expr": "probe_duration_seconds", "legendFormat": "{{env}}" }], + "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] } + }, + { + "id": 7, "type": "timeseries", "title": "Production request phases (DNS, connect, TLS, processing, transfer)", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "refId": "A", "expr": "probe_http_duration_seconds{env=\"production\"}", "legendFormat": "{{phase}}" }], + "fieldConfig": { "defaults": { "unit": "s", "custom": { "stacking": { "mode": "normal" }, "fillOpacity": 30 } }, "overrides": [] } + }, + { + "id": 8, "type": "timeseries", "title": "Production EC2 CPU (CloudWatch, 1-minute detailed monitoring)", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, + "datasource": { "type": "cloudwatch", "uid": "cloudwatch" }, + "targets": [{ + "refId": "A", "datasource": { "type": "cloudwatch", "uid": "cloudwatch" }, + "queryMode": "Metrics", "region": "default", "namespace": "AWS/EC2", "metricName": "CPUUtilization", + "dimensions": { "InstanceId": "*" }, "matchExact": false, "statistic": "Average", "period": "60", + "metricQueryType": 0, "metricEditorMode": 0, "id": "", "expression": "" + }], + "fieldConfig": { "defaults": { "unit": "percent", "min": 0 }, "overrides": [] } + }, + { + "id": 9, "type": "timeseries", "title": "Production EC2 network (CloudWatch)", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, + "datasource": { "type": "cloudwatch", "uid": "cloudwatch" }, + "targets": [ + { + "refId": "A", "datasource": { "type": "cloudwatch", "uid": "cloudwatch" }, + "queryMode": "Metrics", "region": "default", "namespace": "AWS/EC2", "metricName": "NetworkIn", + "dimensions": { "InstanceId": "*" }, "matchExact": false, "statistic": "Sum", "period": "60", + "metricQueryType": 0, "metricEditorMode": 0, "id": "", "expression": "" + }, + { + "refId": "B", "datasource": { "type": "cloudwatch", "uid": "cloudwatch" }, + "queryMode": "Metrics", "region": "default", "namespace": "AWS/EC2", "metricName": "NetworkOut", + "dimensions": { "InstanceId": "*" }, "matchExact": false, "statistic": "Sum", "period": "60", + "metricQueryType": 0, "metricEditorMode": 0, "id": "", "expression": "" + } + ], + "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] } + } + ] +} \ No newline at end of file diff --git a/monitoring/grafana/provisioning/dashboards/cloudpulse.yml b/monitoring/grafana/provisioning/dashboards/cloudpulse.yml new file mode 100644 index 0000000..3113b4b --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/cloudpulse.yml @@ -0,0 +1,8 @@ +apiVersion: 1 +providers: + - name: CloudPulse + folder: CloudPulse + type: file + disableDeletion: true + options: + path: /var/lib/grafana/dashboards \ No newline at end of file diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..13b4ee8 --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,16 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + + # Uses the AWS SDK default credential chain (~/.aws mounted read-only) + - name: CloudWatch + uid: cloudwatch + type: cloudwatch + jsonData: + authType: default + defaultRegion: us-east-1 \ No newline at end of file diff --git a/monitoring/prometheus/alerts.yml b/monitoring/prometheus/alerts.yml new file mode 100644 index 0000000..81b99b4 --- /dev/null +++ b/monitoring/prometheus/alerts.yml @@ -0,0 +1,29 @@ +groups: + - name: cloudpulse + rules: + - alert: DpasteDown + expr: probe_success == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "dpaste is down ({{ $labels.env }})" + description: "HTTP probe of {{ $labels.instance }} has been failing for more than 1 minute." + + - alert: DpasteSlowResponse + expr: probe_duration_seconds > 2 + for: 5m + labels: + severity: warning + annotations: + summary: "dpaste is slow ({{ $labels.env }})" + description: "Probe of {{ $labels.instance }} takes more than 2s for 5 minutes." + + - alert: TLSCertificateExpiringSoon + expr: (probe_ssl_earliest_cert_expiry - time()) / 86400 < 14 + for: 10m + labels: + severity: warning + annotations: + summary: "TLS certificate expires in less than 14 days" + description: "Certificate for {{ $labels.instance }} expires soon; Caddy should have renewed it." \ No newline at end of file diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000..299e592 --- /dev/null +++ b/monitoring/prometheus/prometheus.yml @@ -0,0 +1,35 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +rule_files: + - /etc/prometheus/alerts.yml + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + + # Black-box HTTP probes: local container (static) + production (file-based, + # written by set-production-target.ps1 because the sslip.io hostname changes with the IP) + - job_name: blackbox-http + metrics_path: /probe + params: + module: [http_2xx] + static_configs: + - targets: ["http://dpaste:8000/"] + labels: + env: local + file_sd_configs: + - files: ["/etc/prometheus/targets.d/*.yml"] + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox:9115 + + - job_name: blackbox-exporter + static_configs: + - targets: ["blackbox:9115"] diff --git a/monitoring/prometheus/targets.d/.gitkeep b/monitoring/prometheus/targets.d/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/monitoring/set-production-target.ps1 b/monitoring/set-production-target.ps1 new file mode 100644 index 0000000..b617108 --- /dev/null +++ b/monitoring/set-production-target.ps1 @@ -0,0 +1,9 @@ +# Point the black-box probe at the current production URL. +# The sslip.io hostname is derived from the EC2 public IP, which changes when the +# AWS Academy session restarts, so the target is generated from Terraform output. +$url = (terraform -chdir="$PSScriptRoot\..\terraform" output -raw app_url) +if ($LASTEXITCODE -ne 0 -or -not $url) { throw "Could not read app_url from Terraform (are AWS credentials valid?)" } +$yml = "- targets: [`"$url/`"]`n labels:`n env: production`n" +New-Item -ItemType Directory -Force "$PSScriptRoot\prometheus\targets.d" | Out-Null +[System.IO.File]::WriteAllText("$PSScriptRoot\prometheus\targets.d\production.yml", $yml) +Write-Host "Production probe target: $url/" \ No newline at end of file