diff --git a/.gitignore b/.gitignore index b69a35d..bd967a6 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,6 @@ dpaste/static/ dpaste.egg-info dpaste.sqlite node_modules -**/__pycache__/ \ No newline at end of file +**/__pycache__/ +# Generated monitoring target (changes with the EC2 IP) +monitoring/prometheus/targets.d/*.yml diff --git a/monitoring/blackbox/blackbox.yml b/monitoring/blackbox/blackbox.yml new file mode 100644 index 0000000..ec077e0 --- /dev/null +++ b/monitoring/blackbox/blackbox.yml @@ -0,0 +1,9 @@ +modules: + http_2xx: + prober: http + timeout: 10s + http: + method: GET + valid_status_codes: [200] + follow_redirects: true + preferred_ip_protocol: ip4 \ No newline at end of file diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml new file mode 100644 index 0000000..dcd3a7a --- /dev/null +++ b/monitoring/docker-compose.yml @@ -0,0 +1,71 @@ +# CloudPulse - local observability stack +# Probes the local container AND the live AWS deployment (black-box monitoring: +# no changes to the dpaste application code). +# Usage (from monitoring/): +# powershell -ExecutionPolicy Bypass -File set-production-target.ps1 +# docker compose up -d --build +# Grafana: http://localhost:3000 Prometheus: http://localhost:9090 +name: cloudpulse-monitoring + +services: + dpaste: + build: + context: .. + dockerfile: Dockerfile.hardened + image: cloudpulse:local + container_name: cloudpulse-dpaste + environment: + SECRET_KEY: local-monitoring-demo-not-a-secret + DEBUG: "False" + ALLOWED_HOSTS: localhost,127.0.0.1,dpaste + ports: + - "127.0.0.1:8000:8000" + volumes: + - dpaste_data:/data + restart: unless-stopped + + blackbox: + image: prom/blackbox-exporter:v0.25.0 + container_name: cloudpulse-blackbox + command: ["--config.file=/etc/blackbox/blackbox.yml"] + volumes: + - ./blackbox/blackbox.yml:/etc/blackbox/blackbox.yml:ro + restart: unless-stopped + + + prometheus: + image: prom/prometheus:v3.2.1 + container_name: cloudpulse-prometheus + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.retention.time=7d + volumes: + - ./prometheus:/etc/prometheus:ro + - prometheus_data:/prometheus + ports: + - "127.0.0.1:9090:9090" + restart: unless-stopped + + grafana: + image: grafana/grafana:11.6.0 + container_name: cloudpulse-grafana + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-cloudpulse-local} + GF_USERS_ALLOW_SIGN_UP: "false" + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana_data:/var/lib/grafana + # AWS Academy session credentials, read-only, for the CloudWatch datasource + - ${USERPROFILE}/.aws:/usr/share/grafana/.aws:ro + ports: + - "127.0.0.1:3000:3000" + depends_on: + - prometheus + restart: unless-stopped + +volumes: + dpaste_data: + prometheus_data: + grafana_data: \ No newline at end of file diff --git a/monitoring/grafana/dashboards/cloudpulse-overview.json b/monitoring/grafana/dashboards/cloudpulse-overview.json new file mode 100644 index 0000000..9b82f3c --- /dev/null +++ b/monitoring/grafana/dashboards/cloudpulse-overview.json @@ -0,0 +1,111 @@ +{ + "uid": "cloudpulse-overview", + "title": "CloudPulse - dpaste overview", + "tags": ["cloudpulse"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { "from": "now-1h", "to": "now" }, + "panels": [ + { + "id": 1, "type": "stat", "title": "Production", + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "refId": "A", "expr": "probe_success{env=\"production\"}" }], + "fieldConfig": { "defaults": { + "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }], + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } + }, "overrides": [] }, + "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } } + }, + { + "id": 2, "type": "stat", "title": "Local", + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "refId": "A", "expr": "probe_success{env=\"local\"}" }], + "fieldConfig": { "defaults": { + "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }], + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } + }, "overrides": [] }, + "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } } + }, + { + "id": 3, "type": "stat", "title": "Production HTTP status", + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "refId": "A", "expr": "probe_http_status_code{env=\"production\"}" }], + "fieldConfig": { "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 200 }, { "color": "red", "value": 300 }] } + }, "overrides": [] }, + "options": { "colorMode": "value", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } } + }, + { + "id": 4, "type": "stat", "title": "TLS certificate expires in", + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "refId": "A", "expr": "(probe_ssl_earliest_cert_expiry{env=\"production\"} - time()) / 86400" }], + "fieldConfig": { "defaults": { + "unit": "d", "decimals": 1, + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "orange", "value": 14 }, { "color": "green", "value": 30 }] } + }, "overrides": [] }, + "options": { "colorMode": "value", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } } + }, + { + "id": 5, "type": "stat", "title": "Firing alerts", + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "refId": "A", "expr": "count(ALERTS{alertstate=\"firing\"}) or vector(0)" }], + "fieldConfig": { "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] } + }, "overrides": [] }, + "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } } + }, + { + "id": 6, "type": "timeseries", "title": "Response time (full request)", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "refId": "A", "expr": "probe_duration_seconds", "legendFormat": "{{env}}" }], + "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] } + }, + { + "id": 7, "type": "timeseries", "title": "Production request phases (DNS, connect, TLS, processing, transfer)", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "refId": "A", "expr": "probe_http_duration_seconds{env=\"production\"}", "legendFormat": "{{phase}}" }], + "fieldConfig": { "defaults": { "unit": "s", "custom": { "stacking": { "mode": "normal" }, "fillOpacity": 30 } }, "overrides": [] } + }, + { + "id": 8, "type": "timeseries", "title": "Production EC2 CPU (CloudWatch, 1-minute detailed monitoring)", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, + "datasource": { "type": "cloudwatch", "uid": "cloudwatch" }, + "targets": [{ + "refId": "A", "datasource": { "type": "cloudwatch", "uid": "cloudwatch" }, + "queryMode": "Metrics", "region": "default", "namespace": "AWS/EC2", "metricName": "CPUUtilization", + "dimensions": { "InstanceId": "*" }, "matchExact": false, "statistic": "Average", "period": "60", + "metricQueryType": 0, "metricEditorMode": 0, "id": "", "expression": "" + }], + "fieldConfig": { "defaults": { "unit": "percent", "min": 0 }, "overrides": [] } + }, + { + "id": 9, "type": "timeseries", "title": "Production EC2 network (CloudWatch)", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, + "datasource": { "type": "cloudwatch", "uid": "cloudwatch" }, + "targets": [ + { + "refId": "A", "datasource": { "type": "cloudwatch", "uid": "cloudwatch" }, + "queryMode": "Metrics", "region": "default", "namespace": "AWS/EC2", "metricName": "NetworkIn", + "dimensions": { "InstanceId": "*" }, "matchExact": false, "statistic": "Sum", "period": "60", + "metricQueryType": 0, "metricEditorMode": 0, "id": "", "expression": "" + }, + { + "refId": "B", "datasource": { "type": "cloudwatch", "uid": "cloudwatch" }, + "queryMode": "Metrics", "region": "default", "namespace": "AWS/EC2", "metricName": "NetworkOut", + "dimensions": { "InstanceId": "*" }, "matchExact": false, "statistic": "Sum", "period": "60", + "metricQueryType": 0, "metricEditorMode": 0, "id": "", "expression": "" + } + ], + "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] } + } + ] +} \ No newline at end of file diff --git a/monitoring/grafana/provisioning/dashboards/cloudpulse.yml b/monitoring/grafana/provisioning/dashboards/cloudpulse.yml new file mode 100644 index 0000000..3113b4b --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/cloudpulse.yml @@ -0,0 +1,8 @@ +apiVersion: 1 +providers: + - name: CloudPulse + folder: CloudPulse + type: file + disableDeletion: true + options: + path: /var/lib/grafana/dashboards \ No newline at end of file diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..13b4ee8 --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,16 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + + # Uses the AWS SDK default credential chain (~/.aws mounted read-only) + - name: CloudWatch + uid: cloudwatch + type: cloudwatch + jsonData: + authType: default + defaultRegion: us-east-1 \ No newline at end of file diff --git a/monitoring/prometheus/alerts.yml b/monitoring/prometheus/alerts.yml new file mode 100644 index 0000000..81b99b4 --- /dev/null +++ b/monitoring/prometheus/alerts.yml @@ -0,0 +1,29 @@ +groups: + - name: cloudpulse + rules: + - alert: DpasteDown + expr: probe_success == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "dpaste is down ({{ $labels.env }})" + description: "HTTP probe of {{ $labels.instance }} has been failing for more than 1 minute." + + - alert: DpasteSlowResponse + expr: probe_duration_seconds > 2 + for: 5m + labels: + severity: warning + annotations: + summary: "dpaste is slow ({{ $labels.env }})" + description: "Probe of {{ $labels.instance }} takes more than 2s for 5 minutes." + + - alert: TLSCertificateExpiringSoon + expr: (probe_ssl_earliest_cert_expiry - time()) / 86400 < 14 + for: 10m + labels: + severity: warning + annotations: + summary: "TLS certificate expires in less than 14 days" + description: "Certificate for {{ $labels.instance }} expires soon; Caddy should have renewed it." \ No newline at end of file diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000..299e592 --- /dev/null +++ b/monitoring/prometheus/prometheus.yml @@ -0,0 +1,35 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +rule_files: + - /etc/prometheus/alerts.yml + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + + # Black-box HTTP probes: local container (static) + production (file-based, + # written by set-production-target.ps1 because the sslip.io hostname changes with the IP) + - job_name: blackbox-http + metrics_path: /probe + params: + module: [http_2xx] + static_configs: + - targets: ["http://dpaste:8000/"] + labels: + env: local + file_sd_configs: + - files: ["/etc/prometheus/targets.d/*.yml"] + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox:9115 + + - job_name: blackbox-exporter + static_configs: + - targets: ["blackbox:9115"] diff --git a/monitoring/prometheus/targets.d/.gitkeep b/monitoring/prometheus/targets.d/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/monitoring/set-production-target.ps1 b/monitoring/set-production-target.ps1 new file mode 100644 index 0000000..b617108 --- /dev/null +++ b/monitoring/set-production-target.ps1 @@ -0,0 +1,9 @@ +# Point the black-box probe at the current production URL. +# The sslip.io hostname is derived from the EC2 public IP, which changes when the +# AWS Academy session restarts, so the target is generated from Terraform output. +$url = (terraform -chdir="$PSScriptRoot\..\terraform" output -raw app_url) +if ($LASTEXITCODE -ne 0 -or -not $url) { throw "Could not read app_url from Terraform (are AWS credentials valid?)" } +$yml = "- targets: [`"$url/`"]`n labels:`n env: production`n" +New-Item -ItemType Directory -Force "$PSScriptRoot\prometheus\targets.d" | Out-Null +[System.IO.File]::WriteAllText("$PSScriptRoot\prometheus\targets.d\production.yml", $yml) +Write-Host "Production probe target: $url/" \ No newline at end of file