From 5988c468abf3b4ff8964418d249a7be0b372419b Mon Sep 17 00:00:00 2001 From: Jacob-Sonne Date: Wed, 29 Apr 2026 18:43:05 +0000 Subject: [PATCH 01/16] Updated cicd-prod.server.yml to copy the new monitoring folder --- .github/workflows/cicd-prod.server.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/cicd-prod.server.yml b/.github/workflows/cicd-prod.server.yml index 153b75b..8544f0d 100644 --- a/.github/workflows/cicd-prod.server.yml +++ b/.github/workflows/cicd-prod.server.yml @@ -32,7 +32,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 - # TODO Get confirmation about this split - name: Build and push minitwitapiimage uses: docker/build-push-action@v7 with: @@ -43,7 +42,6 @@ jobs: cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/minitwitapiimage:webbuildcache cache-to: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/minitwitapiimage:webbuildcache,mode=max - # TODO Get confirmation about this split - name: Build and push minitwitwebimage uses: docker/build-push-action@v7 with: @@ -73,7 +71,7 @@ jobs: host: ${{ secrets.SSH_HOST_SWARM }} username: ${{ secrets.SSH_USER_SWARM }} key: ${{ secrets.SSH_KEY_SWARM }} - source: docker-compose.yml,docker-compose.monitoring.yml,nginx.conf,nginx.develop.conf,loki,promtail,grafana + source: docker-compose.yml,docker-compose.monitoring.yml,nginx.conf,monitoring target: "/home/minitwit/" - name: Pull images and deploy to test server From b737047b7a821c432162dceae67e1fef0bda7f1d Mon Sep 17 00:00:00 2001 From: Jacob-Sonne Date: Wed, 29 Apr 2026 19:02:26 +0000 Subject: [PATCH 02/16] removed promtail global mode --- docker-compose.monitoring.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.monitoring.yml b/docker-compose.monitoring.yml index 44bac5f..c799787 100644 --- a/docker-compose.monitoring.yml +++ b/docker-compose.monitoring.yml @@ -55,7 +55,6 @@ services: networks: - minitwit-network deploy: - mode: global restart_policy: condition: on-failure From 15beaf4176946c243293965f4517ff46439e38f0 Mon Sep 17 00:00:00 2001 From: Jacob-Sonne Date: Fri, 8 May 2026 13:45:08 +0000 Subject: [PATCH 03/16] changes... --- Makefile | 6 +++++- monitoring/prometheus/prometheus.yml | 10 ++++++---- nginx.develop.conf | 18 +++++++++--------- test/minitwit_simulator.py | 4 ++-- 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 09de59e..fa27104 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,7 @@ setenv: @set -a && [ -f .env ] && . ./.env || true && set +a deploywebapi: + @set -a && . ./.env && set +a && \ docker compose -f docker-compose.local-db.yml up -d docker stack deploy -c docker-compose.develop.yml $(STACK_NAME) @@ -55,4 +56,7 @@ clean: docker stack rm $(STACK_NAME) || true docker stack rm $(MONITORING_STACK) || true docker compose -f docker-compose.local-db.yml down || true - docker network rm $(NETWORK); \ No newline at end of file + docker network rm $(NETWORK); + +deployall: clean buildlocal initswarm createnetwork deploywebapi deploymonitoring + watch -n 2 docker service ls \ No newline at end of file diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml index b7f3fe2..c504158 100644 --- a/monitoring/prometheus/prometheus.yml +++ b/monitoring/prometheus/prometheus.yml @@ -3,8 +3,10 @@ global: scrape_configs: - job_name: "minitwit" - metrics_path: /metrics - - static_configs: - - targets: ["minitwit-app:5001"] + dns_sd_configs: + - names: + - tasks.api + type: A + port: 5001 + diff --git a/nginx.develop.conf b/nginx.develop.conf index 26e7a49..ccf4f5b 100644 --- a/nginx.develop.conf +++ b/nginx.develop.conf @@ -29,14 +29,14 @@ http { proxy_set_header X-Forwarded-Proto $scheme; } - # location /grafana { - # proxy_pass http://grafana:3000; - # proxy_set_header Host $http_host; - # proxy_set_header X-Real-IP $remote_addr; - # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - # proxy_set_header X-URIScheme https; - # proxy_set_header Upgrade $http_upgrade; - # proxy_set_header Connection "upgrade"; - # } + location /grafana { + proxy_pass http://grafana:3000; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-URIScheme https; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } } } \ No newline at end of file diff --git a/test/minitwit_simulator.py b/test/minitwit_simulator.py index c1a3b7d..6c078c8 100644 --- a/test/minitwit_simulator.py +++ b/test/minitwit_simulator.py @@ -128,7 +128,7 @@ def main(host): data=json.dumps(data), params=params, headers=HEADERS, - timeout=0.3, + timeout=1.0, ) # error handling (204 success, 400 user exists) @@ -344,7 +344,7 @@ def main(host): ) ) - sleep(1000) + sleep(1) if __name__ == "__main__": From 0e734f321d2664d751c3cc89a5ffa385c40bea26 Mon Sep 17 00:00:00 2001 From: Jacob-Sonne Date: Fri, 8 May 2026 14:43:38 +0000 Subject: [PATCH 04/16] Changing Makefile to deploy minitwit api/web/nginx and monitoring in one command --- Makefile | 59 +++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index fa27104..ad21664 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,21 @@ .PHONY: staticcheck gofmt hadolint analysis checkmake all runlocalswarm buildlocal swarm createnetwork env deploy clean test +# Use ONE shell per recipe. Does not work across recipes. +.ONESHELL: +SHELL := /bin/bash + +# Make .env variables accessible to all recipes with $(VARIABLE_NAME) +include .env + +# Export all make variables to shell. Recipes can use $VARIABLE_NAME. Especially useful if we use python scripts to run stuff as we might want to expose variables instead of passing them. +# export + all: staticcheck gofmt hadolint checkmake staticcheck: go install honnef.co/go/tools/cmd/staticcheck@latest staticcheck ./... - gofmt: gofmt -l . @@ -25,7 +34,7 @@ STACK_NAME=minitwit MONITORING_STACK=monitoring NETWORK=minitwit-network -runlocalswarm: buildlocal initswarm createnetwork setenv deploywebapi +runlocalswarm: buildlocal initswarm createnetwork deploywebapi buildlocal: docker build -t minitwit-api:dev -f Dockerfile.api . @@ -41,11 +50,23 @@ createnetwork: echo "Creating overlay network..."; docker network create --driver overlay --attachable $(NETWORK); \ else echo "Network already exists"; fi -setenv: - @set -a && [ -f .env ] && . ./.env || true && set +a +createnetwork2: + @echo "Ensuring overlay network exists..." + @docker network inspect $(NETWORK) >/dev/null 2>&1 || \ + docker network create --driver overlay --attachable $(NETWORK) + + @echo "Waiting for Swarm network propagation..." + @i=0; \ + until docker network inspect $(NETWORK) >/dev/null 2>&1; do \ + i=$$((i+1)); \ + if [ $$i -gt 10 ]; then \ + echo "Network did not become ready in time"; exit 1; \ + fi; \ + sleep 1; \ + done + @echo "Network ready" deploywebapi: - @set -a && . ./.env && set +a && \ docker compose -f docker-compose.local-db.yml up -d docker stack deploy -c docker-compose.develop.yml $(STACK_NAME) @@ -58,5 +79,29 @@ clean: docker compose -f docker-compose.local-db.yml down || true docker network rm $(NETWORK); -deployall: clean buildlocal initswarm createnetwork deploywebapi deploymonitoring - watch -n 2 docker service ls \ No newline at end of file +deployall: clean buildlocal initswarm createnetwork2 deploywebapi deploymonitoring + watch -n 2 docker service ls + +service_watch: + watch -n 2 docker service ls + +log_web: + docker service logs minitwit_web -f --tail 20 + +log_api: + docker service logs minitwit_api -f --tail 20 + +log_proxy: + docker service logs minitwit_nginx -f --tail 20 + +log_grafana: + docker service logs monitoring_grafana -f --tail 20 + +log_loki: + docker service logs monitoring_loki -f --tail 20 + +log_prometheus: + docker service logs monitoring_prometheus -f --tail 20 + +log_promtail: + docker service logs monitoring_promtail -f --tail 20 \ No newline at end of file From e56d16e448dfa21df49ccfe8d27cb69240351feb Mon Sep 17 00:00:00 2001 From: Jacob-Sonne Date: Fri, 8 May 2026 16:02:39 +0000 Subject: [PATCH 05/16] Made changes to Makefile and minitwit_simulator.py so we can spin up a simulator with a make command. --- Makefile | 35 ++++++++++++++++------------------- test/minitwit_simulator.py | 6 ++++-- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index ad21664..6fa2708 100644 --- a/Makefile +++ b/Makefile @@ -50,22 +50,6 @@ createnetwork: echo "Creating overlay network..."; docker network create --driver overlay --attachable $(NETWORK); \ else echo "Network already exists"; fi -createnetwork2: - @echo "Ensuring overlay network exists..." - @docker network inspect $(NETWORK) >/dev/null 2>&1 || \ - docker network create --driver overlay --attachable $(NETWORK) - - @echo "Waiting for Swarm network propagation..." - @i=0; \ - until docker network inspect $(NETWORK) >/dev/null 2>&1; do \ - i=$$((i+1)); \ - if [ $$i -gt 10 ]; then \ - echo "Network did not become ready in time"; exit 1; \ - fi; \ - sleep 1; \ - done - @echo "Network ready" - deploywebapi: docker compose -f docker-compose.local-db.yml up -d docker stack deploy -c docker-compose.develop.yml $(STACK_NAME) @@ -77,11 +61,15 @@ clean: docker stack rm $(STACK_NAME) || true docker stack rm $(MONITORING_STACK) || true docker compose -f docker-compose.local-db.yml down || true - docker network rm $(NETWORK); + docker network rm $(NETWORK) || true; -deployall: clean buildlocal initswarm createnetwork2 deploywebapi deploymonitoring +deployall: createnetwork buildlocal initswarm deploywebapi deploymonitoring watch -n 2 docker service ls +deploy_local_simulator: + cd ./test + python3 minitwit_simulator.py http://127.0.0.1/api + service_watch: watch -n 2 docker service ls @@ -104,4 +92,13 @@ log_prometheus: docker service logs monitoring_prometheus -f --tail 20 log_promtail: - docker service logs monitoring_promtail -f --tail 20 \ No newline at end of file + docker service logs monitoring_promtail -f --tail 20 + +# Used to cleanup VM by removing: unused images, stopped containers, unused volumes, build cache +# and other temporary files. +aggressive_docker_clean: + docker system df + docker system prune -a --volumes + sudo journalctl --vacuum-size=100M + sudo apt-get clean + sudo apt-get autoremove -y \ No newline at end of file diff --git a/test/minitwit_simulator.py b/test/minitwit_simulator.py index 6c078c8..27208b2 100644 --- a/test/minitwit_simulator.py +++ b/test/minitwit_simulator.py @@ -19,6 +19,8 @@ import sqlite3 +REQUESTS_PER_SECOND = 10 +DELAY = 1/REQUESTS_PER_SECOND CSV_FILENAME = "./minitwit_scenario.csv" USERNAME = "simulator" PWD = "super_safe!" @@ -164,7 +166,7 @@ def main(host): params = {"latest": action["latest"], "no": action["no"]} response = requests.post( - url, params=params, headers=HEADERS, timeout=0.3 + url, params=params, headers=HEADERS, timeout=5 ) # error handling (200 success, 403 failure (no headers)) @@ -344,7 +346,7 @@ def main(host): ) ) - sleep(1) + sleep(DELAY) if __name__ == "__main__": From 5885a99d36f8e648a59b64d95014344d6a446754 Mon Sep 17 00:00:00 2001 From: Jacob-Sonne Date: Sun, 10 May 2026 14:24:02 +0000 Subject: [PATCH 06/16] working metrics ready to merge into logging_final --- Makefile | 5 +++-- docker-compose.monitoring.yml | 10 +++++----- nginx.develop.conf | 18 +++++++++--------- test/minitwit_simulator.py | 2 +- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 09de59e..d3813d1 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ .PHONY: staticcheck gofmt hadolint analysis checkmake all runlocalswarm buildlocal swarm createnetwork env deploy clean test - +.ONESHELL: +SHELL := /bin/bash all: staticcheck gofmt hadolint checkmake staticcheck: @@ -25,7 +26,7 @@ STACK_NAME=minitwit MONITORING_STACK=monitoring NETWORK=minitwit-network -runlocalswarm: buildlocal initswarm createnetwork setenv deploywebapi +runlocalswarm: buildlocal initswarm createnetwork setenv deploywebapi deploymonitoring buildlocal: docker build -t minitwit-api:dev -f Dockerfile.api . diff --git a/docker-compose.monitoring.yml b/docker-compose.monitoring.yml index 519941d..8d9aa7a 100644 --- a/docker-compose.monitoring.yml +++ b/docker-compose.monitoring.yml @@ -7,7 +7,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock # ports: # - "9090:9090" - # user: "0:0" Used for local testing + user: "0:0" # Run Prometheus as root to access docker socket and host metrics from cadvisor and node_exporter networks: - minitwit-network deploy: @@ -85,8 +85,8 @@ services: deploy: mode: global labels: - - "prometheus-job=cadvisor" - - "prometheus-port=8080" + - "prometheus_job=cadvisor" + - "prometheus_port=8080" node_exporter: image: quay.io/prometheus/node-exporter:latest @@ -101,8 +101,8 @@ services: deploy: mode: global labels: - - "prometheus-job=node_exporter" - - "prometheus-port=9100" + - "prometheus_job=node_exporter" + - "prometheus_port=9100" networks: # Network shared between services so we can refer to them by name diff --git a/nginx.develop.conf b/nginx.develop.conf index 8f0eb6b..3c28de5 100644 --- a/nginx.develop.conf +++ b/nginx.develop.conf @@ -29,14 +29,14 @@ http { proxy_set_header X-Forwarded-Proto $scheme; } - # location /grafana { - # proxy_pass http://grafana:3000; - # proxy_set_header Host $http_host; - # proxy_set_header X-Real-IP $remote_addr; - # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - # proxy_set_header X-URIScheme https; - # proxy_set_header Upgrade $http_upgrade; - # proxy_set_header Connection "upgrade"; - # } + location /grafana { + proxy_pass http://grafana:3000; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-URIScheme https; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } } } diff --git a/test/minitwit_simulator.py b/test/minitwit_simulator.py index c1a3b7d..8e79f57 100644 --- a/test/minitwit_simulator.py +++ b/test/minitwit_simulator.py @@ -344,7 +344,7 @@ def main(host): ) ) - sleep(1000) + sleep(0.5) if __name__ == "__main__": From 2b06d01e17fc975d920fddbb66c402b667134f39 Mon Sep 17 00:00:00 2001 From: Jacob-Sonne Date: Sun, 10 May 2026 18:36:30 +0000 Subject: [PATCH 07/16] Done with dashboards and monitoring/logging changes --- Makefile | 14 +- docker-compose.monitoring.yml | 8 +- .../grafana/dashboards/Api_Dashboard.json | 1019 ++++++++ .../dashboards/Availability_Dashboard.json | 385 +++ .../dashboards/Resource_Usage_Dashboard.json | 2056 +++++++++++++++++ .../provisioning/alerting/alert-rules.yaml | 107 + .../provisioning/alerting/contact-points.yaml | 24 + .../provisioning/dashboards/dashboards.yml | 1 - .../datasources}/datasource.yml | 0 .../Api_Dashboard.json} | 0 test/minitwit_simulator.py | 2 +- 11 files changed, 3608 insertions(+), 8 deletions(-) create mode 100644 monitoring/grafana/dashboards/Api_Dashboard.json create mode 100644 monitoring/grafana/dashboards/Availability_Dashboard.json create mode 100644 monitoring/grafana/dashboards/Resource_Usage_Dashboard.json create mode 100644 monitoring/grafana/provisioning/alerting/alert-rules.yaml create mode 100644 monitoring/grafana/provisioning/alerting/contact-points.yaml rename monitoring/grafana/{ => provisioning/datasources}/datasource.yml (100%) rename monitoring/grafana/{dashboards/ApiDashboard.json => saved_dashboards/Api_Dashboard.json} (100%) diff --git a/Makefile b/Makefile index 6fa2708..c95ddf6 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: staticcheck gofmt hadolint analysis checkmake all runlocalswarm buildlocal swarm createnetwork env deploy clean test +.PHONY: staticcheck gofmt hadolint analysis checkmake all runlocalswarm buildlocal swarm createnetwork env deploy clean test delete_all_volumes # Use ONE shell per recipe. Does not work across recipes. .ONESHELL: @@ -51,6 +51,7 @@ createnetwork: else echo "Network already exists"; fi deploywebapi: + @set -a; source .env; set +a; docker compose -f docker-compose.local-db.yml up -d docker stack deploy -c docker-compose.develop.yml $(STACK_NAME) @@ -63,6 +64,17 @@ clean: docker compose -f docker-compose.local-db.yml down || true docker network rm $(NETWORK) || true; +delete_grafana_volume: + docker volume rm monitoring_grafana-storage + +delete_loki_volume: + docker volume rm monitoring_loki-storage + +delete_prometheus_volume: + docker volume rm monitoring_prometheus-storage + +delete_all_volumes: delete_grafana_volume delete_loki_volume delete_prometheus_volume + deployall: createnetwork buildlocal initswarm deploywebapi deploymonitoring watch -n 2 docker service ls diff --git a/docker-compose.monitoring.yml b/docker-compose.monitoring.yml index 717071e..562aa2a 100644 --- a/docker-compose.monitoring.yml +++ b/docker-compose.monitoring.yml @@ -5,8 +5,6 @@ services: - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - prometheus-storage:/prometheus - /var/run/docker.sock:/var/run/docker.sock - # ports: - # - "9090:9090" user: "0:0" # Run Prometheus as root to access docker socket and host metrics from cadvisor and node_exporter networks: - minitwit-network @@ -27,10 +25,9 @@ services: - prometheus - loki volumes: - - ./monitoring/grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml - - ./monitoring/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards - - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards - grafana-storage:/var/lib/grafana + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning + - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards networks: - minitwit-network deploy: @@ -69,6 +66,7 @@ services: networks: - minitwit-network deploy: + mode: global restart_policy: condition: on-failure diff --git a/monitoring/grafana/dashboards/Api_Dashboard.json b/monitoring/grafana/dashboards/Api_Dashboard.json new file mode 100644 index 0000000..337ca56 --- /dev/null +++ b/monitoring/grafana/dashboards/Api_Dashboard.json @@ -0,0 +1,1019 @@ +{ + "apiVersion": "dashboard.grafana.app/v2", + "kind": "Dashboard", + "metadata": { + "name": "ad5dg8t", + "generation": 1, + "creationTimestamp": "2026-05-10T18:27:02Z", + "labels": {}, + "annotations": {} + }, + "spec": { + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana", + "version": "v0", + "spec": {} + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "builtIn": true + } + } + ], + "cursorSync": "Off", + "description": "The number of logs with level = error", + "editable": true, + "elements": { + "panel-2": { + "kind": "Panel", + "spec": { + "id": 2, + "title": "Total requests", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "spec": { + "direction": "backward", + "editorMode": "code", + "expr": "sum(count_over_time({endpoint=~\"$endpoint\"} [$__range]))", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "text" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "id": 3, + "title": "Total web requests", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "spec": { + "direction": "backward", + "editorMode": "code", + "expr": "sum by (endpoint) (\n count_over_time({endpoint!=\"\"}[$__interval])\n)", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.0.1", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 80, + "color": "red" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "links": [ + { + "oneClick": true, + "targetBlank": false, + "title": "test", + "url": "http://localhost:3000/d/adz9m5k/register?orgId=1&from=now-6h&to=now&timezone=browser" + } + ], + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-4": { + "kind": "Panel", + "spec": { + "id": 4, + "title": "Raw Logs", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "spec": { + "direction": "backward", + "editorMode": "code", + "expr": "{endpoint=~\"$endpoint\", method=~\"$method\", level=~\"$level\"}", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "logs", + "version": "13.0.1", + "spec": { + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": true, + "showControls": false, + "showFieldSelector": false, + "showLevel": true, + "showTime": false, + "sortOrder": "Descending", + "timestampResolution": "ms", + "unwrappedColumns": false, + "wrapLogMessage": false + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + } + } + }, + "panel-5": { + "kind": "Panel", + "spec": { + "id": 5, + "title": "Errors", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "spec": { + "direction": "backward", + "editorMode": "code", + "expr": "sum(count_over_time({level=\"error\", endpoint=~\"$endpoint\"} [$__range])) or vector(0)", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "dark-red" + } + ] + } + }, + "overrides": [] + } + } + } + } + }, + "panel-6": { + "kind": "Panel", + "spec": { + "id": 6, + "title": "Success", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "spec": { + "direction": "backward", + "editorMode": "code", + "expr": "sum(count_over_time({level=\"info\", endpoint!=\"\", endpoint=~\"$endpoint\"} [$__range]))", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + } + }, + "overrides": [] + } + } + } + } + }, + "panel-7": { + "kind": "Panel", + "spec": { + "id": 7, + "title": "Error Rate %", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "spec": { + "direction": "backward", + "editorMode": "code", + "expr": "100 * (sum(count_over_time({level=\"error\", endpoint=~\"$endpoint\"} [$__range])) or vector(0)) / sum(count_over_time({endpoint=~\"$endpoint\"} [$__range]))", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 80, + "color": "red" + } + ] + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-8": { + "kind": "Panel", + "spec": { + "id": 8, + "title": "New panel", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "spec": { + "direction": "backward", + "editorMode": "builder", + "expr": "sum by (endpoint) (\n count_over_time({endpoint!=\"\"}[$__range])\n)", + "legendFormat": "Total", + "queryType": "instant", + "step": "" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "spec": { + "direction": "backward", + "editorMode": "code", + "expr": "sum by (endpoint) (\n count_over_time({level=\"info\", endpoint!=\"\"}[$__range])\n)", + "legendFormat": "Success", + "queryType": "instant" + } + }, + "refId": "B", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "spec": { + "direction": "backward", + "editorMode": "code", + "expr": "sum by (endpoint) (\n count_over_time({level=\"error\", endpoint!=\"\"}[$__range])\n)", + "legendFormat": "Error", + "queryType": "instant" + } + }, + "refId": "C", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "spec": { + "direction": "backward", + "editorMode": "builder", + "expr": "100 *\nsum by (endpoint) (\n count_over_time({level=\"error\", endpoint!=\"\"}[$__range])\n)\n/\nsum by (endpoint) (\n count_over_time({endpoint!=\"\"}[$__range])\n)", + "legendFormat": "Error Rate %", + "queryType": "instant" + } + }, + "refId": "D", + "hidden": false + } + } + ], + "transformations": [ + { + "kind": "Transformation", + "group": "labelsToFields", + "spec": { + "options": { + "mode": "columns" + } + } + }, + { + "kind": "Transformation", + "group": "reduce", + "spec": { + "options": { + "includeTimeField": false, + "labelsToFields": false, + "mode": "reduceFields", + "reducers": [] + } + } + }, + { + "kind": "Transformation", + "group": "merge", + "spec": { + "options": {} + } + }, + { + "kind": "Transformation", + "group": "organize", + "spec": { + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "orderByMode": "manual", + "renameByName": { + "Time": "awd", + "Value #A": "Total", + "Value #B": "Success", + "Value #C": "Error", + "Value #D": "Error Rate %", + "endpoint": "" + } + } + } + } + ], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "table", + "version": "13.0.1", + "spec": { + "options": { + "cellHeight": "sm", + "frameIndex": 0, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Time" + } + ] + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "dark-green" + } + ] + }, + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "fieldMinMax": false + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "scope": "series", + "options": "Total" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "scope": "series", + "options": "Success" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "scope": "series", + "options": "Error" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "scope": "series", + "options": "Error Rate %" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "scope": "series", + "options": "endpoint" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + } + ] + } + } + } + } + } + }, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 3, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-2" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 3, + "y": 0, + "width": 3, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-6" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 6, + "y": 0, + "width": 3, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-5" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 9, + "y": 0, + "width": 3, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-7" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 3, + "width": 24, + "height": 10, + "element": { + "kind": "ElementReference", + "name": "panel-3" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 13, + "width": 24, + "height": 11, + "element": { + "kind": "ElementReference", + "name": "panel-4" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 24, + "width": 9, + "height": 6, + "element": { + "kind": "ElementReference", + "name": "panel-8" + } + } + } + ] + } + }, + "links": [], + "liveNow": false, + "preload": false, + "tags": [], + "timeSettings": { + "timezone": "browser", + "from": "now-6h", + "to": "now", + "autoRefresh": "", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "hideTimepicker": false, + "fiscalYearStartMonth": 0 + }, + "title": "Api Dashboard", + "variables": [ + { + "kind": "QueryVariable", + "spec": { + "name": "endpoint", + "current": { + "text": "", + "value": "" + }, + "label": "Endpoint", + "hide": "dontHide", + "refresh": "onDashboardLoad", + "skipUrlSync": false, + "description": "This variable is used to filter for a specific endpoint or all of them.", + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "spec": { + "label": "endpoint", + "refId": "LokiVariableQueryEditor-VariableQuery", + "stream": "", + "type": 1 + } + }, + "regex": "", + "regexApplyTo": "value", + "sort": "disabled", + "definition": "", + "options": [], + "multi": false, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "level", + "current": { + "text": "", + "value": "" + }, + "label": "Level", + "hide": "dontHide", + "refresh": "onDashboardLoad", + "skipUrlSync": false, + "description": "Filter based on the level of logs.", + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "spec": { + "label": "level", + "refId": "LokiVariableQueryEditor-VariableQuery", + "stream": "", + "type": 1 + } + }, + "regex": "", + "regexApplyTo": "value", + "sort": "disabled", + "definition": "", + "options": [], + "multi": false, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "method", + "current": { + "text": "", + "value": "" + }, + "label": "Method", + "hide": "dontHide", + "refresh": "onDashboardLoad", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "spec": { + "label": "method", + "refId": "LokiVariableQueryEditor-VariableQuery", + "stream": "", + "type": 1 + } + }, + "regex": "", + "regexApplyTo": "value", + "sort": "disabled", + "definition": "", + "options": [], + "multi": false, + "includeAll": true, + "allowCustomValue": true + } + } + ], + "preferences": { + "layout": { + "kind": "GridLayout", + "spec": { + "items": [] + } + } + } + } +} \ No newline at end of file diff --git a/monitoring/grafana/dashboards/Availability_Dashboard.json b/monitoring/grafana/dashboards/Availability_Dashboard.json new file mode 100644 index 0000000..ec519e4 --- /dev/null +++ b/monitoring/grafana/dashboards/Availability_Dashboard.json @@ -0,0 +1,385 @@ +{ + "apiVersion": "dashboard.grafana.app/v2", + "kind": "Dashboard", + "metadata": { + "name": "adlrfxw", + "generation": 4, + "creationTimestamp": "2026-05-10T17:12:39Z", + "labels": {}, + "annotations": {} + }, + "spec": { + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana", + "version": "v0", + "spec": {} + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "builtIn": true + } + } + ], + "cursorSync": "Off", + "editable": true, + "elements": { + "panel-1": { + "kind": "Panel", + "spec": { + "id": 1, + "title": "Availability State Timeline", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "max by(job) (up) or on(job) (0 * max by(job) (up))", + "legendFormat": "__auto", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "state-timeline", + "version": "13.0.1", + "spec": { + "options": { + "alignValue": "left", + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": false, + "rowHeight": 0.9, + "showValue": "auto", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "fieldConfig": { + "defaults": { + "unit": "bool_on_off", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "dark-red" + }, + { + "value": 1, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 80, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "fieldMinMax": false + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "scope": "series", + "options": "{instance=\"10.0.1.19:8080\", job=\"cadvisor\"}" + }, + "properties": [ + { + "id": "displayName", + "value": "Cadvisor" + } + ] + }, + { + "matcher": { + "id": "byName", + "scope": "series", + "options": "{instance=\"10.0.1.21:9100\", job=\"node_exporter\"}" + }, + "properties": [ + { + "id": "displayName", + "value": "Node Exporter" + } + ] + }, + { + "matcher": { + "id": "byName", + "scope": "series", + "options": "{instance=\"minitwit_api:5001\", job=\"minitwit\"}" + }, + "properties": [ + { + "id": "displayName", + "value": "Minitwit Api" + } + ] + }, + { + "matcher": { + "id": "byName", + "scope": "series", + "options": "{instance=\"minitwit_web:5000\", job=\"minitwit_web\"}" + }, + "properties": [ + { + "id": "displayName", + "value": "Minitwit Web" + } + ] + } + ] + } + } + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "id": 2, + "title": "Availability %", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "avg by(job) (\n avg_over_time(up[$__range])\n) * 100\nor on(job) (0 * avg by(job) (up))", + "legendFormat": "__auto", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + } + }, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 10, + "element": { + "kind": "ElementReference", + "name": "panel-2" + }, + "repeat": { + "mode": "variable", + "value": "instance", + "direction": "h" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 10, + "width": 24, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-1" + }, + "repeat": { + "mode": "variable", + "value": "instance", + "direction": "v" + } + } + } + ] + } + }, + "links": [], + "liveNow": false, + "preload": false, + "tags": [], + "timeSettings": { + "timezone": "browser", + "from": "now-3h", + "to": "now", + "autoRefresh": "5s", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "hideTimepicker": false, + "fiscalYearStartMonth": 0 + }, + "title": "Availability", + "variables": [ + { + "kind": "QueryVariable", + "spec": { + "name": "instance", + "current": { + "text": "", + "value": "" + }, + "label": "Instance", + "hide": "dontHide", + "refresh": "onDashboardLoad", + "skipUrlSync": false, + "description": "", + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "qryType": 1, + "query": "label_values(instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + } + }, + "regex": "", + "regexApplyTo": "value", + "sort": "disabled", + "definition": "label_values(instance)", + "options": [], + "multi": false, + "includeAll": false, + "allowCustomValue": true, + "staticOptions": [ + { + "text": "All", + "value": ".*", + "properties": { + "text": "All", + "value": ".*" + } + } + ] + } + } + ], + "preferences": { + "layout": { + "kind": "GridLayout", + "spec": { + "items": [] + } + } + } + } +} \ No newline at end of file diff --git a/monitoring/grafana/dashboards/Resource_Usage_Dashboard.json b/monitoring/grafana/dashboards/Resource_Usage_Dashboard.json new file mode 100644 index 0000000..fa9fe10 --- /dev/null +++ b/monitoring/grafana/dashboards/Resource_Usage_Dashboard.json @@ -0,0 +1,2056 @@ +{ + "apiVersion": "dashboard.grafana.app/v2", + "kind": "Dashboard", + "metadata": { + "name": "adkmz5r", + "generation": 1, + "creationTimestamp": "2026-05-10T17:40:13Z", + "labels": {}, + "annotations": {} + }, + "spec": { + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana", + "version": "v0", + "spec": {} + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "builtIn": true + } + } + ], + "cursorSync": "Off", + "editable": true, + "elements": { + "panel-11": { + "kind": "Panel", + "spec": { + "id": 11, + "title": "Max Mem %", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "max(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100", + "legendFormat": "__auto", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 80, + "color": "red" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-12": { + "kind": "Panel", + "spec": { + "id": 12, + "title": "Disk Utilization %", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "max((node_filesystem_size_bytes{mountpoint=\"/\", fstype!~\"tmpfs|overlay|squashfs\"}\n-\nnode_filesystem_avail_bytes{mountpoint=\"/\", fstype!~\"tmpfs|overlay|squashfs\"})\n/\nnode_filesystem_size_bytes{mountpoint=\"/\", fstype!~\"tmpfs|overlay|squashfs\"}\n* 100)", + "legendFormat": "__auto", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.0.1", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 80, + "color": "red" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-13": { + "kind": "Panel", + "spec": { + "id": 13, + "title": "Network Received", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "sum(\n rate(\n node_network_receive_bytes_total{\n device!~\"lo|docker.*|veth.*\"\n }[$__rate_interval]\n )\n)", + "legendFormat": "__auto", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.0.1", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-14": { + "kind": "Panel", + "spec": { + "id": 14, + "title": "Network Transmitted", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "sum(\n rate(\n node_network_transmit_bytes_total{\n device!~\"lo|docker.*|veth.*\"\n }[$__rate_interval]\n )\n)", + "legendFormat": "__auto", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.0.1", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-15": { + "kind": "Panel", + "spec": { + "id": 15, + "title": "CPU core utilization per swarm service name", + "description": "25% means 0.25 core is used. a machine can have more than one core.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "sum by(container_label_com_docker_swarm_service_name)(\n rate(\n container_cpu_usage_seconds_total{\n container_label_com_docker_swarm_service_name!=\"\"\n }[$__rate_interval]\n )\n) * 100", + "legendFormat": "__auto", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.0.1", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 80, + "color": "red" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-16": { + "kind": "Panel", + "spec": { + "id": 16, + "title": "Total Mem", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "max(node_memory_MemTotal_bytes)", + "legendFormat": "__auto", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "decbytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-17": { + "kind": "Panel", + "spec": { + "id": 17, + "title": "Mem Usage", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "max(node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes)", + "legendFormat": "__auto", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "decbytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-18": { + "kind": "Panel", + "spec": { + "id": 18, + "title": "Mem Avail", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "max(node_memory_MemAvailable_bytes)", + "legendFormat": "__auto", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "decbytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-19": { + "kind": "Panel", + "spec": { + "id": 19, + "title": "Total Disk", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "builder", + "expr": "max(node_filesystem_size_bytes)", + "legendFormat": "__auto", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "decbytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "id": 2, + "title": "CPU Utilization %", + "description": "How much % of the CPU is used in total.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "100 - (avg by(container_label_com_docker_swarm_service_name) (\n rate(node_cpu_seconds_total{mode=\"idle\"}[$__interval])\n) * 100)", + "legendFormat": "__auto", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.0.1", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 80, + "color": "red" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-20": { + "kind": "Panel", + "spec": { + "id": 20, + "title": "Disk Usage", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "max(node_filesystem_size_bytes - node_filesystem_avail_bytes)", + "legendFormat": "__auto", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "decbytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-21": { + "kind": "Panel", + "spec": { + "id": 21, + "title": "Mem Avail", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "max(node_filesystem_avail_bytes)", + "legendFormat": "__auto", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "decbytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "id": 3, + "title": "Mean CPU %", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "100 - (\n avg by(container_label_com_docker_swarm_service_name)(\n rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])\n ) * 100\n)", + "legendFormat": "__auto", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 80, + "color": "red" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-4": { + "kind": "Panel", + "spec": { + "id": 4, + "title": "Min CPU %", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "100 - (\n avg by(container_label_com_docker_swarm_service_name)(\n rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])\n ) * 100\n)", + "legendFormat": "__auto", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "min" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 80, + "color": "red" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-5": { + "kind": "Panel", + "spec": { + "id": 5, + "title": "Max CPU %", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "100 - (\n avg by(container_label_com_docker_swarm_service_name)(\n rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])\n ) * 100\n)", + "legendFormat": "__auto", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 80, + "color": "red" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-6": { + "kind": "Panel", + "spec": { + "id": 6, + "title": "p95 CPU %", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "100 - (\n avg by(container_label_com_docker_swarm_service_name)(\n rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])\n ) * 100\n)", + "legendFormat": "__auto", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "p95" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 80, + "color": "red" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-7": { + "kind": "Panel", + "spec": { + "id": 7, + "title": "Memory Utilization %", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "max(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100", + "legendFormat": "__auto", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.0.1", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 80, + "color": "red" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-8": { + "kind": "Panel", + "spec": { + "id": 8, + "title": "Min Mem %", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "min(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100", + "legendFormat": "__auto", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "min" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 80, + "color": "red" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-9": { + "kind": "Panel", + "spec": { + "id": 9, + "title": "Mean Mem %", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "avg(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100", + "legendFormat": "__auto", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 80, + "color": "red" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + } + }, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 3, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-4" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 3, + "y": 0, + "width": 3, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-3" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 6, + "y": 0, + "width": 3, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-6" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 9, + "y": 0, + "width": 3, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-5" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 4, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-16" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 16, + "y": 0, + "width": 4, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-17" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 20, + "y": 0, + "width": 4, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-18" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 3, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-2" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 3, + "width": 4, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-8" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 16, + "y": 3, + "width": 4, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-9" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 20, + "y": 3, + "width": 4, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-11" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 6, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-7" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 11, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-15" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 14, + "width": 4, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-19" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 16, + "y": 14, + "width": 4, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-20" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 20, + "y": 14, + "width": 4, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-21" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 17, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-12" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 19, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-13" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 27, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-14" + } + } + } + ] + } + }, + "links": [], + "liveNow": false, + "preload": false, + "tags": [], + "timeSettings": { + "timezone": "browser", + "from": "now-1h", + "to": "now", + "autoRefresh": "5s", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "hideTimepicker": false, + "fiscalYearStartMonth": 0 + }, + "title": "Resource Usage", + "variables": [], + "preferences": { + "layout": { + "kind": "GridLayout", + "spec": { + "items": [] + } + } + } + } +} \ No newline at end of file diff --git a/monitoring/grafana/provisioning/alerting/alert-rules.yaml b/monitoring/grafana/provisioning/alerting/alert-rules.yaml new file mode 100644 index 0000000..16b15db --- /dev/null +++ b/monitoring/grafana/provisioning/alerting/alert-rules.yaml @@ -0,0 +1,107 @@ +apiVersion: 1 +groups: + - orgId: 1 + name: Healthcheck + folder: Alerts + interval: 1m + rules: + - uid: cflgop2cdg9a8c + title: Web Down + condition: C + data: + - refId: A + relativeTimeRange: + from: 120 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + editorMode: code + expr: up{job="minitwit_web"} + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + queryType: expression + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: eq + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: NoData + execErrState: Error + for: 2m + isPaused: false + notification_settings: + receiver: Web Down Discord Webhook + - uid: bflgqzjdk7ldsc + title: Api Down + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + editorMode: code + expr: up{job="minitwit"} + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + queryType: expression + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: eq + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: NoData + execErrState: Error + for: 2m + isPaused: false + notification_settings: + receiver: Api Down Discord Webhook diff --git a/monitoring/grafana/provisioning/alerting/contact-points.yaml b/monitoring/grafana/provisioning/alerting/contact-points.yaml new file mode 100644 index 0000000..2fc5934 --- /dev/null +++ b/monitoring/grafana/provisioning/alerting/contact-points.yaml @@ -0,0 +1,24 @@ +apiVersion: 1 +contactPoints: + - orgId: 1 + name: Api Down Discord Webhook + receivers: + - uid: bflgr4kepsiyod + type: discord + settings: + message: '@everyone Api Service is down!' + title: Api + url: https://discord.com/api/webhooks/1492115087731986433/KIxRzQHCx737Lx20vJdAO1K5tNJCXyZkSzdM_dwUVzZBdwRY_nfyq479hLMX2Kjhjp4L + use_discord_username: false + disableResolveMessage: false + - orgId: 1 + name: Web Down Discord Webhook + receivers: + - uid: eflgokynvkb28d + type: discord + settings: + message: '@everyone Web Service is down!' + title: Web + url: https://discord.com/api/webhooks/1492115087731986433/KIxRzQHCx737Lx20vJdAO1K5tNJCXyZkSzdM_dwUVzZBdwRY_nfyq479hLMX2Kjhjp4L + use_discord_username: true + disableResolveMessage: false diff --git a/monitoring/grafana/provisioning/dashboards/dashboards.yml b/monitoring/grafana/provisioning/dashboards/dashboards.yml index 5ca5974..fe158d5 100644 --- a/monitoring/grafana/provisioning/dashboards/dashboards.yml +++ b/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -3,7 +3,6 @@ apiVersion: 1 providers: - name: default orgId: 1 - folder: Monitoring type: file disableDeletion: false editable: true diff --git a/monitoring/grafana/datasource.yml b/monitoring/grafana/provisioning/datasources/datasource.yml similarity index 100% rename from monitoring/grafana/datasource.yml rename to monitoring/grafana/provisioning/datasources/datasource.yml diff --git a/monitoring/grafana/dashboards/ApiDashboard.json b/monitoring/grafana/saved_dashboards/Api_Dashboard.json similarity index 100% rename from monitoring/grafana/dashboards/ApiDashboard.json rename to monitoring/grafana/saved_dashboards/Api_Dashboard.json diff --git a/test/minitwit_simulator.py b/test/minitwit_simulator.py index 27208b2..6dc5bc2 100644 --- a/test/minitwit_simulator.py +++ b/test/minitwit_simulator.py @@ -19,7 +19,7 @@ import sqlite3 -REQUESTS_PER_SECOND = 10 +REQUESTS_PER_SECOND = 1000 DELAY = 1/REQUESTS_PER_SECOND CSV_FILENAME = "./minitwit_scenario.csv" USERNAME = "simulator" From 6d4aeee46dc98f553029906c15c38e315fb4b488 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 14 May 2026 18:29:19 +0000 Subject: [PATCH 08/16] Update report PDF --- report/build/MSc_group_e.pdf | Bin 135000 -> 135000 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/report/build/MSc_group_e.pdf b/report/build/MSc_group_e.pdf index b32ba50d06c45019c7fecafec21a397df08eb293..7a20cf44fa009addfde9f966ba83e02016175813 100644 GIT binary patch delta 138 zcmcaHkK@KXj)oS-Elm3vRZI*mj4Tb!qBIrsee+XX5=&AQG+eBV42;YSObwxu+aEJB z?d5hebT)8wa&|LvHZ?OfGjKC8aJ6(cb~1GYinth>n>pDj*br6{J3Vnbquh2DKBf>x E09V~3z5oCK delta 138 zcmcaHkK@KXj)oS-Elm3vRg4Wyj7&{UqBIrsee+XX5=&AQG+eBV42;YSObwxu+aEJB z?d5heF*I~DHE}j}u{1Y!c6N0$ax!%^aR%ylbagbcFn6|7upz7@c6#D=M!D@Qd`uyX E08Ef0oB#j- From 57799330f6c3e77c28310a24dff20d853cdc43fa Mon Sep 17 00:00:00 2001 From: "Jacob Sonne (Json)" Date: Thu, 14 May 2026 20:40:55 +0200 Subject: [PATCH 09/16] Comment out .env file inclusion Comment out the inclusion of the .env file in the Makefile. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 091bb4a..1797d0d 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ SHELL := /bin/bash # Make .env variables accessible to all recipes with $(VARIABLE_NAME) -include .env +# include .env # Export all make variables to shell. Recipes can use $VARIABLE_NAME. Especially useful if we use python scripts to run stuff as we might want to expose variables instead of passing them. # export From bf159deeebc5d7b73818dcc7721b869eb107d01a Mon Sep 17 00:00:00 2001 From: "Jacob Sonne (Json)" Date: Thu, 14 May 2026 21:06:31 +0200 Subject: [PATCH 10/16] Replace hardcoded Discord webhook URL with env variable --- monitoring/grafana/provisioning/alerting/contact-points.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/grafana/provisioning/alerting/contact-points.yaml b/monitoring/grafana/provisioning/alerting/contact-points.yaml index 2fc5934..2abde83 100644 --- a/monitoring/grafana/provisioning/alerting/contact-points.yaml +++ b/monitoring/grafana/provisioning/alerting/contact-points.yaml @@ -8,7 +8,7 @@ contactPoints: settings: message: '@everyone Api Service is down!' title: Api - url: https://discord.com/api/webhooks/1492115087731986433/KIxRzQHCx737Lx20vJdAO1K5tNJCXyZkSzdM_dwUVzZBdwRY_nfyq479hLMX2Kjhjp4L + url: $__env{DISCORD_WEBHOOK_URL} use_discord_username: false disableResolveMessage: false - orgId: 1 @@ -19,6 +19,6 @@ contactPoints: settings: message: '@everyone Web Service is down!' title: Web - url: https://discord.com/api/webhooks/1492115087731986433/KIxRzQHCx737Lx20vJdAO1K5tNJCXyZkSzdM_dwUVzZBdwRY_nfyq479hLMX2Kjhjp4L + url: $__env{DISCORD_WEBHOOK_URL} use_discord_username: true disableResolveMessage: false From a2b0b40094084fd19e12331a9c278338cbf3535b Mon Sep 17 00:00:00 2001 From: "Jacob Sonne (Json)" Date: Thu, 14 May 2026 21:07:23 +0200 Subject: [PATCH 11/16] Add loading webhook secret when deploymonitoring target to Makefile --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 1797d0d..52f60b2 100644 --- a/Makefile +++ b/Makefile @@ -59,6 +59,7 @@ deploywebapi: docker stack deploy -c docker-compose.develop.yml $(STACK_NAME) deploymonitoring: + @set -a; source .env; set +a; docker stack deploy -c docker-compose.monitoring.yml $(MONITORING_STACK) clean: From e3c4196ebe8d27cbb92c6ecc5cefd9e29fbd8cb3 Mon Sep 17 00:00:00 2001 From: "Jacob Sonne (Json)" Date: Thu, 14 May 2026 21:08:10 +0200 Subject: [PATCH 12/16] Add DISCORD_WEBHOOK_URL to Grafana environment --- docker-compose.monitoring.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.monitoring.yml b/docker-compose.monitoring.yml index 562aa2a..faaeb1a 100644 --- a/docker-compose.monitoring.yml +++ b/docker-compose.monitoring.yml @@ -21,6 +21,7 @@ services: environment: - GF_SERVER_ROOT_URL=https://minitwit-devops.tech/grafana - GF_SERVER_SERVE_FROM_SUB_PATH=true + - DISCORD_WEBHOOK_URL=${DISCORD_WEBHOOK_URL} depends_on: - prometheus - loki From 09f43313351f3725a270231b11ac0482aa6731cd Mon Sep 17 00:00:00 2001 From: Sebastian Date: Thu, 14 May 2026 19:20:20 +0000 Subject: [PATCH 13/16] update replicas --- docker-compose.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 08f0fd5..97dd67a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ services: timeout: 10s retries: 3 deploy: - replicas: 6 + replicas: 3 restart_policy: condition: on-failure update_config: @@ -38,7 +38,7 @@ services: timeout: 10s retries: 3 deploy: - replicas: 3 + replicas: 2 restart_policy: condition: on-failure update_config: @@ -62,9 +62,16 @@ services: networks: - minitwit-network deploy: + replicas: 2 + restart_policy: + condition: on-failure + update_config: + parallelism: 1 + delay: 10s + failure_action: rollback placement: - constraints: - - node.role == manager + preferences: + - spread: node.role networks: # Network shared between services so we can refer to them by name minitwit-network: From 742a26339ea6f30bd4ba3e97a5d42905b091d438 Mon Sep 17 00:00:00 2001 From: "Jacob Sonne (Json)" Date: Thu, 14 May 2026 21:35:03 +0200 Subject: [PATCH 14/16] Disable Grafana proxy configuration Comment out the Grafana location block in the Nginx configuration. --- nginx.develop.conf | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/nginx.develop.conf b/nginx.develop.conf index 3c28de5..21f388d 100644 --- a/nginx.develop.conf +++ b/nginx.develop.conf @@ -29,14 +29,14 @@ http { proxy_set_header X-Forwarded-Proto $scheme; } - location /grafana { - proxy_pass http://grafana:3000; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-URIScheme https; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - } +# location /grafana { +# proxy_pass http://grafana:3000; +# proxy_set_header Host $http_host; +# proxy_set_header X-Real-IP $remote_addr; +# proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +# proxy_set_header X-URIScheme https; +# proxy_set_header Upgrade $http_upgrade; +# proxy_set_header Connection "upgrade"; +# } } } From e7dfe459e31a5c86a975474202bf83856829e4d1 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 14 May 2026 19:43:08 +0000 Subject: [PATCH 15/16] Update report PDF --- report/build/MSc_group_e.pdf | Bin 135000 -> 135000 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/report/build/MSc_group_e.pdf b/report/build/MSc_group_e.pdf index 7a20cf44fa009addfde9f966ba83e02016175813..9e41741acc542a8a129ec35631944d75cdf309f5 100644 GIT binary patch delta 136 zcmcaHkK@KXj)oS-EldX)l`Kt+4NRjn74&`cQ(O{DQWZ2@tc(nd%nVEoO$@=3+aEJB z?d5iJHFI@yHg+>Lce6A#adR{`aW=CwGcn>pDj*br6{J3Vnbquh2DKBf>x E02}TjSpWb4 From 511918b664b22c6ff8aa02e42b9fb3a6f8e18231 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 14 May 2026 20:06:14 +0000 Subject: [PATCH 16/16] Update report PDF --- report/build/MSc_group_e.pdf | Bin 135000 -> 135000 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/report/build/MSc_group_e.pdf b/report/build/MSc_group_e.pdf index 9e41741acc542a8a129ec35631944d75cdf309f5..314a58559113454de4a56ebcf3a784e3022a0d2a 100644 GIT binary patch delta 137 zcmcaHkK@KXj)oS-EldX(m5mGx%nS{qG!^uH^HW?BOHvgyT&#=?jLZy74NV}D+aEJB z?d5heb~SW1a5Ql=vv4(ZGBq}Ib8$3rv2b%TG%|E`HaD`cQ?MbdBzAh@c1F4FEPPBM Fi~uF+BT4`O delta 137 zcmcaHkK@KXj)oS-EldX(l?^RTj15eqG!^uH^HW?BOHvgyT&#=?jLZy74NV}D+aEJB z?d5iJHFI@yHg+>Lce6A#adR{`aW=CwGc