diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index da2671b..5854d4c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,4 +51,5 @@ jobs: with: tag_name: ${{ steps.next_version.outputs.version }} name: Release ${{ steps.next_version.outputs.version }} - generate_release_notes: true \ No newline at end of file + generate_release_notes: true + files: report/build/MSc_group_p.pdf diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml deleted file mode 100644 index 754a61c..0000000 --- a/.github/workflows/report.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Report - -on: - push: - branches: [ "main", "development" ] - pull_request: - branches: [ "main", "development" ] - workflow_dispatch: - -env: - REPORT_SOURCE: report/report.md - REPORT_OUTPUT: report/build/MSc_group_p.pdf - REPORT_TARGET_BRANCH: development - SOURCE_DATE_EPOCH: "0" - -permissions: - contents: write - -jobs: - build-report: - name: Build Report PDF - runs-on: ubuntu-latest - - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - - name: Prepare report build directory - run: mkdir -p report/build - - - name: Check report word count - run: | - words=$(python3 - <<'PY' - import re - from pathlib import Path - - import os - - text = Path(os.environ["REPORT_SOURCE"]).read_text(encoding="utf-8") - text = re.sub(r"---\n.*?\n---\n", "", text, flags=re.S) - text = re.sub(r"```.*?```", "", text, flags=re.S) - text = re.sub(r"\[[^\]]+\]\([^)]+\)", "", text) - print(len(re.findall(r"\b[\w'-]+\b", text))) - PY - ) - echo "Report word count: ${words}" - if [ "${words}" -gt 2500 ]; then - echo "::warning title=Report word count::Report has ${words} words, which is above the 2500 word limit." - fi - - - name: Build PDF with Pandoc - uses: docker://pandoc/latex:3.6 - with: - args: >- - ${{ env.REPORT_SOURCE }} - --from=markdown - --pdf-engine=xelatex - --number-sections - --metadata=date= - --variable=graphics - --resource-path=.:report - --output=${{ env.REPORT_OUTPUT }} - - - name: Commit generated PDF - if: github.event_name == 'push' && github.ref_name == env.REPORT_TARGET_BRANCH - run: | - cp "${REPORT_OUTPUT}" /tmp/MSc_group_p.pdf - git fetch origin "${REPORT_TARGET_BRANCH}" - git switch -C "${REPORT_TARGET_BRANCH}" "origin/${REPORT_TARGET_BRANCH}" - mkdir -p "$(dirname "${REPORT_OUTPUT}")" - cp /tmp/MSc_group_p.pdf "${REPORT_OUTPUT}" - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add "${REPORT_OUTPUT}" - if git diff --cached --quiet; then - echo "No report PDF changes to commit." - else - git commit -m "Update generated report PDF [skip ci]" - git push origin "HEAD:${REPORT_TARGET_BRANCH}" - fi - - - name: Upload report PDF - uses: actions/upload-artifact@v4 - with: - name: MSc_group_p - path: ${{ env.REPORT_OUTPUT }} diff --git a/Makefile b/Makefile index a62b1de..ccce4c3 100644 --- a/Makefile +++ b/Makefile @@ -106,11 +106,12 @@ ui-e2e: report: @echo "$(CYAN)==> Building report PDF...$(RESET)" mkdir -p report/build - SOURCE_DATE_EPOCH=0 pandoc $(REPORT_SOURCE) --from=markdown --pdf-engine=xelatex --toc --number-sections --metadata=date= --output=$(REPORT_OUTPUT) + npx --yes md-to-pdf $(REPORT_SOURCE) --basedir report + mv report/report.pdf $(REPORT_OUTPUT) @echo "$(GREEN)Report built at $(REPORT_OUTPUT)$(RESET)" report-word-count: - @pandoc $(REPORT_SOURCE) -t plain | wc -w + @python3 -c 'import re, pathlib; text = pathlib.Path("$(REPORT_SOURCE)").read_text(encoding="utf-8"); text = re.sub(r"---\n.*?\n---\n", "", text, flags=re.S); text = re.sub(r"```.*?```", "", text, flags=re.S); text = re.sub(r"<[^>]+>", "", text); text = re.sub(r"\[[^\]]+\]\([^)]+\)", "", text); print(len(re.findall(r"\b[\w'\''-]+\b", text)))' # --------- Execute tests -------------------------- diff --git a/README.md b/README.md index 7733781..ad17d60 100644 --- a/README.md +++ b/README.md @@ -1,49 +1,143 @@ -# Project readme
+# MiniTwit (ITU DevOps) -## Git and branches
-Naming convention: -{fix/feature}//{short_message} +A Twitter-clone. The original Python/Flask + SQLite app has been rewritten in Go + MongoDB and runs on a Docker Swarm of DigitalOcean droplets, with Prometheus/Grafana/Loki for observability and a Discord bot for alerts. -## CI/CD -### Project infrastructure is setup with Vagrant. Run _vagrant up_ to create virtual machines (droplets) on Digital Ocean. -### Everytime a merge is done into the _main_ branch GithubActions perform continuous deployment to update the project with latest changes. +The service exposes two interfaces on the same port: +- UI — server-rendered HTML for end users (`/`, `/login`, `/timeline`, `/user/{username}`, …) +- Simulator API — JSON endpoints consumed by the simulator (`/register`, `/msgs`, `/fllws/{username}`, `/latest`, …) -## How to run env locally -### 1. Prepare local prerequisites for `vagrant up` - export DIGITAL_OCEAN_TOKEN=**** +Public site: -Create an SSH key pair locally at: - ~/.ssh/ssh_key_golang_minitwit +--- -Add the matching public key to DigitalOcean with this exact key name: - ssh_key_golang_minitwit +## Repository layout -Optional if you want TLS/Nginx bootstrap during the same run: - export TLS_DOMAIN=your-domain.com - export TLS_EMAIL=you@example.com +| Path | Purpose | +|---|---| +| [src/](src/) | Go application (gorilla/mux router, handlers, middleware, DB helpers) | +| [src/bot/](src/bot/) | Discord bot — separate Go module with its own `go.mod`, built into its own image | +| [docker/](docker/) | Dockerfiles for the web and bot images | +| [monitoring/](monitoring/) | Prometheus, Grafana, Loki, Promtail configs and dashboards | +| [infra/](infra/) | Terraform definition of the production infra (droplets, firewall, MongoDB)| +| [remote_files/](remote_files/) | Files copied to the Swarm manager: authoritative `docker-stack.yml`, `deploy.sh`, TLS bootstrap | +| [.github/workflows/](.github/workflows/) | CI (static analysis, tests, E2E) and CD (build → push → deploy) | +| [Vagrantfile](Vagrantfile), [setup-swarm.sh](setup-swarm.sh) | Legacy provisioning path (still works) | +| [docker-compose.yml](docker-compose.yml) | Local dev stack | +| [report/](report/) | Report in PDF format | -### 2. Create an `.env` file in root folder and setup deployment environment variables - DOCKER_USERNAME=**** - DISCORD_TOKEN=**** - GRAFANA_ADMIN_USER=**** - GRAFANA_ADMIN_PASSWORD=**** - MONGO_URI=**** +--- -`vagrant up` creates the droplets and then automatically runs `setup-swarm.sh`, so these variables must be available locally when you run it. +## Local development -### 3. Run this command to create infrastructure and deploy the Swarm stack - vagrant up +Requirements: Docker + Docker Compose. Go 1.22+ only needed for running tests outside containers. -### 4. Run this command to start docker locally - docker compose up --build +```bash +docker compose up --build # web on :8080, MongoDB, Prometheus, Grafana, Loki, Promtail +docker compose down -v +``` -#### 4.1 Re-run deployment, routing, or TLS setup on existing droplets - vagrant rsync - export TLS_DOMAIN=your-domain.com - export TLS_EMAIL=you@example.com - ./setup-swarm.sh +Run the Go service directly (needs a MongoDB on `localhost:27017`): -`setup-swarm.sh` loads deployment variables from the local `.env` file or the current shell environment and passes them directly to the manager node. The `.env` file is no longer synced through `Vagrantfile`. +```bash +cd src && go run main.go +``` -### 5. API test -To test API you can use _test-api-routes.sh_ script. +Common dev commands: + +```bash +make verify # full local CI check (build, sim, lint, UI E2E) +make test-sim # run the grading simulator against :8080 +make ui-e2e # Selenium UI tests (needs geckodriver) +./test-api-routes.sh # API smoke tests +``` + +--- + +## Production infrastructure + +Production runs a Docker Swarm of three DigitalOcean droplets (1 manager + 2 workers) with a managed MongoDB cluster. DNS for `itu-minitwit.me` lives at Namecheap (not IaC-managed). + +There are two provisioning paths: + +### Terraform + +```bash +cd infra +cp terraform.tfvars.example terraform.tfvars # fill in tls_email, discord_token, grafana_admin_password, … +export DIGITALOCEAN_TOKEN= +terraform init +terraform plan +terraform apply # ~5–8 min; provisions droplets + MongoDB + deploys the stack +``` + +Outputs include the manager IP and MongoDB URI needed for the `SSH_HOST` and `MONGO_URI` GitHub Secrets. + +### Vagrant + +The original path is still functional. Requires the same `DIGITALOCEAN_TOKEN`, an SSH key at `~/.ssh/ssh_key_golang_minitwit` registered in DO under the same name, and a local `.env` with `DOCKER_USERNAME`, `MONGO_URI`, `DISCORD_TOKEN`, `GRAFANA_ADMIN_USER`, `GRAFANA_ADMIN_PASSWORD`. + +```bash +vagrant up # creates droplets, then runs setup-swarm.sh which deploys the stack +vagrant rsync && ./setup-swarm.sh # re-deploy onto existing droplets +vagrant provision # runs the script in Vagrant file +``` + +`MONGO_URI` should point to an external managed MongoDB (e.g. DO Managed MongoDB or Atlas). + +### TLS / nginx + +The reverse proxy (nginx + Let's Encrypt) is bootstrapped once per manager with [remote_files/bootstrap_droplet_tls.sh](remote_files/bootstrap_droplet_tls.sh). It is rerun automatically by the CD workflow's post-deploy verification step. + +--- + +## How to contribute changes + +### Branch naming + +``` +{fix|feature}/{short-message} +``` + +Examples: `feature/terraform`, `fix/login-redirect`. + +### Workflow + +1. Branch off `development`. +2. Push your branch and open a PR into `development`. CI runs automatically. +3. Once your PR is merged into `development`, the same checks run again. When a release is ready, `development` is merged into `main`. +4. Pushing to `main` triggers continuous deployment to production. + +### CI on push/PR (`development`, `main`) + +[`static-analysis.yml`](.github/workflows/static-analysis.yml) builds the images and runs: +- Docker Scout + Trivy + Semgrep security scans +- `make verify` (simulator test, `go fmt`, `golangci-lint`, Hadolint) +- `make ui-e2e` (Selenium UI E2E tests) + + +### CD on push to `main` + +[`continous-deployment.yml`](.github/workflows/continous-deployment.yml): +1. Builds and pushes the web, bot, Prometheus, and Grafana images to Docker Hub +2. SCPs deploy files and a freshly rendered `.env` to the Swarm manager +3. Runs [`remote_files/deploy.sh`](remote_files/deploy.sh) on the manager +4. Verifies the deploy: nginx config, certbot renewal, TLS chain + +The required GitHub Secrets are: `DOCKER_USERNAME`, `DOCKERHUB_AUTHTOKEN`, `SSH_HOST`, `SSH_USER`, `SSH_PRIVATE_KEY`, `MONGO_URI`, `DISCORD_TOKEN`, `GRAFANA_ADMIN_USER`, `GRAFANA_ADMIN_PASSWORD`, `TLS_DOMAIN`, `TLS_EMAIL`. + +## Observability + +- Prometheus scrapes the webserver's `/metrics` (custom counters/histograms in [src/middleware/metrics.go](src/middleware/metrics.go)). +- Grafana is served at `/grafana/` in production (dashboards and datasources provisioned from [monitoring/grafana/](monitoring/grafana/)). +- Loki + Promtail collect container logs; Promtail runs `mode: global` (one per Swarm node) reading `/var/lib/docker/containers/`. +- Discord bot posts alerts to a Discord channel via the token in `DISCORD_TOKEN`. + +Each request gets an `X-Request-ID` propagated through context; all user-supplied values are sanitized before logging. + +--- + +## Reports & releases + +```bash +make report # build the course report PDF +``` diff --git a/Vagrantfile b/Vagrantfile index 46b983d..1cff860 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -55,11 +55,22 @@ Vagrant.configure("2") do |config| # Allowing SSH connections sudo ufw allow "OpenSSH" - # Open the published MiniTwit HTTP port. + # Open public HTTP/HTTPS only through Nginx. sudo ufw allow 80/tcp - - # Open Grafana for external dashboard access. - sudo ufw allow 3000/tcp + sudo ufw allow 443/tcp + + # Block direct access to services that should only be reached through Nginx. + sudo ufw deny 3000/tcp + sudo ufw deny 8080/tcp + + # Docker can publish ports before UFW sees the traffic. Block public + # access to those published container ports in Docker's user chain using iptables firewall. + sudo iptables -C DOCKER-USER -p tcp ! -s 127.0.0.0/8 --dport 3000 -j DROP 2>/dev/null || sudo iptables -I DOCKER-USER -p tcp ! -s 127.0.0.0/8 --dport 3000 -j DROP + sudo iptables -C DOCKER-USER -p tcp ! -s 127.0.0.0/8 --dport 8080 -j DROP 2>/dev/null || sudo iptables -I DOCKER-USER -p tcp ! -s 127.0.0.0/8 --dport 8080 -j DROP + echo iptables-persistent iptables-persistent/autosave_v4 boolean true | sudo debconf-set-selections + echo iptables-persistent iptables-persistent/autosave_v6 boolean true | sudo debconf-set-selections + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y iptables-persistent + sudo netfilter-persistent save # Enable the firewall only after SSH is allowed, otherwise provisioning # risks locking us out on a fresh host. diff --git a/docker-compose.yml b/docker-compose.yml index ed0d128..93455c1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -74,6 +74,7 @@ services: - grafana_cloud_data:/var/lib/grafana - ./monitoring/grafana/datasources:/etc/grafana/provisioning/datasources:ro - ./monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards:ro + - ./monitoring/grafana/alerting:/etc/grafana/provisioning/alerting:ro deploy: placement: constraints: diff --git a/infra/.gitignore b/infra/.gitignore index 3f68054..02477b5 100644 --- a/infra/.gitignore +++ b/infra/.gitignore @@ -1,10 +1,8 @@ -# Local terraform state and lock data — never commit. .terraform/ .terraform.lock.hcl terraform.tfstate terraform.tfstate.backup crash.log -# Real variable values (may contain secrets). terraform.tfvars *.auto.tfvars diff --git a/infra/main.tf b/infra/main.tf index 6276256..f540db4 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -63,17 +63,6 @@ resource "digitalocean_firewall" "swarm" { port_range = "443" source_addresses = ["0.0.0.0/0", "::/0"] } - inbound_rule { - protocol = "tcp" - port_range = "3000" - source_addresses = ["0.0.0.0/0", "::/0"] - } - inbound_rule { - protocol = "tcp" - port_range = "8080" - source_addresses = ["0.0.0.0/0", "::/0"] - } - # Swarm node-to-node communication, restricted to the cluster's own droplets. inbound_rule { protocol = "tcp" diff --git a/monitoring/grafana/Dockerfile b/monitoring/grafana/Dockerfile index 8f1d9ee..f5165b1 100644 --- a/monitoring/grafana/Dockerfile +++ b/monitoring/grafana/Dockerfile @@ -2,5 +2,6 @@ FROM grafana/grafana:12.1 COPY dashboards /etc/grafana/provisioning/dashboards COPY datasources /etc/grafana/provisioning/datasources +COPY alerting /etc/grafana/provisioning/alerting -EXPOSE 3000 \ No newline at end of file +EXPOSE 3000 diff --git a/monitoring/grafana/alerting/error_rate_alerts.yml b/monitoring/grafana/alerting/error_rate_alerts.yml new file mode 100644 index 0000000..14af5ef --- /dev/null +++ b/monitoring/grafana/alerting/error_rate_alerts.yml @@ -0,0 +1,193 @@ +apiVersion: 1 + +groups: + - orgId: 1 + name: Error Rate Alerts + folder: Error Rate Dashboard + interval: 30s + rules: + - uid: error_rate_above_10_percent + title: Error rate above 10% + condition: C + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: ituminitwitprometheus + model: + datasource: + type: prometheus + uid: ituminitwitprometheus + editorMode: code + expr: | + ( + sum(rate(minitwit_http_responses_total{status=~"[45].."}[5m])) / + sum(rate(minitwit_http_responses_total[5m])) + ) * 100 + instant: true + intervalMs: 1000 + legendFormat: Error Rate % + maxDataPoints: 43200 + range: false + refId: A + - refId: B + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 10 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + dashboardUid: error_rate + panelId: 4 + noDataState: OK + execErrState: Error + for: 0s + annotations: + summary: Error rate is above 10% + description: Overall 5-minute HTTP error rate is above 10%. + labels: + severity: warning + isPaused: false + - orgId: 1 + name: App Availability Alerts + folder: Time Respond Dashboard + interval: 30s + rules: + - uid: app_down + title: App is down + condition: C + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: ituminitwitprometheus + model: + datasource: + type: prometheus + uid: ituminitwitprometheus + editorMode: code + expr: up{job="itu-minittwit-app"} + instant: true + intervalMs: 1000 + legendFormat: Application Status + maxDataPoints: 43200 + range: false + refId: A + - refId: B + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 1 + type: lt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + dashboardUid: Time_respond + panelId: 1 + noDataState: Alerting + execErrState: Alerting + for: 1m + annotations: + summary: App is down + description: Prometheus cannot scrape the Minitwit app target, or the app is reporting up < 1. + labels: + severity: critical + isPaused: false diff --git a/monitoring/grafana/dashboards/error_rate.json b/monitoring/grafana/dashboards/error_rate.json index 63ee32f..3e8cb96 100644 --- a/monitoring/grafana/dashboards/error_rate.json +++ b/monitoring/grafana/dashboards/error_rate.json @@ -290,7 +290,7 @@ "uid": "ituminitwitprometheus" }, "editorMode": "code", - "expr": "sum by (status, path) (rate(minitwit_http_responses_total{status=~\"[45..]\"}[5m]))", + "expr": "sum by (status, path) (rate(minitwit_http_responses_total{status=~\"[45]..\"}[5m]))", "legendFormat": "{{status}} {{path}}", "range": true, "refId": "A" diff --git a/remote_files/bootstrap_droplet_tls.sh b/remote_files/bootstrap_droplet_tls.sh index 7f50cd7..6b9c19d 100644 --- a/remote_files/bootstrap_droplet_tls.sh +++ b/remote_files/bootstrap_droplet_tls.sh @@ -34,6 +34,17 @@ echo "Configuring firewall (ufw) if available..." if command -v ufw >/dev/null 2>&1; then ufw allow OpenSSH || true ufw allow 'Nginx Full' || true + ufw deny 3000/tcp || true + ufw deny "${APP_UPSTREAM_PORT}/tcp" || true +fi + +if command -v iptables >/dev/null 2>&1; then + iptables -C DOCKER-USER -p tcp ! -s 127.0.0.0/8 --dport 3000 -j DROP 2>/dev/null || iptables -I DOCKER-USER -p tcp ! -s 127.0.0.0/8 --dport 3000 -j DROP + iptables -C DOCKER-USER -p tcp ! -s 127.0.0.0/8 --dport "${APP_UPSTREAM_PORT}" -j DROP 2>/dev/null || iptables -I DOCKER-USER -p tcp ! -s 127.0.0.0/8 --dport "${APP_UPSTREAM_PORT}" -j DROP + echo iptables-persistent iptables-persistent/autosave_v4 boolean true | debconf-set-selections + echo iptables-persistent iptables-persistent/autosave_v6 boolean true | debconf-set-selections + DEBIAN_FRONTEND=noninteractive apt-get install -y iptables-persistent + netfilter-persistent save fi NGINX_CONF_PATH="/etc/nginx/sites-available/${DOMAIN}.conf" diff --git a/report/build/MSc_group_p.pdf b/report/build/MSc_group_p.pdf index 06ae604..0d22792 100644 Binary files a/report/build/MSc_group_p.pdf and b/report/build/MSc_group_p.pdf differ diff --git a/report/report.md b/report/report.md index aef2492..49b1440 100644 --- a/report/report.md +++ b/report/report.md @@ -1,220 +1,264 @@ - +

+ ITU logo +

-\begin{center} -\includegraphics[width=1\textwidth]{report/images/itu_logo.jpg} +

Report by group p "EastOps"

-{\LARGE\textbf{Report group P}} -\end{center} + -**Anton Yakovenko anya@itu.dk** -**Janusz Bekas jdbe@itu.dk** -**Mengdi Liao menl@itu.dk** -**Viktor Horvath vhor@itu.dk** - -\newpage +
## Introduction -Itu-minitwit is an evolved version of the MinitTwit project originally written in python and Flask. It was re-written in GoLang with the Gorilla webtoolkit framework. We use MongoDB for our database. Loki, Prometheus, Promtail and Grafana are used as a monitoring stack. +Itu-minitwit is an evolved version of the MiniTwit project, originally written in Python and Flask. We re-wrote it in Go using the Gorilla web toolkit, with MongoDB as the database. Loki, Prometheus, Promtail, and Grafana form the monitoring stack. -The project’s codebase is located on GitHub and utilizes GitHub Actions for CI/CD pipeline. We are using DigitalOcean as our host for Virtual machines and a managed database. +The codebase lives on GitHub and uses GitHub Actions for the CI/CD pipeline. DigitalOcean hosts our virtual machines and managed database. -\newpage +
## System perspective -\begin{center} -\includegraphics[width=0.9\textwidth]{report/images/sys_arch_img.png} - -\emph{System architecture view} -\end{center} +

+ System architecture view +

-The production runtime is Docker Swarm on DigitalOcean. The Swarm has one manager node, minitwit, and two worker nodes, minitwit-web-1 and minitwit-web-2. The remote stack consists of webserver, prometheus, grafana, loki, promtail, and discordbot services on the overlay network minitwit-network. Manager node consists of a monitoring stack (prometheus, grafana, loki, a promtail replica and webserver replica). Worker nodes have webserver replicas and promtail replicas. +

System architecture view

-The web application and Discord bot both connect to the database through MONGO\_URI. +The production runtime is Docker Swarm on DigitalOcean. The Swarm has one manager node, **minitwit**, and two worker nodes, **minitwit-web-1** and **minitwit-web-2**. The remote stack consists of webserver, prometheus, grafana, loki, promtail, and discordbot services on the overlay network minitwit-network. Manager node consists of a monitoring stack (prometheus, grafana, loki, Discord bot, a promtail replica and webserver replica). Worker nodes have webserver replicas and promtail replicas. -Monitoring is implemented using Grafana for visual display of graphs and logs. +The infrastructure itself is provisioned by Vagrant, but can also be provisioned from Terraform (`infra/`). A single `terraform apply` creates the three droplets with Docker pre-installed via cloud-init, a DigitalOcean Cloud Firewall, a managed MongoDB cluster, and a DO Project to group them. Two `null_resource` blocks then run `swarm-setup.sh` to initialize the Swarm and `deploy-stack.sh` to upload the stack files and trigger `deploy.sh` on the manager. -Logs are being created in webserver containers, then Promtail runs as a global Swarm service and ships Docker container logs to Loki. +The web application and Discord bot both connect to the database through `MONGO_URI`. -Graphs are created by prometheus querying the webserver system and passing the information to grafana that interprets information and displays them as informational graphs. +Monitoring is implemented using Grafana for visual display of graphs and logs. Logs originate in the webserver containers; Promtail runs as a global Swarm service and ships Docker container logs to Loki. Graphs are produced by Prometheus, which scrapes the webserver's metrics endpoint and feeds Grafana for visualization. Necessary environment variables are stored in GitHub Secrets. -\newpage +
## Dependencies -**Higher-Level Dependencies** -**DigitalOcean Droplets:** Cloud VMs that host the production Docker Swarm nodes. -**DigitalOcean Managed MongoDB:** Hosted production MongoDB database used by the app and bot. -**Docker:** Runs the application and infrastructure services in containers. -**Docker Compose:** Defines and runs the local development environment. -**Docker Swarm:** Orchestrates the remote production stack across multiple droplets. -**Docker Hub:** Registry where production images are pushed and pulled from. -**Vagrant:** Current tool for provisioning DigitalOcean droplets. -**vagrant-digitalocean:** Vagrant provider plugin/box for creating DigitalOcean droplets. -**GitHub Actions:** CI/CD platform for tests, static analysis, image builds, releases, and deployments. -**Nginx:** Reverse proxy in front of the app and Grafana. -**Certbot:** Automates Let’s Encrypt certificate setup and renewal. -**Let’s Encrypt:** Issues TLS certificates. -**UFW:** Host firewall configured during provisioning. -**Prometheus:** Collects application and service metrics. -**Grafana:** Displays dashboards for metrics and logs. -**Loki:** Stores and queries logs. -**Promtail:** Collects container logs and ships them to Loki. -**rsyslog:** Older/auxiliary centralized log collection component. -**Pandoc / LaTeX:** Builds the project report PDF. -**Firefox / geckodriver / Selenium:** Runs browser-based UI tests. -**Docker Scout:** Scans Docker images for vulnerabilities. -**Trivy:** Performs container vulnerability scanning. -**Semgrep:** Static analysis for security/code patterns. -**CodeQL SARIF upload:** Uploads scan results to GitHub code scanning. -**golangci-lint:** Runs Go linting. -**Hadolint:** Lints Dockerfiles. - -**Container / Image Dependencies** -**\`mongo:8.0\`:** Local MongoDB database image. -**\`grafana/grafana:12.1\`:** Base image for the custom Grafana image. -**\`prom/prometheus:v3.5.1\`:** Base image for the custom Prometheus image. -**\`grafana/loki:latest\`:** Loki log storage/query service image. -**\`grafana/promtail:latest\`:** Promtail log shipper image. -**\`ubuntu:24.04\`:** Base image for the rsyslog container. -**\`golang:1.25.10\`:** Build image for compiling the Go web app. -**\`alpine:3.23\`:** Lightweight runtime image for the Go web app. -**\`pandoc/latex:3.6\`:** CI image used to build the report PDF. -**\`curlimages/curl\`:** Small image used for HTTP checks in older/local helper flows. - -**Main Go App Dependencies** -**\`github.com/gorilla/mux\`:** HTTP router for app routes. -**\`github.com/gorilla/sessions\`:** Cookie/session management. -**\`github.com/prometheus/client\_golang\`:** Exposes Prometheus metrics from the app. -**\`go.mongodb.org/mongo-driver\`:** MongoDB client driver. -**\`golang.org/x/crypto\`:** Crypto utilities, including password hashing support. - -**Discord Bot Dependencies** -**\`github.com/bwmarrin/discordgo\`:** Discord API client library. -**\`go.mongodb.org/mongo-driver\`:** MongoDB client driver used by the bot. - -**Important Go Indirect Dependencies** -**\`github.com/gorilla/securecookie\`:** Secure cookie encoding used by Gorilla sessions. -**\`github.com/gorilla/websocket\`:** WebSocket support, used indirectly by Discord integration. -**\`github.com/prometheus/client\_model\`:** Prometheus metric data model. -**\`github.com/prometheus/common\`:** Shared Prometheus helpers. -**\`github.com/prometheus/procfs\`:** Reads Linux process metrics for Prometheus. -**\`github.com/klauspost/compress\`:** Compression support. -**\`github.com/golang/snappy\`:** Snappy compression support, used by MongoDB-related code. -**\`github.com/xdg-go/scram\`:** SCRAM authentication support for MongoDB. -**\`github.com/xdg-go/pbkdf2\`:** Password-based key derivation used in auth flows. -**\`github.com/xdg-go/stringprep\`:** String preparation used in authentication protocols. -**\`google.golang.org/protobuf\`:** Protocol Buffers support. -**\`golang.org/x/sys\`:** Low-level OS/system calls. -**\`golang.org/x/text\`:** Text encoding and normalization utilities. -**\`golang.org/x/sync\`:** Extra concurrency helpers. -**\`golang.org/x/net\`:** Extended networking libraries. - -**Python / Test Dependencies** -**\`pytest\`:** Python test runner. -**\`pymongo\`:** Python MongoDB client used by tests. -**\`selenium\`:** Browser automation for UI tests. -**\`requests\`:** HTTP client used by simulator/test tooling. - -**GitHub Actions Dependencies** -**\`actions/checkout\`:** Checks out repository code in CI. -**\`docker/login-action\`:** Logs CI into Docker Hub. -**\`docker/setup-buildx-action\`:** Enables Docker Buildx in CI. -**\`docker/build-push-action\`:** Builds and pushes Docker images. -**\`actions/setup-go\`:** Installs/configures Go in CI. -**\`actions/setup-python\`:** Installs/configures Python in CI. -**\`actions/upload-artifact\`:** Uploads generated artifacts like the report PDF. -**\`browser-actions/setup-firefox\`:** Installs Firefox for UI tests. -**\`docker/scout-action\`:** Runs Docker Scout vulnerability checks. -**\`aquasecurity/trivy-action\`:** Runs Trivy scans. -**\`golangci/golangci-lint-action\`:** Runs Go linting in CI. -**\`returntocorp/semgrep-action\`:** Runs Semgrep static analysis. -**\`softprops/action-gh-release\`:** Creates GitHub releases. -**\`zwaldowski/semver-release-action\`:** Handles semantic version release automation. -**\`zwaldowski/match-label-action\`:** Checks labels used for release/version decisions. - -\newpage +### Higher-Level Dependencies + +- **DigitalOcean Droplets** — Cloud VMs that host the production Docker Swarm nodes. +- **DigitalOcean Managed MongoDB** — Hosted production MongoDB database used by the app and bot. +- **Docker** — Runs the application and infrastructure services in containers. +- **Docker Compose** — Defines and runs the local development environment. +- **Docker Swarm** — Orchestrates the remote production stack across multiple droplets. +- **Docker Hub** — Registry where production images are pushed and pulled from. +- **Vagrant** — Tool for provisioning DigitalOcean droplets. +- **vagrant-digitalocean** — Vagrant provider plugin/box for creating DigitalOcean droplets. +- **GitHub Actions** — CI/CD platform for tests, static analysis, image builds, releases, and deployments. +- **Nginx** — Reverse proxy in front of the app and Grafana. +- **Certbot** — Automates Let's Encrypt certificate setup and renewal. +- **Let's Encrypt** — Issues TLS certificates. +- **DigitalOcean Cloud Firewall** — Manages perimeter network rules for the droplets (replaces the per-host UFW from the Vagrant era). +- **Prometheus** — Collects application and service metrics. +- **Grafana** — Displays dashboards for metrics and logs. +- **Loki** — Stores and queries logs. +- **Promtail** — Collects container logs and ships them to Loki. +- **rsyslog** — Older/auxiliary centralized log collection component. +- **Firefox / geckodriver / Selenium** — Runs browser-based UI tests. +- **Docker Scout** — Scans Docker images for vulnerabilities. +- **Trivy** — Performs container vulnerability scanning. +- **Semgrep** — Static analysis for security/code patterns. +- **CodeQL SARIF upload** — Uploads scan results to GitHub code scanning. +- **golangci-lint** — Runs Go linting. +- **Hadolint** — Lints Dockerfiles. + +### Container / Image Dependencies + +- `mongo:8.0` — Local MongoDB database image. +- `grafana/grafana:12.1` — Base image for the custom Grafana image. +- `prom/prometheus:v3.5.1` — Base image for the custom Prometheus image. +- `grafana/loki:latest` — Loki log storage/query service image. +- `grafana/promtail:latest` — Promtail log shipper image. +- `ubuntu:24.04` — Base image for the rsyslog container. +- `golang:1.25.10` — Build image for compiling the Go web app. +- `alpine:3.23` — Lightweight runtime image for the Go web app. +- `curlimages/curl` — Small image used for HTTP checks in older/local helper flows. + +### Main Go App Dependencies + +- `github.com/gorilla/mux` — HTTP router for app routes. +- `github.com/gorilla/sessions` — Cookie/session management. +- `github.com/prometheus/client_golang` — Exposes Prometheus metrics from the app. +- `go.mongodb.org/mongo-driver` — MongoDB client driver. +- `golang.org/x/crypto` — Crypto utilities, including password hashing support. + +### Discord Bot Dependencies + +- `github.com/bwmarrin/discordgo` — Discord API client library. +- `go.mongodb.org/mongo-driver` — MongoDB client driver used by the bot. + +### Important Go Indirect Dependencies + +- `github.com/gorilla/securecookie` — Secure cookie encoding used by Gorilla sessions. +- `github.com/gorilla/websocket` — WebSocket support, used indirectly by Discord integration. +- `github.com/prometheus/client_model` — Prometheus metric data model. +- `github.com/prometheus/common` — Shared Prometheus helpers. +- `github.com/prometheus/procfs` — Reads Linux process metrics for Prometheus. +- `github.com/klauspost/compress` — Compression support. +- `github.com/golang/snappy` — Snappy compression support, used by MongoDB-related code. +- `github.com/xdg-go/scram` — SCRAM authentication support for MongoDB. +- `github.com/xdg-go/pbkdf2` — Password-based key derivation used in auth flows. +- `github.com/xdg-go/stringprep` — String preparation used in authentication protocols. +- `google.golang.org/protobuf` — Protocol Buffers support. +- `golang.org/x/sys` — Low-level OS/system calls. +- `golang.org/x/text` — Text encoding and normalization utilities. +- `golang.org/x/sync` — Extra concurrency helpers. +- `golang.org/x/net` — Extended networking libraries. + +### Python / Test Dependencies + +- `pytest` — Python test runner. +- `pymongo` — Python MongoDB client used by tests. +- `selenium` — Browser automation for UI tests. +- `requests` — HTTP client used by simulator/test tooling. + +### GitHub Actions Dependencies + +- `actions/checkout` — Checks out repository code in CI. +- `docker/login-action` — Logs CI into Docker Hub. +- `docker/setup-buildx-action` — Enables Docker Buildx in CI. +- `docker/build-push-action` — Builds and pushes Docker images. +- `actions/setup-go` — Installs/configures Go in CI. +- `actions/setup-python` — Installs/configures Python in CI. +- `actions/upload-artifact` — Uploads generated artifacts like the report PDF. +- `browser-actions/setup-firefox` — Installs Firefox for UI tests. +- `docker/scout-action` — Runs Docker Scout vulnerability checks. +- `aquasecurity/trivy-action` — Runs Trivy scans. +- `golangci/golangci-lint-action` — Runs Go linting in CI. +- `returntocorp/semgrep-action` — Runs Semgrep static analysis. +- `softprops/action-gh-release` — Creates GitHub releases. +- `zwaldowski/semver-release-action` — Handles semantic version release automation. +- `zwaldowski/match-label-action` — Checks labels used for release/version decisions. + +
## Process perspective -To manage the workflow, we have created a Discord channel where we discuss our ideas and current problems and how to solve them. During the evolution process, we handled bugs that arose along the way. We gathered them on github issues, from where we had a clear view for them and assigned the person responsible for the fix. Once the solution was found, developers created a pull request to merge their branch with the fix into the development branch, where tests from CI/CD pipeline were run. After the tests were marked as successful PR required at least one review from the team member to merge into the development branch. Then we tried to create a PR from development to main every week to have continuous weekly releases. We kept naming conventions for new branches which we described in the readme file. +To manage the workflow, we created a Discord channel where we discuss ideas, raise current problems, and decide how to solve them. During the evolution of the project, we tracked bugs as GitHub issues, which gave us a clear view of open work and let us assign owners for each fix. Once a solution was ready, the developer opened a pull request from their feature branch into `development`, where the CI/CD pipeline ran the full test suite. After CI passed, the PR required at least one review from another team member before it could be merged. We aimed to open a `development` → `main` PR every week to keep a steady flow of weekly releases. Branch naming conventions are documented in the project README. ## CI/CD Pipeline: Stages and Tools -Our CI/CD pipeline ensures security and quality at every stage before deployment. The automated workflow consists of four main stages: +Our CI/CD pipeline ensures security and quality at every stage before deployment. The automated workflow consists of four main stages. + +### 1. Security Tests + +First line of defense, scanning the code and environment for vulnerabilities. + +- **Semgrep (SAST)** — Scans the raw source code to detect bad coding patterns, hardcoded secrets, and OWASP Top 10 vulnerabilities. +- **Trivy** — Scans project dependencies and OS packages for known vulnerabilities (CVEs). +- **Docker Scout** — Analyzes the compiled Docker images, generating an SBOM and ensuring the final artifact is free of critical container vulnerabilities. + +### 2. Static Tests + +Enforces code quality, readability, and best practices without running the application. + +- **Golang Linter** — Detects syntax errors, dead code, and potential bugs in the Go source code. +- **Docker Linter** — Validates Dockerfile configurations to ensure images are optimized and follow security best practices. +- **Golang Formatting** — Ensures consistent code styling across the team. + +### 3. Dynamic Tests + +Runs the compiled application to verify system behavior. + +- **Integration Tests** — Runs the provided simulator, testing thousands of user interactions with our application. +- **End-to-End (E2E) Tests** — Simulate real user interactions from start to finish, ensuring the entire ecosystem (UI, backend, and DB) functions as expected. -1\. Security Tests This stage forms the first line of defense, scanning the code and environment for vulnerabilities. +### 4. Deployment and Release -* Semgrep (SAST): Scans the raw source code to detect bad coding patterns, hardcoded secrets, and OWASP Top 10 vulnerabilities. -* Trivy: Scans project dependencies and OS packages for known vulnerabilities (CVEs). -* Docker Scout: Analyzes the compiled Docker images, generating an SBOM and ensuring the final artifact is free of critical container vulnerabilities. +Triggered only when all previous stages pass successfully. -2\. Static Tests This stage enforces code quality, readability, and best practices without running the application. +- **Create Release on GitHub** — The pipeline automatically generates a new version tag (e.g. `v1.0.2`); versions are saved in our repo, so if we ever need to roll back to a previous deployment we can do so easily. -* Golang Linter: Detects syntax errors, dead code, and potential bugs in the Go source code. -* Docker Linter: Validates Dockerfile configurations to ensure images are optimized and follow security best practices. -* Golang Formatting: Ensures consistent code styling across the team. +### Monitoring -3\. Tests (Dynamic) This stage runs the compiled application to verify system behavior. +- Response time of different endpoints across percentile methodologies (P50, P95, P99 latency). +- Total incoming HTTPS requests for endpoints. +- Overall error rate, plus 4xx and 5xx responses per endpoint. +- Request / success / error rate per endpoint. +- Grafana alert system that posts to our Discord channel when the app is down or total error rate exceeds 10%. -* Integration Tests: Runs provided simulator, testing thousands of user having interactions with our application -* End-to-End (E2E) Tests: Simulate real user interactions from start to finish, ensuring the entire ecosystem (UI, backend, and DB) functions as expected. +### Logging -4\. Deployment and Release Triggered only when all previous stages pass successfully. +We log failures for different endpoints inside webserver containers. Promtail tails the Docker container logs and ships them to Loki, where Grafana queries and visualizes them. -* Create Release on GitHub: The pipeline automatically generates a new version tag (e.g., v1.0.2), versions are saved in our repo and if we would decide that we want to return to previous deployment, we can easily make it +### Security -**Monitoring** -- We measure time respond of different endpoints using different methodologies (P50 Latency, P95 Latency, P99 Latency) -- Total incoming https request for endpoints -- Overall error rate and 4xx and 5xx responds for different endpoints -- Request / success / error rate by different endpoints - -**Logging** - We are logging failures for different endpoints inside webserver containers. We use promtail to go through the docker container logs and aggregate them for grafana. -**Security** - We added following following technologies: -- **UFW Firewall (Perimeter Defense):** The host firewall is strictly configured to deny unauthorized external traffic. Public access is restricted entirely to secure management via OpenSSH and encrypted web traffic via the `Nginx Full` profile (which opens HTTP Port 80 and HTTPS Port 443). -- **Nginx & HTTPS:** Nginx acts as a reverse proxy, buffering the public internet from our internal services. All data in transit is fully encrypted using HTTPS (SSL/TLS). -- **Password Hashing:** User credentials are cryptographically hashed before being stored in the database, ensuring no plaintext passwords are ever saved. -- **Automated CI/CD Scans:** Our DevSecOps pipeline actively blocks vulnerabilities before deployment using **Semgrep** (source code analysis), **Trivy** (dependency scanning), and **Docker Scout** (container image CVEs). -- Our docker container images are running as a non-root user except for promtail. We kept promtail user as a root, because it needs more rights to read logs from other containers. +- **DigitalOcean Cloud Firewall (Perimeter Defense)** — A managed firewall at the cloud-provider level denies all external traffic by default. Only ports 22 (OpenSSH), 80 (HTTP), and 443 (HTTPS) are publicly reachable; swarm-internal ports (2377, 7946, 4789) are restricted to droplets carrying the `minitwit` tag. +- **Nginx & HTTPS** — Nginx acts as a reverse proxy, buffering the public internet from our internal services. All data in transit is fully encrypted using HTTPS (SSL/TLS). +- **Password Hashing** — User credentials are cryptographically hashed before being stored in the database, ensuring no plaintext passwords are ever saved. +- **Automated CI/CD Scans** — Our DevSecOps pipeline actively blocks vulnerabilities before deployment using **Semgrep** (source code analysis), **Trivy** (dependency scanning), and **Docker Scout** (container image CVEs). +- Our Docker container images run as a non-root user, with the exception of Promtail — it needs elevated rights to read logs from other containers. +### Scaling and Availability -**Scaling and availability** - We have implemented docker swarm to help with scaling. Furthermore, we are using rolling updates to reduce down time as much as possible. We have 3 replicas for webserver images and a promtail replica for each of webservers to gather all the necessary logs. +We use Docker Swarm to scale the application horizontally. Rolling updates keep downtime as low as possible during deploys: 3 webserver replicas plus one Promtail replica per node for log collection. -\newpage +
+ +## Current state of the system + +We ran CI's static-analysis stack (golangci-lint, hadolint, Trivy, Semgrep) against `development` on 2026-05-17. + +| Tool | Scope | Result | +|---|---|---| +| golangci-lint | `src/` | 0 issues | +| golangci-lint | `src/bot/` | 3 errcheck issues | +| hadolint | 6 Dockerfiles | 0 issues | +| Trivy — CVEs | web + bot images | 0 | +| Trivy — Dockerfile misconfig | 6 Dockerfiles | 4 HIGH, 6 LOW | +| Trivy — secrets | tracked tree | 0 | +| Semgrep (364 rules) | 93 files | 9 blocking | + +For more context ragrding errors refer to CI/CD pipelines. + +
## Reflection Perspective -**Evolution and refactoring** -The app evolved continuously. First we rewrote minitwit in GoLang and MongoDB. Then we introduced the first version of CI/CD pipeline, which only built and published the latest version of the app to the digital ocean host. Then we further improved the quality of the codebase by structuring the project, adding missing features and implementing new ones. Later we improved our CI/CD pipeline to include various security and quality tests. +### Evolution and Refactoring + +The app evolved continuously. First we rewrote MiniTwit in Go and MongoDB. Then we introduced the first version of our CI/CD pipeline, which only built and published the latest version of the app to the DigitalOcean host. Next we improved the codebase by restructuring the project, adding missing features, and implementing new ones. We later expanded the CI/CD pipeline to include various security and quality tests. Most recently, we added infrastructure provisioning with Terraform, replacing imperative VM creation with a declarative IaC setup that brings up droplets, Cloud Firewall, managed MongoDB, the Swarm bootstrap, and the full app deploy in a single `terraform apply`. + +### Operation and Maintenance + +After encountering our first errors, we added a custom logging system that collected all errors caught inside the handler and stored them in separate files. Later, we transitioned to specialized technologies, using Promtail to collect log lines from different containers and Loki to store them. + +### Encountered Problems + +We encountered multiple problems during development and evolution. The main ones: + +- **Multi-day outage from DB non-indexing** — Lack of indexes caused the droplet to silently run out of memory and crash. We didn't notice for 4–5 days. +- **MongoDB ransomware attack** — MongoDB was reachable from outside the container host on port 27017 with no root username/password configured. A bot attack wiped some tables. +- **Base image selection** — Some barebones Linux distros didn't have `curl` available, which broke the CI/CD pipeline. +- **Swarm creation mishap** — We forgot to add additional droplets on DigitalOcean and mixed up network and volume names, causing a 30-minute downtime. +- **Volume export issues in early logging** — Our first logging system couldn't export logs outside the container. Rather than fixing it, we moved to Promtail and Loki. +- **Testing CI/CD safely** — Some CI/CD changes couldn't be tested directly in the main repository because they need teammate approval, so we used a separate sandbox repo with full rights. +- **Terraform `cloud-init` self-deadlock** — `cloud-init status --wait` was being called from inside `runcmd`, so cloud-init was waiting on itself. Docker never installed and the Swarm bootstrap hung for ~45 minutes before we caught it. +- **DigitalOcean Terraform provider MongoDB-user bug** — Provider v2.86 doesn't send the `user_settings` field that the MongoDB API now requires, so `digitalocean_database_user` fails on apply. We worked around it by using the cluster's auto-created `doadmin` user instead of a custom one. -**Operation maintenance:** +### Use of Generative AI -- After encountering our first errors, we added a custom logging system that collected all errors caught inside the handler and stored them in separate files. Later, we transitioned to specialized technologies, using Promtail to collect log lines from different containers and Loki to store them. - -\newpage +**Tools used:** -**Encountered problems** -We have encountered multiple problems during the development and evolution cycle. Main ones are: +- **Codex 5.5** — Review of our Docker stack. +- **Claude 4.7 and Gemini** — Code review, troubleshooting, and explaining concepts we couldn't solve on our own. +- **GitHub Copilot** — Coding suggestions during day-to-day work. -* DB non-indexing caused the droplet to be down for multiple days before we even noticed. Droplet ran out of memory and crashed everything. This has caused a downtime of 4-5 days; -* A ransomware bot attack stole some tables due to the DB port being open. MongoDB was reachable from outside the container host on port 27017, and there was no Mongo root username/password configured. -* Virtual machine selection for docker images (some barebones linux distros did not have curl for example) and it failed the ci/cd pipeline; -* During docker swarm creation we forgot to add additional droplets on DigitalOcean and mixed up network and volume names which resulted in a 30 minute downtime; -* In our first logging system we struggled with volumes in our server, where we couldn’t export logs and store them outside of the container. Instead of fixing this we moved to promtail and loki. -* Some CI/CD operations couldn't be tested directly in our repository, because changes need to be accepted by other team members. That’s why when we were testing github action we used separate repository which was used as sandbox, with full rights +Generative AI significantly supported our workflow. With these tools, our project accelerated noticeably — especially when we made errors and the AI models were able to detect them and suggest repair steps. They also helped us understand new concepts from lectures and exercises. We remained cautious and manually verified suggestions, as the models occasionally lacked the full context of our specific system architecture. -**Use of Generative AI** -Technologies which we used: +
-Codex 5.5: for the review of our Docker stack. -Powerful models such as Claude 4.7 and Gemini: for code review, troubleshooting, and explaining concepts which we couldn't solve by ourselves. -GitHub Copilot: for coding tips during our work process. +## Reflections on the work process -Reflections on the work process: -Generative AI significantly supported our workflow. With these tools, our project accelerated noticeably, especially when we made errors and the AI models were able to detect them and suggest repair steps. What's more, not only did they help us solve ongoing problems, but they also guided us in understanding new concepts coming from lectures and exercises. However, we had to remain cautious and manually verify the AI's suggestions, as it occasionally lacked the full context of our specific system architecture. \ No newline at end of file +During this project we tried to follow DevOps practices. We communicated actively throughout the developmennt process to indentify and solve issues as they came up. We tried to manage the workload and distribute tasks. +Furthermore, we tried to experiment and improvise during the development as much as time allowed. +We followed weekly release schedule and automated various tests and checks to avoid unnecessary manual work. We utilized GitHub infrastructure to check each others work using Pull Requests. +We have learned what works and on what we should work on to improve ourselves as developers in the future.