Final release hopefully

.github/workflows/release.yml

@@ -51,4 +51,5 @@ jobs:
         with:
           tag_name: ${{ steps.next_version.outputs.version }}
           name: Release ${{ steps.next_version.outputs.version }}
-          generate_release_notes: true
+          generate_release_notes: true
+          files: report/build/MSc_group_p.pdf

Makefile

@@ -106,11 +106,12 @@ ui-e2e:
 report:
 	@echo "$(CYAN)==> Building report PDF...$(RESET)"
 	mkdir -p report/build
-	SOURCE_DATE_EPOCH=0 pandoc $(REPORT_SOURCE) --from=markdown --pdf-engine=xelatex --toc --number-sections --metadata=date= --output=$(REPORT_OUTPUT)
+	npx --yes md-to-pdf $(REPORT_SOURCE) --basedir report
+	mv report/report.pdf $(REPORT_OUTPUT)
 	@echo "$(GREEN)Report built at $(REPORT_OUTPUT)$(RESET)"
 
 report-word-count:
-	@pandoc $(REPORT_SOURCE) -t plain | wc -w
+	@python3 -c 'import re, pathlib; text = pathlib.Path("$(REPORT_SOURCE)").read_text(encoding="utf-8"); text = re.sub(r"---\n.*?\n---\n", "", text, flags=re.S); text = re.sub(r"```.*?```", "", text, flags=re.S); text = re.sub(r"<[^>]+>", "", text); text = re.sub(r"\[[^\]]+\]\([^)]+\)", "", text); print(len(re.findall(r"\b[\w'\''-]+\b", text)))'
 
 # --------- Execute tests --------------------------
 

README.md

@@ -1,49 +1,143 @@
-# Project readme <br>
+# MiniTwit (ITU DevOps)
 
-## Git and branches <br> 
-Naming convention:
-{fix/feature}//{short_message}
+A Twitter-clone. The original Python/Flask + SQLite app has been rewritten in Go + MongoDB and runs on a Docker Swarm of DigitalOcean droplets, with Prometheus/Grafana/Loki for observability and a Discord bot for alerts.
 
-## CI/CD
-### Project infrastructure is setup with Vagrant. Run _vagrant up_ to create virtual machines (droplets) on Digital Ocean.
-### Everytime a merge is done into the _main_ branch GithubActions perform continuous deployment to update the project with latest changes. 
+The service exposes two interfaces on the same port:
+- UI — server-rendered HTML for end users (`/`, `/login`, `/timeline`, `/user/{username}`, …)
+- Simulator API — JSON endpoints consumed by the simulator (`/register`, `/msgs`, `/fllws/{username}`, `/latest`, …)
 
-## How to run env locally
-### 1. Prepare local prerequisites for `vagrant up`
-    export DIGITAL_OCEAN_TOKEN=****
+Public site: <https://itu-minitwit.me>
 
-Create an SSH key pair locally at:
-    ~/.ssh/ssh_key_golang_minitwit
+---
 
-Add the matching public key to DigitalOcean with this exact key name:
-    ssh_key_golang_minitwit
+## Repository layout
 
-Optional if you want TLS/Nginx bootstrap during the same run:
-    export TLS_DOMAIN=your-domain.com
-    export TLS_EMAIL=you@example.com
+| Path | Purpose |
+|---|---|
+| [src/](src/) | Go application (gorilla/mux router, handlers, middleware, DB helpers) |
+| [src/bot/](src/bot/) | Discord bot — separate Go module with its own `go.mod`, built into its own image |
+| [docker/](docker/) | Dockerfiles for the web and bot images |
+| [monitoring/](monitoring/) | Prometheus, Grafana, Loki, Promtail configs and dashboards |
+| [infra/](infra/) | Terraform definition of the production infra (droplets, firewall, MongoDB)|
+| [remote_files/](remote_files/) | Files copied to the Swarm manager: authoritative `docker-stack.yml`, `deploy.sh`, TLS bootstrap |
+| [.github/workflows/](.github/workflows/) | CI (static analysis, tests, E2E) and CD (build → push → deploy) |
+| [Vagrantfile](Vagrantfile), [setup-swarm.sh](setup-swarm.sh) | Legacy provisioning path (still works) |
+| [docker-compose.yml](docker-compose.yml) | Local dev stack |
+| [report/](report/) | Report in PDF format |
 
-### 2. Create an `.env` file in root folder and setup deployment environment variables
-    DOCKER_USERNAME=****
-    DISCORD_TOKEN=****
-    GRAFANA_ADMIN_USER=****
-    GRAFANA_ADMIN_PASSWORD=****
-    MONGO_URI=****
+---
 
-`vagrant up` creates the droplets and then automatically runs `setup-swarm.sh`, so these variables must be available locally when you run it.
+## Local development
 
-### 3. Run this command to create infrastructure and deploy the Swarm stack
-    vagrant up
+Requirements: Docker + Docker Compose. Go 1.22+ only needed for running tests outside containers.
 
-### 4. Run this command to start docker locally
-    docker compose up --build
+```bash
+docker compose up --build      # web on :8080, MongoDB, Prometheus, Grafana, Loki, Promtail
+docker compose down -v
+```
 
-#### 4.1 Re-run deployment, routing, or TLS setup on existing droplets
-    vagrant rsync
-    export TLS_DOMAIN=your-domain.com
-    export TLS_EMAIL=you@example.com
-    ./setup-swarm.sh
+Run the Go service directly (needs a MongoDB on `localhost:27017`):
 
-`setup-swarm.sh` loads deployment variables from the local `.env` file or the current shell environment and passes them directly to the manager node. The `.env` file is no longer synced through `Vagrantfile`.
+```bash
+cd src && go run main.go
+```
 
-### 5. API test
-To test API you can use _test-api-routes.sh_ script.
+Common dev commands:
+
+```bash
+make verify                            # full local CI check (build, sim, lint, UI E2E)
+make test-sim                          # run the grading simulator against :8080
+make ui-e2e                            # Selenium UI tests (needs geckodriver)
+./test-api-routes.sh                   # API smoke tests
+```
+
+---
+
+## Production infrastructure
+
+Production runs a Docker Swarm of three DigitalOcean droplets (1 manager + 2 workers) with a managed MongoDB cluster. DNS for `itu-minitwit.me` lives at Namecheap (not IaC-managed).
+
+There are two provisioning paths:
+
+### Terraform
+
+```bash
+cd infra
+cp terraform.tfvars.example terraform.tfvars   # fill in tls_email, discord_token, grafana_admin_password, …
+export DIGITALOCEAN_TOKEN=<your-token>
+terraform init
+terraform plan
+terraform apply                                 # ~5–8 min; provisions droplets + MongoDB + deploys the stack
+```
+
+Outputs include the manager IP and MongoDB URI needed for the `SSH_HOST` and `MONGO_URI` GitHub Secrets.
+
+### Vagrant
+
+The original path is still functional. Requires the same `DIGITALOCEAN_TOKEN`, an SSH key at `~/.ssh/ssh_key_golang_minitwit` registered in DO under the same name, and a local `.env` with `DOCKER_USERNAME`, `MONGO_URI`, `DISCORD_TOKEN`, `GRAFANA_ADMIN_USER`, `GRAFANA_ADMIN_PASSWORD`.
+
+```bash
+vagrant up                              # creates droplets, then runs setup-swarm.sh which deploys the stack
+vagrant rsync && ./setup-swarm.sh       # re-deploy onto existing droplets
+vagrant provision                       # runs the script in Vagrant file
+```
+
+`MONGO_URI` should point to an external managed MongoDB (e.g. DO Managed MongoDB or Atlas).
+
+### TLS / nginx
+
+The reverse proxy (nginx + Let's Encrypt) is bootstrapped once per manager with [remote_files/bootstrap_droplet_tls.sh](remote_files/bootstrap_droplet_tls.sh). It is rerun automatically by the CD workflow's post-deploy verification step.
+
+---
+
+## How to contribute changes
+
+### Branch naming
+
+```
+{fix|feature}/{short-message}
+```
+
+Examples: `feature/terraform`, `fix/login-redirect`.
+
+### Workflow
+
+1. Branch off `development`.
+2. Push your branch and open a PR into `development`. CI runs automatically.
+3. Once your PR is merged into `development`, the same checks run again. When a release is ready, `development` is merged into `main`.
+4. Pushing to `main` triggers continuous deployment to production.
+
+### CI on push/PR (`development`, `main`)
+
+[`static-analysis.yml`](.github/workflows/static-analysis.yml) builds the images and runs:
+- Docker Scout + Trivy + Semgrep security scans
+- `make verify` (simulator test, `go fmt`, `golangci-lint`, Hadolint)
+- `make ui-e2e` (Selenium UI E2E tests)
+
+
+### CD on push to `main`
+
+[`continous-deployment.yml`](.github/workflows/continous-deployment.yml):
+1. Builds and pushes the web, bot, Prometheus, and Grafana images to Docker Hub
+2. SCPs deploy files and a freshly rendered `.env` to the Swarm manager
+3. Runs [`remote_files/deploy.sh`](remote_files/deploy.sh) on the manager
+4. Verifies the deploy: nginx config, certbot renewal, TLS chain
+
+The required GitHub Secrets are: `DOCKER_USERNAME`, `DOCKERHUB_AUTHTOKEN`, `SSH_HOST`, `SSH_USER`, `SSH_PRIVATE_KEY`, `MONGO_URI`, `DISCORD_TOKEN`, `GRAFANA_ADMIN_USER`, `GRAFANA_ADMIN_PASSWORD`, `TLS_DOMAIN`, `TLS_EMAIL`.
+
+## Observability
+
+- Prometheus scrapes the webserver's `/metrics` (custom counters/histograms in [src/middleware/metrics.go](src/middleware/metrics.go)).
+- Grafana is served at `/grafana/` in production (dashboards and datasources provisioned from [monitoring/grafana/](monitoring/grafana/)).
+- Loki + Promtail collect container logs; Promtail runs `mode: global` (one per Swarm node) reading `/var/lib/docker/containers/`.
+- Discord bot posts alerts to a Discord channel via the token in `DISCORD_TOKEN`.
+
+Each request gets an `X-Request-ID` propagated through context; all user-supplied values are sanitized before logging.
+
+---
+
+## Reports & releases
+
+```bash
+make report      # build the course report PDF
+```

Vagrantfile

@@ -55,11 +55,22 @@ Vagrant.configure("2") do |config|
         # Allowing SSH connections
         sudo ufw allow "OpenSSH"
 
-        # Open the published MiniTwit HTTP port.
+        # Open public HTTP/HTTPS only through Nginx.
         sudo ufw allow 80/tcp
-
-        # Open Grafana for external dashboard access.
-        sudo ufw allow 3000/tcp
+        sudo ufw allow 443/tcp
+
+        # Block direct access to services that should only be reached through Nginx.
+        sudo ufw deny 3000/tcp
+        sudo ufw deny 8080/tcp
+
+        # Docker can publish ports before UFW sees the traffic. Block public
+        # access to those published container ports in Docker's user chain using iptables firewall.
+        sudo iptables -C DOCKER-USER -p tcp ! -s 127.0.0.0/8 --dport 3000 -j DROP 2>/dev/null || sudo iptables -I DOCKER-USER -p tcp ! -s 127.0.0.0/8 --dport 3000 -j DROP
+        sudo iptables -C DOCKER-USER -p tcp ! -s 127.0.0.0/8 --dport 8080 -j DROP 2>/dev/null || sudo iptables -I DOCKER-USER -p tcp ! -s 127.0.0.0/8 --dport 8080 -j DROP
+        echo iptables-persistent iptables-persistent/autosave_v4 boolean true | sudo debconf-set-selections
+        echo iptables-persistent iptables-persistent/autosave_v6 boolean true | sudo debconf-set-selections
+        sudo DEBIAN_FRONTEND=noninteractive apt-get install -y iptables-persistent
+        sudo netfilter-persistent save
 
         # Enable the firewall only after SSH is allowed, otherwise provisioning
         # risks locking us out on a fresh host.

docker-compose.yml

@@ -74,6 +74,7 @@ services:
       - grafana_cloud_data:/var/lib/grafana
       - ./monitoring/grafana/datasources:/etc/grafana/provisioning/datasources:ro
       - ./monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards:ro
+      - ./monitoring/grafana/alerting:/etc/grafana/provisioning/alerting:ro
     deploy:
       placement:
         constraints:

infra/.gitignore

@@ -1,10 +1,8 @@
-# Local terraform state and lock data — never commit.
 .terraform/
 .terraform.lock.hcl
 terraform.tfstate
 terraform.tfstate.backup
 crash.log
 
-# Real variable values (may contain secrets).
 terraform.tfvars
 *.auto.tfvars

infra/main.tf

@@ -63,17 +63,6 @@ resource "digitalocean_firewall" "swarm" {
     port_range       = "443"
     source_addresses = ["0.0.0.0/0", "::/0"]
   }
-  inbound_rule {
-    protocol         = "tcp"
-    port_range       = "3000"
-    source_addresses = ["0.0.0.0/0", "::/0"]
-  }
-  inbound_rule {
-    protocol         = "tcp"
-    port_range       = "8080"
-    source_addresses = ["0.0.0.0/0", "::/0"]
-  }
-
   # Swarm node-to-node communication, restricted to the cluster's own droplets.
   inbound_rule {
     protocol    = "tcp"

monitoring/grafana/Dockerfile

@@ -2,5 +2,6 @@ FROM grafana/grafana:12.1
 
 COPY dashboards /etc/grafana/provisioning/dashboards
 COPY datasources /etc/grafana/provisioning/datasources
+COPY alerting /etc/grafana/provisioning/alerting
 
-EXPOSE 3000
+EXPOSE 3000