Sync fsicos2 fail2ban configuration

devops/files/dokku/deploy-trajectory-viewer

@@ -0,0 +1,31 @@
+#!/bin/bash
+set -euo pipefail
+BRANCH=${1:-trajectory_styled}
+REPO=/opt/trajectory-viewer
+SYNC_REPO=/var/lib/trajectory-viewer-sync.git
+
+echo "==> Deploying branch: $BRANCH"
+cd "$REPO"
+
+FETCHED=0
+if GIT_SSH_COMMAND="ssh -i /root/.ssh/id_rsa_github -o IdentitiesOnly=yes" git fetch origin; then
+  FETCHED=1
+  echo "==> GitHub fetch succeeded"
+else
+  echo "==> Warning: git fetch origin failed; deploying current local branch state" >&2
+fi
+
+git checkout "$BRANCH"
+if [ "$FETCHED" -eq 1 ]; then
+  git merge --ff-only "origin/$BRANCH"
+fi
+
+python3 -m py_compile vsplot/backend/app.py vsplot/backend/callbacks.py vsplot/backend/matomo.py
+echo "==> Syntax OK"
+rm -rf "$SYNC_REPO"
+git clone --bare --no-local "$REPO" "$SYNC_REPO"
+chmod -R a+rX "$SYNC_REPO"
+sudo -u dokku git config --global --add safe.directory "$SYNC_REPO"
+dokku git:sync --build trajectory-viewer "file://$SYNC_REPO" "$BRANCH"
+echo "==> Deploy complete"
+dokku logs trajectory-viewer --tail 20

devops/host_vars/dokku.yml

@@ -0,0 +1,40 @@
+dokku_hostname: dokku
+
+dokku_github_public_key: >-
+  ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDQZTvlk/bboloJVMfjGmy2kgbqig8Zop0f47nV0TNjArYTpqMq3gGIfNM9Ae+WbBCdpyIEq/4sxRbMyQXMmH42rvXiAM7ArJhyKMswGWYAVxUi4U7EFhGqTi4hyaA3b0/tm1vN4IisjKxleXthPOUvbcSDx210r6kZ5azu1WdBBBAwCDzi7FJhalPPj6QH7+5lmueeIzc4qRPJaAB3FINxS9PnUd+AxtVNJMqoi6nfDhQGhzhbfQs5vfilxlXNnp4XbpCK9xHhAHUGas/uaGvqV06WIm7VWQX1Lmbr4bjpVMXOTXVA7hjokGFkh6vDVe28UeCNQjw6PdQ6ErwxblxuMYMC7CNGXYHxLxpZb+It2S0VEUhqls4vsMnYSk1rY0i3bdeyG3k1llTDRV+yLluoG/h0puWnqHXFszO1kTx258vJku3HR44+nuGmoTuBAsnLe8LxPPlbr0KKZlBsN6ng0S6SIP8dvgGlYTis7Ce/waXygGC68bkkbKgOqPFx3VebWujFi+RamLsof/EFS4ezWj1fYHIgV3KSVH/QuE/Oe3wwJPYHFfH6JngfBumZZRcGUMTl4Y19Ut6c1/l4ARy1ETG9KKSnrb8KOd4zeihBzaIkzHaEfGdS6pXQ8slKqrTcdCc6Sv+tobUq3JjRF/ilVx5DRi0jOCH/+7N//9adlQ==
+  Roberts Github ssh key
+
+dokku_trajectory_viewer_name: trajectory-viewer
+dokku_trajectory_viewer_repo: git@github.com:ICOS-Carbon-Portal/trajectory-viewer.git
+dokku_trajectory_viewer_branch: main
+dokku_trajectory_viewer_repo_dir: /opt/trajectory-viewer
+dokku_trajectory_viewer_domain: trajectory.icos-cp.eu
+dokku_trajectory_viewer_storage_mount: /root/data:/data:ro
+dokku_trajectory_viewer_port_map: http:80:8050
+dokku_trajectory_viewer_env:
+  APP_URL: "https://trajectory.icos-cp.eu/"
+  DATA_DIR: /data/trj
+  DOKKU_APP_RESTORE: "1"
+  DOKKU_PROXY_PORT: "80"
+  ICOS_CHROME: "0"
+  ICOS_CHROME_ENABLED: "1"
+  MATOMO_AUTH_TOKEN: 0fcf14d8c6125a74ee621ccf6cf66dc1
+  MATOMO_SITE_ID: "12"
+  MATOMO_URL: https://matomo.icos-cp.eu/matomo.php
+  PORT: "8050"
+  URL_BASE_PATH: /
+
+icosdata_mkdirs:
+  - /data
+  - /data/flexextract4
+  - /data4
+
+icosdata_nfs_mounts:
+  - path: /data4
+    src: fsicos4.int:/tank/data/flexpart
+    fstype: nfs
+    opts: "rw,vers=3,_netdev,hard,timeo=600,retrans=2"
+  - path: /data/flexextract4
+    src: fsicos4.int:/tank/flexextract
+    fstype: nfs
+    opts: "rw,vers=3,_netdev,hard,timeo=600,retrans=2"

devops/host_vars/fsicos2.yml

@@ -5,6 +5,157 @@ bbclient_remotes:
   - fsicos2
   - icos1
 
+fail2ban_config_files:
+  - dest: /etc/fail2ban/jail.d/sshd-allports.local
+    content: |
+      # Since we're running ssh on different ports that fail2ban is used to,
+      # block all ports.
+      [sshd]
+      banaction = iptables-allports
+
+  - dest: /etc/fail2ban/jail.local
+    content: |
+      [DEFAULT]
+      bantime  = 24h
+      findtime = 10m
+      maxretry = 10
+      banaction = iptables-allports
+
+      # -----------------------------------------------
+      # nginx — malicious path detection
+      # Strict — 3 hits = ban. These are always attacks.
+      # -----------------------------------------------
+      [nginx-malicious]
+      enabled   = true
+      port      = http,https
+      filter    = nginx-malicious
+      logpath   = /var/log/nginx/access.log
+                  /var/log/nginx/doi.log
+                  /var/log/nginx/erddap.log
+                  /var/log/nginx/eric-forum.log
+                  /var/log/nginx/kanidm.log
+                  /var/log/nginx/keycloak.log
+                  /var/log/nginx/mailman.log
+                  /var/log/nginx/maps.log
+                  /var/log/nginx/matomo.log
+                  /var/log/nginx/plausible.log
+                  /var/log/nginx/quince.log
+                  /var/log/nginx/rspamd.log
+                  /var/log/nginx/typesense.log
+                  /var/log/nginx/vmagent.log
+                  /var/log/nginx/wordpress.log
+      maxretry  = 3
+      findtime  = 10m
+      bantime   = 48h
+      banaction = iptables-multiport
+
+      # -----------------------------------------------
+      # nginx — 4xx flood (20 hits / 2m)
+      # -----------------------------------------------
+      [nginx-4xx]
+      enabled   = true
+      port      = http,https
+      filter    = nginx-4xx
+      logpath   = /var/log/nginx/access.log
+                  /var/log/nginx/doi.log
+                  /var/log/nginx/erddap.log
+                  /var/log/nginx/eric-forum.log
+                  /var/log/nginx/kanidm.log
+                  /var/log/nginx/keycloak.log
+                  /var/log/nginx/mailman.log
+                  /var/log/nginx/maps.log
+                  /var/log/nginx/matomo.log
+                  /var/log/nginx/plausible.log
+                  /var/log/nginx/quince.log
+                  /var/log/nginx/rspamd.log
+                  /var/log/nginx/typesense.log
+                  /var/log/nginx/vmagent.log
+                  /var/log/nginx/wordpress.log
+      maxretry  = 120
+      findtime  = 10m
+      bantime   = 1h
+      banaction = iptables-multiport
+
+      # -----------------------------------------------
+      # nginx — bot/vulnerability scanning
+      # -----------------------------------------------
+      [nginx-botsearch]
+      enabled   = true
+      port      = http,https
+      filter    = nginx-botsearch
+      logpath   = /var/log/nginx/access.log
+                  /var/log/nginx/doi.log
+                  /var/log/nginx/erddap.log
+                  /var/log/nginx/eric-forum.log
+                  /var/log/nginx/kanidm.log
+                  /var/log/nginx/keycloak.log
+                  /var/log/nginx/mailman.log
+                  /var/log/nginx/maps.log
+                  /var/log/nginx/matomo.log
+                  /var/log/nginx/plausible.log
+                  /var/log/nginx/quince.log
+                  /var/log/nginx/rspamd.log
+                  /var/log/nginx/typesense.log
+                  /var/log/nginx/vmagent.log
+                  /var/log/nginx/wordpress.log
+      maxretry  = 5
+      findtime  = 10m
+      bantime   = 48h
+      banaction = iptables-multiport
+
+      # -----------------------------------------------
+      # nginx — HTTP basic auth brute force
+      # -----------------------------------------------
+      [nginx-http-auth]
+      enabled   = true
+      port      = http,https
+      filter    = nginx-http-auth
+      logpath   = /var/log/nginx/error.log
+      maxretry  = 5
+      findtime  = 10m
+      bantime   = 24h
+      banaction = iptables-multiport
+
+      # -----------------------------------------------
+      # nginx — rate limit violations (DISABLED — no limit_req in nginx config)
+      # -----------------------------------------------
+      [nginx-limit-req]
+      enabled   = false
+
+      # -----------------------------------------------
+      # Recidive — repeat offenders get all-ports 4-week ban
+      # -----------------------------------------------
+      [recidive]
+      enabled   = true
+      filter    = recidive
+      logpath   = /var/log/fail2ban.log
+      maxretry  = 3
+      findtime  = 1d
+      bantime   = 4w
+      banaction = iptables-allports
+
+  - dest: /etc/fail2ban/jail.d/whitelist.local
+    content: |
+      [DEFAULT]
+      ignoreip = 158.248.42.234 158.39.74.161 129.177.63.11 66.249.77.0/24 129.132.144.22 10.7.95.0/24 130.235.0.0/16 172.16.0.0/12 10.7.89.0/24 193.205.145.87 80.187.119.148 195.254.249.168 80.187.107.166 109.37.152.211 147.100.95.201 5.180.2.164 109.37.149.230 128.214.251.97
+
+  - dest: /etc/fail2ban/filter.d/nginx-4xx.conf
+    content: |
+      [Definition]
+      failregex = ^<HOST> - \S+ \[.*?\] "(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \S+ HTTP/\d+\.\d+" (400|401|403|404|405|444) \d+
+      ignoreregex = \.(css|js|png|jpg|jpeg|gif|ico|woff|woff2|ttf|svg)(\?[^ ]*)? HTTP
+
+  - dest: /etc/fail2ban/filter.d/nginx-malicious.conf
+    content: |
+      [Definition]
+      failregex = ^<HOST> - \S+ \[.*?\] "(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) /\S*(wp-admin|wp-login\.php|phpmyadmin|xmlrpc\.php|cgi-bin|actuator|console|boaform|setup\.cgi) HTTP/\S+" \d+ \d+ ".*?" ".*?" ".*?"$
+                  ^<HOST> - \S+ \[.*?\] "(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) /\S*\.env\S* HTTP/\S+" \d+ \d+ ".*?" ".*?" ".*?"$
+                  ^<HOST> - \S+ \[.*?\] "(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) /\S*(\.git|passwd|\.aws|\.ssh|backup|shell|eval) HTTP/\S+" \d+ \d+ ".*?" ".*?" ".*?"$
+                  ^<HOST> - \S+ \[.*?\] "(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) /\S*(\.\.) HTTP/\S+" \d+ \d+ ".*?" ".*?" ".*?"$
+                  ^<HOST> - \S+ \[.*?\] "(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) /\S*(cmd|exec|system|phpinfo)=\S* HTTP/\S+" \d+ \d+ ".*?" ".*?" ".*?"$
+      ignoreregex =
+      # datepattern line removed — fail2ban auto-detects nginx bracketed timestamps
+
 # PROMETHEUS
 nginx_metrics_enable: True
 

devops/host_vars/fsicos4.yml

@@ -0,0 +1,3 @@
+icosdata_exports: |
+  /tank/flexextract 10.10.10.145/32(rw,sync,no_subtree_check,no_root_squash)
+  /tank/data/flexpart 10.10.10.145/32(rw,sync,no_subtree_check,no_root_squash)

devops/icosdata.yml

@@ -18,8 +18,10 @@
 - hosts:
     - fsicos2
     - fsicos3
+    - fsicos4
     - icos1
     - cupcake
+    - dokku
     - pancake
 
   tags:
@@ -79,7 +81,7 @@
       name: Mount nfs data
       tags: mount
       mount:
-        fstype: nfs4
+        fstype: "{{ item.fstype | default('nfs4') }}"
         state: "{{ item.state | default('mounted') }}"
         # The next two default to omit so that they can be left out when state
         # is "unmounted".

devops/production.inventory/hosts.yml

@@ -70,6 +70,10 @@ all:
         cupcake:
           ansible_port: 60603
           user_conf: "{{ vault_cupcake_user_conf }}"
+        dokku:
+          ansible_port: 60613
+          ansible_user: root
+          root_keys: "{{ vault_dokku_root_keys }}"
         pancake:
           ansible_port: 60605
           ansible_user: root
@@ -79,6 +83,9 @@ all:
         fdp:
           ansible_port: 60606
           ansible_user: fdp
+        fsicos4-flexpart:
+          ansible_port: 60611
+          ansible_user: root
 
 
     # CDB VMS
@@ -125,6 +132,8 @@ all:
       vars:
         ansible_host: fsicos3
       hosts:
+        lxc-sftp:
+          ansible_port: 60580
         amalthea:
           ansible_port: 60560
         callisto:
@@ -160,6 +169,3 @@ all:
           ansible_port: 60570
         ctehires:
           ansible_port: 60575
-        dokku:
-          root_keys: "{{ vault_dokku_root_keys }}"
-          ansible_port: 60595

devops/roles/icos.docker2/tasks/ubuntu.yml

@@ -1,8 +1,14 @@
 # https://docs.docker.com/engine/install/ubuntu/
+- name: Ensure apt keyrings directory exists
+  file:
+    path: /etc/apt/keyrings
+    state: directory
+    mode: "0755"
+
 - name: Add docker key
   ansible.builtin.get_url:
     url: https://download.docker.com/linux/ubuntu/gpg
-    dest: /etc/apt/trusted.gpg.d/docker.asc
+    dest: /etc/apt/keyrings/docker.asc
     mode: '0644'
     force: true
   register: _key

devops/roles/icos.dokku/defaults/main.yml

@@ -1,7 +1,9 @@
 # PARAMETERS
 
 # The domain from which we're serving dokku apps.
-# dokku_hostname
+dokku_hostname: "{{ inventory_hostname }}"
+dokku_vhost_enable: true
+dokku_nginx_enable: true
 
 # CONFIG
 dokku_ssh_port: 3022

devops/roles/icos.dokku/tasks/install.yml

@@ -23,17 +23,18 @@
     vtype: "{{ item.vtype }}"
   loop:
     - question: "dokku/vhost_enable"
-      value: "true"
+      value: "{{ 'true' if dokku_vhost_enable else 'false' }}"
       vtype: boolean
     - question: "dokku/hostname"
-      value: "dokku.fsicos3.icos-cp.eu"
+      value: "{{ dokku_hostname }}"
       vtype: "string"
     - question: "dokku/nginx_enable"
-      value: "true"
+      value: "{{ 'true' if dokku_nginx_enable else 'false' }}"
       vtype: "boolean"
 
 - name: Install dokku
   apt:
     name:
       - dokku
+      - git
 

devops/roles/icos.lxc_sftp/defaults/main.yml

@@ -0,0 +1,8 @@
+lxc_sftp_base_dir: /srv/sftp
+
+# List of sftp users.
+# Each entry:
+#   name: username
+#   pubkey: "ssh-rsa ..."      (optional, but recommended)
+#   password: "plaintext"      (optional, avoid in production)
+lxc_sftp_users: []

devops/roles/icos.lxc_sftp/tasks/main.yml

@@ -0,0 +1,2 @@
+- import_tasks: setup.yml
+  tags: lxc_sftp_setup

devops/roles/icos.lxc_sftp/tasks/setup.yml

@@ -0,0 +1,11 @@
+- name: Configure sftp user
+  include_role:
+    name: icos.sftp_user
+  vars:
+    sftp_user_login: "{{ sftp_user.name }}"
+    sftp_user_dir: "{{ lxc_sftp_base_dir }}/{{ sftp_user.name }}"
+    sftp_user_pubkey: "{{ sftp_user.pubkey | default('') }}"
+    sftp_user_password: "{{ sftp_user.password | default('') }}"
+  loop: "{{ lxc_sftp_users }}"
+  loop_control:
+    loop_var: sftp_user

devops/roles/icos.lxd_forward/tasks/main.yml

@@ -7,7 +7,7 @@
       -A PREROUTING -p tcp
       --dport {{ lxd_forward_port }}
       -j DNAT --to-destination {{ lxd_forward_ip }}:22
-  when: lxd_forward_port
+  when: lxd_forward_port | int > 0
 
 - name: Modify /etc/hosts to add lxd_forward_name.lxd
   lineinfile: