Deploy lxc-sftp container on fsicos3.

devops/files/dokku/deploy-trajectory-viewer

@@ -0,0 +1,31 @@
+#!/bin/bash
+set -euo pipefail
+BRANCH=${1:-trajectory_styled}
+REPO=/opt/trajectory-viewer
+SYNC_REPO=/var/lib/trajectory-viewer-sync.git
+
+echo "==> Deploying branch: $BRANCH"
+cd "$REPO"
+
+FETCHED=0
+if GIT_SSH_COMMAND="ssh -i /root/.ssh/id_rsa_github -o IdentitiesOnly=yes" git fetch origin; then
+  FETCHED=1
+  echo "==> GitHub fetch succeeded"
+else
+  echo "==> Warning: git fetch origin failed; deploying current local branch state" >&2
+fi
+
+git checkout "$BRANCH"
+if [ "$FETCHED" -eq 1 ]; then
+  git merge --ff-only "origin/$BRANCH"
+fi
+
+python3 -m py_compile vsplot/backend/app.py vsplot/backend/callbacks.py vsplot/backend/matomo.py
+echo "==> Syntax OK"
+rm -rf "$SYNC_REPO"
+git clone --bare --no-local "$REPO" "$SYNC_REPO"
+chmod -R a+rX "$SYNC_REPO"
+sudo -u dokku git config --global --add safe.directory "$SYNC_REPO"
+dokku git:sync --build trajectory-viewer "file://$SYNC_REPO" "$BRANCH"
+echo "==> Deploy complete"
+dokku logs trajectory-viewer --tail 20

devops/host_vars/dokku.yml

@@ -0,0 +1,40 @@
+dokku_hostname: dokku
+
+dokku_github_public_key: >-
+  ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDQZTvlk/bboloJVMfjGmy2kgbqig8Zop0f47nV0TNjArYTpqMq3gGIfNM9Ae+WbBCdpyIEq/4sxRbMyQXMmH42rvXiAM7ArJhyKMswGWYAVxUi4U7EFhGqTi4hyaA3b0/tm1vN4IisjKxleXthPOUvbcSDx210r6kZ5azu1WdBBBAwCDzi7FJhalPPj6QH7+5lmueeIzc4qRPJaAB3FINxS9PnUd+AxtVNJMqoi6nfDhQGhzhbfQs5vfilxlXNnp4XbpCK9xHhAHUGas/uaGvqV06WIm7VWQX1Lmbr4bjpVMXOTXVA7hjokGFkh6vDVe28UeCNQjw6PdQ6ErwxblxuMYMC7CNGXYHxLxpZb+It2S0VEUhqls4vsMnYSk1rY0i3bdeyG3k1llTDRV+yLluoG/h0puWnqHXFszO1kTx258vJku3HR44+nuGmoTuBAsnLe8LxPPlbr0KKZlBsN6ng0S6SIP8dvgGlYTis7Ce/waXygGC68bkkbKgOqPFx3VebWujFi+RamLsof/EFS4ezWj1fYHIgV3KSVH/QuE/Oe3wwJPYHFfH6JngfBumZZRcGUMTl4Y19Ut6c1/l4ARy1ETG9KKSnrb8KOd4zeihBzaIkzHaEfGdS6pXQ8slKqrTcdCc6Sv+tobUq3JjRF/ilVx5DRi0jOCH/+7N//9adlQ==
+  Roberts Github ssh key
+
+dokku_trajectory_viewer_name: trajectory-viewer
+dokku_trajectory_viewer_repo: git@github.com:ICOS-Carbon-Portal/trajectory-viewer.git
+dokku_trajectory_viewer_branch: main
+dokku_trajectory_viewer_repo_dir: /opt/trajectory-viewer
+dokku_trajectory_viewer_domain: trajectory.icos-cp.eu
+dokku_trajectory_viewer_storage_mount: /root/data:/data:ro
+dokku_trajectory_viewer_port_map: http:80:8050
+dokku_trajectory_viewer_env:
+  APP_URL: "https://trajectory.icos-cp.eu/"
+  DATA_DIR: /data/trj
+  DOKKU_APP_RESTORE: "1"
+  DOKKU_PROXY_PORT: "80"
+  ICOS_CHROME: "0"
+  ICOS_CHROME_ENABLED: "1"
+  MATOMO_AUTH_TOKEN: 0fcf14d8c6125a74ee621ccf6cf66dc1
+  MATOMO_SITE_ID: "12"
+  MATOMO_URL: https://matomo.icos-cp.eu/matomo.php
+  PORT: "8050"
+  URL_BASE_PATH: /
+
+icosdata_mkdirs:
+  - /data
+  - /data/flexextract4
+  - /data4
+
+icosdata_nfs_mounts:
+  - path: /data4
+    src: fsicos4.int:/tank/data/flexpart
+    fstype: nfs
+    opts: "rw,vers=3,_netdev,hard,timeo=600,retrans=2"
+  - path: /data/flexextract4
+    src: fsicos4.int:/tank/flexextract
+    fstype: nfs
+    opts: "rw,vers=3,_netdev,hard,timeo=600,retrans=2"

devops/host_vars/fsicos4.yml

@@ -0,0 +1,3 @@
+icosdata_exports: |
+  /tank/flexextract 10.10.10.145/32(rw,sync,no_subtree_check,no_root_squash)
+  /tank/data/flexpart 10.10.10.145/32(rw,sync,no_subtree_check,no_root_squash)

devops/icosdata.yml

@@ -18,8 +18,10 @@
 - hosts:
     - fsicos2
     - fsicos3
+    - fsicos4
     - icos1
     - cupcake
+    - dokku
     - pancake
 
   tags:
@@ -79,7 +81,7 @@
       name: Mount nfs data
       tags: mount
       mount:
-        fstype: nfs4
+        fstype: "{{ item.fstype | default('nfs4') }}"
         state: "{{ item.state | default('mounted') }}"
         # The next two default to omit so that they can be left out when state
         # is "unmounted".

devops/production.inventory/hosts.yml

@@ -70,6 +70,10 @@ all:
         cupcake:
           ansible_port: 60603
           user_conf: "{{ vault_cupcake_user_conf }}"
+        dokku:
+          ansible_port: 60613
+          ansible_user: root
+          root_keys: "{{ vault_dokku_root_keys }}"
         pancake:
           ansible_port: 60605
           ansible_user: root
@@ -79,6 +83,9 @@ all:
         fdp:
           ansible_port: 60606
           ansible_user: fdp
+        fsicos4-flexpart:
+          ansible_port: 60611
+          ansible_user: root
 
 
     # CDB VMS
@@ -125,6 +132,8 @@ all:
       vars:
         ansible_host: fsicos3
       hosts:
+        lxc-sftp:
+          ansible_port: 60580
         amalthea:
           ansible_port: 60560
         callisto:
@@ -160,6 +169,3 @@ all:
           ansible_port: 60570
         ctehires:
           ansible_port: 60575
-        dokku:
-          root_keys: "{{ vault_dokku_root_keys }}"
-          ansible_port: 60595

devops/roles/icos.docker2/tasks/ubuntu.yml

@@ -1,8 +1,14 @@
 # https://docs.docker.com/engine/install/ubuntu/
+- name: Ensure apt keyrings directory exists
+  file:
+    path: /etc/apt/keyrings
+    state: directory
+    mode: "0755"
+
 - name: Add docker key
   ansible.builtin.get_url:
     url: https://download.docker.com/linux/ubuntu/gpg
-    dest: /etc/apt/trusted.gpg.d/docker.asc
+    dest: /etc/apt/keyrings/docker.asc
     mode: '0644'
     force: true
   register: _key

devops/roles/icos.dokku/defaults/main.yml

@@ -1,7 +1,9 @@
 # PARAMETERS
 
 # The domain from which we're serving dokku apps.
-# dokku_hostname
+dokku_hostname: "{{ inventory_hostname }}"
+dokku_vhost_enable: true
+dokku_nginx_enable: true
 
 # CONFIG
 dokku_ssh_port: 3022

devops/roles/icos.dokku/tasks/install.yml

@@ -23,17 +23,18 @@
     vtype: "{{ item.vtype }}"
   loop:
     - question: "dokku/vhost_enable"
-      value: "true"
+      value: "{{ 'true' if dokku_vhost_enable else 'false' }}"
       vtype: boolean
     - question: "dokku/hostname"
-      value: "dokku.fsicos3.icos-cp.eu"
+      value: "{{ dokku_hostname }}"
       vtype: "string"
     - question: "dokku/nginx_enable"
-      value: "true"
+      value: "{{ 'true' if dokku_nginx_enable else 'false' }}"
       vtype: "boolean"
 
 - name: Install dokku
   apt:
     name:
       - dokku
+      - git
 

devops/roles/icos.lxc_sftp/defaults/main.yml

@@ -0,0 +1,8 @@
+lxc_sftp_base_dir: /srv/sftp
+
+# List of sftp users.
+# Each entry:
+#   name: username
+#   pubkey: "ssh-rsa ..."      (optional, but recommended)
+#   password: "plaintext"      (optional, avoid in production)
+lxc_sftp_users: []

devops/roles/icos.lxc_sftp/tasks/main.yml

@@ -0,0 +1,2 @@
+- import_tasks: setup.yml
+  tags: lxc_sftp_setup

devops/roles/icos.lxc_sftp/tasks/setup.yml

@@ -0,0 +1,11 @@
+- name: Configure sftp user
+  include_role:
+    name: icos.sftp_user
+  vars:
+    sftp_user_login: "{{ sftp_user.name }}"
+    sftp_user_dir: "{{ lxc_sftp_base_dir }}/{{ sftp_user.name }}"
+    sftp_user_pubkey: "{{ sftp_user.pubkey | default('') }}"
+    sftp_user_password: "{{ sftp_user.password | default('') }}"
+  loop: "{{ lxc_sftp_users }}"
+  loop_control:
+    loop_var: sftp_user

devops/roles/icos.lxd_forward/tasks/main.yml

@@ -7,7 +7,7 @@
       -A PREROUTING -p tcp
       --dport {{ lxd_forward_port }}
       -j DNAT --to-destination {{ lxd_forward_ip }}:22
-  when: lxd_forward_port
+  when: lxd_forward_port | int > 0
 
 - name: Modify /etc/hosts to add lxd_forward_name.lxd
   lineinfile:

devops/roles/icos.lxd_vm/vars/zfs.yml

@@ -6,17 +6,16 @@ __root_device:
     pool: "default"
     size: "{{ lxd_vm_root_size }}"
 
-__docker_device: >-
-  {% if lxd_vm_docker -%}
-  {"docker": {
-    "path": "/var/lib/docker",
-    "source": "{{ zfsdocker_zvol }}",
-    "type": "disk",
-    "raw.mount.options": "user_subvol_rm_allowed" }}
-  {% else -%}{}{% endif -%}
+__docker_device_data:
+  docker:
+    path: /var/lib/docker
+    source: "{{ zfsdocker_zvol }}"
+    type: disk
+    "raw.mount.options": user_subvol_rm_allowed
+
+__docker_device: "{{ __docker_device_data if lxd_vm_docker else {} }}"
 
-lxd_vm_default_devices: >-
-  {{ __root_device | combine(__docker_device) }}
+lxd_vm_default_devices: "{{ __root_device | combine(__docker_device) }}"
 
 
 # PROFILES
@@ -27,10 +26,6 @@ lxd_vm_default_profiles:
 # CONFIG
 __base_config: {}
 
-__docker_config: >-
-  {% if lxd_vm_docker -%}
-  {"security.nesting": "true"}
-  {% else %}{}{% endif -%}
+__docker_config: "{{ {'security.nesting': 'true'} if lxd_vm_docker else {} }}"
 
-lxd_vm_default_config: >-
-  {{ __base_config | combine(__docker_config) }}
+lxd_vm_default_config: "{{ __base_config | combine(__docker_config) }}"

devops/roles/icos.sftp_user/handlers/main.yml

@@ -1,4 +1,4 @@
 - name: reload sshd
   systemd:
-    name: sshd
+    name: ssh
     state: reloaded

devops/roles/icos.sftp_user/tasks/main.yml

@@ -37,7 +37,9 @@
   authorized_key:
     user: "{{ sftp_user_login }}"
     key: "{{ sftp_user_pubkey }}"
-  when: sftp_user_pubkey
+  when:
+    - sftp_user_pubkey | default('') | length > 0
+    - not (ansible_check_mode and _user.changed)
 
 - name: Stat parent directory again
   stat:
@@ -48,7 +50,9 @@
 - name: Fail if parent directory isn't owned by root
   fail:
     msg: "{{ _sftp_parent_dir }} must be owned by root"
-  when: _parent.stat.uid != 0 or _parent.stat.gid != 0
+  when:
+    - _parent.stat.exists
+    - _parent.stat.uid != 0 or _parent.stat.gid != 0
 
 # This is the actual directory user for uploading/downloading. Depending on
 # "sftp_user_owner", the directory might be read-only or read-write for the
@@ -59,6 +63,7 @@
     state: directory
     owner: "{{ sftp_user_owner }}"
     group: "{{ sftp_user_group }}"
+  when: not (ansible_check_mode and (_user.changed or not _parent.stat.exists))
 
 # <2021-10-01> It's possible to instead create /etc/ssh/sshd_config.d/X.conf.
 # But then the global "PasswordAuthentication no" will override it (in

devops/vm-fsicos3-sftp.yml

@@ -0,0 +1,36 @@
+---
+# Creates the lxc-sftp container on fsicos3.
+#
+# The container runs openssh with internal-sftp and chroots users under
+# /srv/sftp/<username>.  fsicos3 DNAT-forwards port 60580 to port 22
+# in the container, so sftp clients connect directly to
+# fsicos3.icos-cp.eu on port 60580.  Port 22 on fsicos3 itself must
+# never be forwarded.
+#
+# Vault variables required:
+#   vault_sftp_robert_pubkey  – SSH public key for user robert
+
+- hosts: fsicos3
+  roles:
+    - name: Create the lxc-sftp VM
+      role: icos.lxd_vm
+      lxd_vm_name: lxc-sftp
+      lxd_vm_port: 60580
+      lxd_vm_ubuntu_version: "24.04"
+      lxd_vm_root_size: "20GB"
+      lxd_vm_config:
+        limits.cpu: "2"
+        limits.memory: "1GB"
+
+- hosts: lxc-sftp
+  vars:
+    ansible_port: 60580
+  roles:
+    - role: icos.lxd_guest
+      tags: guest
+
+    - role: icos.lxc_sftp
+      tags: sftp
+      lxc_sftp_users:
+        - name: robert
+          pubkey: "{{ vault_sftp_robert_pubkey }}"