diff --git a/.gitignore b/.gitignore index 861eb4e..707a93f 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,4 @@ backups/ # Docker data data/ +ansible/inventory/inventory.yml diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg new file mode 100644 index 0000000..373dd87 --- /dev/null +++ b/ansible/ansible.cfg @@ -0,0 +1,8 @@ +[defaults] +inventory = inventory/hosts.yml +remote_tmp = /tmp/.ansible-${USER}/tmp +host_key_checking = False +timeout = 30 + +[ssh_connection] +ssh_args = -o ControlMaster=auto -o ControlPersist=60s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10 diff --git a/ansible/deploy.yml b/ansible/deploy.yml new file mode 100644 index 0000000..6b60324 --- /dev/null +++ b/ansible/deploy.yml @@ -0,0 +1,11 @@ +--- +- name: Deploy Bublik PR preview + hosts: preview_server + vars: + environment_name: "production" + environment_domain: "example.com" + + tasks: + - include_role: + name: preview-server + tasks_from: deploy.yml diff --git a/ansible/destroy.yml b/ansible/destroy.yml new file mode 100644 index 0000000..21548c3 --- /dev/null +++ b/ansible/destroy.yml @@ -0,0 +1,8 @@ +- name: Destroy PR preview + hosts: preview_server + vars: + pr_id: 123 + tasks: + - include_role: + name: preview-server + tasks_from: destroy.yml diff --git a/ansible/inventory/inventory.example.yml b/ansible/inventory/inventory.example.yml new file mode 100644 index 0000000..b71aed0 --- /dev/null +++ b/ansible/inventory/inventory.example.yml @@ -0,0 +1,9 @@ +all: + children: + preview_server: + hosts: + preview: + ansible_host: + ansible_user: + ansible_ssh_private_key_file: + ansible_command_timeout: 900 diff --git a/ansible/requirements.yml b/ansible/requirements.yml new file mode 100644 index 0000000..e754709 --- /dev/null +++ b/ansible/requirements.yml @@ -0,0 +1,8 @@ +--- +roles: + - name: geerlingguy.docker + version: 7.5.4 + +collections: + - name: community.docker + version: 4.7.0 diff --git a/ansible/roles/preview-server/defaults/main.yml b/ansible/roles/preview-server/defaults/main.yml new file mode 100644 index 0000000..3599f32 --- /dev/null +++ b/ansible/roles/preview-server/defaults/main.yml @@ -0,0 +1,39 @@ +--- +# Packages to install +additional_packages: + - jq + - git + - python3-pip + - python3-requests + - python3-docker + - wget + - tar + +# Proxy +traefik_version: '3.5.2' +traefik_network: 'traefik_proxy' +traefik_domain: 'example.com' + +# Branches and remotes defaults +docker_repo: 'https://github.com/ts-factory/bublik-docker.git' +docker_branch: 'main' + +frontend_repo: 'https://github.com/ts-factory/bublik-ui.git' +frontend_branch: 'main' + +backend_repo: 'https://github.com/ts-factory/bublik.git' +backend_branch: 'main' + +# Preview user +bublik_user: bublik +django_superuser_email: admin@bublik.com +django_superuser_password: admin + +# Docker +docker_install_compose_plugin: true +docker_compose_package: docker-compose-plugin +docker_users: + - '{{ ansible_user | default(ansible_env.USER) }}' +docker_install_compose: true +docker_pip_package: python3-pip +docker_pip_executable: pip3 diff --git a/ansible/roles/preview-server/tasks/deploy.yml b/ansible/roles/preview-server/tasks/deploy.yml new file mode 100644 index 0000000..ce11958 --- /dev/null +++ b/ansible/roles/preview-server/tasks/deploy.yml @@ -0,0 +1,227 @@ +--- +- name: Create environments directory + file: + path: "/opt/environments" + state: directory + mode: "0755" + owner: "{{ bublik_user }}" + group: "{{ bublik_user }}" + become: true + +- name: Create environment directory + file: + path: "/opt/environments/{{ environment_name }}" + state: directory + mode: "0755" + owner: "{{ bublik_user }}" + group: "{{ bublik_user }}" + become_user: "{{ bublik_user }}" + +- name: Remove existing bublik-docker directory if it exists + file: + path: "/opt/environments/{{ environment_name }}/bublik-docker" + state: absent + become: true + +- name: Clone main bublik-docker repository + git: + repo: "{{ docker_repo }}" + dest: "/opt/environments/{{ environment_name }}/bublik-docker" + version: "{{ docker_branch }}" + force: yes + update: yes + become_user: "{{ bublik_user }}" + +- name: Initialize and update submodules + args: + chdir: /opt/environments/{{ environment_name }}/bublik-docker + shell: | + git submodule init + git submodule update --recursive + become_user: "{{ bublik_user }}" + +- name: Checkout frontend submodule to PR branch + args: + chdir: /opt/environments/{{ environment_name }}/bublik-docker/bublik-ui + shell: | + git remote set-url origin {{ frontend_repo }} + git fetch origin + git checkout {{ frontend_branch }} + git reset --hard origin/{{ frontend_branch }} + become_user: "{{ bublik_user }}" + +- name: Checkout backend submodule to PR branch + args: + chdir: /opt/environments/{{ environment_name }}/bublik-docker/bublik + shell: | + git remote set-url origin {{ backend_repo }} + git fetch origin + git checkout {{ backend_branch }} + git reset --hard origin/{{ backend_branch }} + become_user: "{{ bublik_user }}" + +- name: Setup .env file and django settings files + args: + chdir: /opt/environments/{{ environment_name }}/bublik-docker + shell: task setup + become_user: "{{ bublik_user }}" + +- name: Generate random SECRET_KEY + set_fact: + secret_key: "{{ lookup('ansible.builtin.pipe', 'openssl rand -base64 50 | tr -d \"/+=\\n\"') }}" + no_log: true + +- name: Preprocess .env file - adjust variables for PR preview + lineinfile: + path: "/opt/environments/{{ environment_name }}/bublik-docker/.env" + regexp: "^{{ item.key }}=" + line: "{{ item.key }}={{ item.value }}" + loop: + - { key: "COMPOSE_PROJECT_NAME", value: "{{ environment_name }}" } + - { key: "IMAGE_TAG", value: "{{ environment_name }}" } + - { key: "SECRET_KEY", value: "{{ secret_key }}" } + - { key: "BUBLIK_FQDN", value: "https://{{ environment_domain }}" } + - { key: "SECURE_HTTP", value: "True" } + - { key: "DJANGO_SUPERUSER_EMAIL", value: "{{ django_superuser_email }}" } + - { key: "DB_HOST", value: "db" } + - { key: "REDIS_HOST", value: "redis" } + - { key: "RABBITMQ_HOST", value: "rabbitmq" } + - { key: "FLOWER_HOST", value: "flower" } + - { key: "BUBLIK_DOCKER_DJANGO_HOST_PROXY", value: "django" } + - { + key: "DJANGO_SUPERUSER_PASSWORD", + value: "{{ django_superuser_password }}", + } + no_log: true + become_user: "{{ bublik_user }}" + +- name: Read BUBLIK_DOCKER_PROXY_PORT from .env file + ansible.builtin.slurp: + src: /opt/environments/{{ environment_name }}/bublik-docker/.env + register: env_file + failed_when: env_file.content | b64decode is not regex('BUBLIK_DOCKER_PROXY_PORT=\d+') + +- name: Extract nginx_port from .env file + ansible.builtin.set_fact: + nginx_port: "{{ (env_file.content | b64decode | regex_search('BUBLIK_DOCKER_PROXY_PORT=(\\d+)', '\\1'))[0] }}" + failed_when: nginx_port is not defined or nginx_port | int <= 0 + +- name: Add Traefik labels and networks to nginx service + args: + executable: /bin/bash + chdir: /opt/environments/{{ environment_name }}/bublik-docker + become_user: "{{ bublik_user }}" + shell: | + yq eval -i ' + .services.nginx.labels = [ + "traefik.enable=true", + "traefik.docker.network={{ traefik_network }}", + "traefik.http.routers.bublik-nginx-{{ environment_name }}.rule=Host(`{{ environment_domain }}`)", + "traefik.http.routers.bublik-nginx-{{ environment_name }}.entrypoints=web,websecure", + "traefik.http.routers.bublik-nginx-{{ environment_name }}.tls=true", + "traefik.http.routers.bublik-nginx-{{ environment_name }}.tls.certresolver=letsencrypt", + "traefik.http.services.bublik-nginx-{{ environment_name }}.loadbalancer.server.port={{ nginx_port }}" + ] | + .services.nginx.networks = ["default", "{{ traefik_network }}"] | + .networks.{{ traefik_network }} = {"external": true} + ' docker-compose.yml + +- name: Remove ports from all services + args: + executable: /bin/bash + chdir: /opt/environments/{{ environment_name }}/bublik-docker + become_user: "{{ bublik_user }}" + shell: | + yq eval -i '.services |= with_entries(.value |= del(.ports))' {{ item }} + yq eval -i '.services |= with_entries(.value |= del(.network_mode))' {{ item }} + loop: + - docker-compose.yml + - docker-compose.db.yml + +- name: Build Bublik + args: + executable: /bin/bash + chdir: /opt/environments/{{ environment_name }}/bublik-docker + shell: task build + +- name: Stop Bublik + community.docker.docker_compose_v2: + project_src: /opt/environments/{{ environment_name }}/bublik-docker + state: absent + remove_volumes: true + files: + - docker-compose.yml + - docker-compose.db.yml + when: run_bootstrap is defined and run_bootstrap == "true" + +- name: Start Bublik DB + community.docker.docker_compose_v2: + project_src: /opt/environments/{{ environment_name }}/bublik-docker + state: present + recreate: "always" + files: + - docker-compose.db.yml + +- name: Check if production environment exists + stat: + path: /opt/environments/production/bublik-docker + register: production_dir + when: run_bootstrap is defined and run_bootstrap == "true" + +- name: Check if production docker compose is running + shell: docker compose ps --services --filter "status=running" | wc -l + args: + chdir: /opt/environments/production/bublik-docker + register: production_compose_running + when: + - run_bootstrap is defined and run_bootstrap == "true" + - production_dir.stat.exists + changed_when: false + failed_when: false + +- name: Create Production DB Backup + args: + chdir: /opt/environments/production/bublik-docker + shell: docker compose exec -T db pg_dump -U bublik -d bublik | gzip > /opt/bootstrap/bublik-bootstrap-db.sql.gz + become: true + when: + - run_bootstrap is defined and run_bootstrap == "true" + - production_dir.stat.exists + - production_compose_running.stdout | int > 0 + +- name: Restore backup into staging database + shell: zcat /opt/bootstrap/bublik-bootstrap-db.sql.gz | docker compose exec -T db psql -U {{ db_user | default('bublik') }} -d {{ db_name | default('bublik') }} + args: + chdir: /opt/environments/{{ environment_name }}/bublik-docker + when: + - run_bootstrap is defined and run_bootstrap == "true" + - production_dir.stat.exists + - production_compose_running.stdout | int > 0 + +- name: Start Bublik + community.docker.docker_compose_v2: + project_src: /opt/environments/{{ environment_name }}/bublik-docker + state: present + build: "always" + recreate: "always" + files: + - docker-compose.yml + - docker-compose.db.yml + +- name: Upgrade Configs + args: + chdir: /opt/environments/{{ environment_name }}/bublik-docker + shell: docker compose exec -it django python manage.py reformat_configs + when: + - run_bootstrap is defined and run_bootstrap == "true" + +- name: Run Meta Categorization Task + args: + chdir: /opt/environments/{{ environment_name }}/bublik-docker + shell: task meta-categorization + when: + - run_bootstrap is defined and run_bootstrap == "true" + +- name: Display preview URL + debug: + msg: "{{ environment_name }} is available at: https://{{ environment_domain }}" diff --git a/ansible/roles/preview-server/tasks/destroy.yml b/ansible/roles/preview-server/tasks/destroy.yml new file mode 100644 index 0000000..4efdaff --- /dev/null +++ b/ansible/roles/preview-server/tasks/destroy.yml @@ -0,0 +1,35 @@ +- name: Check if repo directory exists + stat: + path: "/opt/environments/{{ environment_name }}/bublik-docker" + register: repo_dir_check + +- name: Check if compose files exist + stat: + path: "/opt/environments/{{ environment_name }}/bublik-docker/docker-compose.yml" + register: compose_file_check + when: repo_dir_check.stat.exists + +- name: Remove Compose Containers + community.docker.docker_compose_v2: + project_src: /opt/environments/{{ environment_name }}/bublik-docker + state: absent + files: + - docker-compose.yml + - docker-compose.db.yml + remove_volumes: true + when: + - repo_dir_check.stat.exists + - compose_file_check.stat.exists | default(false) + ignore_errors: true + +- name: Delete Preview Directory + file: + path: "/opt/environments/{{ environment_name }}" + state: absent + # Required to delete since volumes belong to root + become: true + +- name: Remove Proxy Domain + file: + path: "/opt/traefik/dynamic/{{ environment_name }}.yml" + state: absent diff --git a/ansible/roles/preview-server/tasks/main.yml b/ansible/roles/preview-server/tasks/main.yml new file mode 100644 index 0000000..fad3697 --- /dev/null +++ b/ansible/roles/preview-server/tasks/main.yml @@ -0,0 +1,114 @@ +--- +- name: Create the new user + user: + name: "{{ bublik_user }}" + state: present + create_home: yes + shell: /bin/bash + +- name: Ensure .ssh directory exists for bublik_user + file: + path: "/home/{{ bublik_user }}/.ssh" + state: directory + owner: "{{ bublik_user }}" + group: "{{ bublik_user }}" + mode: "0700" + +- name: Add SSH public key to bublik_user + authorized_key: + user: "{{ bublik_user }}" + key: "{{ lookup('file', ssh_public_key_path) }}" + state: present + +- name: Allow bublik_user passwordless sudo + copy: + dest: "/etc/sudoers.d/{{ bublik_user }}" + content: "{{ bublik_user }} ALL=(ALL) NOPASSWD:ALL" + owner: root + group: root + mode: "0440" + +- name: Install additional packages + apt: + name: "{{ additional_packages }}" + state: present + update_cache: yes + +- name: Ensure Task is installed + stat: + path: /usr/local/bin/task + register: task_bin + +- name: Install Task + shell: sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin + when: not task_bin.stat.exists + +- name: Ensure yq is installed + stat: + path: /usr/local/bin/yq + register: yq_bin + +- name: Download yq + get_url: + url: "https://github.com/mikefarah/yq/releases/download/v4.47.2/{{ 'yq_linux_amd64' if ansible_architecture == 'x86_64' else 'yq_linux_arm64' }}" + dest: /usr/local/bin/yq + mode: "0755" + when: not yq_bin.stat.exists + +- name: Install Docker using geerlingguy.docker role + include_role: + name: geerlingguy.docker + vars: + docker_install_compose_plugin: true + docker_compose_package: docker-compose-plugin + docker_users: + - "{{ ansible_user }}" + - "{{ bublik_user }}" + +- name: Configure Docker daemon.json + copy: + dest: /etc/docker/daemon.json + content: | + { + "dns": ["8.8.8.8", "8.8.4.4", "1.1.1.1"] + } + owner: root + group: root + mode: "0644" + +- name: Restart Docker + systemd: + name: docker + state: restarted + enabled: yes + +- name: Create Traefik network + community.docker.docker_network: + name: "{{ traefik_network }}" + driver: bridge + +- name: Create Traefik directory + file: + path: /opt/traefik + state: directory + mode: "0755" + owner: "{{ bublik_user }}" + group: "{{ bublik_user }}" + +- name: Create Bootstrap Directory + file: + path: /opt/bootstrap + state: directory + mode: "0755" + owner: "{{ bublik_user }}" + group: "{{ bublik_user }}" + +- name: Copy Traefik docker-compose file + template: + src: docker-compose.traefik.yml.j2 + dest: /opt/traefik/docker-compose.yml + +- name: Start Traefik with Docker Compose + community.docker.docker_compose_v2: + project_src: /opt/traefik + state: present diff --git a/ansible/roles/preview-server/templates/docker-compose.traefik.yml.j2 b/ansible/roles/preview-server/templates/docker-compose.traefik.yml.j2 new file mode 100644 index 0000000..a31b1e3 --- /dev/null +++ b/ansible/roles/preview-server/templates/docker-compose.traefik.yml.j2 @@ -0,0 +1,36 @@ +services: + traefik: + image: traefik:{{ traefik_version }} + command: + - "--api.dashboard=true" + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--providers.file.directory=/dynamic" + - "--providers.file.watch=true" + - "--entrypoints.web.address=:80" + - "--entrypoints.websecure.address=:443" + - "--certificatesresolvers.letsencrypt.acme.httpchallenge=true" + - "--certificatesresolvers.letsencrypt.acme.httpchallenge.entrypoint=web" + - "--certificatesresolvers.letsencrypt.acme.email=admin@{{ traefik_domain }}" + - "--certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json" + # Optional: Enable access logs + - "--accesslog=true" + # Optional: Set log level + - "--log.level=INFO" + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./dynamic:/dynamic:ro + - ./letsencrypt:/letsencrypt + restart: unless-stopped + networks: + - {{ traefik_network }} + ports: + - "80:80" + - "443:443" + labels: + # Disable Traefik for itself to avoid the "port is missing" error + traefik.enable: "false" + +networks: + {{ traefik_network }}: + external: true diff --git a/ansible/setup.yml b/ansible/setup.yml new file mode 100644 index 0000000..5cc7936 --- /dev/null +++ b/ansible/setup.yml @@ -0,0 +1,7 @@ +- name: Setup PR preview server + hosts: preview_server + become: true + roles: + - role: preview-server + vars: + traefik_domain: example.com