From 11da5cee1876cd00e3873d1033c3616eede7b9c2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Sep 2025 03:10:21 +0000 Subject: [PATCH 1/3] Initial plan From fee616c23fd474d682868e29f266392106098f0d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Sep 2025 03:14:48 +0000 Subject: [PATCH 2/3] Copy georgetown scenario directory structure to create kingston scenario Co-authored-by: jodybro <25326943+jodybro@users.noreply.github.com> --- scenarios/kingston/README.md | 0 .../kingston/ansible/playbook-debian.yml | 73 +++++++++++++++++++ scenarios/kingston/files/check.sh | 34 +++++++++ scenarios/kingston/files/mock-build-run.sh | 13 ++++ scenarios/kingston/files/mock-build.service | 9 +++ .../kingston/files/secondary-service.service | 9 +++ scenarios/kingston/files/secondary-service.sh | 2 + .../kingston/packer/aws-debian11.pkr.hcl | 67 +++++++++++++++++ scenarios/kingston/packer/variables.pkr.hcl | 25 +++++++ 9 files changed, 232 insertions(+) create mode 100644 scenarios/kingston/README.md create mode 100644 scenarios/kingston/ansible/playbook-debian.yml create mode 100644 scenarios/kingston/files/check.sh create mode 100644 scenarios/kingston/files/mock-build-run.sh create mode 100644 scenarios/kingston/files/mock-build.service create mode 100644 scenarios/kingston/files/secondary-service.service create mode 100644 scenarios/kingston/files/secondary-service.sh create mode 100644 scenarios/kingston/packer/aws-debian11.pkr.hcl create mode 100644 scenarios/kingston/packer/variables.pkr.hcl diff --git a/scenarios/kingston/README.md b/scenarios/kingston/README.md new file mode 100644 index 0000000..e69de29 diff --git a/scenarios/kingston/ansible/playbook-debian.yml b/scenarios/kingston/ansible/playbook-debian.yml new file mode 100644 index 0000000..e88b58f --- /dev/null +++ b/scenarios/kingston/ansible/playbook-debian.yml @@ -0,0 +1,73 @@ +--- +# ansible-playbook playbook-debian.yml -u admin --private-key=/path/to/private/key -i "$ip," +- name: copy files to remote server + hosts: all + tasks: + # OS + - name: update packages and install lsof + become: true + package: + # This is weird in that it does not exist in the offical docs + # but this module just passes params down to the underlying + # package manager module where `update_cache` is normally a thing so + # it ends up working.... + update_cache: yes + state: latest + name: + - lsof + - util-linux # Ensure that fallocate is installed + + - name: create log file + become: true + file: + path: /var/log/fallocate.log + state: touch + owner: admin + group: admin + + - name: cronjob + cron: + name: "reboot" + special_time: reboot + job: "/home/admin/badlog.py &" + + # check.sh + - name: Create /home/admin/agent directory + ansible.builtin.file: + path: /home/admin/agent + owner: admin + group: admin + mode: a+wx + state: directory + + - name: Copy kingston systemd files + become: true + copy: + src: {{ item.src }} + dest: {{ item.dest }} + with_items: + - { src: ../files/mock-build.service, dest: /etc/systemd/system/mock-build.service } + - { src: ../files/secondary-service.service, dest: /etc/systemd/system/secondary-service.service } + + - name: Start mock build application + become: true + systemd: + name: mock-build + state: started + + - name: Start secondary service + become: true + systemd: + name: secondary-service + state: started + enabled: yes + + - name: copy check.sh + copy: + src: ../files/check.sh + dest: /home/admin/agent/check.sh + + - name: set check.sh + file: + path: /home/admin/agent/check.sh + mode: "+x" diff --git a/scenarios/kingston/files/check.sh b/scenarios/kingston/files/check.sh new file mode 100644 index 0000000..8c383ae --- /dev/null +++ b/scenarios/kingston/files/check.sh @@ -0,0 +1,34 @@ +#!/usr/bin/bash + +BUILD_MOUNT_POINT="/tmp/ephemeral-build" + +# Check if the build mount point exists +function check_build_mount_point { + if [ ! -d $BUILD_MOUNT_POINT ]; then + echo -b "NO" + exit 1 + fi +} + +# Function to check if both example-build-artifact.txt and +# secondary-artifact.txt exist in the build mount point +# BOTH files need to exist concurrently +function check_build_artifacts { + BUILD_FILES=( + "example-build-artifact.txt" + "secondary-artifact.txt" + ) + for file in "${BUILD_FILES[@]}"; do + if [ ! -f "$BUILD_MOUNT_POINT/$file" ]; then + echo -b "NO" + exit 1 + else + echo -b "OK" + exit 0 + fi + done +} + +# Check if one or both functions executed successfull +check_build_mount_point +check_build_artifacts diff --git a/scenarios/kingston/files/mock-build-run.sh b/scenarios/kingston/files/mock-build-run.sh new file mode 100644 index 0000000..0d2f659 --- /dev/null +++ b/scenarios/kingston/files/mock-build-run.sh @@ -0,0 +1,13 @@ +#!/usr/env/bin bash + +MOUNT_POINT="/tmp/ephemeral-build" +EXAMPLE_BUILD_ARTIFACT="example-build-artifact.txt" +MAX_BUILD_ARTIFACT_SIZE=10M + +# Create the example build file +touch $MOUNT_POINT/$EXAMPLE_BUILD_ARTIFACT + +# Use fallocate to allocate the maximum size of the file +fallocate -l "$MAX_SIZE" "$MOUNT_POINT/$EXAMPLE_BUILD_ARTIFACT" + +trap diff --git a/scenarios/kingston/files/mock-build.service b/scenarios/kingston/files/mock-build.service new file mode 100644 index 0000000..335b661 --- /dev/null +++ b/scenarios/kingston/files/mock-build.service @@ -0,0 +1,9 @@ +[Unit] +Description=Mock Build Application +After=network.target + +[Service] +ExecStart=/home/admin/mock_build.sh + +[Install] +WantedBy=multi-user.target diff --git a/scenarios/kingston/files/secondary-service.service b/scenarios/kingston/files/secondary-service.service new file mode 100644 index 0000000..08eb9b9 --- /dev/null +++ b/scenarios/kingston/files/secondary-service.service @@ -0,0 +1,9 @@ +[Unit] +Description=Secondary service Application +After=network.target + +[Service] +ExecStart=/home/admin/secondary-service.sh + +[Install] +WantedBy=multi-user.target diff --git a/scenarios/kingston/files/secondary-service.sh b/scenarios/kingston/files/secondary-service.sh new file mode 100644 index 0000000..40972ef --- /dev/null +++ b/scenarios/kingston/files/secondary-service.sh @@ -0,0 +1,2 @@ +# Secondary service that will continually run in the background +# and try to place a file on tmp artifact directory that is too big to fit on disk diff --git a/scenarios/kingston/packer/aws-debian11.pkr.hcl b/scenarios/kingston/packer/aws-debian11.pkr.hcl new file mode 100644 index 0000000..11c7991 --- /dev/null +++ b/scenarios/kingston/packer/aws-debian11.pkr.hcl @@ -0,0 +1,67 @@ +# Debian + +packer { + required_plugins { + amazon = { + version = "= 1.2.1" + source = "github.com/hashicorp/amazon" + } + } +} + +source "amazon-ebs" "debian" { + ami_name = "scenario-1-saintjohn" + instance_type = "t3a.nano" + region = "${var.region}" + vpc_id = "${var.vpc_id}" + subnet_id = "${var.subnet_id}" + associate_public_ip_address = true + source_ami = "${var.source_ami}" + ssh_username = "admin" +} + +build { + name = "debian-build" + sources = [ + "source.amazon-ebs.debian" + ] + + # OS & scenario packages + provisioner "shell" { + inline = [ + "echo Update packages...", + "sudo apt-get update", + "sudo apt-get install -y lsof", + ] + } + + # badlog.py + provisioner "file" { + source = "../files/badlog.py" + destination = "/tmp/badlog.py" + } + + provisioner "shell" { + inline = [ + "mv /tmp/badlog.py /home/admin/badlog.py", + "chmod +x /home/admin/badlog.py", + "sudo touch /var/log/bad.log", + "sudo chown admin: /var/log/bad.log", + "echo '@reboot /home/admin/badlog.py &' | crontab -", + ] + } + + # check.sh + provisioner "file" { + source = "../files/check.sh" + destination = "/tmp/check.sh" + } + + provisioner "shell" { + inline = [ + "sudo mv /tmp/check.sh /home/admin/agent/check.sh", + "sudo chmod +x /home/admin/agent/check.sh", + ] + } + +} diff --git a/scenarios/kingston/packer/variables.pkr.hcl b/scenarios/kingston/packer/variables.pkr.hcl new file mode 100644 index 0000000..21259e5 --- /dev/null +++ b/scenarios/kingston/packer/variables.pkr.hcl @@ -0,0 +1,25 @@ +# variables for Packer file, adapt to your AWS region, base image, vpc, subnet + +variable "region" { + type = string + default = "us-east-1" +} + +# tested with source Debian 11 image HVM 64-bit (x86) +# change to one in your region +variable "source_ami" { + type = string + default = "ami-" +} + +# change to your vpc +variable "vpc_id" { + type = string + default = "vpc-" +} + +# change to your subnet +variable "subnet_id" { + type = string + default = "subnet-" +} \ No newline at end of file From e7edc323b866baed51394c635d21a96824d40fb0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Sep 2025 03:50:21 +0000 Subject: [PATCH 3/3] Transform kingston scenario into Docker optimization challenge Co-authored-by: jodybro <25326943+jodybro@users.noreply.github.com> --- scenarios/kingston/README.md | 110 ++++++++++++++++ .../kingston/ansible/playbook-debian.yml | 117 +++++++++++------- scenarios/kingston/files/.dockerignore | 49 ++++++++ scenarios/kingston/files/Dockerfile.base | 59 +++++++++ scenarios/kingston/files/Dockerfile.optimized | 94 ++++++++++++++ scenarios/kingston/files/Gemfile | 27 ++++ scenarios/kingston/files/Gemfile.lock | 53 ++++++++ scenarios/kingston/files/check.sh | 82 +++++++----- scenarios/kingston/files/entrypoint.sh | 8 ++ scenarios/kingston/files/mock-build-run.sh | 13 -- scenarios/kingston/files/mock-build.service | 9 -- .../kingston/files/secondary-service.service | 9 -- scenarios/kingston/files/secondary-service.sh | 2 - scenarios/kingston/files/setup_scenario.sh | 71 +++++++++++ scenarios/kingston/files/sidekiq_shutdown.rb | 3 + scenarios/kingston/files/time_builds.sh | 111 +++++++++++++++++ .../kingston/packer/aws-debian11.pkr.hcl | 91 +++++++++++--- 17 files changed, 786 insertions(+), 122 deletions(-) create mode 100644 scenarios/kingston/files/.dockerignore create mode 100644 scenarios/kingston/files/Dockerfile.base create mode 100644 scenarios/kingston/files/Dockerfile.optimized create mode 100644 scenarios/kingston/files/Gemfile create mode 100644 scenarios/kingston/files/Gemfile.lock create mode 100644 scenarios/kingston/files/entrypoint.sh delete mode 100644 scenarios/kingston/files/mock-build-run.sh delete mode 100644 scenarios/kingston/files/mock-build.service delete mode 100644 scenarios/kingston/files/secondary-service.service delete mode 100644 scenarios/kingston/files/secondary-service.sh create mode 100644 scenarios/kingston/files/setup_scenario.sh create mode 100644 scenarios/kingston/files/sidekiq_shutdown.rb create mode 100644 scenarios/kingston/files/time_builds.sh diff --git a/scenarios/kingston/README.md b/scenarios/kingston/README.md index e69de29..201745c 100644 --- a/scenarios/kingston/README.md +++ b/scenarios/kingston/README.md @@ -0,0 +1,110 @@ +# "Kingston": Docker Image Build Optimization + +## Introduction + +Welcome to the Kingston scenario! This challenge focuses on optimizing Docker image build times for a Ruby on Rails application. You'll be working with a real-world Dockerfile that has several optimization opportunities. + +## Scenario Description + +You have been given a Ruby on Rails application with a poorly optimized Dockerfile that takes a long time to build. Your task is to optimize the Docker build process to achieve **at least a 10% reduction in build time** compared to the base implementation. + +## The Challenge + +The scenario includes: +- **Dockerfile.base**: An unoptimized Dockerfile with multiple inefficiencies +- **Dockerfile.optimized**: A pre-optimized version that you can study and improve further +- **Sample Rails app**: A minimal Rails application for testing builds +- **Timing script**: Automated build time comparison tool + +## Common Docker Optimization Techniques + +Here are some optimization strategies you might consider: + +### 1. **Layer Caching Optimization** +- Order instructions from least to most frequently changing +- Copy dependency files before application code +- Use `.dockerignore` to exclude unnecessary files + +### 2. **Reduce Layer Count** +- Combine multiple RUN statements +- Chain commands with `&&` +- Clean up in the same layer where packages are installed + +### 3. **Multi-stage Builds** +- Separate build dependencies from runtime dependencies +- Use a smaller base image for the final stage +- Copy only necessary artifacts between stages + +### 4. **Base Image Selection** +- Use more recent, optimized base images +- Consider Alpine Linux for smaller images +- Use official language-specific images when appropriate + +### 5. **Package Management** +- Use `--no-install-recommends` for apt packages +- Clean package caches in the same layer +- Pin package versions for reproducible builds + +### 6. **Build Context Optimization** +- Minimize build context size +- Use `.dockerignore` effectively +- Avoid sending unnecessary files to Docker daemon + +## Getting Started + +1. **Explore the current setup:** + ```bash + cd /home/admin/docker-optimization + ls -la + ``` + +2. **Examine the base Dockerfile:** + ```bash + cat Dockerfile.base + ``` + +3. **Study the optimized version:** + ```bash + cat Dockerfile.optimized + ``` + +4. **Run the build time comparison:** + ```bash + ./time_builds.sh + ``` + +## Success Criteria + +The scenario is considered complete when: +- Both Dockerfiles build successfully +- The optimized build is **at least 10% faster** than the base build +- The optimization maintains the same functionality + +## Tips for Success + +- **Start with the obvious wins**: Look for repeated `RUN` commands, unnecessary downloads, and poor layer ordering +- **Use Docker's build cache effectively**: Structure your Dockerfile to maximize cache hits +- **Monitor build progress**: Use `docker build --progress=plain` to see detailed build steps +- **Test iteratively**: Make small changes and test frequently +- **Consider the application needs**: Don't break functionality while optimizing + +## Validation + +Run the check script to validate your solution: +```bash +/home/admin/agent/check.sh +``` + +This will verify that your optimization achieves the required 10% improvement in build time. + +## Files in this Scenario + +- `Dockerfile.base` - The unoptimized Dockerfile (starting point) +- `Dockerfile.optimized` - An optimized version to study and improve +- `time_builds.sh` - Script to measure and compare build times +- `setup_scenario.sh` - Initial scenario setup script +- `Gemfile`, `Gemfile.lock` - Rails application dependencies +- `entrypoint.sh` - Docker container entrypoint script +- Sample Rails application files for realistic build testing + +Good luck with your Docker optimization challenge! \ No newline at end of file diff --git a/scenarios/kingston/ansible/playbook-debian.yml b/scenarios/kingston/ansible/playbook-debian.yml index e88b58f..1770ed9 100644 --- a/scenarios/kingston/ansible/playbook-debian.yml +++ b/scenarios/kingston/ansible/playbook-debian.yml @@ -1,37 +1,59 @@ --- # ansible-playbook playbook-debian.yml -u admin --private-key=/path/to/private/key -i "$ip," -- name: copy files to remote server +- name: Setup Docker Image Build Optimization scenario hosts: all tasks: - # OS - - name: update packages and install lsof + # OS packages and Docker installation + - name: Update packages and install prerequisites become: true package: - # This is weird in that it does not exist in the offical docs - # but this module just passes params down to the underlying - # package manager module where `update_cache` is normally a thing so - # it ends up working.... update_cache: yes state: latest name: - - lsof - - util-linux # Ensure that fallocate is installed + - apt-transport-https + - ca-certificates + - curl + - gnupg + - lsb-release + - bc - - name: create log file + - name: Add Docker GPG key become: true - file: - path: /var/log/fallocate.log - state: touch - owner: admin - group: admin + apt_key: + url: https://download.docker.com/linux/debian/gpg + state: present + + - name: Add Docker repository + become: true + apt_repository: + repo: "deb [arch=amd64] https://download.docker.com/linux/debian {{ ansible_distribution_release }} stable" + state: present - - name: cronjob - cron: - name: "reboot" - special_time: reboot - job: "/home/admin/badlog.py &" - - # check.sh + - name: Install Docker + become: true + package: + update_cache: yes + state: latest + name: + - docker-ce + - docker-ce-cli + - containerd.io + + - name: Add admin user to docker group + become: true + user: + name: admin + groups: docker + append: yes + + - name: Start and enable Docker service + become: true + systemd: + name: docker + state: started + enabled: yes + + # Scenario files setup - name: Create /home/admin/agent directory ansible.builtin.file: path: /home/admin/agent @@ -40,34 +62,37 @@ mode: a+wx state: directory - - name: Copy kingston systemd files - become: true + - name: Copy Docker optimization scenario files + become: false copy: - src: {{ item.src }} - dest: {{ item.dest }} + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: "{{ item.mode | default('0644') }}" with_items: - - { src: ../files/mock-build.service, dest: /etc/systemd/system/mock-build.service } - - { src: ../files/secondary-service.service, dest: /etc/systemd/system/secondary-service.service } - - - name: Start mock build application - become: true - systemd: - name: mock-build - state: started + - { src: ../files/Dockerfile.base, dest: /home/admin/Dockerfile.base } + - { src: ../files/Dockerfile.optimized, dest: /home/admin/Dockerfile.optimized } + - { src: ../files/Gemfile, dest: /home/admin/Gemfile } + - { src: ../files/Gemfile.lock, dest: /home/admin/Gemfile.lock } + - { src: ../files/entrypoint.sh, dest: /home/admin/entrypoint.sh, mode: '0755' } + - { src: ../files/sidekiq_shutdown.rb, dest: /home/admin/sidekiq_shutdown.rb, mode: '0755' } + - { src: ../files/time_builds.sh, dest: /home/admin/time_builds.sh, mode: '0755' } + - { src: ../files/setup_scenario.sh, dest: /home/admin/setup_scenario.sh, mode: '0755' } - - name: Start secondary service - become: true - systemd: - name: secondary-service - state: started - enabled: yes - - - name: copy check.sh + - name: Copy check.sh to agent directory copy: src: ../files/check.sh dest: /home/admin/agent/check.sh + mode: '0755' + + - name: Run scenario setup script + become: false + shell: /home/admin/setup_scenario.sh + args: + creates: /home/admin/docker-optimization - - name: set check.sh - file: - path: /home/admin/agent/check.sh - mode: "+x" + - name: Copy timing script to scenario directory + become: false + copy: + src: ../files/time_builds.sh + dest: /home/admin/docker-optimization/time_builds.sh + mode: '0755' diff --git a/scenarios/kingston/files/.dockerignore b/scenarios/kingston/files/.dockerignore new file mode 100644 index 0000000..c1c4ccb --- /dev/null +++ b/scenarios/kingston/files/.dockerignore @@ -0,0 +1,49 @@ +# .dockerignore for optimized Docker builds +.git +.gitignore +README.md +Dockerfile* +.dockerignore +*.md + +# Logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Coverage directory used by tools like istanbul +coverage/ + +# node_modules +node_modules/ + +# Test files +test/ +spec/ +**/*_test.rb +**/*_spec.rb + +# Development dependencies +.bundle/vendor +tmp/ +log/ +public/system/ +storage/ + +# Editor files +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS files +.DS_Store +Thumbs.db \ No newline at end of file diff --git a/scenarios/kingston/files/Dockerfile.base b/scenarios/kingston/files/Dockerfile.base new file mode 100644 index 0000000..bac95a7 --- /dev/null +++ b/scenarios/kingston/files/Dockerfile.base @@ -0,0 +1,59 @@ +FROM ubuntu:16.04 +ENV RUBY_VERSION=2.3.8 +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y curl zlib1g-dev build-essential libssl-dev libreadline-dev libyaml-dev libsqlite3-dev \ + sqlite3 \ + git \ + libxml2-dev \ + libxslt1-dev \ + libcurl4-openssl-dev \ + software-properties-common \ + libffi-dev \ + tzdata \ + # These three are wkhtmltopdf dependencies + wget libfontconfig1 libxrender1 \ + libmysqlclient-dev openssl && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - \ + && apt-get install -y nodejs + +RUN curl -O https://cache.ruby-lang.org/pub/ruby/2.3/ruby-2.3.8.tar.gz && \ + tar -xzvf ruby-2.3.8.tar.gz && \ + cd ruby-2.3.8 && \ + ./configure --disable-install-doc && \ + make && \ + make install && \ + cd / && \ + rm -rf /tmp/ruby-2.3.8 ruby-2.3.8.tar.gz + +# This thing is used by wicked_pdf to generate pdfs from html +RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.3/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz && \ + tar vxf wkhtmltox-0.12.3_linux-generic-amd64.tar.xz && \ + cp wkhtmltox/bin/wk* /usr/bin/ + +COPY . /app + +WORKDIR /app + +COPY Gemfile Gemfile.lock ./ + +# nokogiri needs to be installed separately or it will fail +RUN gem install bundler -v 1.17.0 && \ + gem install rake -v 13.0.6 --force && \ + gem install nokogiri -v '1.9.1' + +RUN bundle install && \ + bundle config --delete bin && \ + rake rails:update:bin && \ + bundle binstubs rails + +COPY . /app +RUN chmod +x /app/entrypoint.sh /app/sidekiq_shutdown.rb + +RUN bundle exec rake RAILS_ENV=production assets:precompile + +CMD ["bundle","exec","rails","server","-p","8000"] \ No newline at end of file diff --git a/scenarios/kingston/files/Dockerfile.optimized b/scenarios/kingston/files/Dockerfile.optimized new file mode 100644 index 0000000..cecb066 --- /dev/null +++ b/scenarios/kingston/files/Dockerfile.optimized @@ -0,0 +1,94 @@ +# Multi-stage build to reduce final image size +FROM ubuntu:20.04 AS builder +ENV RUBY_VERSION=2.7.4 +ENV DEBIAN_FRONTEND=noninteractive + +# Combine RUN statements and use specific versions for better caching +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + curl \ + zlib1g-dev \ + build-essential \ + libssl-dev \ + libreadline-dev \ + libyaml-dev \ + libsqlite3-dev \ + sqlite3 \ + git \ + libxml2-dev \ + libxslt1-dev \ + libcurl4-openssl-dev \ + software-properties-common \ + libffi-dev \ + tzdata \ + wget \ + libfontconfig1 \ + libxrender1 \ + libmysqlclient-dev \ + openssl \ + && curl -sL https://deb.nodesource.com/setup_16.x | bash - \ + && apt-get install -y --no-install-recommends nodejs \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Use Ruby from package manager instead of compiling from source +RUN apt-get update && \ + apt-get install -y --no-install-recommends ruby-full \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Download and install wkhtmltopdf in single layer +RUN wget -q https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.6/wkhtmltox_0.12.6-1.focal_amd64.deb \ + && apt-get update \ + && apt-get install -y --no-install-recommends ./wkhtmltox_0.12.6-1.focal_amd64.deb \ + && rm wkhtmltox_0.12.6-1.focal_amd64.deb \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Copy dependency files first for better layer caching +COPY Gemfile Gemfile.lock ./ + +# Install gems in single RUN statement +RUN gem install bundler -v 2.3.0 \ + && bundle config set --local deployment 'true' \ + && bundle config set --local without 'development test' \ + && bundle install --jobs=$(nproc) \ + && bundle clean --force + +# Copy application code +COPY . . + +# Set permissions and precompile assets in single layer +RUN chmod +x /app/entrypoint.sh /app/sidekiq_shutdown.rb \ + && bundle exec rake RAILS_ENV=production assets:precompile + +# Final runtime stage +FROM ubuntu:20.04 +ENV DEBIAN_FRONTEND=noninteractive + +# Install only runtime dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + ruby \ + nodejs \ + libsqlite3-0 \ + libxml2 \ + libxslt1.1 \ + libcurl4 \ + libmysqlclient21 \ + libfontconfig1 \ + libxrender1 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Copy from builder stage +COPY --from=builder /usr/local/bin/wkhtmlto* /usr/local/bin/ +COPY --from=builder /app . + +EXPOSE 8000 + +CMD ["bundle","exec","rails","server","-p","8000"] \ No newline at end of file diff --git a/scenarios/kingston/files/Gemfile b/scenarios/kingston/files/Gemfile new file mode 100644 index 0000000..a8f5e37 --- /dev/null +++ b/scenarios/kingston/files/Gemfile @@ -0,0 +1,27 @@ +source 'https://rubygems.org' +git_source(:github) { |repo| "https://github.com/#{repo}.git" } + +ruby '2.7.4' + +gem 'rails', '~> 6.1.0' +gem 'sqlite3', '~> 1.4' +gem 'puma', '~> 5.0' +gem 'sass-rails', '>= 6' +gem 'webpacker', '~> 5.0' +gem 'turbo-rails' +gem 'stimulus-rails' +gem 'jbuilder', '~> 2.7' +gem 'bootsnap', '>= 1.4.4', require: false +gem 'nokogiri', '~> 1.12' +gem 'wicked_pdf' +gem 'wkhtmltopdf-binary' + +group :development, :test do + gem 'byebug', platforms: [:mri, :mingw, :x64_mingw] +end + +group :development do + gem 'web-console', '>= 4.1.0' + gem 'listen', '~> 3.3' + gem 'spring' +end \ No newline at end of file diff --git a/scenarios/kingston/files/Gemfile.lock b/scenarios/kingston/files/Gemfile.lock new file mode 100644 index 0000000..ba1c20c --- /dev/null +++ b/scenarios/kingston/files/Gemfile.lock @@ -0,0 +1,53 @@ +GEM + remote: https://rubygems.org/ + specs: + actioncable (6.1.7) + actionpack (= 6.1.7) + activesupport (= 6.1.7) + nio4r (~> 2.0) + websocket-driver (>= 0.6.1) + actionmailbox (6.1.7) + actionpack (= 6.1.7) + activejob (= 6.1.7) + activerecord (= 6.1.7) + activestorage (= 6.1.7) + activesupport (= 6.1.7) + mail (>= 2.7.1) + actionmailer (6.1.7) + actionpack (= 6.1.7) + actionview (= 6.1.7) + activejob (= 6.1.7) + activesupport (= 6.1.7) + mail (~> 2.5, >= 2.5.4) + rails-dom-testing (~> 2.0) + actionpack (6.1.7) + actionview (= 6.1.7) + activesupport (= 6.1.7) + rack (~> 2.0, >= 2.0.9) + rack-test (>= 0.6.3) + rails-dom-testing (~> 2.0) + rails-html-sanitizer (~> 1.0, >= 1.2.0) + rails (6.1.7) + actioncable (= 6.1.7) + actionmailbox (= 6.1.7) + actionmailer (= 6.1.7) + actionpack (= 6.1.7) + actiontext (= 6.1.7) + actionview (= 6.1.7) + activejob (= 6.1.7) + activemodel (= 6.1.7) + activerecord (= 6.1.7) + activestorage (= 6.1.7) + activesupport (= 6.1.7) + bundler (>= 1.15.0) + railties (= 6.1.7) + sprockets-rails (>= 2.0.0) + +PLATFORMS + ruby + +DEPENDENCIES + rails (~> 6.1.0) + +BUNDLED WITH + 2.3.0 \ No newline at end of file diff --git a/scenarios/kingston/files/check.sh b/scenarios/kingston/files/check.sh index 8c383ae..5b1ae02 100644 --- a/scenarios/kingston/files/check.sh +++ b/scenarios/kingston/files/check.sh @@ -1,34 +1,58 @@ -#!/usr/bin/bash +#!/bin/bash -BUILD_MOUNT_POINT="/tmp/ephemeral-build" +# Check script for Docker Image Build Optimization scenario +# Validates that the optimized Docker build achieves >= 10% improvement over base build -# Check if the build mount point exists -function check_build_mount_point { - if [ ! -d $BUILD_MOUNT_POINT ]; then - echo -b "NO" +SCENARIO_DIR="/home/admin/docker-optimization" +IMPROVEMENT_FILE="$SCENARIO_DIR/improvement.txt" +BASE_TIME_FILE="$SCENARIO_DIR/base_time.txt" +OPTIMIZED_TIME_FILE="$SCENARIO_DIR/optimized_time.txt" + +# Check if Docker is installed and running +if ! command -v docker &> /dev/null; then + echo "NO" + exit 1 +fi + +if ! docker info &> /dev/null; then + echo "NO" + exit 1 +fi + +# Check if the scenario directory exists +if [ ! -d "$SCENARIO_DIR" ]; then + echo "NO" + exit 1 +fi + +# Check if Dockerfiles exist +if [ ! -f "$SCENARIO_DIR/Dockerfile.base" ] || [ ! -f "$SCENARIO_DIR/Dockerfile.optimized" ]; then + echo "NO" + exit 1 +fi + +# Check if build timing has been run +if [ ! -f "$IMPROVEMENT_FILE" ] || [ ! -f "$BASE_TIME_FILE" ] || [ ! -f "$OPTIMIZED_TIME_FILE" ]; then + echo "NO" exit 1 - fi -} - -# Function to check if both example-build-artifact.txt and -# secondary-artifact.txt exist in the build mount point -# BOTH files need to exist concurrently -function check_build_artifacts { - BUILD_FILES=( - "example-build-artifact.txt" - "secondary-artifact.txt" - ) - for file in "${BUILD_FILES[@]}"; do - if [ ! -f "$BUILD_MOUNT_POINT/$file" ]; then - echo -b "NO" - exit 1 +fi + +# Read the improvement percentage +if [ -f "$IMPROVEMENT_FILE" ]; then + improvement=$(cat "$IMPROVEMENT_FILE") + + # Extract integer part of improvement (before decimal point) + improvement_int=$(echo "$improvement" | cut -d'.' -f1) + + # Check if improvement is >= 10% + if [ "$improvement_int" -ge 10 ]; then + echo "OK" + exit 0 else - echo -b "OK" - exit 0 + echo "NO" + exit 1 fi - done -} - -# Check if one or both functions executed successfull -check_build_mount_point -check_build_artifacts +else + echo "NO" + exit 1 +fi diff --git a/scenarios/kingston/files/entrypoint.sh b/scenarios/kingston/files/entrypoint.sh new file mode 100644 index 0000000..eb40a5e --- /dev/null +++ b/scenarios/kingston/files/entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +# Remove a potentially pre-existing server.pid for Rails. +rm -f /app/tmp/pids/server.pid + +# Then exec the container's main process (what's set as CMD in the Dockerfile). +exec "$@" \ No newline at end of file diff --git a/scenarios/kingston/files/mock-build-run.sh b/scenarios/kingston/files/mock-build-run.sh deleted file mode 100644 index 0d2f659..0000000 --- a/scenarios/kingston/files/mock-build-run.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/env/bin bash - -MOUNT_POINT="/tmp/ephemeral-build" -EXAMPLE_BUILD_ARTIFACT="example-build-artifact.txt" -MAX_BUILD_ARTIFACT_SIZE=10M - -# Create the example build file -touch $MOUNT_POINT/$EXAMPLE_BUILD_ARTIFACT - -# Use fallocate to allocate the maximum size of the file -fallocate -l "$MAX_SIZE" "$MOUNT_POINT/$EXAMPLE_BUILD_ARTIFACT" - -trap diff --git a/scenarios/kingston/files/mock-build.service b/scenarios/kingston/files/mock-build.service deleted file mode 100644 index 335b661..0000000 --- a/scenarios/kingston/files/mock-build.service +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Mock Build Application -After=network.target - -[Service] -ExecStart=/home/admin/mock_build.sh - -[Install] -WantedBy=multi-user.target diff --git a/scenarios/kingston/files/secondary-service.service b/scenarios/kingston/files/secondary-service.service deleted file mode 100644 index 08eb9b9..0000000 --- a/scenarios/kingston/files/secondary-service.service +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Secondary service Application -After=network.target - -[Service] -ExecStart=/home/admin/secondary-service.sh - -[Install] -WantedBy=multi-user.target diff --git a/scenarios/kingston/files/secondary-service.sh b/scenarios/kingston/files/secondary-service.sh deleted file mode 100644 index 40972ef..0000000 --- a/scenarios/kingston/files/secondary-service.sh +++ /dev/null @@ -1,2 +0,0 @@ -# Secondary service that will continually run in the background -# and try to place a file on tmp artifact directory that is too big to fit on disk diff --git a/scenarios/kingston/files/setup_scenario.sh b/scenarios/kingston/files/setup_scenario.sh new file mode 100644 index 0000000..6b885e7 --- /dev/null +++ b/scenarios/kingston/files/setup_scenario.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# Setup script for Docker optimization scenario +# Creates the scenario environment and copies necessary files + +SCENARIO_DIR="/home/admin/docker-optimization" + +echo "Setting up Docker Image Build Optimization scenario..." + +# Create scenario directory +mkdir -p "$SCENARIO_DIR/app" + +# Copy Dockerfiles +cp /home/admin/Dockerfile.base "$SCENARIO_DIR/" +cp /home/admin/Dockerfile.optimized "$SCENARIO_DIR/" + +# Create basic Rails app structure for Docker builds +mkdir -p "$SCENARIO_DIR/app/tmp/pids" +mkdir -p "$SCENARIO_DIR/app/config" +mkdir -p "$SCENARIO_DIR/app/app/assets" + +# Copy application files +cp /home/admin/Gemfile "$SCENARIO_DIR/app/" +cp /home/admin/Gemfile.lock "$SCENARIO_DIR/app/" +cp /home/admin/entrypoint.sh "$SCENARIO_DIR/app/" +cp /home/admin/sidekiq_shutdown.rb "$SCENARIO_DIR/app/" + +# Create minimal Rails configuration files +cat > "$SCENARIO_DIR/app/config/application.rb" << 'EOF' +require_relative "boot" +require "rails/all" + +Bundler.require(*Rails.groups) + +module DockerOptimizationApp + class Application < Rails::Application + config.load_defaults 6.1 + end +end +EOF + +cat > "$SCENARIO_DIR/app/config/boot.rb" << 'EOF' +ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__) +require "bundler/setup" +require "bootsnap/setup" +EOF + +# Create Rakefile +cat > "$SCENARIO_DIR/app/Rakefile" << 'EOF' +require_relative "config/application" +Rails.application.load_tasks +EOF + +# Create simple asset files for precompilation +mkdir -p "$SCENARIO_DIR/app/app/assets/stylesheets" +echo "/* Application styles */" > "$SCENARIO_DIR/app/app/assets/stylesheets/application.css" + +mkdir -p "$SCENARIO_DIR/app/app/assets/javascripts" +echo "// Application JavaScript" > "$SCENARIO_DIR/app/app/assets/javascripts/application.js" + +# Set permissions +chmod +x "$SCENARIO_DIR/app/entrypoint.sh" +chmod +x "$SCENARIO_DIR/app/sidekiq_shutdown.rb" + +echo "Scenario setup complete!" +echo "" +echo "To run the optimization test:" +echo "1. cd $SCENARIO_DIR" +echo "2. Run: ./time_builds.sh" +echo "" +echo "The goal is to optimize the Dockerfile.optimized to achieve >10% build time improvement over Dockerfile.base" \ No newline at end of file diff --git a/scenarios/kingston/files/sidekiq_shutdown.rb b/scenarios/kingston/files/sidekiq_shutdown.rb new file mode 100644 index 0000000..6acfd2e --- /dev/null +++ b/scenarios/kingston/files/sidekiq_shutdown.rb @@ -0,0 +1,3 @@ +#!/usr/bin/env ruby +# Simple Sidekiq shutdown script placeholder +puts "Shutting down Sidekiq gracefully..." \ No newline at end of file diff --git a/scenarios/kingston/files/time_builds.sh b/scenarios/kingston/files/time_builds.sh new file mode 100644 index 0000000..228e317 --- /dev/null +++ b/scenarios/kingston/files/time_builds.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +# Docker Image Build Time Comparison Script +# This script builds both the base and optimized Dockerfiles and compares build times + +SCENARIO_DIR="/home/admin/docker-optimization" +BASE_DOCKERFILE="$SCENARIO_DIR/Dockerfile.base" +OPTIMIZED_DOCKERFILE="$SCENARIO_DIR/Dockerfile.optimized" +APP_DIR="$SCENARIO_DIR/app" + +# Results file +RESULTS_FILE="$SCENARIO_DIR/build_times.txt" + +echo "Docker Image Build Time Comparison" > "$RESULTS_FILE" +echo "====================================" >> "$RESULTS_FILE" +echo "" >> "$RESULTS_FILE" + +# Function to build and time a Docker image +build_and_time() { + local dockerfile=$1 + local image_name=$2 + local label=$3 + + echo "Building $label..." + echo "Building $label..." >> "$RESULTS_FILE" + + # Clear Docker cache to ensure fair comparison + docker builder prune -f >/dev/null 2>&1 + + # Time the build + start_time=$(date +%s) + + # Build the image (capture both stdout and stderr) + if docker build -f "$dockerfile" -t "$image_name" "$APP_DIR" >/dev/null 2>&1; then + end_time=$(date +%s) + build_time=$((end_time - start_time)) + echo "Build time: ${build_time} seconds" + echo "Build time: ${build_time} seconds" >> "$RESULTS_FILE" + echo "$build_time" + else + echo "Build failed!" + echo "Build failed!" >> "$RESULTS_FILE" + echo "-1" + fi + + echo "" >> "$RESULTS_FILE" +} + +# Ensure we're in the right directory +cd "$SCENARIO_DIR" || exit 1 + +# Build base image +echo "=== Building Base Docker Image ===" +base_time=$(build_and_time "$BASE_DOCKERFILE" "rails-app:base" "Base Image") + +if [ "$base_time" -eq -1 ]; then + echo "Base image build failed. Cannot proceed with comparison." + exit 1 +fi + +# Build optimized image +echo "=== Building Optimized Docker Image ===" +optimized_time=$(build_and_time "$OPTIMIZED_DOCKERFILE" "rails-app:optimized" "Optimized Image") + +if [ "$optimized_time" -eq -1 ]; then + echo "Optimized image build failed. Cannot proceed with comparison." + exit 1 +fi + +# Calculate improvement using shell arithmetic (fallback if bc not available) +echo "=== Results ===" >> "$RESULTS_FILE" +echo "Base build time: ${base_time} seconds" >> "$RESULTS_FILE" +echo "Optimized build time: ${optimized_time} seconds" >> "$RESULTS_FILE" + +if [ "$base_time" -gt 0 ]; then + # Calculate improvement percentage using shell arithmetic + # improvement = ((base_time - optimized_time) * 100) / base_time + time_saved=$((base_time - optimized_time)) + improvement_x100=$(((base_time - optimized_time) * 10000 / base_time)) + improvement=$((improvement_x100 / 100)) + improvement_decimal=$((improvement_x100 % 100)) + + echo "Time saved: ${time_saved} seconds" >> "$RESULTS_FILE" + echo "Improvement: ${improvement}.${improvement_decimal}%" >> "$RESULTS_FILE" + + echo "" + echo "=== Build Time Comparison Results ===" + echo "Base build time: ${base_time} seconds" + echo "Optimized build time: ${optimized_time} seconds" + echo "Time saved: ${time_saved} seconds" + echo "Improvement: ${improvement}.${improvement_decimal}%" + + # Store times for check script + echo "$base_time" > "$SCENARIO_DIR/base_time.txt" + echo "$optimized_time" > "$SCENARIO_DIR/optimized_time.txt" + echo "${improvement}.${improvement_decimal}" > "$SCENARIO_DIR/improvement.txt" + + # Check if improvement is >= 10% (using integer comparison) + if [ "$improvement" -ge 10 ]; then + echo "" + echo "🎉 SUCCESS: Optimization achieved ${improvement}.${improvement_decimal}% improvement (>= 10% required)" + exit 0 + else + echo "" + echo "❌ INSUFFICIENT: Optimization achieved only ${improvement}.${improvement_decimal}% improvement (>= 10% required)" + exit 1 + fi +else + echo "Error: Invalid build time" + exit 1 +fi \ No newline at end of file diff --git a/scenarios/kingston/packer/aws-debian11.pkr.hcl b/scenarios/kingston/packer/aws-debian11.pkr.hcl index 11c7991..1834b84 100644 --- a/scenarios/kingston/packer/aws-debian11.pkr.hcl +++ b/scenarios/kingston/packer/aws-debian11.pkr.hcl @@ -1,4 +1,4 @@ -# Debian +# Debian AMI for Kingston Docker Optimization Scenario packer { required_plugins { @@ -10,8 +10,8 @@ packer { } source "amazon-ebs" "debian" { - ami_name = "scenario-1-saintjohn" - instance_type = "t3a.nano" + ami_name = "scenario-kingston-docker-optimization" + instance_type = "t3a.small" region = "${var.region}" vpc_id = "${var.vpc_id}" subnet_id = "${var.subnet_id}" @@ -26,28 +26,91 @@ build { "source.amazon-ebs.debian" ] - # OS & scenario packages + # OS & Docker setup provisioner "shell" { inline = [ - "echo Update packages...", + "echo Installing Docker prerequisites...", "sudo apt-get update", - "sudo apt-get install -y lsof", + "sudo apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release bc", ] } - # badlog.py + # Install Docker + provisioner "shell" { + inline = [ + "curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg", + "echo \"deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable\" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null", + "sudo apt-get update", + "sudo apt-get install -y docker-ce docker-ce-cli containerd.io", + "sudo usermod -aG docker admin", + ] + } + + # Copy Docker scenario files + provisioner "file" { + source = "../files/Dockerfile.base" + destination = "/tmp/Dockerfile.base" + } + + provisioner "file" { + source = "../files/Dockerfile.optimized" + destination = "/tmp/Dockerfile.optimized" + } + + provisioner "file" { + source = "../files/time_builds.sh" + destination = "/tmp/time_builds.sh" + } + + provisioner "file" { + source = "../files/setup_scenario.sh" + destination = "/tmp/setup_scenario.sh" + } + + provisioner "file" { + source = "../files/Gemfile" + destination = "/tmp/Gemfile" + } + + provisioner "file" { + source = "../files/Gemfile.lock" + destination = "/tmp/Gemfile.lock" + } + provisioner "file" { - source = "../files/badlog.py" - destination = "/tmp/badlog.py" + source = "../files/entrypoint.sh" + destination = "/tmp/entrypoint.sh" + } + + provisioner "file" { + source = "../files/sidekiq_shutdown.rb" + destination = "/tmp/sidekiq_shutdown.rb" + } + + # Setup scenario files + provisioner "shell" { + inline = [ + "sudo mkdir -p /home/admin/agent", + "sudo mv /tmp/Dockerfile.base /home/admin/", + "sudo mv /tmp/Dockerfile.optimized /home/admin/", + "sudo mv /tmp/time_builds.sh /home/admin/", + "sudo mv /tmp/setup_scenario.sh /home/admin/", + "sudo mv /tmp/Gemfile /home/admin/", + "sudo mv /tmp/Gemfile.lock /home/admin/", + "sudo mv /tmp/entrypoint.sh /home/admin/", + "sudo mv /tmp/sidekiq_shutdown.rb /home/admin/", + "sudo chmod +x /home/admin/time_builds.sh", + "sudo chmod +x /home/admin/setup_scenario.sh", + "sudo chmod +x /home/admin/entrypoint.sh", + "sudo chmod +x /home/admin/sidekiq_shutdown.rb", + "sudo chown -R admin:admin /home/admin/", + ] } + # Setup Docker optimization scenario provisioner "shell" { inline = [ - "mv /tmp/badlog.py /home/admin/badlog.py", - "chmod +x /home/admin/badlog.py", - "sudo touch /var/log/bad.log", - "sudo chown admin: /var/log/bad.log", - "echo '@reboot /home/admin/badlog.py &' | crontab -", + "sudo -u admin /home/admin/setup_scenario.sh", ] }