ci.yml

name: CI

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

permissions:
  contents: read

# Pushes to main queue so each merge gets a test result; PR updates cancel the
# run for the superseded head.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name != 'push' }}

jobs:
  test:
    name: Julia ${{ matrix.julia-version }} / ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    # The repository's root .Rprofile sources renv/activate.R, so R started from
    # the checkout auto-activates renv and rewrites the library paths. renv is a
    # local-development convenience; CI installs into the system library and
    # reaches it through R_LIBS_SITE, so the autoloader is switched off here.
    # This variable covers the unprivileged R that Julia embeds through RCall.
    # It cannot cover the installs themselves, which run under sudo: sudo resets
    # the environment, so those commands pass --no-init-file instead.
    env:
      RENV_CONFIG_AUTOLOADER_ENABLED: "FALSE"
    strategy:
      fail-fast: false
      matrix:
        # Pinned rather than tracking latest stable, because a pinned Manifest.toml
        # resolved by an unpinned Julia is not a reproducible build. This must equal
        # julia_version in Manifest.toml; it cannot be read from the pin file, since
        # nothing can be read before Julia exists. test/unit/test_install_pins.jl
        # fails if the three ever disagree.
        julia-version: ["1.12.5"]
        # Explicit, not ubuntu-latest: the R pin below names an apt package revision
        # built for 24.04, and a runner that silently rolled to the next LTS would
        # take that pin with it.
        os: [ubuntu-24.04]

    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4.4.0

      - name: Set up Julia
        uses: julia-actions/setup-julia@4c0cb0fce8556fdb04a90347310e5db8b1f98fb9  # v2.7.0
        with:
          version: ${{ matrix.julia-version }}

      - name: Cache Julia packages
        uses: julia-actions/cache@d10a6fd8f31b12404a54613ebad242900567f2b9  # v2.1.0

      # Every external version CI installs is read from the committed pin file, so CI
      # and a developer's machine cannot drift apart. A temporary environment is used
      # because the project environment cannot be instantiated until R is present:
      # RCall's build step requires it.
      - name: Read pinned versions
        run: |
          julia -e '
            using Pkg
            Pkg.activate(temp = true)
            Pkg.add("YAML")
            using YAML
            pins = YAML.load_file("config/defaults/tool_versions.yml")
            open(ENV["GITHUB_ENV"], "a") do io
                println(io, "R_APT_VERSION=", pins["runtimes"]["r"]["apt_version"])
                println(io, "BUN_VERSION=", pins["toolchain"]["bun"]["version"])
                println(io, "CUTADAPT_SPEC=cutadapt==", pins["tools"]["cutadapt"]["version"])
                println(io, "MULTIQC_SPEC=multiqc==", pins["tools"]["multiqc"]["version"])
                fastqc = pins["tools"]["fastqc"]["archives"]["any"]
                println(io, "FASTQC_URL=", fastqc["url"])
                println(io, "FASTQC_SHA256=", fastqc["sha256"])
                for tool in ("vsearch", "swarm")
                    archive = pins["tools"][tool]["archives"]["linux-x86_64"]
                    println(io, uppercase(tool), "_VERSION=", pins["tools"][tool]["version"])
                    println(io, uppercase(tool), "_URL=", archive["url"])
                    println(io, uppercase(tool), "_SHA256=", archive["sha256"])
                end
            end'

      # R + required packages (needed by RCall)
      # Install R directly via apt to avoid r-lib/actions/setup-r mangling
      # R_HOME and stripping default packages (utils, methods, stats, etc.)
      # The version is pinned to the apt revision named in the pin file, which tracks
      # R.Version in renv.lock; CRAN's Ubuntu repository retains older revisions, so
      # this resolves rather than merely requesting the newest.
      - name: Set up R
        run: |
          wget --https-only -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc \
            | sudo gpg --dearmor -o /usr/share/keyrings/r-project.gpg
          echo "deb [signed-by=/usr/share/keyrings/r-project.gpg] https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/" \
            | sudo tee /etc/apt/sources.list.d/r-project.list
          sudo apt-get update -qq
          sudo apt-get install -y \
            r-base-core="$R_APT_VERSION" \
            r-base-dev="$R_APT_VERSION" \
            r-base-html="$R_APT_VERSION" \
            r-recommended="$R_APT_VERSION" \
            r-base="$R_APT_VERSION"
          Rscript -e 'cat(R.version.string, "\n")'

      - name: Install R system dependencies
        run: sudo apt-get install -y libcurl4-openssl-dev libssl-dev libxml2-dev libfontconfig1-dev

      # The R packages are restored from renv.lock, the same pin a developer restores
      # from, rather than resolved against whatever the repositories serve that day.
      # Asking for a bare package name pins nothing: a Bioconductor release is frozen,
      # so dada2 and its kin held still, but CRAN publishes only its newest revision,
      # so vegan silently moved off the pinned 2.7-3 and test_provenance caught the
      # drift. Restoring into .Library keeps the system library that the RCall steps
      # and R_LIBS_SITE below expect, and renv is invoked explicitly here rather than
      # left to activate itself. --no-init-file stops R sourcing the repository's
      # .Rprofile, and with it the renv autoloader, which would otherwise rebind
      # .Library to its own sandbox under root's cache; the flag is used rather than
      # the environment variable above because sudo resets the environment first.
      #
      # renv's package cache persists across runs, so a restore that fails partway
      # resumes from the packages it already built. The cache sits outside ~ because
      # the restore runs as root. The run id makes each key unique so every attempt
      # saves its progress; restore-keys picks the newest match.
      - name: Restore the renv cache
        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830  # v4.3.0
        with:
          path: /opt/renv-cache
          key: renv-${{ runner.os }}-R${{ env.R_APT_VERSION }}-${{ hashFiles('renv.lock') }}-${{ github.run_id }}
          restore-keys: |
            renv-${{ runner.os }}-R${{ env.R_APT_VERSION }}-${{ hashFiles('renv.lock') }}-
            renv-${{ runner.os }}-R${{ env.R_APT_VERSION }}-

      # R_CLI_DYNAMIC and TERM make renv print progress one line at a time in the log.
      - name: Install R packages
        env:
          RENV_PATHS_CACHE: /opt/renv-cache
        run: |
          sudo mkdir -p "$RENV_PATHS_CACHE"
          sudo chmod 777 "$RENV_PATHS_CACHE"
          sudo env R_CLI_DYNAMIC=false TERM=dumb Rscript --no-init-file -e 'install.packages(c("renv", "BiocManager"), repos="https://cloud.r-project.org", lib=.Library)'
          sudo env RENV_PATHS_CACHE="$RENV_PATHS_CACHE" R_CLI_DYNAMIC=false TERM=dumb Rscript --no-init-file -e 'renv::restore(project=".", library=.Library, prompt=FALSE)'
          Rscript --no-init-file -e 'for (pkg in c("dada2","Biostrings","ShortRead","vegan","dplyr","MASS")) if (!require(pkg,character.only=TRUE,quietly=TRUE)) stop(pkg, " failed to install")'

      # Saved on failure too, since the packages built before a failure are the ones
      # the next attempt should reuse. Root writes the cache, so it is made readable first.
      - name: Make the renv cache readable
        if: always()
        run: sudo chmod -R a+rX /opt/renv-cache || true

      - name: Save the renv cache
        if: always()
        uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830  # v4.3.0
        with:
          path: /opt/renv-cache
          key: renv-${{ runner.os }}-R${{ env.R_APT_VERSION }}-${{ hashFiles('renv.lock') }}-${{ github.run_id }}

      - name: Install cutadapt
        run: pip install "$CUTADAPT_SPEC"

      # The one tool CI does not pin. Ubuntu ships no versioned cd-hit revision worth
      # naming, and 24.04 is frozen, so its cd-hit does not move under us; the version
      # that arrives is recorded at preflight, where a discrepancy is visible.
      - name: Install cd-hit
        run: sudo apt-get install -y cd-hit

      # Downloaded against the pinned URL and refused unless it hashes to the pinned
      # SHA256, so CI runs the same bytes install.jl puts on a developer's machine.
      - name: Install vsearch
        run: |
          source scripts/ci/fetch_pinned.sh
          fetch_pinned "$VSEARCH_URL" vsearch.tar.gz
          echo "$VSEARCH_SHA256  vsearch.tar.gz" | sha256sum -c -
          tar xzf vsearch.tar.gz --warning=no-unknown-keyword
          sudo mv "vsearch-$VSEARCH_VERSION-linux-x86_64/bin/vsearch" /usr/local/bin/vsearch
          rm -rf vsearch.tar.gz "vsearch-$VSEARCH_VERSION-linux-x86_64"
          vsearch --version

      - name: Install swarm
        run: |
          source scripts/ci/fetch_pinned.sh
          fetch_pinned "$SWARM_URL" swarm.tar.gz
          echo "$SWARM_SHA256  swarm.tar.gz" | sha256sum -c -
          tar xzf swarm.tar.gz --warning=no-unknown-keyword
          sudo mv "swarm-$SWARM_VERSION-linux-x86_64/bin/swarm" /usr/local/bin/swarm
          rm -rf swarm.tar.gz "swarm-$SWARM_VERSION-linux-x86_64"
          swarm --version

      - name: Install multiqc
        run: pip install "$MULTIQC_SPEC"

      # FastQC ships as a Java launcher script; `fastqc --version` checks that Java
      # is on PATH as well.
      - name: Install fastqc
        run: |
          source scripts/ci/fetch_pinned.sh
          fetch_pinned "$FASTQC_URL" fastqc.zip
          echo "$FASTQC_SHA256  fastqc.zip" | sha256sum -c -
          sudo unzip -q -d /opt fastqc.zip
          rm -f fastqc.zip
          sudo chmod +x /opt/FastQC/fastqc
          sudo ln -sf /opt/FastQC/fastqc /usr/local/bin/fastqc
          fastqc --version

      - name: Instantiate Julia project
        run: julia --project=. -e 'import Pkg; Pkg.instantiate()'

      - name: Set up Bun
        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6  # v2.2.0
        with:
          bun-version: ${{ env.BUN_VERSION }}

      - name: Build frontend
        working-directory: frontend
        run: |
          bun install --frozen-lockfile --ignore-scripts
          bun run build

      - name: Frontend unit tests
        working-directory: frontend
        run: bun run test

      - name: Download PR2 databases
        run: julia --project=. config/ci/download_databases.jl

      - name: Rebuild RCall against installed R
        run: julia --project=. -e 'import Pkg; Pkg.build("RCall")'

      - name: Verify R packages visible from RCall
        run: |
          julia --project=. -e '
            using RCall
            R"cat(.libPaths(), sep=\"\n\")"
            for pkg in ["dada2", "dplyr", "vegan", "tibble", "MASS"]
              R"if (!require($pkg, character.only=TRUE, quietly=TRUE)) stop($pkg, \" not found\")"
            end
          '

      - name: Run tests
        env:
          CI_SKIP_TAXONOMY: "1"
          R_LIBS_SITE: /usr/local/lib/R/site-library:/usr/lib/R/site-library:/usr/lib/R/library
        # A failure also posts annotations, which the checks API serves without the full
        # log. Annotations are capped at 4096 bytes, so each failing assertion gets its own
        # (up to 8), followed by the summary rows that report a failure or an error.
        shell: bash
        run: |
          set +e
          julia --project=. -t 2 --code-coverage=user --compiled-modules=no test/runtests.jl --integration --server 2>&1 | tee /tmp/julia-tests.log
          status="${PIPESTATUS[0]}"
          if [ "$status" -ne 0 ]; then
            enc() { head -c 3500 | sed -e 's/%/%25/g' -e 's/\r/%0D/g' | awk '{printf "%s%%0A", $0}'; }
            awk '
              /Test Summary:/ { left = 0 }
              /(Test Failed at|Error During Test at)/ || (left == 0 && /Got exception outside of a @test/) { n++; left = 60 }
              left > 0 { print > ("/tmp/julia-failure-" n ".txt"); left-- }
            ' /tmp/julia-tests.log
            shown=0
            for f in $(ls /tmp/julia-failure-*.txt 2>/dev/null | sort -t- -k3 -n); do
              shown=$((shown + 1))
              [ "$shown" -gt 8 ] && break
              echo "::error title=Julia test failure ${shown}::$(enc < "$f")"
            done
            awk '
              /Test Summary:/ { print; p = index($0, "Pass"); pe = (p ? p + 3 : 0); t = index($0, "Total"); next }
              pe && t && index($0, "|") && substr($0, pe + 1, t - pe - 1) ~ /[0-9]/ { print; next }
              /ERROR:|Some tests did not pass/ { print }
            ' /tmp/julia-tests.log | tail -n 60 > /tmp/julia-tests-summary.txt
            echo "::error title=Julia tests failed (summary)::$(enc < /tmp/julia-tests-summary.txt)"
            if [ "$shown" -eq 0 ]; then
              echo "::error title=Julia tests failed (tail)::$(tail -c 3500 /tmp/julia-tests.log | enc)"
            fi
          fi
          exit "$status"

      - name: Process coverage
        uses: julia-actions/julia-processcoverage@03114f09f119417c3242a9fb6e0b722676aedf38  # v1.2.2

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f  # v6.0.2
        with:
          files: lcov.info
          token: ${{ secrets.CODECOV_TOKEN }}
          slug: JoshuaJewell/MetaManifold-WebUI
          fail_ci_if_error: false