diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 39f68f12..d3aad24f 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -17,375 +17,177 @@ jobs: strategy: fail-fast: false name: Build & Tests - steps: - - name: Add apt repo - run: sudo add-apt-repository universe - - # Setup Python - - name: Setup Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - with: - python-version: "3.11" - architecture: x64 - - - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - with: - version: "0.11.21" + steps: - name: Checkout DataGateway API uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - # Prep for using the API for tests - - name: Configure datagateway extension - run: echo "`yq \ - '.datagateway_api.extension="/datagateway-api"' datagateway_api/config.yaml.example`" > datagateway_api/config.yaml.example - - name: Create config.yaml - run: cp datagateway_api/config.yaml.example datagateway_api/config.yaml - - name: Create search_api_mapping.json - run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json - - name: Create logging.ini - run: cp datagateway_api/logging.example.ini datagateway_api/logging.ini - # Run Unit tests - - name: Run unit tests - run: uv run pytest test/unit --cov=datagateway_api --cov-report=xml + - name: Create coverage file + run: touch coverage.xml - # ICAT Ansible clone and install dependencies - - name: Checkout icat-ansible - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - repository: icatproject-contrib/icat-ansible - ref: payara7 - path: icat-ansible - - name: Install Ansible - run: pip install -r icat-ansible/requirements.txt - - # Prep for running the playbook - - name: Create hosts file - run: echo -e "[icatdb_minimal_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts - - name: Prepare vault pass - run: echo -e "icattravispw" > icat-ansible/vault_pass.txt - - name: Move vault to directory it'll get detected by Ansible - run: mv icat-ansible/vault.yml icat-ansible/group_vars/all - - name: Replace default payara user with Actions user - run: | - sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml + - name: Run unit tests + run: docker compose run unit-tests - # Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions - - name: Change hostname to localhost - run: sudo hostname -b localhost + - name: Start test dependencies + run: docker compose --profile dependencies up -d - # Remove existing MySQL installation so it doesn't interfere with GitHub Actions - - name: Remove existing mysql - run: | - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - sudo apt-get remove --purge "mysql*" - sudo rm -rf /var/lib/mysql* /etc/mysql + - name: Wait for test data generation + run: docker wait dg_api_testdata - # Create local instance of ICAT - - name: Run ICAT Ansible Playbook - run: | - ansible-playbook icat-ansible/icatdb_minimal_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv - - # rootUserNames needs editing as anon/anon is used in search API and required to pass endpoint tests - - name: Add anon user to rootUserNames - run: | - awk -F" =" '/rootUserNames/{$2="= simple/root anon/anon";print;next}1' /home/runner/install/icat.server/run.properties > /home/runner/install/icat.server/run.properties.tmp - - name: Apply rootUserNames change - run: | - mv -f /home/runner/install/icat.server/run.properties.tmp /home/runner/install/icat.server/run.properties - - name: Reinstall ICAT Server - run: | - cd /home/runner/install/icat.server/ && ./setup -vv install + - name: Run integration tests + run: docker compose run integration-tests - - name: Checkout DataGateway API - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - name: Upload code coverage + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} - # Prep for using the API for tests - - name: Configure datagateway extension - run: echo "`yq \ - '.datagateway_api.extension="/datagateway-api"' datagateway_api/config.yaml.example`" > datagateway_api/config.yaml.example - - name: Create config.yaml - run: cp datagateway_api/config.yaml.example datagateway_api/config.yaml - - name: Create search_api_mapping.json - run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json - - name: Create logging.ini - run: cp datagateway_api/logging.example.ini datagateway_api/logging.ini - - - name: Add dummy data to icatdb - run: | - uv run python -m util.icat_db_generator + - name: Show logs on failure + if: failure() + run: docker compose logs - # Run integration tests session, saves and uploads a coverage report to codecov - - name: Run integration tests - run: uv run pytest test/integration --cov=datagateway_api --cov-report=xml + - name: Cleanup + if: always() + run: docker compose --profile dependencies down linting: runs-on: ubuntu-24.04 name: Linting + steps: - - name: Setup Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - with: - python-version: "3.11" - architecture: x64 - name: Checkout DataGateway API uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - with: - version: "0.11.21" - - name: Run linting - run: uv run flake8 datagateway_api test util + run: docker compose run lint formatting: runs-on: ubuntu-24.04 name: Code Formatting + steps: - - name: Setup Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - with: - python-version: "3.11" - architecture: x64 - name: Checkout DataGateway API uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - with: - version: "0.11.21" - - name: Install dependencies and check formatting - run: uv run black --check datagateway_api test util + - name: Run formatting checks + run: docker compose run format generator-script-testing: runs-on: ubuntu-24.04 continue-on-error: true name: icatdb Generator Script Consistency Test - steps: - - name: Add apt repo - run: sudo add-apt-repository universe - # Setup Python - - name: Setup Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - with: - python-version: "3.11" - architecture: x64 - - - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - with: - version: "0.11.21" - # ICAT Ansible clone and install dependencies - - name: Checkout icat-ansible + steps: + - name: Checkout PR branch uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - repository: icatproject-contrib/icat-ansible - ref: payara7 - path: icat-ansible - - name: Install Ansible - run: pip install -r icat-ansible/requirements.txt - - # Prep for running the playbook - - name: Create hosts file - run: echo -e "[icatdb_minimal_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts - - name: Prepare vault pass - run: echo -e "icattravispw" > icat-ansible/vault_pass.txt - - name: Move vault to directory it'll get detected by Ansible - run: mv icat-ansible/vault.yml icat-ansible/group_vars/all - - name: Replace default payara user with Actions user - run: | - sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml + path: pr - # Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions - - name: Change hostname to localhost - run: sudo hostname -b localhost + - name: Start dependencies + working-directory: pr + run: docker compose --profile dependencies up -d - # Remove existing MySQL installation so it doesn't interfere with GitHub Actions - - name: Remove existing mysql - run: | - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - sudo apt-get remove --purge "mysql*" - sudo rm -rf /var/lib/mysql* /etc/mysql + - name: Wait for test data generation + run: docker wait dg_api_testdata - # Create local instance of ICAT - - name: Run ICAT Ansible Playbook + - name: Drop modTime and createTime run: | - ansible-playbook icat-ansible/icatdb_minimal_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv + docker exec -i icat_mariadb_container \ + mysql -picatdbuserpw -uicatdbuser icatdb \ + < pr/util/columns_to_drop.sql - - name: Checkout DataGateway API - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - - name: Create config.yaml - run: cd /home/runner/work/datagateway-api/datagateway-api; cp datagateway_api/config.yaml.example datagateway_api/config.yaml - - name: Create search_api_mapping.json - run: cd /home/runner/work/datagateway-api/datagateway-api; cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json - - name: Create logging.ini - run: cd /home/runner/work/datagateway-api/datagateway-api; cp datagateway_api/logging.example.ini datagateway_api/logging.ini - - name: Add dummy data to icatdb - run: uv run python -m util.icat_db_generator - - name: Drop modTime and createTime - run: mysql -picatdbuserpw -uicatdbuser icatdb < /home/runner/work/datagateway-api/datagateway-api/util/columns_to_drop.sql - name: Get SQL dump of dummy data - run: mysqldump -picatdbuserpw -uicatdbuser --skip-comments icatdb > ~/generator_script_dump_1.sql + run: | + docker exec icat_mariadb_container \ + mysqldump \ + -picatdbuserpw \ + -uicatdbuser \ + --skip-comments \ + icatdb > generator_script_dump_1.sql - # Drop and re-create icatdb to remove generated data - - name: Drop icatdb - run: mysql -picatdbuserpw -uicatdbuser -e 'DROP DATABASE icatdb;' - - name: Create icatdb - run: mysql -picatdbuserpw -uicatdbuser -e 'CREATE DATABASE icatdb;' - # Regenerate table structure of icatdb - - name: Reinstall ICAT Server - run: cd /home/runner/install/icat.server; ./setup -vv install + - name: Recreate ICAT environment + working-directory: pr + run: | + docker compose --profile dependencies down + docker compose --profile dependencies up -d - - name: Add ICAT 5 triggers - run: cd /home/runner/install/icat.server; sudo mysql -uroot -D icatdb < create_triggers_mysql_5_0.sql + - name: Wait for regenerated test data + run: docker wait dg_api_testdata - - name: Add (new) dummy data to icatdb - run: | - cd /home/runner/work/datagateway-api/datagateway-api; uv run python -m util.icat_db_generator - name: Drop modTime and createTime - run: mysql -picatdbuserpw -uicatdbuser icatdb < /home/runner/work/datagateway-api/datagateway-api/util/columns_to_drop.sql - - name: Get SQL dump of new dummy data - run: mysqldump -picatdbuserpw -uicatdbuser --skip-comments icatdb > ~/generator_script_dump_2.sql + run: | + docker exec -i icat_mariadb_container \ + mysql -picatdbuserpw -uicatdbuser icatdb \ + < pr/util/columns_to_drop.sql + + - name: Get SQL dump of regenerated data + run: | + docker exec icat_mariadb_container \ + mysqldump \ + -picatdbuserpw \ + -uicatdbuser \ + --skip-comments \ + icatdb > generator_script_dump_2.sql - # Tests that the generator script produces consistent data over two separate runs - name: Diff SQL dumps - run: diff -s ~/generator_script_dump_1.sql ~/generator_script_dump_2.sql - - # Drop and re-create icatdb to remove generated data - - name: Drop icatdb - run: mysql -picatdbuserpw -uicatdbuser -e 'DROP DATABASE icatdb;' - - name: Create icatdb - run: mysql -picatdbuserpw -uicatdbuser -e 'CREATE DATABASE icatdb;' - # Regenerate table structure of icatdb - - name: Reinstall ICAT Server - run: cd /home/runner/install/icat.server; ./setup -vv install - - name: Add ICAT 5 triggers - run: cd /home/runner/install/icat.server; sudo mysql -uroot -D icatdb < create_triggers_mysql_5_0.sql - - - name: Checkout DataGateway API (default branch) + run: diff -s generator_script_dump_1.sql generator_script_dump_2.sql + + - name: Shutdown PR stack + working-directory: pr + run: docker compose --profile dependencies down + + - name: Checkout default branch uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ github.event.repository.default_branch }} + path: main + + - name: Start default branch dependencies + working-directory: main + run: docker compose --profile dependencies up -d - - name: Create config.yaml - run: cd /home/runner/work/datagateway-api/datagateway-api; cp datagateway_api/config.yaml.example datagateway_api/config.yaml - - name: Create search_api_mapping.json - run: cd /home/runner/work/datagateway-api/datagateway-api; cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json - - name: Create logging.ini - run: cd /home/runner/work/datagateway-api/datagateway-api; cp datagateway_api/logging.example.ini datagateway_api/logging.ini + - name: Wait for test data generation + run: docker wait dg_api_testdata - - name: Add dummy data to icatdb - run: uv run python -m util.icat_db_generator - name: Drop modTime and createTime - run: mysql -picatdbuserpw -uicatdbuser icatdb < /home/runner/work/datagateway-api/datagateway-api/util/columns_to_drop.sql - - name: Get SQL dump of dummy data from main's generator script - run: mysqldump -picatdbuserpw -uicatdbuser --skip-comments icatdb > ~/generator_script_dump_main.sql - - # Tests that the generator script produces the same data as is produced with main's version - - # NOTE: If a delibrate change is made to the script that will change the data generated, - # the diff (and therefore this job) will fail. If this is the case, don't be alarmed. - # The `continue-on-error` keyword has been added to this job so the workflow should - # pass when the PR is merged in, even if this job fails. - # But, if you didn't mean to change the output of the script, there is likely a - # problem with the changes made that will make the generator script's data - # different to SciGateway preprod - - name: Diff SQL dumps - run: diff -s ~/generator_script_dump_main.sql ~/generator_script_dump_1.sql + run: | + docker exec -i icat_mariadb_container \ + mysql -picatdbuserpw -uicatdbuser icatdb \ + < main/util/columns_to_drop.sql + + - name: Get SQL dump of dummy data from default branch generator + run: | + docker exec icat_mariadb_container \ + mysqldump \ + -picatdbuserpw \ + -uicatdbuser \ + --skip-comments \ + icatdb > generator_script_dump_main.sql + + # NOTE: If a deliberate change is made to the script that changes + # generated data, this comparison may fail. The job remains + # continue-on-error to avoid blocking merges. + - name: Compare current branch output with default branch + run: diff -s generator_script_dump_main.sql generator_script_dump_1.sql + + - name: Cleanup + if: always() + working-directory: main + run: docker compose --profile dependencies down pip-install-testing: runs-on: ubuntu-24.04 - strategy: - fail-fast: false name: Pip Install Test + steps: - - name: Add apt repo - run: sudo add-apt-repository universe - # Checkout DataGateway API and setup Python - - name: Check out repo - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v3.5.3 - - name: Add apt repo - run: sudo add-apt-repository universe - # Setup Python - - name: Setup Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - with: - python-version: "3.11" - architecture: x64 - # ICAT Ansible clone and install dependencies - - name: Checkout icat-ansible + - name: Checkout DataGateway API uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - repository: icatproject-contrib/icat-ansible - ref: payara7 - path: icat-ansible - - name: Install Ansible - run: pip install -r icat-ansible/requirements.txt - # Prep for running the playbook - - name: Create hosts file - run: echo -e "[icatdb_minimal_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts - - name: Prepare vault pass - run: echo -e "icattravispw" > icat-ansible/vault_pass.txt - - name: Move vault to directory it'll get detected by Ansible - run: mv icat-ansible/vault.yml icat-ansible/group_vars/all - - name: Replace default payara user with Actions user - run: | - sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml - # Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions - - name: Change hostname to localhost - run: sudo hostname -b localhost + - name: Start test dependencies + run: docker compose --profile dependencies up -d - # Remove existing MySQL installation so it doesn't interfere with GitHub Actions - - name: Remove existing mysql - run: | - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - sudo apt-get remove --purge "mysql*" - sudo rm -rf /var/lib/mysql* /etc/mysql - - # Create local instance of ICAT - - name: Run ICAT Ansible Playbook - run: | - ansible-playbook icat-ansible/icatdb_minimal_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv - - # rootUserNames needs editing as anon/anon is used in search API and required to pass endpoint tests - - name: Add anon user to rootUserNames - run: | - awk -F" =" '/rootUserNames/{$2="= simple/root anon/anon";print;next}1' /home/runner/install/icat.server/run.properties > /home/runner/install/icat.server/run.properties.tmp - - name: Apply rootUserNames change - run: | - mv -f /home/runner/install/icat.server/run.properties.tmp /home/runner/install/icat.server/run.properties - - name: Reinstall ICAT Server - run: | - cd /home/runner/install/icat.server/ && ./setup -vv install - - # Create virtual environment and install DataGateway API - - name: Create and activate virtual environment - run: | - python3 -m venv dg-api-venv - source dg-api-venv/bin/activate - - name: Install DataGateway API via pip - run: pip install . - - # Prep for using the API for tests - - name: Configure datagateway extension - run: echo "`yq \ - '.datagateway_api.extension="/datagateway-api"' datagateway_api/config.yaml.example`" > datagateway_api/config.yaml.example - - name: Create config.yaml - run: cp datagateway_api/config.yaml.example datagateway_api/config.yaml - - name: Create search_api_mapping.json - run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json - - name: Create logging.ini - run: cp datagateway_api/logging.example.ini datagateway_api/logging.ini - - # Launch API to see if it starts correctly or has a startup issue - # Code logic used from https://stackoverflow.com/a/63643845 - - name: Start API - run: timeout 10 python -m datagateway_api.main || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi + - name: Run pip install test + run: docker compose run pip-install-test docker: # This job triggers only if all the other jobs succeed and does different things depending on the context. diff --git a/Dockerfile b/Dockerfile index 7785a514..8bbc147f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,6 +20,34 @@ WORKDIR /datagateway-api-run COPY pyproject.toml uv.lock README.md ./ + +######################################################################################################################## +# Stage for testing, linting and formatting development +######################################################################################################################## +FROM base AS test + +RUN --mount=type=cache,target=/root/.cache/uv \ + set -eux; \ + # Lock and install all dependencies but do not install the project \ + uv sync --locked --no-install-project + +COPY datagateway_api/ datagateway_api/ +COPY util/ util/ +COPY test/ test/ +COPY .flake8 .flake8 + +# Create local config files from examples only if they do not already exist. +# This avoids overwriting repository or environment-specific settings. +RUN if [ ! -f datagateway_api/config.yaml ]; then \ + cp datagateway_api/config.yaml.example datagateway_api/config.yaml; \ + fi && \ + if [ ! -f datagateway_api/logging.ini ]; then \ + cp datagateway_api/logging.example.ini datagateway_api/logging.ini; \ + fi && \ + if [ ! -f datagateway_api/search_api_mapping.json ]; then \ + cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json; \ + fi + ######################################################################################################################## # Stage for local development ######################################################################################################################## diff --git a/datagateway_api/config.yaml.example b/datagateway_api/config.yaml.example index 9ef2fc56..f62c5c0b 100644 --- a/datagateway_api/config.yaml.example +++ b/datagateway_api/config.yaml.example @@ -4,7 +4,7 @@ datagateway_api: client_cache_size: 5 client_pool_init_size: 2 client_pool_max_size: 5 - icat_url: "https://localhost:8181" + icat_url: "http://icat_payara_container:8080" icat_check_cert: false use_reader_for_performance: enabled: false @@ -13,11 +13,11 @@ datagateway_api: reader_password: readerpw search_api: extension: "/search-api" - icat_url: "https://localhost:8181" + icat_url: "http://icat_payara_container:8080" icat_check_cert: false - mechanism: "anon" - username: "" - password: "" + mechanism: "simple" + username: "root" + password: "pw" search_scoring: enabled: false api_url: "http://localhost:9000/score" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..6ff2a7cd --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,198 @@ +services: + # Database needed to store the test data + icat_mariadb: + restart: always + # note: the latest version does not support the SQL connector needed by icat + image: mariadb:10.10 + container_name: icat_mariadb_container + # note: in the case that something else is already running on 3306 locally, + # this can be changed or removed without affecting the tests which connect + # directly to the icat_mariadb service without using port forwarding + # ports: + # - "3308:3306" + environment: + MYSQL_ROOT_PASSWORD: pw + MARIADB_DATABASE: icatdb + MARIADB_USER: icatdbuser + MARIADB_PASSWORD: icatdbuserpw + # the health check will tell us when data is in the DB + healthcheck: + test: '/usr/bin/mysql --database=$$MARIADB_DATABASE --user=$$MARIADB_USER --password=$$MARIADB_PASSWORD --execute "SHOW TABLES;"' + interval: 10s + timeout: 2s + retries: 10 + networks: + - dg_network + profiles: [dependencies, tests, full] + + # The ICAT server, available at https://localhost:18181/icat/version + icat_payara: + restart: always + image: harbor.stfc.ac.uk/icat/icat_6:latest + container_name: icat_payara_container + depends_on: + icat_mariadb: + condition: service_healthy + ports: + - "14747:4848" # payara port + - "18181:8181" # https port + - "18080:8080" # https port + volumes: + - ./icat/post-boot-commands.asadmin:/config/post-boot-commands.asadmin + environment: + - POSTBOOT_COMMANDS=/config/post-boot-commands.asadmin + - ROOT_USERS="simple/root anon/anon" + healthcheck: + test: curl --fail http://localhost:8080/icat/version || exit 1 + interval: 10s + timeout: 2s + retries: 10 + networks: + - dg_network + profiles: [dependencies, tests, full] + + auth_simple: + restart: unless-stopped + image: harbor.stfc.ac.uk/icat/icat_auth_simple:latest + container_name: auth_simple_container + ports: + - "28181:8181" + - "24747:4848" + networks: + - dg_network + profiles: [dependencies, tests, full] + + auth_anon: + restart: unless-stopped + image: harbor.stfc.ac.uk/icat/icat_auth_anon:latest + container_name: auth_anon_container + ports: + - "29181:8181" + - "25747:4848" + networks: + - dg_network + profiles: [dependencies, tests, full] + + testdata: + container_name: dg_api_testdata + build: + context: . + target: test + depends_on: + icat_payara: + condition: service_healthy + volumes: + - ./util:/datagateway-api-run/util + command: uv run python -m util.icat_db_generator + restart: "no" + networks: + - dg_network + profiles: [dependencies, tests, full] + + format: + container_name: dg_api_format + build: + context: . + target: test + volumes: + - ./datagateway_api:/datagateway-api-run/datagateway_api + - ./test:/datagateway-api-run/test + - ./util:/datagateway-api-run/util + command: uv run black datagateway_api test util + profiles: [format, checks] + + lint: + container_name: dg_api_lint + build: + context: . + target: test + volumes: + - ./datagateway_api:/datagateway-api-run/datagateway_api + - ./test:/datagateway-api-run/test + - ./util:/datagateway-api-run/util + command: uv run flake8 datagateway_api test util + profiles: [lint, checks] + + unit-tests: + container_name: dg_api_unit_tests + build: + context: . + target: test + volumes: + # use for local development + # - ./datagateway_api:/datagateway-api-run/datagateway_api + # - ./test:/datagateway-api-run/test + - ./coverage.xml:/datagateway-api-run/coverage.xml + command: uv run pytest test/unit --cov=datagateway_api --cov-report=xml + profiles: [tests] + + integration-tests: + container_name: dg_api_integration-tests + build: + context: . + target: test + depends_on: + icat_payara: + condition: service_healthy + auth_simple: + condition: service_started + auth_anon: + condition: service_started + testdata: + condition: service_completed_successfully + + volumes: + # use for local development + # - ./datagateway_api:/datagateway-api-run/datagateway_api + # - ./test:/datagateway-api-run/test + - ./coverage.xml:/datagateway-api-run/coverage.xml + command: uv run pytest test/integration --cov=datagateway_api --cov-report=xml + networks: + - dg_network + profiles: [tests] + + pip-install-test: + container_name: dg_api_pip-install-test + build: + context: . + target: test + + command: + - sh + - -c + - | + timeout 10 uv run python -m datagateway_api.main + test $? -eq 124 -o $? -eq 0 + + # volumes: + # use for local development + # - ./datagateway_api:/datagateway-api-run/datagateway_api + depends_on: + icat_payara: + condition: service_healthy + networks: + - dg_network + profiles: [tests] + + # Build the api from the local docker image + # available at http://localhost:5000/docs + dg_api: + build: + context: . + target: dev + container_name: dg_api_container + depends_on: + icat_payara: + condition: service_healthy + + volumes: + - ./datagateway_api:/datagateway-api-run/datagateway_api + + ports: + - "5000:8000" + networks: + - dg_network + profiles: [full] + +networks: + dg_network: diff --git a/icat/post-boot-commands.asadmin b/icat/post-boot-commands.asadmin new file mode 100644 index 00000000..d273f7cc --- /dev/null +++ b/icat/post-boot-commands.asadmin @@ -0,0 +1,3 @@ +set resources.jdbc-connection-pool.icat.property.url=jdbc:mysql://icat_mariadb_container:3306/icatdb +set resources.jdbc-connection-pool.icat.property.user=icatdbuser +set resources.jdbc-connection-pool.icat.property.password=icatdbuserpw \ No newline at end of file diff --git a/test/integration/search_api/endpoints/test_count_endpoint.py b/test/integration/search_api/endpoints/test_count_endpoint.py index b179cf3c..6a59b331 100644 --- a/test/integration/search_api/endpoints/test_count_endpoint.py +++ b/test/integration/search_api/endpoints/test_count_endpoint.py @@ -93,13 +93,13 @@ def test_valid_count_endpoint( pytest.param( "Datasets", '{"isPublic": true}', - {"count": 120}, + {"count": 119}, id="Dataset count with isPublic condition (True)", ), pytest.param( "Documents", '{"isPublic": true}', - {"count": 60}, + {"count": 59}, id="Document count with isPublic condition (True)", ), pytest.param( diff --git a/test/mock_data.py b/test/mock_data.py index cbe2b3cb..93f20e7f 100644 --- a/test/mock_data.py +++ b/test/mock_data.py @@ -15,7 +15,7 @@ "investigationFacilityCycles": [{"facilityCycle": 1}], "investigationUsers": [{"role": "string", "user": 1}], "summary": "string", - "name": "string", + "name": "Test Data for API Testing, Data Creation 3", "doi": "string", "visitId": "test large", "facility": 1,