feat(docker): dockerise dev workflow (test, lint, format, ICAT, app)

Dockerfile

@@ -20,6 +20,34 @@ WORKDIR /datagateway-api-run
 
 COPY pyproject.toml uv.lock README.md ./
 
+
+########################################################################################################################
+# Stage for testing, linting and formatting development
+########################################################################################################################
+FROM base AS test
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    set -eux; \
+    # Lock and install all dependencies but do not install the project \
+    uv sync --locked --no-install-project
+
+COPY datagateway_api/ datagateway_api/
+COPY util/ util/
+COPY test/ test/
+COPY .flake8 .flake8
+
+# Create local config files from examples only if they do not already exist.
+# This avoids overwriting repository or environment-specific settings.
+RUN if [ ! -f datagateway_api/config.yaml ]; then \
+        cp datagateway_api/config.yaml.example datagateway_api/config.yaml; \
+    fi && \
+    if [ ! -f datagateway_api/logging.ini ]; then \
+        cp datagateway_api/logging.example.ini datagateway_api/logging.ini; \
+    fi && \
+    if [ ! -f datagateway_api/search_api_mapping.json ]; then \
+        cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json; \
+    fi
+
 ########################################################################################################################
 # Stage for local development
 ########################################################################################################################

datagateway_api/config.yaml.example

@@ -4,7 +4,7 @@ datagateway_api:
   client_cache_size: 5
   client_pool_init_size: 2
   client_pool_max_size: 5
-  icat_url: "https://localhost:8181"
+  icat_url: "http://icat_payara_container:8080"
   icat_check_cert: false
   use_reader_for_performance:
     enabled: false
@@ -13,11 +13,11 @@ datagateway_api:
     reader_password: readerpw
 search_api:
   extension: "/search-api"
-  icat_url: "https://localhost:8181"
+  icat_url: "http://icat_payara_container:8080"
   icat_check_cert: false
-  mechanism: "anon"
-  username: ""
-  password: ""
+  mechanism: "simple"
+  username: "root"
+  password: "pw"
   search_scoring:
     enabled: false
     api_url: "http://localhost:9000/score"

docker-compose.yml

@@ -0,0 +1,198 @@
+services:
+  # Database needed to store the test data
+  icat_mariadb:
+    restart: always
+    # note: the latest version does not support the SQL connector needed by icat
+    image: mariadb:10.10
+    container_name: icat_mariadb_container
+    # note: in the case that something else is already running on 3306 locally,
+    # this can be changed or removed without affecting the tests which connect
+    # directly to the icat_mariadb service without using port forwarding
+    # ports:
+    #   - "3308:3306"
+    environment:
+      MYSQL_ROOT_PASSWORD: pw
+      MARIADB_DATABASE: icatdb
+      MARIADB_USER: icatdbuser
+      MARIADB_PASSWORD: icatdbuserpw
+    # the health check will tell us when data is in the DB
+    healthcheck:
+      test: '/usr/bin/mysql --database=$$MARIADB_DATABASE --user=$$MARIADB_USER --password=$$MARIADB_PASSWORD --execute "SHOW TABLES;"'
+      interval: 10s
+      timeout: 2s
+      retries: 10
+    networks:
+      - dg_network
+    profiles: [dependencies, tests, full]
+
+  # The ICAT server, available at https://localhost:18181/icat/version
+  icat_payara:
+    restart: always
+    image: harbor.stfc.ac.uk/icat/icat_6:latest
+    container_name: icat_payara_container
+    depends_on:
+      icat_mariadb:
+        condition: service_healthy
+    ports:
+      - "14747:4848" # payara port
+      - "18181:8181" # https port
+      - "18080:8080" # https port
+    volumes:
+      - ./icat/post-boot-commands.asadmin:/config/post-boot-commands.asadmin
+    environment:
+      - POSTBOOT_COMMANDS=/config/post-boot-commands.asadmin
+      - ROOT_USERS="simple/root anon/anon"
+    healthcheck:
+      test: curl --fail http://localhost:8080/icat/version || exit 1
+      interval: 10s
+      timeout: 2s
+      retries: 10
+    networks:
+      - dg_network
+    profiles: [dependencies, tests, full]
+
+  auth_simple:
+    restart: unless-stopped
+    image: harbor.stfc.ac.uk/icat/icat_auth_simple:latest
+    container_name: auth_simple_container
+    ports:
+      - "28181:8181"
+      - "24747:4848"
+    networks:
+      - dg_network
+    profiles: [dependencies, tests, full]
+
+  auth_anon:
+    restart: unless-stopped
+    image: harbor.stfc.ac.uk/icat/icat_auth_anon:latest
+    container_name: auth_anon_container
+    ports:
+      - "29181:8181"
+      - "25747:4848"
+    networks:
+      - dg_network
+    profiles: [dependencies, tests, full]
+
+  testdata:
+    container_name: dg_api_testdata
+    build:
+      context: .
+      target: test
+    depends_on:
+      icat_payara:
+        condition: service_healthy
+    volumes:
+      - ./util:/datagateway-api-run/util
+    command: uv run python -m util.icat_db_generator
+    restart: "no"
+    networks:
+      - dg_network
+    profiles: [dependencies, tests, full]
+
+  format:
+    container_name: dg_api_format
+    build:
+      context: .
+      target: test
+    volumes:
+      - ./datagateway_api:/datagateway-api-run/datagateway_api
+      - ./test:/datagateway-api-run/test
+      - ./util:/datagateway-api-run/util
+    command: uv run black datagateway_api test util
+    profiles: [format, checks]
+
+  lint:
+    container_name: dg_api_lint
+    build:
+      context: .
+      target: test
+    volumes:
+      - ./datagateway_api:/datagateway-api-run/datagateway_api
+      - ./test:/datagateway-api-run/test
+      - ./util:/datagateway-api-run/util
+    command: uv run flake8 datagateway_api test util
+    profiles: [lint, checks]
+
+  unit-tests:
+    container_name: dg_api_unit_tests
+    build:
+      context: .
+      target: test
+    volumes:
+      # use for local development
+      # - ./datagateway_api:/datagateway-api-run/datagateway_api
+      # - ./test:/datagateway-api-run/test
+      - ./coverage.xml:/datagateway-api-run/coverage.xml
+    command: uv run pytest test/unit --cov=datagateway_api --cov-report=xml
+    profiles: [tests]
+
+  integration-tests:
+    container_name: dg_api_integration-tests
+    build:
+      context: .
+      target: test
+    depends_on:
+      icat_payara:
+        condition: service_healthy
+      auth_simple:
+        condition: service_started
+      auth_anon:
+        condition: service_started
+      testdata:
+        condition: service_completed_successfully
+
+    volumes:
+      # use for local development
+      # - ./datagateway_api:/datagateway-api-run/datagateway_api
+      # - ./test:/datagateway-api-run/test
+      - ./coverage.xml:/datagateway-api-run/coverage.xml
+    command: uv run pytest test/integration  --cov=datagateway_api --cov-report=xml
+    networks:
+      - dg_network
+    profiles: [tests]
+
+  pip-install-test:
+    container_name: dg_api_pip-install-test
+    build:
+      context: .
+      target: test
+
+    command:
+      - sh
+      - -c
+      - |
+        timeout 10 uv run python -m datagateway_api.main
+        test $? -eq 124 -o $? -eq 0
+
+    # volumes:
+    # use for local development
+    # - ./datagateway_api:/datagateway-api-run/datagateway_api
+    depends_on:
+      icat_payara:
+        condition: service_healthy
+    networks:
+      - dg_network
+    profiles: [tests]
+
+  # Build the api from the local docker image
+  # available at http://localhost:5000/docs
+  dg_api:
+    build:
+      context: .
+      target: dev
+    container_name: dg_api_container
+    depends_on:
+      icat_payara:
+        condition: service_healthy
+
+    volumes:
+      - ./datagateway_api:/datagateway-api-run/datagateway_api
+
+    ports:
+      - "5000:8000"
+    networks:
+      - dg_network
+    profiles: [full]
+
+networks:
+  dg_network:

icat/post-boot-commands.asadmin

@@ -0,0 +1,3 @@
+set resources.jdbc-connection-pool.icat.property.url=jdbc:mysql://icat_mariadb_container:3306/icatdb
+set resources.jdbc-connection-pool.icat.property.user=icatdbuser
+set resources.jdbc-connection-pool.icat.property.password=icatdbuserpw

test/integration/search_api/endpoints/test_count_endpoint.py

@@ -93,13 +93,13 @@ def test_valid_count_endpoint(
             pytest.param(
                 "Datasets",
                 '{"isPublic": true}',
-                {"count": 120},
+                {"count": 119},
                 id="Dataset count with isPublic condition (True)",
             ),
             pytest.param(
                 "Documents",
                 '{"isPublic": true}',
-                {"count": 60},
+                {"count": 59},
                 id="Document count with isPublic condition (True)",
             ),
             pytest.param(

test/mock_data.py

@@ -15,7 +15,7 @@
     "investigationFacilityCycles": [{"facilityCycle": 1}],
     "investigationUsers": [{"role": "string", "user": 1}],
     "summary": "string",
-    "name": "string",
+    "name": "Test Data for API Testing, Data Creation 3",
     "doi": "string",
     "visitId": "test large",
     "facility": 1,