Feature 2410 generic checksum column sqlalchemy 2.1

.github/workflows/autotest.yml

@@ -178,19 +178,19 @@ jobs:
       - name: Set repository variable
         id: repo
         run: echo "repo=$(echo '${{ github.repository }}' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT
-      - name: Build Python 3.9 image locally
-        if: needs.runtime_images.outputs.build_locally == 'true' && matrix.cfg.PYTHON == '3.9'
+      - name: Build Python 3.11 image locally
+        if: needs.runtime_images.outputs.build_locally == 'true' && matrix.cfg.PYTHON == '3.11'
         uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
         with:
           context: .
           file: etc/docker/test/runtime.Dockerfile
           target: final
           build-args: |
-            PYTHON=3.9
+            PYTHON=3.11
           push: false
-          tags: ${{ needs.runtime_images.outputs.py39_image }}
+          tags: ${{ needs.runtime_images.outputs.py311_image }}
           outputs: type=docker
-          cache-from: type=registry,ref=ghcr.io/${{ steps.repo.outputs.repo }}/rucio-dev-runtime:py39-buildcache
+          cache-from: type=registry,ref=ghcr.io/${{ steps.repo.outputs.repo }}/rucio-dev-runtime:py311-buildcache
         continue-on-error: false
       - name: Build Python 3.10 image locally
         if: needs.runtime_images.outputs.build_locally == 'true' && matrix.cfg.PYTHON == '3.10'
@@ -208,8 +208,8 @@ jobs:
         continue-on-error: false
       - name: Run test with cfg
         env:
-          PY39_RUNTIME_IMAGE: ${{ needs.runtime_images.outputs.py39_image }}
+          PY311_RUNTIME_IMAGE: ${{ needs.runtime_images.outputs.py311_image }}
           PY310_RUNTIME_IMAGE: ${{ needs.runtime_images.outputs.py310_image }}
           BUILD_LOCALLY: ${{ needs.runtime_images.outputs.build_locally }}
         run: |
-          echo '{"matrix": ${{ toJson(matrix.cfg) }}, "runtime_images": {"3.9": "${{ needs.runtime_images.outputs.py39_image }}", "3.10": "${{ needs.runtime_images.outputs.py310_image }}"}, "build_locally": "${{ needs.runtime_images.outputs.build_locally }}" }' | ./tools/test/run_tests.py
+          echo '{"matrix": ${{ toJson(matrix.cfg) }}, "runtime_images": {"3.11": "${{ needs.runtime_images.outputs.py311_image }}", "3.10": "${{ needs.runtime_images.outputs.py310_image }}"}, "build_locally": "${{ needs.runtime_images.outputs.build_locally }}" }' | ./tools/test/run_tests.py

.github/workflows/integration_tests.yml

@@ -12,7 +12,7 @@ concurrency:
 
 jobs:
   runtime_images:
-    name: Build Python 3.9 Runtime Image
+    name: Build Python 3.10 Runtime Image
     uses: ./.github/workflows/runtime_images.yml
     permissions:
       contents: read
@@ -121,7 +121,7 @@ jobs:
           build-args: |
             PYTHON=3.9
           push: false
-          tags: ${{ needs.runtime_images.outputs.py39_image }}
+          tags: ${{ needs.runtime_images.outputs.py310_image }}
           outputs: type=docker
         continue-on-error: false
       - name: Pull all required images
@@ -133,7 +133,7 @@ jobs:
         shell: bash
         run: |
           docker image ls
-          sed -i 's;image: .*rucio-dev.*;image: ${{ needs.runtime_images.outputs.py39_image }};' \
+          sed -i 's;image: .*rucio-dev.*;image: ${{ needs.runtime_images.outputs.py310_image }};' \
               $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml
       - name: Start containers
         run: |

.github/workflows/runtime_images.yml

@@ -3,9 +3,9 @@ name: Build Runtime Images
 on:
   workflow_call:
     outputs:
-      py39_image:
-        description: "Python 3.9 runtime image tag"
-        value: ${{ jobs.build_runtime_images.outputs.py39_image }}
+      py311_image:
+        description: "Python 3.11 runtime image tag"
+        value: ${{ jobs.build_runtime_images.outputs.py311_image }}
       py310_image:
         description: "Python 3.10 runtime image tag"
         value: ${{ jobs.build_runtime_images.outputs.py310_image }}
@@ -67,14 +67,14 @@ jobs:
           COMMIT_REF="${{ steps.metadata.outputs.commit_ref }}"
           REPO=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')
 
-          for PYVER in 3.9 3.10; do
+          for PYVER in 3.11 3.10; do
             HASH=$(echo "PYTHON=${PYVER}" | cat - $FILES_TO_HASH | sha256sum | cut -d' ' -f1 | head -c 12)
             IMAGE="ghcr.io/${REPO}/rucio-dev-runtime:py${PYVER//.}-${HASH}"
             echo "Python ${PYVER} image tag: $IMAGE"
 
             # Export dynamic outputs
-            if [[ "$PYVER" == "3.9" ]]; then
-              echo "py39_image=$IMAGE" >> $GITHUB_OUTPUT
+            if [[ "$PYVER" == "3.11" ]]; then
+              echo "py311_image=$IMAGE" >> $GITHUB_OUTPUT
             else
               echo "py310_image=$IMAGE" >> $GITHUB_OUTPUT
             fi

etc/docker/test/extra/rucio_autotests_common.cfg

@@ -140,7 +140,7 @@ email_from = Rucio <rucio-dev@cern.ch>
 email_test = spamspamspam@cern.ch
 
 [policy]
-package = atlas_rucio_policy_package
+#package = atlas_rucio_policy_package
 #permission = atlas
 #schema = atlas
 lfn2pfn_algorithm_default = hash

etc/docker/test/extra/rucio_default.cfg

@@ -160,7 +160,7 @@ email_from = Rucio <rucio-dev@cern.ch>
 email_test = spamspamspam@cern.ch
 
 [policy]
-package = atlas_rucio_policy_package
+#package = atlas_rucio_policy_package
 #permission = atlas
 #schema = atlas
 lfn2pfn_algorithm_default = hash

etc/docker/test/matrix.yml

@@ -2,10 +2,10 @@ dists:
   - id: alma9
     allow:
       python:
-        - "3.9"
+        - "3.11"
         - "3.10"
 python:
-  - id: "3.9"
+  - id: "3.11"
     allow:
       suites:
         - remote_dbs

etc/docker/test/matrix_nightly.yml

@@ -2,10 +2,10 @@ dists:
   - id: alma9
     allow:
       python:
-        - "3.9"
+        - "3.10"
 python:
-  - "3.9"
   - "3.10"
+  - "3.11"
 suites:
   - id: client
     RDBMS: postgres14

etc/docker/test/runtime.Dockerfile

@@ -9,6 +9,7 @@ FROM almalinux:9.1 AS base
     ENV PYTHON_VENV="/opt/venv"
     ENV PATH="${PYTHON_VENV}/bin:${PATH}"
     ENV PYTHON_310_PATCH_VERSION="4"
+    ENV PYTHON_311_PATCH_VERSION="11"
     ENV RUCIO_HOME="/opt/rucio"
 
 FROM base AS oracle-client
@@ -17,17 +18,14 @@ FROM base AS oracle-client
         echo "/usr/lib/oracle/19.12/client64/lib" > /etc/ld.so.conf.d/oracle-instantclient.conf;
 
 FROM base AS python
-    RUN if [ "$PYTHON" == "3.9" ] ; then \
-            dnf install -y epel-release.noarch && \
-            dnf install -y 'dnf-command(config-manager)' && \
-            dnf config-manager --set-enabled crb && \
-            dnf -y update && \
-            dnf -y install boost-python3 python3-pip python3-devel && \
-            dnf remove --assumeyes python3-setuptools && \
-            python3 -m pip --no-cache-dir install --upgrade pip && \
-            python3 -m pip --no-cache-dir install --upgrade setuptools wheel; \
-        elif [ "$PYTHON" == "3.10" ] ; then \
+    RUN if [ "$PYTHON" == "3.10" ] ; then \
             PYTHON_VERSION="3.10.${PYTHON_310_PATCH_VERSION}" && \
+            COMPILE_FROM_SOURCE=true; \
+        elif [ "$PYTHON" == "3.11" ] ; then \
+            PYTHON_VERSION="3.11.${PYTHON_311_PATCH_VERSION}" && \
+            COMPILE_FROM_SOURCE=true; \
+        fi && \
+        if [ "$COMPILE_FROM_SOURCE" = "true" ] ; then \
             dnf install -y 'dnf-command(config-manager)' && \
             dnf config-manager --enable crb && \
             dnf -y update && \
@@ -57,11 +55,11 @@ FROM python AS gfal2
         if [ "$PYTHON" == "3.9" ] ; then \
             dnf -y install gfal2-python3 && \
             cp /usr/lib64/python3.9/site-packages/gfal2.so /usr/lib64/gfal2.so; \
-        elif [ "$PYTHON" == "3.10" ] ; then \
+        elif [ "$PYTHON" == "3.10" ] || [ "$PYTHON" == "3.11" ] ; then \
             wget https://archives.boost.io/release/1.80.0/source/boost_1_80_0.tar.gz && \
             tar -xvzf boost_1_80_0.tar.gz && \
             cd boost_1_80_0 && \
-            ./bootstrap.sh --with-libraries=python --with-python=/usr/bin/python3.10 --prefix=/usr --libdir=/usr/local/lib && \
+            ./bootstrap.sh --with-libraries=python --with-python=/usr/bin/python${PYTHON} --prefix=/usr --libdir=/usr/local/lib && \
             ./b2 --with-python --libdir=/usr/local/lib --link=shared && \
             cp /usr/local/src/boost_1_80_0/stage/lib/lib* /usr/lib64/ && \
             dnf install -y git dnf-plugins-core git rpm-build tree which cmake make gcc gcc-c++ && \
@@ -81,7 +79,7 @@ FROM python AS mod_wsgi
     RUN if [ "$PYTHON" == "3.9" ] ; then \
             dnf install -y python3-mod_wsgi && \
             cp /usr/lib64/httpd/modules/mod_wsgi_python3.so /usr/lib64/httpd/modules/mod_wsgi.so; \
-        elif [ "$PYTHON" == "3.10" ] ; then \
+        elif [ "$PYTHON" == "3.10" ] || [ "$PYTHON" == "3.11" ] ; then \
             dnf install -y httpd-devel && \
             curl -sSL https://github.com/GrahamDumpleton/mod_wsgi/archive/4.9.1.tar.gz | tar xzv && \
             cd mod_wsgi-4.9.1 && \

lib/rucio/alembicrevision.py

@@ -12,4 +12,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ALEMBIC_REVISION = '3b943000da18'  # the current alembic head revision
+ALEMBIC_REVISION = '8ab4d628cffb'  # the current alembic head revision
+# ALEMBIC_REVISION = '3b943000da18'  # the previous alembic head revision

lib/rucio/client/uploadclient.py

@@ -727,14 +727,15 @@ def _register_file(
         file_did = {'scope': file_scope, 'name': file_name}
         replica_for_api = self._convert_file_for_api(file)
         try:
-            # if the remote checksum is different, this DID must not be used
+            # if the remote checksum is different, this DID must not be used. Use checksum column (not the legacy columns) to detect this
+            #  and raise an exception
             meta = self.client.get_metadata(file_scope, file_name)
             logger(logging.INFO, 'File DID already exists')
-            logger(logging.DEBUG, 'local checksum: %s, remote checksum: %s' % (file['adler32'], meta['adler32']))
+            logger(logging.DEBUG, 'local checksum: %s, remote checksum: %s' % (file['checksum']['adler32'], meta['checksum']['adler32']))
 
-            if str(meta['adler32']).lstrip('0') != str(file['adler32']).lstrip('0'):
+            if str(meta['checksum']['adler32']).lstrip('0') != str(file['checksum']['adler32']).lstrip('0'):
                 logger(logging.ERROR,
-                       'Local checksum %s does not match remote checksum %s' % (file['adler32'], meta['adler32']))
+                       'Local checksum %s does not match remote checksum %s' % (file['checksum']['adler32'], meta['adler32']))
                 raise DataIdentifierAlreadyExists
 
             # add the file to rse if it is not registered yet
@@ -845,8 +846,7 @@ def _collect_file_info(
         new_item['basename'] = os.path.basename(filepath)
 
         new_item['bytes'] = os.stat(filepath).st_size
-        new_item['adler32'] = adler32(filepath)
-        new_item['md5'] = md5(filepath)
+        new_item['checksum'] = {'md5': md5(filepath), 'adler32': adler32(filepath)}
         new_item['meta'] = {'guid': self._get_file_guid(new_item)}
         new_item['state'] = 'C'
         if not new_item.get('did_scope'):
@@ -975,8 +975,9 @@ def _convert_file_for_api(
         replica['scope'] = file['did_scope']
         replica['name'] = file['did_name']
         replica['bytes'] = file['bytes']
-        replica['adler32'] = file['adler32']
-        replica['md5'] = file['md5']
+        replica['adler32'] = file['checksum'].get('adler32')
+        replica['md5'] = file['checksum'].get('md5')
+        replica['checksum'] = file['checksum']
         replica['meta'] = file['meta']
         replica['state'] = file['state']
         pfn = file.get('pfn')

lib/rucio/common/schema/generic.py

@@ -129,6 +129,11 @@
            "type": "string",
            "pattern": "^[a-fA-F\\d]{8}$"}
 
+CHECKSUM = {"description": "Checksum dictionary",
+            "type": "object",
+            "properties": {"adler32": {"type": "string"},
+                           "md5": {"type": "string"}}}
+
 WEIGHT = {"description": "Rule weight",
           "type": ["string", "null"]}
 
@@ -254,6 +259,7 @@
                       "bytes": BYTES,
                       "adler32": ADLER32,
                       "md5": MD5,
+                      "checksum": CHECKSUM,
                       "state": REPLICA_STATE,
                       "pfn": PFN},
        "required": ["scope", "name"],
@@ -273,6 +279,7 @@
                         "bytes": BYTES,
                         "adler32": ADLER32,
                         "md5": MD5,
+                        "checksum": CHECKSUM,
                         "state": REPLICA_STATE,
                         "pfn": PFN},
          "required": ["scope", "name"],

lib/rucio/common/schema/generic_multi_vo.py

@@ -130,6 +130,11 @@
            "type": "string",
            "pattern": "^[a-fA-F\\d]{8}$"}
 
+CHECKSUM = {"description": "Checksum dictionary",
+            "type": "object",
+            "properties": {"adler32": {"type": "string"},
+                           "md5": {"type": "string"}}}
+
 WEIGHT = {"description": "Rule weight",
           "type": ["string", "null"]}
 
@@ -228,6 +233,7 @@
                       "bytes": BYTES,
                       "adler32": ADLER32,
                       "md5": MD5,
+                      "checksum": CHECKSUM,
                       "state": REPLICA_STATE,
                       "pfn": PFN},
        "required": ["scope", "name"],
@@ -247,6 +253,7 @@
                         "bytes": BYTES,
                         "adler32": ADLER32,
                         "md5": MD5,
+                        "checksum": CHECKSUM,
                         "state": REPLICA_STATE,
                         "pfn": PFN},
          "required": ["scope", "name"],

lib/rucio/common/types.py

@@ -414,6 +414,7 @@ class FileToUploadWithCollectedInfoDict(FileToUploadDict):
     dirname: str
     upload_result: dict
     bytes: int
+    checksum: dict[str, Any]
 
 
 class FileToUploadWithCollectedAndDatasetInfoDict(FileToUploadWithCollectedInfoDict):
@@ -465,6 +466,7 @@ class RequestAttributesDict(TypedDict):
     bytes: int
     md5: str
     adler32: str
+    checksum: dict[str, Any]
     is_intermediate_hop: bool